30 lines
495 B
Plaintext
30 lines
495 B
Plaintext
# Robots.txt for giampy-dogservice
|
|
|
|
# Blocca aree amministrative e private
|
|
User-agent: *
|
|
Disallow: /admin/
|
|
Disallow: /logout/
|
|
|
|
# Blocca file statici non importanti per SEO
|
|
Disallow: /*.json$
|
|
Disallow: *.css
|
|
Disallow: *.js
|
|
Disallow: /static/
|
|
|
|
# Crawl delay - server debole, limitiamo il traffico
|
|
Crawl-delay: 5
|
|
Request-rate: 1/5
|
|
|
|
# Blocca bot cattivi o pesanti
|
|
User-agent: AhrefsBot
|
|
Disallow: /
|
|
|
|
User-agent: SemrushBot
|
|
Disallow: /
|
|
|
|
User-agent: DotBot
|
|
Disallow: /
|
|
|
|
User-agent: MJ12bot
|
|
Disallow: /
|