# robots.txt for http://www.informare.it/ - Stefano Bellio 1999
# List of all files and directories that should be ignored by robots

User-agent: *				# For all robots
Disallow: /cgi-bin/			# Virtual documents
Disallow: /cgi-win/			# VB CGI
Crawl-delay: 10				# 10 secondi di delay per aprire una nuova pagina da parte dei Bot

User-agent: SERankingBacklinksBot
Crawl-delay: 10

# Googlebot (non supporta Crawl-delay)
User-agent: Googlebot
Allow: /

# Google-Extended (AI training)
User-agent: Google-Extended
Disallow: /

# Bingbot
User-agent: Bingbot
Allow: /
Crawl-delay: 5

# BingPreview / Copilot
User-agent: BingPreview
Allow: /
Crawl-delay: 5

# Yandex
User-agent: Yandex
Allow: /
Crawl-delay: 5

# OpenAI GPTBot
User-agent: GPTBot
Allow: /
Crawl-delay: 10

# OpenAI OAI-SearchBot
User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 10

# ByteDance (TikTok)
User-agent: Bytespider
Disallow: /

# Apple Intelligence - Training
User-agent: Applebot-Extended
Disallow: /

# Anthropic ClaudeBot
User-agent: ClaudeBot
Allow: /
Crawl-delay: 2

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Crawl-delay: 2

# Amazonbot (Alexa/AI)
User-agent: Amazonbot
Allow: /
Crawl-delay: 10

# CCBot (Common Crawl)
User-agent: CCBot
Allow: /
Crawl-delay: 25

# Ahref
User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: AhrefsSiteAudit
Allow: /
Crawl-delay: 2