# Data Center Index | robots.txt
# Questions: hello@datacenterindex.ai

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /og/

# Link-preview crawlers. These fetch og:image assets under /og/ to build the
# share card, and they honor robots.txt, so the blanket "Disallow: /og/" above
# silently broke every unfurl (X rendered a grey placeholder card). Named
# groups override the "*" group entirely, so these bots get the whole site
# while generic crawlers still stay out of /og/.
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: redditbot
Allow: /

# AI search & training crawlers (allow-listed explicitly)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: DuckAssistBot
Allow: /

# Crawl-delay courtesy for the noisier ones
User-agent: Bytespider
Crawl-delay: 5

Sitemap: https://datacenterindex.ai/sitemap.xml
Sitemap: https://datacenterindex.ai/sitemap-news.xml