# Sitemap
Sitemap: https://divestats.live/sitemap.xml

User-agent: *
Disallow: /api/
Disallow: /login.html
Disallow: /register.html
Allow: /
# Block AI/LLM crawlers
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Search engines may read the data API: profile and meet pages are built from it,
# so without it they would index empty pages. (Verified search engines are also
# exempt from the site's per-visitor limits; other clients are not.)
# ChatGPT search, Perplexity and Claude search: their index and pages opened for a
# user's question; the training crawlers (GPTBot, ClaudeBot) stay blocked above.
# A crawler follows only its own group, so the log-in pages (noindex, one URL per
# page they return to) are left out here as well as under *.
User-agent: Googlebot
User-agent: Bingbot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: DuckDuckBot
User-agent: Applebot
Disallow: /login.html
Disallow: /register.html
Allow: /