# One origin, one robots.txt. arbe.0sk.ar is both the app and the docs:
# /docs/ is public documentation we *want* crawled and trained on — that's the
# point of documentation — and the rest is auth-gated or ephemeral, so it stays
# out of search.
#
# The agent-facing files at the root are allowed on purpose: an agent pointed at
# this host needs /llms.txt and /install.sh without tripping a disallow, and /api
# is JSON route discovery — cheap to read, no auth needed.
#
# Crawlers obey exactly one User-agent group, so the named-crawler group below
# repeats the app's restrictions rather than inheriting them. Without any group
# at all Cloudflare injects a managed Content Signals placeholder, which reads as
# "invalid" to scanners.

User-agent: *
Allow: /$
Allow: /about
Allow: /privacy
Allow: /docs/
Allow: /llms.txt
Allow: /llms-small.txt
Allow: /llms-full.txt
Allow: /install.sh
Allow: /api$
Disallow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Named AI crawlers, stated explicitly so there's no ambiguity about the docs.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: meta-externalagent
Allow: /$
Allow: /about
Allow: /privacy
Allow: /docs/
Allow: /llms.txt
Allow: /llms-small.txt
Allow: /llms-full.txt
Allow: /install.sh
Allow: /api$
Disallow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Plain-text mirrors of the whole docs site, for agents that want it in one shot:
#   /llms.txt        index
#   /llms-small.txt  condensed
#   /llms-full.txt   everything
# Every docs page also has a markdown twin: /docs/system/architecture/ -> /docs/system/architecture.md

Sitemap: https://arbe.0sk.ar/docs/sitemap-index.xml