# robots.txt for AgenticRepo
# Classic rules + AI user-agents + Content Signals + RSL license + ARD Agentmap.

# --- Search engines: come on in ---
User-agent: Googlebot
User-agent: Bingbot
Allow: /

# --- AI live-retrieval agents (answer engines fetching on a user's behalf): welcome ---
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=no

# --- AI training crawlers: our excuses are proprietary (they are not) ---
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: CCBot
Disallow: /

# --- Everyone else ---
User-agent: *
Allow: /
Disallow: /api/internal/
Content-Signal: search=yes, ai-input=yes, ai-train=no

# --- Extensions (non-group lines) ---
Sitemap: https://agenticrepo.dev/sitemap.xml
License: https://agenticrepo.dev/license.xml
Agentmap: https://agenticrepo.dev/.well-known/ai-catalog.json