Sitemap: https://agentshare.dev/sitemap.xml

# AgentShare / Agent Price API - agentshare.dev
# Price infra for AI agents. Admin UX is noindex/meta; robots.txt does not authenticate.

User-agent: *
Allow: /
# Marketing / HTML pages (explicit for crawler clarity; Allow: / already covers these)
Allow: /pricing
Allow: /about
Allow: /contact
Allow: /docs
Allow: /for-agents
Allow: /registry
# /signup is a thin auth form — let crawlers reach the URL but the page itself
# carries <meta robots="noindex"> so it stays out of result pages.
Allow: /signup
# Machine / API surfaces
Allow: /api/
Allow: /api/llm.txt
Allow: /api/llms.txt
Allow: /llm.txt
Allow: /llms.txt
Allow: /mcp
Allow: /api/v1/tools
Allow: /api/v1/mcp/tools
# /coverage = public JSON "data domain coverage" (categories we track), NOT pytest/HTML coverage reports
Allow: /coverage
Allow: /legal/
Allow: /openapi.json
Allow: /agent.json
Allow: /mcp.json
Allow: /.well-known/
Allow: /public/bot-traffic
# Operator UI — not for search (HTML also uses noindex on login)
Disallow: /admin/
Disallow: /admin-db/

# Preferred AI/train crawlers - explicit Allow / (narrow user-agent overrides * if matched first)
User-agent: GPTBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: xAI-SearchBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Common Crawl (CCBot) — align with content-signal ai-train=no; avoid inviting training corpus ingestion
User-agent: CCBot
Disallow: /

# Scraping-heavy commercial crawlers - reduce noise (optional SEO-tool visibility trade-off)
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Cloudflare content signals (ai-input=yes: welcome AI agents to use our API)
content-signal: ai-input=yes
content-signal: ai-train=no
content-signal: search=yes