# prave.app — robots.txt
# Last updated: 2026-05-13

User-agent: *
Allow: /

# Authenticated app surfaces — gated, no crawl value.
Disallow: /dashboard
Disallow: /skills/new
Disallow: /skills/*/edit
Disallow: /cli-auth

# Protect Stripe / billing redirect endpoints from crawl noise.
Disallow: /billing/
Disallow: /api/

# ────────────────────────────────────────────────────────────────────
# AI-search policy
# ────────────────────────────────────────────────────────────────────
# We distinguish between two classes of AI crawlers:
#
#   1. SEARCH / CITATION bots — these power live AI-search answers
#      (ChatGPT search, Perplexity, Google AI Overviews, Claude with
#      web access, Bing/Copilot, Apple Intelligence). When they can
#      crawl us they can CITE Prave as a source for "best Claude
#      skill platform", "Codex skill registry", "MCP server
#      catalogue" etc. We WANT this — it's the only way users
#      asking AI assistants discover Prave.
#
#   2. TRAINING-ONLY bots — bulk-scrape content to bake into the
#      next foundation-model release. They never cite back. We do
#      NOT consent to this. Vendors who want a licensing
#      conversation can email hello@epplab-studio.de.
#
# Each crawler below is in exactly one bucket.

# Bucket 1 — SEARCH / CITATION bots: explicit allow.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Bucket 2 — TRAINING-ONLY bots: blocked. These are the crawlers
# associated with model pre-training that don't surface citations
# back to the source. We keep them out so our content isn't
# laundered into a competing model without attribution.
User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: FacebookBot
Disallow: /

# Sitemap entries.
Sitemap: https://prave.app/sitemap-index.xml
Sitemap: https://prave.app/sitemap.xml