# robots.txt for consult-with-roy.com
#
# Replaces what this zone served until 2026-09-07: Cloudflare's Content Signals
# *preamble* and nothing after it — twenty-six lines explaining what a content
# signal means, then no signal, no User-agent, no Allow, no Disallow, no
# Sitemap. It returned 200 with `X-Matched-Path: /404` because the app had no
# robots route at all, so the boilerplate was the entire file. It restricted
# nothing and disclosed nothing.
#
# A static file rather than app/robots.ts on purpose: Next's MetadataRoute.Robots
# type emits only User-agent/Allow/Disallow/Crawl-delay/Sitemap and has no way to
# express the Content-Signal line below. tests/seo.spec.ts asserts the Disallow
# list here still matches PRIVATE_PATH_PREFIXES in lib/seo/site.ts, so this
# cannot quietly drift from the per-page robots metadata.

# Policy (Roy, 2026-09-07): indexing yes, AI answers yes, AI training no.
# Being cited in an answer sends a reader here, which is the point. Being
# absorbed into model weights does not, and cannot be undone afterwards.
#
# Content-Signal is a Cloudflare/IETF proposal, not part of the original robots
# grammar, so a crawler that does not understand the line ignores it and obeys
# the ordinary directives below.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Closed surfaces. /r/ is the one that matters most: those URLs carry a
# single-use token that resumes a client's session, so an indexed /r/ URL is a
# published credential rather than merely an awkward privacy leak. The rest are
# authenticated or per-client views with nothing generic to rank for.
Disallow: /admin
Disallow: /admin/
Disallow: /api
Disallow: /api/
Disallow: /r/
Disallow: /consult
Disallow: /consult/
Disallow: /verify
Disallow: /verify/
Disallow: /chat
Disallow: /chat/

# Named explicitly rather than left to the wildcard. These are the crawlers that
# feed answer engines, and an explicit Allow is a clearer statement of intent
# than silence when the same vendor also runs a training crawler we decline.
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

# The training-only crawlers, declined per the policy above. GPTBot is OpenAI's
# training crawler and is distinct from OAI-SearchBot, which is allowed above;
# likewise CCBot, whose corpus is a common training input.
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Applebot-Extended
Disallow: /

Sitemap: https://consult-with-roy.com/sitemap.xml