# zfrqbl.com — crawler policy. Owner decision 2026-10-10: search and live-fetch agents ALLOWED,
# training crawlers BLOCKED.
#
# This file is the declarative record and it covers fetchers that never touch Cloudflare's edge.
# Cloudflare AI Crawl Control also enforces at the network edge, and an edge block wins — so if a
# crawler here says Allow but gets a 403, the dashboard is the thing to check, not this file:
#   dashboard -> zfrqbl.com -> AI Crawl Control -> Crawlers
# Verified 2026-10-10 before this file was written: OAI-SearchBot, ChatGPT-User, Claude-SearchBot,
# PerplexityBot, Googlebot -> 200; GPTBot, ClaudeBot -> 403 (edge).

# --- Search and retrieval: allowed. These cite, refer and answer. ---
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# --- Training crawlers: blocked. No citation value; the corpus is not a training set. ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Bytespider
Disallow: /

# --- Everything else: allowed, minus paths that are not content. ---
User-agent: *
Disallow: /templates/
Disallow: /404
Disallow: /essays/test
Allow: /

Sitemap: https://zfrqbl.com/sitemap.xml