# Balford — robots.txt
# AI / LLM crawlers are explicitly welcome: search indexing, AI answers
# and model training are all permitted. Only internal tooling / ERP,
# admin, API and legacy WordPress endpoints stay excluded.
#
# The Content Signals policy ("search=yes, ai-input=yes, ai-train=yes") is sent
# as an HTTP response header on this file instead of as a robots.txt line.
# "Content-Signal" is not a directive defined by RFC 9309, so a parser that
# treats an unknown token inside a group as terminating the group would drop
# every Allow/Disallow rule that follows it - and Lighthouse's SEO audit
# reported "robots.txt is not valid / Unknown directive" on every run
# (2026-09-25). As a header the same policy is declared, and the file itself
# stays conformant.

User-Agent: *
Allow: /
Disallow: /erp/
Disallow: /admin/
Disallow: /api/
Disallow: /assets/
Disallow: /author/
Disallow: /comments/
Disallow: */comments
Disallow: /trackback/
Disallow: */trackback
Disallow: /feed/
Disallow: */feed/
Disallow: */feed
Disallow: /?s=
Disallow: /?post_type=

# --- AI / LLM crawlers: allowed (crawl, index, quote, train) ---
# The wildcard policy above already permits this. The names are listed so a
# crawler operator can confirm at a glance that they are welcome - they are
# documentation, not an exception list.
User-Agent: GPTBot
User-Agent: OAI-SearchBot
User-Agent: ChatGPT-User
User-Agent: ClaudeBot
User-Agent: Claude-User
User-Agent: Claude-SearchBot
User-Agent: anthropic-ai
User-Agent: Google-Extended
User-Agent: GoogleOther
User-Agent: Google-CloudVertexBot
User-Agent: PerplexityBot
User-Agent: Perplexity-User
User-Agent: DuckAssistBot
User-Agent: Applebot
User-Agent: Applebot-Extended
User-Agent: Amazonbot
User-Agent: Meta-ExternalAgent
User-Agent: Meta-ExternalFetcher
User-Agent: CCBot
User-Agent: AI2Bot
User-Agent: AI2Bot-Dolma
User-Agent: ImagesiftBot
User-Agent: img2dataset
User-Agent: Webzio-Extended
User-Agent: Diffbot
User-Agent: Omgilibot
User-Agent: Timpibot
User-Agent: cohere-ai
User-Agent: cohere-training-data-crawler
User-Agent: MistralAI-User
User-Agent: Youbot
User-Agent: Bytespider
User-Agent: PanguBot
User-Agent: PetalBot
User-Agent: YisouSpider
User-Agent: DeepSeekBot
User-Agent: KimiCrawler
User-Agent: QwenBot
User-Agent: MiniMaxBot
User-Agent: HunyuanBot
User-Agent: DoubaoBot
User-Agent: SemrushBot-OCOB
User-Agent: AhrefsBot
User-Agent: GrokBot
User-Agent: xAI-Bot
User-Agent: OAI-AdsBot
User-Agent: Brightbot
User-Agent: NomicBot
User-Agent: KagiBot
User-Agent: TerraCotta
Allow: /
Disallow: /erp/
Disallow: /admin/
Disallow: /api/
Disallow: /?s=
Disallow: /?post_type=

# Structured feeds for AI agents
# llms.txt:      https://balford.net/llms.txt
# llms-full.txt: https://balford.net/llms-full.txt

# --- Yandex ---
# Clean-param tells Yandex that these query parameters do not change the page,
# so ?utm_source=..., ?yclid=..., ?gclid=... and ?fbclid=... variants are not
# crawled and indexed as separate duplicates.
Clean-param: utm_source&utm_medium&utm_campaign&utm_content&utm_term&yclid&gclid&fbclid&msclkid /

User-Agent: Yandex
Allow: /
Disallow: /erp/
Disallow: /admin/
Disallow: /api/
Disallow: /?s=
Disallow: /?post_type=

Host: https://balford.net
Sitemap: https://balford.net/sitemap.xml

Sitemap: https://balford.net/sitemap-core.xml

# Engineering answer pages (Nuxt app mounted at /ask/).
Sitemap: https://balford.net/ask/sitemap.xml