# ---------------------------------------------------------------------------
# Scrums.com robots.txt  —  ONE file governs the whole www host.
#
# robots.txt is host-scoped: crawlers only read /robots.txt at the root, so
# there is no per-surface robots file. This one file covers the Astro marketing
# IA, the catalog /products/* (products-proxy worker) and the docs /docs/*
# (scrums-docs-proxy). Each surface's URLs are enumerated in the segmented
# sitemap index at /sitemap.xml (generated by
# 03-grow/market/web/agt-web-site/technical-seo/sitemap-gen — see its README).
#
# Deliberately NOT disallowed:
#   - parameter URLs (?filter=, ?sort=, ?page=, …) — they carry per-page
#     noindex,follow + a canonical to their clean parent landing, and Google
#     can only read those signals if the URL stays crawlable
#   - /jobs/{item} postings — live (Active/Open) postings are listed in
#     /sitemap-jobs.xml; every other status is unlisted but kept index,follow
#     so JobPosting schema reaches Google Jobs (lifecycle events also go to
#     the Google Indexing API when its key is provisioned)
# Only paths that should never be fetched at all are disallowed below.
# ---------------------------------------------------------------------------
User-agent: *
Disallow: /api/
Disallow: /login
Disallow: /sign-up
Disallow: /opt-out
Disallow: /thank-you
Disallow: /products/deploy
Allow: /

# ---------------------------------------------------------------------------
# AI + search crawlers: explicitly allowed to crawl the full site (incl.
# /products and /docs). Listed individually so a future tightening of the
# catch-all rule above never silently blocks them.
# ---------------------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Google-Extended
Allow: /

# ---------------------------------------------------------------------------
# Sitemap: the segmented root index ONLY — child sitemaps (company, four
# catalog verticals, solutions, resources, jobs, docs) are referenced from it
# and additionally submitted individually in Search Console for per-segment
# indexation tracking. The human-readable index is /sitemap.
# ---------------------------------------------------------------------------
Sitemap: https://www.scrums.com/sitemap.xml