# robots.txt — Startup Ignition
# AI crawlers, search engines, and archivers are welcome.
# Self-managed: disable Cloudflare's "Manage your robots.txt" so this file is authoritative.

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

# Explicit allow for major AI search/answer/training crawlers.
# (Redundant with the wildcard above, but signals intent and protects against
#  user-agent-specific overrides from Cloudflare or future tooling.)
#
# The Disallow lines are repeated in EVERY group on purpose. A crawler obeys only the
# single most specific matching group, so a group containing just "Allow: /" would
# otherwise grant that crawler access to everything the wildcard blocks. Do not remove
# them as duplication.
#
# /dataroom/ holds gated decks and is blocked outright. The files themselves are in R2 and
# have no URL of their own at all, so there is nothing else to block. /docs/ stays crawlable
# because /docs/api and /docs/mcp are public agent-facing documentation - that separation
# is exactly why the decks moved off /docs/.

User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: Claude-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: anthropic-ai
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: Perplexity-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: Applebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: DuckAssistBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: MistralAI-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: FacebookBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: Amazonbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

# Block training-only crawlers (want citations, not training data)
User-agent: CCBot
Disallow: /

User-agent: ia_archiver
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

User-agent: archive.org_bot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /dataroom/

# Sitemap
Sitemap: https://www.startupignition.com/sitemap-index.xml