# gol.cr — robots policy
#
# We WANT search engines and modern AI search tools to crawl + index
# the marketing landing, our articles (/articulo/), and the
# server-rendered share-preview pages
# (/t/ /l/ /p/ /j/ /c/ /q/ /u/ /g/ /inv/ /el-capi/). The app at /app/
# ships an empty shell to bots, so the share-preview routes + articles
# are what carries indexable content.
#
# /api/ is intentionally Disallow-ed: it's the app's internal data
# layer (rate-limited, auth-aware, no SEO value) and indexing it would
# just burn crawl budget on payloads that duplicate what the
# share-preview pages already render in HTML.

User-agent: *
Allow: /
Allow: /app/
Allow: /t/
Allow: /l/
Allow: /j/
Allow: /p/
Allow: /c/
Allow: /q/
Allow: /u/
Allow: /g/
Allow: /inv/
Allow: /el-capi/
Allow: /articulo/
Disallow: /admin
Disallow: /admin/
Disallow: /api/
Disallow: /metrics
Disallow: /landing/beta-signup
Disallow: /auth/
Disallow: /me/
Disallow: /ingestion/

# AI search / answer tools — opt them in explicitly. Cloudflare's
# default bot-protection rules sometimes treat these as "verified
# bots" that need explicit allowlisting; declaring them here is the
# canonical signal that we welcome the traffic.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Allow: /

Sitemap: https://gol.cr/sitemap.xml