# Lighthouse Intelligence -- https://lighthousedata.io
# For questions: hello@lighthousedata.io

# ── Allow all well-behaved crawlers ──────────────────────────────
User-agent: *
Allow: /
Allow: /stats/
Allow: /data/
Allow: /region/
Allow: /methodology
Allow: /trending
Allow: /compare/
Allow: /journeys/
Allow: /api/docs
Allow: /sitemap.xml
Allow: /sitemap-hubs.xml
Allow: /sitemap-data.xml

# ── Block admin and internal tools ───────────────────────────────
Disallow: /admin/
Disallow: /newsroom
Disallow: /industry
Disallow: /chat
Disallow: /api/
Disallow: /review
Disallow: /fortress
Disallow: /costs
Disallow: /agents
Disallow: /embeddings
Disallow: /assumptions
Disallow: /governance
Disallow: /audit-results
Disallow: /pipeline-health
Disallow: /engineering-ops
Disallow: /intelligence
Disallow: /sources
Disallow: /content-drafts
Disallow: /content-strategy
Disallow: /content-gen
Disallow: /queue
Disallow: /releases
Disallow: /crawlers
Disallow: /investors
Disallow: /investor
Disallow: /launch-plan
Disallow: /sops
Disallow: /security-dashboard
Disallow: /llm-settings
Disallow: /api-integrations
Disallow: /aeo
Disallow: /agentic-stats
Disallow: /partials/

# ── Block known bad bots / scrapers ──────────────────────────────
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ── Explicitly allow AI crawlers (boosts AI Overview citation frequency) ──
User-agent: GPTBot
Allow: /data/
Allow: /stats/
Allow: /methodology
Disallow: /admin/
Disallow: /newsroom
Disallow: /industry
Disallow: /api/

User-agent: ClaudeBot
Allow: /data/
Allow: /stats/
Allow: /methodology
Disallow: /admin/
Disallow: /newsroom
Disallow: /industry
Disallow: /api/

User-agent: PerplexityBot
Allow: /data/
Allow: /stats/
Allow: /methodology
Disallow: /admin/
Disallow: /newsroom
Disallow: /industry
Disallow: /api/

User-agent: Google-Extended
Allow: /data/
Allow: /stats/
Allow: /methodology
Disallow: /admin/
Disallow: /newsroom
Disallow: /industry
Disallow: /api/

Sitemap: https://lighthousedata.io/sitemap.xml
Sitemap: https://lighthousedata.io/sitemap-hubs.xml
Sitemap: https://lighthousedata.io/sitemap-stats.xml
Sitemap: https://lighthousedata.io/sitemap-pipeline.xml
Sitemap: https://lighthousedata.io/sitemap-data.xml
Sitemap: https://lighthousedata.io/llms.txt