User-agent: *
Allow: /
Disallow: /account
Disallow: /api
Disallow: /health
Disallow: /admin/
Disallow: /mcp
Disallow: /webhooks

# Entity detail pages (high volume, low SEO value for now)
Disallow: /recall/
Disallow: /510k/
Disallow: /company/
Disallow: /drug/
Disallow: /device-class/
Disallow: /inspection/
Disallow: /compliance/
Disallow: /shortage/
Disallow: /import-refusal/
Disallow: /facility/

# Entity list, facet, and hub pages (/recalls, /inspections/oai, /state/CA, …)
# are intentionally crawlable: they are bounded, substantive, and listed in
# /sitemap-pages.xml, /sitemap-facets.xml, and /sitemap-hubs.xml.

# Block entity sitemap pages (entities removed from index, but block direct access too)
# Do NOT block /sitemap-pages.xml, /sitemap-facets.xml, /sitemap-hubs.xml (static, wanted)
Disallow: /sitemap-recalls-*.xml
Disallow: /sitemap-510k-*.xml
Disallow: /sitemap-companies-*.xml
Disallow: /sitemap-drugs-*.xml
Disallow: /sitemap-devices-*.xml
Disallow: /sitemap-inspections-*.xml
Disallow: /sitemap-compliance-actions-*.xml
Disallow: /sitemap-drug-shortages-*.xml
Disallow: /sitemap-import-refusals-*.xml
Disallow: /sitemap-facilities-*.xml

# Explicit AI crawler guidance
User-agent: GPTBot
Allow: /
Disallow: /mcp

User-agent: ChatGPT-User
Allow: /
Disallow: /mcp

User-agent: ClaudeBot
Allow: /
Disallow: /mcp

User-agent: Claude-Web
Allow: /
Disallow: /mcp

User-agent: PerplexityBot
Allow: /
Disallow: /mcp

User-agent: Google-Extended
Allow: /
Disallow: /mcp

User-agent: Applebot-Extended
Allow: /
Disallow: /mcp

Sitemap: https://www.regdatalab.com/sitemap.xml