# See https://www.robotstxt.org/robotstxt.html for documentation
#
# AI search and indexing strategy:
#   We explicitly allow AI search bots (GPTBot, ChatGPT-User, PerplexityBot,
#   ClaudeBot, anthropic-ai, Google-Extended, Applebot-Extended) to crawl
#   public content so TLS Radar can be cited in ChatGPT, Perplexity, Claude,
#   Google AI Overviews, Gemini, and Apple Intelligence answers.
#
#   Each AI bot gets its own block below because the robots.txt spec says
#   the most-specific User-agent block wins. Without explicit blocks, AI
#   bots inherit the `User-agent: *` rules - which works today but does
#   not signal intent. Explicit blocks survive future wildcard changes
#   and document the policy for future maintainers.

# ----- Default (all other crawlers) -----
User-agent: *

# Rails internals
Disallow: /rails/
Disallow: /cable
Disallow: /assets/

# Admin & authenticated areas
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Disallow: /profile/
Disallow: /settings/

# Per-scan share URLs use <meta robots="noindex"> instead of Disallow,
# so crawlers can see the directive and follow outbound links back to /scan.

# ----- AI search bots (explicit allow) -----
# Each block repeats the Disallow set because the most-specific
# User-agent block wins; AI bots ignore `*` once they match here.

User-agent: GPTBot
Disallow: /rails/
Disallow: /cable
Disallow: /assets/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Disallow: /profile/
Disallow: /settings/

User-agent: ChatGPT-User
Disallow: /rails/
Disallow: /cable
Disallow: /assets/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Disallow: /profile/
Disallow: /settings/

User-agent: OAI-SearchBot
Disallow: /rails/
Disallow: /cable
Disallow: /assets/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Disallow: /profile/
Disallow: /settings/

User-agent: PerplexityBot
Disallow: /rails/
Disallow: /cable
Disallow: /assets/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Disallow: /profile/
Disallow: /settings/

User-agent: ClaudeBot
Disallow: /rails/
Disallow: /cable
Disallow: /assets/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Disallow: /profile/
Disallow: /settings/

User-agent: anthropic-ai
Disallow: /rails/
Disallow: /cable
Disallow: /assets/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Disallow: /profile/
Disallow: /settings/

User-agent: Google-Extended
Disallow: /rails/
Disallow: /cable
Disallow: /assets/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Disallow: /profile/
Disallow: /settings/

User-agent: Applebot-Extended
Disallow: /rails/
Disallow: /cable
Disallow: /assets/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Disallow: /profile/
Disallow: /settings/

Sitemap: https://tlsradar.com/sitemap.xml