# robots.txt for https://aidefensestack.com
# Welcome search engines, AI search and assistant fetchers (they drive traffic and citations) and the AI
# training and grounding agents listed below; refuse SEO data harvesters and training-only AI scrapers.

# Search engines: allowed
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Googlebot-Image
User-agent: Google-InspectionTool
User-agent: Slurp
User-agent: DuckDuckBot
Allow: /

# AI search and assistant fetchers (they fetch pages to answer a query): allowed
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Claude-SearchBot
User-agent: Perplexity-User
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: meta-webindexer
User-agent: Meta-ExternalFetcher
Allow: /

# AI training and grounding crawlers and control tokens: allowed (site-owner policy choice; Disallow here to opt out)
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: anthropic-ai
User-agent: Applebot-Extended
User-agent: meta-externalagent
Allow: /

# Social link-preview crawlers: allowed
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Slackbot
User-agent: Discordbot
Allow: /

# Google advertising crawlers: allowed (publisher eligibility)
User-agent: Mediapartners-Google
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
Allow: /

# SEO data harvesters and link auditors: blocked
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: Barkrowler
User-agent: MajesticBot
User-agent: rogerbot
User-agent: serpstatbot
User-agent: ZoominfoBot
User-agent: PetalBot
Disallow: /

# Training-only AI scrapers (no search or citation value): blocked
User-agent: CCBot
User-agent: Bytespider
User-agent: cohere-ai
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgili
User-agent: Omgilibot
Disallow: /

# Low-value regional crawlers: blocked
User-agent: YandexBot
User-agent: Baiduspider
Disallow: /

# Everyone else: allowed
User-agent: *
Allow: /

Sitemap: https://aidefensestack.com/sitemap-index.xml