# LLM Crawling Control File
# Updated: 26-02-2026
# Purpose: Base restrictions for major LLMs 2026 (SPA-friendly)

# =========================
# OpenAI
# =========================
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: OAI-ContentFetcher
User-agent: OpenAI-Crawler
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# Anthropic
# =========================
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude 3
User-agent: Claude-User
User-agent: AnthropicBot
User-agent: Anthropic-AI
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# Google (Gemini / Vertex AI)
# =========================
User-agent: Google-Extended
Allow: /

User-agent: Bard
User-agent: Gemini
User-agent: GoogleOther
User-agent: Google-CloudVertexBot
User-agent: DuplexWeb-Google
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# Microsoft (Bing AI / Copilot)
# =========================
User-agent: BingAI
User-agent: CopilotBot
User-agent: BingPreview
User-agent: AdIdxBot
User-agent: MSNBot-Media
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# Meta (LLaMA / Meta AI)
# =========================
User-agent: LLaMA
User-agent: MetaAI
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
User-agent: FacebookExternalHit
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# xAI / Grok
# =========================
User-agent: xAI
User-agent: Grok
User-agent: XAI-Bot
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# Mistral AI
# =========================
User-agent: Mistral
User-agent: MistralAI-User
User-agent: MistralCrawler
User-agent: Mistral-Research
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# Cohere / AI21 / Inflection
# =========================
User-agent: CohereBot
User-agent: Cohere-ai
User-agent: AI21Bot
User-agent: AI21Labs
User-agent: InflectionBot
User-agent: PiBot
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# Stability / Midjourney / Runway
# =========================
User-agent: StabilityBot
User-agent: StableDiffusionCrawler
User-agent: MidjourneyBot
User-agent: RunwayBot
User-agent: PikaBot
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# Hugging Face / Open datasets
# =========================
User-agent: HuggingFace
User-agent: HFDataBot
User-agent: BigScienceBot
User-agent: LAIONBot
User-agent: DatasetBot
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# Amazon / Alexa / Bedrock
# =========================
User-agent: Amazonbot
User-agent: Amazon-Kendra
User-agent: AlexaMediaBot
User-agent: AWS-AI
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# Apple / Siri / Apple Intelligence
# =========================
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: SiriBot
User-agent: AppleAI
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# China / Asia AI ecosystems
# =========================
User-agent: Baiduspider
User-agent: Baidu-AI
User-agent: AlibabaBot
User-agent: AliyunBot
User-agent: TencentBot
User-agent: HuaweiPetalBot
User-agent: 360Spider
User-agent: Sogou
User-agent: NaverBot
User-agent: KakaoBot
User-agent: LineBot
User-agent: YahooJapan
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# Research / Dataset collectors
# =========================
User-agent: CCBot
User-agent: CommonCrawl
User-agent: Diffbot
User-agent: BrightDataBot
User-agent: PetalBot
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: Scrapy
User-agent: python-requests
User-agent: curl
User-agent: wget
User-agent: Go-http-client
User-agent: libwww-perl
User-agent: HTTrack
User-agent: WebCopier
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /

# =========================
# Default for any other bots
# =========================
User-agent: *
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /internal/
Allow: /