# Article 370 Knowledge Base - robots.txt
# Copyright © 2024-2026 Article370.com. All rights reserved.
# Last updated: 2026-01-07

# TEMPORARY: Fully allow all crawlers and tools
User-agent: *
Allow: /
Disallow: /CLAUDE.md
Disallow: /.gitignore

# ----------------------------------------
# ORIGINAL RULES (temporarily disabled)
# ----------------------------------------
# Default: Allow all search engine crawlers
# User-agent: *
# Allow: /
# Allow: /assets/img/
# Allow: /assets/css/
# Disallow: /assets/js/
# Disallow: /assets/tmp/
# Disallow: /*.json$
# Crawl-delay: 1

# # ========================================
# # GOOGLE CRAWLERS - EXPLICITLY ALLOWED
# # ========================================
# User-agent: Googlebot
# Allow: /
# Crawl-delay: 0

# User-agent: Googlebot-Image
# Allow: /assets/img/

# User-agent: Googlebot-Mobile
# Allow: /

# User-agent: AdsBot-Google
# Allow: /

# User-agent: AdsBot-Google-Mobile
# Allow: /

# User-agent: Mediapartners-Google
# Allow: /

# # ========================================
# # BING CRAWLERS - EXPLICITLY ALLOWED
# # ========================================
# User-agent: Bingbot
# Allow: /
# Crawl-delay: 1

# User-agent: BingPreview
# Allow: /

# # ========================================
# # AI TRAINING CRAWLERS - BLOCKED
# # (These are NOT search indexing crawlers)
# # ========================================
# User-agent: CCBot
# Disallow: /

# User-agent: GPTBot
# Disallow: /

# User-agent: ChatGPT-User
# Disallow: /

# User-agent: Google-Extended
# Disallow: /

# User-agent: anthropic-ai
# Disallow: /

# User-agent: ClaudeBot
# Disallow: /

# User-agent: cohere-ai
# Disallow: /

# # ========================================
# # KNOWN SCRAPERS - BLOCKED
# # ========================================
# User-agent: Bytespider
# Disallow: /

# User-agent: PetalBot
# Disallow: /

# User-agent: Scrapy
# Disallow: /

# User-agent: DataForSeoBot
# Disallow: /

# User-agent: Diffbot
# Disallow: /

# # ========================================
# # SITEMAP
# # ========================================
# Sitemap: https://article370.com/sitemap.xml