# ============================================================
# robots.txt for spoonfeeding.in
# Last updated: March 2026
# ============================================================

# ============================================================
# Allow AI Bots for Search, Citations & Indexing
# ============================================================

# OpenAI (ChatGPT, SearchGPT)
User-agent: OAI-SearchBot
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Google Gemini & Vertex AI
User-agent: Google-Extended
Allow: /

# Gemini Deep Research feature
User-agent: Gemini-Deep-Research
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Amazon (Alexa, Amazon Q)
User-agent: Amazonbot
Allow: /

# Meta AI (Facebook / Instagram AI)
User-agent: FacebookBot
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Common Crawl (Large-scale AI training data)
User-agent: CCBot
Allow: /

# ============================================================
# Google Search & AdSense
# ============================================================

# Google Search (Standard) — Required for Gemini Grounding
User-agent: Googlebot
Allow: /

# Google AdSense — Essential for ad revenue
User-agent: Mediapartners-Google
Disallow:

# ============================================================
# General Rules for all other bots
# ============================================================

User-agent: *
Disallow: /feeds/
Disallow: /search
Allow: /

# ============================================================
# Sitemap
# ============================================================

Sitemap: https://www.spoonfeeding.in/sitemap.xml