# robots.txt for https://www.digitalstudio.in/
# Last updated: June 11, 2026

# -------------------------
# Standard Search Engines
# -------------------------
User-agent: *
Allow: /
Crawl-delay: 1

# -------------------------
# AI Search & Discovery (Search-based AI)
# -------------------------
# OpenAI's Search crawler (SearchGPT)
User-agent: OAI-SearchBot
Allow: /

# OpenAI's general crawler
User-agent: GPTBot
Allow: /

# User-initiated ChatGPT requests
User-agent: ChatGPT-User
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

# Google-Extended (Controls Gemini and Vertex AI)
User-agent: Google-Extended
Allow: /

# Apple Intelligence (Applebot-Extended)
User-agent: Applebot-Extended
Allow: /

# -------------------------
# AI Training Scrapers (Foundation Model Seeding)
# -------------------------
# Common Crawl (Feeds many Open Source LLMs like Llama)
User-agent: CCBot
Allow: /

# -------------------------
# Important Files & Metadata
# -------------------------
# Pointing AI to your curated summary
# Llm-policy: https://www.digitalstudio.in/llms.txt
# Ai-Policy: https://www.digitalstudio.in/llms.txt

# Sitemaps
Sitemap: https://www.digitalstudio.in/GoogleMap.xml
Sitemap: https://www.digitalstudio.in/video-sitemap.xml
Sitemap: https://www.digitalstudio.in/image-sitemap.xml