# =========================================================
# Digital Cyril — Robots.txt
# SEO + AI Search + LLM Visibility
# Last reviewed: October 2026
# =========================================================

# ---------------------------------------------------------
# DEFAULT — ALL CRAWLERS
# ---------------------------------------------------------
User-agent: *
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php


# ---------------------------------------------------------
# OPENAI — CHATGPT SEARCH
# ---------------------------------------------------------
# OAI-SearchBot helps surface websites in ChatGPT search.
# Keep public content crawlable.
# ---------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php


# ---------------------------------------------------------
# OPENAI — GPT TRAINING CRAWLER
# ---------------------------------------------------------
# GPTBot controls crawling for OpenAI foundation-model training.
# ---------------------------------------------------------
User-agent: GPTBot
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php


# ---------------------------------------------------------
# ANTHROPIC — CLAUDE
# ---------------------------------------------------------
User-agent: ClaudeBot
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php


# ---------------------------------------------------------
# PERPLEXITY
# ---------------------------------------------------------
User-agent: PerplexityBot
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php


# ---------------------------------------------------------
# GOOGLE — GEMINI / AI SYSTEMS
# ---------------------------------------------------------
# Google-Extended controls use of crawled content for
# Gemini-related model training/grounding.
# It does NOT control Google Search rankings.
# ---------------------------------------------------------
User-agent: Google-Extended
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php


# ---------------------------------------------------------
# BYTE DANCE / TIKTOK
# ---------------------------------------------------------
User-agent: Bytespider
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php


# ---------------------------------------------------------
# SITEMAPS
# ---------------------------------------------------------
Sitemap: https://digitalcyril.in/sitemap.xml
Sitemap: https://digitalcyril.in/page-sitemap.xml
Sitemap: https://digitalcyril.in/post-sitemap.xml
Sitemap: https://digitalcyril.in/sitemap-ai.xml
Sitemap: https://digitalcyril.in/llm-sitemap.xml