# Robots.txt for UDGOK
# https://udgok.com

# ============================================
# ALLOW ALL AI CRAWLERS FOR CITATIONS
# ============================================

# OpenAI - ChatGPT / SearchGPT
User-agent: OAI-SearchBot
Allow: /
User-agent: GPTBot
Allow: /

# Anthropic - Claude
User-agent: Claude-SearchBot
Allow: /
User-agent: Claude-User
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Mistral AI
User-agent: MistralAI-User
Allow: /

# Google - Gemini / AI Overviews
User-agent: Google-Extended
Allow: /

# Common Crawl (used by various AI systems)
User-agent: CCBot
Allow: /

# DuckDuckGo (used by AI features)
User-agent: DuckDuckBot
Allow: /

# ============================================
# GENERAL SETTINGS
# ============================================

User-agent: *
Allow: /

# Sitemap location
Sitemap: https://udgok.com/sitemap.xml

# Crawl-delay (optional, for politeness)
Crawl-delay: 1