User-agent: *
Allow: /

# Disallow authenticated pages
Disallow: /app/
Disallow: /dashboard
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password

# Disallow admin areas
Disallow: /horizon
Disallow: /admin

# Disallow API endpoints
Disallow: /api/
Disallow: /telegram/webhook

# AI Bot Access Control
# OpenAI GPTBot - Allow crawling for AI training
User-agent: GPTBot
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /horizon

# Anthropic Claude-Web - Allow crawling
User-agent: Claude-Web
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /horizon

# Google AI Bot (Gemini)
User-agent: Google-Extended
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /horizon

# Bing AI Bot
User-agent: BingBot
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /horizon

# Facebook AI Bot
User-agent: facebookexternalhit
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/

# Apple AI Bot
User-agent: Applebot-Extended
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /horizon

# CCBot (Common Crawl - used by various AI services)
User-agent: CCBot
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /horizon

Sitemap: https://www.sonodemegunu.com/sitemap.xml
RSS: https://www.sonodemegunu.com/feed