# robots.txt for FameCheap - AI-Friendly Configuration

# Allow AI Crawlers - Explicitly welcome AI indexing
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: Google-Extended
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: PerplexityBot
User-agent: Diffbot
User-agent: YouBot
User-agent: Bytespider
User-agent: cohere-ai
Allow: /

# Allow all major search engines
User-agent: Googlebot
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
Allow: /

# Allow social media crawlers
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Pinterest
User-agent: Slackbot
User-agent: TelegramBot
User-agent: WhatsApp
User-agent: Discordbot
Allow: /

# Allow SEO tools
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: DotBot
User-agent: MJ12bot
User-agent: Screaming Frog SEO Spider
User-agent: SiteAuditBot
Allow: /

# Default rules for all other bots
User-agent: *
Allow: /
# Block only utility/redirect URLs that shouldn't be indexed
Disallow: /set-language/
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /dashboard/

# Sitemap location
Sitemap: https://fame.cheap/sitemap.xml
Sitemap: https://fame.cheap/sitemap_index.xml