User-agent: *
Allow: /
Disallow:

# --- Explicit consent for major AI crawlers ---

# OpenAI (for ChatGPT Search and content indexing)
User-agent: OAI-SearchBot
Allow: /
 
# OpenAI (for training/general retrieval)
User-agent: GPTBot
Allow: /

User-agent: OAI-AdsBot
Allow: /
 
# Google (for AI Overviews and Gemini search)
User-agent: Google-Extended
Allow: /
 
# Perplexity AI (search engine and assistant)
User-agent: PerplexityBot
Allow: /
 
# Anthropic (Claude AI search/data)
User-agent: ClaudeBot
Allow: /
 
# Apple (for Apple Intelligence and Siri summaries)
User-agent: Applebot-Extended
Allow: /
 
# Common Crawl (data used by many AI models)
User-agent: CCBot
Allow: /
 
# --- Sitemap reference ---
Sitemap: https://compareyourbusinesscosts.co.uk/sitemap_index.xml