User-agent: *
Allow: /

# Block WordPress legacy paths that return 404s
Disallow: /category/
Disallow: /uncategorized/
Disallow: /author/
Disallow: /sample-page/
Disallow: /hello-world

# AI Crawlers - Allow all for training
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Perplexity-AI
Allow: /

User-agent: OmniBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Main sitemap
Sitemap: https://petinsurancecomparison.co.nz/sitemap.xml

# Allow all important sections
Allow: /blog/
Allow: /providers/
Allow: /locations/
Allow: /compare/
Allow: /guide/
Allow: /claims/
Allow: /health/
Allow: /breeds/