User-Agent: *
Allow: /
Disallow: /admin
Disallow: /migration
Disallow: /setup-metafields
Disallow: /cart
Disallow: /*?s=
Disallow: /*?orderby=
Disallow: /*?min_price=
Disallow: /*?filter_
Disallow: /*?add-to-cart=

User-agent: AdsBot-Google
Disallow:

# AI assistants & AI-search crawlers are explicitly welcome (see also /llms.txt).
# NOTE: in production Cloudflare may prepend its own "Managed content" block that
# can override these rules (e.g. Disallow: / for GPTBot). To actually be visible
# in AI search, AI-bot access must ALSO be allowed in the Cloudflare dashboard
# (AI Audit / Block AI bots). These groups keep the intent explicit on our side.
User-agent: GPTBot
Disallow:

User-agent: OAI-SearchBot
Disallow:

User-agent: ChatGPT-User
Disallow:

User-agent: Google-Extended
Disallow:

User-agent: ClaudeBot
Disallow:

User-agent: Claude-Web
Disallow:

User-agent: anthropic-ai
Disallow:

User-agent: PerplexityBot
Disallow:

User-agent: Perplexity-User
Disallow:

User-agent: Applebot-Extended
Disallow:

User-agent: CCBot
Disallow:

User-agent: Amazonbot
Disallow:

User-agent: Bytespider
Disallow:

User-agent: meta-externalagent
Disallow:

User-agent: cohere-ai
Disallow:

User-Agent: Baiduspider
Disallow: /

User-Agent: CazoodleBot
Disallow: /

User-Agent: Fasterfox
Disallow: /

User-agent: teracent-feed-processing
Disallow: /

User-Agent: Ingrid
Crawl-delay: 20

User-Agent: Jyxobot
Disallow: /

User-Agent: MJ12bot
Disallow: /

User-Agent: ShopWiki
Disallow: /

User-Agent: msnbot
Crawl-Delay: 20

User-Agent: Slurp
Crawl-delay: 20

User-Agent: Yahoo! Slurp
Crawl-delay: 20

User-agent: rogerbot
Crawl-delay: 2

Sitemap: https://www.xaro.nl/sitemap.xml