# Prevent Crawling Unnecessary Endpoints - Dynamically added by BigScoots
User-agent: *
Disallow: /cdn-cgi/
Disallow: /wp-admin/
Disallow: /wp-json/
Disallow: /*?s=
Disallow: /*?attachment_id=
Disallow: /*?add-to-cart=
Disallow: /*&add-to-cart=
Disallow: /*?add_to_wishlist=
Disallow: /*&add_to_wishlist=

# Adventure Diaries — robots.txt
# Note: Cloudflare's Content Signals block is prepended to this file at the edge.
# robots.txt is advisory only — malicious bots that ignore it are blocked at Cloudflare.

# ─── Default: allow well-behaved search crawlers, protect admin + noise paths ───
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=

# ─── Opt out of AI training / dataset crawlers ───
User-agent: GPTBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: meta-externalagent
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: FriendlyCrawler
Disallow: /

# ─── Block aggressive nuisance crawlers (the polite ones obey; the rest are blocked at Cloudflare) ───
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: BLEXBot
Disallow: /

Sitemap: https://adventurediaries.com/sitemap_index.xml