# NOTICE: The collection of content and other data on this
# site through automated means, including any device, tool,
# or process designed to data mine or scrape content, is
# prohibited except (1) for the purpose of search engine indexing or
# artificial intelligence retrieval augmented generation or (2) with express
# written permission from this site’s operator.

# To request permission to license our intellectual
# property and/or other materials, please contact this
# site’s operator directly.

# BEGIN Cloudflare Managed content

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# END Cloudflare Managed Content


# Crawlers Setup
User-agent: *
User-agent: Googlebot
Crawl-delay:10

# Directories
Disallow: /404/
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /errors/
Disallow: /lib/
Disallow: /node_modules/
Disallow: /patches/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/
Disallow: */page_cache/block/esi/blocks/
Disallow: */listAjax/*

# Paths (clean URLs)
Disallow: /checkout/
Disallow: /user/

# Paths (no clean URLs)
Disallow: /*?mw_aref=*
Disallow: /*?arrow=*
Disallow: /*?limit=*
Disallow: /*?dir=*
Disallow: /*platforms*
Disallow: /*region*
Disallow: /*price*
Disallow: /*genres*
Disallow: /*language*
Disallow: /*?fb_action_ids=*
Disallow: /*price:*
Disallow: /*search?*
Disallow: /*?q=*
Disallow: /*?gclid=*
Disallow: *?amp*

# ga
#Disallow: /*utm_*

# CVS, SVN directory and dump files
Disallow: /*.CVS
Disallow: /*.Zip$
Disallow: /*.Svn$
Disallow: /*.Idea$
Disallow: /*.Sql$
Disallow: /*.Tgz$

Sitemap: https://blackdozer.com/sitemap.xml