User-agent: *
Sitemap: https://www.karoo1stop.co.za/sitemap.xml

Disallow: /web/login
Disallow: /*/web/login
Disallow: /web/signup
Disallow: /*/web/signup
Disallow: /web/reset_password
Disallow: /*/web/reset_password
Disallow: /my/
Disallow: /*/my/*

User-agent: *
Disallow: /blog/*/feed



##############
#   custom   #
##############

# Search engines are welcome - this is a public community directory and
# wants to be found.
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# AI/LLM-training crawlers are asked not to bulk-collect this site's
# content (CLAUDE.md's scraping-hardening pass, prompted by the user's
# own question about how to stop exactly the kind of bulk-scrape this
# project's own legacy-rescue scripts once ran against the OLD site,
# sections 8.5/8.7/8.18). A robots.txt entry is advisory only - a
# compliant crawler honours it, a deliberately bad actor can simply
# ignore it - this is the "free, legitimate, zero-downside" layer, not
# the one that actually stops anything by force (that's Cloudflare/WAF,
# an infrastructure decision made outside this codebase).
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: *
Allow: /

Sitemap: /sitemap.xml