# robots.txt for treatlyme.com
# Optimized for WordPress, WPML, SmartCrawl Pro, & Modern Bots

# -----------------------------------------
# GLOBAL BOT RULES
# -----------------------------------------
User-agent: *

# WP Defender - Begin
Disallow: /a9216b4df098af2f
# WP Defender - End

Crawl-delay: 1

# Allow essential WordPress AJAX functionality
Allow: /wp-admin/admin-ajax.php

# Allow rendering assets (CSS, JS, Fonts, Images)
Allow: /wp-includes/*.css
Allow: /wp-includes/*.js
Allow: /wp-content/themes/*.css
Allow: /wp-content/themes/*.js
Allow: /wp-content/themes/*.woff
Allow: /wp-content/themes/*.woff2
Allow: /wp-content/themes/*.ttf
Allow: /wp-content/themes/*.svg
Allow: /wp-content/plugins/*.css
Allow: /wp-content/plugins/*.js
Allow: /wp-content/plugins/*.svg
Allow: /wp-content/uploads/*.jpg
Allow: /wp-content/uploads/*.jpeg
Allow: /wp-content/uploads/*.png
Allow: /wp-content/uploads/*.gif
Allow: /wp-content/uploads/*.webp
Allow: /favicon.ico

# WPML: Allow language-specific paths
Allow: /*/wp-content/uploads/
Allow: /*/wp-content/themes/

# Block WordPress system files & directories
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/wp-rocket-cache/
Disallow: /wp-content/cache/wp-rocket/
Disallow: /wp-content/upgrade/
Disallow: /wp-content/backup-db/
Disallow: /wp-content/uploads/wpcf7_uploads/
Disallow: /wp-content/uploads/cache/
Disallow: /wp-content/uploads/smush-webp/
Disallow: /wp-content/advanced-cache.php
Disallow: /wp-content/wp-cache-config.php
Disallow: /wp-content/debug.log
Disallow: /readme.html
Disallow: /license.txt
Disallow: /xmlrpc.php
Disallow: /wp-mail.php
Disallow: /wp-config.php
Disallow: /wp-config-sample.php
Disallow: /wp-cron.php
Disallow: /wp-signup.php
Disallow: /wp-login.php
Disallow: /wp-activate.php
Disallow: /wp-comments-post.php
Disallow: /wp-trackback.php

# Block theme sensitive text files
Disallow: /wp-content/themes/*/readme.txt
Disallow: /wp-content/themes/*/README.txt
Disallow: /wp-content/themes/*/license.txt
Disallow: /wp-content/themes/*/LICENSE.txt

# Block URL parameters (Search, Filtering, Sorting)
Disallow: /*?s=*
Disallow: /*&s=*
Disallow: /*?p=*
Disallow: /*&p=*
Disallow: /*?filter=*
Disallow: /*&filter=*
Disallow: /*?orderby=*
Disallow: /*&orderby=*
Disallow: /*?m=*
Disallow: /*&m=*
Disallow: /*?cat=*
Disallow: /*&cat=*
Disallow: /*?lang=*
Disallow: /*&lang=*
Disallow: */replytocom=*
Disallow: /*?b=
Disallow: /*&b=
Disallow: /*?p=
Disallow: /*&p=
Disallow: /*?srch=
Disallow: /*&srch=
Disallow: /products/*?b=
Disallow: /products/*?*&b=
Disallow: /products/*?p=
Disallow: /products/*?*&p=
Disallow: /products/*?srch=
Disallow: /products/*&srch=

# Block sensitive and low-value pages
# Note: Tags, categories, and author pages are intentionally omitted here 
# so SmartCrawl's 'noindex' directives can be properly crawled by Google.
Disallow: */trackback/
Disallow: */feed/
Disallow: */embed/
Disallow: */comments/feed/
Disallow: /404/
Disallow: /search/

# WPML Duplicate content prevention
Disallow: /wp-content/cache/wpml/
Disallow: /wp-content/uploads/cache/wpml/
Disallow: /wp-content/plugins/sitepress-multilingual-cms/

# -----------------------------------------
# SPECIFIC BOT LIMITATIONS & BLOCKS
# -----------------------------------------

# Block AI Data Scrapers & LLM Training Bots Completely
User-agent: GPTBot
User-agent: Google-Extended
User-agent: CCBot
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Bytespider
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: ai2bot
User-agent: ai2bot-dolma
Disallow: /

# Limit AI Search & Citation Bots (Allowed to crawl for AI answers, but throttled)
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Claude-Web
User-agent: Claude-SearchBot
User-agent: PerplexityBot
Crawl-delay: 2
Disallow: /wp-admin/
Disallow: /wp-includes/

# Limit aggressive Social Bots
User-agent: meta-externalagent
User-agent: FacebookBot
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /wp-includes/

# Limit aggressive SEO bots
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
Crawl-delay: 1

# Block potentially harmful or resource-intensive bots completely
User-agent: PetalBot
User-agent: DataForSeoBot
User-agent: DotBot
User-agent: SEOkicks
User-agent: Sogou web spider
User-agent: Sogou inst spider
User-agent: ia_archiver
User-agent: alexibot
User-agent: Barkrowler
User-agent: AspiegelBot
User-agent: Aboundexbot
User-agent: ExtLinksBot
User-agent: spbot
User-agent: twengabot
User-agent: Applebot
User-agent: BLEXBot
User-agent: LinkpadBot
User-agent: magpie-crawler
User-agent: HTTrack
User-agent: wget
User-agent: Nutch
User-agent: WBSearchBot
User-agent: MegaIndex
User-agent: ltx71
User-agent: TheNomad
User-agent: Nimbostratus-Bot
User-agent: DomainStatsBot
Disallow: /

Sitemap: https://treatlyme.com/sitemap.xml