htaccess.txt

# Take what you want from this suggested rule set. It is EXTENSIVE. But the knock-on effect of scraping and training is worse, so...
# These cover partial matches - so will catch similarly named agents *by design*. Everything is escaped. The alphabetic ordering is a bit approximate - I'm very open to collaboration to improve further.
# I generally shorten names to the point where they become unlikely to be anything else, or anything unsuspicious. This also helps when companies with a clear user agent decide to change their user agent slightly.
# For efficiency, I don't usually include any user agent that would be blocked by a generic suspicious name rule unless they're a severe enough operation that it makes sense to double up in case they change it later.
# You can put your other rewrites in here, just watch out for the order.
# It's case insensitive. I just normalised an earlier version once upon a time.
# If you believe something is too broad or doesn't belong on the list, please submit an issue with your reasoning and I'll check it out. Note that any user agent that ignores robots.txt will not be removed.


# BEGIN CODEWORDCREATIVE BOT BLOCK https://github.com/codewordcreative/bot-block-list - this won't link to this, it just makes this easier to maintain.
<IfModule mod_rewrite.c>
   RewriteEngine On

   # Force HTTPS - uncomment the below if you want to integrate the standard http > https redirect into this block. Unrelated to these rules, but perhaps helpful to show where to put it.
   # RewriteCond %{HTTPS} !=on
   # RewriteRule ^ https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301]
    
    # Allow all user agents to access robots.txt. Where they are on both lists, it may stop them from trying again if they obey it. Note: Don't use this if using WordPress or any CMS or plugin that dynamically generates robots.txt. Alternatively, you can use this to take control over robots.txt - just remember to add one manually.
   # RewriteCond %{REQUEST_URI} ^/robots\.txt$ [NC]
   # RewriteRule .* - [L]

  #Allow list - allow through any useful bots that may fail other checks
   RewriteCond %{HTTP_USER_AGENT} "(facebookexternalhit|Facebot|WhatsApp|Twitterbot|LinkedInBot|Slackbot|Discordbot|TeamsBot|Outlook|Applebot|TelegramBot|Pinterestbot|Redditbot)" [NC]
RewriteRule ^ - [L]


   # Block SEO tools you don't use. Commented out, because you need to make a conscious choice about these and be aware that you may want to block ones that I have blocked, or unblock ones I have blocked.
   # RewriteCond %{HTTP_USER_AGENT} (Ahrefs|Barkrowler|BLEXBot|Cocolyzebot|DataForSeoBot|DomainStatsBot|dotbot|hypestat|Keyword\ Density/0\.9|linkdexbot|MegaIndex|MJ12bot|online-webceo-bot|Openfind|Nutch|Rogerbot|Semrush|SenutoBot|SEOkicks|SEOlizer|serpstatpot|scrapy|SiteCheckerBot|SISTRIX|SnapPreviewBot|Sogou|spbot|Zeus|ZoomBot) [NC]
   # RewriteRule ^ - [F,L]

   # Block suspicious or outdated user agents. Script kiddies use these a lot - it's mad how much this cuts out.
   RewriteCond %{HTTP_USER_AGENT} (Windows\ NT\ 5\.1|Firefox/11\.0|Chrome/45\.0|Not-A\.Brand|Mozilla/5\.0\ \(Windows\ NT\ 15\.0\)|MSIE\ 10\.0|Firefox/39\.0) [NC]
   RewriteRule ^ - [F,L]

   # Block suspicious or malicious bots - note that php, python, crawler etc. are pretty broad. You may need to unblock or whitelist some APIs or scripts manually as needed, or just remove from the list.
   RewriteCond %{HTTP_USER_AGENT} (Missigua\ Locator|larbin|libwww|lwp-trivial|Mata\ Hari|tencent|php|crawl|scrape|spider|spyder|harvest|collector|python|curl|nutch|stripper|wget) [NC]
   RewriteRule .* - [F,L]

   # Block specific bots, especially potential scrapers/copiers, also intel gatherers - now also archive.org because AI scraapers use that
   RewriteCond %{HTTP_USER_AGENT} (008|abonti|adbeat_bot|addsearchbot|ai2bot|aihitbot|amazonbot|amazon-kendra|anderspinkbot|andibot|anthropic|archiver|awario|acapbot|acoonbot|alexibot|anthropic|archive\.org|asterias|attackbot|backdor|base64_decode|bash|bdcbot|bedrockbot|bigsur|bitsightbot|blackboard|brandverity|brightbot|builtbottough|bytedance|becomebot|binlar|blackwidow|blekkobot|blex|blowfish|bullseye|bunnys|butterfly|c99|careerbot|casper|censysinspect|ccbot|chatgpt|chatglm|checkpriv|cherrypick|chinaclaw|choppy|cincraw|clshttp|cmsworld|copernic|copier|copyrightcheck|cosmos|cotoyogi|crescent|cheesebot|cherrypicker|claude|cohere|devin|dataprovider|diffbot|download\ ninja|datacha|DeepSeek|diavol|discobot|dotnetdotcom|dumbot|disconnect|echobot|econtext|email|eolasbot|eval|eventures|extract|eyenetie|facebookbot|factset|feedbooster|foobot|feedfinder|flaming|flashget|flicky|fuck|genieo|gptbot|grub-client|g00g1e|getright|gigabot|go-ahead-got|Google-CloudVertex|GoogleOther|gozilla|grabnet|grafula|heritrix|hloader|httrack|icarus6j|imagesiftbot|img2dataset|internetseer|isscyber|jetbot|jetcar|kangaroo|klibweb|kmccrew|TheKnowledgeAI|leechftp|libweb|libwww-perl|liebaofast|linerbot|linkwalker|linkextractor|linkfluence|linko|linkwalker|lwp-download|magpie|majestic|mauibot|mediapartners-google|mediatoolkitbot|mechanize|meta-externalagent|metainspector|miner|mippin|mister\ pix|mistral|moget|muckrack|morfeus|moveoverbot|mozlila|nbot|netEstate|netmechanic|nicerspro|nikto|ninja|nominet|netants|netcraftsurvey|netestate|netvibes|news-please|newspaper|novaact|oa-searchbot|OAI-SearchBot|offline\ explorer|omgili|octopus|operator|oppo|pagegrabber|pandalytics|pangubot|panscient|peer39|perplexity|petalbot|phantom|phindbot|planetwork|poseidon|postrank|propowerbot|proximic|prowebwalker|purebot|quillbot|quora|queryn|queryseeker|radian6|radiation|realdownload|remoteview|rogerbot|repomonkey|sbintuition|scan|scoop|scooter|seekerspid|seekportbot|semalt|sider|smtbot|spinn3r|shell|siclab|sidetrade|sindice|sistrix|sitebot|siteexplorer|sitesnagger|skygrid|smartdownload|snoopy|spankbot|spbot|sqlmap|stackrambler|sucker|surftbot|sux0r|suzukacz|suzuran|takeout|teleport|telesoft|textrazor|thinkbot|true_robots|turingos|turnit|timpibot|trendiction|trendsmap|turnitin|tweetmemebot|twingly|um-ln|unserializ|vampire|velenpublic|virustotal|voideye|w4mwnpbXf3MFAbxOkJRw|WARDBot|web\ downloader|web\ image|webauto|webbandit|webenhancer|webmasterworld|webleacher|webreaper|websauger|website\ quester|webster|webstripper|webvac|webviewer|webwhacker|webzio|webzip|winhttp|wwwoffle|woxbot|wp-bot|www-collector|xenu|xaldon|xxxyy|yak|yandexadditional|yamanalab|yioopbot|youda|youbot|zoominfobot|zeus|zmeu|zune|zyborg) [NC]
   RewriteRule ^ - [F,L]

   # Block IP ranges - these are some IPs used by ChatGPT a while back, but ChatGPT/OpenAI is one of those bigger companies that uses a consistent user agent - so no real need. Commented out and left as an example for other IPs.
   # RewriteCond %{REMOTE_ADDR} ^52\.230\.152\.[0-9]{1,3}$ [OR]
   # RewriteCond %{REMOTE_ADDR} ^52\.233\.106\.[0-9]{1,3}$ [OR]
   # RewriteCond %{REMOTE_ADDR} ^20\.171\.206\.[0-9]{1,3}$
   # RewriteRule .* - [F,L]

   # Block malicious request patterns (8G Firewall). This too is pretty broad - consider unblocking or whitelisting as needed.
   RewriteCond %{QUERY_STRING} (eval\(|UNION.*SELECT|base64_decode\(|/etc/passwd|self/environ|input_file|input_file_name|mosConfig) [NC,OR]
   RewriteCond %{QUERY_STRING} (GLOBALS|_REQUEST|proc/self|wget|curl|python|<script|<iframe) [NC]
   RewriteRule .* - [F,L]

   # Block bad request methods - note that sometimes these have legit purposes, but you'll usually know about them.
   RewriteCond %{REQUEST_METHOD} !^(GET|POST|HEAD|OPTIONS) [NC]
   RewriteRule .* - [F,L]

   # Block URL encoding attacks (8G Firewall)
   RewriteCond %{THE_REQUEST} "%20--" [NC,OR]
   RewriteCond %{THE_REQUEST} "%0A" [NC,OR]
   RewriteCond %{THE_REQUEST} "%0D" [NC]
   RewriteRule .* - [F,L]

   # Block dangerous query strings (8G Firewall)
   RewriteCond %{QUERY_STRING} (\.\.\/|\.\.\\) [NC,OR]
   RewriteCond %{QUERY_STRING} (<|\"|'|>|\%3C|\%3E) [NC]
   RewriteRule ^.*$ - [F,L]

</IfModule>
# END CODEWORD CREATIVE BOT BLOCK