Skip to content

Commit 1747f73

Browse files
author
Mikhail Kobzarev
committed
feat(recrawler-33): Добавить блокировку AI-ботов
1 parent 374075a commit 1747f73

9 files changed

Lines changed: 253 additions & 37 deletions

File tree

‎assets/images/icons/icon.svg‎

Lines changed: 9 additions & 0 deletions
Loading

‎composer.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,14 @@
3333
"prefer-stable": true,
3434
"config": {
3535
"platform": {
36-
"php": "7.4"
36+
"php": "8.1"
3737
},
3838
"allow-plugins": {
3939
"kylekatarnls/update-helper": true
4040
}
4141
},
4242
"require": {
43-
"php": ">=7.4",
43+
"php": ">=8.1",
4444
"ext-json": "*",
4545
"psr/log": "^1.1",
4646
"google/apiclient": "^2.12.1"

‎readme.txt‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ Contributors: mihdan
33
Donate link: https://www.kobzarev.com/donate/
44
Tags: indexnow, yandex, bing, google, seo
55
Requires at least: 6.0
6-
Tested up to: 6.6
7-
Stable tag: 0.1.5
8-
Requires PHP: 7.4
6+
Tested up to: 6.9
7+
Stable tag: 0.2.0
8+
Requires PHP: 8.1
99
License: GPLv2 or later
1010
License URI: https://www.gnu.org/licenses/gpl-2.0.html
1111

@@ -150,6 +150,10 @@ See the documentation available from each search engine for more details about R
150150

151151
== Changelog ==
152152

153+
= 0.2.0 (06.04.2025) =
154+
* Добавлена возможность блокировки ИИ-ботов
155+
* Поднята минимальная версия РНР до 8.1+
156+
153157
= 0.1.5 (09.09.2024) =
154158
* Fixed integration bug with "LuckyWP Table of Contents" plugin
155159

‎recrawler.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
/**
33
* Plugin Name: ReCrawler
44
* Description: ReCrawler is a small WordPress Plugin for quickly notifying search engines whenever their website content is created, updated, or deleted.
5-
* Version: 0.1.5
5+
* Version: 0.2.0
66
* Author: Mikhail Kobzarev
77
* Author URI: https://www.kobzarev.com/
88
* Plugin URI: https://wordpress.org/plugins/recrawler/
@@ -22,7 +22,7 @@
2222
exit;
2323
}
2424

25-
define( 'RECRAWLER_VERSION', '0.1.5' );
25+
define( 'RECRAWLER_VERSION', '0.2.0' );
2626
define( 'RECRAWLER_SLUG', 'recrawler' );
2727
define( 'RECRAWLER_PREFIX', 'recrawler' );
2828
define( 'RECRAWLER_NAME', 'ReCrawler' );

‎src/Enums/BlockAiCrawlerEnum.php‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
<?php
2+
namespace Mihdan\ReCrawler\Enums;
3+
4+
enum BlockAiCrawlerEnum: string
5+
{
6+
case CHATGPT_USER = 'ChatGPT-User';
7+
case BING_AI = 'BingAI';
8+
case OPENAI = 'OpenAI';
9+
case ANTHROPIC_AI = 'AnthropicAI';
10+
case JASPER_AI = 'JasperAI';
11+
case AI_CONTENT_DETECTOR = 'AI Content Detector';
12+
case AI_SEO_CRAWLER = 'AI SEO Crawler';
13+
case GRAMMARLY = 'Grammarly';
14+
case COPYSCAPE = 'Copyscape';
15+
case QUILLBOT = 'QuillBot';
16+
case WRITESONIC = 'Writesonic';
17+
case HYPOTENUSE_AI = 'Hypotenuse AI';
18+
case COPY_AI = 'CopyAI';
19+
case FRASE_AI = 'Frase AI';
20+
case CONTENTBOT = 'ContentBot';
21+
case DEEPAI = 'DeepAI';
22+
case INFERKIT = 'Inferkit';
23+
case SUDOWRITE = 'Sudowrite';
24+
case AI_WRITER = 'AI Writer';
25+
case INK_EDITOR = 'INK Editor';
26+
case SCALENUT = 'Scalenut';
27+
case WRITECREAM = 'Writecream';
28+
case ZIMM_WRITER = 'ZimmWriter';
29+
case SCALENUT_BOT = 'ScalenutBot';
30+
case CONTENTEDGE = 'Contentedge';
31+
case RYTR = 'Rytr';
32+
case ANYWORD = 'Anyword';
33+
case WORDTUNE = 'Wordtune';
34+
case WORDAI = 'WordAI';
35+
case SPIN_REWRITER = 'Spin Rewriter';
36+
case NEURAL_TEXT = 'Neural Text';
37+
case WRITESCOPE = 'Writescope';
38+
case SIMPLIFIED_AI = 'Simplified AI';
39+
case TEXT_BLAZE = 'Text Blaze';
40+
case OPENTEXT_AI = 'OpenText AI';
41+
case DEEPL = 'DeepL';
42+
case SAPLING_AI = 'SaplingAI';
43+
case COPYMATIC = 'Copymatic';
44+
case AI_DUNGEON = 'AI Dungeon';
45+
case NARRATIVE_DEVICE = 'Narrative Device';
46+
case TEXTCORTEX = 'TextCortex';
47+
case AI21_LABS = 'AI21 Labs';
48+
case WRITER_ZEN = 'WriterZen';
49+
case OUTWRITE = 'Outwrite';
50+
case SEO_CONTENT_MACHINE = 'SEO Content Machine';
51+
case CRAWLQ_AI = 'CrawlQ AI';
52+
case SLICKWRITE = 'SlickWrite';
53+
case PROWRITING_AID = 'ProWritingAid';
54+
case HEMINGWAY_EDITOR = 'Hemingway Editor';
55+
case CONTENT_HARMONY = 'Content Harmony';
56+
case CONTENT_KING = 'Content King';
57+
case ROBOT_SPIDER = 'RobotSpider';
58+
case CONTENT_AT_SCALE = 'ContentAtScale';
59+
case SURFER_AI = 'Surfer AI';
60+
case INK_FOR_ALL = 'INKforall';
61+
case CLEARSCOPE = 'ClearScope';
62+
case MARKETMUSE = 'MarketMuse';
63+
case NEURAL_SEO = 'NeuralSEO';
64+
case CONVERSION_AI = 'Conversion AI';
65+
case CONTENT_SAMURAI = 'Content Samurai';
66+
case VIDNAMI_AI = 'Vidnami AI';
67+
case KAFKAI = 'Kafkai';
68+
case PARAPHRASER_IO = 'Paraphraser.io';
69+
case SPINBOT = 'Spinbot';
70+
case ARTICOOLO = 'Articoolo';
71+
case AI_ARTICLE_WRITER = 'AI Article Writer';
72+
case SEO_ROBOT = 'SEO Robot';
73+
case AI_SEARCH_ENGINE = 'AI Search Engine';
74+
case AUTOMATED_WRITER = 'Automated Writer';
75+
case SCRIPTBOOK = 'ScriptBook';
76+
case KEYWORD_DENSITY_AI = 'Keyword Density AI';
77+
case METATAG_BOT = 'MetaTagBot';
78+
case CONTENT_OPTIMIZER = 'Content Optimizer';
79+
case PAGE_ANALYZER_AI = 'Page Analyzer AI';
80+
81+
public static function toArray(): array
82+
{
83+
return array_reduce(
84+
self::cases(),
85+
static fn($carry, $case) => array_merge($carry, [$case->name => $case->value]),
86+
[]
87+
);
88+
}
89+
}

‎src/Main.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
use Mihdan\ReCrawler\Providers\IndexNow\IndexNow;
1616
use Mihdan\ReCrawler\Providers\Seznam\SeznamIndexNow;
1717
use Mihdan\ReCrawler\Providers\Naver\NaverIndexNow;
18+
use Mihdan\ReCrawler\Providers\BlockAiCrawler;
1819
use Mihdan\ReCrawler\Providers\Yandex\YandexIndexNow;
1920
use Mihdan\ReCrawler\Providers\Yandex\YandexWebmaster;
2021
use Mihdan\ReCrawler\Views\HelpTab;
@@ -94,6 +95,7 @@ function( Container $c ) {
9495
$this->wposa = $this->container->get( WPOSA::class );
9596
$this->wposa->setup_hooks();
9697

98+
( $this->container->make( BlockAiCrawler::class ) )->setup_hooks();
9799
( $this->container->make( Hooks::class ) )->setup_hooks();
98100

99101
( $this->container->make( HelpTab::class ) )->setup_hooks();

‎src/Providers/BlockAiCrawler.php‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
<?php
2+
/**
3+
* Block AI crawlers.
4+
*
5+
* @package Mihdan\ReCrawler
6+
*/
7+
8+
namespace Mihdan\ReCrawler\Providers;
9+
10+
use Mihdan\ReCrawler\Enums\BlockAiCrawlerEnum;
11+
use Mihdan\ReCrawler\Views\WPOSA;
12+
13+
class BlockAiCrawler {
14+
15+
private WPOSA $wposa;
16+
17+
public function __construct( WPOSA $wposa ) {
18+
$this->wposa = $wposa;
19+
}
20+
21+
public function setup_hooks(): void {
22+
add_action( 'init', [ $this, 'maybe_block' ] );
23+
add_action( 'do_robots', [ $this, 'add_robots_rules' ] );
24+
}
25+
26+
public function maybe_block(): void {
27+
$bots = $this->wposa->get_option( 'bots', 'ai', [] );
28+
29+
if ( empty( $bots ) || ! is_array( $bots ) ) {
30+
return;
31+
}
32+
33+
$user_agent = $_SERVER['HTTP_USER_AGENT'] ?? ''; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput
34+
35+
if ( empty( $user_agent ) ) {
36+
return;
37+
}
38+
39+
foreach ( BlockAiCrawlerEnum::cases() as $case ) {
40+
if ( ! isset( $bots[ $case->name ] ) ) {
41+
continue;
42+
}
43+
44+
if ( stripos( $user_agent, $case->value ) !== false ) {
45+
$this->block();
46+
return;
47+
}
48+
}
49+
}
50+
51+
public function add_robots_rules(): void {
52+
$bots = $this->wposa->get_option( 'bots', 'ai', [] );
53+
54+
if ( empty( $bots ) || ! is_array( $bots ) ) {
55+
return;
56+
}
57+
58+
foreach ( BlockAiCrawlerEnum::cases() as $case ) {
59+
if ( ! isset( $bots[ $case->name ] ) ) {
60+
continue;
61+
}
62+
63+
echo "User-agent: {$case->value}\n";
64+
echo "Disallow: /\n\n";
65+
}
66+
}
67+
68+
private function block(): void {
69+
status_header( 403 );
70+
nocache_headers();
71+
exit;
72+
}
73+
}

‎src/Views/Settings.php‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
namespace Mihdan\ReCrawler\Views;
99

10+
use Mihdan\ReCrawler\Enums\BlockAiCrawlerEnum;
1011
use Mihdan\ReCrawler\Utils;
1112

1213
/**
@@ -143,7 +144,7 @@ public function setup_fields() {
143144
]
144145
);
145146

146-
$this->wposa->add_section(
147+
$this->wposa->add_tab(
147148
array(
148149
'id' => 'general',
149150
'title' => __( 'General', 'recrawler' ),
@@ -258,7 +259,24 @@ public function setup_fields() {
258259
)
259260
);
260261

261-
$this->wposa->add_section(
262+
$this->wposa->add_tab(
263+
array(
264+
'id' => 'ai',
265+
'title' => __( 'AI', 'recrawler' ),
266+
)
267+
);
268+
269+
$this->wposa->add_field(
270+
'ai',
271+
array(
272+
'id' => 'bots',
273+
'type' => 'multicheck',
274+
'name' => __( 'Block AI Crawlers', 'recrawler' ),
275+
'options' => BlockAiCrawlerEnum::toArray(),
276+
)
277+
);
278+
279+
$this->wposa->add_tab(
262280
array(
263281
'id' => 'index_now',
264282
'title' => __( 'IndexNow', 'recrawler' ),
@@ -310,7 +328,7 @@ public function setup_fields() {
310328
)
311329
);
312330

313-
$this->wposa->add_section(
331+
$this->wposa->add_tab(
314332
array(
315333
'id' => 'bing_webmaster',
316334
'title' => __( 'Bing API', 'recrawler' ),
@@ -338,7 +356,7 @@ public function setup_fields() {
338356
)
339357
);
340358

341-
$this->wposa->add_section(
359+
$this->wposa->add_tab(
342360
array(
343361
'id' => 'google_webmaster',
344362
'title' => __( 'Google API', 'recrawler' ),
@@ -367,7 +385,7 @@ public function setup_fields() {
367385
)
368386
);
369387

370-
$this->wposa->add_section(
388+
$this->wposa->add_tab(
371389
array(
372390
'id' => 'yandex_webmaster',
373391
'title' => __( 'Yandex API', 'recrawler' ),
@@ -485,7 +503,7 @@ public function setup_fields() {
485503
)
486504
);
487505

488-
$this->wposa->add_section(
506+
$this->wposa->add_tab(
489507
array(
490508
'id' => 'logs',
491509
'title' => __( 'Logs', 'recrawler' ),
@@ -554,7 +572,7 @@ public function setup_fields() {
554572
)
555573
);
556574

557-
$this->wposa->add_section(
575+
$this->wposa->add_tab(
558576
array(
559577
'id' => 'plugins',
560578
'reset_button' => false,

0 commit comments

Comments
 (0)