From 6578016ddc4bdaef4616469e73345f84bea6c1da Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Sun, 19 Jul 2026 23:27:06 +0800 Subject: [PATCH] fix: allow opting out of anti-bot blocking --- crawl4ai/async_configs.py | 7 ++++++- crawl4ai/async_webcrawler.py | 14 ++++++++++---- tests/proxy/test_antibot_opt_out.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 tests/proxy/test_antibot_opt_out.py diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 27320fd4b..aebe1e3c7 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -264,7 +264,8 @@ class UntrustedConfigError(ValueError): "verbose", "log_console", "capture_network_requests", "capture_console_messages", "method", "stream", "prefetch", "url", "check_robots_txt", "user_agent", "user_agent_mode", - "user_agent_generator_config", "url_matcher", "match_mode", "max_retries", + "user_agent_generator_config", "url_matcher", "match_mode", + "check_blocked", "max_retries", }, } @@ -1430,6 +1431,7 @@ class CrawlerRunConfig(): Default: False. cache_validation_timeout (float): Timeout in seconds for cache validation HTTP requests. Default: 10.0. + check_blocked (bool): Whether anti-bot matches fail the crawl. Default: True. # Page Navigation and Timing Parameters wait_until (str): The condition to wait for when navigating, e.g. "domcontentloaded". @@ -1701,6 +1703,7 @@ def __init__( # Experimental Parameters experimental: Dict[str, Any] = None, # Anti-Bot Retry Parameters + check_blocked: bool = True, max_retries: int = 0, fallback_fetch_function: Optional[Callable[[str], Awaitable[str]]] = None, ): @@ -1898,6 +1901,7 @@ def __init__( self.experimental = experimental or {} # Anti-Bot Retry Parameters + self.check_blocked = check_blocked self.max_retries = max_retries self.fallback_fetch_function = fallback_fetch_function @@ -2184,6 +2188,7 @@ def to_dict(self): "url_matcher": self.url_matcher, "match_mode": self.match_mode, "experimental": self.experimental, + "check_blocked": self.check_blocked, "max_retries": self.max_retries, } diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py index 8216d19bc..83650f79a 100644 --- a/crawl4ai/async_webcrawler.py +++ b/crawl4ai/async_webcrawler.py @@ -505,7 +505,7 @@ async def arun( # Check if blocked (skip for raw: URLs — # caller-provided content, anti-bot N/A) - if _is_raw_url: + if _is_raw_url or not config.check_blocked: _blocked = False _block_reason = "" else: @@ -554,7 +554,8 @@ async def arun( if _fallback_fn and not _done and not _is_raw_url: _needs_fallback = ( crawl_result is None # All proxies threw exceptions - or is_blocked(crawl_result.status_code, crawl_result.html or "")[0] + or (config.check_blocked and is_blocked( + crawl_result.status_code, crawl_result.html or "")[0]) ) if _needs_fallback: self.logger.warning( @@ -625,7 +626,12 @@ async def arun( # empty by design, and is_blocked() would misread "0 bytes # html" as a block. _has_download = bool(getattr(crawl_result, "downloaded_files", None)) - if not _fallback_succeeded and not _is_raw_url and not _has_download: + if ( + config.check_blocked + and not _fallback_succeeded + and not _is_raw_url + and not _has_download + ): _blocked, _block_reason = is_blocked( crawl_result.status_code, crawl_result.html or "") if _blocked: @@ -1246,4 +1252,4 @@ async def amap_domain( config or DomainMapperConfig(**kwargs) if kwargs else DomainMapperConfig() ) - return await self._domain_mapper.scan(domain, mapper_config) \ No newline at end of file + return await self._domain_mapper.scan(domain, mapper_config) diff --git a/tests/proxy/test_antibot_opt_out.py b/tests/proxy/test_antibot_opt_out.py new file mode 100644 index 000000000..ffef0192c --- /dev/null +++ b/tests/proxy/test_antibot_opt_out.py @@ -0,0 +1,28 @@ +from unittest.mock import AsyncMock, Mock + +import pytest + +from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig +from crawl4ai.models import AsyncCrawlResponse, CrawlResult + + +@pytest.mark.asyncio +async def test_check_blocked_false_skips_antibot_detector(monkeypatch, tmp_path): + url = "https://example.com/small" + html = "small legitimate page" + response = AsyncCrawlResponse(html=html, response_headers={}, status_code=200) + strategy = Mock(crawl=AsyncMock(return_value=response)) + crawler = AsyncWebCrawler(crawler_strategy=strategy, base_directory=str(tmp_path)) + crawler.ready = True + page = CrawlResult(url=url, html=html, success=True) + crawler.aprocess_html = AsyncMock(return_value=page) + monkeypatch.setattr( + "crawl4ai.async_webcrawler.is_blocked", + Mock(side_effect=AssertionError("detector should be skipped")), + ) + + config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, check_blocked=False) + result = await crawler.arun(url, config=config) + + assert result.success is True + assert result.crawl_stats["proxies_used"][0]["blocked"] is False