fix: allow opting out of anti-bot blocking

crawl4ai/async_configs.py

@@ -264,7 +264,8 @@ class UntrustedConfigError(ValueError):
         "verbose", "log_console", "capture_network_requests",
         "capture_console_messages", "method", "stream", "prefetch", "url",
         "check_robots_txt", "user_agent", "user_agent_mode",
-        "user_agent_generator_config", "url_matcher", "match_mode", "max_retries",
+        "user_agent_generator_config", "url_matcher", "match_mode",
+        "check_blocked", "max_retries",
     },
 }
 
@@ -1430,6 +1431,7 @@ class CrawlerRunConfig():
                                       Default: False.
         cache_validation_timeout (float): Timeout in seconds for cache validation HTTP requests.
                                           Default: 10.0.
+        check_blocked (bool): Whether anti-bot matches fail the crawl. Default: True.
 
         # Page Navigation and Timing Parameters
         wait_until (str): The condition to wait for when navigating, e.g. "domcontentloaded".
@@ -1701,6 +1703,7 @@ def __init__(
         # Experimental Parameters
         experimental: Dict[str, Any] = None,
         # Anti-Bot Retry Parameters
+        check_blocked: bool = True,
         max_retries: int = 0,
         fallback_fetch_function: Optional[Callable[[str], Awaitable[str]]] = None,
     ):
@@ -1898,6 +1901,7 @@ def __init__(
         self.experimental = experimental or {}
 
         # Anti-Bot Retry Parameters
+        self.check_blocked = check_blocked
         self.max_retries = max_retries
         self.fallback_fetch_function = fallback_fetch_function
 
@@ -2184,6 +2188,7 @@ def to_dict(self):
             "url_matcher": self.url_matcher,
             "match_mode": self.match_mode,
             "experimental": self.experimental,
+            "check_blocked": self.check_blocked,
             "max_retries": self.max_retries,
         }
 

crawl4ai/async_webcrawler.py

@@ -505,7 +505,7 @@ async def arun(
 
                                 # Check if blocked (skip for raw: URLs —
                                 # caller-provided content, anti-bot N/A)
-                                if _is_raw_url:
+                                if _is_raw_url or not config.check_blocked:
                                     _blocked = False
                                     _block_reason = ""
                                 else:
@@ -554,7 +554,8 @@ async def arun(
                     if _fallback_fn and not _done and not _is_raw_url:
                         _needs_fallback = (
                             crawl_result is None  # All proxies threw exceptions
-                            or is_blocked(crawl_result.status_code, crawl_result.html or "")[0]
+                            or (config.check_blocked and is_blocked(
+                                crawl_result.status_code, crawl_result.html or "")[0])
                         )
                         if _needs_fallback:
                             self.logger.warning(
@@ -625,7 +626,12 @@ async def arun(
                         # empty by design, and is_blocked() would misread "0 bytes
                         # html" as a block.
                         _has_download = bool(getattr(crawl_result, "downloaded_files", None))
-                        if not _fallback_succeeded and not _is_raw_url and not _has_download:
+                        if (
+                            config.check_blocked
+                            and not _fallback_succeeded
+                            and not _is_raw_url
+                            and not _has_download
+                        ):
                             _blocked, _block_reason = is_blocked(
                                 crawl_result.status_code, crawl_result.html or "")
                             if _blocked:
@@ -1246,4 +1252,4 @@ async def amap_domain(
             config or DomainMapperConfig(**kwargs) if kwargs else DomainMapperConfig()
         )
 
-        return await self._domain_mapper.scan(domain, mapper_config)
+        return await self._domain_mapper.scan(domain, mapper_config)

tests/proxy/test_antibot_opt_out.py

@@ -0,0 +1,28 @@
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig
+from crawl4ai.models import AsyncCrawlResponse, CrawlResult
+
+
+@pytest.mark.asyncio
+async def test_check_blocked_false_skips_antibot_detector(monkeypatch, tmp_path):
+    url = "https://example.com/small"
+    html = "<html><body>small legitimate page</body></html>"
+    response = AsyncCrawlResponse(html=html, response_headers={}, status_code=200)
+    strategy = Mock(crawl=AsyncMock(return_value=response))
+    crawler = AsyncWebCrawler(crawler_strategy=strategy, base_directory=str(tmp_path))
+    crawler.ready = True
+    page = CrawlResult(url=url, html=html, success=True)
+    crawler.aprocess_html = AsyncMock(return_value=page)
+    monkeypatch.setattr(
+        "crawl4ai.async_webcrawler.is_blocked",
+        Mock(side_effect=AssertionError("detector should be skipped")),
+    )
+
+    config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, check_blocked=False)
+    result = await crawler.arun(url, config=config)
+
+    assert result.success is True
+    assert result.crawl_stats["proxies_used"][0]["blocked"] is False