diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml index 7aabc6814..094209899 100644 --- a/deploy/docker/config.yml +++ b/deploy/docker/config.yml @@ -84,6 +84,14 @@ crawler: pool: max_pages: 40 # ← GLOBAL_SEM permits idle_ttl_sec: 300 # ← 30 min janitor cutoff + # Pooled browsers are long-lived and reuse one browser context per config. + # Real sites leave state in that context (cookies, localStorage, service + # workers) which is never cleared, and navigation gets measurably slower as + # it builds up. Recycling by pages served bounds it: the context is replaced + # inside the same Chromium process, old contexts drain on their own, and it + # works under sustained load — unlike the janitor, which only closes *idle* + # browsers and so never fires on a busy server. 0 disables. See #2231. + max_pages_before_recycle: 200 browser: kwargs: headless: true diff --git a/deploy/docker/crawler_pool.py b/deploy/docker/crawler_pool.py index 516d9562a..cdde8bba1 100644 --- a/deploy/docker/crawler_pool.py +++ b/deploy/docker/crawler_pool.py @@ -20,9 +20,26 @@ # Config MEM_LIMIT = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0) BASE_IDLE_TTL = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 300) +RECYCLE_PAGES = CONFIG.get("crawler", {}).get("pool", {}).get("max_pages_before_recycle", 0) DEFAULT_CONFIG_SIG = None # Cached sig for default config +def _apply_pool_defaults(cfg: BrowserConfig) -> BrowserConfig: + """Apply pool-owned policy to a browser config before it is pooled. + + Browsers handed out by this pool are long-lived, and the endpoints build + their BrowserConfig from the request body (see api.py handle_crawl_request), + so config.yml's browser kwargs never reach them. Pooling is what makes a + browser long-lived, so the pool is where its recycling policy belongs. + + Applied before _sig() so every request shares one signature and pooling is + unaffected. An explicit per-request value wins. See #2231. + """ + if RECYCLE_PAGES and not cfg.max_pages_before_recycle: + cfg.max_pages_before_recycle = RECYCLE_PAGES + return cfg + + def get_pool_snapshot() -> dict: """Return a point-in-time snapshot of pool state for monitoring. @@ -54,7 +71,7 @@ def _is_default_config(sig: str) -> bool: async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler: """Get crawler from pool with tiered strategy.""" - sig = _sig(cfg) + sig = _sig(_apply_pool_defaults(cfg)) async with LOCK: # Check permanent browser for default config if PERMANENT and _is_default_config(sig): @@ -135,7 +152,7 @@ async def init_permanent(cfg: BrowserConfig): async with LOCK: if PERMANENT: return - DEFAULT_CONFIG_SIG = _sig(cfg) + DEFAULT_CONFIG_SIG = _sig(_apply_pool_defaults(cfg)) logger.info("🔥 Creating permanent default browser") PERMANENT = AsyncWebCrawler(config=cfg, thread_safe=False) await PERMANENT.start() diff --git a/tests/docker/test_pool_recycle_config.py b/tests/docker/test_pool_recycle_config.py new file mode 100644 index 000000000..c89f48273 --- /dev/null +++ b/tests/docker/test_pool_recycle_config.py @@ -0,0 +1,78 @@ +"""Pooled browsers must recycle their context by pages served (#2231). + +The pool's janitor only closes *idle* browsers, so a server under sustained +load never recycles one. Recycling by page count is what bounds the context +state (cookies/localStorage/service workers) that slows navigation down. + +The policy has to be applied by the pool, not by config.yml's browser kwargs: +/crawl and /crawl/stream build their BrowserConfig from the request body +(api.py handle_crawl_request), so those kwargs never reach them. +""" +import sys +from pathlib import Path + +import pytest +import yaml + +from crawl4ai import BrowserConfig + +ROOT = Path(__file__).resolve().parents[2] +DOCKER_DIR = ROOT / "deploy" / "docker" +sys.path.insert(0, str(DOCKER_DIR)) + +pool = pytest.importorskip("crawler_pool", reason="docker server deps not installed") + + +def test_config_enables_recycling(): + cfg = yaml.safe_load((DOCKER_DIR / "config.yml").read_text()) + assert cfg["crawler"]["pool"].get("max_pages_before_recycle", 0) > 0, ( + "config.yml must set crawler.pool.max_pages_before_recycle > 0, else " + "pooled contexts are never recycled under sustained load (#2231)." + ) + + +def test_pool_applies_recycling_to_a_request_supplied_config(): + """A config off the wire carries no recycle setting; the pool must add it.""" + from_request = BrowserConfig() + assert from_request.max_pages_before_recycle == 0 # guard the premise + + pool._apply_pool_defaults(from_request) + assert from_request.max_pages_before_recycle == pool.RECYCLE_PAGES > 0 + + +def test_explicit_caller_value_wins(): + cfg = BrowserConfig(max_pages_before_recycle=7) + pool._apply_pool_defaults(cfg) + assert cfg.max_pages_before_recycle == 7 + + +@pytest.mark.asyncio +async def test_get_crawler_applies_it(monkeypatch): + """The call site matters, not just the helper: a config handed to + get_crawler() must come back carrying the pool's recycle policy.""" + class _FakeCrawler: + def __init__(self, config, **kw): + self.config = config + + async def start(self): + return self + + monkeypatch.setattr(pool, "AsyncWebCrawler", _FakeCrawler) + monkeypatch.setattr(pool, "COLD_POOL", {}) + monkeypatch.setattr(pool, "HOT_POOL", {}) + monkeypatch.setattr(pool, "PERMANENT", None) + monkeypatch.setattr(pool, "get_container_memory_percent", lambda: 10.0) + + cfg = BrowserConfig() + await pool.get_crawler(cfg) + assert cfg.max_pages_before_recycle == pool.RECYCLE_PAGES > 0 + + +def test_signature_is_stable_across_requests(): + """Applying the default must not split the pool into two signatures.""" + a, b = BrowserConfig(), BrowserConfig() + assert pool._sig(pool._apply_pool_defaults(a)) == pool._sig(pool._apply_pool_defaults(b)) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))