From 6ccf610bcb44328815cf521a6c528dc807670ae8 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 7 Sep 2026 13:04:58 +0200 Subject: [PATCH 1/2] fix(docker): recycle pooled browser contexts by pages served (#2231) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pooled browsers get slower with sustained use and the janitor never recycles them, because it only closes browsers that have been *idle* past a TTL — and a server under continuous load never has an idle one. Measured where the slowdown actually lives: after 1000 real page loads on one pooled browser, a fresh context inside the SAME chromium process navigated as fast as a brand-new process (7.1ms vs 7.0ms), while the worn context took 32.0ms. Wiping that context's cookies/localStorage/ service workers in place recovered half of it (16.3ms). A control run with pages that leave no state behind stayed flat over 2000 pages. So the rot is context state, not the browser process, and the mechanism to bound it already exists in browser_manager (version-based recycling, which replaces the context under load without waiting for a quiet moment). It was simply never enabled for the Docker server. Enabling it is one config line — no second lifetime policy in the pool's janitor, which would put the same rule in two places. Measured effect at max_pages_before_recycle=200: penalty drops from ~30ms to ~10-14ms. It bounds the damage to one recycle window rather than removing it, so the threshold is the knob. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019EQGr8ey8riKFr4inp9B8f --- deploy/docker/config.yml | 8 +++++ tests/docker/test_pool_recycle_config.py | 42 ++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 tests/docker/test_pool_recycle_config.py diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml index 7aabc6814..6040ed1ca 100644 --- a/deploy/docker/config.yml +++ b/deploy/docker/config.yml @@ -88,6 +88,14 @@ crawler: kwargs: headless: true text_mode: true + # Pooled browsers are long-lived, and their browser context is reused for + # every crawl that shares a config. Real sites leave state in that context + # (cookies, localStorage, service workers) which is never cleared, and + # navigation gets measurably slower as it builds up. Recycling by pages + # served bounds that: the context is replaced inside the same Chromium + # process, old contexts drain on their own, and it works under sustained + # load (unlike the janitor, which only closes *idle* browsers). See #2231. + max_pages_before_recycle: 200 extra_args: # - "--single-process" # SECURITY: --no-sandbox disables the Chromium renderer sandbox (a renderer diff --git a/tests/docker/test_pool_recycle_config.py b/tests/docker/test_pool_recycle_config.py new file mode 100644 index 000000000..c51a81b0c --- /dev/null +++ b/tests/docker/test_pool_recycle_config.py @@ -0,0 +1,42 @@ +"""The Docker pool must recycle its browser context by pages served (#2231). + +The pool's janitor only closes *idle* browsers, so a server under sustained +load never recycles one. Recycling by page count is what bounds the context +state (cookies/localStorage/service workers) that slows navigation down. +""" +import sys +from pathlib import Path + +import pytest +import yaml + +from crawl4ai import BrowserConfig + +CONFIG_PATH = Path(__file__).resolve().parents[2] / "deploy" / "docker" / "config.yml" + + +@pytest.fixture(scope="module") +def browser_section(): + return yaml.safe_load(CONFIG_PATH.read_text())["crawler"]["browser"] + + +def test_recycle_is_enabled(browser_section): + kwargs = browser_section.get("kwargs", {}) + assert kwargs.get("max_pages_before_recycle", 0) > 0, ( + "deploy/docker/config.yml must set crawler.browser.kwargs." + "max_pages_before_recycle > 0, otherwise pooled browser contexts are " + "never recycled under sustained load (issue #2231)." + ) + + +def test_kwargs_reach_browser_config(browser_section): + """server.py/api.py splat these kwargs straight into BrowserConfig.""" + cfg = BrowserConfig( + extra_args=browser_section.get("extra_args", []), + **browser_section.get("kwargs", {}), + ) + assert cfg.max_pages_before_recycle > 0 + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) From 47ea8d7a817e1062d6a9ca4bab1d2b4111c3a5ed Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 7 Sep 2026 13:14:13 +0200 Subject: [PATCH 2/2] fix(docker): apply the recycle policy in the pool, not in config.yml kwargs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit put max_pages_before_recycle in config.yml's crawler.browser.kwargs, which does not reach the endpoints in the report: /crawl and /crawl/stream build their BrowserConfig from the request body (api.py:687, api.py:903) and nothing merges the server's browser kwargs into it. Only the handful of endpoints that call get_default_browser_config() would have picked it up. Move the setting to crawler.pool (it is a pool policy, next to max_pages and idle_ttl_sec) and apply it in get_crawler()/init_permanent(), which every endpoint goes through. Pooling is what makes a browser long-lived, so the pool is the right owner of its recycling policy. Applied before _sig() so all requests still share one signature and pooling is unchanged; an explicit per-request value wins. Tests cover the call site too, not only the helper — dropping _apply_pool_defaults() from get_crawler() now fails. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019EQGr8ey8riKFr4inp9B8f --- deploy/docker/config.yml | 16 ++--- deploy/docker/crawler_pool.py | 21 ++++++- tests/docker/test_pool_recycle_config.py | 74 ++++++++++++++++++------ 3 files changed, 82 insertions(+), 29 deletions(-) diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml index 6040ed1ca..094209899 100644 --- a/deploy/docker/config.yml +++ b/deploy/docker/config.yml @@ -84,18 +84,18 @@ crawler: pool: max_pages: 40 # ← GLOBAL_SEM permits idle_ttl_sec: 300 # ← 30 min janitor cutoff + # Pooled browsers are long-lived and reuse one browser context per config. + # Real sites leave state in that context (cookies, localStorage, service + # workers) which is never cleared, and navigation gets measurably slower as + # it builds up. Recycling by pages served bounds it: the context is replaced + # inside the same Chromium process, old contexts drain on their own, and it + # works under sustained load — unlike the janitor, which only closes *idle* + # browsers and so never fires on a busy server. 0 disables. See #2231. + max_pages_before_recycle: 200 browser: kwargs: headless: true text_mode: true - # Pooled browsers are long-lived, and their browser context is reused for - # every crawl that shares a config. Real sites leave state in that context - # (cookies, localStorage, service workers) which is never cleared, and - # navigation gets measurably slower as it builds up. Recycling by pages - # served bounds that: the context is replaced inside the same Chromium - # process, old contexts drain on their own, and it works under sustained - # load (unlike the janitor, which only closes *idle* browsers). See #2231. - max_pages_before_recycle: 200 extra_args: # - "--single-process" # SECURITY: --no-sandbox disables the Chromium renderer sandbox (a renderer diff --git a/deploy/docker/crawler_pool.py b/deploy/docker/crawler_pool.py index 516d9562a..cdde8bba1 100644 --- a/deploy/docker/crawler_pool.py +++ b/deploy/docker/crawler_pool.py @@ -20,9 +20,26 @@ # Config MEM_LIMIT = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0) BASE_IDLE_TTL = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 300) +RECYCLE_PAGES = CONFIG.get("crawler", {}).get("pool", {}).get("max_pages_before_recycle", 0) DEFAULT_CONFIG_SIG = None # Cached sig for default config +def _apply_pool_defaults(cfg: BrowserConfig) -> BrowserConfig: + """Apply pool-owned policy to a browser config before it is pooled. + + Browsers handed out by this pool are long-lived, and the endpoints build + their BrowserConfig from the request body (see api.py handle_crawl_request), + so config.yml's browser kwargs never reach them. Pooling is what makes a + browser long-lived, so the pool is where its recycling policy belongs. + + Applied before _sig() so every request shares one signature and pooling is + unaffected. An explicit per-request value wins. See #2231. + """ + if RECYCLE_PAGES and not cfg.max_pages_before_recycle: + cfg.max_pages_before_recycle = RECYCLE_PAGES + return cfg + + def get_pool_snapshot() -> dict: """Return a point-in-time snapshot of pool state for monitoring. @@ -54,7 +71,7 @@ def _is_default_config(sig: str) -> bool: async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler: """Get crawler from pool with tiered strategy.""" - sig = _sig(cfg) + sig = _sig(_apply_pool_defaults(cfg)) async with LOCK: # Check permanent browser for default config if PERMANENT and _is_default_config(sig): @@ -135,7 +152,7 @@ async def init_permanent(cfg: BrowserConfig): async with LOCK: if PERMANENT: return - DEFAULT_CONFIG_SIG = _sig(cfg) + DEFAULT_CONFIG_SIG = _sig(_apply_pool_defaults(cfg)) logger.info("🔥 Creating permanent default browser") PERMANENT = AsyncWebCrawler(config=cfg, thread_safe=False) await PERMANENT.start() diff --git a/tests/docker/test_pool_recycle_config.py b/tests/docker/test_pool_recycle_config.py index c51a81b0c..c89f48273 100644 --- a/tests/docker/test_pool_recycle_config.py +++ b/tests/docker/test_pool_recycle_config.py @@ -1,8 +1,12 @@ -"""The Docker pool must recycle its browser context by pages served (#2231). +"""Pooled browsers must recycle their context by pages served (#2231). The pool's janitor only closes *idle* browsers, so a server under sustained load never recycles one. Recycling by page count is what bounds the context state (cookies/localStorage/service workers) that slows navigation down. + +The policy has to be applied by the pool, not by config.yml's browser kwargs: +/crawl and /crawl/stream build their BrowserConfig from the request body +(api.py handle_crawl_request), so those kwargs never reach them. """ import sys from pathlib import Path @@ -12,30 +16,62 @@ from crawl4ai import BrowserConfig -CONFIG_PATH = Path(__file__).resolve().parents[2] / "deploy" / "docker" / "config.yml" - +ROOT = Path(__file__).resolve().parents[2] +DOCKER_DIR = ROOT / "deploy" / "docker" +sys.path.insert(0, str(DOCKER_DIR)) -@pytest.fixture(scope="module") -def browser_section(): - return yaml.safe_load(CONFIG_PATH.read_text())["crawler"]["browser"] +pool = pytest.importorskip("crawler_pool", reason="docker server deps not installed") -def test_recycle_is_enabled(browser_section): - kwargs = browser_section.get("kwargs", {}) - assert kwargs.get("max_pages_before_recycle", 0) > 0, ( - "deploy/docker/config.yml must set crawler.browser.kwargs." - "max_pages_before_recycle > 0, otherwise pooled browser contexts are " - "never recycled under sustained load (issue #2231)." +def test_config_enables_recycling(): + cfg = yaml.safe_load((DOCKER_DIR / "config.yml").read_text()) + assert cfg["crawler"]["pool"].get("max_pages_before_recycle", 0) > 0, ( + "config.yml must set crawler.pool.max_pages_before_recycle > 0, else " + "pooled contexts are never recycled under sustained load (#2231)." ) -def test_kwargs_reach_browser_config(browser_section): - """server.py/api.py splat these kwargs straight into BrowserConfig.""" - cfg = BrowserConfig( - extra_args=browser_section.get("extra_args", []), - **browser_section.get("kwargs", {}), - ) - assert cfg.max_pages_before_recycle > 0 +def test_pool_applies_recycling_to_a_request_supplied_config(): + """A config off the wire carries no recycle setting; the pool must add it.""" + from_request = BrowserConfig() + assert from_request.max_pages_before_recycle == 0 # guard the premise + + pool._apply_pool_defaults(from_request) + assert from_request.max_pages_before_recycle == pool.RECYCLE_PAGES > 0 + + +def test_explicit_caller_value_wins(): + cfg = BrowserConfig(max_pages_before_recycle=7) + pool._apply_pool_defaults(cfg) + assert cfg.max_pages_before_recycle == 7 + + +@pytest.mark.asyncio +async def test_get_crawler_applies_it(monkeypatch): + """The call site matters, not just the helper: a config handed to + get_crawler() must come back carrying the pool's recycle policy.""" + class _FakeCrawler: + def __init__(self, config, **kw): + self.config = config + + async def start(self): + return self + + monkeypatch.setattr(pool, "AsyncWebCrawler", _FakeCrawler) + monkeypatch.setattr(pool, "COLD_POOL", {}) + monkeypatch.setattr(pool, "HOT_POOL", {}) + monkeypatch.setattr(pool, "PERMANENT", None) + monkeypatch.setattr(pool, "get_container_memory_percent", lambda: 10.0) + + cfg = BrowserConfig() + await pool.get_crawler(cfg) + assert cfg.max_pages_before_recycle == pool.RECYCLE_PAGES > 0 + + +def test_signature_is_stable_across_requests(): + """Applying the default must not split the pool into two signatures.""" + a, b = BrowserConfig(), BrowserConfig() + assert pool._sig(pool._apply_pool_defaults(a)) == pool._sig(pool._apply_pool_defaults(b)) if __name__ == "__main__":