fix(docker): recycle pooled browser contexts by pages served (#2231)

deploy/docker/config.yml

@@ -84,6 +84,14 @@ crawler:
   pool:
     max_pages: 40                          # ← GLOBAL_SEM permits
     idle_ttl_sec: 300                     # ← 30 min janitor cutoff
+    # Pooled browsers are long-lived and reuse one browser context per config.
+    # Real sites leave state in that context (cookies, localStorage, service
+    # workers) which is never cleared, and navigation gets measurably slower as
+    # it builds up. Recycling by pages served bounds it: the context is replaced
+    # inside the same Chromium process, old contexts drain on their own, and it
+    # works under sustained load — unlike the janitor, which only closes *idle*
+    # browsers and so never fires on a busy server. 0 disables. See #2231.
+    max_pages_before_recycle: 200
   browser:
     kwargs:
       headless: true

deploy/docker/crawler_pool.py

@@ -20,9 +20,26 @@
 # Config
 MEM_LIMIT = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0)
 BASE_IDLE_TTL = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 300)
+RECYCLE_PAGES = CONFIG.get("crawler", {}).get("pool", {}).get("max_pages_before_recycle", 0)
 DEFAULT_CONFIG_SIG = None  # Cached sig for default config
 
 
+def _apply_pool_defaults(cfg: BrowserConfig) -> BrowserConfig:
+    """Apply pool-owned policy to a browser config before it is pooled.
+
+    Browsers handed out by this pool are long-lived, and the endpoints build
+    their BrowserConfig from the request body (see api.py handle_crawl_request),
+    so config.yml's browser kwargs never reach them. Pooling is what makes a
+    browser long-lived, so the pool is where its recycling policy belongs.
+
+    Applied before _sig() so every request shares one signature and pooling is
+    unaffected. An explicit per-request value wins. See #2231.
+    """
+    if RECYCLE_PAGES and not cfg.max_pages_before_recycle:
+        cfg.max_pages_before_recycle = RECYCLE_PAGES
+    return cfg
+
+
 def get_pool_snapshot() -> dict:
     """Return a point-in-time snapshot of pool state for monitoring.
 
@@ -54,7 +71,7 @@ def _is_default_config(sig: str) -> bool:
 
 async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
     """Get crawler from pool with tiered strategy."""
-    sig = _sig(cfg)
+    sig = _sig(_apply_pool_defaults(cfg))
     async with LOCK:
         # Check permanent browser for default config
         if PERMANENT and _is_default_config(sig):
@@ -135,7 +152,7 @@ async def init_permanent(cfg: BrowserConfig):
     async with LOCK:
         if PERMANENT:
             return
-        DEFAULT_CONFIG_SIG = _sig(cfg)
+        DEFAULT_CONFIG_SIG = _sig(_apply_pool_defaults(cfg))
         logger.info("🔥 Creating permanent default browser")
         PERMANENT = AsyncWebCrawler(config=cfg, thread_safe=False)
         await PERMANENT.start()

tests/docker/test_pool_recycle_config.py

@@ -0,0 +1,78 @@
+"""Pooled browsers must recycle their context by pages served (#2231).
+
+The pool's janitor only closes *idle* browsers, so a server under sustained
+load never recycles one. Recycling by page count is what bounds the context
+state (cookies/localStorage/service workers) that slows navigation down.
+
+The policy has to be applied by the pool, not by config.yml's browser kwargs:
+/crawl and /crawl/stream build their BrowserConfig from the request body
+(api.py handle_crawl_request), so those kwargs never reach them.
+"""
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+
+from crawl4ai import BrowserConfig
+
+ROOT = Path(__file__).resolve().parents[2]
+DOCKER_DIR = ROOT / "deploy" / "docker"
+sys.path.insert(0, str(DOCKER_DIR))
+
+pool = pytest.importorskip("crawler_pool", reason="docker server deps not installed")
+
+
+def test_config_enables_recycling():
+    cfg = yaml.safe_load((DOCKER_DIR / "config.yml").read_text())
+    assert cfg["crawler"]["pool"].get("max_pages_before_recycle", 0) > 0, (
+        "config.yml must set crawler.pool.max_pages_before_recycle > 0, else "
+        "pooled contexts are never recycled under sustained load (#2231)."
+    )
+
+
+def test_pool_applies_recycling_to_a_request_supplied_config():
+    """A config off the wire carries no recycle setting; the pool must add it."""
+    from_request = BrowserConfig()
+    assert from_request.max_pages_before_recycle == 0  # guard the premise
+
+    pool._apply_pool_defaults(from_request)
+    assert from_request.max_pages_before_recycle == pool.RECYCLE_PAGES > 0
+
+
+def test_explicit_caller_value_wins():
+    cfg = BrowserConfig(max_pages_before_recycle=7)
+    pool._apply_pool_defaults(cfg)
+    assert cfg.max_pages_before_recycle == 7
+
+
+@pytest.mark.asyncio
+async def test_get_crawler_applies_it(monkeypatch):
+    """The call site matters, not just the helper: a config handed to
+    get_crawler() must come back carrying the pool's recycle policy."""
+    class _FakeCrawler:
+        def __init__(self, config, **kw):
+            self.config = config
+
+        async def start(self):
+            return self
+
+    monkeypatch.setattr(pool, "AsyncWebCrawler", _FakeCrawler)
+    monkeypatch.setattr(pool, "COLD_POOL", {})
+    monkeypatch.setattr(pool, "HOT_POOL", {})
+    monkeypatch.setattr(pool, "PERMANENT", None)
+    monkeypatch.setattr(pool, "get_container_memory_percent", lambda: 10.0)
+
+    cfg = BrowserConfig()
+    await pool.get_crawler(cfg)
+    assert cfg.max_pages_before_recycle == pool.RECYCLE_PAGES > 0
+
+
+def test_signature_is_stable_across_requests():
+    """Applying the default must not split the pool into two signatures."""
+    a, b = BrowserConfig(), BrowserConfig()
+    assert pool._sig(pool._apply_pool_defaults(a)) == pool._sig(pool._apply_pool_defaults(b))
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-v"]))