From 12d138bbf313164be2b93bd4bd6c545cd48e1e1b Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Wed, 22 Jul 2026 16:41:50 +0530 Subject: [PATCH] fix(docker): compose v5 compatibility, legacy-field warnings, playground error handling - docker-compose.yml: move the PID cap to deploy.resources.limits.pids (Compose v5 rejects pids_limit alongside a limits block; same behavior on v2.x). - entrypoint.sh: explain the loopback-only bind when no CRAWL4AI_API_TOKEN is set and how to fix it. - server.py: warn when the removed output_path (screenshot/pdf) or legacy hooks.code is sent - fields are ignored, never executed; hooks status reads "ignored" and /crawl/stream sends X-Hooks-Warning. Make "/" public so the /playground redirect works; data routes stay gated. - schemas.py: capture legacy hooks.code so it can be reported (never run). - playground: check response.ok on the streaming branch, surface server error details, hint at the token bar on 401. - tests: add deploy/docker/tests/test_legacy_compat.py (13 tests). --- deploy/docker/entrypoint.sh | 2 + deploy/docker/schemas.py | 8 + deploy/docker/server.py | 40 +++- deploy/docker/static/playground/index.html | 21 ++- deploy/docker/tests/test_legacy_compat.py | 206 +++++++++++++++++++++ docker-compose.yml | 4 +- 6 files changed, 272 insertions(+), 9 deletions(-) create mode 100644 deploy/docker/tests/test_legacy_compat.py diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh index b624a3115..19d6766f6 100644 --- a/deploy/docker/entrypoint.sh +++ b/deploy/docker/entrypoint.sh @@ -31,6 +31,8 @@ else # No credential -> refuse to expose; serve loopback only. GUNICORN_BIND="127.0.0.1:${PORT}" echo "entrypoint: no CRAWL4AI_API_TOKEN set; binding loopback only (${GUNICORN_BIND})." >&2 + echo "entrypoint: WARNING: this is the CONTAINER's loopback - published ports (-p ${PORT}:${PORT}) will NOT work; connections from the host will be reset." >&2 + echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or the .llm.env file with docker compose) and restart." >&2 fi export GUNICORN_BIND diff --git a/deploy/docker/schemas.py b/deploy/docker/schemas.py index f066e5c88..b1065b5a8 100644 --- a/deploy/docker/schemas.py +++ b/deploy/docker/schemas.py @@ -42,6 +42,14 @@ class HookConfig(BaseModel): le=120, description="Timeout in seconds for each hook execution", ) + # Legacy 0.8.x field: inline-Python hook code, removed in 0.9.0 (it was an + # exec()-based RCE surface). Captured here (instead of being dropped by + # pydantic) solely so the server can tell the caller it was NOT executed; + # it is never run. + code: Optional[Dict[str, str]] = Field( + default=None, + description="REMOVED in 0.9.0: inline hook code is accepted for compatibility but never executed", + ) class Config: json_schema_extra = { diff --git a/deploy/docker/server.py b/deploy/docker/server.py index 410b8be38..ea6de2ac0 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -392,7 +392,7 @@ def _current_api_token() -> str: app.add_middleware( AuthGateMiddleware, token_provider=_current_api_token, - public_paths={HEALTH_PATH, "/token"}, + public_paths={HEALTH_PATH, "/token", "/"}, public_prefixes=_UI_PREFIXES, ) @@ -660,6 +660,17 @@ async def get_artifact(artifact_id: str, _td: Dict = Depends(token_dep)): # Screenshot endpoint +_OUTPUT_PATH_WARNING = ( + "output_path was removed in 0.9.0 and is ignored - no file was written. " + "The result is stored server-side; fetch it with an authenticated " + "GET /artifacts/{artifact_id}." +) + +_HOOKS_CODE_WARNING = ( + "Inline hook code (hooks.code) was removed in 0.9.0 and was NOT executed. " + "Use declarative hook actions instead (GET /hooks/info for the schema)." +) + @app.post("/screenshot") @limiter.limit(config["rate_limiting"]["default_limit"]) @@ -684,7 +695,12 @@ async def generate_screenshot( raise HTTPException(500, detail=results[0].error_message or "Crawl failed") screenshot_data = results[0].screenshot art = _store_artifact("png", base64.b64decode(screenshot_data)) - return {"success": True, "screenshot": screenshot_data, **art} + response = {"success": True, "screenshot": screenshot_data, **art} + # Legacy 0.8.x key, no longer a schema field: peek at the raw body + # (already parsed and cached by FastAPI) to warn that it was ignored. + if (await request.json()).get("output_path"): + response["warning"] = _OUTPUT_PATH_WARNING + return response except HTTPException: raise except Exception as e: @@ -719,7 +735,12 @@ async def generate_pdf( raise HTTPException(500, detail=results[0].error_message or "Crawl failed") pdf_data = results[0].pdf art = _store_artifact("pdf", pdf_data) - return {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art} + response = {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art} + # Legacy 0.8.x key, no longer a schema field: peek at the raw body + # (already parsed and cached by FastAPI) to warn that it was ignored. + if (await request.json()).get("output_path"): + response["warning"] = _OUTPUT_PATH_WARNING + return response except HTTPException: raise except Exception as e: @@ -889,7 +910,7 @@ async def crawl( raise HTTPException(400, f"Rejected config: {e}") if crawler_config.stream: return await stream_process(crawl_request=crawl_request) - + # Prepare hooks config if provided hooks_config = None if crawl_request.hooks: @@ -897,7 +918,7 @@ async def crawl( 'hooks': crawl_request.hooks.hooks, 'timeout': crawl_request.hooks.timeout } - + results = await handle_crawl_request( urls=crawl_request.urls, browser_config=crawl_request.browser_config, @@ -906,6 +927,11 @@ async def crawl( hooks_config=hooks_config, crawler_configs=crawl_request.crawler_configs, ) + if crawl_request.hooks and crawl_request.hooks.code: + hooks_resp = results.setdefault("hooks", {"attached": []}) + if not crawl_request.hooks.hooks: + hooks_resp["status"] = "ignored" + hooks_resp["warning"] = _HOOKS_CODE_WARNING # check if all of the results are not successful if all(not result["success"] for result in results["results"]): raise HTTPException(500, f"Crawl request failed: {results['results'][0]['error_message']}") @@ -927,7 +953,7 @@ async def crawl_stream( return await stream_process(crawl_request=crawl_request) async def stream_process(crawl_request: CrawlRequestWithHooks): - + # Prepare hooks config if provided# Prepare hooks config if provided hooks_config = None if crawl_request.hooks: @@ -953,6 +979,8 @@ async def stream_process(crawl_request: CrawlRequestWithHooks): if hooks_info: import json headers["X-Hooks-Status"] = json.dumps(hooks_info['status']['status']) + if crawl_request.hooks and crawl_request.hooks.code: + headers["X-Hooks-Warning"] = _HOOKS_CODE_WARNING return StreamingResponse( stream_results(crawler, gen), diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html index e88c281a0..cf72a303b 100644 --- a/deploy/docker/static/playground/index.html +++ b/deploy/docker/static/playground/index.html @@ -617,6 +617,17 @@

🔥 Stress Test

} } + // Build a useful error message from a failed HTTP response + function httpErrorMessage(response, data) { + let msg = (data && (data.detail || data.error)) || `HTTP ${response.status}`; + if (response.status === 401) { + msg += getToken() + ? ' — token rejected; check the API token in the token bar (top right)' + : ' — set your API token in the token bar (top right)'; + } + return msg; + } + // Generate code snippets function generateSnippets(api, payload, method = 'POST') { // Python snippet @@ -751,7 +762,7 @@

🔥 Stress Test

const time = Math.round(performance.now() - startTime); if (!response.ok) { updateStatus('error', time); - throw new Error(responseData.error || 'Request failed'); + throw new Error(httpErrorMessage(response, responseData)); } updateStatus('success', time); document.querySelector('#response-content code').textContent = JSON.stringify(responseData, null, 2); @@ -765,6 +776,12 @@

🔥 Stress Test

body: JSON.stringify(payload) }); + if (!response.ok) { + const errData = await response.json().catch(() => ({})); + updateStatus('error', Math.round(performance.now() - startTime)); + throw new Error(httpErrorMessage(response, errData)); + } + const reader = response.body.getReader(); let text = ''; let maxMemory = 0; @@ -809,7 +826,7 @@

🔥 Stress Test

if (!response.ok) { updateStatus('error', time); - throw new Error(responseData.error || 'Request failed'); + throw new Error(httpErrorMessage(response, responseData)); } updateStatus( diff --git a/deploy/docker/tests/test_legacy_compat.py b/deploy/docker/tests/test_legacy_compat.py new file mode 100644 index 000000000..b8b8402c5 --- /dev/null +++ b/deploy/docker/tests/test_legacy_compat.py @@ -0,0 +1,206 @@ +""" +Behavioral tests for 0.9.x legacy-compatibility handling: + + * root redirect - "/" is public and redirects to /playground instead of + dying in the auth gate with a bare 401; /monitor and the + data routes stay gated. + * output_path - /screenshot and /pdf still accept the 0.8.x output_path + field but return a warning saying no file was written, + instead of silently dropping it. + * legacy hooks - hooks.code (removed 0.8.x inline Python) is captured, + never executed, and reported as status "ignored" with a + warning when hooks are enabled; any hooks payload is + still refused (403) while hooks are disabled. + * compose file - the PID cap lives under deploy.resources.limits (not + pids_limit), which Compose v5 rejects alongside a + limits block. + +These exercise the running app via TestClient (no browser / Redis needed); +crawl internals are stubbed where a handler would otherwise need a browser. +""" + +import base64 +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from auth import create_access_token # noqa: E402 + + +def _bearer() -> dict: + return {"Authorization": f"Bearer {create_access_token({'sub': 'user@x.com'}, scope='data')}"} + + +# ───────────────────────── root redirect ───────────────────────── + + +class TestRootRedirect: + def test_root_is_public_and_redirects_to_playground(self, stock_client): + r = stock_client.get("/", follow_redirects=False) + assert r.status_code in (302, 307), ( + f"GET / returned {r.status_code}; expected a redirect. The auth " + f"gate must allow the exact path '/' so the redirect route runs." + ) + assert r.headers["location"] == "/playground" + + def test_monitor_and_data_routes_stay_gated(self, stock_client): + assert stock_client.get("/monitor").status_code == 401 + assert stock_client.get("/monitor/health").status_code == 401 + assert stock_client.post("/crawl", json={"urls": ["https://x"]}).status_code == 401 + + +# ───────────────────────── output_path warning ───────────────────────── + + +@pytest.fixture +def stub_crawler(server_module, monkeypatch): + """Stub the crawler pool + artifact store so /screenshot and /pdf run + without a browser. Returns the fake artifact dict for assertions.""" + art = {"artifact_id": "a1", "url": "/artifacts/a1", "mime": "x", "size": 1} + png_b64 = base64.b64encode(b"fake-png").decode() + + fake_result = SimpleNamespace(success=True, screenshot=png_b64, pdf=b"fake-pdf") + + class _FakeCrawler: + async def arun(self, url, config): + return [fake_result] + + async def fake_get_crawler(cfg): + return _FakeCrawler() + + async def fake_release_crawler(crawler): + pass + + monkeypatch.setattr(server_module, "get_crawler", fake_get_crawler) + monkeypatch.setattr(server_module, "release_crawler", fake_release_crawler) + monkeypatch.setattr(server_module, "_store_artifact", lambda kind, data: dict(art)) + return art + + +class TestOutputPathWarning: + @pytest.mark.parametrize("endpoint,payload_key", [("/screenshot", "screenshot"), ("/pdf", "pdf")]) + def test_output_path_accepted_with_warning(self, stock_client, stub_crawler, endpoint, payload_key): + r = stock_client.post( + endpoint, + json={"url": "https://example.com", "output_path": "/tmp/x.bin"}, + headers=_bearer(), + ) + assert r.status_code == 200 + body = r.json() + assert body["success"] is True + assert "warning" in body, "output_path must not be silently dropped" + assert "no file was written" in body["warning"] + assert body["artifact_id"] == stub_crawler["artifact_id"] + + @pytest.mark.parametrize("endpoint", ["/screenshot", "/pdf"]) + def test_no_warning_without_output_path(self, stock_client, stub_crawler, endpoint): + r = stock_client.post(endpoint, json={"url": "https://example.com"}, headers=_bearer()) + assert r.status_code == 200 + assert "warning" not in r.json() + + +# ───────────────────────── legacy hooks.code ───────────────────────── + +LEGACY_HOOKS = {"code": {"before_goto": "async def hook(p, c, u, **kw): return p"}} +DECLARATIVE_HOOKS = {"hooks": [{"action": "scroll_to_bottom", "params": {"max_steps": 2}}]} + + +class TestLegacyHookCode: + def test_hook_config_captures_code_field(self): + """The legacy field must be parsed (not dropped) so it can be reported.""" + from schemas import CrawlRequestWithHooks + + req = CrawlRequestWithHooks(urls=["https://x"], hooks=LEGACY_HOOKS) + assert req.hooks.code == LEGACY_HOOKS["code"] + assert req.hooks.hooks == [] + + @pytest.mark.parametrize("hooks_payload", [LEGACY_HOOKS, DECLARATIVE_HOOKS]) + def test_any_hooks_payload_403_when_disabled(self, server_module, monkeypatch, stock_client, hooks_payload): + monkeypatch.setattr(server_module, "HOOKS_ENABLED", False) + r = stock_client.post( + "/crawl", + json={"urls": ["https://example.com"], "hooks": hooks_payload}, + headers=_bearer(), + ) + assert r.status_code == 403 + + def _stub_crawl(self, server_module, monkeypatch, results): + async def fake_handle_crawl_request(**kwargs): + return dict(results) + + monkeypatch.setattr(server_module, "handle_crawl_request", fake_handle_crawl_request) + + def test_legacy_code_warned_and_ignored_when_enabled(self, server_module, monkeypatch, stock_client): + monkeypatch.setattr(server_module, "HOOKS_ENABLED", True) + # Empty declarative specs -> api.py reports a vacuous success + self._stub_crawl( + server_module, monkeypatch, + {"success": True, "results": [{"success": True}], + "hooks": {"status": "success", "attached": []}}, + ) + r = stock_client.post( + "/crawl", + json={"urls": ["https://example.com"], "hooks": LEGACY_HOOKS}, + headers=_bearer(), + ) + assert r.status_code == 200 + hooks = r.json()["hooks"] + assert hooks["status"] == "ignored", "vacuous 'success' must be rewritten" + assert hooks["attached"] == [] + assert "NOT executed" in hooks["warning"] + + def test_mixed_request_keeps_declarative_status_and_warns(self, server_module, monkeypatch, stock_client): + monkeypatch.setattr(server_module, "HOOKS_ENABLED", True) + self._stub_crawl( + server_module, monkeypatch, + {"success": True, "results": [{"success": True}], + "hooks": {"status": "success", "attached": ["before_retrieve_html"]}}, + ) + r = stock_client.post( + "/crawl", + json={"urls": ["https://example.com"], + "hooks": {**DECLARATIVE_HOOKS, **LEGACY_HOOKS}}, + headers=_bearer(), + ) + assert r.status_code == 200 + hooks = r.json()["hooks"] + assert hooks["status"] == "success", "real declarative execution must not be relabeled" + assert hooks["attached"] == ["before_retrieve_html"] + assert "NOT executed" in hooks["warning"] + + def test_no_hooks_response_untouched(self, server_module, monkeypatch, stock_client): + self._stub_crawl( + server_module, monkeypatch, + {"success": True, "results": [{"success": True}]}, + ) + r = stock_client.post("/crawl", json={"urls": ["https://example.com"]}, headers=_bearer()) + assert r.status_code == 200 + assert "hooks" not in r.json() + + +# ───────────────────────── compose file ───────────────────────── + + +class TestComposeFile: + def test_pid_cap_lives_under_deploy_limits(self): + """Compose v5 rejects pids_limit next to deploy.resources.limits + ("can't set distinct values"); the cap must be expressed once, under + deploy.resources.limits.pids.""" + import os + + override = os.environ.get("CRAWL4AI_COMPOSE_FILE") + if override: + compose_path = Path(override) + else: + here = Path(__file__).resolve() + if len(here.parents) < 4: + pytest.skip("not running from a repo checkout") + compose_path = here.parents[3] / "docker-compose.yml" + if not compose_path.exists(): + pytest.skip(f"docker-compose.yml not found at {compose_path}") + doc = yaml.safe_load(compose_path.read_text()) + base = doc["x-base-config"] + assert "pids_limit" not in base + assert base["deploy"]["resources"]["limits"]["pids"] == 512 diff --git a/docker-compose.yml b/docker-compose.yml index 1cd87ad93..5939b3321 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,6 @@ x-base-config: &base-config - ALL security_opt: - no-new-privileges:true - pids_limit: 512 # Read-only root filesystem; only these paths are writable (tmpfs). read_only: true tmpfs: @@ -39,6 +38,9 @@ x-base-config: &base-config resources: limits: memory: 4G + # PID cap; expressed here (not as pids_limit) so the file stays valid + # on Compose v5+, which rejects pids_limit alongside a limits block. + pids: 512 reservations: memory: 1G restart: unless-stopped