diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh
index b624a3115..19d6766f6 100644
--- a/deploy/docker/entrypoint.sh
+++ b/deploy/docker/entrypoint.sh
@@ -31,6 +31,8 @@ else
# No credential -> refuse to expose; serve loopback only.
GUNICORN_BIND="127.0.0.1:${PORT}"
echo "entrypoint: no CRAWL4AI_API_TOKEN set; binding loopback only (${GUNICORN_BIND})." >&2
+ echo "entrypoint: WARNING: this is the CONTAINER's loopback - published ports (-p ${PORT}:${PORT}) will NOT work; connections from the host will be reset." >&2
+ echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or the .llm.env file with docker compose) and restart." >&2
fi
export GUNICORN_BIND
diff --git a/deploy/docker/schemas.py b/deploy/docker/schemas.py
index f066e5c88..b1065b5a8 100644
--- a/deploy/docker/schemas.py
+++ b/deploy/docker/schemas.py
@@ -42,6 +42,14 @@ class HookConfig(BaseModel):
le=120,
description="Timeout in seconds for each hook execution",
)
+ # Legacy 0.8.x field: inline-Python hook code, removed in 0.9.0 (it was an
+ # exec()-based RCE surface). Captured here (instead of being dropped by
+ # pydantic) solely so the server can tell the caller it was NOT executed;
+ # it is never run.
+ code: Optional[Dict[str, str]] = Field(
+ default=None,
+ description="REMOVED in 0.9.0: inline hook code is accepted for compatibility but never executed",
+ )
class Config:
json_schema_extra = {
diff --git a/deploy/docker/server.py b/deploy/docker/server.py
index 410b8be38..ea6de2ac0 100644
--- a/deploy/docker/server.py
+++ b/deploy/docker/server.py
@@ -392,7 +392,7 @@ def _current_api_token() -> str:
app.add_middleware(
AuthGateMiddleware,
token_provider=_current_api_token,
- public_paths={HEALTH_PATH, "/token"},
+ public_paths={HEALTH_PATH, "/token", "/"},
public_prefixes=_UI_PREFIXES,
)
@@ -660,6 +660,17 @@ async def get_artifact(artifact_id: str, _td: Dict = Depends(token_dep)):
# Screenshot endpoint
+_OUTPUT_PATH_WARNING = (
+ "output_path was removed in 0.9.0 and is ignored - no file was written. "
+ "The result is stored server-side; fetch it with an authenticated "
+ "GET /artifacts/{artifact_id}."
+)
+
+_HOOKS_CODE_WARNING = (
+ "Inline hook code (hooks.code) was removed in 0.9.0 and was NOT executed. "
+ "Use declarative hook actions instead (GET /hooks/info for the schema)."
+)
+
@app.post("/screenshot")
@limiter.limit(config["rate_limiting"]["default_limit"])
@@ -684,7 +695,12 @@ async def generate_screenshot(
raise HTTPException(500, detail=results[0].error_message or "Crawl failed")
screenshot_data = results[0].screenshot
art = _store_artifact("png", base64.b64decode(screenshot_data))
- return {"success": True, "screenshot": screenshot_data, **art}
+ response = {"success": True, "screenshot": screenshot_data, **art}
+ # Legacy 0.8.x key, no longer a schema field: peek at the raw body
+ # (already parsed and cached by FastAPI) to warn that it was ignored.
+ if (await request.json()).get("output_path"):
+ response["warning"] = _OUTPUT_PATH_WARNING
+ return response
except HTTPException:
raise
except Exception as e:
@@ -719,7 +735,12 @@ async def generate_pdf(
raise HTTPException(500, detail=results[0].error_message or "Crawl failed")
pdf_data = results[0].pdf
art = _store_artifact("pdf", pdf_data)
- return {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art}
+ response = {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art}
+ # Legacy 0.8.x key, no longer a schema field: peek at the raw body
+ # (already parsed and cached by FastAPI) to warn that it was ignored.
+ if (await request.json()).get("output_path"):
+ response["warning"] = _OUTPUT_PATH_WARNING
+ return response
except HTTPException:
raise
except Exception as e:
@@ -889,7 +910,7 @@ async def crawl(
raise HTTPException(400, f"Rejected config: {e}")
if crawler_config.stream:
return await stream_process(crawl_request=crawl_request)
-
+
# Prepare hooks config if provided
hooks_config = None
if crawl_request.hooks:
@@ -897,7 +918,7 @@ async def crawl(
'hooks': crawl_request.hooks.hooks,
'timeout': crawl_request.hooks.timeout
}
-
+
results = await handle_crawl_request(
urls=crawl_request.urls,
browser_config=crawl_request.browser_config,
@@ -906,6 +927,11 @@ async def crawl(
hooks_config=hooks_config,
crawler_configs=crawl_request.crawler_configs,
)
+ if crawl_request.hooks and crawl_request.hooks.code:
+ hooks_resp = results.setdefault("hooks", {"attached": []})
+ if not crawl_request.hooks.hooks:
+ hooks_resp["status"] = "ignored"
+ hooks_resp["warning"] = _HOOKS_CODE_WARNING
# check if all of the results are not successful
if all(not result["success"] for result in results["results"]):
raise HTTPException(500, f"Crawl request failed: {results['results'][0]['error_message']}")
@@ -927,7 +953,7 @@ async def crawl_stream(
return await stream_process(crawl_request=crawl_request)
async def stream_process(crawl_request: CrawlRequestWithHooks):
-
+
# Prepare hooks config if provided# Prepare hooks config if provided
hooks_config = None
if crawl_request.hooks:
@@ -953,6 +979,8 @@ async def stream_process(crawl_request: CrawlRequestWithHooks):
if hooks_info:
import json
headers["X-Hooks-Status"] = json.dumps(hooks_info['status']['status'])
+ if crawl_request.hooks and crawl_request.hooks.code:
+ headers["X-Hooks-Warning"] = _HOOKS_CODE_WARNING
return StreamingResponse(
stream_results(crawler, gen),
diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html
index e88c281a0..cf72a303b 100644
--- a/deploy/docker/static/playground/index.html
+++ b/deploy/docker/static/playground/index.html
@@ -617,6 +617,17 @@ 🔥 Stress Test
}
}
+ // Build a useful error message from a failed HTTP response
+ function httpErrorMessage(response, data) {
+ let msg = (data && (data.detail || data.error)) || `HTTP ${response.status}`;
+ if (response.status === 401) {
+ msg += getToken()
+ ? ' — token rejected; check the API token in the token bar (top right)'
+ : ' — set your API token in the token bar (top right)';
+ }
+ return msg;
+ }
+
// Generate code snippets
function generateSnippets(api, payload, method = 'POST') {
// Python snippet
@@ -751,7 +762,7 @@ 🔥 Stress Test
const time = Math.round(performance.now() - startTime);
if (!response.ok) {
updateStatus('error', time);
- throw new Error(responseData.error || 'Request failed');
+ throw new Error(httpErrorMessage(response, responseData));
}
updateStatus('success', time);
document.querySelector('#response-content code').textContent = JSON.stringify(responseData, null, 2);
@@ -765,6 +776,12 @@ 🔥 Stress Test
body: JSON.stringify(payload)
});
+ if (!response.ok) {
+ const errData = await response.json().catch(() => ({}));
+ updateStatus('error', Math.round(performance.now() - startTime));
+ throw new Error(httpErrorMessage(response, errData));
+ }
+
const reader = response.body.getReader();
let text = '';
let maxMemory = 0;
@@ -809,7 +826,7 @@ 🔥 Stress Test
if (!response.ok) {
updateStatus('error', time);
- throw new Error(responseData.error || 'Request failed');
+ throw new Error(httpErrorMessage(response, responseData));
}
updateStatus(
diff --git a/deploy/docker/tests/test_legacy_compat.py b/deploy/docker/tests/test_legacy_compat.py
new file mode 100644
index 000000000..b8b8402c5
--- /dev/null
+++ b/deploy/docker/tests/test_legacy_compat.py
@@ -0,0 +1,206 @@
+"""
+Behavioral tests for 0.9.x legacy-compatibility handling:
+
+ * root redirect - "/" is public and redirects to /playground instead of
+ dying in the auth gate with a bare 401; /monitor and the
+ data routes stay gated.
+ * output_path - /screenshot and /pdf still accept the 0.8.x output_path
+ field but return a warning saying no file was written,
+ instead of silently dropping it.
+ * legacy hooks - hooks.code (removed 0.8.x inline Python) is captured,
+ never executed, and reported as status "ignored" with a
+ warning when hooks are enabled; any hooks payload is
+ still refused (403) while hooks are disabled.
+ * compose file - the PID cap lives under deploy.resources.limits (not
+ pids_limit), which Compose v5 rejects alongside a
+ limits block.
+
+These exercise the running app via TestClient (no browser / Redis needed);
+crawl internals are stubbed where a handler would otherwise need a browser.
+"""
+
+import base64
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import yaml
+
+from auth import create_access_token # noqa: E402
+
+
+def _bearer() -> dict:
+ return {"Authorization": f"Bearer {create_access_token({'sub': 'user@x.com'}, scope='data')}"}
+
+
+# ───────────────────────── root redirect ─────────────────────────
+
+
+class TestRootRedirect:
+ def test_root_is_public_and_redirects_to_playground(self, stock_client):
+ r = stock_client.get("/", follow_redirects=False)
+ assert r.status_code in (302, 307), (
+ f"GET / returned {r.status_code}; expected a redirect. The auth "
+ f"gate must allow the exact path '/' so the redirect route runs."
+ )
+ assert r.headers["location"] == "/playground"
+
+ def test_monitor_and_data_routes_stay_gated(self, stock_client):
+ assert stock_client.get("/monitor").status_code == 401
+ assert stock_client.get("/monitor/health").status_code == 401
+ assert stock_client.post("/crawl", json={"urls": ["https://x"]}).status_code == 401
+
+
+# ───────────────────────── output_path warning ─────────────────────────
+
+
+@pytest.fixture
+def stub_crawler(server_module, monkeypatch):
+ """Stub the crawler pool + artifact store so /screenshot and /pdf run
+ without a browser. Returns the fake artifact dict for assertions."""
+ art = {"artifact_id": "a1", "url": "/artifacts/a1", "mime": "x", "size": 1}
+ png_b64 = base64.b64encode(b"fake-png").decode()
+
+ fake_result = SimpleNamespace(success=True, screenshot=png_b64, pdf=b"fake-pdf")
+
+ class _FakeCrawler:
+ async def arun(self, url, config):
+ return [fake_result]
+
+ async def fake_get_crawler(cfg):
+ return _FakeCrawler()
+
+ async def fake_release_crawler(crawler):
+ pass
+
+ monkeypatch.setattr(server_module, "get_crawler", fake_get_crawler)
+ monkeypatch.setattr(server_module, "release_crawler", fake_release_crawler)
+ monkeypatch.setattr(server_module, "_store_artifact", lambda kind, data: dict(art))
+ return art
+
+
+class TestOutputPathWarning:
+ @pytest.mark.parametrize("endpoint,payload_key", [("/screenshot", "screenshot"), ("/pdf", "pdf")])
+ def test_output_path_accepted_with_warning(self, stock_client, stub_crawler, endpoint, payload_key):
+ r = stock_client.post(
+ endpoint,
+ json={"url": "https://example.com", "output_path": "/tmp/x.bin"},
+ headers=_bearer(),
+ )
+ assert r.status_code == 200
+ body = r.json()
+ assert body["success"] is True
+ assert "warning" in body, "output_path must not be silently dropped"
+ assert "no file was written" in body["warning"]
+ assert body["artifact_id"] == stub_crawler["artifact_id"]
+
+ @pytest.mark.parametrize("endpoint", ["/screenshot", "/pdf"])
+ def test_no_warning_without_output_path(self, stock_client, stub_crawler, endpoint):
+ r = stock_client.post(endpoint, json={"url": "https://example.com"}, headers=_bearer())
+ assert r.status_code == 200
+ assert "warning" not in r.json()
+
+
+# ───────────────────────── legacy hooks.code ─────────────────────────
+
+LEGACY_HOOKS = {"code": {"before_goto": "async def hook(p, c, u, **kw): return p"}}
+DECLARATIVE_HOOKS = {"hooks": [{"action": "scroll_to_bottom", "params": {"max_steps": 2}}]}
+
+
+class TestLegacyHookCode:
+ def test_hook_config_captures_code_field(self):
+ """The legacy field must be parsed (not dropped) so it can be reported."""
+ from schemas import CrawlRequestWithHooks
+
+ req = CrawlRequestWithHooks(urls=["https://x"], hooks=LEGACY_HOOKS)
+ assert req.hooks.code == LEGACY_HOOKS["code"]
+ assert req.hooks.hooks == []
+
+ @pytest.mark.parametrize("hooks_payload", [LEGACY_HOOKS, DECLARATIVE_HOOKS])
+ def test_any_hooks_payload_403_when_disabled(self, server_module, monkeypatch, stock_client, hooks_payload):
+ monkeypatch.setattr(server_module, "HOOKS_ENABLED", False)
+ r = stock_client.post(
+ "/crawl",
+ json={"urls": ["https://example.com"], "hooks": hooks_payload},
+ headers=_bearer(),
+ )
+ assert r.status_code == 403
+
+ def _stub_crawl(self, server_module, monkeypatch, results):
+ async def fake_handle_crawl_request(**kwargs):
+ return dict(results)
+
+ monkeypatch.setattr(server_module, "handle_crawl_request", fake_handle_crawl_request)
+
+ def test_legacy_code_warned_and_ignored_when_enabled(self, server_module, monkeypatch, stock_client):
+ monkeypatch.setattr(server_module, "HOOKS_ENABLED", True)
+ # Empty declarative specs -> api.py reports a vacuous success
+ self._stub_crawl(
+ server_module, monkeypatch,
+ {"success": True, "results": [{"success": True}],
+ "hooks": {"status": "success", "attached": []}},
+ )
+ r = stock_client.post(
+ "/crawl",
+ json={"urls": ["https://example.com"], "hooks": LEGACY_HOOKS},
+ headers=_bearer(),
+ )
+ assert r.status_code == 200
+ hooks = r.json()["hooks"]
+ assert hooks["status"] == "ignored", "vacuous 'success' must be rewritten"
+ assert hooks["attached"] == []
+ assert "NOT executed" in hooks["warning"]
+
+ def test_mixed_request_keeps_declarative_status_and_warns(self, server_module, monkeypatch, stock_client):
+ monkeypatch.setattr(server_module, "HOOKS_ENABLED", True)
+ self._stub_crawl(
+ server_module, monkeypatch,
+ {"success": True, "results": [{"success": True}],
+ "hooks": {"status": "success", "attached": ["before_retrieve_html"]}},
+ )
+ r = stock_client.post(
+ "/crawl",
+ json={"urls": ["https://example.com"],
+ "hooks": {**DECLARATIVE_HOOKS, **LEGACY_HOOKS}},
+ headers=_bearer(),
+ )
+ assert r.status_code == 200
+ hooks = r.json()["hooks"]
+ assert hooks["status"] == "success", "real declarative execution must not be relabeled"
+ assert hooks["attached"] == ["before_retrieve_html"]
+ assert "NOT executed" in hooks["warning"]
+
+ def test_no_hooks_response_untouched(self, server_module, monkeypatch, stock_client):
+ self._stub_crawl(
+ server_module, monkeypatch,
+ {"success": True, "results": [{"success": True}]},
+ )
+ r = stock_client.post("/crawl", json={"urls": ["https://example.com"]}, headers=_bearer())
+ assert r.status_code == 200
+ assert "hooks" not in r.json()
+
+
+# ───────────────────────── compose file ─────────────────────────
+
+
+class TestComposeFile:
+ def test_pid_cap_lives_under_deploy_limits(self):
+ """Compose v5 rejects pids_limit next to deploy.resources.limits
+ ("can't set distinct values"); the cap must be expressed once, under
+ deploy.resources.limits.pids."""
+ import os
+
+ override = os.environ.get("CRAWL4AI_COMPOSE_FILE")
+ if override:
+ compose_path = Path(override)
+ else:
+ here = Path(__file__).resolve()
+ if len(here.parents) < 4:
+ pytest.skip("not running from a repo checkout")
+ compose_path = here.parents[3] / "docker-compose.yml"
+ if not compose_path.exists():
+ pytest.skip(f"docker-compose.yml not found at {compose_path}")
+ doc = yaml.safe_load(compose_path.read_text())
+ base = doc["x-base-config"]
+ assert "pids_limit" not in base
+ assert base["deploy"]["resources"]["limits"]["pids"] == 512
diff --git a/docker-compose.yml b/docker-compose.yml
index 1cd87ad93..5939b3321 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -24,7 +24,6 @@ x-base-config: &base-config
- ALL
security_opt:
- no-new-privileges:true
- pids_limit: 512
# Read-only root filesystem; only these paths are writable (tmpfs).
read_only: true
tmpfs:
@@ -39,6 +38,9 @@ x-base-config: &base-config
resources:
limits:
memory: 4G
+ # PID cap; expressed here (not as pids_limit) so the file stays valid
+ # on Compose v5+, which rejects pids_limit alongside a limits block.
+ pids: 512
reservations:
memory: 1G
restart: unless-stopped