Add compose v5 compatibility, legacy-field warnings, UI error fixes

deploy/docker/entrypoint.sh

@@ -31,6 +31,8 @@ else
     # No credential -> refuse to expose; serve loopback only.
     GUNICORN_BIND="127.0.0.1:${PORT}"
     echo "entrypoint: no CRAWL4AI_API_TOKEN set; binding loopback only (${GUNICORN_BIND})." >&2
+    echo "entrypoint: WARNING: this is the CONTAINER's loopback - published ports (-p ${PORT}:${PORT}) will NOT work; connections from the host will be reset." >&2
+    echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or the .llm.env file with docker compose) and restart." >&2
 fi
 export GUNICORN_BIND
 

deploy/docker/schemas.py

@@ -42,6 +42,14 @@ class HookConfig(BaseModel):
         le=120,
         description="Timeout in seconds for each hook execution",
     )
+    # Legacy 0.8.x field: inline-Python hook code, removed in 0.9.0 (it was an
+    # exec()-based RCE surface). Captured here (instead of being dropped by
+    # pydantic) solely so the server can tell the caller it was NOT executed;
+    # it is never run.
+    code: Optional[Dict[str, str]] = Field(
+        default=None,
+        description="REMOVED in 0.9.0: inline hook code is accepted for compatibility but never executed",
+    )
 
     class Config:
         json_schema_extra = {

deploy/docker/server.py

@@ -392,7 +392,7 @@ def _current_api_token() -> str:
 app.add_middleware(
     AuthGateMiddleware,
     token_provider=_current_api_token,
-    public_paths={HEALTH_PATH, "/token"},
+    public_paths={HEALTH_PATH, "/token", "/"},
     public_prefixes=_UI_PREFIXES,
 )
 
@@ -660,6 +660,17 @@ async def get_artifact(artifact_id: str, _td: Dict = Depends(token_dep)):
 
 # Screenshot endpoint
 
+_OUTPUT_PATH_WARNING = (
+    "output_path was removed in 0.9.0 and is ignored - no file was written. "
+    "The result is stored server-side; fetch it with an authenticated "
+    "GET /artifacts/{artifact_id}."
+)
+
+_HOOKS_CODE_WARNING = (
+    "Inline hook code (hooks.code) was removed in 0.9.0 and was NOT executed. "
+    "Use declarative hook actions instead (GET /hooks/info for the schema)."
+)
+
 
 @app.post("/screenshot")
 @limiter.limit(config["rate_limiting"]["default_limit"])
@@ -684,7 +695,12 @@ async def generate_screenshot(
             raise HTTPException(500, detail=results[0].error_message or "Crawl failed")
         screenshot_data = results[0].screenshot
         art = _store_artifact("png", base64.b64decode(screenshot_data))
-        return {"success": True, "screenshot": screenshot_data, **art}
+        response = {"success": True, "screenshot": screenshot_data, **art}
+        # Legacy 0.8.x key, no longer a schema field: peek at the raw body
+        # (already parsed and cached by FastAPI) to warn that it was ignored.
+        if (await request.json()).get("output_path"):
+            response["warning"] = _OUTPUT_PATH_WARNING
+        return response
     except HTTPException:
         raise
     except Exception as e:
@@ -719,7 +735,12 @@ async def generate_pdf(
             raise HTTPException(500, detail=results[0].error_message or "Crawl failed")
         pdf_data = results[0].pdf
         art = _store_artifact("pdf", pdf_data)
-        return {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art}
+        response = {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art}
+        # Legacy 0.8.x key, no longer a schema field: peek at the raw body
+        # (already parsed and cached by FastAPI) to warn that it was ignored.
+        if (await request.json()).get("output_path"):
+            response["warning"] = _OUTPUT_PATH_WARNING
+        return response
     except HTTPException:
         raise
     except Exception as e:
@@ -889,15 +910,15 @@ async def crawl(
         raise HTTPException(400, f"Rejected config: {e}")
     if crawler_config.stream:
         return await stream_process(crawl_request=crawl_request)
-    
+
     # Prepare hooks config if provided
     hooks_config = None
     if crawl_request.hooks:
         hooks_config = {
             'hooks': crawl_request.hooks.hooks,
             'timeout': crawl_request.hooks.timeout
         }
-    
+
     results = await handle_crawl_request(
         urls=crawl_request.urls,
         browser_config=crawl_request.browser_config,
@@ -906,6 +927,11 @@ async def crawl(
         hooks_config=hooks_config,
         crawler_configs=crawl_request.crawler_configs,
     )
+    if crawl_request.hooks and crawl_request.hooks.code:
+        hooks_resp = results.setdefault("hooks", {"attached": []})
+        if not crawl_request.hooks.hooks:
+            hooks_resp["status"] = "ignored"
+        hooks_resp["warning"] = _HOOKS_CODE_WARNING
     # check if all of the results are not successful
     if all(not result["success"] for result in results["results"]):
         raise HTTPException(500, f"Crawl request failed: {results['results'][0]['error_message']}")
@@ -927,7 +953,7 @@ async def crawl_stream(
     return await stream_process(crawl_request=crawl_request)
 
 async def stream_process(crawl_request: CrawlRequestWithHooks):
-    
+
     # Prepare hooks config if provided# Prepare hooks config if provided
     hooks_config = None
     if crawl_request.hooks:
@@ -953,6 +979,8 @@ async def stream_process(crawl_request: CrawlRequestWithHooks):
     if hooks_info:
         import json
         headers["X-Hooks-Status"] = json.dumps(hooks_info['status']['status'])
+    if crawl_request.hooks and crawl_request.hooks.code:
+        headers["X-Hooks-Warning"] = _HOOKS_CODE_WARNING
 
     return StreamingResponse(
         stream_results(crawler, gen),

deploy/docker/static/playground/index.html

@@ -617,6 +617,17 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
             }
         }
 
+        // Build a useful error message from a failed HTTP response
+        function httpErrorMessage(response, data) {
+            let msg = (data && (data.detail || data.error)) || `HTTP ${response.status}`;
+            if (response.status === 401) {
+                msg += getToken()
+                    ? ' — token rejected; check the API token in the token bar (top right)'
+                    : ' — set your API token in the token bar (top right)';
+            }
+            return msg;
+        }
+
         // Generate code snippets
         function generateSnippets(api, payload, method = 'POST') {
             // Python snippet
@@ -751,7 +762,7 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
                     const time = Math.round(performance.now() - startTime);
                     if (!response.ok) {
                         updateStatus('error', time);
-                        throw new Error(responseData.error || 'Request failed');
+                        throw new Error(httpErrorMessage(response, responseData));
                     }
                     updateStatus('success', time);
                     document.querySelector('#response-content code').textContent = JSON.stringify(responseData, null, 2);
@@ -765,6 +776,12 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
                         body: JSON.stringify(payload)
                     });
 
+                    if (!response.ok) {
+                        const errData = await response.json().catch(() => ({}));
+                        updateStatus('error', Math.round(performance.now() - startTime));
+                        throw new Error(httpErrorMessage(response, errData));
+                    }
+
                     const reader = response.body.getReader();
                     let text = '';
                     let maxMemory = 0;
@@ -809,7 +826,7 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
 
                     if (!response.ok) {
                         updateStatus('error', time);
-                        throw new Error(responseData.error || 'Request failed');
+                        throw new Error(httpErrorMessage(response, responseData));
                     }
 
                     updateStatus(

deploy/docker/tests/test_legacy_compat.py

@@ -0,0 +1,206 @@
+"""
+Behavioral tests for 0.9.x legacy-compatibility handling:
+
+  * root redirect   - "/" is public and redirects to /playground instead of
+                      dying in the auth gate with a bare 401; /monitor and the
+                      data routes stay gated.
+  * output_path     - /screenshot and /pdf still accept the 0.8.x output_path
+                      field but return a warning saying no file was written,
+                      instead of silently dropping it.
+  * legacy hooks    - hooks.code (removed 0.8.x inline Python) is captured,
+                      never executed, and reported as status "ignored" with a
+                      warning when hooks are enabled; any hooks payload is
+                      still refused (403) while hooks are disabled.
+  * compose file    - the PID cap lives under deploy.resources.limits (not
+                      pids_limit), which Compose v5 rejects alongside a
+                      limits block.
+
+These exercise the running app via TestClient (no browser / Redis needed);
+crawl internals are stubbed where a handler would otherwise need a browser.
+"""
+
+import base64
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import yaml
+
+from auth import create_access_token  # noqa: E402
+
+
+def _bearer() -> dict:
+    return {"Authorization": f"Bearer {create_access_token({'sub': 'user@x.com'}, scope='data')}"}
+
+
+# ───────────────────────── root redirect ─────────────────────────
+
+
+class TestRootRedirect:
+    def test_root_is_public_and_redirects_to_playground(self, stock_client):
+        r = stock_client.get("/", follow_redirects=False)
+        assert r.status_code in (302, 307), (
+            f"GET / returned {r.status_code}; expected a redirect. The auth "
+            f"gate must allow the exact path '/' so the redirect route runs."
+        )
+        assert r.headers["location"] == "/playground"
+
+    def test_monitor_and_data_routes_stay_gated(self, stock_client):
+        assert stock_client.get("/monitor").status_code == 401
+        assert stock_client.get("/monitor/health").status_code == 401
+        assert stock_client.post("/crawl", json={"urls": ["https://x"]}).status_code == 401
+
+
+# ───────────────────────── output_path warning ─────────────────────────
+
+
+@pytest.fixture
+def stub_crawler(server_module, monkeypatch):
+    """Stub the crawler pool + artifact store so /screenshot and /pdf run
+    without a browser. Returns the fake artifact dict for assertions."""
+    art = {"artifact_id": "a1", "url": "/artifacts/a1", "mime": "x", "size": 1}
+    png_b64 = base64.b64encode(b"fake-png").decode()
+
+    fake_result = SimpleNamespace(success=True, screenshot=png_b64, pdf=b"fake-pdf")
+
+    class _FakeCrawler:
+        async def arun(self, url, config):
+            return [fake_result]
+
+    async def fake_get_crawler(cfg):
+        return _FakeCrawler()
+
+    async def fake_release_crawler(crawler):
+        pass
+
+    monkeypatch.setattr(server_module, "get_crawler", fake_get_crawler)
+    monkeypatch.setattr(server_module, "release_crawler", fake_release_crawler)
+    monkeypatch.setattr(server_module, "_store_artifact", lambda kind, data: dict(art))
+    return art
+
+
+class TestOutputPathWarning:
+    @pytest.mark.parametrize("endpoint,payload_key", [("/screenshot", "screenshot"), ("/pdf", "pdf")])
+    def test_output_path_accepted_with_warning(self, stock_client, stub_crawler, endpoint, payload_key):
+        r = stock_client.post(
+            endpoint,
+            json={"url": "https://example.com", "output_path": "/tmp/x.bin"},
+            headers=_bearer(),
+        )
+        assert r.status_code == 200
+        body = r.json()
+        assert body["success"] is True
+        assert "warning" in body, "output_path must not be silently dropped"
+        assert "no file was written" in body["warning"]
+        assert body["artifact_id"] == stub_crawler["artifact_id"]
+
+    @pytest.mark.parametrize("endpoint", ["/screenshot", "/pdf"])
+    def test_no_warning_without_output_path(self, stock_client, stub_crawler, endpoint):
+        r = stock_client.post(endpoint, json={"url": "https://example.com"}, headers=_bearer())
+        assert r.status_code == 200
+        assert "warning" not in r.json()
+
+
+# ───────────────────────── legacy hooks.code ─────────────────────────
+
+LEGACY_HOOKS = {"code": {"before_goto": "async def hook(p, c, u, **kw): return p"}}
+DECLARATIVE_HOOKS = {"hooks": [{"action": "scroll_to_bottom", "params": {"max_steps": 2}}]}
+
+
+class TestLegacyHookCode:
+    def test_hook_config_captures_code_field(self):
+        """The legacy field must be parsed (not dropped) so it can be reported."""
+        from schemas import CrawlRequestWithHooks
+
+        req = CrawlRequestWithHooks(urls=["https://x"], hooks=LEGACY_HOOKS)
+        assert req.hooks.code == LEGACY_HOOKS["code"]
+        assert req.hooks.hooks == []
+
+    @pytest.mark.parametrize("hooks_payload", [LEGACY_HOOKS, DECLARATIVE_HOOKS])
+    def test_any_hooks_payload_403_when_disabled(self, server_module, monkeypatch, stock_client, hooks_payload):
+        monkeypatch.setattr(server_module, "HOOKS_ENABLED", False)
+        r = stock_client.post(
+            "/crawl",
+            json={"urls": ["https://example.com"], "hooks": hooks_payload},
+            headers=_bearer(),
+        )
+        assert r.status_code == 403
+
+    def _stub_crawl(self, server_module, monkeypatch, results):
+        async def fake_handle_crawl_request(**kwargs):
+            return dict(results)
+
+        monkeypatch.setattr(server_module, "handle_crawl_request", fake_handle_crawl_request)
+
+    def test_legacy_code_warned_and_ignored_when_enabled(self, server_module, monkeypatch, stock_client):
+        monkeypatch.setattr(server_module, "HOOKS_ENABLED", True)
+        # Empty declarative specs -> api.py reports a vacuous success
+        self._stub_crawl(
+            server_module, monkeypatch,
+            {"success": True, "results": [{"success": True}],
+             "hooks": {"status": "success", "attached": []}},
+        )
+        r = stock_client.post(
+            "/crawl",
+            json={"urls": ["https://example.com"], "hooks": LEGACY_HOOKS},
+            headers=_bearer(),
+        )
+        assert r.status_code == 200
+        hooks = r.json()["hooks"]
+        assert hooks["status"] == "ignored", "vacuous 'success' must be rewritten"
+        assert hooks["attached"] == []
+        assert "NOT executed" in hooks["warning"]
+
+    def test_mixed_request_keeps_declarative_status_and_warns(self, server_module, monkeypatch, stock_client):
+        monkeypatch.setattr(server_module, "HOOKS_ENABLED", True)
+        self._stub_crawl(
+            server_module, monkeypatch,
+            {"success": True, "results": [{"success": True}],
+             "hooks": {"status": "success", "attached": ["before_retrieve_html"]}},
+        )
+        r = stock_client.post(
+            "/crawl",
+            json={"urls": ["https://example.com"],
+                  "hooks": {**DECLARATIVE_HOOKS, **LEGACY_HOOKS}},
+            headers=_bearer(),
+        )
+        assert r.status_code == 200
+        hooks = r.json()["hooks"]
+        assert hooks["status"] == "success", "real declarative execution must not be relabeled"
+        assert hooks["attached"] == ["before_retrieve_html"]
+        assert "NOT executed" in hooks["warning"]
+
+    def test_no_hooks_response_untouched(self, server_module, monkeypatch, stock_client):
+        self._stub_crawl(
+            server_module, monkeypatch,
+            {"success": True, "results": [{"success": True}]},
+        )
+        r = stock_client.post("/crawl", json={"urls": ["https://example.com"]}, headers=_bearer())
+        assert r.status_code == 200
+        assert "hooks" not in r.json()
+
+
+# ───────────────────────── compose file ─────────────────────────
+
+
+class TestComposeFile:
+    def test_pid_cap_lives_under_deploy_limits(self):
+        """Compose v5 rejects pids_limit next to deploy.resources.limits
+        ("can't set distinct values"); the cap must be expressed once, under
+        deploy.resources.limits.pids."""
+        import os
+
+        override = os.environ.get("CRAWL4AI_COMPOSE_FILE")
+        if override:
+            compose_path = Path(override)
+        else:
+            here = Path(__file__).resolve()
+            if len(here.parents) < 4:
+                pytest.skip("not running from a repo checkout")
+            compose_path = here.parents[3] / "docker-compose.yml"
+        if not compose_path.exists():
+            pytest.skip(f"docker-compose.yml not found at {compose_path}")
+        doc = yaml.safe_load(compose_path.read_text())
+        base = doc["x-base-config"]
+        assert "pids_limit" not in base
+        assert base["deploy"]["resources"]["limits"]["pids"] == 512