diff --git a/.agents/skills/crawl-with-crwl/SKILL.md b/.agents/skills/crawl-with-crwl/SKILL.md new file mode 100644 index 000000000..0a4974d6a --- /dev/null +++ b/.agents/skills/crawl-with-crwl/SKILL.md @@ -0,0 +1,36 @@ +--- +name: crawl-with-crwl +description: Crawl public web pages and extract Markdown or structured content with the crwl CLI. Use when an agent needs current page content, focused extraction, a deep crawl, or machine-readable crawl results and the crwl command is available. +--- + +# Crawl with crwl + +Use only the `crwl` executable. Do not call `crwl-remote` directly. A user may +alias `crwl` to a remote backend, so keep commands backend-agnostic. + +## Workflow + +1. Confirm the target URL and requested output. +2. Run the smallest suitable crawl: + +```bash +crwl 'https://example.com' -o markdown +``` + +3. Use JSON only when structured output is needed: + +```bash +crwl 'https://example.com/products' -o json +``` + +4. Use deep crawling only when the task requires linked pages: + +```bash +crwl 'https://example.com/docs' --deep-crawl bfs --max-pages 10 -o markdown +``` + +5. Inspect the exit status and stderr. Report authentication, connectivity, +robots, or extraction failures rather than silently substituting content. + +Quote URLs in shell commands. Never place API tokens or other credentials in +command output, prompts, logs, or committed files. diff --git a/.agents/skills/crawl-with-crwl/agents/openai.yaml b/.agents/skills/crawl-with-crwl/agents/openai.yaml new file mode 100644 index 000000000..eed348953 --- /dev/null +++ b/.agents/skills/crawl-with-crwl/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Crawl with crwl" + short_description: "Crawl web pages through the crwl command" + default_prompt: "Use $crawl-with-crwl to crawl a URL and return its relevant content." diff --git a/README.md b/README.md index 838553762..4c62b8ab8 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,67 @@ crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10 crwl https://www.example.com/products -q "Extract all product prices" ``` +### Use `crwl` with a remote Crawl4AI API + +If Crawl4AI runs on another machine, in Docker, or as a hosted service, the +dependency-light remote client can send the same crawl configuration to its +HTTP API. Save credentials once in `~/.crawl4ai/remote.json`: + +```bash +crwl-remote config +crwl-remote https://www.nbcnews.com/business -o markdown +``` + +The credential file is created with user-only permissions. For automation, +`CRAWL4AI_API_URL` and `CRAWL4AI_API_TOKEN` override saved credentials, and +`--api-url` / `--api-token` take highest priority. + +Browser settings, including an upstream proxy, can be supplied through a +configuration file: + +```json +{ + "proxy_config": { + "server": "http://proxy.example:8080", + "username": "proxy-user", + "password": "proxy-password" + } +} +``` + +```bash +crwl-remote https://ipv4.webshare.io -B browser.json -o markdown +``` + +The server's static `CRAWL4AI_API_TOKEN` is an operator credential, so browser +and crawler configuration supplied with it is trusted, including proxy +settings. Protect proxy credentials with the same care as the remote API token +and do not commit the browser configuration file. + +To install only the command without Crawl4AI's local browser dependencies: + +```bash +pip install --no-deps 'crawl4ai[crwl-remote]' +``` + +The `crwl-remote` executable is also installed. The `--no-deps` flag is +necessary because Python extras can add dependencies but cannot subtract the +normal Crawl4AI dependencies. + +To make remote crawling the shell default while keeping agent and script +commands written as `crwl ...`: + +```bash +# Bash or Zsh +alias crwl='crwl-remote' + +# Fish +alias crwl 'crwl-remote' + +# PowerShell +function crwl { & crwl-remote @args } +``` + ## 💖 Support Crawl4AI > 🎉 **Sponsorship Program Now Open!** After powering 51K+ developers and 1 year of growth, Crawl4AI is launching dedicated support for **startups** and **enterprises**. Be among the first 50 **Founding Sponsors** for permanent recognition in our Hall of Fame. diff --git a/crwl_remote.py b/crwl_remote.py new file mode 100644 index 000000000..3ddcfe2e0 --- /dev/null +++ b/crwl_remote.py @@ -0,0 +1,253 @@ +"""Dependency-light entry point for a remote Crawl4AI API. + +Remote crawls are dispatched before importing Crawl4AI's local browser stack. +This lets the command use a hosted or Docker API without browser dependencies. +""" + +from __future__ import annotations + +import json +import getpass +import os +import sys +from pathlib import Path +from typing import Any, Optional +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +CONFIG_PATH = Path.home() / ".crawl4ai" / "remote.json" + + +def _saved_credentials() -> dict[str, str]: + if not CONFIG_PATH.is_file(): + return {} + try: + data = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return {key: str(value) for key, value in data.items() if value} + + +def _option_value(args: list[str], option: str) -> Optional[str]: + for index, arg in enumerate(args): + if arg == option and index + 1 < len(args): + return args[index + 1] + if arg.startswith(option + "="): + return arg.split("=", 1)[1] + return None + + +def _api_setting(args: list[str], option: str, env_name: str) -> Optional[str]: + config_key = "api_url" if env_name.endswith("_URL") else "api_token" + return ( + _option_value(args, option) + or os.environ.get(env_name) + or _saved_credentials().get(config_key) + ) + + +def _configure(args: list[str]) -> int: + api_url = _option_value(args, "--api-url") + api_token = _option_value(args, "--api-token") + if not api_url: + api_url = input("Crawl4AI API URL: ").strip() + if not api_token: + api_token = getpass.getpass("Crawl4AI API token: ").strip() + if not api_url or not api_token: + print("Error: both API URL and token are required.", file=sys.stderr) + return 2 + + CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True) + CONFIG_PATH.write_text( + json.dumps({"api_url": api_url.rstrip("/"), "api_token": api_token}, indent=2) + "\n", + encoding="utf-8", + ) + CONFIG_PATH.chmod(0o600) + print(f"Remote credentials saved to {CONFIG_PATH}") + return 0 + + +def _load_config(path: Optional[str]) -> dict[str, Any]: + if not path: + return {} + text = Path(path).read_text(encoding="utf-8") + if path.endswith((".yml", ".yaml")): + try: + import yaml + except ImportError as exc: + raise RuntimeError("YAML config files require PyYAML; use JSON in a minimal install") from exc + return yaml.safe_load(text) or {} + return json.loads(text) + + +def _parse_values(value: Optional[str]) -> dict[str, Any]: + if not value: + return {} + result: dict[str, Any] = {} + for item in value.split(","): + key, separator, raw = item.partition("=") + if not separator: + raise ValueError(f"Invalid key=value pair: {item}") + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = raw + result[key.strip()] = parsed + return result + + +def _remote_args(args: list[str]) -> dict[str, Any]: + positional = [arg for arg in args if not arg.startswith("-")] + if positional and positional[0] == "crawl": + positional.pop(0) + option_values = { + value + for option in ("--api-url", "--api-token", "-B", "--browser-config", "-C", + "--crawler-config", "-b", "--browser", "-c", "--crawler", + "-o", "--output", "-O", "--output-file", "--deep-crawl", + "--max-pages") + if (value := _option_value(args, option)) is not None + } + urls = [value for value in positional if value not in option_values] + if not urls: + raise ValueError("URL argument is required") + + browser_path = _option_value(args, "--browser-config") or _option_value(args, "-B") + crawler_path = _option_value(args, "--crawler-config") or _option_value(args, "-C") + browser_config = _load_config(browser_path) + crawler_config = _load_config(crawler_path) + browser_config.update(_parse_values(_option_value(args, "--browser") or _option_value(args, "-b"))) + crawler_config.update(_parse_values(_option_value(args, "--crawler") or _option_value(args, "-c"))) + if "cache_mode" not in crawler_config: + crawler_config["cache_mode"] = "bypass" + deep_crawl = _option_value(args, "--deep-crawl") + if deep_crawl: + strategy_types = { + "bfs": "BFSDeepCrawlStrategy", + "dfs": "DFSDeepCrawlStrategy", + "best-first": "BestFirstCrawlingStrategy", + } + if deep_crawl not in strategy_types: + raise ValueError("--deep-crawl must be bfs, dfs, or best-first") + raw_max_pages = _option_value(args, "--max-pages") or "10" + crawler_config["deep_crawl_strategy"] = { + "type": strategy_types[deep_crawl], + "params": {"max_depth": 3, "max_pages": int(raw_max_pages)}, + } + + return { + "url": urls[0], + "browser_config": browser_config, + "crawler_config": crawler_config, + "output": _option_value(args, "--output") or _option_value(args, "-o") or "all", + "output_file": _option_value(args, "--output-file") or _option_value(args, "-O"), + } + + +def _render(result: dict[str, Any], output: str) -> str: + if output in ("markdown", "md"): + markdown = result.get("markdown") or "" + return markdown.get("raw_markdown", "") if isinstance(markdown, dict) else str(markdown) + if output in ("markdown-fit", "md-fit"): + markdown = result.get("markdown") or "" + return markdown.get("fit_markdown", "") if isinstance(markdown, dict) else str(markdown) + if output == "json": + extracted = result.get("extracted_content") + if isinstance(extracted, str): + try: + extracted = json.loads(extracted) + except json.JSONDecodeError: + return extracted + return json.dumps(extracted, indent=2, ensure_ascii=False) + return json.dumps(result, indent=2, ensure_ascii=False) + + +def _render_results(results: list[dict[str, Any]], output: str) -> str: + if len(results) == 1 or output not in ("markdown", "md", "markdown-fit", "md-fit"): + return _render(results[0], output) + sections = [] + for result in results: + sections.append( + f"{'=' * 60}\n# {result.get('url', '')}\n{'=' * 60}\n\n" + f"{_render(result, output)}" + ) + return "\n\n".join(sections) + + +def _run_remote(args: list[str], api_url: str, api_token: Optional[str]) -> int: + parsed = _remote_args(args) + payload = json.dumps({ + "urls": [parsed["url"]], + "browser_config": parsed["browser_config"], + "crawler_config": parsed["crawler_config"], + }).encode("utf-8") + headers = {"Content-Type": "application/json"} + if api_token: + headers["Authorization"] = f"Bearer {api_token}" + request = Request( + api_url.rstrip("/") + "/crawl", + data=payload, + headers=headers, + method="POST", + ) + try: + with urlopen(request, timeout=120) as response: + body = json.load(response) + except HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + print(f"Error: Crawl4AI API returned HTTP {exc.code}: {detail}", file=sys.stderr) + return 1 + except URLError as exc: + print(f"Error: Cannot connect to Crawl4AI API: {exc.reason}", file=sys.stderr) + return 1 + + if not body.get("success", False): + print(f"Error: Remote crawl failed: {body.get('msg', 'Unknown error')}", file=sys.stderr) + return 1 + results = body.get("results", []) + if not results: + print("Error: Remote crawl returned no results", file=sys.stderr) + return 1 + rendered = _render_results(results, parsed["output"]) + if parsed["output_file"]: + Path(parsed["output_file"]).write_text(rendered, encoding="utf-8") + else: + print(rendered) + return 0 + + +def main() -> None: + args = sys.argv[1:] + if args and args[0] == "config": + raise SystemExit(_configure(args[1:])) + api_url = _api_setting(args, "--api-url", "CRAWL4AI_API_URL") + if any(arg in ("-h", "--help") for arg in args): + print( + "Usage: crwl-remote URL [OPTIONS]\n" + " crwl-remote config [--api-url URL --api-token TOKEN]\n\n" + "Use CRAWL4AI_API_URL and CRAWL4AI_API_TOKEN (or --api-url and\n" + "--api-token) to crawl through a remote Crawl4AI API. Credentials\n" + "saved by `crwl-remote config` are used as the fallback.\n\n" + "Remote options: -B/--browser-config, -C/--crawler-config,\n" + "-b/--browser, -c/--crawler, -o/--output, -O/--output-file" + ) + return + if api_url: + token = _api_setting(args, "--api-token", "CRAWL4AI_API_TOKEN") + try: + raise SystemExit(_run_remote(args, api_url, token)) + except (OSError, ValueError, RuntimeError, json.JSONDecodeError) as exc: + print(f"Error: {exc}", file=sys.stderr) + raise SystemExit(1) + + print( + "Error: API URL is required; run `crwl-remote config`, set " + "CRAWL4AI_API_URL, or use --api-url.", + file=sys.stderr, + ) + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 1756b925f..c7de5c5d8 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -92,6 +92,19 @@ def _attach_declarative_hooks(crawler, hooks_config: dict) -> dict: logger = logging.getLogger(__name__) + +def _load_browser_config( + data: dict, + provenance: Provenance = Provenance.UNTRUSTED, +) -> BrowserConfig: + """Load browser configuration at the caller's established trust level.""" + browser_config = BrowserConfig.load(data, provenance=provenance) + if provenance == Provenance.UNTRUSTED: + from egress_broker import enforce_egress + enforce_egress(browser_config) + return browser_config + + # --- Helper to get memory --- def _get_memory_mb(): try: @@ -651,6 +664,7 @@ async def handle_crawl_request( config: dict, hooks_config: Optional[dict] = None, crawler_configs: Optional[List[dict]] = None, + provenance: Provenance = Provenance.UNTRUSTED, ) -> dict: """Handle non-streaming crawl requests with optional hooks.""" # Track request start @@ -671,10 +685,12 @@ async def handle_crawl_request( try: urls = _normalize_and_validate_seeds(urls) - browser_config = BrowserConfig.load(browser_config, provenance=Provenance.UNTRUSTED) - crawler_config = CrawlerRunConfig.load(crawler_config, provenance=Provenance.UNTRUSTED) - from egress_broker import enforce_egress - enforce_egress(browser_config) + browser_config = _load_browser_config( + browser_config, provenance=provenance + ) + crawler_config = CrawlerRunConfig.load( + crawler_config, provenance=provenance + ) from governor import clamp_deep_crawl clamp_deep_crawl(crawler_config) @@ -699,7 +715,10 @@ async def handle_crawl_request( # Build the config(s) to pass to arun/arun_many if crawler_configs and len(urls) > 1: # Per-URL config list: deserialize each and apply base_config - config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs] + config_list = [ + CrawlerRunConfig.load(cc, provenance=provenance) + for cc in crawler_configs + ] for cfg in config_list: for key, value in base_config.items(): if hasattr(cfg, key): @@ -855,7 +874,8 @@ async def handle_stream_crawl_request( browser_config: dict, crawler_config: dict, config: dict, - hooks_config: Optional[dict] = None + hooks_config: Optional[dict] = None, + provenance: Provenance = Provenance.UNTRUSTED, ) -> Tuple[AsyncWebCrawler, AsyncGenerator, Optional[Dict]]: """Handle streaming crawl requests with optional hooks.""" hooks_info = None @@ -865,12 +885,14 @@ async def handle_stream_crawl_request( # mirroring handle_crawl_request. The streaming path previously skipped # this, leaving /crawl/stream (and /crawl with stream=true) unguarded. urls = _normalize_and_validate_seeds(urls) - browser_config = BrowserConfig.load(browser_config, provenance=Provenance.UNTRUSTED) + browser_config = _load_browser_config( + browser_config, provenance=provenance + ) # browser_config.verbose = True # Set to False or remove for production stress testing browser_config.verbose = False - from egress_broker import enforce_egress - enforce_egress(browser_config) - crawler_config = CrawlerRunConfig.load(crawler_config, provenance=Provenance.UNTRUSTED) + crawler_config = CrawlerRunConfig.load( + crawler_config, provenance=provenance + ) from governor import clamp_deep_crawl clamp_deep_crawl(crawler_config) crawler_config.scraping_strategy = LXMLWebScrapingStrategy() diff --git a/deploy/docker/server.py b/deploy/docker/server.py index 410b8be38..9a9c63691 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -881,14 +881,21 @@ async def crawl( if crawl_request.hooks and not HOOKS_ENABLED: raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.") # Check whether it is a redirection for a streaming request + provenance = ( + Provenance.TRUSTED + if _td and _td.get("via") == "api_token" + else Provenance.UNTRUSTED + ) try: crawler_config = CrawlerRunConfig.load( - crawl_request.crawler_config, provenance=Provenance.UNTRUSTED + crawl_request.crawler_config, provenance=provenance ) except UntrustedConfigError as e: raise HTTPException(400, f"Rejected config: {e}") if crawler_config.stream: - return await stream_process(crawl_request=crawl_request) + return await stream_process( + crawl_request=crawl_request, provenance=provenance + ) # Prepare hooks config if provided hooks_config = None @@ -905,6 +912,7 @@ async def crawl( config=config, hooks_config=hooks_config, crawler_configs=crawl_request.crawler_configs, + provenance=provenance, ) # check if all of the results are not successful if all(not result["success"] for result in results["results"]): @@ -924,9 +932,19 @@ async def crawl_stream( if crawl_request.hooks and not HOOKS_ENABLED: raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.") - return await stream_process(crawl_request=crawl_request) + return await stream_process( + crawl_request=crawl_request, + provenance=( + Provenance.TRUSTED + if _td and _td.get("via") == "api_token" + else Provenance.UNTRUSTED + ), + ) -async def stream_process(crawl_request: CrawlRequestWithHooks): +async def stream_process( + crawl_request: CrawlRequestWithHooks, + provenance: Provenance = Provenance.UNTRUSTED, +): # Prepare hooks config if provided# Prepare hooks config if provided hooks_config = None @@ -941,7 +959,8 @@ async def stream_process(crawl_request: CrawlRequestWithHooks): browser_config=crawl_request.browser_config, crawler_config=crawl_request.crawler_config, config=config, - hooks_config=hooks_config + hooks_config=hooks_config, + provenance=provenance, ) # Add hooks info to response headers if available diff --git a/deploy/docker/tests/test_admin_proxy_config.py b/deploy/docker/tests/test_admin_proxy_config.py new file mode 100644 index 000000000..bfffd8dbf --- /dev/null +++ b/deploy/docker/tests/test_admin_proxy_config.py @@ -0,0 +1,122 @@ +"""Operator-token trusted configuration regression tests.""" + +from unittest.mock import AsyncMock, patch + +import pytest + +from auth import create_access_token +from crawl4ai.async_configs import Provenance, UntrustedConfigError + +pytestmark = pytest.mark.posture + + +def _bearer(token: str) -> dict: + return {"Authorization": f"Bearer {token}"} + + +class TestTrustedConfigLoading: + def test_operator_browser_config_is_fully_trusted(self): + import api + + config = api._load_browser_config( + { + "proxy_config": { + "server": "http://proxy.example:8080", + "username": "user", + "password": "secret", + }, + "extra_args": ["--disable-web-security"], + }, + provenance=Provenance.TRUSTED, + ) + + assert config.proxy_config.server == "http://proxy.example:8080" + assert config.proxy_config.username == "user" + assert config.proxy_config.password == "secret" + assert config.extra_args == ["--disable-web-security"] + + def test_regular_authenticated_config_remains_untrusted(self): + import api + + with pytest.raises(UntrustedConfigError): + api._load_browser_config( + {"proxy_config": {"server": "http://proxy.example:8080"}}, + provenance=Provenance.UNTRUSTED, + ) + + def test_typed_sdk_proxy_config_is_supported_for_operator(self): + import api + + config = api._load_browser_config( + { + "type": "BrowserConfig", + "params": { + "proxy_config": { + "type": "ProxyConfig", + "params": { + "server": "http://proxy.example:8080", + "username": "sdk-user", + "password": "sdk-secret", + }, + }, + }, + }, + provenance=Provenance.TRUSTED, + ) + + assert config.proxy_config.server == "http://proxy.example:8080" + assert config.proxy_config.username == "sdk-user" + assert config.proxy_config.password == "sdk-secret" + + +def test_operator_token_selects_trusted_provenance( + stock_client, server_module, monkeypatch +): + operator_token = "operator-proxy-test-token" + monkeypatch.setitem( + server_module.config["security"], "api_token", operator_token + ) + response_data = { + "success": True, + "results": [{"url": "https://example.com", "success": True}], + } + + with patch.object( + server_module, + "handle_crawl_request", + new=AsyncMock(return_value=response_data), + ) as handler: + response = stock_client.post( + "/crawl", + json={ + "urls": ["https://example.com"], + "browser_config": { + "proxy_config": {"server": "http://proxy.example:8080"} + }, + "crawler_config": {"js_code": "window.operatorConfigured = true"}, + }, + headers=_bearer(operator_token), + ) + + assert response.status_code == 200, response.text + assert handler.await_args.kwargs["provenance"] == Provenance.TRUSTED + + +def test_admin_jwt_does_not_impersonate_operator_token( + stock_client, server_module +): + token = create_access_token({"sub": "admin"}, scope="admin") + + with patch.object( + server_module, + "stream_process", + new=AsyncMock(return_value={"success": True}), + ) as handler: + response = stock_client.post( + "/crawl/stream", + json={"urls": ["https://example.com"]}, + headers=_bearer(token), + ) + + assert response.status_code == 200, response.text + assert handler.await_args.kwargs["provenance"] == Provenance.UNTRUSTED diff --git a/docs/codebase/remote-cli-requirements.md b/docs/codebase/remote-cli-requirements.md new file mode 100644 index 000000000..445c3bab2 --- /dev/null +++ b/docs/codebase/remote-cli-requirements.md @@ -0,0 +1,69 @@ +# Remote CLI Requirements + +## Goal + +Provide Crawl4AI CLI access to a remote Crawl4AI HTTP API without requiring +Playwright, browsers, or other local crawling dependencies. + +## Change discipline + +- Change only files required to implement, test, or document this feature. +- Preserve unrelated files byte-for-byte, including formatting and trailing + newlines, to keep reviews and pull requests focused. + +## Command compatibility + +- Preserve the existing local `crwl` behavior. +- Provide the standalone `crwl-remote ...`. +- Support aliasing `crwl` to `crwl-remote` for backend-agnostic scripts and + agent workflows. +- Do not provide Docker-specific command names; the API may be hosted anywhere. + +## Lightweight installation + +- Support `pip install --no-deps 'crawl4ai[crwl-remote]'`. +- The remote command must use only the Python standard library before + dispatching a request. +- Remote operation must not import or install local browser/crawler libraries. + +## Credentials + +- Provide `crwl-remote config` to save credentials globally in + `~/.crawl4ai/remote.json`. +- Store the credential file with user-only (`0600`) permissions. +- Resolve credentials in this order: + 1. `--api-url` / `--api-token` + 2. `CRAWL4AI_API_URL` / `CRAWL4AI_API_TOKEN` + 3. The global credential file +- Never print, log, or commit API or proxy credentials. + +## Crawl configuration + +- Support browser and crawler configuration files and inline scalar overrides. +- Support existing output formats and output files. +- Support deep-crawl options. +- Support authenticated browser proxy configuration, including proxy server, + username, and password. + +## Proxy authorization and safety + +- Treat the static `CRAWL4AI_API_TOKEN` as an admin/operator credential. +- Trust browser and crawler configuration supplied by callers authenticated + with the admin token, including proxy configuration. + +## Agent skill + +- Include a repository skill that invokes only `crwl`. +- Keep the skill compatible with local `crwl` and a shell alias to + `crwl-remote`. + +## Acceptance criteria + +- Unit and security suites pass without weakening existing security posture. +- Build the modified backend as a Docker/OCI container. +- Run the API with an admin credential. +- Send two `crwl-remote` requests to `https://ipv4.webshare.io` through a + rotating proxy. +- Both requests must succeed and return different public IP addresses. +- Browser execution for acceptance testing must occur only inside the + container; no local Playwright or browser installation is permitted. diff --git a/pyproject.toml b/pyproject.toml index f93d10680..d221f9b99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ torch = ["torch", "nltk", "scikit-learn"] transformer = ["transformers", "tokenizers", "sentence-transformers"] cosine = ["torch", "transformers", "nltk", "sentence-transformers"] sync = ["selenium"] +crwl-remote = [] all = [ "pypdf", "torch", @@ -81,9 +82,11 @@ crawl4ai-migrate = "crawl4ai.migrations:main" crawl4ai-setup = "crawl4ai.install:post_install" crawl4ai-doctor = "crawl4ai.install:doctor" crwl = "crawl4ai.cli:main" +crwl-remote = "crwl_remote:main" [tool.setuptools] packages = {find = {where = ["."], include = ["crawl4ai*"]}} +py-modules = ["crwl_remote"] [tool.setuptools.package-data] crawl4ai = ["js_snippet/*.js", "crawlers/google_search/*.js"] diff --git a/tests/cli/test_crwl_remote.py b/tests/cli/test_crwl_remote.py new file mode 100644 index 000000000..9b6b140fc --- /dev/null +++ b/tests/cli/test_crwl_remote.py @@ -0,0 +1,109 @@ +import io +import json +from unittest.mock import patch + +import crwl_remote + + +class FakeResponse(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +def test_remote_entrypoint_sends_token_and_configs(capsys): + response = FakeResponse(json.dumps({ + "success": True, + "results": [{ + "url": "https://example.com", + "markdown": {"raw_markdown": "# From Docker"}, + }], + }).encode()) + + with patch("crwl_remote.urlopen", return_value=response) as urlopen: + code = crwl_remote._run_remote( + [ + "crawl", + "https://example.com", + "--api-url", "http://backend:11235", + "--api-token", "secret", + "--browser", "headless=true", + "--crawler", "scan_full_page=true", + "--output", "markdown", + ], + "http://backend:11235", + "secret", + ) + + assert code == 0 + assert capsys.readouterr().out.strip() == "# From Docker" + request = urlopen.call_args.args[0] + assert request.full_url == "http://backend:11235/crawl" + assert request.headers["Authorization"] == "Bearer secret" + payload = json.loads(request.data) + assert payload["urls"] == ["https://example.com"] + assert payload["browser_config"] == {"headless": True} + assert payload["crawler_config"] == { + "scan_full_page": True, + "cache_mode": "bypass", + } + + +def test_saved_credentials_enable_remote_mode_from_any_directory(tmp_path, monkeypatch): + config_path = tmp_path / ".crawl4ai" / "remote.json" + monkeypatch.setattr(crwl_remote, "CONFIG_PATH", config_path) + assert crwl_remote._configure([ + "--api-url", "http://backend:11235", + "--api-token", "saved-token", + ]) == 0 + + monkeypatch.chdir(tmp_path) + assert crwl_remote._api_setting([], "--api-url", "CRAWL4AI_API_URL") == "http://backend:11235" + assert crwl_remote._api_setting([], "--api-token", "CRAWL4AI_API_TOKEN") == "saved-token" + assert config_path.stat().st_mode & 0o777 == 0o600 + + +def test_environment_overrides_saved_credentials(tmp_path, monkeypatch): + config_path = tmp_path / "remote.json" + config_path.write_text(json.dumps({ + "api_url": "http://saved", + "api_token": "saved-token", + }), encoding="utf-8") + monkeypatch.setattr(crwl_remote, "CONFIG_PATH", config_path) + monkeypatch.setenv("CRAWL4AI_API_URL", "http://environment") + monkeypatch.setenv("CRAWL4AI_API_TOKEN", "environment-token") + + assert crwl_remote._api_setting([], "--api-url", "CRAWL4AI_API_URL") == "http://environment" + assert crwl_remote._api_setting([], "--api-token", "CRAWL4AI_API_TOKEN") == "environment-token" + + +def test_remote_failure_returns_nonzero(capsys): + response = FakeResponse(json.dumps({ + "success": False, + "msg": "not authorized", + }).encode()) + + with patch("crwl_remote.urlopen", return_value=response): + code = crwl_remote._run_remote( + ["https://example.com"], + "http://backend:11235", + None, + ) + + assert code == 1 + assert "not authorized" in capsys.readouterr().err + + +def test_deep_crawl_options_are_serialized_for_remote_api(): + parsed = crwl_remote._remote_args([ + "https://example.com/docs", + "--deep-crawl", "bfs", + "--max-pages", "7", + ]) + + assert parsed["crawler_config"]["deep_crawl_strategy"] == { + "type": "BFSDeepCrawlStrategy", + "params": {"max_depth": 3, "max_pages": 7}, + }