From 152ac0413378a3f616a395ad6a8180f60d238a79 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Thu, 8 Oct 2026 16:51:01 +0200 Subject: [PATCH 01/20] feat(crawl): add ZenRowsCrawlClient for the Crawl API Add a `zenrows.crawl` package with a typed client for the Crawl API: create, get, list_crawls / iter_crawls, iter_results, stop, get_content, iter_download / download, and a wait helper. Responses parse into pydantic v2 models with open enums; errors raise CrawlAPIError (403 REQS008 reads "Crawl is not enabled for this account"). The shared HTTP transport now takes the error mapping and logger as arguments and gains raw and streaming request helpers. Adds unit tests, an opt-in e2e test (`make test-e2e`, deselected from `make test`), a README section, examples/11_crawl.py and DEVELOPMENT.md notes. --- DEVELOPMENT.md | 59 ++++- Makefile | 9 +- README.md | 98 ++++++- examples/11_crawl.py | 70 +++++ examples/README.md | 4 +- pyproject.toml | 6 +- src/zenrows/__init__.py | 8 +- src/zenrows/batch/_transport.py | 83 ++++-- src/zenrows/crawl/__init__.py | 51 ++++ src/zenrows/crawl/client.py | 335 +++++++++++++++++++++++ src/zenrows/crawl/errors.py | 72 +++++ src/zenrows/crawl/models.py | 193 ++++++++++++++ tests/e2e/test_crawl_e2e.py | 109 ++++++++ tests/test_crawl_client.py | 455 ++++++++++++++++++++++++++++++++ 14 files changed, 1519 insertions(+), 33 deletions(-) create mode 100644 examples/11_crawl.py create mode 100644 src/zenrows/crawl/__init__.py create mode 100644 src/zenrows/crawl/client.py create mode 100644 src/zenrows/crawl/errors.py create mode 100644 src/zenrows/crawl/models.py create mode 100644 tests/e2e/test_crawl_e2e.py create mode 100644 tests/test_crawl_client.py diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7c4b30b..4198496 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -19,14 +19,19 @@ Makefile targets below. ``` src/zenrows/ -├── __init__.py # re-exports both clients +├── __init__.py # re-exports the clients ├── client.py # ZenRowsClient (legacy sync scraper) -└── batch/ - ├── __init__.py # ZenRowsBatchClient + key models +├── batch/ +│ ├── __init__.py # ZenRowsBatchClient + key models +│ ├── client.py # hand-written typed facade +│ ├── _transport.py # httpx wrapper, RFC 7807 → exceptions +│ ├── errors.py # BatchAPIError, ProblemDetail +│ └── models.py # GENERATED — pydantic v2 (do not edit) +└── crawl/ + ├── __init__.py # ZenRowsCrawlClient + models ├── client.py # hand-written typed facade - ├── _transport.py # httpx wrapper, RFC 7807 → exceptions - ├── errors.py # BatchAPIError, ProblemDetail - └── models.py # GENERATED — pydantic v2 (do not edit) + ├── errors.py # CrawlAPIError + └── models.py # hand-written pydantic v2 models ``` The Batch SDK is split deliberately: @@ -42,12 +47,21 @@ This way the wire types stay in lockstep with the OpenAPI document while the ergonomic surface (method names, helpers, retries, URL override) stays in our control. +The Crawl client uses the shared HTTP transport (`_transport.py`: key +header, retries, problem+json) with `CrawlAPIError.from_response` as +its error mapping, and the shared `poll_until` loop. Its models are +hand-written: the surface is a few small schemas, so there is no Crawl +codegen step. +Keep them tolerant: unknown fields are ignored, and every response enum +gets `_missing_ = classmethod(open_enum_missing)`. + ## Common tasks | Make target | What it does | |-----------------|--------------| | `make sync` | `uv sync --all-extras` | -| `make test` | `uv run pytest` | +| `make test` | `uv run pytest` (offline; e2e tests deselected) | +| `make test-e2e` | End-to-end tests against a live API (see below) | | `make check` | `ty check` + `ruff check` + `ruff format --check` (CI mode) | | `make typecheck`| `ty check src` (static types; `models.py` excluded) | | `make lint` | `ruff check --fix` | @@ -56,6 +70,37 @@ override) stays in our control. | `make build` | Build wheel + sdist via hatchling | | `make clean` | Drop caches + build outputs | +## Running the Crawl e2e test + +`tests/e2e/test_crawl_e2e.py` drives a real crawl through the client: +create (with `include_patterns`, `output_format="html"`), wait, read the +results, one page and the NDJSON download, list, stop, and a 404. It +costs a few credits. It is marked `e2e`, which `make test` deselects, +and it skips unless the three required variables below are set. + +| Variable | | +|---|---| +| `ZENROWS_API_KEY` | required: a key with Crawl access | +| `ZENROWS_CRAWL_BASE_URL` | required: the API base, e.g. `https://api.zenrows.com/v1` | +| `ZENROWS_E2E_CRAWL_URL` | required: the page the crawl starts from | +| `ZENROWS_E2E_CRAWL_INCLUDE` | optional: an include pattern; every result must contain it | + +```bash +export ZENROWS_API_KEY=zr_... # a key with Crawl access +export ZENROWS_CRAWL_BASE_URL=https://api.zenrows.com/v1 +export ZENROWS_E2E_CRAWL_URL=https://example.com/products/ +export ZENROWS_E2E_CRAWL_INCLUDE=/product/ # optional +make test-e2e +``` + +Pick a start page whose links include a few pages matching the pattern: +the test crawls at depth 1 with `max_items=3` and `output_format="html"`. + +To test against a local or staging deployment, point +`ZENROWS_CRAWL_BASE_URL` at its `/v1` base instead. When the account +has too many crawls running (429 `too_many_crawls`), the test waits and +retries (up to 5 minutes) before creating its crawl. + ## Refreshing the OpenAPI spec `docs/openapi.yaml` is the SDK-local copy of the spec. `make generate` diff --git a/Makefile b/Makefile index 03233d9..ce7c930 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,18 @@ -.PHONY: install sync test lint format typecheck check generate docs clean build bump release +.PHONY: install sync test test-e2e lint format typecheck check generate docs clean build bump release # Bootstrap: install + dev deps, build the local venv. install sync: uv sync --all-extras -# Run the suite. +# Run the suite (offline; e2e tests are deselected). test: uv run pytest +# Run the end-to-end tests against a live API. Needs ZENROWS_API_KEY, +# ZENROWS_CRAWL_BASE_URL and ZENROWS_E2E_CRAWL_URL; see DEVELOPMENT.md. +test-e2e: + uv run pytest -m e2e -s tests/e2e + # Static type check (ty — Astral). Shipped surface only; tests are # covered by the suite. Generated models.py is excluded in pyproject. typecheck: diff --git a/README.md b/README.md index bf15f52..8133127 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ SDK to access [Zenrows](https://www.zenrows.com/) APIs directly from Python. Zenrows handles proxies rotation, headless browsers, and CAPTCHAs for you. -This package ships two clients: +This package ships three clients: - **`ZenRowsClient`** — the original synchronous scraping client. One URL in, one HTML/JSON response out. Best for ad-hoc scraping. @@ -18,6 +18,9 @@ This package ships two clients: thousands of URLs as one job, poll for results, optionally upload a CSV of URLs in one call. Best for offline / bulk pipelines. _(Private beta — [contact support](mailto:support@zenrows.com) for access.)_ + - **`ZenRowsCrawlClient`** — the Crawl API client. Give it one start + URL; it follows links and returns the URLs (and, optionally, the + pages) it finds behind it. ## Table of Contents @@ -37,6 +40,10 @@ This package ships two clients: - [Act on an id without a GET](#act-on-an-id-without-a-get) - [Scheduled jobs & webhooks](#scheduled-jobs--webhooks) - [Error handling](#error-handling) +- [Quickstart — Crawl API (`ZenRowsCrawlClient`)](#quickstart--crawl-api-zenrowscrawlclient) + - [Read the pages](#read-the-pages) + - [List and stop crawls](#list-and-stop-crawls) + - [Crawl errors](#crawl-errors) - [Contributing](#contributing) - [License](#license) @@ -485,6 +492,95 @@ The full Batch surface (jobs, runs, tasks, results, content, history, file_inputs, HMAC keys) is reachable via methods on `ZenRowsBatchClient`. See `src/zenrows/batch/client.py` or `help(ZenRowsBatchClient)`. +## Quickstart — Crawl API (`ZenRowsCrawlClient`) + +A crawl starts from one URL, follows the links on each page up to +`depth` hops, and keeps the URLs that match your patterns, up to +`max_items`. It runs in the background: `create` returns at once with +`status="running"`, and `wait` blocks until it ends. Without +`output_format` a crawl returns URLs only; `output_format="html"` also +returns each kept URL's page. + +```python +import os +from zenrows import ZenRowsCrawlClient + +client = ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"]) + +crawl = client.create( + "https://example.com/products/", + depth=1, # link hops from the start URL + max_items=50, # stop after keeping 50 URLs (default 10) + max_pages=100, # stop after fetching 100 pages (default 10) + include_patterns=["/product/"], # keep only URLs containing this +) +crawl = client.wait(crawl.crawl_id, timeout=600) # Crawl, terminal +print(crawl.status.value, crawl.stop_reason, crawl.coverage) + +for result in client.iter_results(crawl.crawl_id): # follows next_cursor + print(result.url) +``` + +`wait` returns a `failed` crawl rather than raising (read `crawl.error`), +and raises `WaiterTimeout` on timeout without stopping the crawl. +`iter_results` stops when the server's `next_cursor` is null, which happens +only once the crawl has ended; on a running crawl it follows new results +live until the crawl ends. `get(crawl_id, cursor=..., limit=...)` returns +one raw page. + +### Read the pages + +Pass `output_format="html"` to also fetch the page of every kept URL: + +```python +crawl = client.create(start_url, depth=1, output_format="html") +client.wait(crawl.crawl_id) + +for result in client.iter_results(crawl.crawl_id): + if result.content_id: # content_status == fetched + html = client.get_content(crawl.crawl_id, result) + +# Or everything in one NDJSON file: {"url", "content_status", "content"} per line. +client.download(crawl.crawl_id, "out/crawl.jsonl") +for line in client.iter_download(crawl.crawl_id): # streamed, parsed + print(line.url, len(line.content or "")) +``` + +### List and stop crawls + +```python +for c in client.iter_crawls(): # newest first, without results + print(c.crawl_id, c.status.value) + +client.stop(crawl_id) # idempotent; read final counts with get() +``` + +### Crawl errors + +Every non-2xx raises `CrawlAPIError`. Branch on `status_code` and `code`: + +```python +from zenrows.crawl import CrawlAPIError + +try: + client.create(start_url, depth=1) +except CrawlAPIError as exc: + if exc.not_enabled: # 403 REQS008 + print("Crawl is not enabled for this account") + elif exc.code == "too_many_crawls": # 429: too many crawls running + print("retry in", exc.retry_after, "s") + elif exc.code in ("invalid_parameter", "invalid_start_url"): # 422 + print(exc.detail) + else: + raise +``` + +`crawl_not_found` / `content_not_found` come with 404. A crawl that +ended `failed` is not an error response: read `crawl.error.code` and +`crawl.error.detail` on it. Response enums (`CrawlStatus`, `StopReason`, +`ContentStatus`, ...) are open: a value added later parses as an +`UNKNOWN` member that keeps the raw value. + ## Contributing Pull requests are welcome. For significant changes, please open an issue first to discuss what you would like to change. diff --git a/examples/11_crawl.py b/examples/11_crawl.py new file mode 100644 index 0000000..1d3246e --- /dev/null +++ b/examples/11_crawl.py @@ -0,0 +1,70 @@ +"""11: Crawl a site → wait → read URLs and pages. + +Demonstrates the Crawl client (`ZenRowsCrawlClient`): + - `create(url, depth=..., include_patterns=..., output_format="html")` + starts a crawl and returns at once, `running`. + - `wait(crawl_id)` blocks until it ends and returns the final `Crawl` + (a `failed` crawl is returned, with `error` set). + - `iter_results(crawl_id)` pages through every kept URL. + - `get_content(crawl_id, result)` reads one kept URL's HTML. + - `download(crawl_id, path)` saves every result as NDJSON. + +Run with: + export ZENROWS_API_KEY=zr_... + python examples/11_crawl.py --url https://example.com/products/ +""" + +import argparse +import os + +from zenrows import ZenRowsCrawlClient +from zenrows.crawl import CrawlAPIError, CrawlStatus + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--url", required=True, help="the page to start from") + parser.add_argument("--depth", type=int, default=1) + parser.add_argument("--max-items", type=int, default=5) + parser.add_argument("--include", action="append", default=["/product/"]) + parser.add_argument("--out", default="crawl.jsonl") + args = parser.parse_args() + + client = ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"]) + + try: + crawl = client.create( + args.url, + depth=args.depth, + max_items=args.max_items, + include_patterns=args.include, + output_format="html", + ) + except CrawlAPIError as exc: + if exc.not_enabled: + raise SystemExit("Crawl is not enabled for this account.") from exc + raise + print(f"started {crawl.crawl_id}") + + crawl = client.wait(crawl.crawl_id, timeout=600.0) + cov = crawl.coverage + print( + f"{crawl.crawl_id} {crawl.status.value}: {cov.items_found} URLs kept, " + f"{cov.pages_fetched} pages fetched ({cov.pages_failed} failed)" + ) + if crawl.status is CrawlStatus.FAILED and crawl.error: + raise SystemExit(f"crawl failed: {crawl.error.code.value}: {crawl.error.detail}") + + for result in client.iter_results(crawl.crawl_id): + status = result.content_status.value if result.content_status else "-" + print(f" [{status}] {result.url}") + if result.content_id: + html = client.get_content(crawl.crawl_id, result) + print(f" {len(html)} chars of HTML") + + path = client.download(crawl.crawl_id, args.out) + print(f"wrote {path}") + + +if __name__ == "__main__": + main() diff --git a/examples/README.md b/examples/README.md index ca93890..7f102d6 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,7 @@ # Examples -Runnable samples for the Batch API client (`ZenRowsBatchClient`). +Runnable samples for the Batch API client (`ZenRowsBatchClient`) and +the Crawl API client (`ZenRowsCrawlClient`, `11_crawl.py`). Each script reads the API key from the environment: ```bash @@ -27,6 +28,7 @@ uv run python examples/02_download_to_dir.py --job-id 01J... | `08_download_all_results.py` | `download_all_results()` — server-side export zip of a whole run | | `09_scheduled_jobs.py` | scheduled cadences (`Rate`/`Calendar`/`At`), `pause`/`resume`/`update_schedule`, per-fire runs, `retry_failed` | | `10_lightweight_handles.py` | `client.job(id)` / `client.run(id, rid)` — act on a known id without a GET (`.data` lazily fetches) | +| `11_crawl.py` | Crawl: `create` → `wait` → `iter_results` + `get_content` → `download` (NDJSON) | The samples are deliberately small — each demonstrates one feature end-to-end so they double as living documentation. diff --git a/pyproject.toml b/pyproject.toml index f697fb2..fbad15b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,5 +112,9 @@ exclude = ["src/zenrows/batch/models.py"] # --- pytest --- [tool.pytest.ini_options] testpaths = ["tests"] -addopts = "-ra --strict-markers" +# e2e tests hit a live API; `make test-e2e` selects them with `-m e2e`. +addopts = "-ra --strict-markers -m 'not e2e'" asyncio_mode = "auto" +markers = [ + "e2e: end-to-end tests against a live ZenRows API (opt-in, see DEVELOPMENT.md)", +] diff --git a/src/zenrows/__init__.py b/src/zenrows/__init__.py index 9bbf8c1..4da0a61 100644 --- a/src/zenrows/__init__.py +++ b/src/zenrows/__init__.py @@ -1,6 +1,6 @@ """Zenrows Python SDK. -Two clients live here: +Three clients live here: - `ZenRowsClient` — the original synchronous scraping API client. Backward-compatible with pre-1.4 usage; freshened-up internals. @@ -8,10 +8,14 @@ - `ZenRowsBatchClient` — the new async-job / batch API client. Built on a generated OpenAPI core with a hand-written ergonomic facade on top. + + - `ZenRowsCrawlClient` — the Crawl API client: start a crawl from + one URL and read back the URLs (and pages) it finds. """ from zenrows.__version__ import __version__ from zenrows.batch import ZenRowsBatchClient from zenrows.client import ZenRowsClient +from zenrows.crawl import ZenRowsCrawlClient -__all__ = ["ZenRowsBatchClient", "ZenRowsClient", "__version__"] +__all__ = ["ZenRowsBatchClient", "ZenRowsClient", "ZenRowsCrawlClient", "__version__"] diff --git a/src/zenrows/batch/_transport.py b/src/zenrows/batch/_transport.py index 20e45c5..f6c95bf 100644 --- a/src/zenrows/batch/_transport.py +++ b/src/zenrows/batch/_transport.py @@ -4,10 +4,18 @@ User-Agent, automatic retries for transient failures, and the RFC 7807 → `BatchAPIError` mapping. The facade (`client.py`) is thin glue over this — one method per endpoint, all typed via pydantic v2 models. + +The Crawl client (`zenrows.crawl`) reuses it with its own error type +and logger: both APIs share the key header, problem+json errors and +retry policy. """ +import json +import logging import random import time +from collections.abc import Callable, Iterator +from contextlib import contextmanager from typing import Any, TypeVar import httpx @@ -78,8 +86,15 @@ def __init__( retries: int = _DEFAULT_RETRIES, verify: bool | str = True, httpx_args: dict[str, Any] | None = None, + error_from_response: Callable[[httpx.Response], Exception] = BatchAPIError.from_response, + log: logging.Logger = _log, ): + """`error_from_response` turns a non-2xx response into the + exception to raise; it should expose `.code` and `.detail` for + the error log. `log` receives the per-request records.""" self._retries = max(0, retries) + self._error_from_response = error_from_response + self._log = log self._client = httpx.Client( base_url=base_url.rstrip("/"), headers={ @@ -185,11 +200,25 @@ def request_json( payload = body.model_dump( mode="json", exclude_unset=True, exclude_none=True, by_alias=True ) - import json - content = json.dumps(payload).encode("utf-8") headers = {**(headers or {}), "Content-Type": "application/json"} + response = self.request(method, path, params=params, headers=headers, content=content) + if response.status_code == 204 or not response.content: + return None + return response.json() + + def request( + self, + method: str, + path: str, + *, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + content: bytes | None = None, + ) -> httpx.Response: + """Send a request (with retries) and return the raw response, + raising the mapped error on non-2xx. For non-JSON bodies.""" start = time.monotonic() response = self._send( method, @@ -199,25 +228,41 @@ def request_json( content=content, ) elapsed_ms = (time.monotonic() - start) * 1000 - - if response.status_code >= 400: - err = BatchAPIError.from_response(response) - log_error( - _log, - method=method, - path=path, - status=response.status_code, - code=err.code, - detail=err.problem.detail if err.problem else None, - ) - raise err - + self._raise_for_status(method, path, response) log_request( - _log, method=method, path=path, status=response.status_code, elapsed_ms=elapsed_ms + self._log, method=method, path=path, status=response.status_code, elapsed_ms=elapsed_ms ) - if response.status_code == 204 or not response.content: - return None - return response.json() + return response + + @contextmanager + def stream( + self, + method: str, + path: str, + *, + params: dict[str, Any] | None = None, + ) -> Iterator[httpx.Response]: + """Open a streamed response, raising the mapped error on + non-2xx. Not retried: a stream may be partly consumed.""" + with self._client.stream(method, path, params=_drop_none(params)) as response: + if response.status_code >= 400: + response.read() + self._raise_for_status(method, path, response) + yield response + + def _raise_for_status(self, method: str, path: str, response: httpx.Response) -> None: + if response.status_code < 400: + return + err = self._error_from_response(response) + log_error( + self._log, + method=method, + path=path, + status=response.status_code, + code=getattr(err, "code", "internal"), + detail=getattr(err, "detail", None), + ) + raise err def _drop_none(d: dict[str, Any] | None) -> dict[str, Any] | None: diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py new file mode 100644 index 0000000..2f2021e --- /dev/null +++ b/src/zenrows/crawl/__init__.py @@ -0,0 +1,51 @@ +"""Public surface for the ZenRows Crawl API. + +What's where: + - `client.ZenRowsCrawlClient` — the typed facade: create, get, list, + stop, contents, download, and a `wait` helper. + - `models` — hand-written pydantic v2 response models with open enums. + - `errors.CrawlAPIError` — RFC 9457 problem+json mapping. + +`create(output_format="html")` also returns each kept URL's page; +without it a crawl returns URLs only. +""" + +from zenrows.batch._waiters import WaiterTimeout +from zenrows.crawl.client import DEFAULT_BASE_URL, ZenRowsCrawlClient +from zenrows.crawl.errors import CRAWL_NOT_ENABLED, CrawlAPIError +from zenrows.crawl.models import ( + ContentStatus, + Coverage, + Crawl, + CrawlErrorCode, + CrawlList, + CrawlResult, + CrawlStatus, + CrawlStop, + CrawlWithResults, + DownloadLine, + OutputFormat, + RunError, + StopReason, +) + +__all__ = [ + "CRAWL_NOT_ENABLED", + "DEFAULT_BASE_URL", + "ContentStatus", + "Coverage", + "Crawl", + "CrawlAPIError", + "CrawlErrorCode", + "CrawlList", + "CrawlResult", + "CrawlStatus", + "CrawlStop", + "CrawlWithResults", + "DownloadLine", + "OutputFormat", + "RunError", + "StopReason", + "WaiterTimeout", + "ZenRowsCrawlClient", +] diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py new file mode 100644 index 0000000..30d6549 --- /dev/null +++ b/src/zenrows/crawl/client.py @@ -0,0 +1,335 @@ +"""ZenRowsCrawlClient — the typed facade over the Crawl API. + +A crawl starts from one URL, follows links up to `depth` hops, and +keeps the URLs that match its patterns, up to `max_items`. It runs as +a job: `create` returns at once with `status=running`, and the URLs +grow as pages finish. + + client = ZenRowsCrawlClient(api_key) + crawl = client.create("https://example.com/shop/", depth=1, + include_patterns=["/product/"]) + client.wait(crawl.crawl_id) + for result in client.iter_results(crawl.crawl_id): + print(result.url) + +Requests go through the SDK's shared HTTP transport (auth header, +retries, problem+json mapping), which raises `CrawlAPIError`. +""" + +import json +import logging +import os +import time +from collections.abc import Iterator +from pathlib import Path +from typing import Any, Literal + +import httpx + +from zenrows.__version__ import __version__ +from zenrows.batch._transport import _Transport +from zenrows.batch._waiters import poll_until +from zenrows.crawl.errors import CrawlAPIError +from zenrows.crawl.models import ( + Crawl, + CrawlList, + CrawlResult, + CrawlStatus, + CrawlStop, + CrawlWithResults, + DownloadLine, +) + +# Production endpoint. The `base_url=` kwarg + `ZENROWS_CRAWL_BASE_URL` +# env var are present for advanced use only. +DEFAULT_BASE_URL = "https://api.zenrows.com/v1" +DEFAULT_USER_AGENT = f"zenrows-crawl-python/{__version__}" + +_log = logging.getLogger("zenrows.crawl.transport") + + +class ZenRowsCrawlClient: + """Synchronous, typed client for the ZenRows Crawl API. + + `api_key` is required: + + client = ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"]) + + Every non-2xx raises `CrawlAPIError`; branch on its `status_code` + and `code`. An account without Crawl enabled gets 403 `REQS008` + (`CrawlAPIError.not_enabled`). + """ + + def __init__( + self, + api_key: str, + *, + base_url: str | None = None, + timeout: float | httpx.Timeout = 30.0, + retries: int = 3, + verify_ssl: bool | str = True, + user_agent: str = DEFAULT_USER_AGENT, + httpx_args: dict[str, Any] | None = None, + ): + """`retries` bounds automatic retries of transient failures + (HTTP 429/502/503/504 and network errors) on idempotent + requests — GETs, and a `create` that carries an + `idempotency_key`. Retries use jittered exponential backoff and + honor `Retry-After`; set `retries=0` to disable.""" + if not api_key: + raise ValueError("ZenRowsCrawlClient: api_key is required.") + base_url = base_url or os.environ.get("ZENROWS_CRAWL_BASE_URL") or DEFAULT_BASE_URL + self._t = _Transport( + base_url=base_url, + api_key=api_key, + user_agent=user_agent, + timeout=timeout, + retries=retries, + verify=verify_ssl, + httpx_args=httpx_args, + error_from_response=CrawlAPIError.from_response, + log=_log, + ) + + # ----- lifecycle ----- + + def __enter__(self) -> "ZenRowsCrawlClient": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def close(self) -> None: + self._t.close() + + @property + def base_url(self) -> str: + return self._t.base_url + + # ===== crawls ===== + + def create( + self, + url: str, + *, + depth: int, + max_items: int | None = None, + max_pages: int | None = None, + include_patterns: list[str] | None = None, + exclude_patterns: list[str] | None = None, + output_format: Literal["html"] | None = None, + idempotency_key: str | None = None, + ) -> Crawl: + """`POST /crawls` — start a crawl and return it, `running`. + + - `depth`: link hops to follow from `url` (1 to 100,000). + - `max_items`: stop after keeping this many URLs (server + default 10). + - `max_pages`: stop after fetching this many pages, which + bounds the cost (server default 10). + - `include_patterns` / `exclude_patterns`: substrings matched + against each normalized URL. A URL is kept when it matches + an include pattern (if any) and no exclude pattern. + - `output_format="html"`: also fetch every kept URL's page, + readable with `get_content` / `download`. Omit for URLs only. + - `idempotency_key`: a retry with the same key and body answers + with the crawl the first request created. + + Only the arguments you pass are sent. An account with too many + crawls running gets 429 `too_many_crawls`; retry after the + error's `retry_after` seconds. + """ + if output_format not in (None, "html"): + raise ValueError(f"create: output_format must be 'html' or None, not {output_format!r}") + body: dict[str, Any] = {"url": url, "depth": depth} + optional: dict[str, Any] = { + "max_items": max_items, + "max_pages": max_pages, + "include_patterns": include_patterns, + "exclude_patterns": exclude_patterns, + "output_format": output_format, + } + body.update({k: v for k, v in optional.items() if v is not None}) + headers = {"Content-Type": "application/json"} + if idempotency_key: + headers["Idempotency-Key"] = idempotency_key + response = self._t.request( + "POST", "/crawls", headers=headers, content=json.dumps(body).encode("utf-8") + ) + return Crawl.model_validate(response.json()) + + def get( + self, + crawl_id: str, + *, + cursor: str | None = None, + limit: int | None = None, + ) -> CrawlWithResults: + """`GET /crawls/{crawl_id}` — the crawl and one page of its + results (`limit` 1 to 10,000, server default 1,000). Pass the + page's `next_cursor` as `cursor` for the URLs kept since. For + every result, prefer `iter_results`.""" + return CrawlWithResults.model_validate( + self._t.request_json( + "GET", f"/crawls/{crawl_id}", params={"cursor": cursor, "limit": limit} + ) + ) + + def list_crawls( + self, + *, + cursor: str | None = None, + limit: int | None = None, + ) -> CrawlList: + """`GET /crawls` — one page of the account's crawls, newest + first (`limit` 1 to 100, server default 20), without results. For + most uses prefer the auto-paginating `iter_crawls`.""" + return CrawlList.model_validate( + self._t.request_json("GET", "/crawls", params={"cursor": cursor, "limit": limit}) + ) + + def iter_crawls(self, *, page_size: int | None = None) -> Iterator[Crawl]: + """Auto-paginate `list_crawls`, newest first.""" + cursor: str | None = None + while True: + page = self.list_crawls(cursor=cursor, limit=page_size) + yield from page.crawls + cursor = page.next_cursor + if not cursor: + return + + def iter_results( + self, + crawl_id: str, + *, + page_size: int | None = None, + poll_interval: float = 2.0, + ) -> Iterator[CrawlResult]: + """Yield every URL the crawl keeps, in order, each once. + + Follows `next_cursor` until the server returns null, which it + does only once the crawl has ended and its last page was read. + On a crawl that is still running, the iterator follows it live: + when it has caught up, it sleeps `poll_interval` seconds and + asks again with the same cursor, so it ends when the crawl ends. + Call `wait` first to read a finished crawl without polling. + """ + cursor: str | None = None + while True: + page = self.get(crawl_id, cursor=cursor, limit=page_size) + yield from page.results + if page.next_cursor is None: + return + if not page.results and page.status is CrawlStatus.RUNNING: + time.sleep(poll_interval) + cursor = page.next_cursor + + def stop(self, crawl_id: str) -> CrawlStop: + """`POST /crawls/{crawl_id}/stop` — stop a running crawl. + + Idempotent: a crawl that already ended answers as it ended. + The answer carries no counts; read the final coverage and + results with `get` once pages already in flight have finished. + """ + return CrawlStop.model_validate(self._t.request_json("POST", f"/crawls/{crawl_id}/stop")) + + # ===== contents ===== + + def get_content(self, crawl_id: str, content: str | CrawlResult) -> str: + """`GET /crawls/{crawl_id}/contents/{content_id}` — one kept + URL's page, as HTML text. + + `content` is a `CrawlResult` whose `content_status` is + `fetched`, or its content id (the last segment of its + `content_url`). Only crawls created with an `output_format` + have contents; anything else is 404 `content_not_found`. + """ + if isinstance(content, CrawlResult): + content_id = content.content_id + if not content_id: + raise ValueError( + f"get_content: {content.url} has no content_url " + f"(content_status={content.content_status})" + ) + else: + content_id = content + return self._t.request("GET", f"/crawls/{crawl_id}/contents/{content_id}").text + + def iter_download(self, crawl_id: str) -> Iterator[DownloadLine]: + """`GET /crawls/{crawl_id}/download` — stream every result as + a parsed `DownloadLine` (url, content_status, content). + + On a running crawl the file holds what the crawl has kept so + far (a warning is logged); call `wait` first for the whole of it. + """ + with self._t.stream("GET", f"/crawls/{crawl_id}/download") as response: + _warn_if_partial(crawl_id, response) + for line in response.iter_lines(): + if line.strip(): + yield DownloadLine.model_validate_json(line) + + def download( + self, + crawl_id: str, + target_path: str | Path, + *, + chunk_size: int = 64 * 1024, + ) -> Path: + """`GET /crawls/{crawl_id}/download` — stream the NDJSON file + (one JSON object per result) to `target_path`, and return it. + + On a running crawl the file holds what the crawl has kept so + far (a warning is logged); call `wait` first for the whole of it. + """ + target = Path(target_path) + target.parent.mkdir(parents=True, exist_ok=True) + with self._t.stream("GET", f"/crawls/{crawl_id}/download") as response: + _warn_if_partial(crawl_id, response) + with target.open("wb") as f: + for chunk in response.iter_bytes(chunk_size): + f.write(chunk) + return target + + # ===== waiter ===== + + def wait( + self, + crawl_id: str, + *, + timeout: float = 300.0, + poll_interval: float = 2.0, + max_poll_interval: float = 15.0, + ) -> Crawl: + """Block until the crawl ends (`completed`, `stopped` or + `failed`) and return it, polling with jittered exponential + backoff from `poll_interval` up to `max_poll_interval` seconds. + + A `failed` crawl is returned, not raised: read `error` on it. + Raises `WaiterTimeout` after `timeout` seconds; the crawl keeps + running (call `stop` to end it). + """ + + def fetch() -> Crawl: + # limit=1 keeps each poll cheap; results are not returned. + page = self.get(crawl_id, limit=1) + return Crawl.model_validate(page.model_dump(exclude={"results", "next_cursor"})) + + return poll_until( + fetch=fetch, + is_done=lambda crawl: crawl.is_terminal, + timeout=timeout, + initial_interval=poll_interval, + max_interval=max_poll_interval, + ) + + +def _warn_if_partial(crawl_id: str, response: httpx.Response) -> None: + status = response.headers.get("X-Crawl-Status") + if status == CrawlStatus.RUNNING.value: + logging.getLogger("zenrows.crawl.client").warning( + "crawl %s is still running: the download holds only the results kept so far", + crawl_id, + ) + + +__all__ = ["DEFAULT_BASE_URL", "DEFAULT_USER_AGENT", "ZenRowsCrawlClient"] diff --git a/src/zenrows/crawl/errors.py b/src/zenrows/crawl/errors.py new file mode 100644 index 0000000..b1f5b9d --- /dev/null +++ b/src/zenrows/crawl/errors.py @@ -0,0 +1,72 @@ +"""RFC 9457 problem+json → `CrawlAPIError`. + +The Crawl API answers errors as `application/problem+json` with a +stable `code`. Branch on +`status_code` + `code`; display `detail`, don't parse it. +""" + +import httpx + +from zenrows.batch.errors import ProblemDetail + +# The code of the 403 answer to a key whose account does not have +# Crawl enabled. +CRAWL_NOT_ENABLED = "REQS008" + + +class CrawlAPIError(Exception): + """A non-2xx response from the Crawl API. + + `code` is the problem's `code` member: `crawl_not_found`, + `content_not_found`, `invalid_parameter`, `too_many_crawls`, ..., + `REQS008` when Crawl is not enabled for the account, and the API's + usual auth and credit codes. `problem` + holds the whole body, or None when it was not JSON. + """ + + def __init__( + self, + status_code: int, + problem: ProblemDetail | None, + raw: bytes, + *, + retry_after: float | None = None, + ): + self.status_code = status_code + # Seconds the server asked to wait (`Retry-After`), e.g. on 429 + # `too_many_crawls`. + self.retry_after = retry_after + self.problem = problem + self.raw = raw + self.code: str = problem.code if problem else "internal" + self.detail: str | None = problem.detail if problem else None + if problem and self.code == CRAWL_NOT_ENABLED: + msg = f"{status_code} Crawl is not enabled for this account ({CRAWL_NOT_ENABLED})" + ( + f": {problem.detail}" if problem.detail else "" + ) + elif problem: + msg = f"{status_code} {problem.title}: {problem.detail or problem.code}" + else: + msg = f"{status_code} (no problem body)" + super().__init__(msg) + + @property + def not_enabled(self) -> bool: + """True when the account does not have Crawl enabled (403 REQS008).""" + return self.status_code == 403 and self.code == CRAWL_NOT_ENABLED + + @classmethod + def from_response(cls, response: httpx.Response) -> "CrawlAPIError": + try: + retry_after: float | None = float(response.headers.get("Retry-After", "")) + except ValueError: + retry_after = None + return cls( + status_code=response.status_code, + problem=ProblemDetail.from_response(response), + raw=response.content, + retry_after=retry_after, + ) + + +__all__ = ["CRAWL_NOT_ENABLED", "CrawlAPIError"] diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py new file mode 100644 index 0000000..e3249e0 --- /dev/null +++ b/src/zenrows/crawl/models.py @@ -0,0 +1,193 @@ +"""Pydantic v2 models for the Crawl API's responses. + +Hand-written: the surface is a handful of small schemas. + +Forward compatible: unknown response fields +are ignored (pydantic's default), and every enum is open: a value the +server adds later parses as an `UNKNOWN` member that keeps the raw +value (see `zenrows.batch._open_enum`). +""" + +from datetime import datetime +from enum import Enum + +from pydantic import BaseModel + +from zenrows.batch._open_enum import open_enum_missing + + +class CrawlStatus(Enum): + """Where a crawl stands. `running` until it ends; the other values + are terminal. Clients must accept values not listed here.""" + + RUNNING = "running" + COMPLETED = "completed" + STOPPED = "stopped" + FAILED = "failed" + + _missing_ = classmethod(open_enum_missing) + + +class StopReason(Enum): + """What ended the crawl before nothing was left to open: a limit + (`max_items` / `max_pages`, with `completed`) or the caller (`user`, + with `stopped`).""" + + MAX_ITEMS = "max_items" + MAX_PAGES = "max_pages" + USER = "user" + + _missing_ = classmethod(open_enum_missing) + + +class ContentStatus(Enum): + """Where a kept URL's page stands, when the crawl has an + `output_format`.""" + + PENDING = "pending" + FETCHED = "fetched" + FAILED = "failed" + + _missing_ = classmethod(open_enum_missing) + + +class OutputFormat(Enum): + """The format the crawl returns each kept URL's page in.""" + + HTML = "html" + + _missing_ = classmethod(open_enum_missing) + + +class CrawlErrorCode(Enum): + """Why a crawl failed (`Crawl.error.code`).""" + + INSUFFICIENT_CREDITS = "insufficient_credits" + SEED_UNREACHABLE = "seed_unreachable" + DOMAIN_NOT_ALLOWED = "domain_not_allowed" + NO_ITEMS_FOUND = "no_items_found" + INTERNAL_ERROR = "internal_error" + + _missing_ = classmethod(open_enum_missing) + + +class RunError(BaseModel): + """Why a crawl failed. Present only when `status` is `failed`; part + of the crawl, not an error response.""" + + code: CrawlErrorCode + detail: str + + +class Coverage(BaseModel): + """How far the crawl has got.""" + + pages_fetched: int + """Pages fetched so far, successful or not.""" + pages_failed: int + """Fetched pages whose fetch failed (not charged).""" + items_found: int + """URLs kept so far, bounded by `max_items`.""" + + +class Crawl(BaseModel): + """One crawl, with the parameters it runs with.""" + + crawl_id: str + status: CrawlStatus + url: str + """The page the crawl started from.""" + depth: int + max_items: int + max_pages: int + coverage: Coverage + created_at: datetime + stop_reason: StopReason | None = None + error: RunError | None = None + include_patterns: list[str] | None = None + exclude_patterns: list[str] | None = None + output_format: OutputFormat | None = None + duplicates_removed: int | None = None + """Links met again and dropped as already seen.""" + finished_at: datetime | None = None + """When the crawl reached a terminal status. Absent while it runs.""" + + @property + def is_terminal(self) -> bool: + """True once the crawl has ended (any status but `running`).""" + return self.status is not CrawlStatus.RUNNING + + +class CrawlResult(BaseModel): + """One URL the crawl kept.""" + + url: str + content_status: ContentStatus | None = None + """Present only when the crawl has an `output_format`.""" + content_url: str | None = None + """The page's path (e.g. `/v1/crawls/c_x/contents/ct_y`), present + once `content_status` is `fetched`. Pass the result itself to + `ZenRowsCrawlClient.get_content`.""" + + @property + def content_id(self) -> str | None: + """The content id, the last segment of `content_url`.""" + if not self.content_url: + return None + return self.content_url.rstrip("/").rsplit("/", 1)[-1] + + +class CrawlWithResults(Crawl): + """A crawl and one page of the URLs it has kept (`GET /crawls/{id}`). + + `next_cursor` is never null while the crawl runs: polling with it + returns only URLs kept since. It is null once the crawl has ended + and this page holds its last URLs.""" + + results: list[CrawlResult] + next_cursor: str | None = None + + +class CrawlList(BaseModel): + """One page of the account's crawls, newest first. `next_cursor` is + absent on the last page.""" + + crawls: list[Crawl] + next_cursor: str | None = None + + +class CrawlStop(BaseModel): + """What `stop` answers: where the crawl stands, without counts. Read + the coverage and results with `get`.""" + + crawl_id: str + status: CrawlStatus + stop_reason: StopReason | None = None + error: RunError | None = None + finished_at: datetime | None = None + + +class DownloadLine(BaseModel): + """One line of a crawl's NDJSON download.""" + + url: str + content_status: ContentStatus | None = None + content: str | None = None + """The page HTML, present when `content_status` is `fetched`.""" + + +__all__ = [ + "ContentStatus", + "Coverage", + "Crawl", + "CrawlErrorCode", + "CrawlList", + "CrawlResult", + "CrawlStatus", + "CrawlStop", + "CrawlWithResults", + "DownloadLine", + "OutputFormat", + "RunError", + "StopReason", +] diff --git a/tests/e2e/test_crawl_e2e.py b/tests/e2e/test_crawl_e2e.py new file mode 100644 index 0000000..d37db0d --- /dev/null +++ b/tests/e2e/test_crawl_e2e.py @@ -0,0 +1,109 @@ +"""End-to-end test of ZenRowsCrawlClient against a live Crawl API. + +Opt-in: runs only with `make test-e2e` (pytest `-m e2e`), and only when +`ZENROWS_API_KEY` (a key with Crawl access), `ZENROWS_CRAWL_BASE_URL` +(the API base, e.g. https://api.zenrows.com/v1) and +`ZENROWS_E2E_CRAWL_URL` (the page to crawl from) are set. Otherwise it is +skipped, so `make test` stays offline. When `ZENROWS_E2E_CRAWL_INCLUDE` +is set, the crawl keeps only URLs containing it, and the test checks +that every result does. It starts one small crawl (a few credits) and +reads it back every way the client can. +""" + +import json +import os +import time + +import pytest + +from zenrows.crawl import CrawlAPIError, CrawlStatus, ZenRowsCrawlClient + +_REQUIRED = ("ZENROWS_API_KEY", "ZENROWS_CRAWL_BASE_URL", "ZENROWS_E2E_CRAWL_URL") + +pytestmark = [ + pytest.mark.e2e, + pytest.mark.skipif( + not all(os.environ.get(v) for v in _REQUIRED), + reason="set " + ", ".join(_REQUIRED) + " to run the Crawl e2e test", + ), +] + +START_URL = os.environ.get("ZENROWS_E2E_CRAWL_URL", "") +INCLUDE = os.environ.get("ZENROWS_E2E_CRAWL_INCLUDE") or None +# An account can run only a few crawls at once, so a create can meet +# 429 too_many_crawls while other runs finish. +SLOT_WAIT_SECONDS = 300 + + +@pytest.fixture(scope="module") +def client(): + with ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"]) as c: + yield c + + +def create_when_a_slot_frees(client: ZenRowsCrawlClient): + deadline = time.monotonic() + SLOT_WAIT_SECONDS + while True: + try: + return client.create( + START_URL, + depth=1, + max_items=3, + max_pages=5, + include_patterns=[INCLUDE] if INCLUDE else None, + output_format="html", + ) + except CrawlAPIError as err: + if err.code != "too_many_crawls" or time.monotonic() >= deadline: + raise + wait = err.retry_after or 30.0 + print(f"too_many_crawls: retrying create in {wait:.0f}s") + time.sleep(wait) + + +def test_crawl_end_to_end(client: ZenRowsCrawlClient, tmp_path): + created = create_when_a_slot_frees(client) + print(f"created {created.crawl_id} status={created.status.value}") + assert created.crawl_id + + crawl = client.wait(created.crawl_id, timeout=600) + print( + f"finished {crawl.crawl_id} status={crawl.status.value} " + f"stop_reason={crawl.stop_reason and crawl.stop_reason.value} " + f"pages_fetched={crawl.coverage.pages_fetched} items_found={crawl.coverage.items_found}" + ) + assert crawl.status is CrawlStatus.COMPLETED, crawl.error + + results = list(client.iter_results(crawl.crawl_id)) + print(f"results: {len(results)}") + assert results + if INCLUDE: + assert all(INCLUDE in r.url for r in results), [r.url for r in results] + + fetched = [r for r in results if r.content_id] + assert fetched, "no result has a fetched page" + html = client.get_content(crawl.crawl_id, fetched[0]) + print(f"content of {fetched[0].url}: {len(html)} chars") + assert " ZenRowsCrawlClient: + return ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=0) + + +@pytest.fixture +def no_sleep(monkeypatch): + """Record + swallow sleeps (waiter, scanner and retry backoff all + call `time.sleep`) so tests run instantly.""" + slept: list[float] = [] + monkeypatch.setattr("zenrows.crawl.client.time.sleep", slept.append) + return slept + + +# ----- construction ----- + + +def test_missing_api_key_raises(): + with pytest.raises(ValueError): + ZenRowsCrawlClient(api_key="") + + +def test_default_base_url(monkeypatch): + monkeypatch.delenv("ZENROWS_CRAWL_BASE_URL", raising=False) + assert ZenRowsCrawlClient(api_key=API_KEY).base_url.rstrip("/") == "https://api.zenrows.com/v1" + + +def test_base_url_env_and_kwarg_precedence(monkeypatch): + monkeypatch.setenv("ZENROWS_CRAWL_BASE_URL", "http://env-set/v1") + assert ZenRowsCrawlClient(api_key=API_KEY).base_url.rstrip("/") == "http://env-set/v1" + assert ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL).base_url.rstrip("/") == BASE_URL + + +# ----- create ----- + + +@respx.mock +def test_create_sends_only_required_fields(client: ZenRowsCrawlClient): + route = respx.post(f"{BASE_URL}/crawls").mock( + return_value=Response(202, headers={"Location": "/v1/crawls/c_1"}, json=crawl_body()) + ) + + crawl = client.create("https://example.com/", depth=1) + + assert crawl.crawl_id == "c_1" + assert crawl.status is CrawlStatus.RUNNING + assert not crawl.is_terminal + sent = route.calls.last.request + assert sent.headers["X-API-Key"] == API_KEY + assert sent.headers["Content-Type"] == "application/json" + assert "Idempotency-Key" not in sent.headers + assert json.loads(sent.content) == {"url": "https://example.com/", "depth": 1} + + +@respx.mock +def test_create_sends_every_option_and_idempotency_key(client: ZenRowsCrawlClient): + route = respx.post(f"{BASE_URL}/crawls").mock( + return_value=Response(202, json=crawl_body(output_format="html")) + ) + + crawl = client.create( + "https://example.com/", + depth=2, + max_items=3, + max_pages=5, + include_patterns=["/product/"], + exclude_patterns=["/cart"], + output_format="html", + idempotency_key="idem-1", + ) + + assert crawl.output_format is OutputFormat.HTML + sent = route.calls.last.request + assert sent.headers["Idempotency-Key"] == "idem-1" + body = json.loads(sent.content) + assert body == { + "url": "https://example.com/", + "depth": 2, + "max_items": 3, + "max_pages": 5, + "include_patterns": ["/product/"], + "exclude_patterns": ["/cart"], + "output_format": "html", + } + assert "discovery" not in body + + +def test_create_output_format_accepts_only_html(client: ZenRowsCrawlClient): + with pytest.raises(ValueError, match="output_format"): + client.create("https://example.com/", depth=1, output_format="json") # type: ignore[arg-type] + + +# ----- get / list / stop ----- + + +@respx.mock +def test_get_parses_results_and_open_enums(client: ZenRowsCrawlClient): + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response( + 200, + json=crawl_body( + status="completed", + stop_reason="max_items", + output_format="xml", + finished_at="2026-10-08T09:02:00Z", + some_new_field=True, + results=[ + { + "url": "https://example.com/product/a", + "content_status": "fetched", + "content_url": "/v1/crawls/c_1/contents/ct_a", + }, + {"url": "https://example.com/product/b", "content_status": "archived"}, + ], + next_cursor=None, + ), + ) + ) + + page = client.get("c_1", cursor="cur_1", limit=50) + + assert route.calls.last.request.url.params["cursor"] == "cur_1" + assert route.calls.last.request.url.params["limit"] == "50" + assert page.status is CrawlStatus.COMPLETED + assert page.is_terminal + assert page.stop_reason is StopReason.MAX_ITEMS + assert is_unknown(page.output_format) # type: ignore[arg-type] + assert page.next_cursor is None + assert page.results[0].content_status is ContentStatus.FETCHED + assert page.results[0].content_id == "ct_a" + unknown = page.results[1].content_status + assert unknown is not None and is_unknown(unknown) and unknown.value == "archived" + + +@respx.mock +def test_get_parses_failed_crawl_error(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response( + 200, + json=crawl_body( + status="failed", + error={"code": "seed_unreachable", "detail": "The site may be blocking it."}, + results=[], + next_cursor=None, + ), + ) + ) + page = client.get("c_1") + assert page.status is CrawlStatus.FAILED + assert page.error is not None and page.error.code is CrawlErrorCode.SEED_UNREACHABLE + + +@respx.mock +def test_get_omits_unset_query_params(client: ZenRowsCrawlClient): + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response(200, json=crawl_body(results=[], next_cursor="cur_0")) + ) + client.get("c_1") + assert dict(route.calls.last.request.url.params) == {} + + +@respx.mock +def test_iter_crawls_paginates_until_cursor_absent(client: ZenRowsCrawlClient): + route = respx.get(f"{BASE_URL}/crawls").mock( + side_effect=[ + Response(200, json={"crawls": [crawl_body(crawl_id="c_2")], "next_cursor": "n1"}), + Response(200, json={"crawls": [crawl_body(crawl_id="c_1")]}), + ] + ) + + ids = [c.crawl_id for c in client.iter_crawls(page_size=1)] + + assert ids == ["c_2", "c_1"] + assert route.calls[0].request.url.params["limit"] == "1" + assert "cursor" not in route.calls[0].request.url.params + assert route.calls[1].request.url.params["cursor"] == "n1" + + +@respx.mock +def test_stop_posts_without_body(client: ZenRowsCrawlClient): + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + return_value=Response( + 200, + json={ + "crawl_id": "c_1", + "status": "stopped", + "stop_reason": "user", + "finished_at": "2026-10-08T09:01:00Z", + }, + ) + ) + + stopped = client.stop("c_1") + + assert stopped.status is CrawlStatus.STOPPED + assert stopped.stop_reason is StopReason.USER + assert route.calls.last.request.content == b"" + + +# ----- results scanner ----- + + +@respx.mock +def test_iter_results_stops_on_null_cursor(client: ZenRowsCrawlClient, no_sleep): + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + side_effect=[ + Response(200, json=crawl_body(results=[{"url": "https://a"}], next_cursor="k1")), + # Caught up with a running crawl: no results, same cursor back. + Response(200, json=crawl_body(results=[], next_cursor="k1")), + Response( + 200, + json=crawl_body( + status="completed", results=[{"url": "https://b"}], next_cursor=None + ), + ), + ] + ) + + urls = [r.url for r in client.iter_results("c_1", poll_interval=0.5)] + + assert urls == ["https://a", "https://b"] + assert route.call_count == 3 + assert "cursor" not in route.calls[0].request.url.params + assert route.calls[1].request.url.params["cursor"] == "k1" + assert route.calls[2].request.url.params["cursor"] == "k1" + assert no_sleep == [0.5] # slept only on the empty page of a running crawl + + +# ----- contents / download ----- + + +@respx.mock +def test_get_content_accepts_result_or_id(client: ZenRowsCrawlClient): + route = respx.get(f"{BASE_URL}/crawls/c_1/contents/ct_a").mock( + return_value=Response( + 200, headers={"Content-Type": "text/html"}, text="product" + ) + ) + result = CrawlResult( + url="https://a", content_status="fetched", content_url="/v1/crawls/c_1/contents/ct_a" + ) # type: ignore[arg-type] + + assert client.get_content("c_1", result) == "product" + assert client.get_content("c_1", "ct_a") == "product" + assert route.call_count == 2 + + +def test_get_content_without_content_url_raises(client: ZenRowsCrawlClient): + with pytest.raises(ValueError, match="content_url"): + client.get_content("c_1", CrawlResult(url="https://a", content_status="pending")) # type: ignore[arg-type] + + +NDJSON = ( + b'{"url":"https://a","content_status":"fetched","content":"a"}\n' + b'{"url":"https://b","content_status":"failed"}\n' +) + + +@respx.mock +def test_iter_download_parses_lines(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/c_1/download").mock( + return_value=Response( + 200, + headers={"Content-Type": "application/x-ndjson", "X-Crawl-Status": "completed"}, + content=NDJSON, + ) + ) + lines = list(client.iter_download("c_1")) + assert [line.url for line in lines] == ["https://a", "https://b"] + assert lines[0].content == "a" + assert lines[1].content_status is ContentStatus.FAILED + + +@respx.mock +def test_download_writes_file_and_warns_when_partial(client: ZenRowsCrawlClient, tmp_path, caplog): + respx.get(f"{BASE_URL}/crawls/c_1/download").mock( + return_value=Response(200, headers={"X-Crawl-Status": "running"}, content=NDJSON) + ) + target = client.download("c_1", tmp_path / "out" / "c_1.jsonl") + assert target.read_bytes() == NDJSON + assert "still running" in caplog.text + + +@respx.mock +def test_download_maps_errors(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/nope/download").mock(return_value=problem(404, "crawl_not_found")) + with pytest.raises(CrawlAPIError) as exc_info: + list(client.iter_download("nope")) + assert exc_info.value.code == "crawl_not_found" + + +# ----- errors ----- + + +@respx.mock +def test_not_found_maps_to_crawl_api_error(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/nope").mock(return_value=problem(404, "crawl_not_found")) + with pytest.raises(CrawlAPIError) as exc_info: + client.get("nope") + err = exc_info.value + assert err.status_code == 404 + assert err.code == "crawl_not_found" + assert err.detail == "detail for crawl_not_found" + assert not err.not_enabled + + +@respx.mock +def test_validation_error_maps_to_crawl_api_error(client: ZenRowsCrawlClient): + respx.post(f"{BASE_URL}/crawls").mock(return_value=problem(422, "invalid_start_url")) + with pytest.raises(CrawlAPIError) as exc_info: + client.create("http://10.0.0.1/", depth=1) + assert exc_info.value.status_code == 422 + assert exc_info.value.code == "invalid_start_url" + + +@respx.mock +def test_too_many_crawls_carries_retry_after(client: ZenRowsCrawlClient): + respx.post(f"{BASE_URL}/crawls").mock( + return_value=problem(429, "too_many_crawls", headers={"Retry-After": "30"}) + ) + with pytest.raises(CrawlAPIError) as exc_info: + client.create("https://example.com/", depth=1) + assert exc_info.value.code == "too_many_crawls" + assert exc_info.value.retry_after == 30.0 + + +@respx.mock +def test_crawl_not_enabled_has_clear_message(client: ZenRowsCrawlClient): + respx.post(f"{BASE_URL}/crawls").mock( + return_value=problem( + 403, + "REQS008", + title="Crawl is not enabled for this account.", + type="https://docs.zenrows.com/api-error-codes#REQS008", + ) + ) + with pytest.raises(CrawlAPIError) as exc_info: + client.create("https://example.com/", depth=1) + err = exc_info.value + assert err.not_enabled + assert err.code == "REQS008" + assert "Crawl is not enabled for this account" in str(err) + + +@respx.mock +def test_get_retries_transient_status(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + side_effect=[Response(503), Response(200, json=crawl_body(results=[], next_cursor="k"))] + ) + client.get("c_1") + assert route.call_count == 2 + assert len(no_sleep) == 1 + + +# ----- waiter ----- + + +@respx.mock +def test_wait_polls_until_terminal(client: ZenRowsCrawlClient, no_sleep): + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + side_effect=[ + Response(200, json=crawl_body(results=[], next_cursor="k")), + Response( + 200, + json=crawl_body( + status="completed", + finished_at="2026-10-08T09:02:00Z", + results=[{"url": "https://a"}], + next_cursor="k2", + ), + ), + ] + ) + + crawl = client.wait("c_1", poll_interval=1.0) + + assert crawl.status is CrawlStatus.COMPLETED + assert not hasattr(crawl, "results") + assert route.call_count == 2 + assert all(c.request.url.params["limit"] == "1" for c in route.calls) + assert len(no_sleep) == 1 + + +@respx.mock +def test_wait_times_out(client: ZenRowsCrawlClient, no_sleep, monkeypatch): + respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response(200, json=crawl_body(results=[], next_cursor="k")) + ) + clock = iter(range(0, 1000, 10)) + monkeypatch.setattr("zenrows.batch._waiters.time.monotonic", lambda: next(clock)) + + with pytest.raises(WaiterTimeout): + client.wait("c_1", timeout=25) From 1fc91d32fefd142a91eb1d09aed285955e55a387 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 9 Oct 2026 12:37:28 +0200 Subject: [PATCH 02/20] fix(crawl): align the Crawl client with the production API Mark Crawl as Beta in the README and the module and client docstrings. iter_results no longer polls. It returns at the first empty page. iter_download and download return the X-Crawl-Status status with the lines or the saved path. Download lines accept object content. CrawlAPIError.code is None when the body has no code. The client takes the base URL only from the constructor. create never retries 429 too_many_crawls. Batch retries do not change. wait defaults to 600 s. The README lists every Crawl error code. The example lets --include replace the default pattern. Add a py.typed marker so type checkers use the package types. --- DEVELOPMENT.md | 7 +- README.md | 54 +++++++---- examples/11_crawl.py | 19 ++-- src/zenrows/batch/_transport.py | 23 +++-- src/zenrows/batch/errors.py | 8 +- src/zenrows/crawl/__init__.py | 13 ++- src/zenrows/crawl/client.py | 164 +++++++++++++++++++++----------- src/zenrows/crawl/errors.py | 9 +- src/zenrows/crawl/models.py | 6 +- src/zenrows/py.typed | 0 tests/e2e/test_crawl_e2e.py | 17 ++-- tests/test_batch_client.py | 11 +++ tests/test_crawl_client.py | 113 +++++++++++++++++----- 13 files changed, 302 insertions(+), 142 deletions(-) create mode 100644 src/zenrows/py.typed diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 4198496..1589c68 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -81,7 +81,7 @@ and it skips unless the three required variables below are set. | Variable | | |---|---| | `ZENROWS_API_KEY` | required: a key with Crawl access | -| `ZENROWS_CRAWL_BASE_URL` | required: the API base, e.g. `https://api.zenrows.com/v1` | +| `ZENROWS_CRAWL_BASE_URL` | required: the API base, e.g. `https://api.zenrows.com/v1`; the test passes it as `base_url=` | | `ZENROWS_E2E_CRAWL_URL` | required: the page the crawl starts from | | `ZENROWS_E2E_CRAWL_INCLUDE` | optional: an include pattern; every result must contain it | @@ -98,8 +98,9 @@ the test crawls at depth 1 with `max_items=3` and `output_format="html"`. To test against a local or staging deployment, point `ZENROWS_CRAWL_BASE_URL` at its `/v1` base instead. When the account -has too many crawls running (429 `too_many_crawls`), the test waits and -retries (up to 5 minutes) before creating its crawl. +has reached its limit of active jobs (429 `too_many_crawls`: 3 by default, +shared with its Batch jobs), the test waits and retries (up to 5 minutes) +before creating its crawl. ## Refreshing the OpenAPI spec diff --git a/README.md b/README.md index 8133127..5068b99 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ This package ships three clients: _(Private beta — [contact support](mailto:support@zenrows.com) for access.)_ - **`ZenRowsCrawlClient`** — the Crawl API client. Give it one start URL; it follows links and returns the URLs (and, optionally, the - pages) it finds behind it. + pages) it finds behind it. _(Beta.)_ ## Table of Contents @@ -40,7 +40,7 @@ This package ships three clients: - [Act on an id without a GET](#act-on-an-id-without-a-get) - [Scheduled jobs & webhooks](#scheduled-jobs--webhooks) - [Error handling](#error-handling) -- [Quickstart — Crawl API (`ZenRowsCrawlClient`)](#quickstart--crawl-api-zenrowscrawlclient) +- [Quickstart — Crawl API (beta) (`ZenRowsCrawlClient`)](#quickstart--crawl-api-beta-zenrowscrawlclient) - [Read the pages](#read-the-pages) - [List and stop crawls](#list-and-stop-crawls) - [Crawl errors](#crawl-errors) @@ -492,11 +492,14 @@ The full Batch surface (jobs, runs, tasks, results, content, history, file_inputs, HMAC keys) is reachable via methods on `ZenRowsBatchClient`. See `src/zenrows/batch/client.py` or `help(ZenRowsBatchClient)`. -## Quickstart — Crawl API (`ZenRowsCrawlClient`) +## Quickstart — Crawl API (beta) (`ZenRowsCrawlClient`) + +Crawl is in Beta: its API may still change. A crawl starts from one URL, follows the links on each page up to `depth` hops, and keeps the URLs that match your patterns, up to -`max_items`. It runs in the background: `create` returns at once with +`max_items`. It stays on the start URL's registrable domain; +subdomains count. It runs in the background: `create` returns at once with `status="running"`, and `wait` blocks until it ends. Without `output_format` a crawl returns URLs only; `output_format="html"` also returns each kept URL's page. @@ -514,7 +517,7 @@ crawl = client.create( max_pages=100, # stop after fetching 100 pages (default 10) include_patterns=["/product/"], # keep only URLs containing this ) -crawl = client.wait(crawl.crawl_id, timeout=600) # Crawl, terminal +crawl = client.wait(crawl.crawl_id) # Crawl, terminal (timeout 600 s) print(crawl.status.value, crawl.stop_reason, crawl.coverage) for result in client.iter_results(crawl.crawl_id): # follows next_cursor @@ -523,10 +526,10 @@ for result in client.iter_results(crawl.crawl_id): # follows next_cursor `wait` returns a `failed` crawl rather than raising (read `crawl.error`), and raises `WaiterTimeout` on timeout without stopping the crawl. -`iter_results` stops when the server's `next_cursor` is null, which happens -only once the crawl has ended; on a running crawl it follows new results -live until the crawl ends. `get(crawl_id, cursor=..., limit=...)` returns -one raw page. +`iter_results` does not poll: it returns at the first empty page, so on a +running crawl it yields only the URLs kept so far. Call `wait` first to +read every result. `get(crawl_id, cursor=..., limit=...)` returns one raw +page. ### Read the pages @@ -541,11 +544,18 @@ for result in client.iter_results(crawl.crawl_id): html = client.get_content(crawl.crawl_id, result) # Or everything in one NDJSON file: {"url", "content_status", "content"} per line. -client.download(crawl.crawl_id, "out/crawl.jsonl") -for line in client.iter_download(crawl.crawl_id): # streamed, parsed - print(line.url, len(line.content or "")) +saved = client.download(crawl.crawl_id, "out/crawl.jsonl") +print(saved.path, saved.status) # status from X-Crawl-Status + +with client.iter_download(crawl.crawl_id) as download: # streamed, parsed + print(download.status) + for line in download: + print(line.url, len(line.content or "")) ``` +Both downloads carry the crawl's `status` when the file was read. On a +`running` crawl the file holds only the results kept so far. + ### List and stop crawls ```python @@ -567,17 +577,27 @@ try: except CrawlAPIError as exc: if exc.not_enabled: # 403 REQS008 print("Crawl is not enabled for this account") - elif exc.code == "too_many_crawls": # 429: too many crawls running + elif exc.code == "too_many_crawls": # 429: active jobs limit reached print("retry in", exc.retry_after, "s") - elif exc.code in ("invalid_parameter", "invalid_start_url"): # 422 + elif exc.status_code in (400, 422): print(exc.detail) else: raise ``` -`crawl_not_found` / `content_not_found` come with 404. A crawl that -ended `failed` is not an error response: read `crawl.error.code` and -`crawl.error.detail` on it. Response enums (`CrawlStatus`, `StopReason`, +| Status | `code` | What to do | +|---|---|---| +| 400 | `invalid_request`, `unknown_parameter`, `invalid_cursor` | Fix the request. Do not retry it as is. | +| 403 | `REQS008` | Crawl is not enabled for the account (`exc.not_enabled`). | +| 404 | `crawl_not_found`, `content_not_found` | Check the id. | +| 409 | `idempotency_request_in_flight` | Retry after the first request with this key finishes. | +| 422 | `invalid_parameter`, `invalid_start_url`, `domain_not_allowed` | Fix the request. Do not retry it as is. | +| 422 | `idempotency_key_reused` | Use a new key, or no key. Do not retry it as is. | +| 429 | `too_many_crawls` | The account has reached its limit of active jobs (3 by default), shared with its Batch jobs. Retry after `exc.retry_after` seconds. `create` does not retry it. | + +`code` is None when the response has no problem body with a code (for +example a 502 from a proxy). A crawl that ended `failed` is not an error +response: read `crawl.error.code` and `crawl.error.detail` on it. Response enums (`CrawlStatus`, `StopReason`, `ContentStatus`, ...) are open: a value added later parses as an `UNKNOWN` member that keeps the raw value. diff --git a/examples/11_crawl.py b/examples/11_crawl.py index 1d3246e..fbeb982 100644 --- a/examples/11_crawl.py +++ b/examples/11_crawl.py @@ -1,13 +1,14 @@ -"""11: Crawl a site → wait → read URLs and pages. +"""11: Crawl a site → wait → read URLs and pages. Crawl is in Beta. Demonstrates the Crawl client (`ZenRowsCrawlClient`): - `create(url, depth=..., include_patterns=..., output_format="html")` starts a crawl and returns at once, `running`. - `wait(crawl_id)` blocks until it ends and returns the final `Crawl` (a `failed` crawl is returned, with `error` set). - - `iter_results(crawl_id)` pages through every kept URL. + - `iter_results(crawl_id)` pages through the kept URLs. - `get_content(crawl_id, result)` reads one kept URL's HTML. - - `download(crawl_id, path)` saves every result as NDJSON. + - `download(crawl_id, path)` saves every result as NDJSON and + returns the path and the crawl's status. Run with: export ZENROWS_API_KEY=zr_... @@ -26,7 +27,9 @@ def main() -> None: parser.add_argument("--url", required=True, help="the page to start from") parser.add_argument("--depth", type=int, default=1) parser.add_argument("--max-items", type=int, default=5) - parser.add_argument("--include", action="append", default=["/product/"]) + parser.add_argument( + "--include", action="append", help="keep URLs containing this (default /product/)" + ) parser.add_argument("--out", default="crawl.jsonl") args = parser.parse_args() @@ -37,7 +40,7 @@ def main() -> None: args.url, depth=args.depth, max_items=args.max_items, - include_patterns=args.include, + include_patterns=args.include or ["/product/"], output_format="html", ) except CrawlAPIError as exc: @@ -46,7 +49,7 @@ def main() -> None: raise print(f"started {crawl.crawl_id}") - crawl = client.wait(crawl.crawl_id, timeout=600.0) + crawl = client.wait(crawl.crawl_id) cov = crawl.coverage print( f"{crawl.crawl_id} {crawl.status.value}: {cov.items_found} URLs kept, " @@ -62,8 +65,8 @@ def main() -> None: html = client.get_content(crawl.crawl_id, result) print(f" {len(html)} chars of HTML") - path = client.download(crawl.crawl_id, args.out) - print(f"wrote {path}") + saved = client.download(crawl.crawl_id, args.out) + print(f"wrote {saved.path}") if __name__ == "__main__": diff --git a/src/zenrows/batch/_transport.py b/src/zenrows/batch/_transport.py index f6c95bf..1c3e2e5 100644 --- a/src/zenrows/batch/_transport.py +++ b/src/zenrows/batch/_transport.py @@ -117,17 +117,18 @@ def _send( params: dict[str, Any] | None, headers: dict[str, str] | None, content: bytes | None = None, + retry_statuses: frozenset[int] = _RETRYABLE_STATUSES, ) -> httpx.Response: """Issue the request, retrying transient failures on idempotent requests. - Retries `{429, 502, 503, 504}` and transient network errors up - to `retries` times, with jittered exponential backoff (honoring - `Retry-After` when present). Only idempotent requests are - replayed — `GET`/`PUT`/`DELETE`/`HEAD`/`OPTIONS`, plus `POST` - when the caller supplied an `Idempotency-Key`. Our own timeouts - (`httpx.TimeoutException`) are never retried: the caller set - that budget. + Retries `retry_statuses` (default `{429, 502, 503, 504}`) and + transient network errors up to `retries` times, with jittered + exponential backoff (honoring `Retry-After` when present). Only + idempotent requests are replayed — `GET`/`PUT`/`DELETE`/`HEAD`/ + `OPTIONS`, plus `POST` when the caller supplied an + `Idempotency-Key`. Our own timeouts (`httpx.TimeoutException`) + are never retried: the caller set that budget. """ idempotent = method.upper() in _IDEMPOTENT_METHODS or ( method.upper() == "POST" and _has_idempotency_key(headers) @@ -149,11 +150,7 @@ def _send( continue raise - if ( - idempotent - and attempt < self._retries - and response.status_code in _RETRYABLE_STATUSES - ): + if idempotent and attempt < self._retries and response.status_code in retry_statuses: wait_ms = _retry_after_ms(response) or _backoff_ms(attempt) response.close() time.sleep(wait_ms / 1000) @@ -216,6 +213,7 @@ def request( params: dict[str, Any] | None = None, headers: dict[str, str] | None = None, content: bytes | None = None, + retry_statuses: frozenset[int] = _RETRYABLE_STATUSES, ) -> httpx.Response: """Send a request (with retries) and return the raw response, raising the mapped error on non-2xx. For non-JSON bodies.""" @@ -226,6 +224,7 @@ def request( params=_drop_none(params), headers=headers, content=content, + retry_statuses=retry_statuses, ) elapsed_ms = (time.monotonic() - start) * 1000 self._raise_for_status(method, path, response) diff --git a/src/zenrows/batch/errors.py b/src/zenrows/batch/errors.py index 940a2be..40725b4 100644 --- a/src/zenrows/batch/errors.py +++ b/src/zenrows/batch/errors.py @@ -24,7 +24,7 @@ class ProblemDetail: type: str title: str status: int - code: str + code: str | None detail: str | None = None instance: str | None = None extras: dict[str, Any] | None = None @@ -48,7 +48,7 @@ def from_response(cls, response: httpx.Response) -> "ProblemDetail | None": type=body.get("type", "about:blank"), title=body.get("title", "Error"), status=int(body.get("status", response.status_code)), - code=body.get("code", "internal"), + code=body.get("code"), detail=body.get("detail"), instance=body.get("instance"), extras=extras or None, @@ -68,10 +68,10 @@ def __init__(self, status_code: int, problem: ProblemDetail | None, raw: bytes): self.status_code = status_code self.problem = problem self.raw = raw - self.code: str = problem.code if problem else "internal" + self.code: str = (problem.code if problem else None) or "internal" self.detail: str | None = problem.detail if problem else None msg = ( - f"{status_code} {problem.title}: {problem.detail or problem.code}" + f"{status_code} {problem.title}: {problem.detail or self.code}" if problem else f"{status_code} (no problem body)" ) diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py index 2f2021e..8bfc28f 100644 --- a/src/zenrows/crawl/__init__.py +++ b/src/zenrows/crawl/__init__.py @@ -1,4 +1,6 @@ -"""Public surface for the ZenRows Crawl API. +"""Public surface for the ZenRows Crawl API (Beta). + +Crawl is in Beta: its surface may still change. What's where: - `client.ZenRowsCrawlClient` — the typed facade: create, get, list, @@ -11,7 +13,12 @@ """ from zenrows.batch._waiters import WaiterTimeout -from zenrows.crawl.client import DEFAULT_BASE_URL, ZenRowsCrawlClient +from zenrows.crawl.client import ( + DEFAULT_BASE_URL, + CrawlDownload, + CrawlDownloadFile, + ZenRowsCrawlClient, +) from zenrows.crawl.errors import CRAWL_NOT_ENABLED, CrawlAPIError from zenrows.crawl.models import ( ContentStatus, @@ -36,6 +43,8 @@ "Coverage", "Crawl", "CrawlAPIError", + "CrawlDownload", + "CrawlDownloadFile", "CrawlErrorCode", "CrawlList", "CrawlResult", diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 30d6549..6fca9c0 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -1,4 +1,4 @@ -"""ZenRowsCrawlClient — the typed facade over the Crawl API. +"""ZenRowsCrawlClient — the typed facade over the Crawl API (Beta). A crawl starts from one URL, follows links up to `depth` hops, and keeps the URLs that match its patterns, up to `max_items`. It runs as @@ -18,16 +18,16 @@ import json import logging -import os -import time -from collections.abc import Iterator +from collections.abc import Callable, Iterator +from contextlib import ExitStack +from dataclasses import dataclass from pathlib import Path from typing import Any, Literal import httpx from zenrows.__version__ import __version__ -from zenrows.batch._transport import _Transport +from zenrows.batch._transport import _RETRYABLE_STATUSES, _Transport from zenrows.batch._waiters import poll_until from zenrows.crawl.errors import CrawlAPIError from zenrows.crawl.models import ( @@ -40,16 +40,59 @@ DownloadLine, ) -# Production endpoint. The `base_url=` kwarg + `ZENROWS_CRAWL_BASE_URL` -# env var are present for advanced use only. DEFAULT_BASE_URL = "https://api.zenrows.com/v1" DEFAULT_USER_AGENT = f"zenrows-crawl-python/{__version__}" _log = logging.getLogger("zenrows.crawl.transport") +# 429 `too_many_crawls` is a capacity limit, not a transient failure. +_CREATE_RETRY_STATUSES = _RETRYABLE_STATUSES - {429} + + +class CrawlDownload: + """A crawl's NDJSON download, open and ready to read. + + `status` is the crawl's status when the file was read + (`X-Crawl-Status`): `running` means the file holds only what the + crawl has kept so far. Iterate it once for the parsed lines; the + connection closes when the iteration ends. Use it as a context + manager to close it without reading to the end. + """ + + def __init__(self, response: httpx.Response, close: Callable[[], None]): + self.status: CrawlStatus | None = _crawl_status(response) + self._response = response + self._close = close + + def __iter__(self) -> Iterator[DownloadLine]: + try: + for line in self._response.iter_lines(): + if line.strip(): + yield DownloadLine.model_validate_json(line) + finally: + self.close() + + def close(self) -> None: + self._close() + + def __enter__(self) -> "CrawlDownload": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + +@dataclass(slots=True, frozen=True) +class CrawlDownloadFile: + """A crawl's NDJSON download, saved to `path`. `status` is as on + `CrawlDownload`.""" + + path: Path + status: CrawlStatus | None + class ZenRowsCrawlClient: - """Synchronous, typed client for the ZenRows Crawl API. + """Synchronous, typed client for the ZenRows Crawl API (Beta). `api_key` is required: @@ -71,16 +114,16 @@ def __init__( user_agent: str = DEFAULT_USER_AGENT, httpx_args: dict[str, Any] | None = None, ): - """`retries` bounds automatic retries of transient failures - (HTTP 429/502/503/504 and network errors) on idempotent - requests — GETs, and a `create` that carries an - `idempotency_key`. Retries use jittered exponential backoff and - honor `Retry-After`; set `retries=0` to disable.""" + """`base_url` defaults to `DEFAULT_BASE_URL`. `retries` bounds + automatic retries of transient failures (HTTP 429/502/503/504 + and network errors) on idempotent requests — GETs, and a + `create` that carries an `idempotency_key` (never on 429 for + `create`). Retries use jittered exponential backoff and honor + `Retry-After`; set `retries=0` to disable.""" if not api_key: raise ValueError("ZenRowsCrawlClient: api_key is required.") - base_url = base_url or os.environ.get("ZENROWS_CRAWL_BASE_URL") or DEFAULT_BASE_URL self._t = _Transport( - base_url=base_url, + base_url=base_url or DEFAULT_BASE_URL, api_key=api_key, user_agent=user_agent, timeout=timeout, @@ -129,15 +172,17 @@ def create( bounds the cost (server default 10). - `include_patterns` / `exclude_patterns`: substrings matched against each normalized URL. A URL is kept when it matches - an include pattern (if any) and no exclude pattern. + an include pattern (if any) and no exclude pattern. A crawl + stays on the start URL's registrable domain; subdomains count. - `output_format="html"`: also fetch every kept URL's page, readable with `get_content` / `download`. Omit for URLs only. - `idempotency_key`: a retry with the same key and body answers with the crawl the first request created. - Only the arguments you pass are sent. An account with too many - crawls running gets 429 `too_many_crawls`; retry after the - error's `retry_after` seconds. + Only the arguments you pass are sent. When the account has + reached its limit of active jobs (3 by default), shared with its + Batch jobs, `create` raises 429 `too_many_crawls` without + retrying; retry after the error's `retry_after` seconds. """ if output_format not in (None, "html"): raise ValueError(f"create: output_format must be 'html' or None, not {output_format!r}") @@ -154,7 +199,11 @@ def create( if idempotency_key: headers["Idempotency-Key"] = idempotency_key response = self._t.request( - "POST", "/crawls", headers=headers, content=json.dumps(body).encode("utf-8") + "POST", + "/crawls", + headers=headers, + content=json.dumps(body).encode("utf-8"), + retry_statuses=_CREATE_RETRY_STATUSES, ) return Crawl.model_validate(response.json()) @@ -203,25 +252,20 @@ def iter_results( crawl_id: str, *, page_size: int | None = None, - poll_interval: float = 2.0, ) -> Iterator[CrawlResult]: - """Yield every URL the crawl keeps, in order, each once. - - Follows `next_cursor` until the server returns null, which it - does only once the crawl has ended and its last page was read. - On a crawl that is still running, the iterator follows it live: - when it has caught up, it sleeps `poll_interval` seconds and - asks again with the same cursor, so it ends when the crawl ends. - Call `wait` first to read a finished crawl without polling. + """Yield the URLs the crawl has kept, in order, each once. + + Follows `next_cursor` and returns at the first empty page or + null cursor. It does not poll: on a running crawl it yields + only what the crawl has kept so far. Call `wait` first to read + every result. """ cursor: str | None = None while True: page = self.get(crawl_id, cursor=cursor, limit=page_size) yield from page.results - if page.next_cursor is None: + if not page.results or page.next_cursor is None: return - if not page.results and page.status is CrawlStatus.RUNNING: - time.sleep(poll_interval) cursor = page.next_cursor def stop(self, crawl_id: str) -> CrawlStop: @@ -255,18 +299,18 @@ def get_content(self, crawl_id: str, content: str | CrawlResult) -> str: content_id = content return self._t.request("GET", f"/crawls/{crawl_id}/contents/{content_id}").text - def iter_download(self, crawl_id: str) -> Iterator[DownloadLine]: - """`GET /crawls/{crawl_id}/download` — stream every result as - a parsed `DownloadLine` (url, content_status, content). + def iter_download(self, crawl_id: str) -> CrawlDownload: + """`GET /crawls/{crawl_id}/download` — open the NDJSON file and + return it as a `CrawlDownload`: iterate it for parsed + `DownloadLine`s (url, content_status, content), and read its + `status`. On a running crawl the file holds what the crawl has kept so - far (a warning is logged); call `wait` first for the whole of it. + far, and `status` is `running`; call `wait` first for all of it. """ - with self._t.stream("GET", f"/crawls/{crawl_id}/download") as response: - _warn_if_partial(crawl_id, response) - for line in response.iter_lines(): - if line.strip(): - yield DownloadLine.model_validate_json(line) + stack = ExitStack() + response = stack.enter_context(self._t.stream("GET", f"/crawls/{crawl_id}/download")) + return CrawlDownload(response, stack.close) def download( self, @@ -274,21 +318,23 @@ def download( target_path: str | Path, *, chunk_size: int = 64 * 1024, - ) -> Path: + ) -> CrawlDownloadFile: """`GET /crawls/{crawl_id}/download` — stream the NDJSON file - (one JSON object per result) to `target_path`, and return it. + (one JSON object per result) to `target_path`. Returns its + `path` and the crawl's `status` when the file was read. On a running crawl the file holds what the crawl has kept so - far (a warning is logged); call `wait` first for the whole of it. + far, and `status` is `running`; call `wait` first for all of it. """ target = Path(target_path) target.parent.mkdir(parents=True, exist_ok=True) - with self._t.stream("GET", f"/crawls/{crawl_id}/download") as response: - _warn_if_partial(crawl_id, response) - with target.open("wb") as f: - for chunk in response.iter_bytes(chunk_size): - f.write(chunk) - return target + with ( + self._t.stream("GET", f"/crawls/{crawl_id}/download") as response, + target.open("wb") as f, + ): + for chunk in response.iter_bytes(chunk_size): + f.write(chunk) + return CrawlDownloadFile(path=target, status=_crawl_status(response)) # ===== waiter ===== @@ -296,7 +342,7 @@ def wait( self, crawl_id: str, *, - timeout: float = 300.0, + timeout: float = 600.0, poll_interval: float = 2.0, max_poll_interval: float = 15.0, ) -> Crawl: @@ -323,13 +369,15 @@ def fetch() -> Crawl: ) -def _warn_if_partial(crawl_id: str, response: httpx.Response) -> None: +def _crawl_status(response: httpx.Response) -> CrawlStatus | None: status = response.headers.get("X-Crawl-Status") - if status == CrawlStatus.RUNNING.value: - logging.getLogger("zenrows.crawl.client").warning( - "crawl %s is still running: the download holds only the results kept so far", - crawl_id, - ) + return CrawlStatus(status) if status else None -__all__ = ["DEFAULT_BASE_URL", "DEFAULT_USER_AGENT", "ZenRowsCrawlClient"] +__all__ = [ + "DEFAULT_BASE_URL", + "DEFAULT_USER_AGENT", + "CrawlDownload", + "CrawlDownloadFile", + "ZenRowsCrawlClient", +] diff --git a/src/zenrows/crawl/errors.py b/src/zenrows/crawl/errors.py index b1f5b9d..d81f67b 100644 --- a/src/zenrows/crawl/errors.py +++ b/src/zenrows/crawl/errors.py @@ -20,8 +20,8 @@ class CrawlAPIError(Exception): `code` is the problem's `code` member: `crawl_not_found`, `content_not_found`, `invalid_parameter`, `too_many_crawls`, ..., `REQS008` when Crawl is not enabled for the account, and the API's - usual auth and credit codes. `problem` - holds the whole body, or None when it was not JSON. + usual auth and credit codes; None when the body has no code. + `problem` holds the whole body, or None when it was not JSON. """ def __init__( @@ -38,14 +38,15 @@ def __init__( self.retry_after = retry_after self.problem = problem self.raw = raw - self.code: str = problem.code if problem else "internal" + self.code: str | None = problem.code if problem else None self.detail: str | None = problem.detail if problem else None if problem and self.code == CRAWL_NOT_ENABLED: msg = f"{status_code} Crawl is not enabled for this account ({CRAWL_NOT_ENABLED})" + ( f": {problem.detail}" if problem.detail else "" ) elif problem: - msg = f"{status_code} {problem.title}: {problem.detail or problem.code}" + reason = problem.detail or problem.code + msg = f"{status_code} {problem.title}" + (f": {reason}" if reason else "") else: msg = f"{status_code} (no problem body)" super().__init__(msg) diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index e3249e0..d6a07e2 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -10,6 +10,7 @@ from datetime import datetime from enum import Enum +from typing import Any from pydantic import BaseModel @@ -172,8 +173,9 @@ class DownloadLine(BaseModel): url: str content_status: ContentStatus | None = None - content: str | None = None - """The page HTML, present when `content_status` is `fetched`.""" + content: str | dict[str, Any] | None = None + """The page, present when `content_status` is `fetched`: HTML text, + or an object for a crawl whose output format returns parsed data.""" __all__ = [ diff --git a/src/zenrows/py.typed b/src/zenrows/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/e2e/test_crawl_e2e.py b/tests/e2e/test_crawl_e2e.py index d37db0d..8357507 100644 --- a/tests/e2e/test_crawl_e2e.py +++ b/tests/e2e/test_crawl_e2e.py @@ -30,14 +30,16 @@ START_URL = os.environ.get("ZENROWS_E2E_CRAWL_URL", "") INCLUDE = os.environ.get("ZENROWS_E2E_CRAWL_INCLUDE") or None -# An account can run only a few crawls at once, so a create can meet -# 429 too_many_crawls while other runs finish. +# An account has a limit of active jobs (3 by default, shared with its +# Batch jobs), so a create can meet 429 too_many_crawls while others finish. SLOT_WAIT_SECONDS = 300 @pytest.fixture(scope="module") def client(): - with ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"]) as c: + with ZenRowsCrawlClient( + api_key=os.environ["ZENROWS_API_KEY"], base_url=os.environ["ZENROWS_CRAWL_BASE_URL"] + ) as c: yield c @@ -86,11 +88,14 @@ def test_crawl_end_to_end(client: ZenRowsCrawlClient, tmp_path): print(f"content of {fetched[0].url}: {len(html)} chars") assert " ZenRowsCrawlClient: @pytest.fixture def no_sleep(monkeypatch): - """Record + swallow sleeps (waiter, scanner and retry backoff all - call `time.sleep`) so tests run instantly.""" + """Record + swallow sleeps (waiter and retry backoff both call + `time.sleep`) so tests run instantly.""" slept: list[float] = [] - monkeypatch.setattr("zenrows.crawl.client.time.sleep", slept.append) + monkeypatch.setattr("time.sleep", slept.append) return slept @@ -81,14 +82,9 @@ def test_missing_api_key_raises(): ZenRowsCrawlClient(api_key="") -def test_default_base_url(monkeypatch): - monkeypatch.delenv("ZENROWS_CRAWL_BASE_URL", raising=False) - assert ZenRowsCrawlClient(api_key=API_KEY).base_url.rstrip("/") == "https://api.zenrows.com/v1" - - -def test_base_url_env_and_kwarg_precedence(monkeypatch): +def test_base_url_is_a_constructor_option_only(monkeypatch): monkeypatch.setenv("ZENROWS_CRAWL_BASE_URL", "http://env-set/v1") - assert ZenRowsCrawlClient(api_key=API_KEY).base_url.rstrip("/") == "http://env-set/v1" + assert ZenRowsCrawlClient(api_key=API_KEY).base_url.rstrip("/") == "https://api.zenrows.com/v1" assert ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL).base_url.rstrip("/") == BASE_URL @@ -262,12 +258,10 @@ def test_stop_posts_without_body(client: ZenRowsCrawlClient): @respx.mock -def test_iter_results_stops_on_null_cursor(client: ZenRowsCrawlClient, no_sleep): +def test_iter_results_stops_on_null_cursor(client: ZenRowsCrawlClient): route = respx.get(f"{BASE_URL}/crawls/c_1").mock( side_effect=[ Response(200, json=crawl_body(results=[{"url": "https://a"}], next_cursor="k1")), - # Caught up with a running crawl: no results, same cursor back. - Response(200, json=crawl_body(results=[], next_cursor="k1")), Response( 200, json=crawl_body( @@ -277,14 +271,31 @@ def test_iter_results_stops_on_null_cursor(client: ZenRowsCrawlClient, no_sleep) ] ) - urls = [r.url for r in client.iter_results("c_1", poll_interval=0.5)] + urls = [r.url for r in client.iter_results("c_1")] assert urls == ["https://a", "https://b"] - assert route.call_count == 3 + assert route.call_count == 2 assert "cursor" not in route.calls[0].request.url.params assert route.calls[1].request.url.params["cursor"] == "k1" - assert route.calls[2].request.url.params["cursor"] == "k1" - assert no_sleep == [0.5] # slept only on the empty page of a running crawl + + +@respx.mock +def test_iter_results_returns_at_first_empty_page_of_running_crawl( + client: ZenRowsCrawlClient, no_sleep +): + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + side_effect=[ + Response(200, json=crawl_body(results=[{"url": "https://a"}], next_cursor="k1")), + # Caught up with a running crawl: no results, same cursor back. + Response(200, json=crawl_body(results=[], next_cursor="k1")), + ] + ) + + urls = [r.url for r in client.iter_results("c_1")] + + assert urls == ["https://a"] + assert route.call_count == 2 + assert no_sleep == [] # ----- contents / download ----- @@ -314,11 +325,12 @@ def test_get_content_without_content_url_raises(client: ZenRowsCrawlClient): NDJSON = ( b'{"url":"https://a","content_status":"fetched","content":"a"}\n' b'{"url":"https://b","content_status":"failed"}\n' + b'{"url":"https://c","content_status":"fetched","content":{"title":"C"}}\n' ) @respx.mock -def test_iter_download_parses_lines(client: ZenRowsCrawlClient): +def test_iter_download_parses_lines_and_status(client: ZenRowsCrawlClient): respx.get(f"{BASE_URL}/crawls/c_1/download").mock( return_value=Response( 200, @@ -326,27 +338,39 @@ def test_iter_download_parses_lines(client: ZenRowsCrawlClient): content=NDJSON, ) ) - lines = list(client.iter_download("c_1")) - assert [line.url for line in lines] == ["https://a", "https://b"] + download = client.iter_download("c_1") + assert download.status is CrawlStatus.COMPLETED + lines = list(download) + assert [line.url for line in lines] == ["https://a", "https://b", "https://c"] assert lines[0].content == "a" assert lines[1].content_status is ContentStatus.FAILED + assert lines[2].content == {"title": "C"} + + +@respx.mock +def test_iter_download_without_status_header(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/c_1/download").mock(return_value=Response(200, content=b"")) + with client.iter_download("c_1") as download: + assert download.status is None + assert list(download) == [] @respx.mock -def test_download_writes_file_and_warns_when_partial(client: ZenRowsCrawlClient, tmp_path, caplog): +def test_download_writes_file_and_returns_status(client: ZenRowsCrawlClient, tmp_path): respx.get(f"{BASE_URL}/crawls/c_1/download").mock( return_value=Response(200, headers={"X-Crawl-Status": "running"}, content=NDJSON) ) - target = client.download("c_1", tmp_path / "out" / "c_1.jsonl") - assert target.read_bytes() == NDJSON - assert "still running" in caplog.text + target = tmp_path / "out" / "c_1.jsonl" + saved = client.download("c_1", target) + assert saved == CrawlDownloadFile(path=target, status=CrawlStatus.RUNNING) + assert saved.path.read_bytes() == NDJSON @respx.mock def test_download_maps_errors(client: ZenRowsCrawlClient): respx.get(f"{BASE_URL}/crawls/nope/download").mock(return_value=problem(404, "crawl_not_found")) with pytest.raises(CrawlAPIError) as exc_info: - list(client.iter_download("nope")) + client.iter_download("nope") assert exc_info.value.code == "crawl_not_found" @@ -403,6 +427,43 @@ def test_crawl_not_enabled_has_clear_message(client: ZenRowsCrawlClient): assert "Crawl is not enabled for this account" in str(err) +@pytest.mark.parametrize( + "response", + [Response(502, text="bad gateway"), Response(500, json={"title": "Server error"})], +) +@respx.mock +def test_error_without_code_has_none_code(client: ZenRowsCrawlClient, response): + respx.get(f"{BASE_URL}/crawls/c_1").mock(return_value=response) + with pytest.raises(CrawlAPIError) as exc_info: + client.get("c_1") + assert exc_info.value.status_code == response.status_code + assert exc_info.value.code is None + + +@respx.mock +def test_keyed_create_does_not_retry_too_many_crawls(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls").mock( + return_value=problem(429, "too_many_crawls", headers={"Retry-After": "1"}) + ) + with pytest.raises(CrawlAPIError) as exc_info: + client.create("https://example.com/", depth=1, idempotency_key="idem-1") + assert exc_info.value.code == "too_many_crawls" + assert route.call_count == 1 + assert no_sleep == [] + + +@respx.mock +def test_keyed_create_retries_transient_status(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls").mock( + side_effect=[Response(503), Response(202, json=crawl_body())] + ) + client.create("https://example.com/", depth=1, idempotency_key="idem-1") + assert route.call_count == 2 + assert len(no_sleep) == 1 + + @respx.mock def test_get_retries_transient_status(no_sleep): client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) From 42b3dc55c5b40f981bd56d05208dc8c4fa0418b0 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 9 Oct 2026 17:34:45 +0200 Subject: [PATCH 03/20] refactor(crawl): match the shared Crawl client interface Rename the methods to the names that all Crawl clients share: list_crawls is now list, iter_results is now results (with limit for the page size), get_content is now content, and iter_download is now download. download returns a CrawlDownload with the status and the parsed lines. Remove iter_crawls, the download-to-path helper with CrawlDownloadFile, and the wait poll interval options. The 2 s, x1.5, 15 s backoff stays internal. wait now returns the crawl when the timeout (600 s by default) runs out. The crawl is still running. It no longer raises WaiterTimeout, so the crawl package no longer exports it. --- DEVELOPMENT.md | 2 +- README.md | 40 +++++------ examples/11_crawl.py | 25 ++++--- examples/README.md | 2 +- src/zenrows/crawl/__init__.py | 6 +- src/zenrows/crawl/client.py | 124 ++++++++++------------------------ src/zenrows/crawl/models.py | 2 +- tests/e2e/test_crawl_e2e.py | 22 +++--- tests/test_crawl_client.py | 103 ++++++++++++++++------------ 9 files changed, 145 insertions(+), 181 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 1589c68..1ae3631 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -49,7 +49,7 @@ override) stays in our control. The Crawl client uses the shared HTTP transport (`_transport.py`: key header, retries, problem+json) with `CrawlAPIError.from_response` as -its error mapping, and the shared `poll_until` loop. Its models are +its error mapping. Its models are hand-written: the surface is a few small schemas, so there is no Crawl codegen step. Keep them tolerant: unknown fields are ignored, and every response enum diff --git a/README.md b/README.md index 5068b99..934b245 100644 --- a/README.md +++ b/README.md @@ -517,19 +517,20 @@ crawl = client.create( max_pages=100, # stop after fetching 100 pages (default 10) include_patterns=["/product/"], # keep only URLs containing this ) -crawl = client.wait(crawl.crawl_id) # Crawl, terminal (timeout 600 s) +crawl = client.wait(crawl.crawl_id) # polls up to timeout=600 s print(crawl.status.value, crawl.stop_reason, crawl.coverage) -for result in client.iter_results(crawl.crawl_id): # follows next_cursor +for result in client.results(crawl.crawl_id): # follows next_cursor print(result.url) ``` -`wait` returns a `failed` crawl rather than raising (read `crawl.error`), -and raises `WaiterTimeout` on timeout without stopping the crawl. -`iter_results` does not poll: it returns at the first empty page, so on a +`wait` returns the crawl when it ends, or when `timeout` (seconds, +default 600) runs out. It does not raise: on timeout the crawl is still +`running` (call `wait` again, or `stop`), and a `failed` crawl comes +back with `crawl.error` set. `results` does not poll: it returns at the first empty page, so on a running crawl it yields only the URLs kept so far. Call `wait` first to -read every result. `get(crawl_id, cursor=..., limit=...)` returns one raw -page. +read every result. `results(crawl_id, limit=...)` sets the page size of +each request. `get(crawl_id, cursor=..., limit=...)` returns one raw page. ### Read the pages @@ -539,28 +540,29 @@ Pass `output_format="html"` to also fetch the page of every kept URL: crawl = client.create(start_url, depth=1, output_format="html") client.wait(crawl.crawl_id) -for result in client.iter_results(crawl.crawl_id): - if result.content_id: # content_status == fetched - html = client.get_content(crawl.crawl_id, result) +for result in client.results(crawl.crawl_id): + if result.content_id: # content_status == fetched + html = client.content(crawl.crawl_id, result) # Or everything in one NDJSON file: {"url", "content_status", "content"} per line. -saved = client.download(crawl.crawl_id, "out/crawl.jsonl") -print(saved.path, saved.status) # status from X-Crawl-Status - -with client.iter_download(crawl.crawl_id) as download: # streamed, parsed - print(download.status) +with client.download(crawl.crawl_id) as download: # streamed, parsed + print(download.status) # from X-Crawl-Status for line in download: - print(line.url, len(line.content or "")) + print(line.url, line.content_status) ``` -Both downloads carry the crawl's `status` when the file was read. On a -`running` crawl the file holds only the results kept so far. +`content` takes a result or its content id. `download` returns a +`CrawlDownload`: its `status` is the crawl's status when the file was +read, and iterating it yields parsed lines. On a `running` crawl the +file holds only the results kept so far. ### List and stop crawls ```python -for c in client.iter_crawls(): # newest first, without results +page = client.list(limit=20) # newest first, without results +for c in page.crawls: print(c.crawl_id, c.status.value) +# Pass page.next_cursor as cursor= for the next page; it is None on the last. client.stop(crawl_id) # idempotent; read final counts with get() ``` diff --git a/examples/11_crawl.py b/examples/11_crawl.py index fbeb982..049d785 100644 --- a/examples/11_crawl.py +++ b/examples/11_crawl.py @@ -3,12 +3,12 @@ Demonstrates the Crawl client (`ZenRowsCrawlClient`): - `create(url, depth=..., include_patterns=..., output_format="html")` starts a crawl and returns at once, `running`. - - `wait(crawl_id)` blocks until it ends and returns the final `Crawl` - (a `failed` crawl is returned, with `error` set). - - `iter_results(crawl_id)` pages through the kept URLs. - - `get_content(crawl_id, result)` reads one kept URL's HTML. - - `download(crawl_id, path)` saves every result as NDJSON and - returns the path and the crawl's status. + - `wait(crawl_id)` polls until it ends, or 600 s pass, and returns + the `Crawl` (still `running` on timeout; `error` set when `failed`). + - `results(crawl_id)` pages through the kept URLs. + - `content(crawl_id, result)` reads one kept URL's HTML. + - `download(crawl_id)` reads every result as NDJSON, with the crawl's + status; the example saves the lines to a file. Run with: export ZENROWS_API_KEY=zr_... @@ -16,6 +16,7 @@ """ import argparse +import json import os from zenrows import ZenRowsCrawlClient @@ -57,16 +58,20 @@ def main() -> None: ) if crawl.status is CrawlStatus.FAILED and crawl.error: raise SystemExit(f"crawl failed: {crawl.error.code.value}: {crawl.error.detail}") + if crawl.status is CrawlStatus.RUNNING: + print("still running: the results below are what it has kept so far") - for result in client.iter_results(crawl.crawl_id): + for result in client.results(crawl.crawl_id): status = result.content_status.value if result.content_status else "-" print(f" [{status}] {result.url}") if result.content_id: - html = client.get_content(crawl.crawl_id, result) + html = client.content(crawl.crawl_id, result) print(f" {len(html)} chars of HTML") - saved = client.download(crawl.crawl_id, args.out) - print(f"wrote {saved.path}") + with client.download(crawl.crawl_id) as download, open(args.out, "w") as f: + for line in download: + f.write(json.dumps(line.model_dump(mode="json", exclude_none=True)) + "\n") + print(f"wrote {args.out} (crawl {download.status.value if download.status else '-'})") if __name__ == "__main__": diff --git a/examples/README.md b/examples/README.md index 7f102d6..555f5b4 100644 --- a/examples/README.md +++ b/examples/README.md @@ -28,7 +28,7 @@ uv run python examples/02_download_to_dir.py --job-id 01J... | `08_download_all_results.py` | `download_all_results()` — server-side export zip of a whole run | | `09_scheduled_jobs.py` | scheduled cadences (`Rate`/`Calendar`/`At`), `pause`/`resume`/`update_schedule`, per-fire runs, `retry_failed` | | `10_lightweight_handles.py` | `client.job(id)` / `client.run(id, rid)` — act on a known id without a GET (`.data` lazily fetches) | -| `11_crawl.py` | Crawl: `create` → `wait` → `iter_results` + `get_content` → `download` (NDJSON) | +| `11_crawl.py` | Crawl: `create` → `wait` → `results` + `content` → `download` (NDJSON) | The samples are deliberately small — each demonstrates one feature end-to-end so they double as living documentation. diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py index 8bfc28f..15f1448 100644 --- a/src/zenrows/crawl/__init__.py +++ b/src/zenrows/crawl/__init__.py @@ -4,7 +4,7 @@ What's where: - `client.ZenRowsCrawlClient` — the typed facade: create, get, list, - stop, contents, download, and a `wait` helper. + results, content, download, stop, and wait. - `models` — hand-written pydantic v2 response models with open enums. - `errors.CrawlAPIError` — RFC 9457 problem+json mapping. @@ -12,11 +12,9 @@ without it a crawl returns URLs only. """ -from zenrows.batch._waiters import WaiterTimeout from zenrows.crawl.client import ( DEFAULT_BASE_URL, CrawlDownload, - CrawlDownloadFile, ZenRowsCrawlClient, ) from zenrows.crawl.errors import CRAWL_NOT_ENABLED, CrawlAPIError @@ -44,7 +42,6 @@ "Crawl", "CrawlAPIError", "CrawlDownload", - "CrawlDownloadFile", "CrawlErrorCode", "CrawlList", "CrawlResult", @@ -55,6 +52,5 @@ "OutputFormat", "RunError", "StopReason", - "WaiterTimeout", "ZenRowsCrawlClient", ] diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 6fca9c0..5d26533 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -9,7 +9,7 @@ crawl = client.create("https://example.com/shop/", depth=1, include_patterns=["/product/"]) client.wait(crawl.crawl_id) - for result in client.iter_results(crawl.crawl_id): + for result in client.results(crawl.crawl_id): print(result.url) Requests go through the SDK's shared HTTP transport (auth header, @@ -18,17 +18,16 @@ import json import logging +import random +import time from collections.abc import Callable, Iterator from contextlib import ExitStack -from dataclasses import dataclass -from pathlib import Path from typing import Any, Literal import httpx from zenrows.__version__ import __version__ from zenrows.batch._transport import _RETRYABLE_STATUSES, _Transport -from zenrows.batch._waiters import poll_until from zenrows.crawl.errors import CrawlAPIError from zenrows.crawl.models import ( Crawl, @@ -48,6 +47,10 @@ # 429 `too_many_crawls` is a capacity limit, not a transient failure. _CREATE_RETRY_STATUSES = _RETRYABLE_STATUSES - {429} +_POLL_INTERVAL = 2.0 +_POLL_BACKOFF = 1.5 +_MAX_POLL_INTERVAL = 15.0 + class CrawlDownload: """A crawl's NDJSON download, open and ready to read. @@ -82,15 +85,6 @@ def __exit__(self, *exc: object) -> None: self.close() -@dataclass(slots=True, frozen=True) -class CrawlDownloadFile: - """A crawl's NDJSON download, saved to `path`. `status` is as on - `CrawlDownload`.""" - - path: Path - status: CrawlStatus | None - - class ZenRowsCrawlClient: """Synchronous, typed client for the ZenRows Crawl API (Beta). @@ -175,7 +169,7 @@ def create( an include pattern (if any) and no exclude pattern. A crawl stays on the start URL's registrable domain; subdomains count. - `output_format="html"`: also fetch every kept URL's page, - readable with `get_content` / `download`. Omit for URLs only. + readable with `content` / `download`. Omit for URLs only. - `idempotency_key`: a retry with the same key and body answers with the crawl the first request created. @@ -217,52 +211,43 @@ def get( """`GET /crawls/{crawl_id}` — the crawl and one page of its results (`limit` 1 to 10,000, server default 1,000). Pass the page's `next_cursor` as `cursor` for the URLs kept since. For - every result, prefer `iter_results`.""" + every result, prefer `results`.""" return CrawlWithResults.model_validate( self._t.request_json( "GET", f"/crawls/{crawl_id}", params={"cursor": cursor, "limit": limit} ) ) - def list_crawls( + def list( self, *, cursor: str | None = None, limit: int | None = None, ) -> CrawlList: """`GET /crawls` — one page of the account's crawls, newest - first (`limit` 1 to 100, server default 20), without results. For - most uses prefer the auto-paginating `iter_crawls`.""" + first (`limit` 1 to 100, server default 20), without results. + Pass the page's `next_cursor` as `cursor` for the next page; it + is None on the last page.""" return CrawlList.model_validate( self._t.request_json("GET", "/crawls", params={"cursor": cursor, "limit": limit}) ) - def iter_crawls(self, *, page_size: int | None = None) -> Iterator[Crawl]: - """Auto-paginate `list_crawls`, newest first.""" - cursor: str | None = None - while True: - page = self.list_crawls(cursor=cursor, limit=page_size) - yield from page.crawls - cursor = page.next_cursor - if not cursor: - return - - def iter_results( + def results( self, crawl_id: str, *, - page_size: int | None = None, + limit: int | None = None, ) -> Iterator[CrawlResult]: """Yield the URLs the crawl has kept, in order, each once. - Follows `next_cursor` and returns at the first empty page or - null cursor. It does not poll: on a running crawl it yields - only what the crawl has kept so far. Call `wait` first to read - every result. + Follows `next_cursor`, `limit` URLs per request, and returns at + the first empty page or null cursor. It does not poll: on a + running crawl it yields only what the crawl has kept so far. + Call `wait` first to read every result. """ cursor: str | None = None while True: - page = self.get(crawl_id, cursor=cursor, limit=page_size) + page = self.get(crawl_id, cursor=cursor, limit=limit) yield from page.results if not page.results or page.next_cursor is None: return @@ -279,7 +264,7 @@ def stop(self, crawl_id: str) -> CrawlStop: # ===== contents ===== - def get_content(self, crawl_id: str, content: str | CrawlResult) -> str: + def content(self, crawl_id: str, content: str | CrawlResult) -> str: """`GET /crawls/{crawl_id}/contents/{content_id}` — one kept URL's page, as HTML text. @@ -292,14 +277,14 @@ def get_content(self, crawl_id: str, content: str | CrawlResult) -> str: content_id = content.content_id if not content_id: raise ValueError( - f"get_content: {content.url} has no content_url " + f"content: {content.url} has no content_url " f"(content_status={content.content_status})" ) else: content_id = content return self._t.request("GET", f"/crawls/{crawl_id}/contents/{content_id}").text - def iter_download(self, crawl_id: str) -> CrawlDownload: + def download(self, crawl_id: str) -> CrawlDownload: """`GET /crawls/{crawl_id}/download` — open the NDJSON file and return it as a `CrawlDownload`: iterate it for parsed `DownloadLine`s (url, content_status, content), and read its @@ -312,61 +297,27 @@ def iter_download(self, crawl_id: str) -> CrawlDownload: response = stack.enter_context(self._t.stream("GET", f"/crawls/{crawl_id}/download")) return CrawlDownload(response, stack.close) - def download( - self, - crawl_id: str, - target_path: str | Path, - *, - chunk_size: int = 64 * 1024, - ) -> CrawlDownloadFile: - """`GET /crawls/{crawl_id}/download` — stream the NDJSON file - (one JSON object per result) to `target_path`. Returns its - `path` and the crawl's `status` when the file was read. - - On a running crawl the file holds what the crawl has kept so - far, and `status` is `running`; call `wait` first for all of it. - """ - target = Path(target_path) - target.parent.mkdir(parents=True, exist_ok=True) - with ( - self._t.stream("GET", f"/crawls/{crawl_id}/download") as response, - target.open("wb") as f, - ): - for chunk in response.iter_bytes(chunk_size): - f.write(chunk) - return CrawlDownloadFile(path=target, status=_crawl_status(response)) - # ===== waiter ===== - def wait( - self, - crawl_id: str, - *, - timeout: float = 600.0, - poll_interval: float = 2.0, - max_poll_interval: float = 15.0, - ) -> Crawl: - """Block until the crawl ends (`completed`, `stopped` or - `failed`) and return it, polling with jittered exponential - backoff from `poll_interval` up to `max_poll_interval` seconds. + def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl: + """Poll the crawl until it ends (`completed`, `stopped` or + `failed`) or `timeout` seconds pass, and return it. A `failed` crawl is returned, not raised: read `error` on it. - Raises `WaiterTimeout` after `timeout` seconds; the crawl keeps - running (call `stop` to end it). + On timeout the crawl is returned still `running`; call `wait` + again to keep waiting, or `stop` to end it. """ - - def fetch() -> Crawl: + deadline = time.monotonic() + timeout + interval = _POLL_INTERVAL + while True: # limit=1 keeps each poll cheap; results are not returned. page = self.get(crawl_id, limit=1) - return Crawl.model_validate(page.model_dump(exclude={"results", "next_cursor"})) - - return poll_until( - fetch=fetch, - is_done=lambda crawl: crawl.is_terminal, - timeout=timeout, - initial_interval=poll_interval, - max_interval=max_poll_interval, - ) + crawl = Crawl.model_validate(page.model_dump(exclude={"results", "next_cursor"})) + remaining = deadline - time.monotonic() + if crawl.is_terminal or remaining <= 0: + return crawl + time.sleep(min(interval * random.uniform(0.8, 1.2), remaining)) + interval = min(interval * _POLL_BACKOFF, _MAX_POLL_INTERVAL) def _crawl_status(response: httpx.Response) -> CrawlStatus | None: @@ -378,6 +329,5 @@ def _crawl_status(response: httpx.Response) -> CrawlStatus | None: "DEFAULT_BASE_URL", "DEFAULT_USER_AGENT", "CrawlDownload", - "CrawlDownloadFile", "ZenRowsCrawlClient", ] diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index d6a07e2..ec2743d 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -128,7 +128,7 @@ class CrawlResult(BaseModel): content_url: str | None = None """The page's path (e.g. `/v1/crawls/c_x/contents/ct_y`), present once `content_status` is `fetched`. Pass the result itself to - `ZenRowsCrawlClient.get_content`.""" + `ZenRowsCrawlClient.content`.""" @property def content_id(self) -> str | None: diff --git a/tests/e2e/test_crawl_e2e.py b/tests/e2e/test_crawl_e2e.py index 8357507..2c00b0f 100644 --- a/tests/e2e/test_crawl_e2e.py +++ b/tests/e2e/test_crawl_e2e.py @@ -10,7 +10,6 @@ reads it back every way the client can. """ -import json import os import time @@ -63,7 +62,7 @@ def create_when_a_slot_frees(client: ZenRowsCrawlClient): time.sleep(wait) -def test_crawl_end_to_end(client: ZenRowsCrawlClient, tmp_path): +def test_crawl_end_to_end(client: ZenRowsCrawlClient): created = create_when_a_slot_frees(client) print(f"created {created.crawl_id} status={created.status.value}") assert created.crawl_id @@ -76,7 +75,7 @@ def test_crawl_end_to_end(client: ZenRowsCrawlClient, tmp_path): ) assert crawl.status is CrawlStatus.COMPLETED, crawl.error - results = list(client.iter_results(crawl.crawl_id)) + results = list(client.results(crawl.crawl_id)) print(f"results: {len(results)}") assert results if INCLUDE: @@ -84,24 +83,19 @@ def test_crawl_end_to_end(client: ZenRowsCrawlClient, tmp_path): fetched = [r for r in results if r.content_id] assert fetched, "no result has a fetched page" - html = client.get_content(crawl.crawl_id, fetched[0]) + html = client.content(crawl.crawl_id, fetched[0]) print(f"content of {fetched[0].url}: {len(html)} chars") assert "product" - assert client.get_content("c_1", "ct_a") == "product" + assert client.content("c_1", result) == "product" + assert client.content("c_1", "ct_a") == "product" assert route.call_count == 2 -def test_get_content_without_content_url_raises(client: ZenRowsCrawlClient): +def test_content_without_content_url_raises(client: ZenRowsCrawlClient): with pytest.raises(ValueError, match="content_url"): - client.get_content("c_1", CrawlResult(url="https://a", content_status="pending")) # type: ignore[arg-type] + client.content("c_1", CrawlResult(url="https://a", content_status="pending")) # type: ignore[arg-type] NDJSON = ( @@ -330,7 +328,7 @@ def test_get_content_without_content_url_raises(client: ZenRowsCrawlClient): @respx.mock -def test_iter_download_parses_lines_and_status(client: ZenRowsCrawlClient): +def test_download_parses_lines_and_status(client: ZenRowsCrawlClient): respx.get(f"{BASE_URL}/crawls/c_1/download").mock( return_value=Response( 200, @@ -338,7 +336,7 @@ def test_iter_download_parses_lines_and_status(client: ZenRowsCrawlClient): content=NDJSON, ) ) - download = client.iter_download("c_1") + download = client.download("c_1") assert download.status is CrawlStatus.COMPLETED lines = list(download) assert [line.url for line in lines] == ["https://a", "https://b", "https://c"] @@ -348,29 +346,18 @@ def test_iter_download_parses_lines_and_status(client: ZenRowsCrawlClient): @respx.mock -def test_iter_download_without_status_header(client: ZenRowsCrawlClient): +def test_download_without_status_header(client: ZenRowsCrawlClient): respx.get(f"{BASE_URL}/crawls/c_1/download").mock(return_value=Response(200, content=b"")) - with client.iter_download("c_1") as download: + with client.download("c_1") as download: assert download.status is None assert list(download) == [] -@respx.mock -def test_download_writes_file_and_returns_status(client: ZenRowsCrawlClient, tmp_path): - respx.get(f"{BASE_URL}/crawls/c_1/download").mock( - return_value=Response(200, headers={"X-Crawl-Status": "running"}, content=NDJSON) - ) - target = tmp_path / "out" / "c_1.jsonl" - saved = client.download("c_1", target) - assert saved == CrawlDownloadFile(path=target, status=CrawlStatus.RUNNING) - assert saved.path.read_bytes() == NDJSON - - @respx.mock def test_download_maps_errors(client: ZenRowsCrawlClient): respx.get(f"{BASE_URL}/crawls/nope/download").mock(return_value=problem(404, "crawl_not_found")) with pytest.raises(CrawlAPIError) as exc_info: - client.iter_download("nope") + client.download("nope") assert exc_info.value.code == "crawl_not_found" @@ -495,7 +482,7 @@ def test_wait_polls_until_terminal(client: ZenRowsCrawlClient, no_sleep): ] ) - crawl = client.wait("c_1", poll_interval=1.0) + crawl = client.wait("c_1") assert crawl.status is CrawlStatus.COMPLETED assert not hasattr(crawl, "results") @@ -505,12 +492,42 @@ def test_wait_polls_until_terminal(client: ZenRowsCrawlClient, no_sleep): @respx.mock -def test_wait_times_out(client: ZenRowsCrawlClient, no_sleep, monkeypatch): - respx.get(f"{BASE_URL}/crawls/c_1").mock( +def test_wait_returns_running_crawl_on_timeout(client: ZenRowsCrawlClient, monkeypatch): + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( return_value=Response(200, json=crawl_body(results=[], next_cursor="k")) ) - clock = iter(range(0, 1000, 10)) - monkeypatch.setattr("zenrows.batch._waiters.time.monotonic", lambda: next(clock)) + now = [0.0] + slept: list[float] = [] + + def sleep(seconds: float) -> None: + slept.append(seconds) + now[0] += seconds + + monkeypatch.setattr("time.monotonic", lambda: now[0]) + monkeypatch.setattr("time.sleep", sleep) + + crawl = client.wait("c_1", timeout=25) + + assert crawl.status is CrawlStatus.RUNNING + assert route.call_count == len(slept) + 1 + assert sum(slept) == pytest.approx(25) + assert max(slept) <= 15.0 * 1.2 - with pytest.raises(WaiterTimeout): - client.wait("c_1", timeout=25) + +@respx.mock +def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep): + respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response( + 200, + json=crawl_body( + status="failed", + error={"code": "no_items_found", "detail": "No URL matched."}, + results=[], + next_cursor=None, + ), + ) + ) + crawl = client.wait("c_1") + assert crawl.status is CrawlStatus.FAILED + assert crawl.error is not None and crawl.error.code is CrawlErrorCode.NO_ITEMS_FOUND + assert no_sleep == [] From b3f38ad4fff70e72307ba346cecb043cf4312279 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 9 Oct 2026 17:48:53 +0200 Subject: [PATCH 04/20] refactor(crawl): accept content_url and expose download lines - content() accepts a result, its content_url, or a content id. - CrawlDownload exposes its parsed lines as the lines property. - Rename CRAWL_NOT_ENABLED to CRAWL_NOT_ENABLED_CODE and remove CrawlAPIError.not_enabled. - Remove CrawlResult.content_id. - Remove the runtime check on output_format; the type hint stays. - Document the client options base_url, retries and timeout. --- README.md | 24 ++++++++++++++++-------- examples/11_crawl.py | 8 ++++---- src/zenrows/crawl/__init__.py | 4 ++-- src/zenrows/crawl/client.py | 30 ++++++++++++++++-------------- src/zenrows/crawl/errors.py | 13 ++++--------- src/zenrows/crawl/models.py | 7 ------- tests/e2e/test_crawl_e2e.py | 4 ++-- tests/test_crawl_client.py | 22 +++++++++------------- 8 files changed, 53 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 934b245..11bc3c9 100644 --- a/README.md +++ b/README.md @@ -532,6 +532,14 @@ running crawl it yields only the URLs kept so far. Call `wait` first to read every result. `results(crawl_id, limit=...)` sets the page size of each request. `get(crawl_id, cursor=..., limit=...)` returns one raw page. +The client takes these options: `base_url` (default +`https://api.zenrows.com/v1`), `retries` (default 3; set 0 to disable) +and `timeout` (seconds per request, default 30): + +```python +client = ZenRowsCrawlClient(api_key, base_url="https://api.zenrows.com/v1", retries=3, timeout=30) +``` + ### Read the pages Pass `output_format="html"` to also fetch the page of every kept URL: @@ -541,19 +549,19 @@ crawl = client.create(start_url, depth=1, output_format="html") client.wait(crawl.crawl_id) for result in client.results(crawl.crawl_id): - if result.content_id: # content_status == fetched + if result.content_url: # content_status == fetched html = client.content(crawl.crawl_id, result) # Or everything in one NDJSON file: {"url", "content_status", "content"} per line. with client.download(crawl.crawl_id) as download: # streamed, parsed print(download.status) # from X-Crawl-Status - for line in download: + for line in download.lines: print(line.url, line.content_status) ``` -`content` takes a result or its content id. `download` returns a -`CrawlDownload`: its `status` is the crawl's status when the file was -read, and iterating it yields parsed lines. On a `running` crawl the +`content` takes a result, its `content_url`, or its content id. +`download` returns a `CrawlDownload`: its `status` is the crawl's status +when the file was read, and its `lines` yields parsed lines. On a `running` crawl the file holds only the results kept so far. ### List and stop crawls @@ -572,12 +580,12 @@ client.stop(crawl_id) # idempotent; read final counts with get Every non-2xx raises `CrawlAPIError`. Branch on `status_code` and `code`: ```python -from zenrows.crawl import CrawlAPIError +from zenrows.crawl import CRAWL_NOT_ENABLED_CODE, CrawlAPIError try: client.create(start_url, depth=1) except CrawlAPIError as exc: - if exc.not_enabled: # 403 REQS008 + if exc.code == CRAWL_NOT_ENABLED_CODE: # 403 REQS008 print("Crawl is not enabled for this account") elif exc.code == "too_many_crawls": # 429: active jobs limit reached print("retry in", exc.retry_after, "s") @@ -590,7 +598,7 @@ except CrawlAPIError as exc: | Status | `code` | What to do | |---|---|---| | 400 | `invalid_request`, `unknown_parameter`, `invalid_cursor` | Fix the request. Do not retry it as is. | -| 403 | `REQS008` | Crawl is not enabled for the account (`exc.not_enabled`). | +| 403 | `REQS008` | Crawl is not enabled for the account (`CRAWL_NOT_ENABLED_CODE`). | | 404 | `crawl_not_found`, `content_not_found` | Check the id. | | 409 | `idempotency_request_in_flight` | Retry after the first request with this key finishes. | | 422 | `invalid_parameter`, `invalid_start_url`, `domain_not_allowed` | Fix the request. Do not retry it as is. | diff --git a/examples/11_crawl.py b/examples/11_crawl.py index 049d785..8965534 100644 --- a/examples/11_crawl.py +++ b/examples/11_crawl.py @@ -20,7 +20,7 @@ import os from zenrows import ZenRowsCrawlClient -from zenrows.crawl import CrawlAPIError, CrawlStatus +from zenrows.crawl import CRAWL_NOT_ENABLED_CODE, CrawlAPIError, CrawlStatus def main() -> None: @@ -45,7 +45,7 @@ def main() -> None: output_format="html", ) except CrawlAPIError as exc: - if exc.not_enabled: + if exc.code == CRAWL_NOT_ENABLED_CODE: raise SystemExit("Crawl is not enabled for this account.") from exc raise print(f"started {crawl.crawl_id}") @@ -64,12 +64,12 @@ def main() -> None: for result in client.results(crawl.crawl_id): status = result.content_status.value if result.content_status else "-" print(f" [{status}] {result.url}") - if result.content_id: + if result.content_url: html = client.content(crawl.crawl_id, result) print(f" {len(html)} chars of HTML") with client.download(crawl.crawl_id) as download, open(args.out, "w") as f: - for line in download: + for line in download.lines: f.write(json.dumps(line.model_dump(mode="json", exclude_none=True)) + "\n") print(f"wrote {args.out} (crawl {download.status.value if download.status else '-'})") diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py index 15f1448..c89c229 100644 --- a/src/zenrows/crawl/__init__.py +++ b/src/zenrows/crawl/__init__.py @@ -17,7 +17,7 @@ CrawlDownload, ZenRowsCrawlClient, ) -from zenrows.crawl.errors import CRAWL_NOT_ENABLED, CrawlAPIError +from zenrows.crawl.errors import CRAWL_NOT_ENABLED_CODE, CrawlAPIError from zenrows.crawl.models import ( ContentStatus, Coverage, @@ -35,7 +35,7 @@ ) __all__ = [ - "CRAWL_NOT_ENABLED", + "CRAWL_NOT_ENABLED_CODE", "DEFAULT_BASE_URL", "ContentStatus", "Coverage", diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 5d26533..b08e838 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -57,8 +57,8 @@ class CrawlDownload: `status` is the crawl's status when the file was read (`X-Crawl-Status`): `running` means the file holds only what the - crawl has kept so far. Iterate it once for the parsed lines; the - connection closes when the iteration ends. Use it as a context + crawl has kept so far. Iterate `lines` once for the parsed lines; + the connection closes when the iteration ends. Use it as a context manager to close it without reading to the end. """ @@ -67,7 +67,11 @@ def __init__(self, response: httpx.Response, close: Callable[[], None]): self._response = response self._close = close - def __iter__(self) -> Iterator[DownloadLine]: + @property + def lines(self) -> Iterator[DownloadLine]: + return self._lines() + + def _lines(self) -> Iterator[DownloadLine]: try: for line in self._response.iter_lines(): if line.strip(): @@ -94,7 +98,7 @@ class ZenRowsCrawlClient: Every non-2xx raises `CrawlAPIError`; branch on its `status_code` and `code`. An account without Crawl enabled gets 403 `REQS008` - (`CrawlAPIError.not_enabled`). + (`CRAWL_NOT_ENABLED_CODE`). """ def __init__( @@ -178,8 +182,6 @@ def create( Batch jobs, `create` raises 429 `too_many_crawls` without retrying; retry after the error's `retry_after` seconds. """ - if output_format not in (None, "html"): - raise ValueError(f"create: output_format must be 'html' or None, not {output_format!r}") body: dict[str, Any] = {"url": url, "depth": depth} optional: dict[str, Any] = { "max_items": max_items, @@ -269,24 +271,24 @@ def content(self, crawl_id: str, content: str | CrawlResult) -> str: URL's page, as HTML text. `content` is a `CrawlResult` whose `content_status` is - `fetched`, or its content id (the last segment of its - `content_url`). Only crawls created with an `output_format` - have contents; anything else is 404 `content_not_found`. + `fetched`, its `content_url`, or its content id (the last + segment of the `content_url`). Only crawls created with an + `output_format` have contents; anything else is 404 + `content_not_found`. """ if isinstance(content, CrawlResult): - content_id = content.content_id - if not content_id: + if not content.content_url: raise ValueError( f"content: {content.url} has no content_url " f"(content_status={content.content_status})" ) - else: - content_id = content + content = content.content_url + content_id = content.rstrip("/").rsplit("/", 1)[-1] return self._t.request("GET", f"/crawls/{crawl_id}/contents/{content_id}").text def download(self, crawl_id: str) -> CrawlDownload: """`GET /crawls/{crawl_id}/download` — open the NDJSON file and - return it as a `CrawlDownload`: iterate it for parsed + return it as a `CrawlDownload`: iterate its `lines` for parsed `DownloadLine`s (url, content_status, content), and read its `status`. diff --git a/src/zenrows/crawl/errors.py b/src/zenrows/crawl/errors.py index d81f67b..6c4a660 100644 --- a/src/zenrows/crawl/errors.py +++ b/src/zenrows/crawl/errors.py @@ -11,7 +11,7 @@ # The code of the 403 answer to a key whose account does not have # Crawl enabled. -CRAWL_NOT_ENABLED = "REQS008" +CRAWL_NOT_ENABLED_CODE = "REQS008" class CrawlAPIError(Exception): @@ -40,8 +40,8 @@ def __init__( self.raw = raw self.code: str | None = problem.code if problem else None self.detail: str | None = problem.detail if problem else None - if problem and self.code == CRAWL_NOT_ENABLED: - msg = f"{status_code} Crawl is not enabled for this account ({CRAWL_NOT_ENABLED})" + ( + if problem and self.code == CRAWL_NOT_ENABLED_CODE: + msg = f"{status_code} Crawl is not enabled for this account ({self.code})" + ( f": {problem.detail}" if problem.detail else "" ) elif problem: @@ -51,11 +51,6 @@ def __init__( msg = f"{status_code} (no problem body)" super().__init__(msg) - @property - def not_enabled(self) -> bool: - """True when the account does not have Crawl enabled (403 REQS008).""" - return self.status_code == 403 and self.code == CRAWL_NOT_ENABLED - @classmethod def from_response(cls, response: httpx.Response) -> "CrawlAPIError": try: @@ -70,4 +65,4 @@ def from_response(cls, response: httpx.Response) -> "CrawlAPIError": ) -__all__ = ["CRAWL_NOT_ENABLED", "CrawlAPIError"] +__all__ = ["CRAWL_NOT_ENABLED_CODE", "CrawlAPIError"] diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index ec2743d..da42f1b 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -130,13 +130,6 @@ class CrawlResult(BaseModel): once `content_status` is `fetched`. Pass the result itself to `ZenRowsCrawlClient.content`.""" - @property - def content_id(self) -> str | None: - """The content id, the last segment of `content_url`.""" - if not self.content_url: - return None - return self.content_url.rstrip("/").rsplit("/", 1)[-1] - class CrawlWithResults(Crawl): """A crawl and one page of the URLs it has kept (`GET /crawls/{id}`). diff --git a/tests/e2e/test_crawl_e2e.py b/tests/e2e/test_crawl_e2e.py index 2c00b0f..0e95212 100644 --- a/tests/e2e/test_crawl_e2e.py +++ b/tests/e2e/test_crawl_e2e.py @@ -81,7 +81,7 @@ def test_crawl_end_to_end(client: ZenRowsCrawlClient): if INCLUDE: assert all(INCLUDE in r.url for r in results), [r.url for r in results] - fetched = [r for r in results if r.content_id] + fetched = [r for r in results if r.content_url] assert fetched, "no result has a fetched page" html = client.content(crawl.crawl_id, fetched[0]) print(f"content of {fetched[0].url}: {len(html)} chars") @@ -89,7 +89,7 @@ def test_crawl_end_to_end(client: ZenRowsCrawlClient): download = client.download(crawl.crawl_id) assert download.status is CrawlStatus.COMPLETED - lines = list(download) + lines = list(download.lines) print(f"download lines: {len(lines)}") assert len(lines) == len(results) diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 21d2226..e2f88b5 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -15,6 +15,7 @@ from zenrows import ZenRowsCrawlClient from zenrows.batch._open_enum import is_unknown from zenrows.crawl import ( + CRAWL_NOT_ENABLED_CODE, ContentStatus, CrawlAPIError, CrawlErrorCode, @@ -140,11 +141,6 @@ def test_create_sends_every_option_and_idempotency_key(client: ZenRowsCrawlClien assert "discovery" not in body -def test_create_output_format_accepts_only_html(client: ZenRowsCrawlClient): - with pytest.raises(ValueError, match="output_format"): - client.create("https://example.com/", depth=1, output_format="json") # type: ignore[arg-type] - - # ----- get / list / stop ----- @@ -182,7 +178,7 @@ def test_get_parses_results_and_open_enums(client: ZenRowsCrawlClient): assert is_unknown(page.output_format) # type: ignore[arg-type] assert page.next_cursor is None assert page.results[0].content_status is ContentStatus.FETCHED - assert page.results[0].content_id == "ct_a" + assert page.results[0].content_url == "/v1/crawls/c_1/contents/ct_a" unknown = page.results[1].content_status assert unknown is not None and is_unknown(unknown) and unknown.value == "archived" @@ -300,7 +296,7 @@ def test_results_returns_at_first_empty_page_of_running_crawl(client: ZenRowsCra @respx.mock -def test_content_accepts_result_or_id(client: ZenRowsCrawlClient): +def test_content_accepts_result_url_or_id(client: ZenRowsCrawlClient): route = respx.get(f"{BASE_URL}/crawls/c_1/contents/ct_a").mock( return_value=Response( 200, headers={"Content-Type": "text/html"}, text="product" @@ -311,8 +307,9 @@ def test_content_accepts_result_or_id(client: ZenRowsCrawlClient): ) # type: ignore[arg-type] assert client.content("c_1", result) == "product" + assert client.content("c_1", "/v1/crawls/c_1/contents/ct_a") == "product" assert client.content("c_1", "ct_a") == "product" - assert route.call_count == 2 + assert route.call_count == 3 def test_content_without_content_url_raises(client: ZenRowsCrawlClient): @@ -338,7 +335,7 @@ def test_download_parses_lines_and_status(client: ZenRowsCrawlClient): ) download = client.download("c_1") assert download.status is CrawlStatus.COMPLETED - lines = list(download) + lines = list(download.lines) assert [line.url for line in lines] == ["https://a", "https://b", "https://c"] assert lines[0].content == "a" assert lines[1].content_status is ContentStatus.FAILED @@ -350,7 +347,7 @@ def test_download_without_status_header(client: ZenRowsCrawlClient): respx.get(f"{BASE_URL}/crawls/c_1/download").mock(return_value=Response(200, content=b"")) with client.download("c_1") as download: assert download.status is None - assert list(download) == [] + assert list(download.lines) == [] @respx.mock @@ -373,7 +370,7 @@ def test_not_found_maps_to_crawl_api_error(client: ZenRowsCrawlClient): assert err.status_code == 404 assert err.code == "crawl_not_found" assert err.detail == "detail for crawl_not_found" - assert not err.not_enabled + assert err.code != CRAWL_NOT_ENABLED_CODE @respx.mock @@ -409,8 +406,7 @@ def test_crawl_not_enabled_has_clear_message(client: ZenRowsCrawlClient): with pytest.raises(CrawlAPIError) as exc_info: client.create("https://example.com/", depth=1) err = exc_info.value - assert err.not_enabled - assert err.code == "REQS008" + assert err.code == CRAWL_NOT_ENABLED_CODE assert "Crawl is not enabled for this account" in str(err) From 7072517e73ceb0515d77a44997825490d85446a9 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 9 Oct 2026 17:59:22 +0200 Subject: [PATCH 05/20] refactor(crawl): remove is_terminal and the DEFAULT_BASE_URL export Wait checks the status against running directly. The default base URL stays internal to the client. --- src/zenrows/crawl/__init__.py | 2 -- src/zenrows/crawl/client.py | 9 ++++----- src/zenrows/crawl/models.py | 5 ----- tests/test_crawl_client.py | 2 -- 4 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py index c89c229..59f2a6c 100644 --- a/src/zenrows/crawl/__init__.py +++ b/src/zenrows/crawl/__init__.py @@ -13,7 +13,6 @@ """ from zenrows.crawl.client import ( - DEFAULT_BASE_URL, CrawlDownload, ZenRowsCrawlClient, ) @@ -36,7 +35,6 @@ __all__ = [ "CRAWL_NOT_ENABLED_CODE", - "DEFAULT_BASE_URL", "ContentStatus", "Coverage", "Crawl", diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index b08e838..1f203b6 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -39,7 +39,7 @@ DownloadLine, ) -DEFAULT_BASE_URL = "https://api.zenrows.com/v1" +_DEFAULT_BASE_URL = "https://api.zenrows.com/v1" DEFAULT_USER_AGENT = f"zenrows-crawl-python/{__version__}" _log = logging.getLogger("zenrows.crawl.transport") @@ -112,7 +112,7 @@ def __init__( user_agent: str = DEFAULT_USER_AGENT, httpx_args: dict[str, Any] | None = None, ): - """`base_url` defaults to `DEFAULT_BASE_URL`. `retries` bounds + """`base_url` defaults to `https://api.zenrows.com/v1`. `retries` bounds automatic retries of transient failures (HTTP 429/502/503/504 and network errors) on idempotent requests — GETs, and a `create` that carries an `idempotency_key` (never on 429 for @@ -121,7 +121,7 @@ def __init__( if not api_key: raise ValueError("ZenRowsCrawlClient: api_key is required.") self._t = _Transport( - base_url=base_url or DEFAULT_BASE_URL, + base_url=base_url or _DEFAULT_BASE_URL, api_key=api_key, user_agent=user_agent, timeout=timeout, @@ -316,7 +316,7 @@ def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl: page = self.get(crawl_id, limit=1) crawl = Crawl.model_validate(page.model_dump(exclude={"results", "next_cursor"})) remaining = deadline - time.monotonic() - if crawl.is_terminal or remaining <= 0: + if crawl.status is not CrawlStatus.RUNNING or remaining <= 0: return crawl time.sleep(min(interval * random.uniform(0.8, 1.2), remaining)) interval = min(interval * _POLL_BACKOFF, _MAX_POLL_INTERVAL) @@ -328,7 +328,6 @@ def _crawl_status(response: httpx.Response) -> CrawlStatus | None: __all__ = [ - "DEFAULT_BASE_URL", "DEFAULT_USER_AGENT", "CrawlDownload", "ZenRowsCrawlClient", diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index da42f1b..eaea5d9 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -113,11 +113,6 @@ class Crawl(BaseModel): finished_at: datetime | None = None """When the crawl reached a terminal status. Absent while it runs.""" - @property - def is_terminal(self) -> bool: - """True once the crawl has ended (any status but `running`).""" - return self.status is not CrawlStatus.RUNNING - class CrawlResult(BaseModel): """One URL the crawl kept.""" diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index e2f88b5..0384a66 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -100,7 +100,6 @@ def test_create_sends_only_required_fields(client: ZenRowsCrawlClient): assert crawl.crawl_id == "c_1" assert crawl.status is CrawlStatus.RUNNING - assert not crawl.is_terminal sent = route.calls.last.request assert sent.headers["X-API-Key"] == API_KEY assert sent.headers["Content-Type"] == "application/json" @@ -173,7 +172,6 @@ def test_get_parses_results_and_open_enums(client: ZenRowsCrawlClient): assert route.calls.last.request.url.params["cursor"] == "cur_1" assert route.calls.last.request.url.params["limit"] == "50" assert page.status is CrawlStatus.COMPLETED - assert page.is_terminal assert page.stop_reason is StopReason.MAX_ITEMS assert is_unknown(page.output_format) # type: ignore[arg-type] assert page.next_cursor is None From 42b35bc276a55a5515098f766492c682e08e1a8d Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 9 Oct 2026 18:03:05 +0200 Subject: [PATCH 06/20] docs(crawl): say New, not Beta Customer-facing Crawl text now says New. The README Crawl section and the package doc carry one still-evolving line. --- README.md | 8 ++++---- examples/11_crawl.py | 2 +- src/zenrows/crawl/__init__.py | 5 +++-- src/zenrows/crawl/client.py | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 11bc3c9..7b92847 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ This package ships three clients: _(Private beta — [contact support](mailto:support@zenrows.com) for access.)_ - **`ZenRowsCrawlClient`** — the Crawl API client. Give it one start URL; it follows links and returns the URLs (and, optionally, the - pages) it finds behind it. _(Beta.)_ + pages) it finds behind it. _(New.)_ ## Table of Contents @@ -40,7 +40,7 @@ This package ships three clients: - [Act on an id without a GET](#act-on-an-id-without-a-get) - [Scheduled jobs & webhooks](#scheduled-jobs--webhooks) - [Error handling](#error-handling) -- [Quickstart — Crawl API (beta) (`ZenRowsCrawlClient`)](#quickstart--crawl-api-beta-zenrowscrawlclient) +- [Quickstart — Crawl API (new) (`ZenRowsCrawlClient`)](#quickstart--crawl-api-new-zenrowscrawlclient) - [Read the pages](#read-the-pages) - [List and stop crawls](#list-and-stop-crawls) - [Crawl errors](#crawl-errors) @@ -492,9 +492,9 @@ The full Batch surface (jobs, runs, tasks, results, content, history, file_inputs, HMAC keys) is reachable via methods on `ZenRowsBatchClient`. See `src/zenrows/batch/client.py` or `help(ZenRowsBatchClient)`. -## Quickstart — Crawl API (beta) (`ZenRowsCrawlClient`) +## Quickstart — Crawl API (new) (`ZenRowsCrawlClient`) -Crawl is in Beta: its API may still change. +Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. A crawl starts from one URL, follows the links on each page up to `depth` hops, and keeps the URLs that match your patterns, up to diff --git a/examples/11_crawl.py b/examples/11_crawl.py index 8965534..1832256 100644 --- a/examples/11_crawl.py +++ b/examples/11_crawl.py @@ -1,4 +1,4 @@ -"""11: Crawl a site → wait → read URLs and pages. Crawl is in Beta. +"""11: Crawl a site → wait → read URLs and pages. Demonstrates the Crawl client (`ZenRowsCrawlClient`): - `create(url, depth=..., include_patterns=..., output_format="html")` diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py index 59f2a6c..696a9d7 100644 --- a/src/zenrows/crawl/__init__.py +++ b/src/zenrows/crawl/__init__.py @@ -1,6 +1,7 @@ -"""Public surface for the ZenRows Crawl API (Beta). +"""Public surface for the ZenRows Crawl API (New). -Crawl is in Beta: its surface may still change. +Crawl is still evolving: new features are coming, limits may be tuned, +and the changelog announces each change. What's where: - `client.ZenRowsCrawlClient` — the typed facade: create, get, list, diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 1f203b6..d43b893 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -1,4 +1,4 @@ -"""ZenRowsCrawlClient — the typed facade over the Crawl API (Beta). +"""ZenRowsCrawlClient — the typed facade over the Crawl API (New). A crawl starts from one URL, follows links up to `depth` hops, and keeps the URLs that match its patterns, up to `max_items`. It runs as @@ -90,7 +90,7 @@ def __exit__(self, *exc: object) -> None: class ZenRowsCrawlClient: - """Synchronous, typed client for the ZenRows Crawl API (Beta). + """Synchronous, typed client for the ZenRows Crawl API (New). `api_key` is required: From 77be02c0024c973344a51288b45bbc42fa464a47 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 9 Oct 2026 21:35:13 +0200 Subject: [PATCH 07/20] fix(crawl): retry stop on 503 crawl_busy, honouring Retry-After A stop that collides with the crawl saving its own progress answers 503 crawl_busy with Retry-After. Nothing was saved and the crawl keeps running, and the stop is safe to repeat, but stop raised at once. stop now retries that answer up to `retries` times, waiting Retry-After seconds between tries, then raises it. Other errors, and other 503s, are raised as before. --- src/zenrows/crawl/client.py | 24 +++++++++++++++++-- tests/test_crawl_client.py | 47 +++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index d43b893..2b61a45 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -47,6 +47,10 @@ # 429 `too_many_crawls` is a capacity limit, not a transient failure. _CREATE_RETRY_STATUSES = _RETRYABLE_STATUSES - {429} +# 503 `crawl_busy` on stop: nothing was saved and the crawl is still +# running, so the stop is safe to repeat after `Retry-After`. +_CRAWL_BUSY = "crawl_busy" + _POLL_INTERVAL = 2.0 _POLL_BACKOFF = 1.5 _MAX_POLL_INTERVAL = 15.0 @@ -117,9 +121,11 @@ def __init__( and network errors) on idempotent requests — GETs, and a `create` that carries an `idempotency_key` (never on 429 for `create`). Retries use jittered exponential backoff and honor - `Retry-After`; set `retries=0` to disable.""" + `Retry-After`; set `retries=0` to disable. It also bounds the + retries of a `stop` answered 503 `crawl_busy`.""" if not api_key: raise ValueError("ZenRowsCrawlClient: api_key is required.") + self._retries = max(0, retries) self._t = _Transport( base_url=base_url or _DEFAULT_BASE_URL, api_key=api_key, @@ -261,8 +267,22 @@ def stop(self, crawl_id: str) -> CrawlStop: Idempotent: a crawl that already ended answers as it ended. The answer carries no counts; read the final coverage and results with `get` once pages already in flight have finished. + + A 503 `crawl_busy` means the stop was not saved and the crawl is + still running; `stop` retries it up to `retries` times, waiting + `Retry-After` seconds each time, then raises it. """ - return CrawlStop.model_validate(self._t.request_json("POST", f"/crawls/{crawl_id}/stop")) + attempt = 0 + while True: + try: + return CrawlStop.model_validate( + self._t.request_json("POST", f"/crawls/{crawl_id}/stop") + ) + except CrawlAPIError as err: + if err.status_code != 503 or err.code != _CRAWL_BUSY or attempt >= self._retries: + raise + time.sleep(err.retry_after if err.retry_after is not None else 1.0) + attempt += 1 # ===== contents ===== diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 0384a66..d48256e 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -248,6 +248,53 @@ def test_stop_posts_without_body(client: ZenRowsCrawlClient): assert route.calls.last.request.content == b"" +@respx.mock +def test_stop_retries_crawl_busy_after_retry_after(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + side_effect=[ + problem(503, "crawl_busy", headers={"Retry-After": "1"}), + Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}), + ] + ) + + stopped = client.stop("c_1") + + assert stopped.status is CrawlStatus.STOPPED + assert route.call_count == 2 + assert no_sleep == [1.0] + + +@respx.mock +def test_stop_surfaces_crawl_busy_after_retries(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + return_value=problem(503, "crawl_busy", headers={"Retry-After": "1"}) + ) + + with pytest.raises(CrawlAPIError) as exc_info: + client.stop("c_1") + + assert exc_info.value.status_code == 503 + assert exc_info.value.code == "crawl_busy" + assert route.call_count == 3 + assert no_sleep == [1.0, 1.0] + + +@respx.mock +def test_stop_does_not_retry_other_503(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + return_value=problem(503, "service_unavailable", headers={"Retry-After": "1"}) + ) + + with pytest.raises(CrawlAPIError): + client.stop("c_1") + + assert route.call_count == 1 + assert no_sleep == [] + + # ----- results scanner ----- From c5f167271dc63a6d8a1f072cec7e3c1428a0583d Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 9 Oct 2026 21:36:32 +0200 Subject: [PATCH 08/20] docs(crawl): list 503 crawl_busy in the crawl errors table --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7b92847..d05ad1a 100644 --- a/README.md +++ b/README.md @@ -604,6 +604,7 @@ except CrawlAPIError as exc: | 422 | `invalid_parameter`, `invalid_start_url`, `domain_not_allowed` | Fix the request. Do not retry it as is. | | 422 | `idempotency_key_reused` | Use a new key, or no key. Do not retry it as is. | | 429 | `too_many_crawls` | The account has reached its limit of active jobs (3 by default), shared with its Batch jobs. Retry after `exc.retry_after` seconds. `create` does not retry it. | +| 503 | `crawl_busy` | `stop` could not be saved yet; the crawl is still running. `stop` retries it up to `retries` times, waiting `exc.retry_after` seconds, then raises it. Repeating the stop is safe. | `code` is None when the response has no problem body with a code (for example a 502 from a proxy). A crawl that ended `failed` is not an error From 42605d9edfd5f6ad01f23c69a57a4157b4c89eef Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 13:27:10 +0200 Subject: [PATCH 09/20] fix(crawl): cap a Retry-After wait at 30 seconds, drop unusable values - A retry honored any Retry-After the server sent, so a value of 3600 made a call wait an hour. The shared transport now waits at most 30 s. - CrawlAPIError.retry_after is None for a negative, NaN or infinite Retry-After, so code that sleeps for it cannot crash. --- src/zenrows/batch/_transport.py | 13 ++++++++++--- src/zenrows/crawl/errors.py | 5 +++++ tests/test_crawl_client.py | 24 ++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/zenrows/batch/_transport.py b/src/zenrows/batch/_transport.py index 1c3e2e5..b29ccc7 100644 --- a/src/zenrows/batch/_transport.py +++ b/src/zenrows/batch/_transport.py @@ -12,6 +12,7 @@ import json import logging +import math import random import time from collections.abc import Callable, Iterator @@ -40,6 +41,9 @@ # capped at 10s. _BACKOFF_BASE_MS = 250 _BACKOFF_CAP_MS = 10_000 +# A `Retry-After` above this waits only this long, so a large server value +# cannot hang a call. +_RETRY_AFTER_CAP_MS = 30_000 _DEFAULT_RETRIES = 3 @@ -54,7 +58,8 @@ def _backoff_ms(attempt: int) -> float: def _retry_after_ms(response: httpx.Response) -> float | None: - """Parse a `Retry-After` header (delta-seconds only) to milliseconds.""" + """Parse a `Retry-After` header (delta-seconds only) to milliseconds, + at most `_RETRY_AFTER_CAP_MS`.""" raw = response.headers.get("Retry-After") if not raw: return None @@ -62,7 +67,9 @@ def _retry_after_ms(response: httpx.Response) -> float | None: secs = float(raw) except ValueError: return None - return secs * 1000 if secs >= 0 else None + if not math.isfinite(secs) or secs < 0: + return None + return min(secs * 1000, _RETRY_AFTER_CAP_MS) class _Transport: @@ -124,7 +131,7 @@ def _send( Retries `retry_statuses` (default `{429, 502, 503, 504}`) and transient network errors up to `retries` times, with jittered - exponential backoff (honoring `Retry-After` when present). Only + exponential backoff (honoring `Retry-After` up to 30s). Only idempotent requests are replayed — `GET`/`PUT`/`DELETE`/`HEAD`/ `OPTIONS`, plus `POST` when the caller supplied an `Idempotency-Key`. Our own timeouts (`httpx.TimeoutException`) diff --git a/src/zenrows/crawl/errors.py b/src/zenrows/crawl/errors.py index 6c4a660..aa32e87 100644 --- a/src/zenrows/crawl/errors.py +++ b/src/zenrows/crawl/errors.py @@ -5,6 +5,8 @@ `status_code` + `code`; display `detail`, don't parse it. """ +import math + import httpx from zenrows.batch.errors import ProblemDetail @@ -57,6 +59,9 @@ def from_response(cls, response: httpx.Response) -> "CrawlAPIError": retry_after: float | None = float(response.headers.get("Retry-After", "")) except ValueError: retry_after = None + # A negative, NaN or infinite value is no wait a caller can sleep for. + if retry_after is not None and (not math.isfinite(retry_after) or retry_after < 0): + retry_after = None return cls( status_code=response.status_code, problem=ProblemDetail.from_response(response), diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 0384a66..1a466ce 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -525,3 +525,27 @@ def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep): assert crawl.status is CrawlStatus.FAILED assert crawl.error is not None and crawl.error.code is CrawlErrorCode.NO_ITEMS_FOUND assert no_sleep == [] + + +@respx.mock +def test_retry_after_above_30_seconds_waits_30_seconds(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + respx.get(f"{BASE_URL}/crawls/c_1").mock( + side_effect=[ + problem(429, "rate_limited", headers={"Retry-After": "3600"}), + Response(200, json=crawl_body(status="completed", results=[], next_cursor=None)), + ] + ) + client.get("c_1") + assert no_sleep == [30.0] + + +@pytest.mark.parametrize("raw", ["-1", "nan", "inf", "soon"]) +@respx.mock +def test_an_unusable_retry_after_is_not_reported(client: ZenRowsCrawlClient, raw: str): + respx.post(f"{BASE_URL}/crawls").mock( + return_value=problem(429, "too_many_crawls", headers={"Retry-After": raw}) + ) + with pytest.raises(CrawlAPIError) as exc_info: + client.create("https://example.com/", depth=1) + assert exc_info.value.retry_after is None From fb66ab9d65d56335c6d581ebb454887f6690d912 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 13:59:31 +0200 Subject: [PATCH 10/20] feat(crawl): export CrawlAPIError from the zenrows package `from zenrows import CrawlAPIError` now works, next to ZenRowsCrawlClient. The README example now crawls the scrapingcourse.com demo shop, whose product URLs match /product/. --- README.md | 5 +++-- src/zenrows/__init__.py | 10 ++++++++-- tests/test_crawl_client.py | 7 +++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7b92847..465e94a 100644 --- a/README.md +++ b/README.md @@ -511,7 +511,7 @@ from zenrows import ZenRowsCrawlClient client = ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"]) crawl = client.create( - "https://example.com/products/", + "https://www.scrapingcourse.com/ecommerce/", depth=1, # link hops from the start URL max_items=50, # stop after keeping 50 URLs (default 10) max_pages=100, # stop after fetching 100 pages (default 10) @@ -580,7 +580,8 @@ client.stop(crawl_id) # idempotent; read final counts with get Every non-2xx raises `CrawlAPIError`. Branch on `status_code` and `code`: ```python -from zenrows.crawl import CRAWL_NOT_ENABLED_CODE, CrawlAPIError +from zenrows import CrawlAPIError +from zenrows.crawl import CRAWL_NOT_ENABLED_CODE try: client.create(start_url, depth=1) diff --git a/src/zenrows/__init__.py b/src/zenrows/__init__.py index 4da0a61..f9e7159 100644 --- a/src/zenrows/__init__.py +++ b/src/zenrows/__init__.py @@ -16,6 +16,12 @@ from zenrows.__version__ import __version__ from zenrows.batch import ZenRowsBatchClient from zenrows.client import ZenRowsClient -from zenrows.crawl import ZenRowsCrawlClient +from zenrows.crawl import CrawlAPIError, ZenRowsCrawlClient -__all__ = ["ZenRowsBatchClient", "ZenRowsClient", "ZenRowsCrawlClient", "__version__"] +__all__ = [ + "CrawlAPIError", + "ZenRowsBatchClient", + "ZenRowsClient", + "ZenRowsCrawlClient", + "__version__", +] diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 1a466ce..534e2a3 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -549,3 +549,10 @@ def test_an_unusable_retry_after_is_not_reported(client: ZenRowsCrawlClient, raw with pytest.raises(CrawlAPIError) as exc_info: client.create("https://example.com/", depth=1) assert exc_info.value.retry_after is None + + +def test_crawl_api_error_imports_from_the_package_root(): + import zenrows + from zenrows.crawl import CrawlAPIError as FromCrawl + + assert zenrows.CrawlAPIError is FromCrawl From 2e6ca6dd5ad498797b57a36d357e7d26c6be1374 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 14:03:56 +0200 Subject: [PATCH 11/20] test(crawl): add a first-install acceptance test for the Crawl client acceptance/crawl/ builds the release artifact from this checkout, installs it in a clean container the way a new user would, and runs every Crawl feature against the live API: README example, create, wait, results, content, download, stop, errors, naming and docs. One JSON result per check and a markdown report land in acceptance/crawl/out/ (git-ignored). It is kept out of the published package. --- acceptance/crawl/.gitignore | 1 + acceptance/crawl/README.md | 110 ++++ acceptance/crawl/builders/build-python.sh | 7 + acceptance/crawl/builders/fetch-source.sh | 11 + acceptance/crawl/run.sh | 148 +++++ acceptance/crawl/surface/run-in-container.sh | 10 + acceptance/crawl/surface/test.py | 571 +++++++++++++++++++ acceptance/crawl/surface/typecheck.py | 26 + acceptance/crawl/tools/build_failed.py | 17 + acceptance/crawl/tools/cleanup.py | 64 +++ acceptance/crawl/tools/report.py | 190 ++++++ acceptance/crawl/tools/scan_key.py | 21 + acceptance/crawl/versions.env | 23 + pyproject.toml | 1 + 14 files changed, 1200 insertions(+) create mode 100644 acceptance/crawl/.gitignore create mode 100644 acceptance/crawl/README.md create mode 100644 acceptance/crawl/builders/build-python.sh create mode 100644 acceptance/crawl/builders/fetch-source.sh create mode 100644 acceptance/crawl/run.sh create mode 100644 acceptance/crawl/surface/run-in-container.sh create mode 100644 acceptance/crawl/surface/test.py create mode 100644 acceptance/crawl/surface/typecheck.py create mode 100644 acceptance/crawl/tools/build_failed.py create mode 100644 acceptance/crawl/tools/cleanup.py create mode 100644 acceptance/crawl/tools/report.py create mode 100644 acceptance/crawl/tools/scan_key.py create mode 100644 acceptance/crawl/versions.env diff --git a/acceptance/crawl/.gitignore b/acceptance/crawl/.gitignore new file mode 100644 index 0000000..89f9ac0 --- /dev/null +++ b/acceptance/crawl/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md new file mode 100644 index 0000000..f8e9e34 --- /dev/null +++ b/acceptance/crawl/README.md @@ -0,0 +1,110 @@ +# Crawl first-install acceptance test: Python SDK + +This folder tests the Python SDK (`zenrows` on PyPI) the way a new customer meets it. It builds the release +artifact, installs it in a clean container with nothing else, follows the README's Crawl +quickstart, and checks every Crawl feature against the live API. The result is one report. + +It tests this repo only, so it runs on its own. + +## What it needs + +- bash, git and Docker. Nothing else on the host. +- Network access to GitHub, the package registries and the Crawl API. +- A Zenrows API key with Crawl access, in `ZENROWS_API_KEY`. + +## How to run it + +From the repo root: + +```bash +export ZENROWS_API_KEY="$(cat path/to/key-file)" # keep it out of your shell history +bash acceptance/crawl/run.sh # build, run every check, write the report +bash acceptance/crawl/run.sh --no-key # only the checks that need no key; the live checks are SKIP +bash acceptance/crawl/run.sh --report # rebuild the report from out/ +REF=v1.2.3 bash acceptance/crawl/run.sh # test a tag, branch or SHA from GitHub instead of this checkout +``` + +Without `ZENROWS_API_KEY`, `run.sh` stops and asks for it. The report is +`acceptance/crawl/out/crawl-first-install-report.md`; `out/` is not committed. + +Every input is an environment variable with a default in `versions.env`: `CRAWL_URL` +(default `https://www.scrapingcourse.com/ecommerce/`), `CRAWL_INCLUDE` (default `/product/`), +`REF`, `CRAWL_API_BASE`, the images (`node:20`, `python:3.12-slim`, `golang:1.25`, pinned by +digest) and the tool versions. `OUT_DIR` moves the output. + +## What a run does + +1. **Source.** By default, this checkout, including changes not yet committed. With `REF`, the + repo at that ref from GitHub. +2. **Build.** In its own container, the release artifact: `pip install` of the wheel `python -m build` makes. +3. **Clean test container.** It gets only the artifact (read-only), the checks in `surface/` + (read-only), the inputs, and the key by name (`-e ZENROWS_API_KEY`). It has no source, no + caches from the build, and no tokens. +4. **README first.** It runs the README's Crawl quickstart as written. The only changes are the + ones the cost rules need: the key, `CRAWL_URL`, `CRAWL_INCLUDE`, `max_pages` 10 or less, and + no html. The report lists each change. +5. **Checks 1 to 21 and the extras.** Each writes one JSON result: + `{surface, check, title, status, expected, actual, evidence}`, with `status` PASS, FAIL or + SKIP. The evidence is the command and its trimmed output, with the key redacted. +6. **Clean-up.** It stops every crawl the run started and reads each one's final status and + `coverage.pages_fetched`. It runs even when a check fails, or on Ctrl-C. +7. **Report.** Verdict, environment, install steps, feature matrix, findings, cost and + clean-up. Last, the run checks that the key appears in no file under `out/`. + +Some checks don't use the live API, so they behave the same on every run. They point the +client at a small local stand-in server that records what it sends. These are: "only the +fields you set are sent", "the error code is empty when the body has none", and the timing +and signal checks. + +## The checks + +| # | Feature | Expected | +| --- | --- | --- | +| 1 | Create | returns at once: `running`, a `crawl_id`, the echoed parameters; only the fields set are sent | +| 2 | Create with html | results get `content_status` and, once fetched, `content_url` | +| 3 | Idempotency | same key and body → same `crawl_id`; different body → 422 `idempotency_key_reused` (MCP, CLI: `CRAWL_INVALID_REQUEST` with the advice to send a new key) | +| 4 | Wait | returns the ended crawl; `failed` is returned, not raised (CLI: exit 1) | +| 5 | Wait runs out | a 1 s timeout on a running crawl returns it `running`, no error (CLI: exit 0; MCP, CLI: a `note`) | +| 6 | Get | `limit` 2, then the cursor: one page each; `next_cursor` moves on and is null on the last page | +| 7 | List | `limit` 2, then the next page: newest first; `next_cursor` absent or null on the last page | +| 8 | Results | every kept URL once, each matching `CRAWL_INCLUDE`; on a running crawl, what is kept so far (MCP, CLI: `partial: true`) | +| 9 | Content | by content id and by `content_url`: the same HTML | +| 10 | Download | a status, and one line per result with `url`, `content_status`, `content` (MCP has no download) | +| 11 | Stop | running → `stopped`, `stop_reason: user`; ended → answers as it ended | +| 12 | Not found | 404 `crawl_not_found` (MCP, CLI: `CRAWL_NOT_FOUND` with `crawl_id` and `server_code`) | +| 13 | Invalid input | `depth` 0 → 422 `invalid_parameter` (MCP, CLI: `CRAWL_INVALID_REQUEST`) | +| 14 | Gated option | `output_format` json → 422 `invalid_parameter`; if it is accepted, the key is in a beta segment, not a bug | +| 15 | Bad key | the key `invalid` → 401 (MCP, CLI: `AUTH_INVALID`) | +| 16 | Errors carry the server code | SDKs: `code` holds it, empty when the body has none; MCP, CLI: `server_code` | +| 17 | Naming | the API's operation and parameter names, in this surface's casing | +| 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | +| 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | +| 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | +| 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (10) | + +Extra checks for this surface: `mypy --strict` sees the package as typed (`py.typed`) and reads the real signatures. + +## What it costs + +Every crawl uses `depth` 1 and `max_pages` 10 or less. A run starts five crawls: the README run, +one URLs-only crawl, one idempotency crawl that is stopped at once, one html crawl that is +stopped with its pages in flight (check 20), and one that keeps nothing (check 19). On the default `CRAWL_URL`, a URLs-only crawl fetches 1 page and the +html crawl 6, so a run costs about **10 pages**, and at most 50 if every crawl reaches its +`max_pages`. The report lists each crawl id and its `pages_fetched`. + +An account can run 3 active jobs at once, shared with Batch jobs. A run has at most 2 crawls +running at a time. If a create still gets a 429, the run waits `Retry-After` and tries again. +It never sends requests to trigger the limit. Don't run two copies against one account at the +same time. + +## How to read the report + +- **Verdict:** ready, unless there is a blocker or major finding. +- **Feature matrix:** PASS, FAIL or SKIP, with a short reason. SKIP means the check does not + apply to this surface, the client rejects the input before the API, a crawl ended before + the check could act on it, or there was no key. +- **Findings:** one entry per FAIL, most severe first, with the command and output. An install + or README failure is a blocker; a wrong answer from a feature is major; naming, copy, the + defaults and the extras are minor. +- `out/results/checks/*.json` holds every result; `out/results/container.log` is the full + output. diff --git a/acceptance/crawl/builders/build-python.sh b/acceptance/crawl/builders/build-python.sh new file mode 100644 index 0000000..ec399e7 --- /dev/null +++ b/acceptance/crawl/builders/build-python.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Runs in the Python image. Builds the wheel a release would publish. Only the wheel leaves. +set -euo pipefail +cp -a /out/source /build && cd /build +pip install --quiet --disable-pip-version-check --root-user-action=ignore "build==$PY_BUILD_VERSION" +python -m build --wheel --outdir /out/artifact +ls -l /out/artifact diff --git a/acceptance/crawl/builders/fetch-source.sh b/acceptance/crawl/builders/fetch-source.sh new file mode 100644 index 0000000..51f1209 --- /dev/null +++ b/acceptance/crawl/builders/fetch-source.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Runs in the Node image when REF is set: downloads ZenRows/$REPO at $REF as GitHub's source +# tarball and reads the full commit SHA from it. +set -euo pipefail +cd /out +curl -fsSL --retry 4 -o src.tar.gz "https://codeload.github.com/ZenRows/$REPO/tar.gz/$REF" +# git get-tar-commit-id stops reading after the header, so the pipe breaks: ignore that. +(gunzip -c src.tar.gz | git get-tar-commit-id > sha) || true +[ -s sha ] || { echo "no commit id in the tarball of $REPO@$REF" >&2; exit 1; } +rm -rf source && mkdir -p source && tar -xzf src.tar.gz -C source --strip-components 1 && rm src.tar.gz +echo "fetched $REPO@$REF = $(cat sha)" diff --git a/acceptance/crawl/run.sh b/acceptance/crawl/run.sh new file mode 100644 index 0000000..03d63bb --- /dev/null +++ b/acceptance/crawl/run.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Crawl first-install acceptance test for this repo. See README.md in this folder. +# +# ZENROWS_API_KEY=... bash run.sh build the release artifact, run every check, write the report +# bash run.sh --no-key only the checks that need no key; the live checks are SKIP +# bash run.sh --report rebuild the report from the results already in out/ +# +# Needs bash, git and Docker. The key reaches containers by name only (-e ZENROWS_API_KEY). +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=versions.env +source "$HERE/versions.env" +OUT_DIR="${OUT_DIR:-$HERE/out}" + +no_key=0; report_only=0 +for arg in "$@"; do + case "$arg" in + --no-key) no_key=1 ;; + --report) report_only=1 ;; + -h|--help) sed -n '2,8p' "$0"; exit 0 ;; + *) echo "unknown argument: $arg (expected --no-key or --report)" >&2; exit 2 ;; + esac +done + +if [ "$report_only" = 0 ] && [ "$no_key" = 0 ] && [ -z "${ZENROWS_API_KEY:-}" ]; then + echo "ZENROWS_API_KEY is not set. Set it (a key with Crawl access), or run 'bash run.sh --no-key' for the offline checks." >&2 + exit 2 +fi +if [ "$no_key" = 1 ]; then unset ZENROWS_API_KEY; fi +export ZENROWS_API_KEY="${ZENROWS_API_KEY:-}" CRAWL_URL CRAWL_INCLUDE CRAWL_API_BASE + +log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; } +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +CMDLINE="bash run.sh $*" +# Unique per run, so clean-up never touches the containers of another run. +LABEL="crawl-acceptance.run=$RUN_ID-$$-$RANDOM" +docker_run() { docker run --rm --label "$LABEL" "$@"; } + +# ---- 1. Source: this checkout (default), or the repo on GitHub at $REF ---- +prepare_source() { + rm -rf "$OUT_DIR/build"; mkdir -p "$OUT_DIR/build/source" + if [ -n "${REF:-}" ]; then + log "source: ZenRows/$REPO@$REF from GitHub" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e REPO="$REPO" -e REF="$REF" "$NODE_IMAGE" bash /builders/fetch-source.sh >"$OUT_DIR/build/fetch.log" 2>&1 + echo "github:$REF" >"$OUT_DIR/build/origin" + else + local root; root="$(git -C "$HERE" rev-parse --show-toplevel)" + log "source: this checkout ($root)" + # Tracked and untracked-but-not-ignored files, as a release build would see them. + (cd "$root" && git ls-files -co --exclude-standard -z | tar --null -T - -cf -) | tar -C "$OUT_DIR/build/source" -xf - + local sha; sha="$(git -C "$root" rev-parse HEAD)" + if [ -n "$(git -C "$root" status --porcelain -- . ":(exclude)$(realpath --relative-to="$root" "$HERE")")" ]; then + sha="$sha+local-changes" + fi + echo "$sha" >"$OUT_DIR/build/sha" + echo "checkout" >"$OUT_DIR/build/origin" + fi + # The test never reads the harness from the source; only the artifact goes in. + rm -rf "$OUT_DIR/build/source/acceptance" +} + +# ---- 2. Builder: the release artifact, in its own container ---- +build_artifact() { + mkdir -p "$OUT_DIR/build/artifact" + case "$SURFACE" in + node|mcp|cli) + log "build: npm pack" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e SURFACE -e PNPM_VERSION "$NODE_IMAGE" bash /builders/build-npm.sh >"$OUT_DIR/build/build.log" 2>&1 ;; + python) + log "build: the wheel" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e PY_BUILD_VERSION "$PYTHON_IMAGE" bash /builders/build-python.sh >"$OUT_DIR/build/build.log" 2>&1 ;; + go) + # Go has no artifact: the clean container runs `go get` at a pushed commit, as a user would. + local sha; sha="$(cat "$OUT_DIR/build/sha")" + case "$sha" in *+local-changes) + log "go: local changes are not tested; go get installs the commit ${sha%+local-changes} from GitHub" ;; + esac + echo "${sha%+local-changes}" >"$OUT_DIR/build/go_ref" ;; + esac +} + +# ---- 3. Clean test container: only the artifact, the checks and the inputs go in ---- +run_checks() { + rm -rf "$OUT_DIR/results"; mkdir -p "$OUT_DIR/results" + local image + case "$SURFACE" in node|mcp|cli) image="$NODE_IMAGE" ;; python) image="$PYTHON_IMAGE" ;; go) image="$GO_IMAGE" ;; esac + log "checks: a clean $image" + local rc=0 + docker_run --init \ + -v "$OUT_DIR/build/artifact:/artifact:ro" \ + -v "$HERE/surface:/harness:ro" \ + -v "$OUT_DIR/results:/results" \ + -e ZENROWS_API_KEY -e CRAWL_URL -e CRAWL_INCLUDE -e CRAWL_API_BASE \ + -e SURFACE -e GO_REF="$(cat "$OUT_DIR/build/go_ref" 2>/dev/null || true)" \ + -e MCP_SDK_VERSION -e TYPESCRIPT_VERSION -e MYPY_VERSION \ + -w /work --tmpfs /work:exec,size=2g \ + "$image" bash /harness/run-in-container.sh >"$OUT_DIR/results/container.log" 2>&1 || rc=$? + log "checks: the container exited $rc (out/results/container.log)" +} + +# ---- 4. Clean-up: stop every crawl this run started. Runs even when a check fails. ---- +cleanup() { + [ "$report_only" = 1 ] && return 0 + docker ps -q --filter "label=$LABEL" | xargs -r docker kill >/dev/null 2>&1 || true + if [ -n "${ZENROWS_API_KEY:-}" ] && [ -d "$OUT_DIR/results" ]; then + log "clean-up: stopping every crawl this run started" + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" \ + -e ZENROWS_API_KEY -e CRAWL_API_BASE "$PYTHON_IMAGE" python -I /tools/cleanup.py /out || log "clean-up failed" + fi +} + +report() { + log "report: out/crawl-first-install-report.md" + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" \ + -e CMDLINE="$CMDLINE" -e SURFACE -e REPO -e CRAWL_URL -e CRAWL_INCLUDE -e CRAWL_API_BASE \ + -e NODE_IMAGE -e PYTHON_IMAGE -e GO_IMAGE -e MCP_SDK_VERSION -e TYPESCRIPT_VERSION \ + -e MYPY_VERSION -e PY_BUILD_VERSION -e PNPM_VERSION \ + "$PYTHON_IMAGE" python -I /tools/report.py /out + # The key must appear in nothing the run wrote. Checked by value, inside a container. + if [ -n "${ZENROWS_API_KEY:-}" ]; then + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" -e ZENROWS_API_KEY "$PYTHON_IMAGE" \ + python -I /tools/scan_key.py /out + fi +} + +export SURFACE REPO NODE_IMAGE PYTHON_IMAGE GO_IMAGE PNPM_VERSION PY_BUILD_VERSION \ + MCP_SDK_VERSION TYPESCRIPT_VERSION MYPY_VERSION +if [ "$report_only" = 1 ]; then report; exit 0; fi + +mkdir -p "$OUT_DIR" +echo "$RUN_ID" >"$OUT_DIR/run_id" +trap cleanup EXIT +trap 'exit 130' INT TERM +if prepare_source && build_artifact; then + run_checks +else + log "the source or build step failed (out/build/*.log)" + mkdir -p "$OUT_DIR/results" + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" -e SURFACE "$PYTHON_IMAGE" python -I /tools/build_failed.py /out +fi +cleanup +trap - EXIT +report +log "done: $OUT_DIR/crawl-first-install-report.md" diff --git a/acceptance/crawl/surface/run-in-container.sh b/acceptance/crawl/surface/run-in-container.sh new file mode 100644 index 0000000..e4598dd --- /dev/null +++ b/acceptance/crawl/surface/run-in-container.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Entry point in the clean python container: install the wheel as `pip install zenrows` would, +# install the pinned mypy for the typing check, record the versions, run the checks. +set -uo pipefail +export PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_ROOT_USER_ACTION=ignore PIP_NO_CACHE_DIR=1 +pip install /artifact/*.whl > /results/pip-install.log 2>&1 || { cat /results/pip-install.log; exit 1; } +pip install "mypy==$MYPY_VERSION" > /results/pip-mypy.log 2>&1 +printf '{"python":"%s","pip":"%s","mypy":"%s","os":"%s"}\n' "$(python -V 2>&1)" "$(pip -V | cut -d' ' -f2)" \ + "$MYPY_VERSION" "$(. /etc/os-release; echo "$PRETTY_NAME")" > /results/env.json +cd /work && python /harness/test.py diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py new file mode 100644 index 0000000..223017b --- /dev/null +++ b/acceptance/crawl/surface/test.py @@ -0,0 +1,571 @@ +"""Python SDK (`zenrows`): the README walk-through, checks 1-21 and the Python extra check. + +Runs in a clean python image. run-in-container.sh installs the wheel first; this script then +imports it the way a user would. +""" + +import http.server +import json +import os +import re +import subprocess +import sys +import threading +import time +import traceback +import urllib.request +import uuid +from datetime import UTC, datetime +from importlib import metadata + +SURFACE = "python" +KEY = os.environ.get("ZENROWS_API_KEY", "") +HAS_KEY = bool(KEY) +CRAWL_URL = os.environ["CRAWL_URL"] +INCLUDE = os.environ["CRAWL_INCLUDE"] +API_BASE = os.environ.get("CRAWL_API_BASE", "https://api.zenrows.com/v1") +NO_MATCH = "/crawl-acceptance-matches-nothing/" # no URL contains it: the crawl keeps nothing (check 19) +DOCUMENTED_DEFAULT = 10 # the default the README documents for max_pages and max_items (check 21) +RESULTS = "/results" +PASS, FAIL, SKIP = "PASS", "FAIL", "SKIP" +TERMINAL = {"completed", "stopped", "failed"} +os.makedirs(f"{RESULTS}/checks", exist_ok=True) +_seq = 0 + + +def redact(v): + s = v if isinstance(v, str) else json.dumps(v, default=str) + return s.replace(KEY, "") if KEY else s + + +def trim(v, n): + s = redact(v) + return s if len(s) <= n else f"{s[:n]}… [{len(s) - n} more chars]" + + +def record(check, title, status, expected, actual, evidence=""): + global _seq + _seq += 1 + r = {"surface": SURFACE, "check": str(check), "title": title, "status": status, "expected": expected, + "actual": trim(actual, 800), "evidence": trim(evidence, 2500)} + name = f"{_seq:03d}-{re.sub(r'[^A-Za-z0-9_.-]+', '_', str(check))}.json" + with open(f"{RESULTS}/checks/{name}", "w") as f: + json.dump(r, f, indent=2) + print(f"[{status}] python {check} {title}: {r['actual'].splitlines()[0][:200] if r['actual'] else ''}", flush=True) + + +def pass_if(ok): + return PASS if ok else FAIL + + +def run_check(check, title, expected, fn, live=False): + if live and not HAS_KEY: + return record(check, title, SKIP, expected, "ZENROWS_API_KEY not set (run without --no-key)") + try: + status, actual, evidence = fn() + record(check, title, status, expected, actual, evidence) + except Exception as e: # an unexpected exception is a FAIL with its traceback + record(check, title, FAIL, expected, f"raised {type(e).__name__}: {e}", traceback.format_exc()) + + +def ledger(crawl_id, purpose): + if crawl_id: + with open(f"{RESULTS}/crawls.jsonl", "a") as f: + f.write(json.dumps({"crawl_id": crawl_id, "purpose": purpose}) + "\n") + + +def sh(args, **kw): + p = subprocess.run(args, capture_output=True, text=True, **kw) + return p.returncode, p.stdout + p.stderr + + +def crawls_since(t): + if not HAS_KEY: + return [] + req = urllib.request.Request(f"{API_BASE}/crawls?limit=100", headers={"X-API-Key": KEY}) + with urllib.request.urlopen(req, timeout=60) as resp: + body = json.load(resp) + out = [] + for c in body["crawls"]: + created = datetime.strptime(c["created_at"][:19], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC) + if created.timestamp() >= t: + out.append(c["crawl_id"]) + return out + + +# ---------- README: install (run-in-container.sh) and the Crawl quickstart ---------- +with open(f"{RESULTS}/pip-install.log") as f: + install_log = f.read() +record("readme-1", "Install (README: `pip install zenrows`; here the release wheel)", PASS, "the package installs", + f"zenrows {metadata.version('zenrows')} installed", f"$ pip install \n{install_log[-1200:]}") + +readme = metadata.metadata("zenrows").get_payload() or "" +HEADING = "## Quickstart — Crawl API (new)" +section = readme.split(HEADING, 1)[1].split("\n## ", 1)[0] if HEADING in readme else "" +m = re.search(r"```python\n(.*?)```", section, re.S) +block = m.group(1) if m else None +record("readme-2", "README has a Crawl quickstart", pass_if(bool(block)), "a ```python block under the Crawl quickstart", + f"{len(block.splitlines())} lines" if block else "not found", block or section[:600]) + +subs = [ + ('"https://example.com/products/"', json.dumps(CRAWL_URL), "start URL → CRAWL_URL"), + ("max_items=50,", "max_items=5,", "max_items 50 → 5"), + ("max_pages=100,", "max_pages=10,", "max_pages 100 → 10 (at most 10)"), + ('include_patterns=["/product/"]', f"include_patterns=[{json.dumps(INCLUDE)}]", "include → CRAWL_INCLUDE"), +] +quick, applied = block or "", [] +for a, b, why in subs: + if a in quick: + quick = quick.replace(a, b) + applied.append(why) + else: + applied.append(f"not found, left as is: {why}") +applied.append("the key comes from ZENROWS_API_KEY, as the quickstart reads it") +with open("/work/quickstart.py", "w") as f: + f.write(quick) + + +def readme_run(): + since = time.time() - 5 + code, out = sh([sys.executable, "/work/quickstart.py"], timeout=12 * 60) + mine = crawls_since(since) + for i in mine: + ledger(i, "README quickstart") + return pass_if(code == 0), f"exit {code}; crawls: {', '.join(mine) or 'none'}", \ + f"substitutions: {'; '.join(applied)}\n$ python quickstart.py\n{out}" + + +run_check("readme-3", "Run the README Crawl quickstart", "it runs to the end without an error", readme_run, live=True) + + +# ---------- extra: a typed package (py.typed) under mypy --strict ---------- +def mypy_check(): + code, out = sh([sys.executable, "-m", "mypy", "--strict", "--no-incremental", "/harness/typecheck.py"]) + untyped = "missing library stubs or py.typed" in out or "module is installed, but missing" in out + revealed = "def (url: builtins.str" in out + errors = [line for line in out.splitlines() if ": error:" in line] + return pass_if(not untyped and revealed and not errors), \ + f"mypy exit {code}; py.typed seen {not untyped}; real signature revealed {revealed}; {len(errors)} errors", \ + f"$ mypy --strict typecheck.py (mypy {metadata.version('mypy')})\n{out}" + + +run_check("x-python-typed", "mypy --strict sees the package as typed (py.typed)", "no missing-stubs error; the real signatures", + mypy_check) + +# ---------- check 1, offline half: only the fields you set are sent ---------- +captured = [] + + +class StandIn(http.server.BaseHTTPRequestHandler): + """A local stand-in for the Crawl API that records what the client sends.""" + + def log_message(self, *a): + pass + + def do_POST(self): + n = int(self.headers.get("Content-Length") or 0) + body = json.loads(self.rfile.read(n) or b"null") + captured.append(body) + self.send_response(201) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({ + "crawl_id": "c_capture", "status": "running", "url": body.get("url"), "depth": body.get("depth"), + "max_items": 10, "max_pages": 10, "created_at": datetime.now(UTC).isoformat(), + "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0}}).encode()) + + def do_GET(self): + if "c_nobody" in self.path: + self.send_response(502) + self.send_header("Content-Length", "0") + self.end_headers() + return + self.send_response(404) + self.send_header("Content-Type", "application/problem+json") + self.end_headers() + self.wfile.write(b'{"status":404,"title":"Not Found","code":"crawl_not_found"}') + + +srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), StandIn) +threading.Thread(target=srv.serve_forever, daemon=True).start() + +from zenrows import ZenRowsCrawlClient # noqa: E402 (imported once installed, as a user would) +from zenrows.crawl import CrawlAPIError # noqa: E402 + +cap = ZenRowsCrawlClient("capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0) +cap.create("https://example.com/", depth=1) +cap.create("https://example.com/", depth=1, max_items=5, max_pages=10, include_patterns=["/p/"], idempotency_key="idem-1") +sent_ok = sorted(captured[0]) == ["depth", "url"] and \ + sorted(captured[1]) == sorted(["url", "depth", "max_items", "max_pages", "include_patterns"]) +sent_line = f"sent (stand-in server): minimal={json.dumps(captured[0])} full={json.dumps(captured[1])}" + +# ---------- live ---------- +client = ZenRowsCrawlClient(KEY or "none", base_url=API_BASE) +BASE = {"depth": 1, "max_items": 5, "max_pages": 10, "include_patterns": [INCLUDE]} +state = {} + + +def create(**kw): + for _ in range(10): + try: + c = client.create(CRAWL_URL, **kw) + ledger(c.crawl_id, "check") + return c + except CrawlAPIError as e: + if e.status_code != 429: + raise + time.sleep(e.retry_after or 30) # the account's active jobs are shared: wait as told + raise RuntimeError("still 429 after 10 tries") + + +def err_of(fn): + try: + return fn(), None + except CrawlAPIError as e: + return None, e + + +def esum(e): + return None if e is None else {"status_code": e.status_code, "code": e.code, "detail": e.detail} + + +def c1(): + if not HAS_KEY: + return (SKIP if sent_ok else FAIL), f"live part skipped (no key); only set fields sent: {sent_ok}", sent_line + t = time.time() + a = state["A"] = create(**BASE) + echo = {"url": a.url, "depth": a.depth, "max_items": a.max_items, "max_pages": a.max_pages, + "include_patterns": a.include_patterns} + want = {"url": CRAWL_URL, **BASE} + ok = sent_ok and a.status.value == "running" and a.crawl_id.startswith("c_") and echo == want + return pass_if(ok), f"{a.status.value}, {a.crawl_id}, {int((time.time() - t) * 1000)} ms, echo {json.dumps(echo)}; " \ + f"only set fields sent: {sent_ok}", f"{a.model_dump_json()}\n{sent_line}" + + +def c5(): + a = state.get("A") + if not a: + return SKIP, "no crawl from check 1", "" + t = time.time() + w = client.wait(a.crawl_id, timeout=1) + ms = int((time.time() - t) * 1000) + if w.status.value != "running": + return SKIP, f"the crawl had already ended ({w.status.value}) after {ms} ms", "" + return pass_if(ms < 5000), f"{w.status.value} after {ms} ms, no exception", w.model_dump_json() + + +def partial_read(): + a = state.get("A") + if not a: + return + running = client.get(a.crawl_id, limit=1).status.value == "running" + t = time.time() + items, e = err_of(lambda: [r.url for r in client.results(a.crawl_id)]) + state["partial"] = (running, len(items or []), int((time.time() - t) * 1000), e) + + +def c3(): + key = f"acc-{uuid.uuid4()}" + body = dict(BASE, max_items=3, max_pages=3) + c = state["C"] = create(idempotency_key=key, **body) + again = create(idempotency_key=key, **body) + v, e = err_of(lambda: client.create(CRAWL_URL, idempotency_key=key, **dict(body, max_items=4))) + if v: + ledger(v.crawl_id, "idempotency, different body (unexpected create)") + ok = c.crawl_id == again.crawl_id and e is not None and e.status_code == 422 and e.code == "idempotency_key_reused" + third = f"{e.status_code} {e.code}" if e else f"created {v.crawl_id}" + return pass_if(ok), f"first {c.crawl_id}, second {again.crawl_id}; different body → {third}", json.dumps(esum(e)) + + +def c11(): + c = state.get("C") + if not c: + return SKIP, "no crawl from check 3", "" + first = client.stop(c.crawl_id) + reason = first.stop_reason.value if first.stop_reason else None + if first.status.value == "completed" and reason != "user": + return SKIP, f"the crawl had already ended on its own ({first.status.value}/{reason})", "" + second, e = err_of(lambda: client.stop(c.crawl_id)) + ok = first.status.value == "stopped" and reason == "user" and e is None and second.status == first.status + two = f"error {e}" if e else second.status.value + return pass_if(ok), f"first: {first.status.value}/{reason}; second: {two}", first.model_dump_json() + + +def c4(): + a = state.get("A") + if not a: + return SKIP, "no crawl from check 1", "" + e = state["ended"] = client.wait(a.crawl_id) + reason = f"/{e.stop_reason.value}" if e.stop_reason else "" + return pass_if(e.status.value in TERMINAL), f"{e.status.value}{reason}; coverage {e.coverage.model_dump_json()}", \ + e.model_dump_json() + + +def c6(): + e = state.get("ended") + if not e: + return SKIP, "no ended crawl", "" + pages, cursor = [], None + for _ in range(20): + p = client.get(e.crawl_id, limit=2, cursor=cursor) + pages.append((len(p.results), p.next_cursor, [r.url for r in p.results])) + if p.next_cursor is None: + break + cursor = p.next_cursor + urls = [u for _, _, us in pages for u in us] + curs = [c for _, c, _ in pages if c is not None] + ok = all(n <= 2 for n, _, _ in pages) and pages[-1][1] is None and len(set(curs)) == len(curs) \ + and len(set(urls)) == len(urls) and (len(pages) >= 2 or e.coverage.items_found <= 2) + sizes = ",".join(str(n) for n, _, _ in pages) + return pass_if(ok), f"{len(pages)} pages of {sizes}; last next_cursor {pages[-1][1]}; {len(urls)} URLs", "" + + +def c7(): + p1 = client.list(limit=2) + p2 = client.list(limit=2, cursor=p1.next_cursor) if p1.next_cursor else None + allc = p1.crawls + (p2.crawls if p2 else []) + times = [c.created_at for c in allc] + newest = all(times[i - 1] >= times[i] for i in range(1, len(times))) + overlap = len({c.crawl_id for c in allc}) != len(allc) + cursor, pages = (p2.next_cursor if p2 else None), 2 + while cursor and pages < 60: + cursor = client.list(limit=100, cursor=cursor).next_cursor + pages += 1 + ok = len(p1.crawls) == 2 and newest and not overlap and not cursor + return pass_if(ok), f"newest first {newest}, no overlap {not overlap}; last page after {pages} pages: " \ + f"next_cursor {json.dumps(cursor)}", "" + + +def c8(): + e = state.get("ended") + if not e: + return SKIP, "no ended crawl", "" + urls = [r.url for r in client.results(e.crawl_id, limit=2)] + distinct, match = len(set(urls)) == len(urls), all(INCLUDE in u for u in urls) + p = state.get("partial") + run = (f"while running ({'running' if p[0] else 'already ended'}): {p[1]} URLs in {p[2]} ms, " + f"{'error ' + str(p[3]) if p[3] else 'returned'}") if p else "not read while running" + ok = bool(urls) and distinct and match and len(urls) == e.coverage.items_found and not (p and p[3]) + return pass_if(ok), f"{len(urls)} URLs (items_found {e.coverage.items_found}), distinct {distinct}, " \ + f"all match {match}; {run}", "\n".join(urls) + + +def settle(crawl_id): + """Poll until no result is still pending (the pages in flight finished), at most 90 s.""" + for _ in range(45): + page = client.get(crawl_id, limit=100) + if not any(r.content_status and r.content_status.value == "pending" for r in page.results): + return "settled" + time.sleep(2) + return "still pending after 90 s" + + +def stop_mid_flight(): + """Create the html crawl and stop it as soon as it has kept URLs, while their pages are fetched.""" + try: + b = state["B"] = create(output_format="html", **BASE) + g = None + for _ in range(100): + g = client.get(b.crawl_id, limit=1) + if g.status.value != "running" or g.coverage.items_found > 0: + break + time.sleep(0.15) + s = client.stop(b.crawl_id) + state["stop"] = {"items": g.coverage.items_found, "status": s.status.value} + except Exception as e: # recorded by check 20 + state["stop"] = {"error": e} + + +def c20(): + b, st = state.get("B"), state.get("stop") or {} + if not b or "error" in st: + return FAIL, f"setup failed: {st.get('error')}", "" + if st["status"] != "stopped": + return SKIP, f"the crawl had already ended ({st['status']}) before the stop", "" + t = time.time() + box = {} + # A daemon thread, so a results() that pages forever fails the check instead of hanging the run. + reader = threading.Thread(target=lambda: box.update(now=list(client.results(b.crawl_id))), daemon=True) + reader.start() + reader.join(60) + if "now" not in box: + return FAIL, "results() did not finish within 60 s on a crawl stopped with pages in flight", "" + now, ms = box["now"], int((time.time() - t) * 1000) + settled = settle(b.crawl_id) + if settled != "settled": + settled += " (an API issue: the stop left these pages pending on an ended crawl)" + later = [r.url for r in client.results(b.crawl_id)] + same = sorted(r.url for r in now) == sorted(later) + pending = sum(1 for r in now if r.content_status and r.content_status.value == "pending") + return pass_if(same and bool(now)), f"stopped with {st['items']} kept; read {len(now)} URLs ({pending} pending) in {ms} ms, " \ + f"then {settled}: {len(later)} URLs, same set {same}", "" + + +def c2(): + b = state.get("B") + if not b: + return SKIP, "no html crawl", "" + if (state.get("stop") or {}).get("status") == "stopped": + settle(b.crawl_id) + rs = list(client.results(b.crawl_id)) + ok = b.output_format is not None and b.output_format.value == "html" and bool(rs) and all( + r.content_status and (r.content_status.value != "fetched" or r.content_url) for r in rs) + with_url = sum(1 for r in rs if r.content_url) + return pass_if(ok), f"{b.crawl_id} ({(state.get('stop') or {}).get('status', '-')}); {len(rs)} results; " \ + f"with content_url {with_url}", json.dumps([r.model_dump(mode="json") for r in rs[:2]]) + + +def c9(): + b = state.get("B") + if not b: + return SKIP, "no html crawl", "" + r = next((x for x in client.results(b.crawl_id) if x.content_url), None) + if not r: + if (state.get("stop") or {}).get("status") == "stopped": + return SKIP, "the stop cancelled every page fetch, so no page has content to read (see check 20)", "" + return FAIL, "no result with content_url", "" + cid = r.content_url.rstrip("/").rsplit("/", 1)[-1] + by_id, by_url = client.content(b.crawl_id, cid), client.content(b.crawl_id, r.content_url) + ok = by_id == by_url and re.search(r"", "").lstrip("-").replace("-", "_") + return re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", n).lower() + + +def naming(rows): + p = os.path.join(OUT, "results", "naming.json") + if not os.path.exists(p): + return + with open(p) as f: + d = json.load(f) + diffs = [] + for op in CANON_OPS: + got = d["operations"].get(op) + if got is None: + diffs.append(f"operation `{op}` missing") + elif snake(got) != op: + diffs.append(f"operation `{op}` is `{got}`") + for prm in CANON_PARAMS: + got = d["params"].get(prm) + if got is None: + diffs.append(f"parameter `{prm}` missing") + elif snake(got) != prm: + diffs.append(f"parameter `{prm}` is `{got}`") + rows["17"] = {"surface": SURFACE, "check": "17", "title": "Naming", + "status": "FAIL" if diffs else "PASS", + "expected": "the API's operation and parameter names, in this surface's casing", + "actual": "; ".join(diffs) or "matches", + "evidence": json.dumps({k: d[k] for k in ("operations", "params")})} + + +def severity(check): + if check in ("17", "18", "21") or check.startswith("x-"): + return "minor" + if check.startswith("readme") or check == "build": + return "blocker" + return "major" + + +def first_line(s, n=110): + line = re.sub(r"\s+", " ", (s or "").split("\n")[0]) + return line if len(line) <= n else line[: n - 1] + "…" + + +def main(): + rows = {} + for p in sorted(glob.glob(os.path.join(OUT, "results", "checks", "*.json"))): + with open(p) as f: + r = json.load(f) + rows[r["check"]] = r + naming(rows) + env = json.loads(read(os.path.join(OUT, "results", "env.json"), "{}") or "{}") + cleanup = json.loads(read(os.path.join(OUT, "cleanup.json"), "null") or "null") + sha = read(os.path.join(OUT, "build", "sha"), "-") + origin = read(os.path.join(OUT, "build", "origin"), "-") + artifacts = [os.path.basename(p) for p in glob.glob(os.path.join(OUT, "build", "artifact", "*"))] + gomod = read(os.path.join(OUT, "results", "go-module.txt")) + + findings = [{"check": c, "severity": severity(c), "r": r} for c, r in rows.items() if r["status"] == "FAIL"] + order = {"blocker": 0, "major": 1, "minor": 2} + findings.sort(key=lambda f: order[f["severity"]]) + blocking = [f for f in findings if f["severity"] != "minor"] + counts = {} + for r in rows.values(): + counts[r["status"]] = counts.get(r["status"], 0) + 1 + + lines = [] + w = lines.append + w(f"# Crawl first-install test: {NAMES[SURFACE]}\n") + w(f"Run `{read(os.path.join(OUT, 'run_id'), '-')}`. The client was built as a release artifact, installed in a " + "clean container, set up from its README, then tested against the live Crawl API.\n") + + w("## 1. Verdict\n") + if blocking: + w("**Not ready.** " + "; ".join(f"{f['check']} {f['r']['title']}: {first_line(f['r']['actual'], 90)}" for f in blocking)) + else: + w("**Ready.**" + (f" {len(findings)} minor finding(s), see Findings." if findings else "")) + w("\n" + ", ".join(f"{v} {k}" for k, v in sorted(counts.items())) + ".\n") + + w("## 2. Environment\n") + w(f"- Command: `{os.environ.get('CMDLINE') or 'bash run.sh'}`") + w(f"- Repo: ZenRows/{REPO}; source: {origin}; SHA tested: `{sha}`") + w(f"- Artifact: {', '.join(artifacts) or ('go get ' + gomod.split()[0] if gomod else '-')}") + w(f"- `CRAWL_URL`: {os.environ.get('CRAWL_URL')}; `CRAWL_INCLUDE`: `{os.environ.get('CRAWL_INCLUDE')}`; " + f"API: {os.environ.get('CRAWL_API_BASE')}") + image = {"node": "NODE_IMAGE", "mcp": "NODE_IMAGE", "cli": "NODE_IMAGE", "python": "PYTHON_IMAGE", "go": "GO_IMAGE"}[SURFACE] + w(f"- Image: `{os.environ.get(image)}`; container: " + ", ".join(f"{k} {v}" for k, v in env.items())) + tools = {"node": f"typescript {os.environ.get('TYPESCRIPT_VERSION')}, pnpm {os.environ.get('PNPM_VERSION')}", + "mcp": f"@modelcontextprotocol/sdk {os.environ.get('MCP_SDK_VERSION')}", "cli": "none", + "python": f"mypy {os.environ.get('MYPY_VERSION')}, build {os.environ.get('PY_BUILD_VERSION')}", "go": "none"}[SURFACE] + w(f"- Pinned tools: {tools}\n") + + w("## 3. First-time install\n") + for check, r in rows.items(): + if check.startswith("readme") or check == "build": + w(f"- **{r['status']}** {r['title']}: {first_line(r['actual'], 200)}") + ev = r.get("evidence") or "" + if ev and (r["status"] != "PASS" or "substitutions" in ev): + w("\n ```\n " + "\n ".join(ev.splitlines()[:16]) + "\n ```") + w("") + + w("## 4. Feature matrix\n") + w("| # | Check | Result |") + w("| --- | --- | --- |") + for c in [str(i) for i in range(1, 22)]: + r = rows.get(c) + cell = f"**{r['status']}** {first_line(r['actual'])}".replace("|", "\\|") if r else "—" + w(f"| {c} | {CHECK_TITLES[c]} | {cell} |") + for c, r in rows.items(): + if c.startswith("x-"): + w(f"| extra | {r['title']} | " + f"**{r['status']}** {first_line(r['actual'])}".replace("|", "\\|") + " |") + w("") + + w("## 5. Findings\n") + if not findings: + w("None.\n") + for i, f in enumerate(findings, 1): + r = f["r"] + w(f"### {i}. {r['title']} ({f['severity']})\n") + w(f"- What happened: {r['actual']}") + w(f"- Expected: {r['expected']}") + w("- Evidence:\n\n ```\n " + "\n ".join((r.get("evidence") or "").splitlines()[:20]) + "\n ```\n") + + w("## 6. Cost\n") + if cleanup and cleanup["crawls"]: + w("| Crawl | Purpose | Final status | pages_fetched |") + w("| --- | --- | --- | --- |") + total = 0 + for c in cleanup["crawls"]: + total += c.get("pages_fetched") or 0 + w(f"| `{c['crawl_id']}` | {', '.join(c['purposes'])}{' (html)' if c.get('output_format') == 'html' else ''} | " + f"{c.get('status')}{'/' + c['stop_reason'] if c.get('stop_reason') else ''} | {c.get('pages_fetched')} |") + w(f"\nTotal pages fetched: **{total}**.\n") + else: + w("No crawls.\n") + + w("## 7. Clean-up\n") + if cleanup and cleanup["crawls"]: + stopped = [c["crawl_id"] for c in cleanup["crawls"] if c.get("stopped_by_cleanup")] + w(f"Checked at {cleanup['checked_at']}: {len(cleanup['crawls'])} crawls started by this run, " + f"{', '.join(cleanup['still_running']) or 'none'} still running. " + + (f"Clean-up stopped {', '.join(stopped)}." if stopped else "Clean-up had nothing to stop.") + "\n") + else: + w("No crawls to clean up.\n") + + with open(os.path.join(OUT, "crawl-first-install-report.md"), "w") as f: + f.write("\n".join(lines)) + with open(os.path.join(OUT, "results.json"), "w") as f: + json.dump(list(rows.values()), f, indent=2) + print(f"{SURFACE}: " + ", ".join(f"{k} {v}" for k, v in sorted(counts.items())) + f"; findings: {len(findings)}") + + +main() diff --git a/acceptance/crawl/tools/scan_key.py b/acceptance/crawl/tools/scan_key.py new file mode 100644 index 0000000..f6f0419 --- /dev/null +++ b/acceptance/crawl/tools/scan_key.py @@ -0,0 +1,21 @@ +"""Fail the run if the API key appears in any file under out/. Reads the key from the env.""" +import os +import sys + +key = os.environ.get("ZENROWS_API_KEY", "").encode() +hits = [] +for root, _, files in os.walk(sys.argv[1]): + if os.sep + "build" in root: + continue + for f in files: + p = os.path.join(root, f) + try: + with open(p, "rb") as fh: + if key and key in fh.read(): + hits.append(p) + except OSError: + pass +if hits: + print("THE API KEY APPEARS IN:", *hits, sep="\n ") + sys.exit(1) +print("key scan: the key appears in no result, log or report file") diff --git a/acceptance/crawl/versions.env b/acceptance/crawl/versions.env new file mode 100644 index 0000000..dc1f030 --- /dev/null +++ b/acceptance/crawl/versions.env @@ -0,0 +1,23 @@ +# Pinned inputs for the Crawl acceptance test. Every value can be overridden from the environment. +SURFACE=python +REPO=zenrows-python-sdk + +# The official images, pinned by digest so two runs agree. +NODE_IMAGE="${NODE_IMAGE:-node:20@sha256:8f693eaa7e0a8e71560c9a82b55fd54c2ae920a2ba5d2cde28bac7d1c01c9ba5}" +PYTHON_IMAGE="${PYTHON_IMAGE:-python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1}" +GO_IMAGE="${GO_IMAGE:-golang:1.25@sha256:699337d620559a59b4a2bb298ad59611e535d2ee755a34cf2d2a98f37578dc80}" + +# Tools the build and the checks install. +PNPM_VERSION="${PNPM_VERSION:-9.4.0}" +PY_BUILD_VERSION="${PY_BUILD_VERSION:-1.2.2.post1}" +MCP_SDK_VERSION="${MCP_SDK_VERSION:-1.27.1}" +TYPESCRIPT_VERSION="${TYPESCRIPT_VERSION:-5.7.3}" +MYPY_VERSION="${MYPY_VERSION:-1.15.0}" + +# What to crawl. +CRAWL_URL="${CRAWL_URL:-https://www.scrapingcourse.com/ecommerce/}" +CRAWL_INCLUDE="${CRAWL_INCLUDE:-/product/}" +CRAWL_API_BASE="${CRAWL_API_BASE:-https://api.zenrows.com/v1}" + +# REF unset tests this checkout. Set it to a SHA, branch or tag to test that commit from GitHub. +REF="${REF:-}" diff --git a/pyproject.toml b/pyproject.toml index fbad15b..4cb04f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ packages = ["src/zenrows"] [tool.hatch.build.targets.sdist] include = ["src/zenrows", "README.md", "LICENSE"] +exclude = ["/acceptance"] # the acceptance test harness, never shipped # --- uv --- [tool.uv] From 3522a639bbcb22669c418bd1872e3e2c7a9489c0 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 14:26:31 +0200 Subject: [PATCH 12/20] feat(crawl)!: name content()'s argument content_id content(crawl_id, content_id) matches the API's content_id and the other clients. It still accepts a content_url or a CrawlResult. A caller that passed content= by keyword now passes content_id=. --- src/zenrows/crawl/client.py | 18 +++++++++--------- tests/test_crawl_client.py | 8 ++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index d43b893..1a8a6ac 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -266,25 +266,25 @@ def stop(self, crawl_id: str) -> CrawlStop: # ===== contents ===== - def content(self, crawl_id: str, content: str | CrawlResult) -> str: + def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: """`GET /crawls/{crawl_id}/contents/{content_id}` — one kept URL's page, as HTML text. - `content` is a `CrawlResult` whose `content_status` is + `content_id` is a `CrawlResult` whose `content_status` is `fetched`, its `content_url`, or its content id (the last segment of the `content_url`). Only crawls created with an `output_format` have contents; anything else is 404 `content_not_found`. """ - if isinstance(content, CrawlResult): - if not content.content_url: + if isinstance(content_id, CrawlResult): + if not content_id.content_url: raise ValueError( - f"content: {content.url} has no content_url " - f"(content_status={content.content_status})" + f"content: {content_id.url} has no content_url " + f"(content_status={content_id.content_status})" ) - content = content.content_url - content_id = content.rstrip("/").rsplit("/", 1)[-1] - return self._t.request("GET", f"/crawls/{crawl_id}/contents/{content_id}").text + content_id = content_id.content_url + segment = content_id.rstrip("/").rsplit("/", 1)[-1] + return self._t.request("GET", f"/crawls/{crawl_id}/contents/{segment}").text def download(self, crawl_id: str) -> CrawlDownload: """`GET /crawls/{crawl_id}/download` — open the NDJSON file and diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 534e2a3..eba3b15 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -556,3 +556,11 @@ def test_crawl_api_error_imports_from_the_package_root(): from zenrows.crawl import CrawlAPIError as FromCrawl assert zenrows.CrawlAPIError is FromCrawl + + +@respx.mock +def test_content_takes_content_id_by_keyword(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/c_1/contents/ct_9").mock( + return_value=Response(200, text="") + ) + assert client.content("c_1", content_id="/v1/crawls/c_1/contents/ct_9") == "" From 76034ff217307f5277621d777080d0910831e952 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 14:29:25 +0200 Subject: [PATCH 13/20] test(crawl): check content_id in the naming check The naming check now covers content_id, the argument of the content operation, so a client that names it differently fails check 17. The MCP surface calls crawl_content with content_id. --- acceptance/crawl/surface/test.py | 3 ++- acceptance/crawl/tools/report.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 223017b..b78821d 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -539,7 +539,8 @@ def c21(): with open(f"{RESULTS}/naming.json", "w") as f: json.dump({"operations": {k: k for k in ["create", "get", "list", "results", "content", "download", "stop", "wait"]}, "params": {k: k for k in ["url", "depth", "max_items", "max_pages", "include_patterns", "exclude_patterns", - "output_format", "idempotency_key", "cursor", "limit", "timeout"]}}, f, indent=2) + "output_format", "idempotency_key", "cursor", "limit", "timeout", + "content_id"]}}, f, indent=2) record("17", "Naming", SKIP, "the API's names in this surface's casing", "computed by the report") diff --git a/acceptance/crawl/tools/report.py b/acceptance/crawl/tools/report.py index fd085e1..94488f8 100644 --- a/acceptance/crawl/tools/report.py +++ b/acceptance/crawl/tools/report.py @@ -23,7 +23,7 @@ # The API's operation and parameter names. Each surface uses them in its own casing. CANON_OPS = ["create", "get", "list", "results", "content", "download", "stop", "wait"] CANON_PARAMS = ["url", "depth", "max_items", "max_pages", "include_patterns", "exclude_patterns", "output_format", - "idempotency_key", "cursor", "limit", "timeout"] + "idempotency_key", "cursor", "limit", "timeout", "content_id"] def read(path, default=None): From 27671664d4085d6bbab595f546ba98ad1334b4c2 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 15:16:59 +0200 Subject: [PATCH 14/20] test(crawl): give check 9 a finished crawl to read Check 9 read content from the crawl check 20 stops; a stop that lands before any page is fetched left nothing to read. It now falls back to a one-item html crawl that runs to the end. --- acceptance/crawl/surface/test.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index b78821d..06888ff 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -419,13 +419,22 @@ def c9(): b = state.get("B") if not b: return SKIP, "no html crawl", "" - r = next((x for x in client.results(b.crawl_id) if x.content_url), None) + def first_fetched(crawl_id): + return next((x for x in client.results(crawl_id) if x.content_url), None) + + # The stop in check 20 can land before any page is fetched. Then read a + # one-item html crawl that runs to the end instead. + src = b.crawl_id + r = first_fetched(src) + if not r: + own = create(output_format="html", **{**BASE, "max_items": 1}) + client.wait(own.crawl_id) + src = own.crawl_id + r = first_fetched(src) if not r: - if (state.get("stop") or {}).get("status") == "stopped": - return SKIP, "the stop cancelled every page fetch, so no page has content to read (see check 20)", "" - return FAIL, "no result with content_url", "" + return FAIL, f"no result with content_url on {src}", "" cid = r.content_url.rstrip("/").rsplit("/", 1)[-1] - by_id, by_url = client.content(b.crawl_id, cid), client.content(b.crawl_id, r.content_url) + by_id, by_url = client.content(src, cid), client.content(src, r.content_url) ok = by_id == by_url and re.search(r" Date: Mon, 12 Oct 2026 15:49:09 +0200 Subject: [PATCH 15/20] fix(crawl): wait at most 30 s between crawl_busy stop retries --- src/zenrows/crawl/client.py | 16 ++++++++++++++-- tests/test_crawl_client.py | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 2b61a45..e4e439a 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -18,6 +18,7 @@ import json import logging +import math import random import time from collections.abc import Callable, Iterator @@ -50,6 +51,9 @@ # 503 `crawl_busy` on stop: nothing was saved and the crawl is still # running, so the stop is safe to repeat after `Retry-After`. _CRAWL_BUSY = "crawl_busy" +# The longest a crawl_busy `Retry-After` may make a stop wait, so a large +# server value cannot hang it. +_MAX_STOP_RETRY_AFTER = 30.0 _POLL_INTERVAL = 2.0 _POLL_BACKOFF = 1.5 @@ -270,7 +274,7 @@ def stop(self, crawl_id: str) -> CrawlStop: A 503 `crawl_busy` means the stop was not saved and the crawl is still running; `stop` retries it up to `retries` times, waiting - `Retry-After` seconds each time, then raises it. + `Retry-After` seconds (at most 30) each time, then raises it. """ attempt = 0 while True: @@ -281,7 +285,7 @@ def stop(self, crawl_id: str) -> CrawlStop: except CrawlAPIError as err: if err.status_code != 503 or err.code != _CRAWL_BUSY or attempt >= self._retries: raise - time.sleep(err.retry_after if err.retry_after is not None else 1.0) + time.sleep(_stop_retry_wait(err.retry_after)) attempt += 1 # ===== contents ===== @@ -342,6 +346,14 @@ def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl: interval = min(interval * _POLL_BACKOFF, _MAX_POLL_INTERVAL) +def _stop_retry_wait(retry_after: float | None) -> float: + """Seconds `stop` waits after a 503 crawl_busy: the server's + `Retry-After`, 1s when it is missing or unusable, at most 30s.""" + if retry_after is None or not math.isfinite(retry_after) or retry_after < 0: + return 1.0 + return min(retry_after, _MAX_STOP_RETRY_AFTER) + + def _crawl_status(response: httpx.Response) -> CrawlStatus | None: status = response.headers.get("X-Crawl-Status") return CrawlStatus(status) if status else None diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index d48256e..b56671c 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -572,3 +572,19 @@ def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep): assert crawl.status is CrawlStatus.FAILED assert crawl.error is not None and crawl.error.code is CrawlErrorCode.NO_ITEMS_FOUND assert no_sleep == [] + + +@pytest.mark.parametrize( + ("retry_after", "slept"), [("3600", 30.0), ("-1", 1.0), ("nan", 1.0), ("2", 2.0)] +) +@respx.mock +def test_stop_waits_at_most_30_seconds_for_crawl_busy(no_sleep, retry_after: str, slept: float): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=1) + respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + side_effect=[ + problem(503, "crawl_busy", headers={"Retry-After": retry_after}), + Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}), + ] + ) + client.stop("c_1") + assert no_sleep == [slept] From 94aadd170096a7680d10cc6676efc1fdb5085b85 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 16:22:40 +0200 Subject: [PATCH 16/20] docs(acceptance): correct the download row and the crawl count --- acceptance/crawl/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index f8e9e34..9a45870 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -69,7 +69,7 @@ and signal checks. | 7 | List | `limit` 2, then the next page: newest first; `next_cursor` absent or null on the last page | | 8 | Results | every kept URL once, each matching `CRAWL_INCLUDE`; on a running crawl, what is kept so far (MCP, CLI: `partial: true`) | | 9 | Content | by content id and by `content_url`: the same HTML | -| 10 | Download | a status, and one line per result with `url`, `content_status`, `content` (MCP has no download) | +| 10 | Download | a status, and one line per result with `url`, `content_status`, `content` | | 11 | Stop | running → `stopped`, `stop_reason: user`; ended → answers as it ended | | 12 | Not found | 404 `crawl_not_found` (MCP, CLI: `CRAWL_NOT_FOUND` with `crawl_id` and `server_code`) | | 13 | Invalid input | `depth` 0 → 422 `invalid_parameter` (MCP, CLI: `CRAWL_INVALID_REQUEST`) | @@ -90,7 +90,8 @@ Every crawl uses `depth` 1 and `max_pages` 10 or less. A run starts five crawls: one URLs-only crawl, one idempotency crawl that is stopped at once, one html crawl that is stopped with its pages in flight (check 20), and one that keeps nothing (check 19). On the default `CRAWL_URL`, a URLs-only crawl fetches 1 page and the html crawl 6, so a run costs about **10 pages**, and at most 50 if every crawl reaches its -`max_pages`. The report lists each crawl id and its `pages_fetched`. +`max_pages`. Check 9 starts a sixth crawl, one html item, when the stop in check 20 lands +before any page is fetched. It adds at most 10 pages. The report lists each crawl id and its `pages_fetched`. An account can run 3 active jobs at once, shared with Batch jobs. A run has at most 2 crawls running at a time. If a create still gets a 429, the run waits `Retry-After` and tries again. From 987dc207d6620123953dfa14b638fd97feddf62f Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 16:36:51 +0200 Subject: [PATCH 17/20] fix(crawl): keep the batch problem code a string The crawl client needs to know when an error body has no code. That change made Batch's public ProblemDetail.code optional. Restore the Batch contract and read the missing code from the crawl error body. --- src/zenrows/batch/errors.py | 8 ++++---- src/zenrows/crawl/errors.py | 15 +++++++++++++-- tests/test_batch_client.py | 1 + 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/zenrows/batch/errors.py b/src/zenrows/batch/errors.py index 40725b4..940a2be 100644 --- a/src/zenrows/batch/errors.py +++ b/src/zenrows/batch/errors.py @@ -24,7 +24,7 @@ class ProblemDetail: type: str title: str status: int - code: str | None + code: str detail: str | None = None instance: str | None = None extras: dict[str, Any] | None = None @@ -48,7 +48,7 @@ def from_response(cls, response: httpx.Response) -> "ProblemDetail | None": type=body.get("type", "about:blank"), title=body.get("title", "Error"), status=int(body.get("status", response.status_code)), - code=body.get("code"), + code=body.get("code", "internal"), detail=body.get("detail"), instance=body.get("instance"), extras=extras or None, @@ -68,10 +68,10 @@ def __init__(self, status_code: int, problem: ProblemDetail | None, raw: bytes): self.status_code = status_code self.problem = problem self.raw = raw - self.code: str = (problem.code if problem else None) or "internal" + self.code: str = problem.code if problem else "internal" self.detail: str | None = problem.detail if problem else None msg = ( - f"{status_code} {problem.title}: {problem.detail or self.code}" + f"{status_code} {problem.title}: {problem.detail or problem.code}" if problem else f"{status_code} (no problem body)" ) diff --git a/src/zenrows/crawl/errors.py b/src/zenrows/crawl/errors.py index aa32e87..2b70fb4 100644 --- a/src/zenrows/crawl/errors.py +++ b/src/zenrows/crawl/errors.py @@ -5,6 +5,7 @@ `status_code` + `code`; display `detail`, don't parse it. """ +import json import math import httpx @@ -16,6 +17,16 @@ CRAWL_NOT_ENABLED_CODE = "REQS008" +def _problem_code(raw: bytes) -> str | None: + # Read from the body: Batch's `ProblemDetail.code` defaults a missing + # code to "internal". + try: + code = json.loads(raw).get("code") + except (ValueError, AttributeError): + return None + return code if isinstance(code, str) else None + + class CrawlAPIError(Exception): """A non-2xx response from the Crawl API. @@ -40,14 +51,14 @@ def __init__( self.retry_after = retry_after self.problem = problem self.raw = raw - self.code: str | None = problem.code if problem else None + self.code: str | None = _problem_code(raw) if problem else None self.detail: str | None = problem.detail if problem else None if problem and self.code == CRAWL_NOT_ENABLED_CODE: msg = f"{status_code} Crawl is not enabled for this account ({self.code})" + ( f": {problem.detail}" if problem.detail else "" ) elif problem: - reason = problem.detail or problem.code + reason = problem.detail or self.code msg = f"{status_code} {problem.title}" + (f": {reason}" if reason else "") else: msg = f"{status_code} (no problem body)" diff --git a/tests/test_batch_client.py b/tests/test_batch_client.py index 3a2cd12..09d1f21 100644 --- a/tests/test_batch_client.py +++ b/tests/test_batch_client.py @@ -97,6 +97,7 @@ def test_problem_without_code_falls_back_to_internal(client: ZenRowsBatchClient) with pytest.raises(BatchAPIError) as exc_info: client.get_job("missing") assert exc_info.value.code == "internal" + assert exc_info.value.problem.code == "internal" assert str(exc_info.value) == "500 Server error: internal" From c57f2e0573189b7f7bc2aa48018c20677d6a7660 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 16:36:51 +0200 Subject: [PATCH 18/20] fix(acceptance): make the crawl harness portable and its clean-up robust realpath --relative-to is GNU-only, so on macOS every run was tagged +local-changes. The clean-up now continues past a network error and retries a stop answered 503 crawl_busy. The key scan skips only the source copy, and run.sh drops the builders this repo does not have. --- acceptance/crawl/run.sh | 30 ++++++---------------- acceptance/crawl/tools/cleanup.py | 40 ++++++++++++++++++++++-------- acceptance/crawl/tools/scan_key.py | 7 ++++-- 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/acceptance/crawl/run.sh b/acceptance/crawl/run.sh index 03d63bb..ec8cc1d 100644 --- a/acceptance/crawl/run.sh +++ b/acceptance/crawl/run.sh @@ -51,7 +51,7 @@ prepare_source() { # Tracked and untracked-but-not-ignored files, as a release build would see them. (cd "$root" && git ls-files -co --exclude-standard -z | tar --null -T - -cf -) | tar -C "$OUT_DIR/build/source" -xf - local sha; sha="$(git -C "$root" rev-parse HEAD)" - if [ -n "$(git -C "$root" status --porcelain -- . ":(exclude)$(realpath --relative-to="$root" "$HERE")")" ]; then + if [ -n "$(git -C "$root" status --porcelain -- . ":(exclude)$(git -C "$HERE" rev-parse --show-prefix)")" ]; then sha="$sha+local-changes" fi echo "$sha" >"$OUT_DIR/build/sha" @@ -64,41 +64,25 @@ prepare_source() { # ---- 2. Builder: the release artifact, in its own container ---- build_artifact() { mkdir -p "$OUT_DIR/build/artifact" - case "$SURFACE" in - node|mcp|cli) - log "build: npm pack" - docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ - -e SURFACE -e PNPM_VERSION "$NODE_IMAGE" bash /builders/build-npm.sh >"$OUT_DIR/build/build.log" 2>&1 ;; - python) - log "build: the wheel" - docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ - -e PY_BUILD_VERSION "$PYTHON_IMAGE" bash /builders/build-python.sh >"$OUT_DIR/build/build.log" 2>&1 ;; - go) - # Go has no artifact: the clean container runs `go get` at a pushed commit, as a user would. - local sha; sha="$(cat "$OUT_DIR/build/sha")" - case "$sha" in *+local-changes) - log "go: local changes are not tested; go get installs the commit ${sha%+local-changes} from GitHub" ;; - esac - echo "${sha%+local-changes}" >"$OUT_DIR/build/go_ref" ;; - esac + log "build: the wheel" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e PY_BUILD_VERSION "$PYTHON_IMAGE" bash /builders/build-python.sh >"$OUT_DIR/build/build.log" 2>&1 } # ---- 3. Clean test container: only the artifact, the checks and the inputs go in ---- run_checks() { rm -rf "$OUT_DIR/results"; mkdir -p "$OUT_DIR/results" - local image - case "$SURFACE" in node|mcp|cli) image="$NODE_IMAGE" ;; python) image="$PYTHON_IMAGE" ;; go) image="$GO_IMAGE" ;; esac - log "checks: a clean $image" + log "checks: a clean $PYTHON_IMAGE" local rc=0 docker_run --init \ -v "$OUT_DIR/build/artifact:/artifact:ro" \ -v "$HERE/surface:/harness:ro" \ -v "$OUT_DIR/results:/results" \ -e ZENROWS_API_KEY -e CRAWL_URL -e CRAWL_INCLUDE -e CRAWL_API_BASE \ - -e SURFACE -e GO_REF="$(cat "$OUT_DIR/build/go_ref" 2>/dev/null || true)" \ + -e SURFACE \ -e MCP_SDK_VERSION -e TYPESCRIPT_VERSION -e MYPY_VERSION \ -w /work --tmpfs /work:exec,size=2g \ - "$image" bash /harness/run-in-container.sh >"$OUT_DIR/results/container.log" 2>&1 || rc=$? + "$PYTHON_IMAGE" bash /harness/run-in-container.sh >"$OUT_DIR/results/container.log" 2>&1 || rc=$? log "checks: the container exited $rc (out/results/container.log)" } diff --git a/acceptance/crawl/tools/cleanup.py b/acceptance/crawl/tools/cleanup.py index 7228432..5213bae 100644 --- a/acceptance/crawl/tools/cleanup.py +++ b/acceptance/crawl/tools/cleanup.py @@ -4,6 +4,7 @@ stopped; then each crawl is read for its final status and coverage.pages_fetched, which the report uses for the cost table. Writes out/cleanup.json. """ +import http.client import json import os import sys @@ -20,12 +21,26 @@ def call(method, path): req = urllib.request.Request(BASE + path, method=method, headers={"X-API-Key": KEY}) try: with urllib.request.urlopen(req, timeout=60) as r: - return r.status, json.loads(r.read() or b"null") + return r.status, json.loads(r.read() or b"null"), r.headers except urllib.error.HTTPError as e: try: - return e.code, json.loads(e.read() or b"null") + return e.code, json.loads(e.read() or b"null"), e.headers except ValueError: - return e.code, None + return e.code, None, e.headers + + +def stop(crawl_id): + """POST stop, retrying 503 crawl_busy up to 3 times. Returns the final HTTP status.""" + for attempt in range(4): + status, body, headers = call("POST", f"/crawls/{crawl_id}/stop") + busy = status == 503 and isinstance(body, dict) and body.get("code") == "crawl_busy" + if not busy or attempt == 3: + return status + try: + wait = float(headers.get("Retry-After", 1)) + except ValueError: + wait = 1 + time.sleep(min(wait if wait >= 0 else 1, 30)) crawls = {} @@ -41,13 +56,18 @@ def call(method, path): rows = [] for crawl_id, entry in crawls.items(): - status, body = call("GET", f"/crawls/{crawl_id}?limit=1") stopped = False - if status == 200 and body and body.get("status") == "running": - call("POST", f"/crawls/{crawl_id}/stop") - stopped = True - time.sleep(2) - status, body = call("GET", f"/crawls/{crawl_id}?limit=1") + try: + status, body, _ = call("GET", f"/crawls/{crawl_id}?limit=1") + if status == 200 and body and body.get("status") == "running": + stopped = stop(crawl_id) < 300 + time.sleep(2) + status, body, _ = call("GET", f"/crawls/{crawl_id}?limit=1") + except (OSError, http.client.HTTPException, ValueError) as e: + # One unreachable crawl must not keep the others running. + rows.append({"crawl_id": crawl_id, **entry, "stopped_by_cleanup": stopped, "error": repr(e)}) + print(f"{crawl_id} not checked: {e!r}") + continue row = {"crawl_id": crawl_id, **entry, "stopped_by_cleanup": stopped, "http": status} if status == 200 and body: row.update(status=body.get("status"), stop_reason=body.get("stop_reason"), @@ -57,7 +77,7 @@ def call(method, path): print(f"{crawl_id} {row.get('status')} pages_fetched={row.get('pages_fetched')}" f"{' (stopped by clean-up)' if stopped else ''}") -still = [r["crawl_id"] for r in rows if r.get("status") == "running"] +still = [r["crawl_id"] for r in rows if r.get("status") == "running" or "error" in r] with open(os.path.join(OUT, "cleanup.json"), "w") as f: json.dump({"crawls": rows, "still_running": still, "checked_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, f, indent=2) diff --git a/acceptance/crawl/tools/scan_key.py b/acceptance/crawl/tools/scan_key.py index f6f0419..29350fd 100644 --- a/acceptance/crawl/tools/scan_key.py +++ b/acceptance/crawl/tools/scan_key.py @@ -3,9 +3,12 @@ import sys key = os.environ.get("ZENROWS_API_KEY", "").encode() +# The copy of the source under test holds the repo, not anything the run wrote. +source_dir = os.path.join(sys.argv[1], "build", "source") hits = [] -for root, _, files in os.walk(sys.argv[1]): - if os.sep + "build" in root: +for root, dirs, files in os.walk(sys.argv[1]): + if root == source_dir: + dirs.clear() continue for f in files: p = os.path.join(root, f) From 45c8a493dca1fd81a5b9652057e93a115de550f0 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 16:38:07 +0200 Subject: [PATCH 19/20] chore(acceptance): lint and format the acceptance harness make check covered only src/ and tests/, so the harness had about 96 ruff errors. Bring acceptance/ into the lint and format targets and fix the errors. --- Makefile | 8 +- acceptance/crawl/surface/test.py | 554 ++++++++++++++++++++----- acceptance/crawl/surface/typecheck.py | 23 +- acceptance/crawl/tools/build_failed.py | 16 +- acceptance/crawl/tools/cleanup.py | 31 +- acceptance/crawl/tools/report.py | 163 ++++++-- acceptance/crawl/tools/scan_key.py | 1 + 7 files changed, 634 insertions(+), 162 deletions(-) diff --git a/Makefile b/Makefile index ce7c930..216456f 100644 --- a/Makefile +++ b/Makefile @@ -20,14 +20,14 @@ typecheck: # Lint + format + type check (CI mode — no fixes). check: typecheck - uv run ruff check src/ tests/ - uv run ruff format --check src/ tests/ + uv run ruff check src/ tests/ acceptance/ + uv run ruff format --check src/ tests/ acceptance/ # Lint + format (writes fixes). lint: - uv run ruff check --fix src/ tests/ + uv run ruff check --fix src/ tests/ acceptance/ format: - uv run ruff format src/ tests/ + uv run ruff format src/ tests/ acceptance/ # Regenerate the pydantic v2 models from the backend's canonical spec. # docs/openapi.yaml is the SDK-local copy of the spec; refresh it from the diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 06888ff..410f5cb 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -24,7 +24,9 @@ CRAWL_URL = os.environ["CRAWL_URL"] INCLUDE = os.environ["CRAWL_INCLUDE"] API_BASE = os.environ.get("CRAWL_API_BASE", "https://api.zenrows.com/v1") -NO_MATCH = "/crawl-acceptance-matches-nothing/" # no URL contains it: the crawl keeps nothing (check 19) +NO_MATCH = ( + "/crawl-acceptance-matches-nothing/" # no URL contains it: the crawl keeps nothing (check 19) +) DOCUMENTED_DEFAULT = 10 # the default the README documents for max_pages and max_items (check 21) RESULTS = "/results" PASS, FAIL, SKIP = "PASS", "FAIL", "SKIP" @@ -46,12 +48,20 @@ def trim(v, n): def record(check, title, status, expected, actual, evidence=""): global _seq _seq += 1 - r = {"surface": SURFACE, "check": str(check), "title": title, "status": status, "expected": expected, - "actual": trim(actual, 800), "evidence": trim(evidence, 2500)} + r = { + "surface": SURFACE, + "check": str(check), + "title": title, + "status": status, + "expected": expected, + "actual": trim(actual, 800), + "evidence": trim(evidence, 2500), + } name = f"{_seq:03d}-{re.sub(r'[^A-Za-z0-9_.-]+', '_', str(check))}.json" with open(f"{RESULTS}/checks/{name}", "w") as f: json.dump(r, f, indent=2) - print(f"[{status}] python {check} {title}: {r['actual'].splitlines()[0][:200] if r['actual'] else ''}", flush=True) + first = r["actual"].splitlines()[0][:200] if r["actual"] else "" + print(f"[{status}] python {check} {title}: {first}", flush=True) def pass_if(ok): @@ -60,12 +70,16 @@ def pass_if(ok): def run_check(check, title, expected, fn, live=False): if live and not HAS_KEY: - return record(check, title, SKIP, expected, "ZENROWS_API_KEY not set (run without --no-key)") + return record( + check, title, SKIP, expected, "ZENROWS_API_KEY not set (run without --no-key)" + ) try: status, actual, evidence = fn() record(check, title, status, expected, actual, evidence) except Exception as e: # an unexpected exception is a FAIL with its traceback - record(check, title, FAIL, expected, f"raised {type(e).__name__}: {e}", traceback.format_exc()) + record( + check, title, FAIL, expected, f"raised {type(e).__name__}: {e}", traceback.format_exc() + ) def ledger(crawl_id, purpose): @@ -96,22 +110,38 @@ def crawls_since(t): # ---------- README: install (run-in-container.sh) and the Crawl quickstart ---------- with open(f"{RESULTS}/pip-install.log") as f: install_log = f.read() -record("readme-1", "Install (README: `pip install zenrows`; here the release wheel)", PASS, "the package installs", - f"zenrows {metadata.version('zenrows')} installed", f"$ pip install \n{install_log[-1200:]}") +record( + "readme-1", + "Install (README: `pip install zenrows`; here the release wheel)", + PASS, + "the package installs", + f"zenrows {metadata.version('zenrows')} installed", + f"$ pip install \n{install_log[-1200:]}", +) readme = metadata.metadata("zenrows").get_payload() or "" HEADING = "## Quickstart — Crawl API (new)" section = readme.split(HEADING, 1)[1].split("\n## ", 1)[0] if HEADING in readme else "" m = re.search(r"```python\n(.*?)```", section, re.S) block = m.group(1) if m else None -record("readme-2", "README has a Crawl quickstart", pass_if(bool(block)), "a ```python block under the Crawl quickstart", - f"{len(block.splitlines())} lines" if block else "not found", block or section[:600]) +record( + "readme-2", + "README has a Crawl quickstart", + pass_if(bool(block)), + "a ```python block under the Crawl quickstart", + f"{len(block.splitlines())} lines" if block else "not found", + block or section[:600], +) subs = [ ('"https://example.com/products/"', json.dumps(CRAWL_URL), "start URL → CRAWL_URL"), ("max_items=50,", "max_items=5,", "max_items 50 → 5"), ("max_pages=100,", "max_pages=10,", "max_pages 100 → 10 (at most 10)"), - ('include_patterns=["/product/"]', f"include_patterns=[{json.dumps(INCLUDE)}]", "include → CRAWL_INCLUDE"), + ( + 'include_patterns=["/product/"]', + f"include_patterns=[{json.dumps(INCLUDE)}]", + "include → CRAWL_INCLUDE", + ), ] quick, applied = block or "", [] for a, b, why in subs: @@ -131,26 +161,46 @@ def readme_run(): mine = crawls_since(since) for i in mine: ledger(i, "README quickstart") - return pass_if(code == 0), f"exit {code}; crawls: {', '.join(mine) or 'none'}", \ - f"substitutions: {'; '.join(applied)}\n$ python quickstart.py\n{out}" + return ( + pass_if(code == 0), + f"exit {code}; crawls: {', '.join(mine) or 'none'}", + f"substitutions: {'; '.join(applied)}\n$ python quickstart.py\n{out}", + ) -run_check("readme-3", "Run the README Crawl quickstart", "it runs to the end without an error", readme_run, live=True) +run_check( + "readme-3", + "Run the README Crawl quickstart", + "it runs to the end without an error", + readme_run, + live=True, +) # ---------- extra: a typed package (py.typed) under mypy --strict ---------- def mypy_check(): - code, out = sh([sys.executable, "-m", "mypy", "--strict", "--no-incremental", "/harness/typecheck.py"]) - untyped = "missing library stubs or py.typed" in out or "module is installed, but missing" in out + code, out = sh( + [sys.executable, "-m", "mypy", "--strict", "--no-incremental", "/harness/typecheck.py"] + ) + untyped = ( + "missing library stubs or py.typed" in out or "module is installed, but missing" in out + ) revealed = "def (url: builtins.str" in out errors = [line for line in out.splitlines() if ": error:" in line] - return pass_if(not untyped and revealed and not errors), \ - f"mypy exit {code}; py.typed seen {not untyped}; real signature revealed {revealed}; {len(errors)} errors", \ - f"$ mypy --strict typecheck.py (mypy {metadata.version('mypy')})\n{out}" + return ( + pass_if(not untyped and revealed and not errors), + f"mypy exit {code}; py.typed seen {not untyped}; " + f"real signature revealed {revealed}; {len(errors)} errors", + f"$ mypy --strict typecheck.py (mypy {metadata.version('mypy')})\n{out}", + ) -run_check("x-python-typed", "mypy --strict sees the package as typed (py.typed)", "no missing-stubs error; the real signatures", - mypy_check) +run_check( + "x-python-typed", + "mypy --strict sees the package as typed (py.typed)", + "no missing-stubs error; the real signatures", + mypy_check, +) # ---------- check 1, offline half: only the fields you set are sent ---------- captured = [] @@ -169,10 +219,20 @@ def do_POST(self): self.send_response(201) self.send_header("Content-Type", "application/json") self.end_headers() - self.wfile.write(json.dumps({ - "crawl_id": "c_capture", "status": "running", "url": body.get("url"), "depth": body.get("depth"), - "max_items": 10, "max_pages": 10, "created_at": datetime.now(UTC).isoformat(), - "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0}}).encode()) + self.wfile.write( + json.dumps( + { + "crawl_id": "c_capture", + "status": "running", + "url": body.get("url"), + "depth": body.get("depth"), + "max_items": 10, + "max_pages": 10, + "created_at": datetime.now(UTC).isoformat(), + "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0}, + } + ).encode() + ) def do_GET(self): if "c_nobody" in self.path: @@ -192,12 +252,24 @@ def do_GET(self): from zenrows import ZenRowsCrawlClient # noqa: E402 (imported once installed, as a user would) from zenrows.crawl import CrawlAPIError # noqa: E402 -cap = ZenRowsCrawlClient("capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0) +cap = ZenRowsCrawlClient( + "capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0 +) cap.create("https://example.com/", depth=1) -cap.create("https://example.com/", depth=1, max_items=5, max_pages=10, include_patterns=["/p/"], idempotency_key="idem-1") -sent_ok = sorted(captured[0]) == ["depth", "url"] and \ - sorted(captured[1]) == sorted(["url", "depth", "max_items", "max_pages", "include_patterns"]) -sent_line = f"sent (stand-in server): minimal={json.dumps(captured[0])} full={json.dumps(captured[1])}" +cap.create( + "https://example.com/", + depth=1, + max_items=5, + max_pages=10, + include_patterns=["/p/"], + idempotency_key="idem-1", +) +sent_ok = sorted(captured[0]) == ["depth", "url"] and sorted(captured[1]) == sorted( + ["url", "depth", "max_items", "max_pages", "include_patterns"] +) +sent_line = ( + f"sent (stand-in server): minimal={json.dumps(captured[0])} full={json.dumps(captured[1])}" +) # ---------- live ---------- client = ZenRowsCrawlClient(KEY or "none", base_url=API_BASE) @@ -231,15 +303,28 @@ def esum(e): def c1(): if not HAS_KEY: - return (SKIP if sent_ok else FAIL), f"live part skipped (no key); only set fields sent: {sent_ok}", sent_line + return ( + (SKIP if sent_ok else FAIL), + f"live part skipped (no key); only set fields sent: {sent_ok}", + sent_line, + ) t = time.time() a = state["A"] = create(**BASE) - echo = {"url": a.url, "depth": a.depth, "max_items": a.max_items, "max_pages": a.max_pages, - "include_patterns": a.include_patterns} + echo = { + "url": a.url, + "depth": a.depth, + "max_items": a.max_items, + "max_pages": a.max_pages, + "include_patterns": a.include_patterns, + } want = {"url": CRAWL_URL, **BASE} ok = sent_ok and a.status.value == "running" and a.crawl_id.startswith("c_") and echo == want - return pass_if(ok), f"{a.status.value}, {a.crawl_id}, {int((time.time() - t) * 1000)} ms, echo {json.dumps(echo)}; " \ - f"only set fields sent: {sent_ok}", f"{a.model_dump_json()}\n{sent_line}" + return ( + pass_if(ok), + f"{a.status.value}, {a.crawl_id}, {int((time.time() - t) * 1000)} ms, " + f"echo {json.dumps(echo)}; only set fields sent: {sent_ok}", + f"{a.model_dump_json()}\n{sent_line}", + ) def c5(): @@ -272,9 +357,18 @@ def c3(): v, e = err_of(lambda: client.create(CRAWL_URL, idempotency_key=key, **dict(body, max_items=4))) if v: ledger(v.crawl_id, "idempotency, different body (unexpected create)") - ok = c.crawl_id == again.crawl_id and e is not None and e.status_code == 422 and e.code == "idempotency_key_reused" + ok = ( + c.crawl_id == again.crawl_id + and e is not None + and e.status_code == 422 + and e.code == "idempotency_key_reused" + ) third = f"{e.status_code} {e.code}" if e else f"created {v.crawl_id}" - return pass_if(ok), f"first {c.crawl_id}, second {again.crawl_id}; different body → {third}", json.dumps(esum(e)) + return ( + pass_if(ok), + f"first {c.crawl_id}, second {again.crawl_id}; different body → {third}", + json.dumps(esum(e)), + ) def c11(): @@ -286,9 +380,18 @@ def c11(): if first.status.value == "completed" and reason != "user": return SKIP, f"the crawl had already ended on its own ({first.status.value}/{reason})", "" second, e = err_of(lambda: client.stop(c.crawl_id)) - ok = first.status.value == "stopped" and reason == "user" and e is None and second.status == first.status + ok = ( + first.status.value == "stopped" + and reason == "user" + and e is None + and second.status == first.status + ) two = f"error {e}" if e else second.status.value - return pass_if(ok), f"first: {first.status.value}/{reason}; second: {two}", first.model_dump_json() + return ( + pass_if(ok), + f"first: {first.status.value}/{reason}; second: {two}", + first.model_dump_json(), + ) def c4(): @@ -297,8 +400,11 @@ def c4(): return SKIP, "no crawl from check 1", "" e = state["ended"] = client.wait(a.crawl_id) reason = f"/{e.stop_reason.value}" if e.stop_reason else "" - return pass_if(e.status.value in TERMINAL), f"{e.status.value}{reason}; coverage {e.coverage.model_dump_json()}", \ - e.model_dump_json() + return ( + pass_if(e.status.value in TERMINAL), + f"{e.status.value}{reason}; coverage {e.coverage.model_dump_json()}", + e.model_dump_json(), + ) def c6(): @@ -314,10 +420,19 @@ def c6(): cursor = p.next_cursor urls = [u for _, _, us in pages for u in us] curs = [c for _, c, _ in pages if c is not None] - ok = all(n <= 2 for n, _, _ in pages) and pages[-1][1] is None and len(set(curs)) == len(curs) \ - and len(set(urls)) == len(urls) and (len(pages) >= 2 or e.coverage.items_found <= 2) + ok = ( + all(n <= 2 for n, _, _ in pages) + and pages[-1][1] is None + and len(set(curs)) == len(curs) + and len(set(urls)) == len(urls) + and (len(pages) >= 2 or e.coverage.items_found <= 2) + ) sizes = ",".join(str(n) for n, _, _ in pages) - return pass_if(ok), f"{len(pages)} pages of {sizes}; last next_cursor {pages[-1][1]}; {len(urls)} URLs", "" + return ( + pass_if(ok), + f"{len(pages)} pages of {sizes}; last next_cursor {pages[-1][1]}; {len(urls)} URLs", + "", + ) def c7(): @@ -332,8 +447,12 @@ def c7(): cursor = client.list(limit=100, cursor=cursor).next_cursor pages += 1 ok = len(p1.crawls) == 2 and newest and not overlap and not cursor - return pass_if(ok), f"newest first {newest}, no overlap {not overlap}; last page after {pages} pages: " \ - f"next_cursor {json.dumps(cursor)}", "" + return ( + pass_if(ok), + f"newest first {newest}, no overlap {not overlap}; last page after {pages} pages: " + f"next_cursor {json.dumps(cursor)}", + "", + ) def c8(): @@ -343,11 +462,27 @@ def c8(): urls = [r.url for r in client.results(e.crawl_id, limit=2)] distinct, match = len(set(urls)) == len(urls), all(INCLUDE in u for u in urls) p = state.get("partial") - run = (f"while running ({'running' if p[0] else 'already ended'}): {p[1]} URLs in {p[2]} ms, " - f"{'error ' + str(p[3]) if p[3] else 'returned'}") if p else "not read while running" - ok = bool(urls) and distinct and match and len(urls) == e.coverage.items_found and not (p and p[3]) - return pass_if(ok), f"{len(urls)} URLs (items_found {e.coverage.items_found}), distinct {distinct}, " \ - f"all match {match}; {run}", "\n".join(urls) + run = ( + ( + f"while running ({'running' if p[0] else 'already ended'}): {p[1]} URLs in {p[2]} ms, " + f"{'error ' + str(p[3]) if p[3] else 'returned'}" + ) + if p + else "not read while running" + ) + ok = ( + bool(urls) + and distinct + and match + and len(urls) == e.coverage.items_found + and not (p and p[3]) + ) + return ( + pass_if(ok), + f"{len(urls)} URLs (items_found {e.coverage.items_found}), distinct {distinct}, " + f"all match {match}; {run}", + "\n".join(urls), + ) def settle(crawl_id): @@ -361,7 +496,10 @@ def settle(crawl_id): def stop_mid_flight(): - """Create the html crawl and stop it as soon as it has kept URLs, while their pages are fetched.""" + """Create the html crawl and stop it as soon as it has kept URLs. + + The stop lands while their pages are fetched. + """ try: b = state["B"] = create(output_format="html", **BASE) g = None @@ -385,11 +523,17 @@ def c20(): t = time.time() box = {} # A daemon thread, so a results() that pages forever fails the check instead of hanging the run. - reader = threading.Thread(target=lambda: box.update(now=list(client.results(b.crawl_id))), daemon=True) + reader = threading.Thread( + target=lambda: box.update(now=list(client.results(b.crawl_id))), daemon=True + ) reader.start() reader.join(60) if "now" not in box: - return FAIL, "results() did not finish within 60 s on a crawl stopped with pages in flight", "" + return ( + FAIL, + "results() did not finish within 60 s on a crawl stopped with pages in flight", + "", + ) now, ms = box["now"], int((time.time() - t) * 1000) settled = settle(b.crawl_id) if settled != "settled": @@ -397,8 +541,12 @@ def c20(): later = [r.url for r in client.results(b.crawl_id)] same = sorted(r.url for r in now) == sorted(later) pending = sum(1 for r in now if r.content_status and r.content_status.value == "pending") - return pass_if(same and bool(now)), f"stopped with {st['items']} kept; read {len(now)} URLs ({pending} pending) in {ms} ms, " \ - f"then {settled}: {len(later)} URLs, same set {same}", "" + return ( + pass_if(same and bool(now)), + f"stopped with {st['items']} kept; read {len(now)} URLs ({pending} pending) in {ms} ms, " + f"then {settled}: {len(later)} URLs, same set {same}", + "", + ) def c2(): @@ -408,17 +556,28 @@ def c2(): if (state.get("stop") or {}).get("status") == "stopped": settle(b.crawl_id) rs = list(client.results(b.crawl_id)) - ok = b.output_format is not None and b.output_format.value == "html" and bool(rs) and all( - r.content_status and (r.content_status.value != "fetched" or r.content_url) for r in rs) + ok = ( + b.output_format is not None + and b.output_format.value == "html" + and bool(rs) + and all( + r.content_status and (r.content_status.value != "fetched" or r.content_url) for r in rs + ) + ) with_url = sum(1 for r in rs if r.content_url) - return pass_if(ok), f"{b.crawl_id} ({(state.get('stop') or {}).get('status', '-')}); {len(rs)} results; " \ - f"with content_url {with_url}", json.dumps([r.model_dump(mode="json") for r in rs[:2]]) + return ( + pass_if(ok), + f"{b.crawl_id} ({(state.get('stop') or {}).get('status', '-')}); {len(rs)} results; " + f"with content_url {with_url}", + json.dumps([r.model_dump(mode="json") for r in rs[:2]]), + ) def c9(): b = state.get("B") if not b: return SKIP, "no html crawl", "" + def first_fetched(crawl_id): return next((x for x in client.results(crawl_id) if x.content_url), None) @@ -436,7 +595,11 @@ def first_fetched(crawl_id): cid = r.content_url.rstrip("/").rsplit("/", 1)[-1] by_id, by_url = client.content(src, cid), client.content(src, r.content_url) ok = by_id == by_url and re.search(r" Date: Tue, 13 Oct 2026 11:02:15 +0200 Subject: [PATCH 20/20] feat(crawl): optional depth with 100-page and 100-item defaults, and stop retries on 429, 502, 503 and 504 Folds the two stacked changes into this branch so the Crawl client ships as one pull request: - depth is optional and unbounded; max_pages and max_items default to 100 - stop retries 429, 502, 503 and 504, waiting Retry-After up to 30 s, or 1 s when it is missing or unusable --- README.md | 14 +++--- acceptance/crawl/README.md | 12 +++-- acceptance/crawl/surface/test.py | 37 +++++++------- src/zenrows/crawl/client.py | 43 +++++++++------- src/zenrows/crawl/models.py | 4 +- tests/test_crawl_client.py | 84 +++++++++++++++++++++++++++----- 6 files changed, 136 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 97a67ac..9550c8e 100644 --- a/README.md +++ b/README.md @@ -496,9 +496,11 @@ See `src/zenrows/batch/client.py` or `help(ZenRowsBatchClient)`. Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. -A crawl starts from one URL, follows the links on each page up to -`depth` hops, and keeps the URLs that match your patterns, up to -`max_items`. It stays on the start URL's registrable domain; +A crawl starts from one URL, follows the links on each page, and keeps +the URLs that match your patterns, up to `max_items`. It stops at +`max_items` or `max_pages`, or when no link is left. `depth` limits the +link hops from the start URL; without it the crawl has no depth limit. +It stays on the start URL's registrable domain; subdomains count. It runs in the background: `create` returns at once with `status="running"`, and `wait` blocks until it ends. Without `output_format` a crawl returns URLs only; `output_format="html"` also @@ -512,9 +514,9 @@ client = ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"]) crawl = client.create( "https://www.scrapingcourse.com/ecommerce/", - depth=1, # link hops from the start URL - max_items=50, # stop after keeping 50 URLs (default 10) - max_pages=100, # stop after fetching 100 pages (default 10) + depth=1, # link hops from the start URL (default: no limit) + max_items=50, # stop after keeping 50 URLs (default 100) + max_pages=100, # stop after fetching 100 pages (default 100) include_patterns=["/product/"], # keep only URLs containing this ) crawl = client.wait(crawl.crawl_id) # polls up to timeout=600 s diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index 9a45870..86d782f 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -60,7 +60,7 @@ and signal checks. | # | Feature | Expected | | --- | --- | --- | -| 1 | Create | returns at once: `running`, a `crawl_id`, the echoed parameters; only the fields set are sent | +| 1 | Create | returns at once: `running`, a `crawl_id`, the echoed parameters; only the fields set are sent, and a create without `depth` sends none and reads back none | | 2 | Create with html | results get `content_status` and, once fetched, `content_url` | | 3 | Idempotency | same key and body → same `crawl_id`; different body → 422 `idempotency_key_reused` (MCP, CLI: `CRAWL_INVALID_REQUEST` with the advice to send a new key) | | 4 | Wait | returns the ended crawl; `failed` is returned, not raised (CLI: exit 1) | @@ -80,17 +80,19 @@ and signal checks. | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | | 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | -| 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (10) | +| 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | Extra checks for this surface: `mypy --strict` sees the package as typed (`py.typed`) and reads the real signatures. ## What it costs -Every crawl uses `depth` 1 and `max_pages` 10 or less. A run starts five crawls: the README run, +Every crawl uses `depth` 1. Every crawl but check 19's sets `max_pages` 10 or less; check 19's +leaves it out so check 21 can read the default (100). A run starts five crawls: the README run, one URLs-only crawl, one idempotency crawl that is stopped at once, one html crawl that is stopped with its pages in flight (check 20), and one that keeps nothing (check 19). On the default `CRAWL_URL`, a URLs-only crawl fetches 1 page and the -html crawl 6, so a run costs about **10 pages**, and at most 50 if every crawl reaches its -`max_pages`. Check 9 starts a sixth crawl, one html item, when the stop in check 20 lands +html crawl 6, so a run costs about **10 pages**. Check 19's crawl keeps nothing and fetches only +the start page at `depth` 1, so its default of 100 does not raise the cost. At most 140 if every +crawl reaches its `max_pages`. Check 9 starts a sixth crawl, one html item, when the stop in check 20 lands before any page is fetched. It adds at most 10 pages. The report lists each crawl id and its `pages_fetched`. An account can run 3 active jobs at once, shared with Batch jobs. A run has at most 2 crawls diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 410f5cb..1f1ba40 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -27,7 +27,7 @@ NO_MATCH = ( "/crawl-acceptance-matches-nothing/" # no URL contains it: the crawl keeps nothing (check 19) ) -DOCUMENTED_DEFAULT = 10 # the default the README documents for max_pages and max_items (check 21) +DOCUMENTED_DEFAULT = 100 # the default the README documents for max_pages and max_items (check 21) RESULTS = "/results" PASS, FAIL, SKIP = "PASS", "FAIL", "SKIP" TERMINAL = {"completed", "stopped", "failed"} @@ -219,20 +219,18 @@ def do_POST(self): self.send_response(201) self.send_header("Content-Type", "application/json") self.end_headers() - self.wfile.write( - json.dumps( - { - "crawl_id": "c_capture", - "status": "running", - "url": body.get("url"), - "depth": body.get("depth"), - "max_items": 10, - "max_pages": 10, - "created_at": datetime.now(UTC).isoformat(), - "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0}, - } - ).encode() - ) + crawl = { + "crawl_id": "c_capture", + "status": "running", + "url": body.get("url"), + "max_items": DOCUMENTED_DEFAULT, + "max_pages": DOCUMENTED_DEFAULT, + "created_at": datetime.now(UTC).isoformat(), + "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0}, + } + if "depth" in body: # as the API: a crawl created without depth omits it + crawl["depth"] = body["depth"] + self.wfile.write(json.dumps(crawl).encode()) def do_GET(self): if "c_nobody" in self.path: @@ -255,7 +253,7 @@ def do_GET(self): cap = ZenRowsCrawlClient( "capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0 ) -cap.create("https://example.com/", depth=1) +minimal = cap.create("https://example.com/") # depth is optional: no depth limit cap.create( "https://example.com/", depth=1, @@ -264,8 +262,11 @@ def do_GET(self): include_patterns=["/p/"], idempotency_key="idem-1", ) -sent_ok = sorted(captured[0]) == ["depth", "url"] and sorted(captured[1]) == sorted( - ["url", "depth", "max_items", "max_pages", "include_patterns"] +sent_ok = ( + sorted(captured[0]) == ["url"] + and minimal.depth is None + and sorted(captured[1]) + == sorted(["url", "depth", "max_items", "max_pages", "include_patterns"]) ) sent_line = ( f"sent (stand-in server): minimal={json.dumps(captured[0])} full={json.dumps(captured[1])}" diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 59bff3d..20bf213 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -1,7 +1,8 @@ """ZenRowsCrawlClient — the typed facade over the Crawl API (New). -A crawl starts from one URL, follows links up to `depth` hops, and -keeps the URLs that match its patterns, up to `max_items`. It runs as +A crawl starts from one URL, follows links (up to `depth` hops when +set), and keeps the URLs that match its patterns, up to `max_items`. It +stops at `max_items` or `max_pages`, or when no link is left. It runs as a job: `create` returns at once with `status=running`, and the URLs grow as pages finish. @@ -48,11 +49,13 @@ # 429 `too_many_crawls` is a capacity limit, not a transient failure. _CREATE_RETRY_STATUSES = _RETRYABLE_STATUSES - {429} -# 503 `crawl_busy` on stop: nothing was saved and the crawl is still -# running, so the stop is safe to repeat after `Retry-After`. -_CRAWL_BUSY = "crawl_busy" -# The longest a crawl_busy `Retry-After` may make a stop wait, so a large -# server value cannot hang it. +# A stop is safe to repeat (an ended crawl answers as it ended), so +# `stop` retries these statuses itself even though it is a POST without +# an idempotency key. 503 includes `crawl_busy`: nothing was saved and +# the crawl is still running. +_STOP_RETRY_STATUSES = _RETRYABLE_STATUSES +# The longest a `Retry-After` may make a stop wait, so a large server +# value cannot hang it. _MAX_STOP_RETRY_AFTER = 30.0 _POLL_INTERVAL = 2.0 @@ -126,7 +129,7 @@ def __init__( `create` that carries an `idempotency_key` (never on 429 for `create`). Retries use jittered exponential backoff and honor `Retry-After`; set `retries=0` to disable. It also bounds the - retries of a `stop` answered 503 `crawl_busy`.""" + retries of a `stop` answered 429, 502, 503 or 504.""" if not api_key: raise ValueError("ZenRowsCrawlClient: api_key is required.") self._retries = max(0, retries) @@ -163,7 +166,7 @@ def create( self, url: str, *, - depth: int, + depth: int | None = None, max_items: int | None = None, max_pages: int | None = None, include_patterns: list[str] | None = None, @@ -174,10 +177,12 @@ def create( """`POST /crawls` — start a crawl and return it, `running`. - `depth`: link hops to follow from `url` (1 to 100,000). + Omit it for no depth limit: the crawl follows links until + `max_items` or `max_pages` stops it, or no link is left. - `max_items`: stop after keeping this many URLs (server - default 10). + default 100). - `max_pages`: stop after fetching this many pages, which - bounds the cost (server default 10). + bounds the cost (server default 100). - `include_patterns` / `exclude_patterns`: substrings matched against each normalized URL. A URL is kept when it matches an include pattern (if any) and no exclude pattern. A crawl @@ -192,8 +197,9 @@ def create( Batch jobs, `create` raises 429 `too_many_crawls` without retrying; retry after the error's `retry_after` seconds. """ - body: dict[str, Any] = {"url": url, "depth": depth} + body: dict[str, Any] = {"url": url} optional: dict[str, Any] = { + "depth": depth, "max_items": max_items, "max_pages": max_pages, "include_patterns": include_patterns, @@ -272,9 +278,12 @@ def stop(self, crawl_id: str) -> CrawlStop: The answer carries no counts; read the final coverage and results with `get` once pages already in flight have finished. - A 503 `crawl_busy` means the stop was not saved and the crawl is - still running; `stop` retries it up to `retries` times, waiting - `Retry-After` seconds (at most 30) each time, then raises it. + Because repeating a stop is safe, `stop` retries a 429, 502, + 503 (including `crawl_busy`, which means the stop was not saved + and the crawl is still running) or 504 up to `retries` times, + waiting `Retry-After` seconds (at most 30; 1 when missing or + unusable) each time, then raises the last error. Network errors + are not retried. """ attempt = 0 while True: @@ -283,7 +292,7 @@ def stop(self, crawl_id: str) -> CrawlStop: self._t.request_json("POST", f"/crawls/{crawl_id}/stop") ) except CrawlAPIError as err: - if err.status_code != 503 or err.code != _CRAWL_BUSY or attempt >= self._retries: + if err.status_code not in _STOP_RETRY_STATUSES or attempt >= self._retries: raise time.sleep(_stop_retry_wait(err.retry_after)) attempt += 1 @@ -347,7 +356,7 @@ def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl: def _stop_retry_wait(retry_after: float | None) -> float: - """Seconds `stop` waits after a 503 crawl_busy: the server's + """Seconds `stop` waits after a 429/502/503/504: the server's `Retry-After`, 1s when it is missing or unusable, at most 30s.""" if retry_after is None or not math.isfinite(retry_after) or retry_after < 0: return 1.0 diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index eaea5d9..e1ab707 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -98,7 +98,9 @@ class Crawl(BaseModel): status: CrawlStatus url: str """The page the crawl started from.""" - depth: int + depth: int | None = None + """Link hops the crawl follows from `url`. Absent when the crawl was + created without one: it has no depth limit.""" max_items: int max_pages: int coverage: Coverage diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index c93fd20..bed4cc9 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -29,18 +29,20 @@ API_KEY = "test-key" -def crawl_body(**overrides): +def crawl_body(*, omit: tuple[str, ...] = (), **overrides): body = { "crawl_id": "c_1", "status": "running", "url": "https://example.com/", "depth": 1, - "max_items": 10, - "max_pages": 10, + "max_items": 100, + "max_pages": 100, "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0}, "created_at": "2026-10-08T09:00:00Z", } body.update(overrides) + for key in omit: + del body[key] return body @@ -93,18 +95,34 @@ def test_base_url_is_a_constructor_option_only(monkeypatch): @respx.mock def test_create_sends_only_required_fields(client: ZenRowsCrawlClient): route = respx.post(f"{BASE_URL}/crawls").mock( - return_value=Response(202, headers={"Location": "/v1/crawls/c_1"}, json=crawl_body()) + return_value=Response( + 202, headers={"Location": "/v1/crawls/c_1"}, json=crawl_body(omit=("depth",)) + ) ) - crawl = client.create("https://example.com/", depth=1) + crawl = client.create("https://example.com/") assert crawl.crawl_id == "c_1" assert crawl.status is CrawlStatus.RUNNING + assert crawl.depth is None # created without depth: no depth limit, none echoed sent = route.calls.last.request assert sent.headers["X-API-Key"] == API_KEY assert sent.headers["Content-Type"] == "application/json" assert "Idempotency-Key" not in sent.headers - assert json.loads(sent.content) == {"url": "https://example.com/", "depth": 1} + assert json.loads(sent.content) == {"url": "https://example.com/"} + + +@respx.mock +def test_create_sends_depth_when_given(client: ZenRowsCrawlClient): + route = respx.post(f"{BASE_URL}/crawls").mock(return_value=Response(202, json=crawl_body())) + + crawl = client.create("https://example.com/", depth=1) + + assert crawl.depth == 1 + assert json.loads(route.calls.last.request.content) == { + "url": "https://example.com/", + "depth": 1, + } @respx.mock @@ -213,7 +231,7 @@ def test_list_returns_one_page(client: ZenRowsCrawlClient): route = respx.get(f"{BASE_URL}/crawls").mock( side_effect=[ Response(200, json={"crawls": [crawl_body(crawl_id="c_2")], "next_cursor": "n1"}), - Response(200, json={"crawls": [crawl_body(crawl_id="c_1")]}), + Response(200, json={"crawls": [crawl_body(crawl_id="c_1", omit=("depth",))]}), ] ) @@ -222,6 +240,8 @@ def test_list_returns_one_page(client: ZenRowsCrawlClient): assert [c.crawl_id for c in first.crawls] == ["c_2"] assert [c.crawl_id for c in last.crawls] == ["c_1"] + assert first.crawls[0].depth == 1 + assert last.crawls[0].depth is None # created without depth assert last.next_cursor is None assert dict(route.calls[0].request.url.params) == {"limit": "1"} assert route.calls[1].request.url.params["cursor"] == "n1" @@ -281,11 +301,50 @@ def test_stop_surfaces_crawl_busy_after_retries(no_sleep): assert no_sleep == [1.0, 1.0] +@pytest.mark.parametrize( + ("status", "code"), + [(429, "rate_limited"), (502, "bad_gateway"), (503, "service_unavailable"), (504, None)], +) +@respx.mock +def test_stop_retries_transient_status(no_sleep, status: int, code: str | None): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + first = ( + problem(status, code, headers={"Retry-After": "2"}) + if code + else Response(status, headers={"Retry-After": "2"}) + ) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + side_effect=[ + first, + Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}), + ] + ) + + stopped = client.stop("c_1") + + assert stopped.status is CrawlStatus.STOPPED + assert route.call_count == 2 + assert no_sleep == [2.0] + + +@respx.mock +def test_stop_surfaces_502_after_retries(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=3) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock(return_value=problem(502, "bad_gateway")) + + with pytest.raises(CrawlAPIError) as exc_info: + client.stop("c_1") + + assert exc_info.value.status_code == 502 + assert route.call_count == 4 + assert no_sleep == [1.0, 1.0, 1.0] + + @respx.mock -def test_stop_does_not_retry_other_503(no_sleep): +def test_stop_does_not_retry_other_status(no_sleep): client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( - return_value=problem(503, "service_unavailable", headers={"Retry-After": "1"}) + return_value=problem(500, "internal_error") ) with pytest.raises(CrawlAPIError): @@ -577,12 +636,15 @@ def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep): @pytest.mark.parametrize( ("retry_after", "slept"), [("3600", 30.0), ("-1", 1.0), ("nan", 1.0), ("2", 2.0)] ) +@pytest.mark.parametrize(("status", "code"), [(503, "crawl_busy"), (429, "rate_limited")]) @respx.mock -def test_stop_waits_at_most_30_seconds_for_crawl_busy(no_sleep, retry_after: str, slept: float): +def test_stop_waits_at_most_30_seconds( + no_sleep, status: int, code: str, retry_after: str, slept: float +): client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=1) respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( side_effect=[ - problem(503, "crawl_busy", headers={"Retry-After": retry_after}), + problem(status, code, headers={"Retry-After": retry_after}), Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}), ] )