From 9b73985cfe73bc2a5f40343adfb8351757a7284a Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Wed, 14 Oct 2026 14:37:16 +0000 Subject: [PATCH 1/9] feat(crawl): return each page as Markdown `crawl create --output-format markdown` stores each kept URL's page converted to Markdown, read back with `crawl content` and `crawl download` as `--output-format html` already was. A value that is neither is still refused locally, before any network call. A crawl also reports `discovery`, the strategies it found URLs with, so `crawl get --json` carries it instead of dropping it. --- acceptance/crawl/surface/test.mjs | 28 ++++++++++++++++++++++++++++ skills/crawl/SKILL.md | 14 ++++++++------ src/cli/commands/crawl.ts | 12 +++++++----- src/core/crawl-api.ts | 12 +++++++++--- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index f1eb617..c5ef609 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -840,6 +840,34 @@ await liveCheck( }, ); +await liveCheck( + "22", + "Markdown output", + 'crawl create --output-format markdown is accepted and the crawl echoes it', + async () => { + const r = create(["--output-format", "markdown", "--max-items", "1", "--max-pages", "1"]); + if (r.code !== 0 || !r.json?.crawl_id) { + return { status: FAIL, actual: H.trim(show(r), 400) }; + } + z(["crawl", "stop", r.json.crawl_id, "--json"]); + const got = r.json.output_format ?? r.json.crawl?.output_format; + return { status: passIf(got === "markdown"), actual: `${r.json.crawl_id}: output_format ${got}` }; + }, +); +await liveCheck( + "23", + "Rejected output format", + "crawl create --output-format pdf is refused locally, before any network call", + async () => { + const r = create(["--output-format", "pdf"]); + const code = r.json?.error?.code; + return { + status: passIf(r.code === 1 && code === "INVALID_USAGE"), + actual: `exit ${r.code}, code ${code}`, + }; + }, +); + H.naming({ operations: { create: "crawl create", diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index cac5d8f..77737c6 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -11,8 +11,8 @@ Give Crawl one start URL; it follows the links on that page (and on the pages behind it, up to the depth you set) and returns the URLs it keeps. Use it when you know where the items live (a category page, a blog index, a docs section) but not the item URLs themselves. Once you have the URLs, read pages -with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page's HTML -with `--output-format html`. +with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page itself +with `--output-format html` or `--output-format markdown`. Crawl stays on the start URL's registrable domain; subdomains count. An account without Crawl access gets 403 `REQS008` → `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account"); @@ -50,16 +50,18 @@ zenrows crawl results --out urls.jsonl # same, as JSONL `partial: true`. Read again once the crawl ends. `--limit` sets the page size of each request. -## Page HTML +## Page content ``` zenrows crawl create --depth 1 --output-format html --max-pages 20 --follow +zenrows crawl create --depth 1 --output-format markdown --max-pages 20 --follow zenrows crawl results --json # fetched results carry content_url -zenrows crawl content # one page's HTML (or pass the content_url) -zenrows crawl download --out pages.jsonl # every URL + HTML, NDJSON +zenrows crawl content # one page (or pass the content_url) +zenrows crawl download --out pages.jsonl # every URL + page, NDJSON ``` -`--output-format html` uses up the page budget: each kept page is one more +`--output-format html` stores each page as fetched; `--output-format markdown` +stores it converted to Markdown. Either uses up the page budget: each kept page is one more fetch, and `--max-pages` counts it. Raise `--max-pages` to keep the same number of URLs. A result's `content_status` is `pending`, `fetched` or `failed`; only `fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). `download` diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index eeb615b..0fdfa71 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -53,8 +53,9 @@ export const crawl: Command = { " the local policy max_pages_per_run caps it (default 1000)", " --include-patterns

keep only URLs containing this substring (repeatable)", " --exclude-patterns

drop URLs containing this substring (repeatable)", - " --output-format html also fetch each kept URL's page HTML (read with content/download);", - " each kept page is one more fetch counted by --max-pages", + " --output-format also fetch each kept URL's page, as HTML or Markdown", + " (read with content/download); each kept page is one more", + " fetch counted by --max-pages", " --follow wait until the crawl ends (Ctrl-C stops the wait, not the crawl)", " --timeout with --follow: stop waiting after s seconds (default 600)", " --idempotency-key so a retried create starts no second crawl", @@ -151,12 +152,13 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { if (exclude.length) params.exclude_patterns = exclude; const outputFormat = asString(values["output-format"]); if (outputFormat !== undefined) { - if (outputFormat !== "html") { + if (outputFormat !== "html" && outputFormat !== "markdown") { throw new ToolkitError({ code: "INVALID_USAGE", message: `Invalid --output-format value '${outputFormat}'.`, - likely_cause: "--output-format takes html only. Without it, a crawl returns URLs only.", - next_action: "Pass --output-format html, or leave it out.", + likely_cause: + "--output-format takes html or markdown. Without it, a crawl returns URLs only.", + next_action: "Pass --output-format html or --output-format markdown, or leave it out.", }); } params.output_format = outputFormat; diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index b0f2871..150f64d 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -10,7 +10,7 @@ * Crawl SDKs do (see `retryStatuses`). `fetchImpl` / `sleepImpl` are injectable * for tests. * - * `output_format` takes `html` only (absent = URLs only). Response fields we do + * `output_format` takes `html` or `markdown` (absent = URLs only). Response fields we do * not model pass through untouched. */ import { Readable } from "node:stream"; @@ -66,6 +66,8 @@ export interface Crawl { include_patterns?: string[]; exclude_patterns?: string[]; output_format?: string; + /** The strategies the crawl finds URLs with; `["links"]` when the create left it out. */ + discovery?: string[]; duplicates_removed?: number; finished_at?: string; [k: string]: unknown; @@ -114,7 +116,7 @@ export interface CreateCrawlParams { include_patterns?: string[]; exclude_patterns?: string[]; /** `html` returns each kept URL's page; absent = URLs only. */ - output_format?: "html"; + output_format?: CrawlOutputFormat; } /** RFC 9457 problem body. */ @@ -129,6 +131,10 @@ interface ProblemJson { /** The API's "Crawl is not enabled for this account" code (403). */ export const CRAWL_NOT_ENABLED_CODE = "REQS008"; +/** The formats a crawl can store each kept URL's page in. */ +export type CrawlOutputFormat = "html" | "markdown"; + + /** A crawl stops progressing once its status is anything but `running`. */ export function isTerminal(status: string | undefined): boolean { return !!status && status !== "running"; @@ -463,7 +469,7 @@ export function stopCrawl(id: string, opts: CallOpts): Promise { return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { ...opts, crawlId: id, retryStop: true }); } -/** The page of one kept URL (HTML for `output_format: html`). */ +/** The page of one kept URL, in the crawl's `output_format`. */ export async function getCrawlContent( id: string, contentId: string, From fde76804371e2c89d2512f221d658bdfea4d5c53 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 08:27:55 +0000 Subject: [PATCH 2/9] chore(crawl): record the contract this surface follows The lock names the API description this surface was synced to, by its blob id, so a later drift check compares against it instead of reporting the surface as never synced. Two operations in that description are not served to callers and no surface exposes them; the lock lists them as pending, so a change to either does not read as drift here. --- acceptance/crawl/contract.lock | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 acceptance/crawl/contract.lock diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock new file mode 100644 index 0000000..435df9b --- /dev/null +++ b/acceptance/crawl/contract.lock @@ -0,0 +1,3 @@ +1adcbc2fd87e3f508317f3ce24c3d48573aec657 +pending GET /healthz +pending POST /internal/v1/crawls/{crawl_id}/stop From cd7f4d6b13c8fe2e97d7e8ec290bb0b414940ecc Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 12:54:34 +0000 Subject: [PATCH 3/9] fix(crawl): exit 1 when a crawl stops because credits ran out The Crawl API now ends a crawl that runs out of credits, or whose API key hits a credit cap, as `stopped` with `error.code` `insufficient_credits` or `api_key_cap_reached` and no `stop_reason`, instead of `failed`. The CLI treated every `stopped` crawl as a deliberate stop, so such a crawl would have exited 0 with `ok: true`. It now exits 1 with `CRAWL_FAILED` and the API's code as `server_code`, as it did when the crawl ended `failed`, with next-step advice for both codes. `crawl stop` on such a crawl says it had already ended. The crawl skill, the trace-debug skill and the README say so. --- README.md | 5 ++++- skills/crawl/SKILL.md | 10 +++++++--- skills/trace-debug/SKILL.md | 3 ++- src/cli/commands/crawl.ts | 24 ++++++++++++++++-------- src/core/crawl-api.ts | 2 +- tests/crawl-command.test.ts | 24 ++++++++++++++++++++++++ 6 files changed, 54 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1f7180f..b49c68f 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,10 @@ run out (default 600). When the time runs out, the command exits 0 and prints the crawl with status `running` and the hint `zenrows crawl wait ` (under `--json`, in `note`); the crawl keeps running. Ctrl-C stops the wait, not the crawl, and exits 130; resume with `zenrows crawl wait `. A crawl that ends `failed` exits 1 -with `CRAWL_FAILED`. +with `CRAWL_FAILED`. So does a crawl that ends `stopped` because the account ran +out of credits or the API key hit a credit cap (`server_code` +`insufficient_credits` or `api_key_cap_reached`); its fetched pages stay +readable. Errors carry the API's code as `server_code`, `crawl_id` on any call about one crawl, and `retry_after` on a 429. When the account has reached its limit of diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index 77737c6..547a017 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -77,8 +77,12 @@ zenrows crawl stop # stop it; kept URLs stay readable ``` A crawl ends `completed` (nothing left, or a cap hit: `stop_reason` -`max_items` / `max_pages`), `stopped` (you stopped it) or `failed` (`error.code` -and `error.detail` say why). Only `failed` exits non-zero (`CRAWL_FAILED`). +`max_items` / `max_pages`), `stopped` or `failed` (`error.code` and +`error.detail` say why). `stopped` with `stop_reason` `user` means you stopped +it. `stopped` with no `stop_reason` and `error.code` `insufficient_credits` or +`api_key_cap_reached` means the account ran out of credits or the API key hit a +credit cap; the pages fetched before that stay readable. `failed`, and a +`stopped` crawl with an `error`, exit non-zero (`CRAWL_FAILED`). `stop` on a crawl that already ended prints the status it ended with. With `--json`, `create`, `get`, `wait` and `stop` print @@ -120,7 +124,7 @@ The "Retry?" column is for errors that remain after these retries. | `CRAWL_TOO_MANY_CRAWLS` | 429 `too_many_crawls` on `create`: the account's active-job limit | yes, after `retry_after` | | `CRAWL_RATE_LIMITED` | any other 429: requests are being throttled | yes, after `retry_after` | | `CRAWL_BUSY` | 503 `crawl_busy` on `stop`: the stop was not saved yet and the crawl still runs | yes, run the stop again after `retry_after` | -| `CRAWL_FAILED` | the crawl ended `failed`, or another error | 5xx: yes, with backoff | +| `CRAWL_FAILED` | the crawl ended `failed`, or `stopped` on credits (`server_code` `insufficient_credits` / `api_key_cap_reached`), or another error | 5xx: yes, with backoff; credits: after they are added or the cap resets | | `BACKEND_UNAVAILABLE` | the API could not be reached | yes | Rules: start small (`--depth 1`, low caps) and check the results before a diff --git a/skills/trace-debug/SKILL.md b/skills/trace-debug/SKILL.md index 0948509..0a8f61b 100644 --- a/skills/trace-debug/SKILL.md +++ b/skills/trace-debug/SKILL.md @@ -54,7 +54,8 @@ zenrows trace export # JSON for sharing its limit of active jobs (3 by default), shared with its Batch jobs. Nothing was created; retry after `retry_after` seconds (see `zenrows crawl list`). -- `CRAWL_FAILED` → the crawl ended `failed`; `error.code` in +- `CRAWL_FAILED` → the crawl ended `failed`, or `stopped` because credits + ran out or the key hit a credit cap; `error.code` in `zenrows crawl get --json` says why, with `error.detail`. See [[crawl]]. - `PARAM_CONFLICT_AUTO_MANUAL` → drop the managed flags or add `--manual`. diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 0fdfa71..685dede 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -405,7 +405,9 @@ async function stopCmd(rest: string[], ctx: RunContext): Promise { assertUsable("crawl"); const apiKey = requireApiKey(); const s = await stopCrawl(id, { apiKey }); - const headline = s.status === "stopped" ? `Stopped crawl ${id}` : `Nothing to stop: crawl ${id} had already ended`; + // A crawl stopped on credits carries `error`: it had already ended, this call did not stop it. + const headline = + s.status === "stopped" && !s.error ? `Stopped crawl ${id}` : `Nothing to stop: crawl ${id} had already ended`; return printCrawl(s, ctx.json || values.json === true, headline); } @@ -428,22 +430,27 @@ async function waitUntilEnd(id: string, opts: { apiKey: string; timeoutMs?: numb } /** - * The error for a crawl that ended `failed`, or null for any other state. The - * crawl's own `error.code` picks the next action. + * The error for a crawl that ended `failed`, or `stopped` with an `error` (the + * account ran out of credits or the API key hit a credit cap), or null for any + * other state. The crawl's own `error.code` picks the next action. */ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { - if (c.status !== "failed") return null; + const creditStop = c.status === "stopped" && !!c.error; + if (c.status !== "failed" && !creditStop) return null; const code = c.error?.code; const why = [code, c.error?.detail].filter(Boolean).join(": "); const next: Record = { - insufficient_credits: "The account ran out of credits. Check `zenrows usage`, add credits, then start a new crawl.", + insufficient_credits: + "The account ran out of credits. The pages fetched so far stay readable. Check `zenrows usage`, add credits, then start a new crawl.", + api_key_cap_reached: + "This API key reached one of its credit caps. The pages fetched so far stay readable. Raise or remove the cap, or wait for it to reset, then start a new crawl.", seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch `.", no_items_found: "Nothing matched. Loosen --include-patterns / --exclude-patterns, raise or drop --depth, or check the start page with `zenrows fetch `.", }; return new ToolkitError({ code: "CRAWL_FAILED", - message: `Crawl ${c.crawl_id} failed.`, + message: creditStop ? `Crawl ${c.crawl_id} stopped before it finished.` : `Crawl ${c.crawl_id} failed.`, likely_cause: why || "The crawl ended in status failed without a reason.", next_action: (code && next[code]) || "Read the error, fix the cause, then start a new crawl.", suggested_commands: [`zenrows crawl get ${c.crawl_id}`], @@ -454,8 +461,9 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { /** * Print a crawl (or a stop answer), structured under --json, and return the exit - * code: 1 when it ended `failed`, else 0. `stopped` is a deliberate outcome, so - * it exits 0 but prints as a warning. `extra` carries the results a `get` read, + * code: 1 when it ended `failed`, or `stopped` because credits ran out or the + * API key hit a credit cap, else 0. A `stopped` crawl the caller stopped is a + * deliberate outcome, so it exits 0 but prints as a warning. `extra` carries the results a `get` read, * or the note of a wait that ran out. */ function printCrawl( diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index 150f64d..c4565c6 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -42,7 +42,7 @@ export interface CrawlCoverage { items_found: number; } -/** Why a crawl ended `failed` (part of the crawl, not an error response). */ +/** Why a crawl ended `failed`, or `stopped` because credits ran out or the API key hit a credit cap (part of the crawl, not an error response). */ export interface CrawlRunError { code: string; detail: string; diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index b46952f..05f18dc 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -335,6 +335,30 @@ test("crawl get exits 0 for completed and stopped crawls", async () => { } }); +test("crawl get exits 1 with CRAWL_FAILED on a crawl stopped on credits", async () => { + for (const code of ["insufficient_credits", "api_key_cap_reached"]) { + await withCrawlWorkspace( + () => + json({ + ...crawlBody, + status: "stopped", + error: { code, detail: "Batch stopped the crawl." }, + results: [{ url: "https://example.com/product/a" }], + next_cursor: null, + }), + async () => { + const { code: exit, out } = await run(["get", "c_1"]); + assert.equal(exit, 1, code); + assert.equal(out.ok, false); + assert.equal(out.status, "stopped"); + assert.equal(out.error.code, "CRAWL_FAILED"); + assert.equal(out.error.server_code, code); + assert.equal(out.results.length, 1); + }, + ); + } +}); + test("crawl get on an unknown id exits 1 with CRAWL_NOT_FOUND", async () => { await withCrawlWorkspace( () => json({ code: "crawl_not_found", title: "Crawl not found", detail: "No crawl c_x.", status: 404 }, 404), From abda2b5899049785290067d45b5ca36e95c4c29d Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 13:08:09 +0000 Subject: [PATCH 4/9] fix(crawl): report credit stops with the credit error code A crawl that ends `stopped` because the account ran out of credits or the API key hit a credit cap now exits 1 with `CRAWL_INSUFFICIENT_CREDITS` or `CRAWL_KEY_CAP_REACHED`, the codes a create refused with 402 already gets, instead of `CRAWL_FAILED`. It carries the same next-step advice as those codes, the API's `error.code` as `server_code`, its `error.detail` in `likely_cause`, and `crawl_id`. A `failed` crawl with either of those codes, as older deployments report it, maps the same way, so callers see one code whatever the server version. Any other error on a stopped or failed crawl stays `CRAWL_FAILED`. `quotaExhausted` takes `status: null`, as `keyCapReached` does, for an error read from a crawl rather than an HTTP 402. The crawl skill, the trace-debug skill and the README say so. --- README.md | 10 +++++---- skills/crawl/SKILL.md | 11 +++++---- skills/trace-debug/SKILL.md | 7 ++++-- src/cli/commands/crawl.ts | 30 +++++++++++++++++-------- src/core/errors.ts | 9 +++++--- tests/crawl-command.test.ts | 45 ++++++++++++++++++++++++++++--------- 6 files changed, 80 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index b49c68f..f931abd 100644 --- a/README.md +++ b/README.md @@ -203,10 +203,12 @@ run out (default 600). When the time runs out, the command exits 0 and prints the crawl with status `running` and the hint `zenrows crawl wait ` (under `--json`, in `note`); the crawl keeps running. Ctrl-C stops the wait, not the crawl, and exits 130; resume with `zenrows crawl wait `. A crawl that ends `failed` exits 1 -with `CRAWL_FAILED`. So does a crawl that ends `stopped` because the account ran -out of credits or the API key hit a credit cap (`server_code` -`insufficient_credits` or `api_key_cap_reached`); its fetched pages stay -readable. +with `CRAWL_FAILED`. A crawl that ends `stopped` because the account ran out of +credits or the API key hit a credit cap also exits 1, with the code a refused +create gets: `CRAWL_INSUFFICIENT_CREDITS` or `CRAWL_KEY_CAP_REACHED`, and the +API's `insufficient_credits` or `api_key_cap_reached` as `server_code`. Older +deployments end such a crawl `failed`; it gets the same codes. Its fetched pages +stay readable. Errors carry the API's code as `server_code`, `crawl_id` on any call about one crawl, and `retry_after` on a 429. When the account has reached its limit of diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index 547a017..0171a69 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -82,7 +82,10 @@ A crawl ends `completed` (nothing left, or a cap hit: `stop_reason` it. `stopped` with no `stop_reason` and `error.code` `insufficient_credits` or `api_key_cap_reached` means the account ran out of credits or the API key hit a credit cap; the pages fetched before that stay readable. `failed`, and a -`stopped` crawl with an `error`, exit non-zero (`CRAWL_FAILED`). +`stopped` crawl with an `error`, exit non-zero: `CRAWL_INSUFFICIENT_CREDITS` or +`CRAWL_KEY_CAP_REACHED` when credits ended it (the same codes a refused +`create` gets, also when an older deployment reports it `failed`), else +`CRAWL_FAILED`. `server_code` carries the API's `error.code`. `stop` on a crawl that already ended prints the status it ended with. With `--json`, `create`, `get`, `wait` and `stop` print @@ -116,15 +119,15 @@ The "Retry?" column is for errors that remain after these retries. | --- | --- | --- | | `CRAWL_NOT_ENABLED` | 403 `REQS008`: Crawl is not enabled for this account | no | | `AUTH_INVALID` | 401: the API key was rejected | no, log in again | -| `CRAWL_INSUFFICIENT_CREDITS` | 402: the account is out of credits (claim the account or add credits) | after credits renew or are added | -| `CRAWL_KEY_CAP_REACHED` | 402 `AUTH014` or `api_key_cap_reached`: this API key reached its credit cap | after the cap resets or is raised | +| `CRAWL_INSUFFICIENT_CREDITS` | 402 on `create`, or a crawl that ended on `insufficient_credits`: the account is out of credits (claim the account or add credits) | after credits renew or are added | +| `CRAWL_KEY_CAP_REACHED` | 402 `AUTH014` or `api_key_cap_reached` on `create`, or a crawl that ended on `api_key_cap_reached`: this API key reached its credit cap | after the cap resets or is raised | | `CRAWL_INVALID_REQUEST` | 400 or 422: fix the request. For `idempotency_key_reused`, send a new key or no key | no, not as is | | `CRAWL_REQUEST_IN_FLIGHT` | 409: the same Idempotency-Key is still in flight | yes, once the first request ends | | `CRAWL_NOT_FOUND` / `CRAWL_CONTENT_NOT_FOUND` | 404: wrong id, or the page has no content | no | | `CRAWL_TOO_MANY_CRAWLS` | 429 `too_many_crawls` on `create`: the account's active-job limit | yes, after `retry_after` | | `CRAWL_RATE_LIMITED` | any other 429: requests are being throttled | yes, after `retry_after` | | `CRAWL_BUSY` | 503 `crawl_busy` on `stop`: the stop was not saved yet and the crawl still runs | yes, run the stop again after `retry_after` | -| `CRAWL_FAILED` | the crawl ended `failed`, or `stopped` on credits (`server_code` `insufficient_credits` / `api_key_cap_reached`), or another error | 5xx: yes, with backoff; credits: after they are added or the cap resets | +| `CRAWL_FAILED` | the crawl ended `failed`, or `stopped` with an error other than credits, or another error | 5xx: yes, with backoff | | `BACKEND_UNAVAILABLE` | the API could not be reached | yes | Rules: start small (`--depth 1`, low caps) and check the results before a diff --git a/skills/trace-debug/SKILL.md b/skills/trace-debug/SKILL.md index 0a8f61b..54f243e 100644 --- a/skills/trace-debug/SKILL.md +++ b/skills/trace-debug/SKILL.md @@ -49,13 +49,16 @@ zenrows trace export # JSON for sharing enabled for this account. Do not retry; ask Zenrows support for access. - `CRAWL_INSUFFICIENT_CREDITS` / `CRAWL_KEY_CAP_REACHED` → Crawl's 402: the account is out of credits, or this key reached its credit cap (`AUTH014`). + The same codes come back for a crawl that credits ended (`stopped`, or + `failed` on older deployments; `server_code` `insufficient_credits` / + `api_key_cap_reached`); its fetched pages stay readable. Same advice as `POLICY_MAX_CREDITS_EXCEEDED` / `KEY_CREDIT_CAP_REACHED`. - `CRAWL_TOO_MANY_CRAWLS` → 429 `too_many_crawls`: the account has reached its limit of active jobs (3 by default), shared with its Batch jobs. Nothing was created; retry after `retry_after` seconds (see `zenrows crawl list`). -- `CRAWL_FAILED` → the crawl ended `failed`, or `stopped` because credits - ran out or the key hit a credit cap; `error.code` in +- `CRAWL_FAILED` → the crawl ended `failed`, or `stopped` with an error + other than credits; `error.code` in `zenrows crawl get --json` says why, with `error.detail`. See [[crawl]]. - `PARAM_CONFLICT_AUTO_MANUAL` → drop the managed flags or add `--manual`. diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 685dede..23a6369 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -30,7 +30,8 @@ import { type CreateCrawlParams, } from "../../core/crawl-api.ts"; import { asNumber, asString, parse, type Command, type RunContext } from "../command.ts"; -import { ToolkitError } from "../../core/errors.ts"; +import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "../../core/errors.ts"; +import { readAccount } from "../../core/agent-account.ts"; import { printError, writeOut, writeOutStream } from "../output.ts"; /** The API's default for `max_pages` (and `max_items`) when the flag is unset. */ @@ -432,18 +433,30 @@ async function waitUntilEnd(id: string, opts: { apiKey: string; timeoutMs?: numb /** * The error for a crawl that ended `failed`, or `stopped` with an `error` (the * account ran out of credits or the API key hit a credit cap), or null for any - * other state. The crawl's own `error.code` picks the next action. + * other state. A crawl the credits ended gets the code and advice a create + * refused with 402 gets (`CRAWL_INSUFFICIENT_CREDITS`, `CRAWL_KEY_CAP_REACHED`), + * whether the API reports it `stopped` or, as older deployments do, `failed`. + * Any other `error.code` picks the next action of a `CRAWL_FAILED`. */ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { const creditStop = c.status === "stopped" && !!c.error; if (c.status !== "failed" && !creditStop) return null; const code = c.error?.code; + const ids = { server_code: code, crawl_id: c.crawl_id }; + const where = `crawl ${c.crawl_id}`; + const detail = c.error?.detail || undefined; + const getCmd = `zenrows crawl get ${c.crawl_id}`; + if (code && isKeyCapReached(code)) { + const cap = keyCapReached(where, { status: null, detail }).toJSON(); + return new ToolkitError({ ...cap, ...ids, code: "CRAWL_KEY_CAP_REACHED", suggested_commands: [...(cap.suggested_commands ?? []), getCmd] }); + } + if (code === "insufficient_credits") { + const acct = readAccount(); + const quota = quotaExhausted(where, acct?.unclaimed ? acct.claimUrl : undefined, { status: null, detail }).toJSON(); + return new ToolkitError({ ...quota, ...ids, code: "CRAWL_INSUFFICIENT_CREDITS", suggested_commands: [...(quota.suggested_commands ?? []), getCmd] }); + } const why = [code, c.error?.detail].filter(Boolean).join(": "); const next: Record = { - insufficient_credits: - "The account ran out of credits. The pages fetched so far stay readable. Check `zenrows usage`, add credits, then start a new crawl.", - api_key_cap_reached: - "This API key reached one of its credit caps. The pages fetched so far stay readable. Raise or remove the cap, or wait for it to reset, then start a new crawl.", seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch `.", no_items_found: "Nothing matched. Loosen --include-patterns / --exclude-patterns, raise or drop --depth, or check the start page with `zenrows fetch `.", @@ -453,9 +466,8 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { message: creditStop ? `Crawl ${c.crawl_id} stopped before it finished.` : `Crawl ${c.crawl_id} failed.`, likely_cause: why || "The crawl ended in status failed without a reason.", next_action: (code && next[code]) || "Read the error, fix the cause, then start a new crawl.", - suggested_commands: [`zenrows crawl get ${c.crawl_id}`], - server_code: code, - crawl_id: c.crawl_id, + suggested_commands: [getCmd], + ...ids, }); } diff --git a/src/core/errors.ts b/src/core/errors.ts index 1c4da84..118d9f8 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -109,12 +109,14 @@ export class ToolkitError extends Error { * (so `claimUrl` is surfaced); for a real logged-in account there is nothing to * claim — the step is to add credits / upgrade in the dashboard. `status`/`detail` * let the caller record the actual upstream response (402 usage-limit, 429 - * quota, …) instead of a hardcoded status. + * quota, …) instead of a hardcoded status. `status: null` is for a crawl the + * credits stopped: its status call succeeded, so there is no HTTP error to cite, + * only `url` (the crawl). */ export function quotaExhausted( url: string, claimUrl?: string, - opts: { status?: number; detail?: string } = {}, + opts: { status?: number | null; detail?: string } = {}, ): ToolkitError { // Say that the allowance comes back. Without it this reads as a permanent paywall, // which is how the API's own AUTH004 text reads ("Purchase a new subscription to @@ -126,10 +128,11 @@ export function quotaExhausted( ? `You are on the Zenrows Free plan. Claim your account to keep your usage and add credits: ${claimUrl}. ${renewLine}` : `You are out of Zenrows credits. ${renewLine} To carry on now, add a credit pack (${BILLING_TOPUP_URL} opens the purchase directly) or upgrade your plan: ${PLANS_URL}`; const detail = opts.detail ? `${opts.detail.replace(/\.\s*$/, "")}. ` : ""; + const where = opts.status === null ? `Stopped ${url}` : `HTTP ${opts.status ?? 429} for ${url}`; return new ToolkitError({ code: "POLICY_MAX_CREDITS_EXCEEDED", message: "Zenrows request quota exhausted.", - likely_cause: `${detail}HTTP ${opts.status ?? 429} for ${url}`, + likely_cause: `${detail}${where}`, next_action: claimLine, suggested_commands: ["zenrows usage"], }); diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 05f18dc..3b4ebe2 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -335,30 +335,55 @@ test("crawl get exits 0 for completed and stopped crawls", async () => { } }); -test("crawl get exits 1 with CRAWL_FAILED on a crawl stopped on credits", async () => { - for (const code of ["insufficient_credits", "api_key_cap_reached"]) { +test("a crawl the credits ended exits 1 with the code a 402 create gets, stopped or failed", async () => { + const cases = [ + { status: "stopped", server: "insufficient_credits", code: "CRAWL_INSUFFICIENT_CREDITS", advice: /zenrows usage/ }, + { status: "stopped", server: "api_key_cap_reached", code: "CRAWL_KEY_CAP_REACHED", advice: /cap to reset/ }, + // Older deployments end such a crawl `failed`: callers see the same code. + { status: "failed", server: "insufficient_credits", code: "CRAWL_INSUFFICIENT_CREDITS", advice: /zenrows usage/ }, + { status: "failed", server: "api_key_cap_reached", code: "CRAWL_KEY_CAP_REACHED", advice: /cap to reset/ }, + ]; + for (const k of cases) { + const label = `${k.status} ${k.server}`; await withCrawlWorkspace( () => json({ ...crawlBody, - status: "stopped", - error: { code, detail: "Batch stopped the crawl." }, + status: k.status, + error: { code: k.server, detail: "Batch stopped the crawl." }, results: [{ url: "https://example.com/product/a" }], next_cursor: null, }), async () => { const { code: exit, out } = await run(["get", "c_1"]); - assert.equal(exit, 1, code); - assert.equal(out.ok, false); - assert.equal(out.status, "stopped"); - assert.equal(out.error.code, "CRAWL_FAILED"); - assert.equal(out.error.server_code, code); - assert.equal(out.results.length, 1); + assert.equal(exit, 1, label); + assert.equal(out.ok, false, label); + assert.equal(out.status, k.status, label); + assert.equal(out.error.code, k.code, label); + assert.equal(out.error.server_code, k.server, label); + assert.equal(out.error.crawl_id, "c_1", label); + assert.match(out.error.likely_cause, /Batch stopped the crawl\. Stopped crawl c_1/, label); + assert.match(out.error.next_action, k.advice, label); + assert.ok(out.error.suggested_commands.includes("zenrows crawl get c_1"), label); + assert.equal(out.results.length, 1, label); }, ); } }); +test("a crawl stopped with any other error code exits 1 with CRAWL_FAILED", async () => { + await withCrawlWorkspace( + () => json({ ...crawlBody, status: "stopped", error: { code: "internal", detail: "Something broke." }, results: [], next_cursor: null }), + async () => { + const { code, out } = await run(["get", "c_1"]); + assert.equal(code, 1); + assert.equal(out.error.code, "CRAWL_FAILED"); + assert.equal(out.error.server_code, "internal"); + assert.match(out.error.message, /stopped before it finished/); + }, + ); +}); + test("crawl get on an unknown id exits 1 with CRAWL_NOT_FOUND", async () => { await withCrawlWorkspace( () => json({ code: "crawl_not_found", title: "Crawl not found", detail: "No crawl c_x.", status: 404 }, 404), From 9ea2b08880c15defeed4aa212529a6d91b49964c Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 13:46:58 +0000 Subject: [PATCH 5/9] test(crawl): use the customer wording for credit-stop details The fixtures now carry the error.detail the server returns when credits end a crawl, one per error code. --- tests/crawl-command.test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 3b4ebe2..cc70b6c 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -335,6 +335,12 @@ test("crawl get exits 0 for completed and stopped crawls", async () => { } }); +// What the server says when credits end a crawl, as error.detail. +const creditStopDetail: Record = { + insufficient_credits: "The crawl stopped because your account has no credits left.", + api_key_cap_reached: "The crawl stopped because your API key reached one of its credit caps.", +}; + test("a crawl the credits ended exits 1 with the code a 402 create gets, stopped or failed", async () => { const cases = [ { status: "stopped", server: "insufficient_credits", code: "CRAWL_INSUFFICIENT_CREDITS", advice: /zenrows usage/ }, @@ -350,7 +356,7 @@ test("a crawl the credits ended exits 1 with the code a 402 create gets, stopped json({ ...crawlBody, status: k.status, - error: { code: k.server, detail: "Batch stopped the crawl." }, + error: { code: k.server, detail: creditStopDetail[k.server] }, results: [{ url: "https://example.com/product/a" }], next_cursor: null, }), @@ -362,7 +368,7 @@ test("a crawl the credits ended exits 1 with the code a 402 create gets, stopped assert.equal(out.error.code, k.code, label); assert.equal(out.error.server_code, k.server, label); assert.equal(out.error.crawl_id, "c_1", label); - assert.match(out.error.likely_cause, /Batch stopped the crawl\. Stopped crawl c_1/, label); + assert.ok(out.error.likely_cause.includes(`${creditStopDetail[k.server]} Stopped crawl c_1`), label); assert.match(out.error.next_action, k.advice, label); assert.ok(out.error.suggested_commands.includes("zenrows crawl get c_1"), label); assert.equal(out.results.length, 1, label); From 0d57b73fdb14ae5dd28178fe1b7a4c81d339f758 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 17:41:02 +0000 Subject: [PATCH 6/9] fix(crawl): treat patterns as a filter on the URLs a crawl returns The Crawl API now applies `include_patterns` and `exclude_patterns` only to the URLs a crawl returns. The crawl still opens on-site pages that fail them, to reach pages that match, and each of those fetches counts toward `max_pages` and is billed. A crawl whose patterns match no URL it read ends `completed` with no results instead of `failed` with `no_items_found`, and `no_items_found` now only means the pages read held nothing to follow. `crawl get`, `crawl wait` and `create --follow` print a note when a completed crawl with patterns kept nothing, telling the caller to loosen the patterns or raise `--depth` or `--max-pages`. The exit code stays 0. The `no_items_found` next action points at the start page instead of the patterns. The `--max-pages` and pattern help, the README, the crawl skill and acceptance check 19 say so. --- README.md | 10 +++++-- acceptance/crawl/README.md | 2 +- acceptance/crawl/surface/test.mjs | 21 ++++++------- skills/crawl/SKILL.md | 9 +++++- src/cli/commands/crawl.ts | 25 ++++++++++++++-- tests/crawl-command.test.ts | 50 +++++++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index f931abd..f5ab907 100644 --- a/README.md +++ b/README.md @@ -184,10 +184,16 @@ zenrows crawl stop # stop a running crawl `--depth` (link hops, 1 to 100000) is optional: leave it out and the crawl follows links until `--max-items` or `--max-pages` stops it, or nothing is left to open. `--max-items` and `--max-pages` cap the crawl (API default 100 each), -and `--max-pages` bounds the cost, since each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default +and `--max-pages` bounds the cost, since each page is one fetch. Every page the +crawl opens counts and is billed, including pages opened only to walk through to +matching URLs. The local policy `max_pages_per_run` caps `--max-pages` (default 1000). `--output-format html` uses up the page budget: each kept page is one more fetch, and `--max-pages` counts it. `--include-patterns` / -`--exclude-patterns` are repeatable substring filters on the URL. `get` and +`--exclude-patterns` are repeatable substring filters on the URLs the crawl +returns, not on the pages it opens: it still opens on-site pages that fail them +to reach pages that match, and `--max-pages` counts those fetches. A crawl whose +patterns match no URL it read ends `completed` with no results, exits 0, and +prints a note (under `--json`, in `note`). `get` and `list` read one page and take `--cursor` and `--limit`. `results` follows every page (`--limit` is the page size); on a running crawl it returns the URLs kept so far, with `partial: true`. `download` reports the crawl's `status` (`null` diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index 06388b8..cf2e43e 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -78,7 +78,7 @@ and signal checks. | 16 | Errors carry the server code | SDKs: `code` holds it, empty when the body has none; MCP, CLI: `server_code` | | 17 | Naming | the API's operation and parameter names, in this surface's casing | | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | -| 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | +| 19 | No match | an include pattern that matches nothing ends `completed` with 0 results and no `error`; SDKs return it; MCP and CLI say `ok: true` (CLI: exit 0, with a `note` about the patterns) | | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index c5ef609..13e6264 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -761,11 +761,11 @@ await check( }, ); -// ---------- 19-21: a failed crawl, results after a stop, the documented defaults ---------- +// ---------- 19-21: a crawl that matches nothing, results after a stop, the documented defaults ---------- await liveCheck( "19", - "Failed crawl", - "a crawl that keeps nothing ends failed (no_items_found); crawl wait exits 1 with ok: false and CRAWL_FAILED", + "No match", + "a crawl whose pattern matches nothing ends completed with 0 results and no error; crawl wait exits 0 with ok: true and a note", async () => { const r = createWith([CRAWL_URL, "--depth", "1", "--include-patterns", H.NO_MATCH]); F = r.json; @@ -773,15 +773,16 @@ await liveCheck( const w = z(["crawl", "wait", F.crawl_id, "--json"]); const j = w.json ?? {}; const ok = - w.code === 1 && - j.ok === false && - j.status === "failed" && - j.crawl?.error?.code === "no_items_found" && - j.error?.code === "CRAWL_FAILED" && - j.error?.server_code === "no_items_found"; + w.code === 0 && + j.ok === true && + j.status === "completed" && + j.crawl?.coverage?.items_found === 0 && + j.crawl?.error === undefined && + j.error === undefined && + typeof j.note === "string"; return { status: passIf(ok), - actual: `${F.crawl_id}: exit ${w.code}, status ${j.status}, ok ${j.ok}, error ${j.error?.code}/${j.error?.server_code}`, + actual: `${F.crawl_id}: exit ${w.code}, status ${j.status}, ok ${j.ok}, items ${j.crawl?.coverage?.items_found}, error ${j.crawl?.error?.code ?? "none"}, note ${j.note ? "yes" : "no"}`, evidence: show(w), }; }, diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index 0171a69..395b95c 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -34,11 +34,18 @@ zenrows crawl results --out urls.jsonl # same, as JSONL a depth has no `depth` field in its output. - `--max-items` (kept URLs) and `--max-pages` (fetched pages) cap the crawl; both default to 100. `--max-pages` bounds the cost: each page is one fetch. + Every page the crawl opens counts and is billed, including pages opened only + to walk through to matching URLs. The local policy `max_pages_per_run` caps `--max-pages` (default 1000); above it the CLI refuses with `POLICY_LIMIT_EXCEEDED` before any request. - `--include-patterns` / `--exclude-patterns` are substring matches on the normalized URL and repeat: - `--include-patterns /product/ --exclude-patterns ?add-to-cart`. + `--include-patterns /product/ --exclude-patterns ?add-to-cart`. They filter + the URLs the crawl returns, not the pages it opens: it still opens on-site + pages that fail them (such as `/page/2/`) to reach pages that match, and + `--max-pages` counts and bills those fetches. A crawl whose patterns match no + URL it read ends `completed` with no results, exits 0, and prints a note + (under `--json`, in `note`) to loosen them or raise `--depth` / `--max-pages`. - `create` returns at once. `--follow` (or `zenrows crawl wait `) polls until the crawl ends or `--timeout` seconds run out (default 600). When the time runs out, the command exits 0 and prints the crawl with status diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 23a6369..33ca5b1 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -51,9 +51,13 @@ export const crawl: Command = { " leave it out for no depth limit (the caps below stop the crawl)", " --max-items stop after keeping n URLs (API default 100)", " --max-pages stop after fetching n pages (API default 100; bounds cost);", + " every page opened counts and is billed, incl. pages opened", + " only to walk through to matching URLs;", " the local policy max_pages_per_run caps it (default 1000)", " --include-patterns

keep only URLs containing this substring (repeatable)", - " --exclude-patterns

drop URLs containing this substring (repeatable)", + " --exclude-patterns

drop URLs containing this substring (repeatable);", + " patterns filter the URLs kept, not the pages opened: the", + " crawl still opens on-site pages that fail them", " --output-format also fetch each kept URL's page, as HTML or Markdown", " (read with content/download); each kept page is one more", " fetch counted by --max-pages", @@ -76,7 +80,8 @@ export const crawl: Command = { " a crawl still running then is printed, not stopped", " --json print structured output: create/get/wait/stop print", " {ok, crawl_id, status, crawl}; get adds results and next_cursor;", - " a wait that runs out adds note", + " a wait that runs out, or a completed crawl whose patterns", + " matched nothing, adds note", ].join("\n"), async run(argv: string[], ctx: RunContext): Promise { const [sub, ...rest] = argv; @@ -459,7 +464,7 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { const next: Record = { seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch `.", no_items_found: - "Nothing matched. Loosen --include-patterns / --exclude-patterns, raise or drop --depth, or check the start page with `zenrows fetch `.", + "The pages the crawl read held nothing to follow (no links, or a block, consent or empty page). Check the start page with `zenrows fetch `, or start from a listing page.", }; return new ToolkitError({ code: "CRAWL_FAILED", @@ -471,6 +476,17 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { }); } +/** + * Set only when a crawl with patterns completed and kept nothing: the patterns + * matched no URL it read. The API ends that crawl `completed`, not `failed`. + */ +function noMatchNote(c: Crawl | CrawlStop): string | undefined { + const crawl = c as Crawl; + const patterns = [...(crawl.include_patterns ?? []), ...(crawl.exclude_patterns ?? [])]; + if (crawl.status !== "completed" || crawl.coverage?.items_found !== 0 || !patterns.length) return undefined; + return "No URL the crawl read matched --include-patterns / --exclude-patterns. Loosen them, or raise --depth or --max-pages."; +} + /** * Print a crawl (or a stop answer), structured under --json, and return the exit * code: 1 when it ended `failed`, or `stopped` because credits ran out or the @@ -485,6 +501,7 @@ function printCrawl( extra: { results?: CrawlResult[]; next_cursor?: string | null; note?: string } = {}, ): number { const err = crawlFailure(c); + const note = extra.note ?? noMatchNote(c); const crawlFields = { ...c } as Record; // A wait polls with one result to stay cheap; that page is not "the results". delete crawlFields.results; @@ -498,6 +515,7 @@ function printCrawl( status: c.status, crawl: crawlFields, ...extra, + note, ...(err ? { error: err.toJSON() } : {}), }, null, @@ -515,6 +533,7 @@ function printCrawl( log.info(` ${cov.items_found} item(s) found · ${cov.pages_fetched} page(s) fetched · ${cov.pages_failed} failed`); } if (extra.note) log.dim(` next: zenrows crawl wait ${c.crawl_id}`); + else if (note) log.warn(` ${note}`); if (err) printError(err, false); return err ? 1 : 0; } diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index cc70b6c..bbd5d8e 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -335,6 +335,56 @@ test("crawl get exits 0 for completed and stopped crawls", async () => { } }); +test("a completed crawl whose patterns matched nothing exits 0 with a note about the patterns", async () => { + const empty = { ...crawlBody, status: "completed", results: [], next_cursor: null }; + await withCrawlWorkspace( + () => json({ ...empty, include_patterns: ["/product/"] }), + async () => { + const { code, out } = await run(["get", "c_1"]); + assert.equal(code, 0); + assert.equal(out.ok, true); + assert.equal(out.error, undefined); + assert.match(out.note, /matched --include-patterns \/ --exclude-patterns/); + + let human = -1; + const err = await captureErr(async () => { + human = await crawl.run(["get", "c_1"], { json: false, yes: false }); + }); + assert.equal(human, 0); + assert.match(err, /Loosen them, or raise --depth or --max-pages/); + }, + ); + await withCrawlWorkspace( + () => json(empty), + async () => { + const { out } = await run(["get", "c_1"]); + assert.equal(out.note, undefined, "no patterns, no note"); + }, + ); +}); + +test("a no_items_found crawl points at the start page, not the patterns", async () => { + await withCrawlWorkspace( + () => + json({ + ...crawlBody, + status: "failed", + include_patterns: ["/product/"], + error: { code: "no_items_found", detail: "The crawl found no items." }, + results: [], + next_cursor: null, + }), + async () => { + const { code, out } = await run(["get", "c_1"]); + assert.equal(code, 1); + assert.equal(out.error.code, "CRAWL_FAILED"); + assert.match(out.error.next_action, /zenrows fetch /); + assert.doesNotMatch(out.error.next_action, /patterns/); + assert.equal(out.note, undefined); + }, + ); +}); + // What the server says when credits end a crawl, as error.detail. const creditStopDetail: Record = { insufficient_credits: "The crawl stopped because your account has no credits left.", From d5c5c708a7db3732b72d96ca598d7be2fe59abd1 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Fri, 16 Oct 2026 12:26:02 +0200 Subject: [PATCH 7/9] test(crawl): drop the surface lock The lock lives beside the contract, not in a surface. main already dropped it, and this branch had brought it back. --- acceptance/crawl/contract.lock | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 acceptance/crawl/contract.lock diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock deleted file mode 100644 index 435df9b..0000000 --- a/acceptance/crawl/contract.lock +++ /dev/null @@ -1,3 +0,0 @@ -1adcbc2fd87e3f508317f3ce24c3d48573aec657 -pending GET /healthz -pending POST /internal/v1/crawls/{crawl_id}/stop From 11f9443d163f22e7b77a8cd67e156eb873c8aa85 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 16 Oct 2026 11:19:51 +0000 Subject: [PATCH 8/9] fix(crawl): stop saying patterns only filter the output, since they limit the walk too Patterns limit both the pages a crawl opens and the URLs it returns, so the help, README and skill no longer say the crawl opens or bills pages that fail them. A completed crawl with 0 results still exits 0, and with patterns set its note says to check them against the URLs the start page links to. no_items_found now means Extract found no data on the start page, and seed_unreachable also covers an empty start page. --- README.md | 13 +++++------ skills/crawl/SKILL.md | 17 +++++++-------- src/cli/commands/crawl.ts | 22 +++++++++---------- tests/crawl-command.test.ts | 43 +++++++++++++++++++++++++++++++------ 4 files changed, 60 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index f5ab907..1844868 100644 --- a/README.md +++ b/README.md @@ -184,16 +184,13 @@ zenrows crawl stop # stop a running crawl `--depth` (link hops, 1 to 100000) is optional: leave it out and the crawl follows links until `--max-items` or `--max-pages` stops it, or nothing is left to open. `--max-items` and `--max-pages` cap the crawl (API default 100 each), -and `--max-pages` bounds the cost, since each page is one fetch. Every page the -crawl opens counts and is billed, including pages opened only to walk through to -matching URLs. The local policy `max_pages_per_run` caps `--max-pages` (default +and `--max-pages` bounds the cost, since each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default 1000). `--output-format html` uses up the page budget: each kept page is one more fetch, and `--max-pages` counts it. `--include-patterns` / -`--exclude-patterns` are repeatable substring filters on the URLs the crawl -returns, not on the pages it opens: it still opens on-site pages that fail them -to reach pages that match, and `--max-pages` counts those fetches. A crawl whose -patterns match no URL it read ends `completed` with no results, exits 0, and -prints a note (under `--json`, in `note`). `get` and +`--exclude-patterns` are repeatable substring filters on the URL. A crawl whose +pages held nothing new to follow ends `completed` with no results and exits 0. +When patterns were set, it prints a note (under `--json`, in `note`) to check +them against the URLs the start page links to. `get` and `list` read one page and take `--cursor` and `--limit`. `results` follows every page (`--limit` is the page size); on a running crawl it returns the URLs kept so far, with `partial: true`. `download` reports the crawl's `status` (`null` diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index a2c7895..d3fe8ec 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -34,18 +34,15 @@ zenrows crawl results --out urls.jsonl # same, as JSONL a depth has no `depth` field in its output. - `--max-items` (kept URLs) and `--max-pages` (fetched pages) cap the crawl; both default to 100. `--max-pages` bounds the cost: each page is one fetch. - Every page the crawl opens counts and is billed, including pages opened only - to walk through to matching URLs. The local policy `max_pages_per_run` caps `--max-pages` (default 1000); above it the CLI refuses with `POLICY_LIMIT_EXCEEDED` before any request. - `--include-patterns` / `--exclude-patterns` are substring matches on the normalized URL and repeat: - `--include-patterns /product/ --exclude-patterns ?add-to-cart`. They filter - the URLs the crawl returns, not the pages it opens: it still opens on-site - pages that fail them (such as `/page/2/`) to reach pages that match, and - `--max-pages` counts and bills those fetches. A crawl whose patterns match no - URL it read ends `completed` with no results, exits 0, and prints a note - (under `--json`, in `note`) to loosen them or raise `--depth` / `--max-pages`. + `--include-patterns /product/ --exclude-patterns ?add-to-cart`. +- A crawl whose pages held nothing new to follow (no link, only links to other + sites, or only links the patterns reject) ends `completed` with no results + and exits 0. When patterns were set, it prints a note (under `--json`, in + `note`) to check them against the URLs the start page links to. - `create` returns at once. `--follow` (or `zenrows crawl wait `) polls until the crawl ends or `--timeout` seconds run out (default 600). When the time runs out, the command exits 0 and prints the crawl with status @@ -92,7 +89,9 @@ credit cap; the pages fetched before that stay readable. `failed`, and a `stopped` crawl with an `error`, exit non-zero: `CRAWL_INSUFFICIENT_CREDITS` or `CRAWL_KEY_CAP_REACHED` when credits ended it (the same codes a refused `create` gets, also when an older deployment reports it `failed`), else -`CRAWL_FAILED`. `server_code` carries the API's `error.code`. +`CRAWL_FAILED`. `server_code` carries the API's `error.code`: +`seed_unreachable` when the start URL could not be fetched or returned an empty +page, `no_items_found` when Extract read the start page and found no data on it. `stop` on a crawl that already ended prints the status it ended with. With `--json`, `create`, `get`, `wait` and `stop` print diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index abd27a3..0a4a5a6 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -51,13 +51,9 @@ export const crawl: Command = { " leave it out for no depth limit (the caps below stop the crawl)", " --max-items stop after keeping n URLs (API default 100)", " --max-pages stop after fetching n pages (API default 100; bounds cost);", - " every page opened counts and is billed, incl. pages opened", - " only to walk through to matching URLs;", " the local policy max_pages_per_run caps it (default 1000)", " --include-patterns

keep only URLs containing this substring (repeatable)", - " --exclude-patterns

drop URLs containing this substring (repeatable);", - " patterns filter the URLs kept, not the pages opened: the", - " crawl still opens on-site pages that fail them", + " --exclude-patterns

drop URLs containing this substring (repeatable)", " --output-format also fetch each kept URL's page, as HTML or Markdown", " (read with content/download); each kept page is one more", " fetch counted by --max-pages", @@ -84,8 +80,8 @@ export const crawl: Command = { " a crawl still running then is printed, not stopped", " --json print structured output: create/get/wait/stop print", " {ok, crawl_id, status, crawl}; get adds results and next_cursor;", - " a wait that runs out, or a completed crawl whose patterns", - " matched nothing, adds note", + " a wait that runs out, or a completed crawl with patterns", + " and 0 results, adds note", ].join("\n"), async run(argv: string[], ctx: RunContext): Promise { const [sub, ...rest] = argv; @@ -496,9 +492,10 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { } const why = [code, c.error?.detail].filter(Boolean).join(": "); const next: Record = { - seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch `.", + seed_unreachable: + "The start URL could not be fetched or returned an empty page (the site may be blocking it). Check it with `zenrows fetch `.", no_items_found: - "The pages the crawl read held nothing to follow (no links, or a block, consent or empty page). Check the start page with `zenrows fetch `, or start from a listing page.", + "Extract read the start page and found no data on it. Start from a page Extract reads, such as a listing or a product page.", }; return new ToolkitError({ code: "CRAWL_FAILED", @@ -511,14 +508,15 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { } /** - * Set only when a crawl with patterns completed and kept nothing: the patterns - * matched no URL it read. The API ends that crawl `completed`, not `failed`. + * Set only when a crawl with patterns completed and kept nothing. The API ends a + * crawl whose pages held nothing new to follow `completed`, and does not count + * the links the patterns rejected, so the patterns are the first thing to check. */ function noMatchNote(c: Crawl | CrawlStop): string | undefined { const crawl = c as Crawl; const patterns = [...(crawl.include_patterns ?? []), ...(crawl.exclude_patterns ?? [])]; if (crawl.status !== "completed" || crawl.coverage?.items_found !== 0 || !patterns.length) return undefined; - return "No URL the crawl read matched --include-patterns / --exclude-patterns. Loosen them, or raise --depth or --max-pages."; + return "The crawl completed with no results. Check --include-patterns / --exclude-patterns against the URLs the start page links to."; } /** diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 8e91ad7..cf2193f 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -335,7 +335,7 @@ test("crawl get exits 0 for completed and stopped crawls", async () => { } }); -test("a completed crawl whose patterns matched nothing exits 0 with a note about the patterns", async () => { +test("a completed crawl with patterns and 0 results exits 0 with a note to check the patterns", async () => { const empty = { ...crawlBody, status: "completed", results: [], next_cursor: null }; await withCrawlWorkspace( () => json({ ...empty, include_patterns: ["/product/"] }), @@ -344,14 +344,14 @@ test("a completed crawl whose patterns matched nothing exits 0 with a note about assert.equal(code, 0); assert.equal(out.ok, true); assert.equal(out.error, undefined); - assert.match(out.note, /matched --include-patterns \/ --exclude-patterns/); + assert.match(out.note, /Check --include-patterns \/ --exclude-patterns against the URLs the start page links to/); let human = -1; const err = await captureErr(async () => { human = await crawl.run(["get", "c_1"], { json: false, yes: false }); }); assert.equal(human, 0); - assert.match(err, /Loosen them, or raise --depth or --max-pages/); + assert.match(err, /against the URLs the start page links to/); }, ); await withCrawlWorkspace( @@ -363,14 +363,18 @@ test("a completed crawl whose patterns matched nothing exits 0 with a note about ); }); -test("a no_items_found crawl points at the start page, not the patterns", async () => { +test("a no_items_found crawl says Extract found no data on the start page, not that the patterns matched nothing", async () => { await withCrawlWorkspace( () => json({ ...crawlBody, status: "failed", include_patterns: ["/product/"], - error: { code: "no_items_found", detail: "The crawl found no items." }, + error: { + code: "no_items_found", + detail: + "Extract read the start page and found no data on it, so the crawl had nothing to follow. Start from a page Extract reads, such as a listing or a product page.", + }, results: [], next_cursor: null, }), @@ -378,13 +382,40 @@ test("a no_items_found crawl points at the start page, not the patterns", async const { code, out } = await run(["get", "c_1"]); assert.equal(code, 1); assert.equal(out.error.code, "CRAWL_FAILED"); - assert.match(out.error.next_action, /zenrows fetch /); + assert.equal(out.error.server_code, "no_items_found"); + assert.match(out.error.next_action, /Extract read the start page and found no data on it/); + assert.match(out.error.next_action, /listing or a product page/); assert.doesNotMatch(out.error.next_action, /patterns/); assert.equal(out.note, undefined); }, ); }); +test("an empty start page fails seed_unreachable with exit 1 and CRAWL_FAILED", async () => { + await withCrawlWorkspace( + () => + json({ + ...crawlBody, + status: "failed", + error: { + code: "seed_unreachable", + detail: "The start URL returned an empty page, so the crawl could not begin. Check the URL.", + }, + results: [], + next_cursor: null, + }), + async () => { + const { code, out } = await run(["get", "c_1"]); + assert.equal(code, 1); + assert.equal(out.ok, false); + assert.equal(out.error.code, "CRAWL_FAILED"); + assert.equal(out.error.server_code, "seed_unreachable"); + assert.match(out.error.next_action, /could not be fetched or returned an empty page/); + assert.equal(out.note, undefined); + }, + ); +}); + // What the server says when credits end a crawl, as error.detail. const creditStopDetail: Record = { insufficient_credits: "The crawl stopped because your account has no credits left.", From 63d5b758197484f9090cdef006f6d3ef13362fdb Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 16 Oct 2026 11:24:53 +0000 Subject: [PATCH 9/9] fix(crawl): describe no_items_found without product names, so the copy matches the API The next_action and the skill now say the crawl read the start page but found no data it could use, and suggest starting from a listing or product page. --- skills/crawl/SKILL.md | 3 ++- src/cli/commands/crawl.ts | 2 +- tests/crawl-command.test.ts | 9 +++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index d3fe8ec..607f93b 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -91,7 +91,8 @@ credit cap; the pages fetched before that stay readable. `failed`, and a `create` gets, also when an older deployment reports it `failed`), else `CRAWL_FAILED`. `server_code` carries the API's `error.code`: `seed_unreachable` when the start URL could not be fetched or returned an empty -page, `no_items_found` when Extract read the start page and found no data on it. +page, `no_items_found` when the crawl read the start page but found no data it +could use. `stop` on a crawl that already ended prints the status it ended with. With `--json`, `create`, `get`, `wait` and `stop` print diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 0a4a5a6..e4a7b3f 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -495,7 +495,7 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { seed_unreachable: "The start URL could not be fetched or returned an empty page (the site may be blocking it). Check it with `zenrows fetch `.", no_items_found: - "Extract read the start page and found no data on it. Start from a page Extract reads, such as a listing or a product page.", + "The crawl read the start page but found no data it could use, so it had nothing to follow. Start from a listing or product page.", }; return new ToolkitError({ code: "CRAWL_FAILED", diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index cf2193f..2d3abf5 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -363,7 +363,7 @@ test("a completed crawl with patterns and 0 results exits 0 with a note to check ); }); -test("a no_items_found crawl says Extract found no data on the start page, not that the patterns matched nothing", async () => { +test("a no_items_found crawl says the start page held no data the crawl could use, not that the patterns matched nothing", async () => { await withCrawlWorkspace( () => json({ @@ -373,7 +373,7 @@ test("a no_items_found crawl says Extract found no data on the start page, not t error: { code: "no_items_found", detail: - "Extract read the start page and found no data on it, so the crawl had nothing to follow. Start from a page Extract reads, such as a listing or a product page.", + "The crawl read the start page but found no data it could use, so it had nothing to follow. Start from a listing or product page.", }, results: [], next_cursor: null, @@ -383,8 +383,9 @@ test("a no_items_found crawl says Extract found no data on the start page, not t assert.equal(code, 1); assert.equal(out.error.code, "CRAWL_FAILED"); assert.equal(out.error.server_code, "no_items_found"); - assert.match(out.error.next_action, /Extract read the start page and found no data on it/); - assert.match(out.error.next_action, /listing or a product page/); + assert.match(out.error.next_action, /read the start page but found no data it could use/); + assert.match(out.error.next_action, /Start from a listing or product page/); + assert.doesNotMatch(out.error.next_action, /Extract|Fetch|Batch/); assert.doesNotMatch(out.error.next_action, /patterns/); assert.equal(out.note, undefined); },