From 9b73985cfe73bc2a5f40343adfb8351757a7284a Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Wed, 14 Oct 2026 14:37:16 +0000 Subject: [PATCH 1/6] feat(crawl): return each page as Markdown `crawl create --output-format markdown` stores each kept URL's page converted to Markdown, read back with `crawl content` and `crawl download` as `--output-format html` already was. A value that is neither is still refused locally, before any network call. A crawl also reports `discovery`, the strategies it found URLs with, so `crawl get --json` carries it instead of dropping it. --- acceptance/crawl/surface/test.mjs | 28 ++++++++++++++++++++++++++++ skills/crawl/SKILL.md | 14 ++++++++------ src/cli/commands/crawl.ts | 12 +++++++----- src/core/crawl-api.ts | 12 +++++++++--- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index f1eb617..c5ef609 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -840,6 +840,34 @@ await liveCheck( }, ); +await liveCheck( + "22", + "Markdown output", + 'crawl create --output-format markdown is accepted and the crawl echoes it', + async () => { + const r = create(["--output-format", "markdown", "--max-items", "1", "--max-pages", "1"]); + if (r.code !== 0 || !r.json?.crawl_id) { + return { status: FAIL, actual: H.trim(show(r), 400) }; + } + z(["crawl", "stop", r.json.crawl_id, "--json"]); + const got = r.json.output_format ?? r.json.crawl?.output_format; + return { status: passIf(got === "markdown"), actual: `${r.json.crawl_id}: output_format ${got}` }; + }, +); +await liveCheck( + "23", + "Rejected output format", + "crawl create --output-format pdf is refused locally, before any network call", + async () => { + const r = create(["--output-format", "pdf"]); + const code = r.json?.error?.code; + return { + status: passIf(r.code === 1 && code === "INVALID_USAGE"), + actual: `exit ${r.code}, code ${code}`, + }; + }, +); + H.naming({ operations: { create: "crawl create", diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index cac5d8f..77737c6 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -11,8 +11,8 @@ Give Crawl one start URL; it follows the links on that page (and on the pages behind it, up to the depth you set) and returns the URLs it keeps. Use it when you know where the items live (a category page, a blog index, a docs section) but not the item URLs themselves. Once you have the URLs, read pages -with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page's HTML -with `--output-format html`. +with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page itself +with `--output-format html` or `--output-format markdown`. Crawl stays on the start URL's registrable domain; subdomains count. An account without Crawl access gets 403 `REQS008` → `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account"); @@ -50,16 +50,18 @@ zenrows crawl results --out urls.jsonl # same, as JSONL `partial: true`. Read again once the crawl ends. `--limit` sets the page size of each request. -## Page HTML +## Page content ``` zenrows crawl create --depth 1 --output-format html --max-pages 20 --follow +zenrows crawl create --depth 1 --output-format markdown --max-pages 20 --follow zenrows crawl results --json # fetched results carry content_url -zenrows crawl content # one page's HTML (or pass the content_url) -zenrows crawl download --out pages.jsonl # every URL + HTML, NDJSON +zenrows crawl content # one page (or pass the content_url) +zenrows crawl download --out pages.jsonl # every URL + page, NDJSON ``` -`--output-format html` uses up the page budget: each kept page is one more +`--output-format html` stores each page as fetched; `--output-format markdown` +stores it converted to Markdown. Either uses up the page budget: each kept page is one more fetch, and `--max-pages` counts it. Raise `--max-pages` to keep the same number of URLs. A result's `content_status` is `pending`, `fetched` or `failed`; only `fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). `download` diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index eeb615b..0fdfa71 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -53,8 +53,9 @@ export const crawl: Command = { " the local policy max_pages_per_run caps it (default 1000)", " --include-patterns

keep only URLs containing this substring (repeatable)", " --exclude-patterns

drop URLs containing this substring (repeatable)", - " --output-format html also fetch each kept URL's page HTML (read with content/download);", - " each kept page is one more fetch counted by --max-pages", + " --output-format also fetch each kept URL's page, as HTML or Markdown", + " (read with content/download); each kept page is one more", + " fetch counted by --max-pages", " --follow wait until the crawl ends (Ctrl-C stops the wait, not the crawl)", " --timeout with --follow: stop waiting after s seconds (default 600)", " --idempotency-key so a retried create starts no second crawl", @@ -151,12 +152,13 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { if (exclude.length) params.exclude_patterns = exclude; const outputFormat = asString(values["output-format"]); if (outputFormat !== undefined) { - if (outputFormat !== "html") { + if (outputFormat !== "html" && outputFormat !== "markdown") { throw new ToolkitError({ code: "INVALID_USAGE", message: `Invalid --output-format value '${outputFormat}'.`, - likely_cause: "--output-format takes html only. Without it, a crawl returns URLs only.", - next_action: "Pass --output-format html, or leave it out.", + likely_cause: + "--output-format takes html or markdown. Without it, a crawl returns URLs only.", + next_action: "Pass --output-format html or --output-format markdown, or leave it out.", }); } params.output_format = outputFormat; diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index b0f2871..150f64d 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -10,7 +10,7 @@ * Crawl SDKs do (see `retryStatuses`). `fetchImpl` / `sleepImpl` are injectable * for tests. * - * `output_format` takes `html` only (absent = URLs only). Response fields we do + * `output_format` takes `html` or `markdown` (absent = URLs only). Response fields we do * not model pass through untouched. */ import { Readable } from "node:stream"; @@ -66,6 +66,8 @@ export interface Crawl { include_patterns?: string[]; exclude_patterns?: string[]; output_format?: string; + /** The strategies the crawl finds URLs with; `["links"]` when the create left it out. */ + discovery?: string[]; duplicates_removed?: number; finished_at?: string; [k: string]: unknown; @@ -114,7 +116,7 @@ export interface CreateCrawlParams { include_patterns?: string[]; exclude_patterns?: string[]; /** `html` returns each kept URL's page; absent = URLs only. */ - output_format?: "html"; + output_format?: CrawlOutputFormat; } /** RFC 9457 problem body. */ @@ -129,6 +131,10 @@ interface ProblemJson { /** The API's "Crawl is not enabled for this account" code (403). */ export const CRAWL_NOT_ENABLED_CODE = "REQS008"; +/** The formats a crawl can store each kept URL's page in. */ +export type CrawlOutputFormat = "html" | "markdown"; + + /** A crawl stops progressing once its status is anything but `running`. */ export function isTerminal(status: string | undefined): boolean { return !!status && status !== "running"; @@ -463,7 +469,7 @@ export function stopCrawl(id: string, opts: CallOpts): Promise { return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { ...opts, crawlId: id, retryStop: true }); } -/** The page of one kept URL (HTML for `output_format: html`). */ +/** The page of one kept URL, in the crawl's `output_format`. */ export async function getCrawlContent( id: string, contentId: string, From fde76804371e2c89d2512f221d658bdfea4d5c53 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 08:27:55 +0000 Subject: [PATCH 2/6] chore(crawl): record the contract this surface follows The lock names the API description this surface was synced to, by its blob id, so a later drift check compares against it instead of reporting the surface as never synced. Two operations in that description are not served to callers and no surface exposes them; the lock lists them as pending, so a change to either does not read as drift here. --- acceptance/crawl/contract.lock | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 acceptance/crawl/contract.lock diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock new file mode 100644 index 0000000..435df9b --- /dev/null +++ b/acceptance/crawl/contract.lock @@ -0,0 +1,3 @@ +1adcbc2fd87e3f508317f3ce24c3d48573aec657 +pending GET /healthz +pending POST /internal/v1/crawls/{crawl_id}/stop From 380d3357b07cc0c26be234d5f7ddcba7338e9882 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 14:12:45 +0000 Subject: [PATCH 3/6] feat(crawl): choose how a crawl finds the URLs it follows `crawl create --discovery links` asks for the strategy a crawl finds URLs with on each page it walks: the links in each page's HTML, which is also what it runs with when the flag is left out. Any other value is refused locally, before any network call. The acceptance harness also passes the run's base URL to the CLI. The CLI reads its own variable, which the harness never set, so a run against another base URL still sent its calls to production. --- acceptance/crawl/surface/test.mjs | 26 +++++++++++++++++++++++++- src/cli/commands/crawl.ts | 16 ++++++++++++++++ src/core/crawl-api.ts | 5 +++++ tests/crawl-command.test.ts | 19 ++++++++++++++++++- 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index c5ef609..062f2e1 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -28,7 +28,15 @@ mkdirSync(APP, { recursive: true }); mkdirSync(HOME, { recursive: true }); // Without a key, "invalid" keeps the CLI from creating an account. -const baseEnv = { ...process.env, HOME, NO_COLOR: "1", ZENROWS_API_KEY: HAS_KEY ? KEY : "invalid" }; +// The CLI reads its own variable, not CRAWL_API_BASE: without it every call +// would reach production whatever the run targets. +const baseEnv = { + ...process.env, + HOME, + NO_COLOR: "1", + ZENROWS_API_KEY: HAS_KEY ? KEY : "invalid", + ZENROWS_CRAWL_API_BASE: H.API_BASE, +}; const parse = (s) => { try { return JSON.parse(s); @@ -868,6 +876,21 @@ await liveCheck( }, ); +await liveCheck( + "24", + "Discovery", + "crawl create --discovery links is echoed back, and --discovery pagination is refused before any network call", + async () => { + const r = create(["--discovery", "links", "--max-items", "1", "--max-pages", "1"]); + if (r.code !== 0 || !r.json?.crawl_id) return { status: FAIL, actual: H.trim(show(r), 400) }; + z(["crawl", "stop", r.json.crawl_id, "--json"]); + const got = r.json.crawl?.discovery ?? r.json.discovery; + const refused = create(["--discovery", "pagination"]); + const ok = JSON.stringify(got) === JSON.stringify(["links"]) && refused.code === 1 && refused.json?.error?.code === "INVALID_USAGE"; + return { status: passIf(ok), actual: `discovery ${JSON.stringify(got)}; pagination exit ${refused.code} ${refused.json?.error?.code}` }; + }, +); + H.naming({ operations: { create: "crawl create", @@ -887,6 +910,7 @@ H.naming({ include_patterns: "--include-patterns", exclude_patterns: "--exclude-patterns", output_format: "--output-format", + discovery: "--discovery", idempotency_key: "--idempotency-key", cursor: "--cursor", limit: "--limit", diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 0fdfa71..606c59a 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -56,6 +56,8 @@ export const crawl: Command = { " --output-format also fetch each kept URL's page, as HTML or Markdown", " (read with content/download); each kept page is one more", " fetch counted by --max-pages", + " --discovery links how the crawl finds the URLs it follows; links (the", + " default) follows the links in each page's HTML", " --follow wait until the crawl ends (Ctrl-C stops the wait, not the crawl)", " --timeout with --follow: stop waiting after s seconds (default 600)", " --idempotency-key so a retried create starts no second crawl", @@ -122,6 +124,7 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { "include-patterns": { type: "string", multiple: true }, "exclude-patterns": { type: "string", multiple: true }, "output-format": { type: "string" }, + discovery: { type: "string", multiple: true }, follow: { type: "boolean" }, timeout: { type: "string" }, "idempotency-key": { type: "string" }, @@ -163,6 +166,19 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { } params.output_format = outputFormat; } + const discovery = patterns(values.discovery); + if (discovery.length) { + const other = discovery.find((d) => d !== "links"); + if (other !== undefined) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Invalid --discovery value '${other}'.`, + likely_cause: "--discovery takes links. Without it, the crawl follows the links in each page's HTML.", + next_action: "Pass --discovery links, or leave it out.", + }); + } + params.discovery = ["links"]; + } const timeoutMs = timeoutFlag(values.timeout); const idempotencyKey = asString(values["idempotency-key"]); diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index 150f64d..2d7c99a 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -117,6 +117,8 @@ export interface CreateCrawlParams { exclude_patterns?: string[]; /** `html` returns each kept URL's page; absent = URLs only. */ output_format?: CrawlOutputFormat; + /** How the crawl finds the URLs it follows on each page it walks. */ + discovery?: CrawlDiscoveryStrategy[]; } /** RFC 9457 problem body. */ @@ -133,6 +135,8 @@ export const CRAWL_NOT_ENABLED_CODE = "REQS008"; /** The formats a crawl can store each kept URL's page in. */ export type CrawlOutputFormat = "html" | "markdown"; +/** How a crawl finds the URLs it follows on each page it walks. */ +export type CrawlDiscoveryStrategy = "links"; /** A crawl stops progressing once its status is anything but `running`. */ @@ -443,6 +447,7 @@ export function createCrawl(params: CreateCrawlParams, opts: CallOpts & { idempo if (params.include_patterns?.length) body.include_patterns = params.include_patterns; if (params.exclude_patterns?.length) body.exclude_patterns = params.exclude_patterns; if (params.output_format !== undefined) body.output_format = params.output_format; + if (params.discovery?.length) body.discovery = params.discovery; return crawlJson("POST", "/crawls", { ...opts, body, diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index b46952f..b4d4b0f 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -86,7 +86,7 @@ test("crawl create rejects flags it does not declare, before any network call", await withCrawlWorkspace( () => json(crawlBody, 202), async (calls) => { - for (const flag of ["--pagination", "--discovery", "--json-output", "--wait", "--html", "--include", "--exclude"]) { + for (const flag of ["--pagination", "--json-output", "--wait", "--html", "--include", "--exclude"]) { const { code, out } = await run(["create", "https://example.com/products/", "--depth", "1", flag, "x"]); assert.equal(code, 1, flag); assert.equal(out.error.code, "UNKNOWN_FLAG", flag); @@ -591,3 +591,20 @@ test("unknown crawl subcommand is INVALID_USAGE", async () => { assert.equal(code, 1); assert.equal(out.error.code, "INVALID_USAGE"); }); + +test("crawl create sends --discovery links, and refuses any other strategy before any network call", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202), + async (calls) => { + for (const value of ["pagination", "json"]) { + const { code, out } = await run(["create", "https://example.com/products/", "--discovery", value]); + assert.equal(code, 1, value); + assert.equal(out.error.code, "INVALID_USAGE", value); + } + assert.equal(calls.length, 0); + const { code } = await run(["create", "https://example.com/products/", "--discovery", "links"]); + assert.equal(code, 0); + assert.deepEqual(JSON.parse(String(calls[0]!.init?.body)).discovery, ["links"]); + }, + ); +}); From 054eeb5cfdaf9641ff1cb8dced83ea6c0ac82106 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Thu, 15 Oct 2026 17:03:13 +0200 Subject: [PATCH 4/6] fix(crawl): follow the API's next_cursor in crawl results crawl results said partial false on any ended crawl. After a stop, the API keeps next_cursor set while pages still in flight can add URLs, so the list was not complete yet. partial now follows next_cursor, and a partial read carries a note to read the results again later. --- acceptance/crawl/README.md | 2 +- acceptance/crawl/surface/test.mjs | 6 +++--- src/cli/commands/crawl.ts | 16 ++++++++++++---- src/core/crawl-api.ts | 6 +++--- tests/crawl-api.test.ts | 4 ++-- tests/crawl-command.test.ts | 17 +++++++++++++++++ 6 files changed, 38 insertions(+), 13 deletions(-) diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index 06388b8..af66718 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -79,7 +79,7 @@ and signal checks. | 17 | Naming | the API's operation and parameter names, in this surface's casing | | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | | 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | -| 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | +| 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results loses no URL (MCP and CLI: `partial` follows the API's `next_cursor`, with a note to read again while it is set), and the same URLs come back once the pages in flight finish | | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | Extra checks for this surface: every subcommand prints JSON with `--json` (`create`, `get`, `wait` and `stop` print `{ok, crawl_id, status, crawl, note?}`), Ctrl-C during `crawl wait` prints the resume hint and exits 130, and `--wait` and `--html` are rejected as unknown flags. diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index 062f2e1..252ab82 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -580,7 +580,7 @@ if (HAS_KEY) { await liveCheck( "20", "Results after stop", - "crawl results on a crawl stopped with pages in flight: partial false, and no URL lost", + "crawl results on a crawl stopped with pages in flight: no URL lost; partial follows the API, and a partial read carries a note to read again", async () => { if (!B || stopInfo?.error) return { status: FAIL, actual: `setup failed: ${stopInfo?.error}` }; if (stopInfo.status !== "stopped") { @@ -601,8 +601,8 @@ await liveCheck( const same = urls(j.results) === urls(later); const pending = (j.results ?? []).filter((x) => x.content_status === "pending").length; return { - status: passIf(now.code === 0 && j.partial === false && same), - actual: `stopped with ${stopInfo.items} kept; first read: exit ${now.code}, ${j.count} URLs (${pending} pending), partial ${j.partial}; then ${settledNote}: ${later.length} URLs, same set ${same}`, + status: passIf(now.code === 0 && same && (!j.partial || Boolean(j.note))), + actual: `stopped with ${stopInfo.items} kept; first read: exit ${now.code}, ${j.count} URLs (${pending} pending), partial ${j.partial}, note: ${j.note ?? "none"}; then ${settledNote}: ${later.length} URLs, same set ${same}`, evidence: show(now), }; }, diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 606c59a..0782b78 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -65,8 +65,9 @@ export const crawl: Command = { " get [--cursor ] [--limit ]", " the crawl and one page of its results (limit up to 10000)", " list [--cursor ] [--limit ] your crawls, newest first (limit up to 100)", - " results [--limit ] every kept URL as JSONL; on a running crawl, those kept so far", - " (partial). --limit is the page size per request", + " results [--limit ] every kept URL as JSONL; partial while more can come (a running", + " crawl, or pages still in flight after a stop). --limit is the page size", + " per request", " --out write the results to a file instead of printing", " content print one kept URL's page (content_id or content_url)", " --out write it to a file instead", @@ -325,9 +326,10 @@ function printResults( o: { outFile?: string; json: boolean; partial: boolean }, ): number { if (o.outFile) writeOut(o.outFile, toJsonl(results)); + const note = o.partial ? partialNote(status) : undefined; if (o.json) { const rows = o.outFile ? { file: o.outFile } : { results }; - log.out(JSON.stringify({ ok: true, crawl_id: id, status, count: results.length, ...rows, partial: o.partial }, null, 2)); + log.out(JSON.stringify({ ok: true, crawl_id: id, status, count: results.length, ...rows, partial: o.partial, ...(note ? { note } : {}) }, null, 2)); return 0; } if (o.outFile) { @@ -336,10 +338,16 @@ function printResults( log.info(`${results.length} result(s) for crawl ${id} (status: ${status}):`); log.out(toJsonl(results).trimEnd()); } - if (o.partial) log.warn(" The crawl is still running: these results are partial. Read them again once it ends."); + if (note) log.warn(` ${note}`); return 0; } +function partialNote(status: string): string { + return isTerminal(status) + ? "The crawl ended, but pages fetched before the end can still add URLs for up to 10 minutes: read the results again later." + : "The crawl is still running: these results are partial. Read them again once it ends."; +} + async function contentCmd(rest: string[], ctx: RunContext): Promise { const { values, positionals } = parse(rest, { out: { type: "string" }, json: { type: "boolean" } }); const json = ctx.json || values.json === true; diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index 2d7c99a..0e28ef3 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -556,9 +556,9 @@ export async function listAllResults( const rows = page.results ?? []; results.push(...rows); if (!rows.length || !page.next_cursor || page.next_cursor === cursor) { - // Every page was read. An ended crawl keeps no new URLs, even while pages in flight after - // a stop keep next_cursor set, so only a running crawl's results are partial. - return { crawl: page, results, partial: !isTerminal(page.status) }; + // Every page was read. The API keeps next_cursor set while URLs can still come: on a + // running crawl, and for up to 10 minutes after a stop, from the pages still in flight. + return { crawl: page, results, partial: page.next_cursor != null }; } cursor = page.next_cursor; } diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts index 681bbd2..a4a5add 100644 --- a/tests/crawl-api.test.ts +++ b/tests/crawl-api.test.ts @@ -571,14 +571,14 @@ test("listAllResults returns what a running crawl kept, ending at the first empt assert.equal(calls(), 2); }); -test("listAllResults on a stopped crawl is complete at the first empty page, even with next_cursor still set", async () => { +test("listAllResults on a stopped crawl is partial while the API keeps next_cursor set", async () => { const { impl, calls } = pagedFetch([ { status: "stopped", results: [{ url: "https://example.com/product/1" }], next_cursor: "c1" }, { status: "stopped", results: [], next_cursor: "c1" }, ]); const { results, partial } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl }); assert.deepEqual(results.map((r) => r.url), ["https://example.com/product/1"]); - assert.equal(partial, false); + assert.equal(partial, true); assert.equal(calls(), 2); }); diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index b4d4b0f..3413b5d 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -387,6 +387,23 @@ test("crawl results on a running crawl returns what is kept with partial: true", ); }); +test("crawl results on a stopped crawl is partial, with a note, while the API keeps next_cursor set", async () => { + const pages: Record = { + "": { ...crawlBody, status: "stopped", results: [{ url: "https://example.com/product/1" }], next_cursor: "p2" }, + p2: { ...crawlBody, status: "stopped", results: [], next_cursor: "p2" }, + }; + await withCrawlWorkspace( + (url) => json(pages[url.searchParams.get("cursor") ?? ""]), + async () => { + const { code, out } = await run(["results", "c_1"]); + assert.equal(code, 0); + assert.equal(out.partial, true); + assert.match(out.note, /read the results again later/); + assert.equal(out.count, 1); + }, + ); +}); + test("crawl get --cursor reads one page and returns next_cursor while running", async () => { await withCrawlWorkspace( () => json({ ...crawlBody, results: [{ url: "https://example.com/product/3" }], next_cursor: "p4" }), From 2469c5c7fc9974307e937d8feca75b275c648222 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 15:25:00 +0000 Subject: [PATCH 5/6] test(crawl): record the gated output and discovery values in the lock The values that the json output and pagination discovery use stay off for every user. The lock now records them as held, so the drift check reads their flag on each pass. --- acceptance/crawl/contract.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock index 435df9b..0d3bf74 100644 --- a/acceptance/crawl/contract.lock +++ b/acceptance/crawl/contract.lock @@ -1,3 +1,5 @@ 1adcbc2fd87e3f508317f3ce24c3d48573aec657 pending GET /healthz pending POST /internal/v1/crawls/{crawl_id}/stop +held output_format: json +held discovery: pagination From 4217197eee14cb3a0c4b8f41e31037d962a0cb5c Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 16:10:08 +0000 Subject: [PATCH 6/6] fix(crawl): say a download after a stop is partial while pages still arrive A stop reads stopped at once, but the API still reads the pages in flight for up to 10 minutes and builds the file from what it kept until then. download now asks the API, as results does, and says to download again later while it is not done. --- src/cli/commands/crawl.ts | 18 ++++++++++++++---- tests/crawl-command.test.ts | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 0782b78..47a5a90 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -72,7 +72,8 @@ export const crawl: Command = { " content print one kept URL's page (content_id or content_url)", " --out write it to a file instead", " download [--out ] save the NDJSON export (with page HTML for html crawls),", - " default .jsonl; partial while the crawl runs", + " default .jsonl; partial while more can come (a running", + " crawl, or pages still in flight after a stop)", " stop stop a running crawl (an ended crawl answers as it ended)", " wait [--timeout ] poll until the crawl ends or s seconds (default 600) run out;", " a crawl still running then is printed, not stopped", @@ -308,12 +309,15 @@ async function downloadCmd(rest: string[], ctx: RunContext): Promise { assertUsable("crawl"); const apiKey = requireApiKey(); const { status, count } = await downloadCrawl(id, (ndjson) => writeOutStream(file, ndjson), { apiKey }); - const partial = status === "running"; + // A stop reads `stopped` at once, but the API still reads the pages in flight for up to 10 + // minutes and builds the file from what it has kept until then. Its next_cursor says when it is done. + const partial = status === "running" || (status === "stopped" && (await listAllResults(id, { apiKey, pageSize: 10_000 })).partial); + const note = partial ? downloadNote(status) : undefined; if (json) { - log.out(JSON.stringify({ ok: true, crawl_id: id, status: status ?? null, partial, count, file }, null, 2)); + log.out(JSON.stringify({ ok: true, crawl_id: id, status: status ?? null, partial, ...(note ? { note } : {}), count, file }, null, 2)); } else { log.success(`Downloaded ${count} result(s) → ${file}`); - if (partial) log.warn(" The crawl is still running: this file is partial. Download again once it ends."); + if (note) log.warn(` ${note}`); } return 0; } @@ -342,6 +346,12 @@ function printResults( return 0; } +function downloadNote(status: string | undefined): string { + return status === "running" + ? "The crawl is still running: this file is partial. Download again once it ends." + : "The crawl was stopped, but pages fetched before the stop can still add lines for up to 10 minutes: download again later."; +} + function partialNote(status: string): string { return isTerminal(status) ? "The crawl ended, but pages fetched before the end can still add URLs for up to 10 minutes: read the results again later." diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 3413b5d..78e7fda 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -438,6 +438,44 @@ test("crawl download writes the NDJSON export to .jsonl", async () => { ); }); +test("crawl download on a stopped crawl is partial, with a note, while the API keeps next_cursor set", async () => { + await withCrawlWorkspace( + (url) => + url.pathname.endsWith("/download") + ? new Response('{"url":"https://example.com/product/1"}\n', { + status: 200, + headers: { "content-type": "application/x-ndjson", "x-crawl-status": "stopped" }, + }) + : json({ ...crawlBody, status: "stopped", results: [], next_cursor: "p2" }), + async (calls) => { + const { code, out } = await run(["download", "c_1"]); + assert.equal(code, 0); + assert.equal(out.status, "stopped"); + assert.equal(out.partial, true); + assert.match(out.note, /download again later/); + assert.equal(calls[1]!.url.pathname, "/v1/crawls/c_1"); + }, + ); +}); + +test("crawl download on a stopped crawl is final once the API's next_cursor is null", async () => { + await withCrawlWorkspace( + (url) => + url.pathname.endsWith("/download") + ? new Response('{"url":"https://example.com/product/1"}\n', { + status: 200, + headers: { "content-type": "application/x-ndjson", "x-crawl-status": "stopped" }, + }) + : json({ ...crawlBody, status: "stopped", results: [{ url: "https://example.com/product/1" }], next_cursor: null }), + async () => { + const { code, out } = await run(["download", "c_1"]); + assert.equal(code, 0); + assert.equal(out.partial, false); + assert.equal(out.note, undefined); + }, + ); +}); + test("crawl download keeps an earlier file when the export fails", async () => { await withCrawlWorkspace( () => json({ code: "crawl_not_found", title: "Not found", status: 404 }, 404),