diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index 06388b8..af66718 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -79,7 +79,7 @@ and signal checks. | 17 | Naming | the API's operation and parameter names, in this surface's casing | | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | | 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | -| 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | +| 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results loses no URL (MCP and CLI: `partial` follows the API's `next_cursor`, with a note to read again while it is set), and the same URLs come back once the pages in flight finish | | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | Extra checks for this surface: every subcommand prints JSON with `--json` (`create`, `get`, `wait` and `stop` print `{ok, crawl_id, status, crawl, note?}`), Ctrl-C during `crawl wait` prints the resume hint and exits 130, and `--wait` and `--html` are rejected as unknown flags. diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock new file mode 100644 index 0000000..0d3bf74 --- /dev/null +++ b/acceptance/crawl/contract.lock @@ -0,0 +1,5 @@ +1adcbc2fd87e3f508317f3ce24c3d48573aec657 +pending GET /healthz +pending POST /internal/v1/crawls/{crawl_id}/stop +held output_format: json +held discovery: pagination diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index f1eb617..252ab82 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -28,7 +28,15 @@ mkdirSync(APP, { recursive: true }); mkdirSync(HOME, { recursive: true }); // Without a key, "invalid" keeps the CLI from creating an account. -const baseEnv = { ...process.env, HOME, NO_COLOR: "1", ZENROWS_API_KEY: HAS_KEY ? KEY : "invalid" }; +// The CLI reads its own variable, not CRAWL_API_BASE: without it every call +// would reach production whatever the run targets. +const baseEnv = { + ...process.env, + HOME, + NO_COLOR: "1", + ZENROWS_API_KEY: HAS_KEY ? KEY : "invalid", + ZENROWS_CRAWL_API_BASE: H.API_BASE, +}; const parse = (s) => { try { return JSON.parse(s); @@ -572,7 +580,7 @@ if (HAS_KEY) { await liveCheck( "20", "Results after stop", - "crawl results on a crawl stopped with pages in flight: partial false, and no URL lost", + "crawl results on a crawl stopped with pages in flight: no URL lost; partial follows the API, and a partial read carries a note to read again", async () => { if (!B || stopInfo?.error) return { status: FAIL, actual: `setup failed: ${stopInfo?.error}` }; if (stopInfo.status !== "stopped") { @@ -593,8 +601,8 @@ await liveCheck( const same = urls(j.results) === urls(later); const pending = (j.results ?? []).filter((x) => x.content_status === "pending").length; return { - status: passIf(now.code === 0 && j.partial === false && same), - actual: `stopped with ${stopInfo.items} kept; first read: exit ${now.code}, ${j.count} URLs (${pending} pending), partial ${j.partial}; then ${settledNote}: ${later.length} URLs, same set ${same}`, + status: passIf(now.code === 0 && same && (!j.partial || Boolean(j.note))), + actual: `stopped with ${stopInfo.items} kept; first read: exit ${now.code}, ${j.count} URLs (${pending} pending), partial ${j.partial}, note: ${j.note ?? "none"}; then ${settledNote}: ${later.length} URLs, same set ${same}`, evidence: show(now), }; }, @@ -840,6 +848,49 @@ await liveCheck( }, ); +await liveCheck( + "22", + "Markdown output", + 'crawl create --output-format markdown is accepted and the crawl echoes it', + async () => { + const r = create(["--output-format", "markdown", "--max-items", "1", "--max-pages", "1"]); + if (r.code !== 0 || !r.json?.crawl_id) { + return { status: FAIL, actual: H.trim(show(r), 400) }; + } + z(["crawl", "stop", r.json.crawl_id, "--json"]); + const got = r.json.output_format ?? r.json.crawl?.output_format; + return { status: passIf(got === "markdown"), actual: `${r.json.crawl_id}: output_format ${got}` }; + }, +); +await liveCheck( + "23", + "Rejected output format", + "crawl create --output-format pdf is refused locally, before any network call", + async () => { + const r = create(["--output-format", "pdf"]); + const code = r.json?.error?.code; + return { + status: passIf(r.code === 1 && code === "INVALID_USAGE"), + actual: `exit ${r.code}, code ${code}`, + }; + }, +); + +await liveCheck( + "24", + "Discovery", + "crawl create --discovery links is echoed back, and --discovery pagination is refused before any network call", + async () => { + const r = create(["--discovery", "links", "--max-items", "1", "--max-pages", "1"]); + if (r.code !== 0 || !r.json?.crawl_id) return { status: FAIL, actual: H.trim(show(r), 400) }; + z(["crawl", "stop", r.json.crawl_id, "--json"]); + const got = r.json.crawl?.discovery ?? r.json.discovery; + const refused = create(["--discovery", "pagination"]); + const ok = JSON.stringify(got) === JSON.stringify(["links"]) && refused.code === 1 && refused.json?.error?.code === "INVALID_USAGE"; + return { status: passIf(ok), actual: `discovery ${JSON.stringify(got)}; pagination exit ${refused.code} ${refused.json?.error?.code}` }; + }, +); + H.naming({ operations: { create: "crawl create", @@ -859,6 +910,7 @@ H.naming({ include_patterns: "--include-patterns", exclude_patterns: "--exclude-patterns", output_format: "--output-format", + discovery: "--discovery", idempotency_key: "--idempotency-key", cursor: "--cursor", limit: "--limit", diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index cac5d8f..77737c6 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -11,8 +11,8 @@ Give Crawl one start URL; it follows the links on that page (and on the pages behind it, up to the depth you set) and returns the URLs it keeps. Use it when you know where the items live (a category page, a blog index, a docs section) but not the item URLs themselves. Once you have the URLs, read pages -with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page's HTML -with `--output-format html`. +with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page itself +with `--output-format html` or `--output-format markdown`. Crawl stays on the start URL's registrable domain; subdomains count. An account without Crawl access gets 403 `REQS008` → `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account"); @@ -50,16 +50,18 @@ zenrows crawl results --out urls.jsonl # same, as JSONL `partial: true`. Read again once the crawl ends. `--limit` sets the page size of each request. -## Page HTML +## Page content ``` zenrows crawl create --depth 1 --output-format html --max-pages 20 --follow +zenrows crawl create --depth 1 --output-format markdown --max-pages 20 --follow zenrows crawl results --json # fetched results carry content_url -zenrows crawl content # one page's HTML (or pass the content_url) -zenrows crawl download --out pages.jsonl # every URL + HTML, NDJSON +zenrows crawl content # one page (or pass the content_url) +zenrows crawl download --out pages.jsonl # every URL + page, NDJSON ``` -`--output-format html` uses up the page budget: each kept page is one more +`--output-format html` stores each page as fetched; `--output-format markdown` +stores it converted to Markdown. Either uses up the page budget: each kept page is one more fetch, and `--max-pages` counts it. Raise `--max-pages` to keep the same number of URLs. A result's `content_status` is `pending`, `fetched` or `failed`; only `fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). `download` diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index eeb615b..47a5a90 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -53,8 +53,11 @@ export const crawl: Command = { " the local policy max_pages_per_run caps it (default 1000)", " --include-patterns

keep only URLs containing this substring (repeatable)", " --exclude-patterns

drop URLs containing this substring (repeatable)", - " --output-format html also fetch each kept URL's page HTML (read with content/download);", - " each kept page is one more fetch counted by --max-pages", + " --output-format also fetch each kept URL's page, as HTML or Markdown", + " (read with content/download); each kept page is one more", + " fetch counted by --max-pages", + " --discovery links how the crawl finds the URLs it follows; links (the", + " default) follows the links in each page's HTML", " --follow wait until the crawl ends (Ctrl-C stops the wait, not the crawl)", " --timeout with --follow: stop waiting after s seconds (default 600)", " --idempotency-key so a retried create starts no second crawl", @@ -62,13 +65,15 @@ export const crawl: Command = { " get [--cursor ] [--limit ]", " the crawl and one page of its results (limit up to 10000)", " list [--cursor ] [--limit ] your crawls, newest first (limit up to 100)", - " results [--limit ] every kept URL as JSONL; on a running crawl, those kept so far", - " (partial). --limit is the page size per request", + " results [--limit ] every kept URL as JSONL; partial while more can come (a running", + " crawl, or pages still in flight after a stop). --limit is the page size", + " per request", " --out write the results to a file instead of printing", " content print one kept URL's page (content_id or content_url)", " --out write it to a file instead", " download [--out ] save the NDJSON export (with page HTML for html crawls),", - " default .jsonl; partial while the crawl runs", + " default .jsonl; partial while more can come (a running", + " crawl, or pages still in flight after a stop)", " stop stop a running crawl (an ended crawl answers as it ended)", " wait [--timeout ] poll until the crawl ends or s seconds (default 600) run out;", " a crawl still running then is printed, not stopped", @@ -121,6 +126,7 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { "include-patterns": { type: "string", multiple: true }, "exclude-patterns": { type: "string", multiple: true }, "output-format": { type: "string" }, + discovery: { type: "string", multiple: true }, follow: { type: "boolean" }, timeout: { type: "string" }, "idempotency-key": { type: "string" }, @@ -151,16 +157,30 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { if (exclude.length) params.exclude_patterns = exclude; const outputFormat = asString(values["output-format"]); if (outputFormat !== undefined) { - if (outputFormat !== "html") { + if (outputFormat !== "html" && outputFormat !== "markdown") { throw new ToolkitError({ code: "INVALID_USAGE", message: `Invalid --output-format value '${outputFormat}'.`, - likely_cause: "--output-format takes html only. Without it, a crawl returns URLs only.", - next_action: "Pass --output-format html, or leave it out.", + likely_cause: + "--output-format takes html or markdown. Without it, a crawl returns URLs only.", + next_action: "Pass --output-format html or --output-format markdown, or leave it out.", }); } params.output_format = outputFormat; } + const discovery = patterns(values.discovery); + if (discovery.length) { + const other = discovery.find((d) => d !== "links"); + if (other !== undefined) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Invalid --discovery value '${other}'.`, + likely_cause: "--discovery takes links. Without it, the crawl follows the links in each page's HTML.", + next_action: "Pass --discovery links, or leave it out.", + }); + } + params.discovery = ["links"]; + } const timeoutMs = timeoutFlag(values.timeout); const idempotencyKey = asString(values["idempotency-key"]); @@ -289,12 +309,15 @@ async function downloadCmd(rest: string[], ctx: RunContext): Promise { assertUsable("crawl"); const apiKey = requireApiKey(); const { status, count } = await downloadCrawl(id, (ndjson) => writeOutStream(file, ndjson), { apiKey }); - const partial = status === "running"; + // A stop reads `stopped` at once, but the API still reads the pages in flight for up to 10 + // minutes and builds the file from what it has kept until then. Its next_cursor says when it is done. + const partial = status === "running" || (status === "stopped" && (await listAllResults(id, { apiKey, pageSize: 10_000 })).partial); + const note = partial ? downloadNote(status) : undefined; if (json) { - log.out(JSON.stringify({ ok: true, crawl_id: id, status: status ?? null, partial, count, file }, null, 2)); + log.out(JSON.stringify({ ok: true, crawl_id: id, status: status ?? null, partial, ...(note ? { note } : {}), count, file }, null, 2)); } else { log.success(`Downloaded ${count} result(s) → ${file}`); - if (partial) log.warn(" The crawl is still running: this file is partial. Download again once it ends."); + if (note) log.warn(` ${note}`); } return 0; } @@ -307,9 +330,10 @@ function printResults( o: { outFile?: string; json: boolean; partial: boolean }, ): number { if (o.outFile) writeOut(o.outFile, toJsonl(results)); + const note = o.partial ? partialNote(status) : undefined; if (o.json) { const rows = o.outFile ? { file: o.outFile } : { results }; - log.out(JSON.stringify({ ok: true, crawl_id: id, status, count: results.length, ...rows, partial: o.partial }, null, 2)); + log.out(JSON.stringify({ ok: true, crawl_id: id, status, count: results.length, ...rows, partial: o.partial, ...(note ? { note } : {}) }, null, 2)); return 0; } if (o.outFile) { @@ -318,10 +342,22 @@ function printResults( log.info(`${results.length} result(s) for crawl ${id} (status: ${status}):`); log.out(toJsonl(results).trimEnd()); } - if (o.partial) log.warn(" The crawl is still running: these results are partial. Read them again once it ends."); + if (note) log.warn(` ${note}`); return 0; } +function downloadNote(status: string | undefined): string { + return status === "running" + ? "The crawl is still running: this file is partial. Download again once it ends." + : "The crawl was stopped, but pages fetched before the stop can still add lines for up to 10 minutes: download again later."; +} + +function partialNote(status: string): string { + return isTerminal(status) + ? "The crawl ended, but pages fetched before the end can still add URLs for up to 10 minutes: read the results again later." + : "The crawl is still running: these results are partial. Read them again once it ends."; +} + async function contentCmd(rest: string[], ctx: RunContext): Promise { const { values, positionals } = parse(rest, { out: { type: "string" }, json: { type: "boolean" } }); const json = ctx.json || values.json === true; diff --git a/src/core/crawl-api.ts b/src/core/crawl-api.ts index b0f2871..0e28ef3 100644 --- a/src/core/crawl-api.ts +++ b/src/core/crawl-api.ts @@ -10,7 +10,7 @@ * Crawl SDKs do (see `retryStatuses`). `fetchImpl` / `sleepImpl` are injectable * for tests. * - * `output_format` takes `html` only (absent = URLs only). Response fields we do + * `output_format` takes `html` or `markdown` (absent = URLs only). Response fields we do * not model pass through untouched. */ import { Readable } from "node:stream"; @@ -66,6 +66,8 @@ export interface Crawl { include_patterns?: string[]; exclude_patterns?: string[]; output_format?: string; + /** The strategies the crawl finds URLs with; `["links"]` when the create left it out. */ + discovery?: string[]; duplicates_removed?: number; finished_at?: string; [k: string]: unknown; @@ -114,7 +116,9 @@ export interface CreateCrawlParams { include_patterns?: string[]; exclude_patterns?: string[]; /** `html` returns each kept URL's page; absent = URLs only. */ - output_format?: "html"; + output_format?: CrawlOutputFormat; + /** How the crawl finds the URLs it follows on each page it walks. */ + discovery?: CrawlDiscoveryStrategy[]; } /** RFC 9457 problem body. */ @@ -129,6 +133,12 @@ interface ProblemJson { /** The API's "Crawl is not enabled for this account" code (403). */ export const CRAWL_NOT_ENABLED_CODE = "REQS008"; +/** The formats a crawl can store each kept URL's page in. */ +export type CrawlOutputFormat = "html" | "markdown"; +/** How a crawl finds the URLs it follows on each page it walks. */ +export type CrawlDiscoveryStrategy = "links"; + + /** A crawl stops progressing once its status is anything but `running`. */ export function isTerminal(status: string | undefined): boolean { return !!status && status !== "running"; @@ -437,6 +447,7 @@ export function createCrawl(params: CreateCrawlParams, opts: CallOpts & { idempo if (params.include_patterns?.length) body.include_patterns = params.include_patterns; if (params.exclude_patterns?.length) body.exclude_patterns = params.exclude_patterns; if (params.output_format !== undefined) body.output_format = params.output_format; + if (params.discovery?.length) body.discovery = params.discovery; return crawlJson("POST", "/crawls", { ...opts, body, @@ -463,7 +474,7 @@ export function stopCrawl(id: string, opts: CallOpts): Promise { return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { ...opts, crawlId: id, retryStop: true }); } -/** The page of one kept URL (HTML for `output_format: html`). */ +/** The page of one kept URL, in the crawl's `output_format`. */ export async function getCrawlContent( id: string, contentId: string, @@ -545,9 +556,9 @@ export async function listAllResults( const rows = page.results ?? []; results.push(...rows); if (!rows.length || !page.next_cursor || page.next_cursor === cursor) { - // Every page was read. An ended crawl keeps no new URLs, even while pages in flight after - // a stop keep next_cursor set, so only a running crawl's results are partial. - return { crawl: page, results, partial: !isTerminal(page.status) }; + // Every page was read. The API keeps next_cursor set while URLs can still come: on a + // running crawl, and for up to 10 minutes after a stop, from the pages still in flight. + return { crawl: page, results, partial: page.next_cursor != null }; } cursor = page.next_cursor; } diff --git a/tests/crawl-api.test.ts b/tests/crawl-api.test.ts index 681bbd2..a4a5add 100644 --- a/tests/crawl-api.test.ts +++ b/tests/crawl-api.test.ts @@ -571,14 +571,14 @@ test("listAllResults returns what a running crawl kept, ending at the first empt assert.equal(calls(), 2); }); -test("listAllResults on a stopped crawl is complete at the first empty page, even with next_cursor still set", async () => { +test("listAllResults on a stopped crawl is partial while the API keeps next_cursor set", async () => { const { impl, calls } = pagedFetch([ { status: "stopped", results: [{ url: "https://example.com/product/1" }], next_cursor: "c1" }, { status: "stopped", results: [], next_cursor: "c1" }, ]); const { results, partial } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl }); assert.deepEqual(results.map((r) => r.url), ["https://example.com/product/1"]); - assert.equal(partial, false); + assert.equal(partial, true); assert.equal(calls(), 2); }); diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index b46952f..78e7fda 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -86,7 +86,7 @@ test("crawl create rejects flags it does not declare, before any network call", await withCrawlWorkspace( () => json(crawlBody, 202), async (calls) => { - for (const flag of ["--pagination", "--discovery", "--json-output", "--wait", "--html", "--include", "--exclude"]) { + for (const flag of ["--pagination", "--json-output", "--wait", "--html", "--include", "--exclude"]) { const { code, out } = await run(["create", "https://example.com/products/", "--depth", "1", flag, "x"]); assert.equal(code, 1, flag); assert.equal(out.error.code, "UNKNOWN_FLAG", flag); @@ -387,6 +387,23 @@ test("crawl results on a running crawl returns what is kept with partial: true", ); }); +test("crawl results on a stopped crawl is partial, with a note, while the API keeps next_cursor set", async () => { + const pages: Record = { + "": { ...crawlBody, status: "stopped", results: [{ url: "https://example.com/product/1" }], next_cursor: "p2" }, + p2: { ...crawlBody, status: "stopped", results: [], next_cursor: "p2" }, + }; + await withCrawlWorkspace( + (url) => json(pages[url.searchParams.get("cursor") ?? ""]), + async () => { + const { code, out } = await run(["results", "c_1"]); + assert.equal(code, 0); + assert.equal(out.partial, true); + assert.match(out.note, /read the results again later/); + assert.equal(out.count, 1); + }, + ); +}); + test("crawl get --cursor reads one page and returns next_cursor while running", async () => { await withCrawlWorkspace( () => json({ ...crawlBody, results: [{ url: "https://example.com/product/3" }], next_cursor: "p4" }), @@ -421,6 +438,44 @@ test("crawl download writes the NDJSON export to .jsonl", async () => { ); }); +test("crawl download on a stopped crawl is partial, with a note, while the API keeps next_cursor set", async () => { + await withCrawlWorkspace( + (url) => + url.pathname.endsWith("/download") + ? new Response('{"url":"https://example.com/product/1"}\n', { + status: 200, + headers: { "content-type": "application/x-ndjson", "x-crawl-status": "stopped" }, + }) + : json({ ...crawlBody, status: "stopped", results: [], next_cursor: "p2" }), + async (calls) => { + const { code, out } = await run(["download", "c_1"]); + assert.equal(code, 0); + assert.equal(out.status, "stopped"); + assert.equal(out.partial, true); + assert.match(out.note, /download again later/); + assert.equal(calls[1]!.url.pathname, "/v1/crawls/c_1"); + }, + ); +}); + +test("crawl download on a stopped crawl is final once the API's next_cursor is null", async () => { + await withCrawlWorkspace( + (url) => + url.pathname.endsWith("/download") + ? new Response('{"url":"https://example.com/product/1"}\n', { + status: 200, + headers: { "content-type": "application/x-ndjson", "x-crawl-status": "stopped" }, + }) + : json({ ...crawlBody, status: "stopped", results: [{ url: "https://example.com/product/1" }], next_cursor: null }), + async () => { + const { code, out } = await run(["download", "c_1"]); + assert.equal(code, 0); + assert.equal(out.partial, false); + assert.equal(out.note, undefined); + }, + ); +}); + test("crawl download keeps an earlier file when the export fails", async () => { await withCrawlWorkspace( () => json({ code: "crawl_not_found", title: "Not found", status: 404 }, 404), @@ -591,3 +646,20 @@ test("unknown crawl subcommand is INVALID_USAGE", async () => { assert.equal(code, 1); assert.equal(out.error.code, "INVALID_USAGE"); }); + +test("crawl create sends --discovery links, and refuses any other strategy before any network call", async () => { + await withCrawlWorkspace( + () => json(crawlBody, 202), + async (calls) => { + for (const value of ["pagination", "json"]) { + const { code, out } = await run(["create", "https://example.com/products/", "--discovery", value]); + assert.equal(code, 1, value); + assert.equal(out.error.code, "INVALID_USAGE", value); + } + assert.equal(calls.length, 0); + const { code } = await run(["create", "https://example.com/products/", "--discovery", "links"]); + assert.equal(code, 0); + assert.deepEqual(JSON.parse(String(calls[0]!.init?.body)).discovery, ["links"]); + }, + ); +});