diff --git a/README.md b/README.md index f931abd..1844868 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,10 @@ to open. `--max-items` and `--max-pages` cap the crawl (API default 100 each), and `--max-pages` bounds the cost, since each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default 1000). `--output-format html` uses up the page budget: each kept page is one more fetch, and `--max-pages` counts it. `--include-patterns` / -`--exclude-patterns` are repeatable substring filters on the URL. `get` and +`--exclude-patterns` are repeatable substring filters on the URL. A crawl whose +pages held nothing new to follow ends `completed` with no results and exits 0. +When patterns were set, it prints a note (under `--json`, in `note`) to check +them against the URLs the start page links to. `get` and `list` read one page and take `--cursor` and `--limit`. `results` follows every page (`--limit` is the page size); on a running crawl it returns the URLs kept so far, with `partial: true`. `download` reports the crawl's `status` (`null` diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index af66718..c79f89f 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -78,7 +78,7 @@ and signal checks. | 16 | Errors carry the server code | SDKs: `code` holds it, empty when the body has none; MCP, CLI: `server_code` | | 17 | Naming | the API's operation and parameter names, in this surface's casing | | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | -| 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | +| 19 | No match | an include pattern that matches nothing ends `completed` with 0 results and no `error`; SDKs return it; MCP and CLI say `ok: true` (CLI: exit 0, with a `note` about the patterns) | | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results loses no URL (MCP and CLI: `partial` follows the API's `next_cursor`, with a note to read again while it is set), and the same URLs come back once the pages in flight finish | | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index 252ab82..ba18a63 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -769,11 +769,11 @@ await check( }, ); -// ---------- 19-21: a failed crawl, results after a stop, the documented defaults ---------- +// ---------- 19-21: a crawl that matches nothing, results after a stop, the documented defaults ---------- await liveCheck( "19", - "Failed crawl", - "a crawl that keeps nothing ends failed (no_items_found); crawl wait exits 1 with ok: false and CRAWL_FAILED", + "No match", + "a crawl whose pattern matches nothing ends completed with 0 results and no error; crawl wait exits 0 with ok: true and a note", async () => { const r = createWith([CRAWL_URL, "--depth", "1", "--include-patterns", H.NO_MATCH]); F = r.json; @@ -781,15 +781,16 @@ await liveCheck( const w = z(["crawl", "wait", F.crawl_id, "--json"]); const j = w.json ?? {}; const ok = - w.code === 1 && - j.ok === false && - j.status === "failed" && - j.crawl?.error?.code === "no_items_found" && - j.error?.code === "CRAWL_FAILED" && - j.error?.server_code === "no_items_found"; + w.code === 0 && + j.ok === true && + j.status === "completed" && + j.crawl?.coverage?.items_found === 0 && + j.crawl?.error === undefined && + j.error === undefined && + typeof j.note === "string"; return { status: passIf(ok), - actual: `${F.crawl_id}: exit ${w.code}, status ${j.status}, ok ${j.ok}, error ${j.error?.code}/${j.error?.server_code}`, + actual: `${F.crawl_id}: exit ${w.code}, status ${j.status}, ok ${j.ok}, items ${j.crawl?.coverage?.items_found}, error ${j.crawl?.error?.code ?? "none"}, note ${j.note ? "yes" : "no"}`, evidence: show(w), }; }, diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index 27b7b86..607f93b 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -39,6 +39,10 @@ zenrows crawl results --out urls.jsonl # same, as JSONL - `--include-patterns` / `--exclude-patterns` are substring matches on the normalized URL and repeat: `--include-patterns /product/ --exclude-patterns ?add-to-cart`. +- A crawl whose pages held nothing new to follow (no link, only links to other + sites, or only links the patterns reject) ends `completed` with no results + and exits 0. When patterns were set, it prints a note (under `--json`, in + `note`) to check them against the URLs the start page links to. - `create` returns at once. `--follow` (or `zenrows crawl wait `) polls until the crawl ends or `--timeout` seconds run out (default 600). When the time runs out, the command exits 0 and prints the crawl with status @@ -85,7 +89,10 @@ credit cap; the pages fetched before that stay readable. `failed`, and a `stopped` crawl with an `error`, exit non-zero: `CRAWL_INSUFFICIENT_CREDITS` or `CRAWL_KEY_CAP_REACHED` when credits ended it (the same codes a refused `create` gets, also when an older deployment reports it `failed`), else -`CRAWL_FAILED`. `server_code` carries the API's `error.code`. +`CRAWL_FAILED`. `server_code` carries the API's `error.code`: +`seed_unreachable` when the start URL could not be fetched or returned an empty +page, `no_items_found` when the crawl read the start page but found no data it +could use. `stop` on a crawl that already ended prints the status it ended with. With `--json`, `create`, `get`, `wait` and `stop` print diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index b41041d..e4a7b3f 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -80,7 +80,8 @@ export const crawl: Command = { " a crawl still running then is printed, not stopped", " --json print structured output: create/get/wait/stop print", " {ok, crawl_id, status, crawl}; get adds results and next_cursor;", - " a wait that runs out adds note", + " a wait that runs out, or a completed crawl with patterns", + " and 0 results, adds note", ].join("\n"), async run(argv: string[], ctx: RunContext): Promise { const [sub, ...rest] = argv; @@ -491,9 +492,10 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { } const why = [code, c.error?.detail].filter(Boolean).join(": "); const next: Record = { - seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch `.", + seed_unreachable: + "The start URL could not be fetched or returned an empty page (the site may be blocking it). Check it with `zenrows fetch `.", no_items_found: - "Nothing matched. Loosen --include-patterns / --exclude-patterns, raise or drop --depth, or check the start page with `zenrows fetch `.", + "The crawl read the start page but found no data it could use, so it had nothing to follow. Start from a listing or product page.", }; return new ToolkitError({ code: "CRAWL_FAILED", @@ -505,6 +507,18 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null { }); } +/** + * Set only when a crawl with patterns completed and kept nothing. The API ends a + * crawl whose pages held nothing new to follow `completed`, and does not count + * the links the patterns rejected, so the patterns are the first thing to check. + */ +function noMatchNote(c: Crawl | CrawlStop): string | undefined { + const crawl = c as Crawl; + const patterns = [...(crawl.include_patterns ?? []), ...(crawl.exclude_patterns ?? [])]; + if (crawl.status !== "completed" || crawl.coverage?.items_found !== 0 || !patterns.length) return undefined; + return "The crawl completed with no results. Check --include-patterns / --exclude-patterns against the URLs the start page links to."; +} + /** * Print a crawl (or a stop answer), structured under --json, and return the exit * code: 1 when it ended `failed`, or `stopped` because credits ran out or the @@ -519,6 +533,7 @@ function printCrawl( extra: { results?: CrawlResult[]; next_cursor?: string | null; note?: string } = {}, ): number { const err = crawlFailure(c); + const note = extra.note ?? noMatchNote(c); const crawlFields = { ...c } as Record; // A wait polls with one result to stay cheap; that page is not "the results". delete crawlFields.results; @@ -532,6 +547,7 @@ function printCrawl( status: c.status, crawl: crawlFields, ...extra, + note, ...(err ? { error: err.toJSON() } : {}), }, null, @@ -549,6 +565,7 @@ function printCrawl( log.info(` ${cov.items_found} item(s) found · ${cov.pages_fetched} page(s) fetched · ${cov.pages_failed} failed`); } if (extra.note) log.dim(` next: zenrows crawl wait ${c.crawl_id}`); + else if (note) log.warn(` ${note}`); if (err) printError(err, false); return err ? 1 : 0; } diff --git a/tests/crawl-command.test.ts b/tests/crawl-command.test.ts index 6556f78..2d3abf5 100644 --- a/tests/crawl-command.test.ts +++ b/tests/crawl-command.test.ts @@ -335,6 +335,88 @@ test("crawl get exits 0 for completed and stopped crawls", async () => { } }); +test("a completed crawl with patterns and 0 results exits 0 with a note to check the patterns", async () => { + const empty = { ...crawlBody, status: "completed", results: [], next_cursor: null }; + await withCrawlWorkspace( + () => json({ ...empty, include_patterns: ["/product/"] }), + async () => { + const { code, out } = await run(["get", "c_1"]); + assert.equal(code, 0); + assert.equal(out.ok, true); + assert.equal(out.error, undefined); + assert.match(out.note, /Check --include-patterns \/ --exclude-patterns against the URLs the start page links to/); + + let human = -1; + const err = await captureErr(async () => { + human = await crawl.run(["get", "c_1"], { json: false, yes: false }); + }); + assert.equal(human, 0); + assert.match(err, /against the URLs the start page links to/); + }, + ); + await withCrawlWorkspace( + () => json(empty), + async () => { + const { out } = await run(["get", "c_1"]); + assert.equal(out.note, undefined, "no patterns, no note"); + }, + ); +}); + +test("a no_items_found crawl says the start page held no data the crawl could use, not that the patterns matched nothing", async () => { + await withCrawlWorkspace( + () => + json({ + ...crawlBody, + status: "failed", + include_patterns: ["/product/"], + error: { + code: "no_items_found", + detail: + "The crawl read the start page but found no data it could use, so it had nothing to follow. Start from a listing or product page.", + }, + results: [], + next_cursor: null, + }), + async () => { + const { code, out } = await run(["get", "c_1"]); + assert.equal(code, 1); + assert.equal(out.error.code, "CRAWL_FAILED"); + assert.equal(out.error.server_code, "no_items_found"); + assert.match(out.error.next_action, /read the start page but found no data it could use/); + assert.match(out.error.next_action, /Start from a listing or product page/); + assert.doesNotMatch(out.error.next_action, /Extract|Fetch|Batch/); + assert.doesNotMatch(out.error.next_action, /patterns/); + assert.equal(out.note, undefined); + }, + ); +}); + +test("an empty start page fails seed_unreachable with exit 1 and CRAWL_FAILED", async () => { + await withCrawlWorkspace( + () => + json({ + ...crawlBody, + status: "failed", + error: { + code: "seed_unreachable", + detail: "The start URL returned an empty page, so the crawl could not begin. Check the URL.", + }, + results: [], + next_cursor: null, + }), + async () => { + const { code, out } = await run(["get", "c_1"]); + assert.equal(code, 1); + assert.equal(out.ok, false); + assert.equal(out.error.code, "CRAWL_FAILED"); + assert.equal(out.error.server_code, "seed_unreachable"); + assert.match(out.error.next_action, /could not be fetched or returned an empty page/); + assert.equal(out.note, undefined); + }, + ); +}); + // What the server says when credits end a crawl, as error.detail. const creditStopDetail: Record = { insufficient_credits: "The crawl stopped because your account has no credits left.",