fix(crawl): say a crawl with nothing new to follow completes, so users stop chasing their patterns

README.md

@@ -187,7 +187,10 @@ to open. `--max-items` and `--max-pages` cap the crawl (API default 100 each),
 and `--max-pages` bounds the cost, since each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default
 1000). `--output-format html` uses up the page budget: each kept page is one
 more fetch, and `--max-pages` counts it. `--include-patterns` /
-`--exclude-patterns` are repeatable substring filters on the URL. `get` and
+`--exclude-patterns` are repeatable substring filters on the URL. A crawl whose
+pages held nothing new to follow ends `completed` with no results and exits 0.
+When patterns were set, it prints a note (under `--json`, in `note`) to check
+them against the URLs the start page links to. `get` and
 `list` read one page and take `--cursor` and `--limit`. `results` follows every
 page (`--limit` is the page size); on a running crawl it returns the URLs kept
 so far, with `partial: true`. `download` reports the crawl's `status` (`null`

acceptance/crawl/README.md

@@ -78,7 +78,7 @@ and signal checks.
 | 16 | Errors carry the server code | SDKs: `code` holds it, empty when the body has none; MCP, CLI: `server_code` |
 | 17 | Naming | the API's operation and parameter names, in this surface's casing |
 | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch |
-| 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) |
+| 19 | No match | an include pattern that matches nothing ends `completed` with 0 results and no `error`; SDKs return it; MCP and CLI say `ok: true` (CLI: exit 0, with a `note` about the patterns) |
 | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results loses no URL (MCP and CLI: `partial` follows the API's `next_cursor`, with a note to read again while it is set), and the same URLs come back once the pages in flight finish |
 | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) |
 

acceptance/crawl/surface/test.mjs

@@ -769,27 +769,28 @@ await check(
   },
 );
 
-// ---------- 19-21: a failed crawl, results after a stop, the documented defaults ----------
+// ---------- 19-21: a crawl that matches nothing, results after a stop, the documented defaults ----------
 await liveCheck(
   "19",
-  "Failed crawl",
-  "a crawl that keeps nothing ends failed (no_items_found); crawl wait exits 1 with ok: false and CRAWL_FAILED",
+  "No match",
+  "a crawl whose pattern matches nothing ends completed with 0 results and no error; crawl wait exits 0 with ok: true and a note",
   async () => {
     const r = createWith([CRAWL_URL, "--depth", "1", "--include-patterns", H.NO_MATCH]);
     F = r.json;
     if (!F?.crawl_id) return { status: FAIL, actual: show(r) };
     const w = z(["crawl", "wait", F.crawl_id, "--json"]);
     const j = w.json ?? {};
     const ok =
-      w.code === 1 &&
-      j.ok === false &&
-      j.status === "failed" &&
-      j.crawl?.error?.code === "no_items_found" &&
-      j.error?.code === "CRAWL_FAILED" &&
-      j.error?.server_code === "no_items_found";
+      w.code === 0 &&
+      j.ok === true &&
+      j.status === "completed" &&
+      j.crawl?.coverage?.items_found === 0 &&
+      j.crawl?.error === undefined &&
+      j.error === undefined &&
+      typeof j.note === "string";
     return {
       status: passIf(ok),
-      actual: `${F.crawl_id}: exit ${w.code}, status ${j.status}, ok ${j.ok}, error ${j.error?.code}/${j.error?.server_code}`,
+      actual: `${F.crawl_id}: exit ${w.code}, status ${j.status}, ok ${j.ok}, items ${j.crawl?.coverage?.items_found}, error ${j.crawl?.error?.code ?? "none"}, note ${j.note ? "yes" : "no"}`,
       evidence: show(w),
     };
   },

skills/crawl/SKILL.md

@@ -39,6 +39,10 @@ zenrows crawl results <id> --out urls.jsonl  # same, as JSONL
 - `--include-patterns` / `--exclude-patterns` are substring matches on the
   normalized URL and repeat:
   `--include-patterns /product/ --exclude-patterns ?add-to-cart`.
+- A crawl whose pages held nothing new to follow (no link, only links to other
+  sites, or only links the patterns reject) ends `completed` with no results
+  and exits 0. When patterns were set, it prints a note (under `--json`, in
+  `note`) to check them against the URLs the start page links to.
 - `create` returns at once. `--follow` (or `zenrows crawl wait <id>`) polls
   until the crawl ends or `--timeout` seconds run out (default 600). When the
   time runs out, the command exits 0 and prints the crawl with status
@@ -85,7 +89,10 @@ credit cap; the pages fetched before that stay readable. `failed`, and a
 `stopped` crawl with an `error`, exit non-zero: `CRAWL_INSUFFICIENT_CREDITS` or
 `CRAWL_KEY_CAP_REACHED` when credits ended it (the same codes a refused
 `create` gets, also when an older deployment reports it `failed`), else
-`CRAWL_FAILED`. `server_code` carries the API's `error.code`.
+`CRAWL_FAILED`. `server_code` carries the API's `error.code`:
+`seed_unreachable` when the start URL could not be fetched or returned an empty
+page, `no_items_found` when the crawl read the start page but found no data it
+could use.
 `stop` on a crawl that already ended prints the status it ended with.
 
 With `--json`, `create`, `get`, `wait` and `stop` print

src/cli/commands/crawl.ts

@@ -80,7 +80,8 @@ export const crawl: Command = {
     "                                    a crawl still running then is printed, not stopped",
     "  --json                            print structured output: create/get/wait/stop print",
     "                                    {ok, crawl_id, status, crawl}; get adds results and next_cursor;",
-    "                                    a wait that runs out adds note",
+    "                                    a wait that runs out, or a completed crawl with patterns",
+    "                                    and 0 results, adds note",
   ].join("\n"),
   async run(argv: string[], ctx: RunContext): Promise<number> {
     const [sub, ...rest] = argv;
@@ -491,9 +492,10 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null {
   }
   const why = [code, c.error?.detail].filter(Boolean).join(": ");
   const next: Record<string, string> = {
-    seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch <url>`.",
+    seed_unreachable:
+      "The start URL could not be fetched or returned an empty page (the site may be blocking it). Check it with `zenrows fetch <url>`.",
     no_items_found:
-      "Nothing matched. Loosen --include-patterns / --exclude-patterns, raise or drop --depth, or check the start page with `zenrows fetch <url>`.",
+      "The crawl read the start page but found no data it could use, so it had nothing to follow. Start from a listing or product page.",
   };
   return new ToolkitError({
     code: "CRAWL_FAILED",
@@ -505,6 +507,18 @@ export function crawlFailure(c: Crawl | CrawlStop): ToolkitError | null {
   });
 }
 
+/**
+ * Set only when a crawl with patterns completed and kept nothing. The API ends a
+ * crawl whose pages held nothing new to follow `completed`, and does not count
+ * the links the patterns rejected, so the patterns are the first thing to check.
+ */
+function noMatchNote(c: Crawl | CrawlStop): string | undefined {
+  const crawl = c as Crawl;
+  const patterns = [...(crawl.include_patterns ?? []), ...(crawl.exclude_patterns ?? [])];
+  if (crawl.status !== "completed" || crawl.coverage?.items_found !== 0 || !patterns.length) return undefined;
+  return "The crawl completed with no results. Check --include-patterns / --exclude-patterns against the URLs the start page links to.";
+}
+
 /**
  * Print a crawl (or a stop answer), structured under --json, and return the exit
  * code: 1 when it ended `failed`, or `stopped` because credits ran out or the
@@ -519,6 +533,7 @@ function printCrawl(
   extra: { results?: CrawlResult[]; next_cursor?: string | null; note?: string } = {},
 ): number {
   const err = crawlFailure(c);
+  const note = extra.note ?? noMatchNote(c);
   const crawlFields = { ...c } as Record<string, unknown>;
   // A wait polls with one result to stay cheap; that page is not "the results".
   delete crawlFields.results;
@@ -532,6 +547,7 @@ function printCrawl(
           status: c.status,
           crawl: crawlFields,
           ...extra,
+          note,
           ...(err ? { error: err.toJSON() } : {}),
         },
         null,
@@ -549,6 +565,7 @@ function printCrawl(
     log.info(`  ${cov.items_found} item(s) found · ${cov.pages_fetched} page(s) fetched · ${cov.pages_failed} failed`);
   }
   if (extra.note) log.dim(`  next: zenrows crawl wait ${c.crawl_id}`);
+  else if (note) log.warn(`  ${note}`);
   if (err) printError(err, false);
   return err ? 1 : 0;
 }

tests/crawl-command.test.ts

@@ -335,6 +335,88 @@ test("crawl get exits 0 for completed and stopped crawls", async () => {
   }
 });
 
+test("a completed crawl with patterns and 0 results exits 0 with a note to check the patterns", async () => {
+  const empty = { ...crawlBody, status: "completed", results: [], next_cursor: null };
+  await withCrawlWorkspace(
+    () => json({ ...empty, include_patterns: ["/product/"] }),
+    async () => {
+      const { code, out } = await run(["get", "c_1"]);
+      assert.equal(code, 0);
+      assert.equal(out.ok, true);
+      assert.equal(out.error, undefined);
+      assert.match(out.note, /Check --include-patterns \/ --exclude-patterns against the URLs the start page links to/);
+
+      let human = -1;
+      const err = await captureErr(async () => {
+        human = await crawl.run(["get", "c_1"], { json: false, yes: false });
+      });
+      assert.equal(human, 0);
+      assert.match(err, /against the URLs the start page links to/);
+    },
+  );
+  await withCrawlWorkspace(
+    () => json(empty),
+    async () => {
+      const { out } = await run(["get", "c_1"]);
+      assert.equal(out.note, undefined, "no patterns, no note");
+    },
+  );
+});
+
+test("a no_items_found crawl says the start page held no data the crawl could use, not that the patterns matched nothing", async () => {
+  await withCrawlWorkspace(
+    () =>
+      json({
+        ...crawlBody,
+        status: "failed",
+        include_patterns: ["/product/"],
+        error: {
+          code: "no_items_found",
+          detail:
+            "The crawl read the start page but found no data it could use, so it had nothing to follow. Start from a listing or product page.",
+        },
+        results: [],
+        next_cursor: null,
+      }),
+    async () => {
+      const { code, out } = await run(["get", "c_1"]);
+      assert.equal(code, 1);
+      assert.equal(out.error.code, "CRAWL_FAILED");
+      assert.equal(out.error.server_code, "no_items_found");
+      assert.match(out.error.next_action, /read the start page but found no data it could use/);
+      assert.match(out.error.next_action, /Start from a listing or product page/);
+      assert.doesNotMatch(out.error.next_action, /Extract|Fetch|Batch/);
+      assert.doesNotMatch(out.error.next_action, /patterns/);
+      assert.equal(out.note, undefined);
+    },
+  );
+});
+
+test("an empty start page fails seed_unreachable with exit 1 and CRAWL_FAILED", async () => {
+  await withCrawlWorkspace(
+    () =>
+      json({
+        ...crawlBody,
+        status: "failed",
+        error: {
+          code: "seed_unreachable",
+          detail: "The start URL returned an empty page, so the crawl could not begin. Check the URL.",
+        },
+        results: [],
+        next_cursor: null,
+      }),
+    async () => {
+      const { code, out } = await run(["get", "c_1"]);
+      assert.equal(code, 1);
+      assert.equal(out.ok, false);
+      assert.equal(out.error.code, "CRAWL_FAILED");
+      assert.equal(out.error.server_code, "seed_unreachable");
+      assert.match(out.error.next_action, /could not be fetched or returned an empty page/);
+      assert.equal(out.note, undefined);
+    },
+  );
+});
+
 // What the server says when credits end a crawl, as error.detail.
 const creditStopDetail: Record<string, string> = {
   insufficient_credits: "The crawl stopped because your account has no credits left.",