feat(crawl): return each page as Markdown, and choose how a crawl finds URLs

acceptance/crawl/README.md

@@ -79,7 +79,7 @@ and signal checks.
 | 17 | Naming | the API's operation and parameter names, in this surface's casing |
 | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch |
 | 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) |
-| 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish |
+| 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results loses no URL (MCP and CLI: `partial` follows the API's `next_cursor`, with a note to read again while it is set), and the same URLs come back once the pages in flight finish |
 | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) |
 
 Extra checks for this surface: every subcommand prints JSON with `--json` (`create`, `get`, `wait` and `stop` print `{ok, crawl_id, status, crawl, note?}`), Ctrl-C during `crawl wait` prints the resume hint and exits 130, and `--wait` and `--html` are rejected as unknown flags.

acceptance/crawl/contract.lock

@@ -0,0 +1,5 @@
+1adcbc2fd87e3f508317f3ce24c3d48573aec657
+pending GET /healthz
+pending POST /internal/v1/crawls/{crawl_id}/stop
+held output_format: json
+held discovery: pagination

acceptance/crawl/surface/test.mjs

@@ -28,7 +28,15 @@ mkdirSync(APP, { recursive: true });
 mkdirSync(HOME, { recursive: true });
 
 // Without a key, "invalid" keeps the CLI from creating an account.
-const baseEnv = { ...process.env, HOME, NO_COLOR: "1", ZENROWS_API_KEY: HAS_KEY ? KEY : "invalid" };
+// The CLI reads its own variable, not CRAWL_API_BASE: without it every call
+// would reach production whatever the run targets.
+const baseEnv = {
+  ...process.env,
+  HOME,
+  NO_COLOR: "1",
+  ZENROWS_API_KEY: HAS_KEY ? KEY : "invalid",
+  ZENROWS_CRAWL_API_BASE: H.API_BASE,
+};
 const parse = (s) => {
   try {
     return JSON.parse(s);
@@ -572,7 +580,7 @@ if (HAS_KEY) {
 await liveCheck(
   "20",
   "Results after stop",
-  "crawl results on a crawl stopped with pages in flight: partial false, and no URL lost",
+  "crawl results on a crawl stopped with pages in flight: no URL lost; partial follows the API, and a partial read carries a note to read again",
   async () => {
     if (!B || stopInfo?.error) return { status: FAIL, actual: `setup failed: ${stopInfo?.error}` };
     if (stopInfo.status !== "stopped") {
@@ -593,8 +601,8 @@ await liveCheck(
     const same = urls(j.results) === urls(later);
     const pending = (j.results ?? []).filter((x) => x.content_status === "pending").length;
     return {
-      status: passIf(now.code === 0 && j.partial === false && same),
-      actual: `stopped with ${stopInfo.items} kept; first read: exit ${now.code}, ${j.count} URLs (${pending} pending), partial ${j.partial}; then ${settledNote}: ${later.length} URLs, same set ${same}`,
+      status: passIf(now.code === 0 && same && (!j.partial || Boolean(j.note))),
+      actual: `stopped with ${stopInfo.items} kept; first read: exit ${now.code}, ${j.count} URLs (${pending} pending), partial ${j.partial}, note: ${j.note ?? "none"}; then ${settledNote}: ${later.length} URLs, same set ${same}`,
       evidence: show(now),
     };
   },
@@ -840,6 +848,49 @@ await liveCheck(
   },
 );
 
+await liveCheck(
+  "22",
+  "Markdown output",
+  'crawl create --output-format markdown is accepted and the crawl echoes it',
+  async () => {
+    const r = create(["--output-format", "markdown", "--max-items", "1", "--max-pages", "1"]);
+    if (r.code !== 0 || !r.json?.crawl_id) {
+      return { status: FAIL, actual: H.trim(show(r), 400) };
+    }
+    z(["crawl", "stop", r.json.crawl_id, "--json"]);
+    const got = r.json.output_format ?? r.json.crawl?.output_format;
+    return { status: passIf(got === "markdown"), actual: `${r.json.crawl_id}: output_format ${got}` };
+  },
+);
+await liveCheck(
+  "23",
+  "Rejected output format",
+  "crawl create --output-format pdf is refused locally, before any network call",
+  async () => {
+    const r = create(["--output-format", "pdf"]);
+    const code = r.json?.error?.code;
+    return {
+      status: passIf(r.code === 1 && code === "INVALID_USAGE"),
+      actual: `exit ${r.code}, code ${code}`,
+    };
+  },
+);
+
+await liveCheck(
+  "24",
+  "Discovery",
+  "crawl create --discovery links is echoed back, and --discovery pagination is refused before any network call",
+  async () => {
+    const r = create(["--discovery", "links", "--max-items", "1", "--max-pages", "1"]);
+    if (r.code !== 0 || !r.json?.crawl_id) return { status: FAIL, actual: H.trim(show(r), 400) };
+    z(["crawl", "stop", r.json.crawl_id, "--json"]);
+    const got = r.json.crawl?.discovery ?? r.json.discovery;
+    const refused = create(["--discovery", "pagination"]);
+    const ok = JSON.stringify(got) === JSON.stringify(["links"]) && refused.code === 1 && refused.json?.error?.code === "INVALID_USAGE";
+    return { status: passIf(ok), actual: `discovery ${JSON.stringify(got)}; pagination exit ${refused.code} ${refused.json?.error?.code}` };
+  },
+);
+
 H.naming({
   operations: {
     create: "crawl create",
@@ -859,6 +910,7 @@ H.naming({
     include_patterns: "--include-patterns",
     exclude_patterns: "--exclude-patterns",
     output_format: "--output-format",
+    discovery: "--discovery",
     idempotency_key: "--idempotency-key",
     cursor: "--cursor",
     limit: "--limit",

skills/crawl/SKILL.md

@@ -11,8 +11,8 @@ Give Crawl one start URL; it follows the links on that page (and on the
 pages behind it, up to the depth you set) and returns the URLs it keeps. Use it
 when you know where the items live (a category page, a blog index, a docs
 section) but not the item URLs themselves. Once you have the URLs, read pages
-with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page's HTML
-with `--output-format html`.
+with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page itself
+with `--output-format html` or `--output-format markdown`.
 
 Crawl stays on the start URL's registrable domain; subdomains count. An account without Crawl access gets
 403 `REQS008` → `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account");
@@ -50,16 +50,18 @@ zenrows crawl results <id> --out urls.jsonl  # same, as JSONL
   `partial: true`. Read again once the crawl ends. `--limit` sets the page size
   of each request.
 
-## Page HTML
+## Page content
 
 ```
 zenrows crawl create <url> --depth 1 --output-format html --max-pages 20 --follow
+zenrows crawl create <url> --depth 1 --output-format markdown --max-pages 20 --follow
 zenrows crawl results <id> --json        # fetched results carry content_url
-zenrows crawl content <id> <content_id>  # one page's HTML (or pass the content_url)
-zenrows crawl download <id> --out pages.jsonl  # every URL + HTML, NDJSON
+zenrows crawl content <id> <content_id>  # one page (or pass the content_url)
+zenrows crawl download <id> --out pages.jsonl  # every URL + page, NDJSON
 ```
 
-`--output-format html` uses up the page budget: each kept page is one more
+`--output-format html` stores each page as fetched; `--output-format markdown`
+stores it converted to Markdown. Either uses up the page budget: each kept page is one more
 fetch, and `--max-pages` counts it. Raise `--max-pages` to keep the same number
 of URLs. A result's `content_status` is `pending`, `fetched` or `failed`; only
 `fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). `download`

src/cli/commands/crawl.ts

@@ -53,22 +53,27 @@ export const crawl: Command = {
     "                                    the local policy max_pages_per_run caps it (default 1000)",
     "    --include-patterns <p>          keep only URLs containing this substring (repeatable)",
     "    --exclude-patterns <p>          drop URLs containing this substring (repeatable)",
-    "    --output-format html            also fetch each kept URL's page HTML (read with content/download);",
-    "                                    each kept page is one more fetch counted by --max-pages",
+    "    --output-format <html|markdown> also fetch each kept URL's page, as HTML or Markdown",
+    "                                    (read with content/download); each kept page is one more",
+    "                                    fetch counted by --max-pages",
+    "    --discovery links               how the crawl finds the URLs it follows; links (the",
+    "                                    default) follows the links in each page's HTML",
     "    --follow                        wait until the crawl ends (Ctrl-C stops the wait, not the crawl)",
     "    --timeout <s>                   with --follow: stop waiting after s seconds (default 600)",
     "    --idempotency-key <key>         so a retried create starts no second crawl",
     "    --no-signup                     do not auto-create a Free plan account if no key exists",
     "  get <id> [--cursor <c>] [--limit <n>]",
     "                                    the crawl and one page of its results (limit up to 10000)",
     "  list [--cursor <c>] [--limit <n>] your crawls, newest first (limit up to 100)",
-    "  results <id> [--limit <n>]        every kept URL as JSONL; on a running crawl, those kept so far",
-    "                                    (partial). --limit is the page size per request",
+    "  results <id> [--limit <n>]        every kept URL as JSONL; partial while more can come (a running",
+    "                                    crawl, or pages still in flight after a stop). --limit is the page size",
+    "                                    per request",
     "    --out <file>                    write the results to a file instead of printing",
     "  content <id> <content_id>         print one kept URL's page (content_id or content_url)",
     "    --out <file>                    write it to a file instead",
     "  download <id> [--out <file>]      save the NDJSON export (with page HTML for html crawls),",
-    "                                    default <id>.jsonl; partial while the crawl runs",
+    "                                    default <id>.jsonl; partial while more can come (a running",
+    "                                    crawl, or pages still in flight after a stop)",
     "  stop <id>                         stop a running crawl (an ended crawl answers as it ended)",
     "  wait <id> [--timeout <s>]         poll until the crawl ends or s seconds (default 600) run out;",
     "                                    a crawl still running then is printed, not stopped",
@@ -121,6 +126,7 @@ async function createCmd(rest: string[], ctx: RunContext): Promise<number> {
     "include-patterns": { type: "string", multiple: true },
     "exclude-patterns": { type: "string", multiple: true },
     "output-format": { type: "string" },
+    discovery: { type: "string", multiple: true },
     follow: { type: "boolean" },
     timeout: { type: "string" },
     "idempotency-key": { type: "string" },
@@ -151,16 +157,30 @@ async function createCmd(rest: string[], ctx: RunContext): Promise<number> {
   if (exclude.length) params.exclude_patterns = exclude;
   const outputFormat = asString(values["output-format"]);
   if (outputFormat !== undefined) {
-    if (outputFormat !== "html") {
+    if (outputFormat !== "html" && outputFormat !== "markdown") {
       throw new ToolkitError({
         code: "INVALID_USAGE",
         message: `Invalid --output-format value '${outputFormat}'.`,
-        likely_cause: "--output-format takes html only. Without it, a crawl returns URLs only.",
-        next_action: "Pass --output-format html, or leave it out.",
+        likely_cause:
+          "--output-format takes html or markdown. Without it, a crawl returns URLs only.",
+        next_action: "Pass --output-format html or --output-format markdown, or leave it out.",
       });
     }
     params.output_format = outputFormat;
   }
+  const discovery = patterns(values.discovery);
+  if (discovery.length) {
+    const other = discovery.find((d) => d !== "links");
+    if (other !== undefined) {
+      throw new ToolkitError({
+        code: "INVALID_USAGE",
+        message: `Invalid --discovery value '${other}'.`,
+        likely_cause: "--discovery takes links. Without it, the crawl follows the links in each page's HTML.",
+        next_action: "Pass --discovery links, or leave it out.",
+      });
+    }
+    params.discovery = ["links"];
+  }
   const timeoutMs = timeoutFlag(values.timeout);
   const idempotencyKey = asString(values["idempotency-key"]);
 
@@ -289,12 +309,15 @@ async function downloadCmd(rest: string[], ctx: RunContext): Promise<number> {
   assertUsable("crawl");
   const apiKey = requireApiKey();
   const { status, count } = await downloadCrawl(id, (ndjson) => writeOutStream(file, ndjson), { apiKey });
-  const partial = status === "running";
+  // A stop reads `stopped` at once, but the API still reads the pages in flight for up to 10
+  // minutes and builds the file from what it has kept until then. Its next_cursor says when it is done.
+  const partial = status === "running" || (status === "stopped" && (await listAllResults(id, { apiKey, pageSize: 10_000 })).partial);
+  const note = partial ? downloadNote(status) : undefined;
   if (json) {
-    log.out(JSON.stringify({ ok: true, crawl_id: id, status: status ?? null, partial, count, file }, null, 2));
+    log.out(JSON.stringify({ ok: true, crawl_id: id, status: status ?? null, partial, ...(note ? { note } : {}), count, file }, null, 2));
   } else {
     log.success(`Downloaded ${count} result(s) → ${file}`);
-    if (partial) log.warn("  The crawl is still running: this file is partial. Download again once it ends.");
+    if (note) log.warn(`  ${note}`);
   }
   return 0;
 }
@@ -307,9 +330,10 @@ function printResults(
   o: { outFile?: string; json: boolean; partial: boolean },
 ): number {
   if (o.outFile) writeOut(o.outFile, toJsonl(results));
+  const note = o.partial ? partialNote(status) : undefined;
   if (o.json) {
     const rows = o.outFile ? { file: o.outFile } : { results };
-    log.out(JSON.stringify({ ok: true, crawl_id: id, status, count: results.length, ...rows, partial: o.partial }, null, 2));
+    log.out(JSON.stringify({ ok: true, crawl_id: id, status, count: results.length, ...rows, partial: o.partial, ...(note ? { note } : {}) }, null, 2));
     return 0;
   }
   if (o.outFile) {
@@ -318,10 +342,22 @@ function printResults(
     log.info(`${results.length} result(s) for crawl ${id} (status: ${status}):`);
     log.out(toJsonl(results).trimEnd());
   }
-  if (o.partial) log.warn("  The crawl is still running: these results are partial. Read them again once it ends.");
+  if (note) log.warn(`  ${note}`);
   return 0;
 }
 
+function downloadNote(status: string | undefined): string {
+  return status === "running"
+    ? "The crawl is still running: this file is partial. Download again once it ends."
+    : "The crawl was stopped, but pages fetched before the stop can still add lines for up to 10 minutes: download again later.";
+}
+
+function partialNote(status: string): string {
+  return isTerminal(status)
+    ? "The crawl ended, but pages fetched before the end can still add URLs for up to 10 minutes: read the results again later."
+    : "The crawl is still running: these results are partial. Read them again once it ends.";
+}
+
 async function contentCmd(rest: string[], ctx: RunContext): Promise<number> {
   const { values, positionals } = parse(rest, { out: { type: "string" }, json: { type: "boolean" } });
   const json = ctx.json || values.json === true;

src/core/crawl-api.ts

@@ -10,7 +10,7 @@
  * Crawl SDKs do (see `retryStatuses`). `fetchImpl` / `sleepImpl` are injectable
  * for tests.
  *
- * `output_format` takes `html` only (absent = URLs only). Response fields we do
+ * `output_format` takes `html` or `markdown` (absent = URLs only). Response fields we do
  * not model pass through untouched.
  */
 import { Readable } from "node:stream";
@@ -66,6 +66,8 @@ export interface Crawl {
   include_patterns?: string[];
   exclude_patterns?: string[];
   output_format?: string;
+  /** The strategies the crawl finds URLs with; `["links"]` when the create left it out. */
+  discovery?: string[];
   duplicates_removed?: number;
   finished_at?: string;
   [k: string]: unknown;
@@ -114,7 +116,9 @@ export interface CreateCrawlParams {
   include_patterns?: string[];
   exclude_patterns?: string[];
   /** `html` returns each kept URL's page; absent = URLs only. */
-  output_format?: "html";
+  output_format?: CrawlOutputFormat;
+  /** How the crawl finds the URLs it follows on each page it walks. */
+  discovery?: CrawlDiscoveryStrategy[];
 }
 
 /** RFC 9457 problem body. */
@@ -129,6 +133,12 @@ interface ProblemJson {
 /** The API's "Crawl is not enabled for this account" code (403). */
 export const CRAWL_NOT_ENABLED_CODE = "REQS008";
 
+/** The formats a crawl can store each kept URL's page in. */
+export type CrawlOutputFormat = "html" | "markdown";
+/** How a crawl finds the URLs it follows on each page it walks. */
+export type CrawlDiscoveryStrategy = "links";
+
+
 /** A crawl stops progressing once its status is anything but `running`. */
 export function isTerminal(status: string | undefined): boolean {
   return !!status && status !== "running";
@@ -437,6 +447,7 @@ export function createCrawl(params: CreateCrawlParams, opts: CallOpts & { idempo
   if (params.include_patterns?.length) body.include_patterns = params.include_patterns;
   if (params.exclude_patterns?.length) body.exclude_patterns = params.exclude_patterns;
   if (params.output_format !== undefined) body.output_format = params.output_format;
+  if (params.discovery?.length) body.discovery = params.discovery;
   return crawlJson<Crawl>("POST", "/crawls", {
     ...opts,
     body,
@@ -463,7 +474,7 @@ export function stopCrawl(id: string, opts: CallOpts): Promise<CrawlStop> {
   return crawlJson<CrawlStop>("POST", `/crawls/${encodeURIComponent(id)}/stop`, { ...opts, crawlId: id, retryStop: true });
 }
 
-/** The page of one kept URL (HTML for `output_format: html`). */
+/** The page of one kept URL, in the crawl's `output_format`. */
 export async function getCrawlContent(
   id: string,
   contentId: string,
@@ -545,9 +556,9 @@ export async function listAllResults(
     const rows = page.results ?? [];
     results.push(...rows);
     if (!rows.length || !page.next_cursor || page.next_cursor === cursor) {
-      // Every page was read. An ended crawl keeps no new URLs, even while pages in flight after
-      // a stop keep next_cursor set, so only a running crawl's results are partial.
-      return { crawl: page, results, partial: !isTerminal(page.status) };
+      // Every page was read. The API keeps next_cursor set while URLs can still come: on a
+      // running crawl, and for up to 10 minutes after a stop, from the pages still in flight.
+      return { crawl: page, results, partial: page.next_cursor != null };
     }
     cursor = page.next_cursor;
   }

tests/crawl-api.test.ts

@@ -571,14 +571,14 @@ test("listAllResults returns what a running crawl kept, ending at the first empt
   assert.equal(calls(), 2);
 });
 
-test("listAllResults on a stopped crawl is complete at the first empty page, even with next_cursor still set", async () => {
+test("listAllResults on a stopped crawl is partial while the API keeps next_cursor set", async () => {
   const { impl, calls } = pagedFetch([
     { status: "stopped", results: [{ url: "https://example.com/product/1" }], next_cursor: "c1" },
     { status: "stopped", results: [], next_cursor: "c1" },
   ]);
   const { results, partial } = await listAllResults("c_1", { apiKey: "k", fetchImpl: impl });
   assert.deepEqual(results.map((r) => r.url), ["https://example.com/product/1"]);
-  assert.equal(partial, false);
+  assert.equal(partial, true);
   assert.equal(calls(), 2);
 });