From 2ed95e92012085c4b6082bd06a3dc2f6437bf058 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 13 Oct 2026 15:37:38 +0200 Subject: [PATCH 1/7] fix(extract): drop the "domain-gated open beta" copy from the README and help The README on npm, the extract help text, the skills and the capability registry still described extract=auto as a "domain-gated open beta". That has not been true since the gate was removed: Extract works on any domain Zenrows has prepared, with any active key, and a domain that isn't prepared yet answers REQS007, which the CLI already falls back from to Autoparse. Say that instead, and replace "contact Zenrows support to enable this domain" in the AUTH010 and REQS007 errors with the self-serve path (POST /v1/extract/prepared-domains). Copy only; no behaviour changes. The npm page updates on the next publish (1.5.1). Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- agent-plugin/skills/extract/SKILL.md | 6 +++--- docs/capabilities.md | 2 +- registry/capabilities.json | 2 +- skills/extract/SKILL.md | 6 +++--- src/adapters/extract.ts | 2 +- src/adapters/protected-fetch.ts | 2 +- src/cli/commands/extract.ts | 4 ++-- src/core/capabilities.ts | 4 ++-- src/core/http.ts | 11 ++++++----- templates/extract-pricing-pages/README.md | 2 +- 11 files changed, 22 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 9a92c60..9822361 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ that endpoint with extraction parameters, not a separate product. | Command | What it does | Status (this build) | | --- | --- | --- | | `zenrows fetch` | Fetch — retrieve a protected page | **available** — `GET https://api.zenrows.com/v1/` | -| `zenrows extract` | Extract — `extract=auto` (domain-gated open beta; falls back to Autoparse) / CSS / Markdown | **beta** — same `/v1/` | +| `zenrows extract` | Extract — `extract=auto` (any prepared domain; falls back to Autoparse on one not prepared yet) / CSS / Markdown | **beta** — same `/v1/` | | `zenrows batch` | Batch — fan out over many URLs | beta — cloud works with beta access; local validate/estimate always | | `zenrows crawl` | Crawl (new) — collect the URLs behind one start page, optionally with each page's HTML | **available** — `https://api.zenrows.com/v1/crawls` | | `zenrows browser` | Browser Sessions REST API (same backend as MCP `browser_*`) | **available** — escalation-only; bills by bandwidth + time | diff --git a/agent-plugin/skills/extract/SKILL.md b/agent-plugin/skills/extract/SKILL.md index dab9be1..80e731f 100644 --- a/agent-plugin/skills/extract/SKILL.md +++ b/agent-plugin/skills/extract/SKILL.md @@ -20,8 +20,8 @@ extract({ url, mode: "css", css_extractor: '{"title":"h1","price":".price"}' }) extract({ url, proxy_country: "US" }) # geo-targeted ``` -`mode=auto` (`extract=auto`) is **open beta**: richest on enabled domains, -currently free; billing may apply later. If the domain is not enabled (AUTH010) +`mode=auto` (`extract=auto`) is in **beta**: works on any domain Zenrows has +prepared, with any key, and is currently free; billing may apply later. If the domain is not enabled (AUTH010) or not prepared yet (REQS007), the tool retries once with Autoparse by default (`fallback_autoparse`). @@ -45,7 +45,7 @@ scrape({ url, outputs: "emails,links" }) # or "*" for all built-ins ## When to use which -- **`mode=auto`**: site-tailored fields on beta-enabled domains. +- **`mode=auto`**: site-tailored fields on any prepared domain. - **`mode=autoparse`**: any website; also the automatic AUTH010 / REQS007 fallback. - **`mode=css`**: known fields/selectors; deterministic. - **Markdown via `scrape`**: feed clean content to an LLM yourself. diff --git a/docs/capabilities.md b/docs/capabilities.md index fd9ced3..786e2b9 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -20,7 +20,7 @@ Classification is based on the public Zenrows documentation: | Capability | Backend evidence | Status | | --- | --- | --- | | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | -| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (domain-gated open beta; CLI falls back to `autoparse`), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | beta | +| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (beta; any prepared domain, CLI falls back to `autoparse` on one not prepared yet), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | beta | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header) — open beta. Local JSONL spec validation + credit estimation work with no key. | beta | | `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL, up to `depth` when set (optional; no depth limit when omitted), on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Subcommands: `create`, `get`, `list`, `results`, `content`, `download`, `stop`, `wait`. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | diff --git a/registry/capabilities.json b/registry/capabilities.json index fce5b92..855febe 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -17,7 +17,7 @@ "command": "zenrows extract", "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, autoparse, css_extractor, outputs, response_type)", "requiresAuth": true, - "notes": "Open beta. Default is extract=auto (domain-gated); CLI falls back to autoparse on AUTH010. Autoparse / CSS / outputs / markdown work on any domain." + "notes": "Beta. Default is extract=auto, which works on any prepared domain with any key; CLI falls back to autoparse on REQS007 (domain not prepared yet) or AUTH010. Autoparse / CSS / outputs / markdown work on any domain." }, "batch": { "key": "batch", diff --git a/skills/extract/SKILL.md b/skills/extract/SKILL.md index 87bb271..597d49c 100644 --- a/skills/extract/SKILL.md +++ b/skills/extract/SKILL.md @@ -12,7 +12,7 @@ access + extraction**, not generic LLM parsing. ## Methods ``` -zenrows extract # extract=auto (default); falls back to Autoparse if domain not enabled +zenrows extract # extract=auto (default); falls back to Autoparse if the domain is not prepared yet zenrows extract --autoparse # general-purpose Autoparse (any domain) zenrows extract --css '{"title":"h1","price":".price"}' # selector map zenrows extract --outputs emails,links # built-in output filters → JSON @@ -21,8 +21,8 @@ zenrows extract --validate # fail if not valid JSON ``` ## When to use which -- **Extract** (`extract=auto`, default): richest fields on domains enabled for the open beta. -- **Autoparse**: any website; also the automatic fallback when Extract is not enabled for the domain. +- **Extract** (`extract=auto`, default): richest fields; works on any domain Zenrows has prepared. +- **Autoparse**: any website; also the automatic fallback when the domain is not prepared for Extract yet. - **CSS**: you know the exact fields/selectors and want determinism. - **Markdown/plaintext**: feed clean content to an LLM yourself. diff --git a/src/adapters/extract.ts b/src/adapters/extract.ts index 0c4c7f1..00ba92e 100644 --- a/src/adapters/extract.ts +++ b/src/adapters/extract.ts @@ -2,7 +2,7 @@ * Extract adapter. * * Structured extraction on the same `/v1/` Fetch and Extract API via: - * - extract=auto → site-tailored Extract (open beta, domain-gated) + * - extract=auto → site-tailored Extract (beta, any prepared domain) * - autoparse=true → general-purpose Autoparse (any domain) * - css_extractor= → selector-based field extraction * - outputs= → built-in output filters → JSON diff --git a/src/adapters/protected-fetch.ts b/src/adapters/protected-fetch.ts index db317f7..9a30c10 100644 --- a/src/adapters/protected-fetch.ts +++ b/src/adapters/protected-fetch.ts @@ -34,7 +34,7 @@ export interface FetchOptions { cssExtractor?: string; autoparse?: boolean; /** - * Site-tailored Extract (`extract=auto`). Open beta, domain-gated. + * Site-tailored Extract (`extract=auto`). Beta; works on any prepared domain. * Takes precedence over autoparse / css_extractor / outputs / response_type. */ extract?: boolean; diff --git a/src/cli/commands/extract.ts b/src/cli/commands/extract.ts index 444829d..3032e8f 100644 --- a/src/cli/commands/extract.ts +++ b/src/cli/commands/extract.ts @@ -1,7 +1,7 @@ /** * `zenrows extract [flags]` → Extract (structured extraction on `/v1/`). * - * Default: `extract=auto` (site-tailored Extract, open beta). If the domain is + * Default: `extract=auto` (site-tailored Extract, beta). If the domain is * not enabled or prepared (AUTH010 / REQS007), falls back once to Autoparse. Opt into Autoparse * directly with `--autoparse`. Other methods: `--css`, `--outputs`, `--output`. */ @@ -24,7 +24,7 @@ export const extract: Command = { usage: "zenrows extract [--autoparse | --css | --outputs | --output md|text] [flags]", help: [ "Methods:", - " (default) extract=auto — site-tailored Extract (open beta); falls back to Autoparse if the domain is not enabled or prepared", + " (default) extract=auto — site-tailored Extract (beta); falls back to Autoparse if the domain is not enabled or prepared", " --autoparse general-purpose Autoparse (any domain, no Extract fallback)", " --css CSS selector map, e.g. '{\"title\":\"h1\",\"price\":\".price\"}'", " --outputs built-in output filters → JSON: emails, phone_numbers, headings,", diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index 2bfb15a..a460708 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -28,8 +28,8 @@ export function getCapability(key: string): Capability | undefined { const USABLE: ReadonlySet = new Set([ "available", "available-but-needs-confirmation", - // Open beta: any key can call; product-specific limits (e.g. Extract domain - // coverage, Batch 403) are handled by the adapters / API errors, not here. + // Open beta: any key can call; product-specific limits (e.g. Extract domains + // not prepared yet, Batch 403) are handled by the adapters / API errors, not here. "beta", ]); diff --git a/src/core/http.ts b/src/core/http.ts index 1cc98eb..c076165 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -147,7 +147,8 @@ export async function scrape( code: "EXTRACT_DOMAIN_NOT_ENABLED", message: "Extract has not been prepared for this domain yet.", likely_cause: zrErrorDetail(body) ?? "This domain is not prepared for Extract.", - next_action: "Retry with --autoparse for general-purpose extraction on any site.", + next_action: + "Retry with --autoparse for general-purpose extraction on any site, or get the domain prepared for Extract (POST /v1/extract/prepared-domains).", suggested_commands: [`zenrows extract ${params.url} --autoparse`], }); } @@ -187,16 +188,16 @@ export async function scrape( } } if (res.status === 402 && isZenrowsErrorEnvelope(body)) { - // AUTH010 on extract=auto means the target domain is not in the Extract - // beta — distinct from credit exhaustion (AUTH004). Let the extract + // AUTH010 on extract=auto means the target domain is not enabled for + // Extract — distinct from credit exhaustion (AUTH004). Let the extract // adapter fall back to autoparse. if (params.extract !== undefined && zrErrorCode(body) === "AUTH010") { throw new ToolkitError({ code: "EXTRACT_DOMAIN_NOT_ENABLED", message: "Extract is not enabled for this domain yet.", - likely_cause: zrErrorDetail(body) ?? "This domain is not part of the Extract open beta.", + likely_cause: zrErrorDetail(body) ?? "This domain is not enabled for Extract.", next_action: - "Retry with --autoparse for general-purpose extraction on any site, or contact Zenrows support to enable this domain for Extract.", + "Retry with --autoparse for general-purpose extraction on any site, or get the domain prepared for Extract (POST /v1/extract/prepared-domains).", suggested_commands: [`zenrows extract ${params.url} --autoparse`], }); } diff --git a/templates/extract-pricing-pages/README.md b/templates/extract-pricing-pages/README.md index eb1ddbd..f26358a 100644 --- a/templates/extract-pricing-pages/README.md +++ b/templates/extract-pricing-pages/README.md @@ -13,5 +13,5 @@ zenrows extract https://www.scrapingcourse.com/ecommerce/ --autoparse zenrows extract --css '{"title":"h1","price":".price"}' --validate ``` -Requires `extract` (open beta). Start on one page, confirm the fields, then +Requires `extract` (beta). Start on one page, confirm the fields, then fan out `zenrows extract` across the rest of your URLs. From 3444e19cdb02b96efe928342c446723199a91217 Mon Sep 17 00:00:00 2001 From: Ander Date: Tue, 13 Oct 2026 16:08:16 +0200 Subject: [PATCH 2/7] docs(extract): call Extract beta, not open beta, and drop em dashes from the touched copy --- README.md | 2 +- agent-plugin/skills/cost-control/SKILL.md | 2 +- agent-plugin/skills/extract/SKILL.md | 6 +++--- agent-plugin/skills/zenrows/SKILL.md | 2 +- docs/capabilities.md | 4 ++-- src/cli/commands/extract.ts | 2 +- src/core/capabilities.ts | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 9822361..185e547 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ that endpoint with extraction parameters, not a separate product. | Command | What it does | Status (this build) | | --- | --- | --- | | `zenrows fetch` | Fetch — retrieve a protected page | **available** — `GET https://api.zenrows.com/v1/` | -| `zenrows extract` | Extract — `extract=auto` (any prepared domain; falls back to Autoparse on one not prepared yet) / CSS / Markdown | **beta** — same `/v1/` | +| `zenrows extract` | Extract: `extract=auto` (any prepared domain; falls back to Autoparse on one not prepared yet), CSS, or Markdown | **beta**, same `/v1/` | | `zenrows batch` | Batch — fan out over many URLs | beta — cloud works with beta access; local validate/estimate always | | `zenrows crawl` | Crawl (new) — collect the URLs behind one start page, optionally with each page's HTML | **available** — `https://api.zenrows.com/v1/crawls` | | `zenrows browser` | Browser Sessions REST API (same backend as MCP `browser_*`) | **available** — escalation-only; bills by bandwidth + time | diff --git a/agent-plugin/skills/cost-control/SKILL.md b/agent-plugin/skills/cost-control/SKILL.md index d74488d..81aac30 100644 --- a/agent-plugin/skills/cost-control/SKILL.md +++ b/agent-plugin/skills/cost-control/SKILL.md @@ -18,7 +18,7 @@ These multipliers apply when you enable the flags (on `scrape` or `extract`): - Premium proxies (`premium_proxy`): **10×** - Both: **25×** -`extract` `mode=auto` is **open beta** and currently free; billing may apply +`extract` `mode=auto` is in **beta** and currently free; billing may apply later. Do not promise permanent free extract pricing. Autoparse / CSS still follow the multipliers above when stealth flags are set. diff --git a/agent-plugin/skills/extract/SKILL.md b/agent-plugin/skills/extract/SKILL.md index 80e731f..625e7eb 100644 --- a/agent-plugin/skills/extract/SKILL.md +++ b/agent-plugin/skills/extract/SKILL.md @@ -21,9 +21,9 @@ extract({ url, proxy_country: "US" }) # geo-targeted ``` `mode=auto` (`extract=auto`) is in **beta**: works on any domain Zenrows has -prepared, with any key, and is currently free; billing may apply later. If the domain is not enabled (AUTH010) -or not prepared yet (REQS007), the tool retries once with Autoparse by default -(`fallback_autoparse`). +prepared, with any key, and is currently free; billing may apply later. If the +domain is not enabled (AUTH010) or not prepared yet (REQS007), the tool retries +once with Autoparse by default (`fallback_autoparse`). Every mode uses Adaptive Stealth Mode by default: pass just the URL, even for protected or dynamic sites. Set `js_render` or `premium_proxy` only to force a diff --git a/agent-plugin/skills/zenrows/SKILL.md b/agent-plugin/skills/zenrows/SKILL.md index f33f994..7d6db20 100644 --- a/agent-plugin/skills/zenrows/SKILL.md +++ b/agent-plugin/skills/zenrows/SKILL.md @@ -66,7 +66,7 @@ If the user needs login, clicks, forms, sessions, or persistent state: | Primitive | MCP tools | Notes | | --- | --- | --- | | Protected Fetch | `scrape` | Default markdown; HTML / plaintext / PDF / screenshot options | -| Extract | `extract` | Structured JSON (`extract=auto`, autoparse, css_extractor). `extract=auto` open beta (currently free; billing may apply later) | +| Extract | `extract` | Structured JSON (`extract=auto`, autoparse, css_extractor). `extract=auto` in beta (currently free; billing may apply later) | | Batch | `batch_create`, `batch_status`, `batch_results`, `batch_cancel` | Beta; may return `BATCH_ACCESS_DENIED` without access | | Browser | `browser_*` (start with `browser_navigate`) | Escalation-only; same backend as Browser Sessions | diff --git a/docs/capabilities.md b/docs/capabilities.md index 786e2b9..3c4052c 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -8,8 +8,8 @@ the single source of truth that prevents hallucinated execution. - **available** — a documented endpoint exists and the command does real work. - **available-but-needs-confirmation** — likely available; verify per account. - **experimental** — exists but not yet promoted to a stable status. -- **beta** — open beta; usable by any key. Product-specific limits (e.g. Extract - domain coverage) are handled by adapters / API errors, not by blocking the CLI. +- **beta** — usable by any key. Product-specific limits (e.g. Extract domains + not prepared yet) are handled by adapters / API errors, not by blocking the CLI. - **planned** — no documented endpoint yet; local spec / validation only. - **not-implemented** / **deprecated** — not usable. diff --git a/src/cli/commands/extract.ts b/src/cli/commands/extract.ts index 3032e8f..09a267d 100644 --- a/src/cli/commands/extract.ts +++ b/src/cli/commands/extract.ts @@ -24,7 +24,7 @@ export const extract: Command = { usage: "zenrows extract [--autoparse | --css | --outputs | --output md|text] [flags]", help: [ "Methods:", - " (default) extract=auto — site-tailored Extract (beta); falls back to Autoparse if the domain is not enabled or prepared", + " (default) extract=auto: site-tailored Extract (beta); falls back to Autoparse if the domain is not enabled or prepared", " --autoparse general-purpose Autoparse (any domain, no Extract fallback)", " --css CSS selector map, e.g. '{\"title\":\"h1\",\"price\":\".price\"}'", " --outputs built-in output filters → JSON: emails, phone_numbers, headings,", diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index a460708..86ad685 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -28,8 +28,8 @@ export function getCapability(key: string): Capability | undefined { const USABLE: ReadonlySet = new Set([ "available", "available-but-needs-confirmation", - // Open beta: any key can call; product-specific limits (e.g. Extract domains - // not prepared yet, Batch 403) are handled by the adapters / API errors, not here. + // Beta: any key can call; product-specific limits (e.g. Extract domains not + // prepared yet, Batch 403) are handled by the adapters / API errors, not here. "beta", ]); From 93943a57778d8436ed20d013750f299f69a691a4 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 13 Oct 2026 19:03:50 +0200 Subject: [PATCH 3/7] docs(extract): drop the beta label now that Extract is generally available Extract is out of beta, so stop labelling it that way: the README commands table, the capability and skill registries (extract status is now `available`), the `extract --help` text, code comments, the docs and the skills, workflow and template that describe it. Pricing wording stays as is: extract=auto is still described as currently free. Batch keeps its beta label; it is unchanged here. See https://docs.zenrows.com/extract/introduction Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- agent-plugin/skills/cost-control/SKILL.md | 2 +- agent-plugin/skills/extract/SKILL.md | 4 ++-- agent-plugin/skills/zenrows/SKILL.md | 2 +- docs/capabilities.md | 6 +++--- registry/capabilities.json | 4 ++-- registry/skills.json | 2 +- skills/zenrows/SKILL.md | 2 +- src/adapters/extract.ts | 6 +++--- src/adapters/protected-fetch.ts | 2 +- src/cli/commands/extract.ts | 4 ++-- src/core/capabilities.ts | 4 ++-- templates/extract-pricing-pages/README.md | 2 +- tests/capabilities.test.ts | 2 +- tests/registry.test.ts | 2 +- workflows/competitor-intelligence/WORKFLOW.md | 2 +- 16 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 185e547..122d9bf 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ that endpoint with extraction parameters, not a separate product. | Command | What it does | Status (this build) | | --- | --- | --- | | `zenrows fetch` | Fetch — retrieve a protected page | **available** — `GET https://api.zenrows.com/v1/` | -| `zenrows extract` | Extract: `extract=auto` (any prepared domain; falls back to Autoparse on one not prepared yet), CSS, or Markdown | **beta**, same `/v1/` | +| `zenrows extract` | Extract: `extract=auto` (any prepared domain; falls back to Autoparse on one not prepared yet), CSS, or Markdown | available, same `/v1/` | | `zenrows batch` | Batch — fan out over many URLs | beta — cloud works with beta access; local validate/estimate always | | `zenrows crawl` | Crawl (new) — collect the URLs behind one start page, optionally with each page's HTML | **available** — `https://api.zenrows.com/v1/crawls` | | `zenrows browser` | Browser Sessions REST API (same backend as MCP `browser_*`) | **available** — escalation-only; bills by bandwidth + time | diff --git a/agent-plugin/skills/cost-control/SKILL.md b/agent-plugin/skills/cost-control/SKILL.md index 81aac30..8249039 100644 --- a/agent-plugin/skills/cost-control/SKILL.md +++ b/agent-plugin/skills/cost-control/SKILL.md @@ -18,7 +18,7 @@ These multipliers apply when you enable the flags (on `scrape` or `extract`): - Premium proxies (`premium_proxy`): **10×** - Both: **25×** -`extract` `mode=auto` is in **beta** and currently free; billing may apply +`extract` `mode=auto` is currently free; billing may apply later. Do not promise permanent free extract pricing. Autoparse / CSS still follow the multipliers above when stealth flags are set. diff --git a/agent-plugin/skills/extract/SKILL.md b/agent-plugin/skills/extract/SKILL.md index 625e7eb..4d08cc9 100644 --- a/agent-plugin/skills/extract/SKILL.md +++ b/agent-plugin/skills/extract/SKILL.md @@ -20,8 +20,8 @@ extract({ url, mode: "css", css_extractor: '{"title":"h1","price":".price"}' }) extract({ url, proxy_country: "US" }) # geo-targeted ``` -`mode=auto` (`extract=auto`) is in **beta**: works on any domain Zenrows has -prepared, with any key, and is currently free; billing may apply later. If the +`mode=auto` (`extract=auto`) works on any domain Zenrows has prepared, with any +key, and is currently free; billing may apply later. If the domain is not enabled (AUTH010) or not prepared yet (REQS007), the tool retries once with Autoparse by default (`fallback_autoparse`). diff --git a/agent-plugin/skills/zenrows/SKILL.md b/agent-plugin/skills/zenrows/SKILL.md index 7d6db20..566e1c1 100644 --- a/agent-plugin/skills/zenrows/SKILL.md +++ b/agent-plugin/skills/zenrows/SKILL.md @@ -66,7 +66,7 @@ If the user needs login, clicks, forms, sessions, or persistent state: | Primitive | MCP tools | Notes | | --- | --- | --- | | Protected Fetch | `scrape` | Default markdown; HTML / plaintext / PDF / screenshot options | -| Extract | `extract` | Structured JSON (`extract=auto`, autoparse, css_extractor). `extract=auto` in beta (currently free; billing may apply later) | +| Extract | `extract` | Structured JSON (`extract=auto`, autoparse, css_extractor). `extract=auto` is currently free; billing may apply later | | Batch | `batch_create`, `batch_status`, `batch_results`, `batch_cancel` | Beta; may return `BATCH_ACCESS_DENIED` without access | | Browser | `browser_*` (start with `browser_navigate`) | Escalation-only; same backend as Browser Sessions | diff --git a/docs/capabilities.md b/docs/capabilities.md index 3c4052c..4cd03d5 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -8,8 +8,8 @@ the single source of truth that prevents hallucinated execution. - **available** — a documented endpoint exists and the command does real work. - **available-but-needs-confirmation** — likely available; verify per account. - **experimental** — exists but not yet promoted to a stable status. -- **beta** — usable by any key. Product-specific limits (e.g. Extract domains - not prepared yet) are handled by adapters / API errors, not by blocking the CLI. +- **beta**: usable by any key. Product-specific limits (e.g. Batch access) + are handled by adapters / API errors, not by blocking the CLI. - **planned** — no documented endpoint yet; local spec / validation only. - **not-implemented** / **deprecated** — not usable. @@ -20,7 +20,7 @@ Classification is based on the public Zenrows documentation: | Capability | Backend evidence | Status | | --- | --- | --- | | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | -| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (beta; any prepared domain, CLI falls back to `autoparse` on one not prepared yet), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | beta | +| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (any prepared domain; CLI falls back to `autoparse` on one not prepared yet), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | available | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header) — open beta. Local JSONL spec validation + credit estimation work with no key. | beta | | `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL, up to `depth` when set (optional; no depth limit when omitted), on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Subcommands: `create`, `get`, `list`, `results`, `content`, `download`, `stop`, `wait`. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | diff --git a/registry/capabilities.json b/registry/capabilities.json index 855febe..6be044a 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -13,11 +13,11 @@ "extract": { "key": "extract", "label": "Extract (extract=auto / Autoparse / CSS / Markdown)", - "status": "beta", + "status": "available", "command": "zenrows extract", "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, autoparse, css_extractor, outputs, response_type)", "requiresAuth": true, - "notes": "Beta. Default is extract=auto, which works on any prepared domain with any key; CLI falls back to autoparse on REQS007 (domain not prepared yet) or AUTH010. Autoparse / CSS / outputs / markdown work on any domain." + "notes": "Default is extract=auto, which works on any prepared domain with any key; CLI falls back to autoparse on REQS007 (domain not prepared yet) or AUTH010. Autoparse / CSS / outputs / markdown work on any domain." }, "batch": { "key": "batch", diff --git a/registry/skills.json b/registry/skills.json index d4fb03f..0c95ead 100644 --- a/registry/skills.json +++ b/registry/skills.json @@ -26,7 +26,7 @@ "name": "extract", "type": "skill", "description": "Turn protected pages into structured data with Autoparse / CSS / Markdown.", - "status": "beta", + "status": "available", "requires_backend_capabilities": ["extract"], "requires_auth": true, "version": "0.1.0", diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index 2c6956e..6bb5a2f 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -72,7 +72,7 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | Primitive | Command | Status | | --- | --- | --- | | Protected Fetch | `zenrows fetch` | available (`GET /v1/`) | -| Extract (extract=auto / Autoparse/CSS/Markdown) | `zenrows extract` | beta (same `/v1/`; extract=auto falls back to autoparse) | +| Extract (extract=auto / Autoparse/CSS/Markdown) | `zenrows extract` | available (same `/v1/`; extract=auto falls back to autoparse) | | Batch | `zenrows batch` | beta — runs cloud jobs (create/status/results); cheapest path at scale. Estimate specs locally with no key | | Crawl | `zenrows crawl` | new — collect the URLs behind one start page, optionally with each page's HTML | | Browser | `zenrows browser` | available (Browser Sessions REST API / MCP) | diff --git a/src/adapters/extract.ts b/src/adapters/extract.ts index 00ba92e..b7606b0 100644 --- a/src/adapters/extract.ts +++ b/src/adapters/extract.ts @@ -2,7 +2,7 @@ * Extract adapter. * * Structured extraction on the same `/v1/` Fetch and Extract API via: - * - extract=auto → site-tailored Extract (beta, any prepared domain) + * - extract=auto → site-tailored Extract (any prepared domain) * - autoparse=true → general-purpose Autoparse (any domain) * - css_extractor= → selector-based field extraction * - outputs= → built-in output filters → JSON @@ -38,7 +38,7 @@ export interface ExtractOutcome extends FetchOutcome { method: ExtractMethod; /** Parsed JSON when the method yields structured data; otherwise undefined. */ data?: unknown; - /** Raw HTML from `extract=auto` beta responses (validation aid). */ + /** Raw HTML from `extract=auto` responses (validation aid). */ html?: string; /** True when Extract fell back to Autoparse because the domain is not enabled. */ fellBackToAutoparse?: boolean; @@ -150,7 +150,7 @@ function parseExtractBody( let data: unknown = parsed; let html: string | undefined; - // extract=auto beta shape: { parsed, html }. Prefer `parsed` for callers. + // extract=auto shape: { parsed, html }. Prefer `parsed` for callers. if (method === "extract" && parsed && typeof parsed === "object" && !Array.isArray(parsed)) { const envelope = parsed as { parsed?: unknown; html?: unknown }; if ("parsed" in envelope) { diff --git a/src/adapters/protected-fetch.ts b/src/adapters/protected-fetch.ts index 9a30c10..7b68c25 100644 --- a/src/adapters/protected-fetch.ts +++ b/src/adapters/protected-fetch.ts @@ -34,7 +34,7 @@ export interface FetchOptions { cssExtractor?: string; autoparse?: boolean; /** - * Site-tailored Extract (`extract=auto`). Beta; works on any prepared domain. + * Site-tailored Extract (`extract=auto`). Works on any prepared domain. * Takes precedence over autoparse / css_extractor / outputs / response_type. */ extract?: boolean; diff --git a/src/cli/commands/extract.ts b/src/cli/commands/extract.ts index 09a267d..cb91dae 100644 --- a/src/cli/commands/extract.ts +++ b/src/cli/commands/extract.ts @@ -1,7 +1,7 @@ /** * `zenrows extract [flags]` → Extract (structured extraction on `/v1/`). * - * Default: `extract=auto` (site-tailored Extract, beta). If the domain is + * Default: `extract=auto` (site-tailored Extract). If the domain is * not enabled or prepared (AUTH010 / REQS007), falls back once to Autoparse. Opt into Autoparse * directly with `--autoparse`. Other methods: `--css`, `--outputs`, `--output`. */ @@ -24,7 +24,7 @@ export const extract: Command = { usage: "zenrows extract [--autoparse | --css | --outputs | --output md|text] [flags]", help: [ "Methods:", - " (default) extract=auto: site-tailored Extract (beta); falls back to Autoparse if the domain is not enabled or prepared", + " (default) extract=auto: site-tailored Extract; falls back to Autoparse if the domain is not enabled or prepared", " --autoparse general-purpose Autoparse (any domain, no Extract fallback)", " --css CSS selector map, e.g. '{\"title\":\"h1\",\"price\":\".price\"}'", " --outputs built-in output filters → JSON: emails, phone_numbers, headings,", diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index 86ad685..73f2f74 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -28,8 +28,8 @@ export function getCapability(key: string): Capability | undefined { const USABLE: ReadonlySet = new Set([ "available", "available-but-needs-confirmation", - // Beta: any key can call; product-specific limits (e.g. Extract domains not - // prepared yet, Batch 403) are handled by the adapters / API errors, not here. + // Beta: any key can call; product-specific limits (e.g. Batch 403) are + // handled by the adapters / API errors, not here. "beta", ]); diff --git a/templates/extract-pricing-pages/README.md b/templates/extract-pricing-pages/README.md index f26358a..81e4001 100644 --- a/templates/extract-pricing-pages/README.md +++ b/templates/extract-pricing-pages/README.md @@ -13,5 +13,5 @@ zenrows extract https://www.scrapingcourse.com/ecommerce/ --autoparse zenrows extract --css '{"title":"h1","price":".price"}' --validate ``` -Requires `extract` (beta). Start on one page, confirm the fields, then +Requires `extract`. Start on one page, confirm the fields, then fan out `zenrows extract` across the rest of your URLs. diff --git a/tests/capabilities.test.ts b/tests/capabilities.test.ts index 73c7da6..caff159 100644 --- a/tests/capabilities.test.ts +++ b/tests/capabilities.test.ts @@ -6,7 +6,7 @@ import { ToolkitError } from "../src/core/errors.ts"; test("capability matrix loads with the honest classifications", () => { const caps = loadCapabilities(); assert.equal(caps.protected_fetch?.status, "available"); - assert.equal(caps.extract?.status, "beta"); + assert.equal(caps.extract?.status, "available"); assert.equal(caps.browser?.status, "available"); // GA product (formerly Scraping Browser) assert.equal(caps.mcp?.status, "available"); // Batch is a real product in open beta. diff --git a/tests/registry.test.ts b/tests/registry.test.ts index 3ca7fb7..7ae3acc 100644 --- a/tests/registry.test.ts +++ b/tests/registry.test.ts @@ -50,7 +50,7 @@ test("assetRunnable reflects backend capability status", () => { const batchSkill = loadRegistry("skill").find((s) => s.name === "batch-jobs")!; assert.equal(assetRunnable(batchSkill), true); // needs batch (open beta = usable) const extractSkill = loadRegistry("skill").find((s) => s.name === "extract")!; - assert.equal(assetRunnable(extractSkill), true); // needs extract (open beta = usable) + assert.equal(assetRunnable(extractSkill), true); // needs extract (available) const crawlSkill = loadRegistry("skill").find((s) => s.name === "crawl")!; assert.equal(assetRunnable(crawlSkill), true); // needs crawl (available) }); diff --git a/workflows/competitor-intelligence/WORKFLOW.md b/workflows/competitor-intelligence/WORKFLOW.md index 002ca65..d0618c0 100644 --- a/workflows/competitor-intelligence/WORKFLOW.md +++ b/workflows/competitor-intelligence/WORKFLOW.md @@ -20,7 +20,7 @@ simple per-URL loop). ``` zenrows fetch https://competitor.example/products ``` -2. **Validate extraction** on one page (extract is beta; extract=auto falls back to autoparse). +2. **Validate extraction** on one page (extract=auto falls back to autoparse on a domain not prepared yet). ``` zenrows extract https://competitor.example/p/123 --validate ``` From ab0428cb46161c2f891d17dce99883e1e397e9b8 Mon Sep 17 00:00:00 2001 From: Ander Date: Tue, 13 Oct 2026 19:34:57 +0200 Subject: [PATCH 4/7] docs(extract): drop the em dash from the AUTH010 comment the PR touches Co-Authored-By: Claude Opus 5.5 --- src/core/http.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/http.ts b/src/core/http.ts index c076165..840d05f 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -189,7 +189,7 @@ export async function scrape( } if (res.status === 402 && isZenrowsErrorEnvelope(body)) { // AUTH010 on extract=auto means the target domain is not enabled for - // Extract — distinct from credit exhaustion (AUTH004). Let the extract + // Extract, distinct from credit exhaustion (AUTH004). Let the extract // adapter fall back to autoparse. if (params.extract !== undefined && zrErrorCode(body) === "AUTH010") { throw new ToolkitError({ From e8affaad3d447d06a9b99e8881832551947400b2 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 13 Oct 2026 23:22:25 +0200 Subject: [PATCH 5/7] docs(capabilities): stop using Batch as the example of a beta status No primitive is in beta once Extract and Batch are generally available, so drop the beta entry from the status legend in docs/capabilities.md and the Batch example from the comment on the usable statuses. The beta status value itself stays in the code. Co-Authored-By: Claude Opus 5.5 --- docs/capabilities.md | 2 -- src/core/capabilities.ts | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/capabilities.md b/docs/capabilities.md index 4cd03d5..aa6dd2f 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -8,8 +8,6 @@ the single source of truth that prevents hallucinated execution. - **available** — a documented endpoint exists and the command does real work. - **available-but-needs-confirmation** — likely available; verify per account. - **experimental** — exists but not yet promoted to a stable status. -- **beta**: usable by any key. Product-specific limits (e.g. Batch access) - are handled by adapters / API errors, not by blocking the CLI. - **planned** — no documented endpoint yet; local spec / validation only. - **not-implemented** / **deprecated** — not usable. diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index 73f2f74..5ae108d 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -28,8 +28,8 @@ export function getCapability(key: string): Capability | undefined { const USABLE: ReadonlySet = new Set([ "available", "available-but-needs-confirmation", - // Beta: any key can call; product-specific limits (e.g. Batch 403) are - // handled by the adapters / API errors, not here. + // Beta: any key can call; product-specific limits are handled by the + // adapters / API errors, not here. "beta", ]); From f90dfe9b4e721d19af56ba7a90666363725cd37d Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 14 Oct 2026 00:04:18 +0200 Subject: [PATCH 6/7] fix(extract): report 422 no_data_extracted as an empty result When Extract knows a kind of page but none of the fields come back filled, the API answers 422 with code no_data_extracted. The request is not charged, and repeating it returns the same answer. The CLI treated every 422 as a failed fetch and suggested escalating with --js-render --premium-proxy, which reads as if the site could not be reached. It now returns that answer to the extract adapter as an empty result: one request, no Autoparse fallback, a warning that says no data was found and how to check the page rendered fully, and `noDataExtracted: true` in --json output. Under --validate it fails with EXTRACT_VALIDATION_FAILED. Other 422s keep the existing handling. Co-Authored-By: Claude Opus 5.5 --- src/adapters/extract.ts | 19 ++++++++++++ src/cli/commands/extract.ts | 7 ++++- src/core/http.ts | 13 +++++++++ tests/extract-outputs.test.ts | 54 +++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) diff --git a/src/adapters/extract.ts b/src/adapters/extract.ts index b7606b0..0876e96 100644 --- a/src/adapters/extract.ts +++ b/src/adapters/extract.ts @@ -47,6 +47,11 @@ export interface ExtractOutcome extends FetchOutcome { * array / non-JSON). Lets callers warn instead of reporting a silent success. */ empty?: boolean; + /** + * True when Extract answered `no_data_extracted` (HTTP 422): it knows this kind + * of page but no field came back filled. Always reported with `empty: true`. + */ + noDataExtracted?: boolean; } /** True when a parsed extraction result carries no usable structured data. */ @@ -117,6 +122,20 @@ async function runExtractOnce( }; const outcome = await runFetch(fetchOpts, config, policy, apiKey); + if (outcome.result.noDataExtracted) { + if (opts.validate) { + throw new ToolkitError({ + code: "EXTRACT_VALIDATION_FAILED", + message: "Extract found no data on this page.", + likely_cause: + "Extract knows this kind of page, but none of the fields came back filled (HTTP 422 no_data_extracted). The request was not charged.", + next_action: + "Check that the page rendered fully: try --manual --js-render with --wait-for . Protected sites may also need --premium-proxy.", + suggested_commands: [`zenrows extract ${opts.url} --manual --js-render --premium-proxy`], + }); + } + return { ...outcome, method, empty: true, noDataExtracted: true }; + } const { data, html, empty } = parseExtractBody(method, outcome.result.body, opts); return { ...outcome, method, data, html, empty }; diff --git a/src/cli/commands/extract.ts b/src/cli/commands/extract.ts index cb91dae..3128720 100644 --- a/src/cli/commands/extract.ts +++ b/src/cli/commands/extract.ts @@ -165,7 +165,11 @@ export const extract: Command = { `Extracted via ${via} · ${outcome.result.body.length} bytes · ${formatRequestCost(outcome.result.costUsd, outcome.result.costCredits)} · run ${runId}`, ); if (runDir) log.dim(` artifact: ${runDir}`); - if (outcome.empty) { + if (outcome.noDataExtracted) { + log.warn( + "Extract found no data on this page (HTTP 422 no_data_extracted, not charged). Check that the page rendered fully: try --manual --js-render with --wait-for . Protected sites may also need --premium-proxy.", + ); + } else if (outcome.empty) { log.warn( "No structured data extracted — the page may render via JavaScript, or this layout isn't auto-extractable. Try --manual --js-render, or --css with explicit selectors.", ); @@ -180,6 +184,7 @@ export const extract: Command = { method: outcome.method, fellBackToAutoparse: outcome.fellBackToAutoparse ?? false, empty: outcome.empty ?? false, + noDataExtracted: outcome.noDataExtracted ?? false, httpStatus: outcome.result.status, data: outcome.data ?? null, bytes: outcome.result.body.length, diff --git a/src/core/http.ts b/src/core/http.ts index 840d05f..eb09553 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -37,6 +37,12 @@ export interface ScraperResult { concurrencyRemaining: number | null; /** The request URL with the apikey redacted — safe to log/persist. */ redactedUrl: string; + /** + * True when an Extract request answered 422 with code `no_data_extracted`: + * Extract knows this kind of page but none of the fields came back filled. + * The request is not charged, and the same request returns the same answer. + */ + noDataExtracted?: boolean; } export interface ScraperParams { @@ -152,6 +158,13 @@ export async function scrape( suggested_commands: [`zenrows extract ${params.url} --autoparse`], }); } + // 422 no_data_extracted on an Extract request: the page was processed and had + // nothing to extract. It is an answer, not a failure to reach the target, so + // return it for the extract adapter to report as an empty result instead of + // advising escalation (which would repeat a slow, deterministic request). + if (res.status === 422 && params.extract !== undefined && zrErrorCode(body) === "NO_DATA_EXTRACTED") { + return { ...result, noDataExtracted: true }; + } if (res.status === 401 || res.status === 403) { // Distinguish Zenrows auth errors from target-site 403s. Zenrows auth // failures surface AUTH00x codes in the body. diff --git a/tests/extract-outputs.test.ts b/tests/extract-outputs.test.ts index 3ddb694..35f0f8f 100644 --- a/tests/extract-outputs.test.ts +++ b/tests/extract-outputs.test.ts @@ -329,3 +329,57 @@ test("runExtract --validate fails on an empty result", async () => { ); }); }); + +// --- 422 no_data_extracted: an answer, not a failure ------------------------ +const NO_DATA = JSON.stringify({ + code: "no_data_extracted", + detail: "Extract found no data on this page.", + status: 422, + title: "No data extracted", +}); + +test("runExtract reports 422 no_data_extracted as an empty result, in one request, without fallback", async () => { + const calls: string[] = []; + await withFetchHandler((url) => { + calls.push(url); + return new Response(NO_DATA, { status: 422, headers: { "content-type": "application/problem+json" } }); + }, async () => { + const outcome = await runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"); + assert.equal(outcome.method, "extract"); + assert.equal(outcome.empty, true); + assert.equal(outcome.noDataExtracted, true); + assert.equal(outcome.fellBackToAutoparse, undefined); + assert.equal(outcome.data, undefined); + assert.equal(calls.length, 1); + assert.match(calls[0]!, /extract=auto/); + }); +}); + +test("runExtract under --validate turns 422 no_data_extracted into EXTRACT_VALIDATION_FAILED", async () => { + await withFetch(NO_DATA, async () => { + await assert.rejects( + () => runExtract({ url: "https://x.com", validate: true }, cfg, defaultPolicy(), "testkey"), + (e: unknown) => + e instanceof ToolkitError && e.code === "EXTRACT_VALIDATION_FAILED" && /no data/i.test(e.message), + ); + }, { status: 422 }); +}); + +test("runExtract keeps treating other 422s as a failed fetch", async () => { + const RESP001 = JSON.stringify({ code: "RESP001", detail: "Could not get content.", status: 422 }); + await withFetch(RESP001, async () => { + await assert.rejects( + () => runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"), + (e: unknown) => e instanceof ToolkitError && e.code === "FETCH_FAILED", + ); + }, { status: 422 }); +}); + +test("a 422 no_data_extracted outside an Extract request is still a failed fetch", async () => { + await withFetch(NO_DATA, async () => { + await assert.rejects( + () => runExtract({ url: "https://x.com", method: "autoparse" }, cfg, defaultPolicy(), "testkey"), + (e: unknown) => e instanceof ToolkitError && e.code === "FETCH_FAILED", + ); + }, { status: 422 }); +}); From 2c7f8ba511cd63b40891afc13a151b813be30499 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 14 Oct 2026 07:34:21 +0200 Subject: [PATCH 7/7] fix(extract): explain 422 blocked_document instead of reporting a failed fetch An Extract request whose fetched page was a block or error page answers 422 blocked_document (not charged). The CLI reported it as "Protected Fetch failed with HTTP 422". It now says what happened and suggests the stronger fetch that can get past it. Co-Authored-By: Claude Opus 5.5 --- src/core/http.ts | 13 +++++++++++++ tests/extract-outputs.test.ts | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/core/http.ts b/src/core/http.ts index eb09553..8619ccb 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -165,6 +165,19 @@ export async function scrape( if (res.status === 422 && params.extract !== undefined && zrErrorCode(body) === "NO_DATA_EXTRACTED") { return { ...result, noDataExtracted: true }; } + // 422 blocked_document on an Extract request: the fetched page was a bot wall, + // an error page or an empty shell, so Extract had nothing real to read. Not + // charged. The fix is a stronger fetch of the same URL, not another extractor. + if (res.status === 422 && params.extract !== undefined && zrErrorCode(body) === "BLOCKED_DOCUMENT") { + throw new ToolkitError({ + code: "EXTRACT_FAILED", + message: "Extract could not read this page: the site answered with a block or error page (HTTP 422 blocked_document, not charged).", + likely_cause: zrErrorDetail(body) ?? "The fetched page was a bot challenge, an error page or a page that had not finished loading.", + next_action: + "Retry with a stronger fetch: --manual --js-render --premium-proxy. Check the URL if the site may be returning an error page.", + suggested_commands: [`zenrows extract ${params.url} --manual --js-render --premium-proxy`], + }); + } if (res.status === 401 || res.status === 403) { // Distinguish Zenrows auth errors from target-site 403s. Zenrows auth // failures surface AUTH00x codes in the body. diff --git a/tests/extract-outputs.test.ts b/tests/extract-outputs.test.ts index 35f0f8f..2a4158e 100644 --- a/tests/extract-outputs.test.ts +++ b/tests/extract-outputs.test.ts @@ -383,3 +383,27 @@ test("a 422 no_data_extracted outside an Extract request is still a failed fetch ); }, { status: 422 }); }); + +// --- 422 blocked_document: a block page, not an extraction miss -------------- +test("runExtract turns 422 blocked_document into EXTRACT_FAILED with a stronger-fetch hint, without fallback", async () => { + const BLOCKED = JSON.stringify({ + code: "blocked_document", + detail: "The page fetched for this request had no real content.", + status: 422, + }); + const calls: string[] = []; + await withFetchHandler((url) => { + calls.push(url); + return new Response(BLOCKED, { status: 422, headers: { "content-type": "application/problem+json" } }); + }, async () => { + await assert.rejects( + () => runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"), + (e: unknown) => + e instanceof ToolkitError && + e.code === "EXTRACT_FAILED" && + /blocked_document/.test(e.message) && + /--js-render --premium-proxy/.test(e.next_action ?? ""), + ); + assert.equal(calls.length, 1); + }); +});