From 593f5c91ad99c95049675fdc87104bf20f088e80 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 08:47:13 +0000 Subject: [PATCH 1/8] feat(extract): report REQS007 with the prepare step instead of falling back to autoparse On 403 REQS007 (domain not prepared yet) or the legacy 402 AUTH010, `zenrows extract` used to retry with the deprecated autoparse. That hid the prepare step, charged for an Autoparse call and often returned unusable data. The error is now reported, not retried, and names the next step: Extract hasn't prepared yet. Prepare it with POST /v1/extract/prepared-domains (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse. `--autoparse` stays as the explicit opt-in and now warns that Autoparse is deprecated. The adapter's `fallbackAutoparse` option defaults to false. The empty-result check now treats a payload whose fields are all null or empty (recursively) as empty, so page scaffolding no longer reads as data. Skills, docs and the capability matrix describe the prepare step, mark autoparse deprecated and drop the fallback wording. --- README.md | 6 +- agent-plugin/README.md | 2 +- agent-plugin/skills/cost-control/SKILL.md | 4 +- agent-plugin/skills/extract/SKILL.md | 37 ++++-- agent-plugin/skills/protected-fetch/SKILL.md | 2 +- agent-plugin/skills/zenrows/SKILL.md | 4 +- docs/capabilities.md | 2 +- registry/capabilities.json | 6 +- registry/skills.json | 4 +- registry/templates.json | 2 +- skills/extract/SKILL.md | 33 +++++- skills/zenrows/SKILL.md | 7 +- src/adapters/extract.ts | 32 ++--- src/cli/commands/extract.ts | 15 ++- src/core/http.ts | 55 +++++---- templates/extract-pricing-pages/README.md | 8 +- tests/capabilities.test.ts | 7 ++ tests/extract-outputs.test.ts | 110 +++++++++++------- tests/http.test.ts | 4 + workflows/competitor-intelligence/WORKFLOW.md | 4 +- 20 files changed, 220 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index 1f7180f..a0d2412 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ that endpoint with extraction parameters, not a separate product. | Command | What it does | Status (this build) | | --- | --- | --- | | `zenrows fetch` | Fetch: retrieve a protected page | **available**: `GET https://api.zenrows.com/v1/` | -| `zenrows extract` | Extract: turn a page into structured data with `extract=auto` on any prepared domain (falls back to Autoparse on one not prepared yet), CSS selectors, or Markdown | **available**: `GET https://api.zenrows.com/v1/?extract=auto` | +| `zenrows extract` | Extract: turn a page into structured data with `extract=auto` on any prepared domain (you can prepare one yourself), CSS selectors, or Markdown | **available**: `GET https://api.zenrows.com/v1/?extract=auto` | | `zenrows batch` | Batch: run Fetch and Extract over many URLs in one job; specs validate and estimate locally with no key | **available**: `POST https://async.api.zenrows.com/v1/jobs` | | `zenrows crawl` | Crawl (new): collect the URLs behind one start page, optionally with each page's HTML | **available**: `POST https://api.zenrows.com/v1/crawls` | | `zenrows browser` | Browser Sessions: a real cloud browser over REST (same backend as MCP `browser_*`); escalation-only, billed by bandwidth + time | **available**: `POST https://mcp.zenrows.com/browser/sessions` | @@ -56,7 +56,7 @@ npm install -g @zenrows/cli # puts `zenrows` on PATH (466 ms, zero deps) zenrows init zenrows fetch https://httpbin.io/html # auto-provisions a Free plan account on first use zenrows extract https://www.owler.com/company/meltwater # extract=auto on a prepared domain -zenrows extract https://www.scrapingcourse.com/ecommerce/ --autoparse # Autoparse (any domain) +zenrows extract https://www.scrapingcourse.com/ecommerce/ --css '{"title":"h1"}' # CSS selectors (any domain) ``` > Prefer no global install? Prefix each command with `npx -y @zenrows/cli` @@ -117,7 +117,7 @@ zenrows fetch --proxy-country us --wait-for ".price" ```bash zenrows extract https://www.owler.com/company/meltwater # extract=auto (prepared domain) -zenrows extract https://www.scrapingcourse.com/ecommerce/ --autoparse # Autoparse (any domain) +zenrows extract https://www.scrapingcourse.com/ecommerce/ --autoparse # deprecated Autoparse (any domain) zenrows extract --css '{"title":"h1","price":".price"}' --validate zenrows extract --outputs emails,links # built-in output filters → JSON zenrows extract --output markdown diff --git a/agent-plugin/README.md b/agent-plugin/README.md index 351d5c6..52e1da7 100644 --- a/agent-plugin/README.md +++ b/agent-plugin/README.md @@ -62,7 +62,7 @@ Use MCP tool names — not CLI `zenrows fetch`: | --- | --- | | `zenrows` | Decision tree across all primitives | | `protected-fetch` | `scrape` | -| `extract` | `extract` (fallback: `scrape` + autoparse / css_extractor) | +| `extract` | `extract` (fallback: `scrape` + css_extractor) | | `batch-jobs` | `batch_create`, `batch_status`, `batch_results`, `batch_cancel` | | `interact-browser` | `browser_*` (escalation only) | | `cost-control` | Cost multipliers + escalate-with-evidence | diff --git a/agent-plugin/skills/cost-control/SKILL.md b/agent-plugin/skills/cost-control/SKILL.md index d6a76f0..4adf57f 100644 --- a/agent-plugin/skills/cost-control/SKILL.md +++ b/agent-plugin/skills/cost-control/SKILL.md @@ -19,8 +19,8 @@ These multipliers apply when you enable the flags (on `scrape` or `extract`): - Both: **25×** `extract` `mode=auto` is currently free; billing may apply -later. Do not promise permanent free extract pricing. Autoparse / CSS still -follow the multipliers above when stealth flags are set. +later. Do not promise permanent free extract pricing. CSS (and the deprecated +Autoparse) still follow the multipliers above when stealth flags are set. ## Browser sessions (`browser_*`) diff --git a/agent-plugin/skills/extract/SKILL.md b/agent-plugin/skills/extract/SKILL.md index 4d08cc9..3ec4ce3 100644 --- a/agent-plugin/skills/extract/SKILL.md +++ b/agent-plugin/skills/extract/SKILL.md @@ -14,28 +14,46 @@ Convert protected pages into structured data. Value is **protected page access Use **`extract`** when you need structured JSON fields rather than a full page: ``` -extract({ url }) # mode=auto (default) -extract({ url, mode: "autoparse" }) # general Autoparse (any domain) +extract({ url }) # mode=auto (default), returns `parsed` only +extract({ url, include_html: true }) # also return the full page HTML +extract({ url, mode: "autoparse" }) # deprecated Autoparse (any domain) extract({ url, mode: "css", css_extractor: '{"title":"h1","price":".price"}' }) extract({ url, proxy_country: "US" }) # geo-targeted ``` -`mode=auto` (`extract=auto`) works on any domain Zenrows has prepared, with any -key, and is currently free; billing may apply later. If the -domain is not enabled (AUTH010) or not prepared yet (REQS007), the tool retries -once with Autoparse by default (`fallback_autoparse`). +`mode=auto` (`extract=auto`) works on any prepared domain, with any key, and +is currently free; billing may apply later. The result carries `parsed` only; +the page HTML is most of the response size, so ask for it with +`include_html: true` only when you need it. Every mode uses Adaptive Stealth Mode by default: pass just the URL, even for protected or dynamic sites. Set `js_render` or `premium_proxy` only to force a fixed configuration; that turns Adaptive Stealth Mode off. Check `empty` in the result: true means no field came back with a value. +## Domain not prepared yet (REQS007) + +`mode=auto` works on any prepared domain, and you can prepare one yourself. On +a domain Extract hasn't prepared yet, `extract` returns a tool error (`REQS007`; +legacy `AUTH010`) that names the next step. It does not retry with Autoparse +unless you set `fallback_autoparse: true`. To prepare the domain: + +``` +POST https://api.zenrows.com/v1/extract/prepared-domains +X-API-Key: +{"domain":"example.com","example_urls":["https://example.com/p/1","https://example.com/p/2"]} +``` + +Give 1 to 5 example URLs of the pages you want. Preparation takes a few +minutes and uses no credits. Poll `GET /v1/extract/prepared-domains/{domain}` +until `status` is `ready` (or `failed`), then call `extract` again. + ## Fallback: `scrape` extraction params If `extract` is unavailable, similar outcomes are available on **`scrape`**: ``` -scrape({ url, autoparse: true }) +scrape({ url, autoparse: true }) # deprecated scrape({ url, css_extractor: '{"title":"h1","price":".price"}' }) scrape({ url, outputs: "emails,links" }) # or "*" for all built-ins ``` @@ -46,7 +64,8 @@ scrape({ url, outputs: "emails,links" }) # or "*" for all built-ins ## When to use which - **`mode=auto`**: site-tailored fields on any prepared domain. -- **`mode=autoparse`**: any website; also the automatic AUTH010 / REQS007 fallback. +- **`mode=autoparse`** (deprecated): any website, only when asked for. Do not + build new work on it. - **`mode=css`**: known fields/selectors; deterministic. - **Markdown via `scrape`**: feed clean content to an LLM yourself. @@ -57,7 +76,7 @@ Stealth flags (`js_render`, `premium_proxy`) still apply the usual multipliers ## Rules - Validate on a single page before `batch_create` ([[batch-jobs]]). -- If Autoparse misses fields, switch to `css_extractor`, or add `js_render` +- If a field is missing, switch to `css_extractor`, or add `js_render` for JS-heavy pages. - Prefer MCP `extract` / `scrape` over CLI `zenrows extract` when the MCP server is connected. diff --git a/agent-plugin/skills/protected-fetch/SKILL.md b/agent-plugin/skills/protected-fetch/SKILL.md index c6cc969..6b1c60c 100644 --- a/agent-plugin/skills/protected-fetch/SKILL.md +++ b/agent-plugin/skills/protected-fetch/SKILL.md @@ -34,7 +34,7 @@ scrape({ url, js_render: true, wait_for: ".price" }) - Prefer `response_type: "markdown"` for LLM consumption; use `html` only when you need raw markup. - For **structured** fields, prefer the `extract` tool (or `scrape` with - `autoparse` / `css_extractor` / `outputs`) — see [[extract]]. + `css_extractor` / `outputs`) — see [[extract]]. - On failure, escalate with evidence before `browser_*` ([[interact-browser]]). - Cost model: [[cost-control]]. diff --git a/agent-plugin/skills/zenrows/SKILL.md b/agent-plugin/skills/zenrows/SKILL.md index 1cae7d3..8efaa9c 100644 --- a/agent-plugin/skills/zenrows/SKILL.md +++ b/agent-plugin/skills/zenrows/SKILL.md @@ -36,7 +36,7 @@ If the user has a known URL and wants page content: If the user has a known URL and wants structured data: → Use Extract. (MCP tool: extract) - Fallback: scrape with autoparse / css_extractor / outputs + Fallback: scrape with css_extractor / outputs If the user has many URLs (bulk): → Use Batch. (validate one page with scrape/extract first, @@ -66,7 +66,7 @@ If the user needs login, clicks, forms, sessions, or persistent state: | Primitive | MCP tools | Notes | | --- | --- | --- | | Protected Fetch | `scrape` | Default markdown; HTML / plaintext / PDF / screenshot options | -| Extract | `extract` | Structured JSON (`extract=auto`, autoparse, css_extractor). `extract=auto` is currently free; billing may apply later | +| Extract | `extract` | Structured JSON (`extract=auto` on any prepared domain, css_extractor; autoparse deprecated). `extract=auto` is currently free; billing may apply later | | Batch | `batch_create`, `batch_status`, `batch_results`, `batch_cancel` | Same credits as Fetch; a 403 maps to `BATCH_ACCESS_DENIED` | | Browser | `browser_*` (start with `browser_navigate`) | Escalation-only; same backend as Browser Sessions | diff --git a/docs/capabilities.md b/docs/capabilities.md index 36c2409..416ef96 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -18,7 +18,7 @@ Classification is based on the public Zenrows documentation: | Capability | Backend evidence | Status | | --- | --- | --- | | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | -| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (any prepared domain; CLI falls back to `autoparse` on one not prepared yet), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | available | +| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (any prepared domain; on one not prepared yet the CLI reports REQS007 with the prepare step, `POST /v1/extract/prepared-domains`), plus deprecated `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | available | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header). Local JSONL spec validation + credit estimation work with no key. | available | | `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL, up to `depth` when set (optional; no depth limit when omitted), on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Subcommands: `create`, `get`, `list`, `results`, `content`, `download`, `stop`, `wait`. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | diff --git a/registry/capabilities.json b/registry/capabilities.json index d81b872..6558452 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -12,12 +12,12 @@ }, "extract": { "key": "extract", - "label": "Extract (extract=auto / Autoparse / CSS / Markdown)", + "label": "Extract (extract=auto / CSS / Markdown)", "status": "available", "command": "zenrows extract", - "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, autoparse, css_extractor, outputs, response_type)", + "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, css_extractor, outputs, response_type; autoparse deprecated)", "requiresAuth": true, - "notes": "Default is extract=auto, which works on any prepared domain with any key; CLI falls back to autoparse on REQS007 (domain not prepared yet) or AUTH010. Autoparse / CSS / outputs / markdown work on any domain." + "notes": "Default is extract=auto, which works on any prepared domain with any key. On a domain not prepared yet (REQS007, legacy AUTH010) the CLI reports the error and does not retry; prepare the domain with POST /v1/extract/prepared-domains (a few example URLs, a few minutes, no credits). CSS / outputs / markdown work on any domain; autoparse is deprecated and runs only with --autoparse." }, "batch": { "key": "batch", diff --git a/registry/skills.json b/registry/skills.json index cdc806c..2169684 100644 --- a/registry/skills.json +++ b/registry/skills.json @@ -25,13 +25,13 @@ { "name": "extract", "type": "skill", - "description": "Turn protected pages into structured data with Autoparse / CSS / Markdown.", + "description": "Turn protected pages into structured data with Extract / CSS / Markdown (Autoparse is deprecated).", "status": "available", "requires_backend_capabilities": ["extract"], "requires_auth": true, "version": "0.1.0", "path": "skills/extract", - "tags": ["extract", "autoparse", "structured-data"] + "tags": ["extract", "structured-data"] }, { "name": "batch-jobs", diff --git a/registry/templates.json b/registry/templates.json index 39f25f6..8686d6e 100644 --- a/registry/templates.json +++ b/registry/templates.json @@ -14,7 +14,7 @@ { "name": "extract-pricing-pages", "type": "template", - "description": "Extract structured pricing fields from product pages (Autoparse / CSS).", + "description": "Extract structured pricing fields from product pages (extract=auto / CSS).", "status": "available", "requires_backend_capabilities": ["extract"], "requires_auth": true, diff --git a/skills/extract/SKILL.md b/skills/extract/SKILL.md index 597d49c..f583a1d 100644 --- a/skills/extract/SKILL.md +++ b/skills/extract/SKILL.md @@ -1,6 +1,6 @@ --- name: extract -description: Turn protected pages into structured data with Extract / Autoparse / CSS / Markdown. +description: Turn protected pages into structured data with Extract / CSS / Markdown (Autoparse is deprecated). version: 0.1.0 requires_backend_capabilities: [extract] --- @@ -12,8 +12,8 @@ access + extraction**, not generic LLM parsing. ## Methods ``` -zenrows extract # extract=auto (default); falls back to Autoparse if the domain is not prepared yet -zenrows extract --autoparse # general-purpose Autoparse (any domain) +zenrows extract # extract=auto (default), any prepared domain +zenrows extract --autoparse # deprecated general-purpose Autoparse (any domain) zenrows extract --css '{"title":"h1","price":".price"}' # selector map zenrows extract --outputs emails,links # built-in output filters → JSON zenrows extract --output markdown # Markdown conversion @@ -21,14 +21,35 @@ zenrows extract --validate # fail if not valid JSON ``` ## When to use which -- **Extract** (`extract=auto`, default): richest fields; works on any domain Zenrows has prepared. -- **Autoparse**: any website; also the automatic fallback when the domain is not prepared for Extract yet. +- **Extract** (`extract=auto`, default): richest fields; works on any prepared + domain (you can prepare one yourself, see below). +- **Autoparse** (deprecated): only with `--autoparse`. Never automatic; do not + build new work on it. - **CSS**: you know the exact fields/selectors and want determinism. - **Markdown/plaintext**: feed clean content to an LLM yourself. +## Domain not prepared yet (REQS007) + +`extract=auto` works on any prepared domain, and you can prepare one yourself. +On a domain Extract hasn't prepared yet the API answers `403 REQS007` (legacy: +`402 AUTH010`) and `zenrows extract` reports it; it does not retry with +Autoparse. To prepare the domain: + +``` +POST https://api.zenrows.com/v1/extract/prepared-domains +X-API-Key: +{"domain":"example.com","example_urls":["https://example.com/p/1","https://example.com/p/2"]} +``` + +Give 1 to 5 example URLs of the pages you want. Preparation takes a few +minutes and uses no credits. Poll `GET /v1/extract/prepared-domains/{domain}` +until `status` is `ready` (or `failed`), then retry the extract. + ## Rules - Validate on a single page before scaling across many URLs. -- If Autoparse misses fields, switch to `--css` with explicit selectors. Auto +- Check `empty` in `--json` output: true means every field came back null or + empty, so the result is not data. +- If a field is missing, switch to `--css` with explicit selectors. Auto mode already renders JS when the target needs it, so reach for `--manual --js-render` only after auto mode has returned an incomplete page: it costs several times a basic call ([[cost-control]]) and it makes the diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index c782a36..70a2ab0 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -32,7 +32,7 @@ If the user has a known URL and wants page content: → Use Protected Fetch. (zenrows fetch ) If the user has a known URL and wants structured data: - → Use Extract. (zenrows extract | --autoparse | --css) + → Use Extract. (zenrows extract | --css) If the user has a start page (listing, index, section) but not the item URLs: → Use Crawl. (zenrows crawl create --depth 1 --follow, @@ -72,13 +72,14 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | Primitive | Command | Status | | --- | --- | --- | | Protected Fetch | `zenrows fetch` | **available**: `GET https://api.zenrows.com/v1/` | -| Extract (`extract=auto`, Autoparse, CSS, Markdown) | `zenrows extract` | **available**: `GET https://api.zenrows.com/v1/?extract=auto` | +| Extract (`extract=auto`, CSS, Markdown; Autoparse deprecated) | `zenrows extract` | **available**: `GET https://api.zenrows.com/v1/?extract=auto` | | Batch | `zenrows batch` | **available**: `POST https://async.api.zenrows.com/v1/jobs` | | Crawl (new) | `zenrows crawl` | **available**: `POST https://api.zenrows.com/v1/crawls` | | Browser | `zenrows browser` | **available**: `POST https://mcp.zenrows.com/browser/sessions` | | MCP | `zenrows mcp` | **available**: `https://mcp.zenrows.com/mcp` | -- Extract: `extract=auto` falls back to Autoparse on a domain not prepared yet. +- Extract: `extract=auto` works on any prepared domain. On one not prepared yet + (REQS007), prepare it with `POST /v1/extract/prepared-domains`; see [[extract]]. - Batch: the cheapest path at scale (create, status, results). Estimate specs locally with no key. - Crawl: collects the URLs behind one start page, optionally with each page's HTML. - Browser: escalation-only, for pages Fetch can't handle; same backend as the MCP `browser_*` tools. diff --git a/src/adapters/extract.ts b/src/adapters/extract.ts index 0876e96..f673521 100644 --- a/src/adapters/extract.ts +++ b/src/adapters/extract.ts @@ -8,10 +8,10 @@ * - outputs= → built-in output filters → JSON * - response_type=markdown|plaintext * - * Default method is `extract`. On AUTH010 / REQS007 (domain not enabled or not - * prepared for Extract) we - * automatically retry once with Autoparse unless the caller opted into - * `--autoparse` (or another explicit method) or disabled the fallback. + * Default method is `extract`. REQS007 (or the legacy AUTH010: Extract has not + * prepared the domain yet) is reported with the prepare step, not retried. + * Autoparse is deprecated and runs only when asked for: `--autoparse`, or + * `fallbackAutoparse: true`. */ import type { Policy, ToolkitConfig } from "../types/index.ts"; import { ToolkitError } from "../core/errors.ts"; @@ -28,8 +28,9 @@ export interface ExtractOptions extends Omit [flags]` → Extract (structured extraction on `/v1/`). * - * Default: `extract=auto` (site-tailored Extract). If the domain is - * not enabled or prepared (AUTH010 / REQS007), falls back once to Autoparse. Opt into Autoparse - * directly with `--autoparse`. Other methods: `--css`, `--outputs`, `--output`. + * Default: `extract=auto` (site-tailored Extract). If Extract has not prepared + * the domain yet (REQS007 / AUTH010), the error names the prepare step; nothing + * is retried. `--autoparse` runs the deprecated Autoparse explicitly. Other + * methods: `--css`, `--outputs`, `--output`. */ import { ensureApiKey } from "../../core/ensure-key.ts"; import { maybeNudgeClaim } from "../../core/nudge.ts"; @@ -24,8 +25,9 @@ export const extract: Command = { usage: "zenrows extract [--autoparse | --css | --outputs | --output md|text] [flags]", help: [ "Methods:", - " (default) extract=auto: site-tailored Extract; falls back to Autoparse if the domain is not enabled or prepared", - " --autoparse general-purpose Autoparse (any domain, no Extract fallback)", + " (default) extract=auto: site-tailored Extract on any prepared domain", + " (not prepared yet? prepare it: POST /v1/extract/prepared-domains)", + " --autoparse deprecated general-purpose Autoparse (any domain)", " --css CSS selector map, e.g. '{\"title\":\"h1\",\"price\":\".price\"}'", " --outputs built-in output filters → JSON: emails, phone_numbers, headings,", " images, audios, videos, links, menus, hashtags, metadata, tables,", @@ -122,6 +124,9 @@ export const extract: Command = { const runId = newRunId(); const startedAt = new Date().toISOString(); log.step(`Extract ${url} (method=${opts.method ?? "extract"})…`); + if (opts.method === "autoparse") { + log.warn("Autoparse is deprecated. Prefer the default extract=auto on a prepared domain, or --css."); + } try { const outcome = await runExtract(opts, config, policy, apiKey); diff --git a/src/core/http.ts b/src/core/http.ts index 8619ccb..6be00cd 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -145,18 +145,10 @@ export async function scrape( redactedUrl: redacted, }; - // REQS007 on extract=auto: the domain hasn't been prepared for Extract yet. - // Same recovery as AUTH010 (the extract adapter falls back to autoparse), so it - // maps to the same code. Checked before the 403 auth branch below. + // REQS007 on extract=auto: Extract has not prepared the domain yet. Reported, + // never retried: the fix is to prepare the domain, which the error names. if (res.status === 403 && params.extract !== undefined && zrErrorCode(body) === "REQS007") { - throw new ToolkitError({ - code: "EXTRACT_DOMAIN_NOT_ENABLED", - message: "Extract has not been prepared for this domain yet.", - likely_cause: zrErrorDetail(body) ?? "This domain is not prepared for Extract.", - next_action: - "Retry with --autoparse for general-purpose extraction on any site, or get the domain prepared for Extract (POST /v1/extract/prepared-domains).", - suggested_commands: [`zenrows extract ${params.url} --autoparse`], - }); + throw extractDomainNotPrepared(params.url, zrErrorDetail(body)); } // 422 no_data_extracted on an Extract request: the page was processed and had // nothing to extract. It is an answer, not a failure to reach the target, so @@ -214,18 +206,10 @@ export async function scrape( } } if (res.status === 402 && isZenrowsErrorEnvelope(body)) { - // AUTH010 on extract=auto means the target domain is not enabled for - // Extract, distinct from credit exhaustion (AUTH004). Let the extract - // adapter fall back to autoparse. + // AUTH010 on extract=auto is the legacy code for REQS007: the domain is + // not prepared for Extract. Distinct from credit exhaustion (AUTH004). if (params.extract !== undefined && zrErrorCode(body) === "AUTH010") { - throw new ToolkitError({ - code: "EXTRACT_DOMAIN_NOT_ENABLED", - message: "Extract is not enabled for this domain yet.", - likely_cause: zrErrorDetail(body) ?? "This domain is not enabled for Extract.", - next_action: - "Retry with --autoparse for general-purpose extraction on any site, or get the domain prepared for Extract (POST /v1/extract/prepared-domains).", - suggested_commands: [`zenrows extract ${params.url} --autoparse`], - }); + throw extractDomainNotPrepared(params.url, zrErrorDetail(body)); } // AUTH014: this key hit one of its credit caps; the account still has credits. if (isKeyCapReached(zrErrorCode(body) ?? undefined)) { @@ -322,6 +306,33 @@ export async function scrape( * the operator to check connectivity would send them at the wrong problem. The * elapsed time is included so the 90s server budget is visible in the output. */ +/** How to prepare a domain for Extract, shared by every surface that reports REQS007. */ +export const PREPARE_DOMAIN_STEP = + "Prepare it with POST /v1/extract/prepared-domains (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse."; + +/** + * REQS007 (or the legacy AUTH010) on extract=auto: Extract has not prepared the + * URL's domain yet. The message names the next step, because agents copy it. + */ +export function extractDomainNotPrepared(url: string, detail: string | null): ToolkitError { + const domain = hostnameOf(url); + return new ToolkitError({ + code: "EXTRACT_DOMAIN_NOT_ENABLED", + message: `Extract hasn't prepared ${domain} yet. ${PREPARE_DOMAIN_STEP}`, + likely_cause: detail ?? `Extract has not prepared ${domain} yet.`, + next_action: `${PREPARE_DOMAIN_STEP.replace("Prepare it", `Prepare ${domain}`)} The body is {"domain":"${domain}","example_urls":["${url}"]} (1 to 5 URLs) with your key in the X-API-Key header; poll GET /v1/extract/prepared-domains/${domain} until status is ready.`, + suggested_commands: [`zenrows extract ${url} --autoparse`], + }); +} + +function hostnameOf(url: string): string { + try { + return new URL(url).hostname.replace(/^www\./, ""); + } catch { + return url; + } +} + export function requestTimeout(url: string, timeoutMs: number, elapsedMs: number): ToolkitError { const suggestedMs = Math.max(timeoutMs + 60_000, SERVER_BUDGET_MS + 60_000); const atBudget = elapsedMs >= SERVER_BUDGET_MS; diff --git a/templates/extract-pricing-pages/README.md b/templates/extract-pricing-pages/README.md index 81e4001..59814bd 100644 --- a/templates/extract-pricing-pages/README.md +++ b/templates/extract-pricing-pages/README.md @@ -3,13 +3,11 @@ Extract structured pricing fields from product pages with Zenrows Extract. ```bash -# Default: extract=auto on an enabled domain (site-tailored JSON) +# Default: extract=auto on a prepared domain (site-tailored JSON). +# Not prepared yet? Prepare it: POST /v1/extract/prepared-domains (see the extract skill). zenrows extract https://www.owler.com/company/meltwater -# Autoparse — general-purpose JSON on any domain -zenrows extract https://www.scrapingcourse.com/ecommerce/ --autoparse - -# Or a precise CSS selector map +# A precise CSS selector map, on any domain zenrows extract --css '{"title":"h1","price":".price"}' --validate ``` diff --git a/tests/capabilities.test.ts b/tests/capabilities.test.ts index 1ea23fd..03970e3 100644 --- a/tests/capabilities.test.ts +++ b/tests/capabilities.test.ts @@ -13,6 +13,13 @@ test("capability matrix loads with the honest classifications", () => { assert.equal(caps.crawl?.status, "available"); }); +test("extract notes name the prepare step and no automatic autoparse fallback", () => { + const notes = loadCapabilities().extract?.notes ?? ""; + assert.match(notes, /POST \/v1\/extract\/prepared-domains/); + assert.match(notes, /autoparse is deprecated/); + assert.doesNotMatch(notes, /falls? back/i); +}); + test("isUsable true for available + open beta, false for absent capabilities", () => { assert.equal(isUsable("protected_fetch"), true); assert.equal(isUsable("extract"), true); diff --git a/tests/extract-outputs.test.ts b/tests/extract-outputs.test.ts index 2a4158e..15bc6d7 100644 --- a/tests/extract-outputs.test.ts +++ b/tests/extract-outputs.test.ts @@ -2,7 +2,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { normalizeOutputs } from "../src/cli/commands/extract.ts"; import { buildParams, type FetchOptions } from "../src/adapters/protected-fetch.ts"; -import { runExtract } from "../src/adapters/extract.ts"; +import { isEmptyData, runExtract } from "../src/adapters/extract.ts"; import { defaultConfig } from "../src/core/config.ts"; import { defaultPolicy } from "../src/core/policy.ts"; import { ToolkitError } from "../src/core/errors.ts"; @@ -174,36 +174,48 @@ test("runExtract --autoparse skips extract and does not unwrap envelope", async }); }); -test("runExtract falls back to autoparse on AUTH010 for extract=auto", async () => { +test("runExtract reports REQS007 with the prepare step and does not retry", async () => { + const REQS007 = JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for x.com yet.", status: 403 }); const calls: string[] = []; await withFetchHandler((url) => { calls.push(url); - if (url.includes("extract=auto")) { - return new Response(AUTH010, { status: 402, headers: { "content-type": "application/json" } }); - } - return new Response('{"title":"fallback"}', { - status: 200, - headers: { - "content-type": "application/json", - "x-request-cost": "0.0003625", - "x-request-credits": "1", + return new Response(REQS007, { status: 403, headers: { "content-type": "application/json" } }); + }, async () => { + await assert.rejects( + () => runExtract({ url: "https://www.x.com/item/1" }, cfg, defaultPolicy(), "testkey"), + (e: unknown) => { + assert.ok(e instanceof ToolkitError); + assert.equal(e.code, "EXTRACT_DOMAIN_NOT_ENABLED"); + assert.equal( + e.message, + "Extract hasn't prepared x.com yet. Prepare it with POST /v1/extract/prepared-domains (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse.", + ); + assert.match(e.toJSON().next_action, /"example_urls":\["https:\/\/www\.x\.com\/item\/1"\]/); + return true; }, - }); + ); + assert.equal(calls.length, 1, "REQS007 must not be retried with autoparse"); + }); +}); + +test("runExtract reports legacy AUTH010 like REQS007 and does not retry", async () => { + const calls: string[] = []; + await withFetchHandler((url) => { + calls.push(url); + return new Response(AUTH010, { status: 402, headers: { "content-type": "application/json" } }); }, async () => { - const outcome = await runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"); - assert.equal(outcome.method, "autoparse"); - assert.equal(outcome.fellBackToAutoparse, true); - assert.equal(outcome.params.autoparse, true); - assert.deepEqual(outcome.data, { title: "fallback" }); - assert.equal(outcome.result.costCredits, 1); - assert.equal(calls.length, 2); - assert.match(calls[0]!, /extract=auto/); - assert.match(calls[1]!, /autoparse=true/); - assert.doesNotMatch(calls[1]!, /extract=auto/); + await assert.rejects( + () => runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"), + (e: unknown) => + e instanceof ToolkitError && + e.code === "EXTRACT_DOMAIN_NOT_ENABLED" && + e.message.startsWith("Extract hasn't prepared x.com yet. Prepare it with POST /v1/extract/prepared-domains"), + ); + assert.equal(calls.length, 1, "AUTH010 must not be retried with autoparse"); }); }); -test("runExtract falls back to autoparse on REQS007 (domain not prepared) for extract=auto", async () => { +test("runExtract retries with autoparse on REQS007 only when fallbackAutoparse is true", async () => { const REQS007 = JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for x.com yet.", status: 403 }); const calls: string[] = []; await withFetchHandler((url) => { @@ -211,31 +223,22 @@ test("runExtract falls back to autoparse on REQS007 (domain not prepared) for ex if (url.includes("extract=auto")) { return new Response(REQS007, { status: 403, headers: { "content-type": "application/json" } }); } - return new Response('{"title":"fallback"}', { status: 200, headers: { "content-type": "application/json" } }); + return new Response('{"title":"fallback"}', { + status: 200, + headers: { "content-type": "application/json", "x-request-credits": "1" }, + }); }, async () => { - const outcome = await runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"); + const outcome = await runExtract({ url: "https://x.com", fallbackAutoparse: true }, cfg, defaultPolicy(), "testkey"); assert.equal(outcome.method, "autoparse"); assert.equal(outcome.fellBackToAutoparse, true); + assert.deepEqual(outcome.data, { title: "fallback" }); assert.equal(calls.length, 2); + assert.match(calls[0]!, /extract=auto/); assert.match(calls[1]!, /autoparse=true/); + assert.doesNotMatch(calls[1]!, /extract=auto/); }); }); -test("runExtract does not fall back when fallbackAutoparse is false", async () => { - await withFetch(AUTH010, async () => { - await assert.rejects( - () => - runExtract( - { url: "https://x.com", method: "extract", fallbackAutoparse: false }, - cfg, - defaultPolicy(), - "testkey", - ), - (e: unknown) => e instanceof ToolkitError && e.code === "EXTRACT_DOMAIN_NOT_ENABLED", - ); - }, { status: 402 }); -}); - test("runExtract does not fall back on AUTH004 (credits exhausted)", async () => { let calls = 0; await withFetchHandler(() => { @@ -250,7 +253,7 @@ test("runExtract does not fall back on AUTH004 (credits exhausted)", async () => }); }); -test("runExtract surfaces autoparse failure after AUTH010 fallback", async () => { +test("runExtract surfaces autoparse failure after an opted-in AUTH010 fallback", async () => { await withFetchHandler((url) => { if (url.includes("extract=auto")) { return new Response(AUTH010, { status: 402, headers: { "content-type": "application/json" } }); @@ -258,7 +261,7 @@ test("runExtract surfaces autoparse failure after AUTH010 fallback", async () => return new Response("upstream boom", { status: 500, headers: { "content-type": "text/plain" } }); }, async () => { await assert.rejects( - () => runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"), + () => runExtract({ url: "https://x.com", fallbackAutoparse: true }, cfg, defaultPolicy(), "testkey"), (e: unknown) => e instanceof ToolkitError && e.code === "FETCH_FAILED", ); }); @@ -286,7 +289,7 @@ test("runExtract --css without a selector map fails locally (no network)", async ); }); -// --- ACT-1514: empty extraction must not be a silent success ---------------- +// --- empty extraction must not be a silent success -------------------------- test("runExtract flags an empty extract=auto result (empty=true)", async () => { await withFetch(JSON.stringify({ parsed: {}, html: "" }), async () => { const outcome = await runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"); @@ -308,6 +311,27 @@ test("runExtract flags an empty autoparse result (empty=true)", async () => { }); }); +test("runExtract flags a result whose fields are all null or empty (empty=true)", async () => { + const scaffolding = { + title: null, + author: { name: "", url: null }, + tags: [], + comments: [{ body: null, date: " " }], + }; + await withFetch(JSON.stringify({ parsed: scaffolding, html: "" }), async () => { + const outcome = await runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"); + assert.deepEqual(outcome.data, scaffolding); + assert.equal(outcome.empty, true); + }); +}); + +test("isEmptyData keeps 0, false and one filled nested field as data", () => { + assert.equal(isEmptyData({ count: 0 }), false); + assert.equal(isEmptyData({ in_stock: false }), false); + assert.equal(isEmptyData({ title: null, items: [{ name: null }, { name: "A" }] }), false); + assert.equal(isEmptyData([[], {}, null, ""]), true); +}); + test("runExtract does not flag a non-empty result", async () => { await withFetch(JSON.stringify({ parsed: { title: "X" }, html: "" }), async () => { const outcome = await runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"); diff --git a/tests/http.test.ts b/tests/http.test.ts index c40bcdf..7bfea54 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -179,6 +179,10 @@ test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async () => scrape("https://api.zenrows.com/v1/", "test-key", { url: "https://example.net", extract: "auto" }), (err: unknown) => { assert.equal((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_ENABLED"); + assert.equal( + (err as { message: string }).message, + "Extract hasn't prepared example.net yet. Prepare it with POST /v1/extract/prepared-domains (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse.", + ); return true; }, ); diff --git a/workflows/competitor-intelligence/WORKFLOW.md b/workflows/competitor-intelligence/WORKFLOW.md index d0618c0..5bdb05f 100644 --- a/workflows/competitor-intelligence/WORKFLOW.md +++ b/workflows/competitor-intelligence/WORKFLOW.md @@ -9,7 +9,7 @@ status: available A higher-level process built on Zenrows **Fetch and Extract**: retrieve protected pages with `zenrows fetch`, then turn them into structured data with -`zenrows extract` (the same API's Autoparse / CSS / Markdown — not a separate +`zenrows extract` (the same API's `extract=auto` / CSS / Markdown — not a separate product). Scheduling and scaling use the tools you already have (cron/CI, a simple per-URL loop). @@ -20,7 +20,7 @@ simple per-URL loop). ``` zenrows fetch https://competitor.example/products ``` -2. **Validate extraction** on one page (extract=auto falls back to autoparse on a domain not prepared yet). +2. **Validate extraction** on one page (on a domain not prepared yet, prepare it first: see the extract skill). ``` zenrows extract https://competitor.example/p/123 --validate ``` From 80a3c6afc858102f253d44a25bacade9bd2e2d38 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 09:39:27 +0000 Subject: [PATCH 2/8] feat(extract): add zenrows extract prepare to prepare a domain Removing the autoparse fallback leaves REQS007 as the answer on a domain Extract has not prepared yet, so CLI users need a way to prepare one. - `zenrows extract prepare --url [--url ...]` submits the domain with 1 to 5 example URLs to POST /v1/extract/prepared-domains and returns at once. `--wait` (off by default) polls until ready or failed, up to `--timeout`. On 409 in_progress it follows the running preparation. - `zenrows extract prepare status ` shows progress, the failure code and detail, the account's monthly allowance and the next step. - Errors name the next step: EXTRACT_PREPARE_INVALID (400), EXTRACT_PREPARE_NOT_FOUND (404), EXTRACT_PREPARE_IN_PROGRESS (409), EXTRACT_PREPARE_OUT_OF_ALLOWANCE (429), EXTRACT_PREPARE_UNAVAILABLE (503 overloaded / disabled), EXTRACT_PREPARE_FAILED (failed status, with the step for its failure code), EXTRACT_PREPARE_TIMEOUT. - The REQS007 error now prints the exact command: zenrows extract prepare --url . Skills, README and the capability matrix teach the flow: extract, REQS007, prepare, poll, extract again. --- README.md | 12 +- agent-plugin/README.md | 2 +- agent-plugin/skills/extract/SKILL.md | 25 +- agent-plugin/skills/zenrows/SKILL.md | 1 + docs/capabilities.md | 2 +- registry/capabilities.json | 2 +- skills/extract/SKILL.md | 27 +- skills/zenrows/SKILL.md | 3 +- src/cli/commands/extract-prepare.ts | 189 ++++++++++++ src/cli/commands/extract.ts | 14 +- src/core/errors.ts | 7 + src/core/http.ts | 25 +- src/core/prepared-domains-api.ts | 292 ++++++++++++++++++ templates/extract-pricing-pages/README.md | 2 +- tests/capabilities.test.ts | 2 + tests/extract-outputs.test.ts | 10 +- tests/extract-prepare.test.ts | 246 +++++++++++++++ tests/http.test.ts | 2 +- workflows/competitor-intelligence/WORKFLOW.md | 2 +- 19 files changed, 815 insertions(+), 50 deletions(-) create mode 100644 src/cli/commands/extract-prepare.ts create mode 100644 src/core/prepared-domains-api.ts create mode 100644 tests/extract-prepare.test.ts diff --git a/README.md b/README.md index a0d2412..3674564 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ that endpoint with extraction parameters, not a separate product. | Command | What it does | Status (this build) | | --- | --- | --- | | `zenrows fetch` | Fetch: retrieve a protected page | **available**: `GET https://api.zenrows.com/v1/` | -| `zenrows extract` | Extract: turn a page into structured data with `extract=auto` on any prepared domain (you can prepare one yourself), CSS selectors, or Markdown | **available**: `GET https://api.zenrows.com/v1/?extract=auto` | +| `zenrows extract` | Extract: turn a page into structured data with `extract=auto` on any prepared domain (prepare one yourself with `zenrows extract prepare`), CSS selectors, or Markdown | **available**: `GET https://api.zenrows.com/v1/?extract=auto` | | `zenrows batch` | Batch: run Fetch and Extract over many URLs in one job; specs validate and estimate locally with no key | **available**: `POST https://async.api.zenrows.com/v1/jobs` | | `zenrows crawl` | Crawl (new): collect the URLs behind one start page, optionally with each page's HTML | **available**: `POST https://api.zenrows.com/v1/crawls` | | `zenrows browser` | Browser Sessions: a real cloud browser over REST (same backend as MCP `browser_*`); escalation-only, billed by bandwidth + time | **available**: `POST https://mcp.zenrows.com/browser/sessions` | @@ -123,6 +123,16 @@ zenrows extract --outputs emails,links # built- zenrows extract --output markdown ``` +On a domain Extract hasn't prepared yet, `zenrows extract` fails with REQS007 +and prints the command to prepare it. Preparing takes a few minutes and costs +no credits; each new domain uses one slot of the monthly allowance. + +```bash +zenrows extract prepare example.com --url https://example.com/p/1 --url https://example.com/list --wait +zenrows extract prepare status example.com # progress + monthly allowance +zenrows extract https://example.com/p/1 # then extract again +``` + ## 9. Batch Zenrows **Batch** (`https://async.api.zenrows.com/v1`) fans a diff --git a/agent-plugin/README.md b/agent-plugin/README.md index 52e1da7..0d96adc 100644 --- a/agent-plugin/README.md +++ b/agent-plugin/README.md @@ -62,7 +62,7 @@ Use MCP tool names — not CLI `zenrows fetch`: | --- | --- | | `zenrows` | Decision tree across all primitives | | `protected-fetch` | `scrape` | -| `extract` | `extract` (fallback: `scrape` + css_extractor) | +| `extract` | `extract` (fallback: `scrape` + css_extractor); on REQS007 `prepare_domain` then `prepared_domain_status` | | `batch-jobs` | `batch_create`, `batch_status`, `batch_results`, `batch_cancel` | | `interact-browser` | `browser_*` (escalation only) | | `cost-control` | Cost multipliers + escalate-with-evidence | diff --git a/agent-plugin/skills/extract/SKILL.md b/agent-plugin/skills/extract/SKILL.md index 3ec4ce3..76b9019 100644 --- a/agent-plugin/skills/extract/SKILL.md +++ b/agent-plugin/skills/extract/SKILL.md @@ -31,22 +31,29 @@ protected or dynamic sites. Set `js_render` or `premium_proxy` only to force a fixed configuration; that turns Adaptive Stealth Mode off. Check `empty` in the result: true means no field came back with a value. -## Domain not prepared yet (REQS007) +## Domain not prepared yet (REQS007): prepare it, then extract again `mode=auto` works on any prepared domain, and you can prepare one yourself. On a domain Extract hasn't prepared yet, `extract` returns a tool error (`REQS007`; -legacy `AUTH010`) that names the next step. It does not retry with Autoparse -unless you set `fallback_autoparse: true`. To prepare the domain: +legacy `AUTH010`) whose `next_tool` holds the `prepare_domain` arguments. It does +not retry with Autoparse unless you set `fallback_autoparse: true`. The flow: ``` -POST https://api.zenrows.com/v1/extract/prepared-domains -X-API-Key: -{"domain":"example.com","example_urls":["https://example.com/p/1","https://example.com/p/2"]} +extract({ url }) # 1. error REQS007 +prepare_domain({ domain: "example.com", + example_urls: ["https://example.com/p/1", "https://example.com/list"] }) + # 2. returns at once, status "preparing" +prepared_domain_status({ domain: "example.com" }) # 3. every ~30 s until status "ready" +extract({ url }) # 4. extract again ``` -Give 1 to 5 example URLs of the pages you want. Preparation takes a few -minutes and uses no credits. Poll `GET /v1/extract/prepared-domains/{domain}` -until `status` is `ready` (or `failed`), then call `extract` again. +- `example_urls`: 1 to 5 real pages you want to extract, one per kind of page. +- Preparing takes a few minutes and costs no credits. Each new domain uses one + slot of the account's monthly allowance; `prepared_domain_status` returns it + (`used` / `limit` / `remaining`) and a `next` field with what to do now. +- `out_of_allowance`: no slots left this month; use `mode=css`. + `overloaded` / `disabled`: retry in a few minutes, do not loop. + A `failed` status carries `failure.code`; follow `next`. ## Fallback: `scrape` extraction params diff --git a/agent-plugin/skills/zenrows/SKILL.md b/agent-plugin/skills/zenrows/SKILL.md index 8efaa9c..2082d5c 100644 --- a/agent-plugin/skills/zenrows/SKILL.md +++ b/agent-plugin/skills/zenrows/SKILL.md @@ -67,6 +67,7 @@ If the user needs login, clicks, forms, sessions, or persistent state: | --- | --- | --- | | Protected Fetch | `scrape` | Default markdown; HTML / plaintext / PDF / screenshot options | | Extract | `extract` | Structured JSON (`extract=auto` on any prepared domain, css_extractor; autoparse deprecated). `extract=auto` is currently free; billing may apply later | +| Prepare a domain | `prepare_domain`, `prepared_domain_status` | After an `extract` REQS007: prepare the domain with 1 to 5 example URLs, poll until ready, extract again. No credits; uses the monthly allowance | | Batch | `batch_create`, `batch_status`, `batch_results`, `batch_cancel` | Same credits as Fetch; a 403 maps to `BATCH_ACCESS_DENIED` | | Browser | `browser_*` (start with `browser_navigate`) | Escalation-only; same backend as Browser Sessions | diff --git a/docs/capabilities.md b/docs/capabilities.md index 416ef96..62b1476 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -18,7 +18,7 @@ Classification is based on the public Zenrows documentation: | Capability | Backend evidence | Status | | --- | --- | --- | | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | -| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (any prepared domain; on one not prepared yet the CLI reports REQS007 with the prepare step, `POST /v1/extract/prepared-domains`), plus deprecated `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | available | +| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (any prepared domain; on one not prepared yet the CLI reports REQS007 with the command to prepare it: `zenrows extract prepare --url `, which calls `POST /v1/extract/prepared-domains`), plus deprecated `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | available | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header). Local JSONL spec validation + credit estimation work with no key. | available | | `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL, up to `depth` when set (optional; no depth limit when omitted), on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Subcommands: `create`, `get`, `list`, `results`, `content`, `download`, `stop`, `wait`. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | diff --git a/registry/capabilities.json b/registry/capabilities.json index 6558452..9596ba4 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -17,7 +17,7 @@ "command": "zenrows extract", "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, css_extractor, outputs, response_type; autoparse deprecated)", "requiresAuth": true, - "notes": "Default is extract=auto, which works on any prepared domain with any key. On a domain not prepared yet (REQS007, legacy AUTH010) the CLI reports the error and does not retry; prepare the domain with POST /v1/extract/prepared-domains (a few example URLs, a few minutes, no credits). CSS / outputs / markdown work on any domain; autoparse is deprecated and runs only with --autoparse." + "notes": "Default is extract=auto, which works on any prepared domain with any key. On a domain not prepared yet (REQS007, legacy AUTH010) the CLI reports the error and does not retry; prepare the domain with `zenrows extract prepare --url [--wait]` (POST /v1/extract/prepared-domains: 1 to 5 example URLs, a few minutes, no credits) and check it with `zenrows extract prepare status `. CSS / outputs / markdown work on any domain; autoparse is deprecated and runs only with --autoparse." }, "batch": { "key": "batch", diff --git a/skills/extract/SKILL.md b/skills/extract/SKILL.md index f583a1d..c1bf980 100644 --- a/skills/extract/SKILL.md +++ b/skills/extract/SKILL.md @@ -28,22 +28,29 @@ zenrows extract --validate # fail if not valid JSON - **CSS**: you know the exact fields/selectors and want determinism. - **Markdown/plaintext**: feed clean content to an LLM yourself. -## Domain not prepared yet (REQS007) +## Domain not prepared yet (REQS007): prepare it, then extract again `extract=auto` works on any prepared domain, and you can prepare one yourself. -On a domain Extract hasn't prepared yet the API answers `403 REQS007` (legacy: -`402 AUTH010`) and `zenrows extract` reports it; it does not retry with -Autoparse. To prepare the domain: +On a domain Extract hasn't prepared yet, `zenrows extract` fails with +`EXTRACT_DOMAIN_NOT_ENABLED` (API code `403 REQS007`; legacy `402 AUTH010`) and +prints the command to run. It does not retry with Autoparse. The flow: ``` -POST https://api.zenrows.com/v1/extract/prepared-domains -X-API-Key: -{"domain":"example.com","example_urls":["https://example.com/p/1","https://example.com/p/2"]} +zenrows extract https://example.com/p/1 # 1. fails: Extract hasn't prepared example.com yet +zenrows extract prepare example.com \ + --url https://example.com/p/1 --url https://example.com/list --wait + # 2. prepare it and wait until ready (a few minutes) +zenrows extract https://example.com/p/1 # 3. extract again ``` -Give 1 to 5 example URLs of the pages you want. Preparation takes a few -minutes and uses no credits. Poll `GET /v1/extract/prepared-domains/{domain}` -until `status` is `ready` (or `failed`), then retry the extract. +- `--url`: 1 to 5 real pages you want to extract, one per kind of page. +- Without `--wait` the command returns at once; then poll with + `zenrows extract prepare status example.com` until it says `ready`. +- Preparing costs no credits. Each new domain uses one slot of the account's + monthly allowance; `prepare status` shows `used` / `limit` / `remaining`. +- `EXTRACT_PREPARE_OUT_OF_ALLOWANCE`: no slots left this month; use `--css`. + `EXTRACT_PREPARE_UNAVAILABLE`: retry in a few minutes, do not loop. + A `failed` status prints the next step for its failure code. ## Rules - Validate on a single page before scaling across many URLs. diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index 70a2ab0..0e6ca80 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -79,7 +79,8 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | MCP | `zenrows mcp` | **available**: `https://mcp.zenrows.com/mcp` | - Extract: `extract=auto` works on any prepared domain. On one not prepared yet - (REQS007), prepare it with `POST /v1/extract/prepared-domains`; see [[extract]]. + (REQS007), run `zenrows extract prepare --url --wait`, then + extract again; see [[extract]]. - Batch: the cheapest path at scale (create, status, results). Estimate specs locally with no key. - Crawl: collects the URLs behind one start page, optionally with each page's HTML. - Browser: escalation-only, for pages Fetch can't handle; same backend as the MCP `browser_*` tools. diff --git a/src/cli/commands/extract-prepare.ts b/src/cli/commands/extract-prepare.ts new file mode 100644 index 0000000..38e0554 --- /dev/null +++ b/src/cli/commands/extract-prepare.ts @@ -0,0 +1,189 @@ +/** + * `zenrows extract prepare --url [--url ...] [--wait]` and + * `zenrows extract prepare status `. + * + * The way out of REQS007: prepare the domain so `extract=auto` works on it. + * Submitting returns at once; `--wait` polls until the preparation is ready or + * failed (a few minutes). Costs no credits; each new domain uses one slot of the + * account's monthly allowance. + */ +import { ensureApiKey } from "../../core/ensure-key.ts"; +import { loadConfig } from "../../core/config.ts"; +import { loadPolicy } from "../../core/policy.ts"; +import { log } from "../../core/logger.ts"; +import { ToolkitError } from "../../core/errors.ts"; +import { + DEFAULT_WAIT_TIMEOUT_MS, + MAX_EXAMPLE_URLS, + failureNextStep, + getAllowance, + getPreparedDomain, + normalizeDomain, + prepareDomain, + statusNextStep, + waitForPreparedDomain, + type Allowance, + type PreparedDomain, + type PrepareRequestOpts, +} from "../../core/prepared-domains-api.ts"; +import { parse, asString, type RunContext } from "../command.ts"; +import { printError } from "../output.ts"; + +export const PREPARE_USAGE = + "zenrows extract prepare --url [--url ...] [--wait [--timeout ]]\n" + + " zenrows extract prepare status "; + +export const PREPARE_HELP = [ + "Prepare a domain so `zenrows extract ` (extract=auto) works on it. Use it after", + "an extract fails with REQS007 (Extract hasn't prepared the domain yet).", + "", + ` --url 1 to ${MAX_EXAMPLE_URLS} real pages you want to extract, one per kind of page (repeat --url)`, + " --wait poll until the domain is ready or failed (a few minutes); off by default", + ` --timeout give up waiting after this long (default ${DEFAULT_WAIT_TIMEOUT_MS / 1000})`, + " status show progress and the account's monthly allowance", + " --json structured result", + "", + "Costs no credits. Each new domain uses one slot of the account's monthly allowance.", +].join("\n"); + +/** Injectable for tests: network, sleep and clock. */ +export interface PrepareDeps { + fetchImpl?: typeof fetch; + sleepImpl?: (ms: number) => Promise; + now?: () => number; + intervalMs?: number; + apiKey?: string; +} + +export async function runPrepare(argv: string[], ctx: RunContext, deps: PrepareDeps = {}): Promise { + const isStatus = argv[0] === "status"; + const { values, positionals } = parse(isStatus ? argv.slice(1) : argv, { + url: { type: "string", multiple: true }, + wait: { type: "boolean" }, + timeout: { type: "string" }, + "no-signup": { type: "boolean" }, + json: { type: "boolean" }, + help: { type: "boolean" }, + }); + const json = ctx.json || values.json === true; + try { + const raw = positionals[0]; + if (!raw) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: "Missing domain.", + likely_cause: "No was given.", + next_action: `Usage: ${PREPARE_USAGE}`, + suggested_commands: ["zenrows extract prepare example.com --url https://example.com/products/1"], + }); + } + const domain = normalizeDomain(raw); + const urls = isStatus ? [] : validateUrls(values.url, domain); + const timeoutMs = parseTimeout(asString(values.timeout)); + + const policy = loadPolicy(); + const apiKey = + deps.apiKey ?? + (await ensureApiKey(isStatus || values["no-signup"] ? { ...policy, auto_signup: false } : policy, { + onProvision: (a) => { + log.info("No API key found — created a Zenrows Free plan account for you."); + log.dim(`Claim it anytime (keeps your usage): ${a.claimUrl}`); + }, + })); + const opts: PrepareRequestOpts = { apiKey, apiBase: loadConfig().apiBase, fetchImpl: deps.fetchImpl }; + + if (isStatus) { + const [record, allowance] = await Promise.all([getPreparedDomain(domain, opts), getAllowance(opts).catch(() => undefined)]); + report(record, allowance, json); + return 0; + } + + log.step(`Prepare ${domain} for Extract (${urls.length} example URL${urls.length === 1 ? "" : "s"})…`); + let record: PreparedDomain = await prepareDomain(domain, urls, opts); + if (record.already_in_progress) log.info(`${domain} was already being prepared; following that preparation.`); + if (values.wait === true && record.status !== "ready" && record.status !== "failed") { + record = await waitForPreparedDomain(domain, { + ...opts, + timeoutMs, + intervalMs: deps.intervalMs, + sleepImpl: deps.sleepImpl, + now: deps.now, + onPoll: (r) => log.dim(` ${r.domain}: ${r.status} (${r.state ?? r.status})`), + }); + } + if (record.status === "failed") { + throw new ToolkitError({ + code: "EXTRACT_PREPARE_FAILED", + message: `Preparing ${domain} failed${record.failure?.code ? ` (${record.failure.code})` : ""}.`, + likely_cause: record.failure?.detail ?? "The preparation did not finish.", + next_action: failureNextStep(domain, record.failure?.code), + server_code: record.failure?.code, + }); + } + report(record, undefined, json, urls[0]); + return 0; + } catch (err) { + printError(err, json); + return 1; + } +} + +function validateUrls(raw: unknown, domain: string): string[] { + const urls = (Array.isArray(raw) ? raw : raw === undefined ? [] : [raw]).map(String); + if (urls.length < 1 || urls.length > MAX_EXAMPLE_URLS) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Pass 1 to ${MAX_EXAMPLE_URLS} example page URLs with --url (got ${urls.length}).`, + likely_cause: "Preparation learns the domain from real pages you want to extract.", + next_action: "Add one --url per kind of page, e.g. a product page and a listing page.", + suggested_commands: [`zenrows extract prepare ${domain} --url https://${domain}/`], + }); + } + for (const u of urls) { + let parsed: URL | undefined; + try { + parsed = new URL(u); + } catch { + // reported below + } + if (!parsed || (parsed.protocol !== "http:" && parsed.protocol !== "https:")) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `--url ${u} is not a full http(s) URL.`, + likely_cause: "Each --url must be a page URL including its scheme.", + next_action: `Use full URLs, e.g. --url https://${domain}/.`, + }); + } + } + return urls; +} + +function parseTimeout(v?: string): number { + if (v === undefined) return DEFAULT_WAIT_TIMEOUT_MS; + const n = Number(v); + if (!Number.isFinite(n) || n <= 0) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `--timeout must be a positive number of seconds (got ${v}).`, + likely_cause: "The value is not a number.", + next_action: "Pass seconds, e.g. --timeout 600.", + }); + } + return n * 1000; +} + +function report(record: PreparedDomain, allowance: Allowance | undefined, json: boolean, exampleUrl?: string): void { + const next = statusNextStep(record, exampleUrl); + if (json) { + log.out(JSON.stringify({ ok: true, ...record, ...(allowance ? { allowance } : {}), next }, null, 2)); + return; + } + const line = `${record.domain}: ${record.status}${record.state && record.state !== record.status ? ` (${record.state})` : ""}`; + if (record.status === "ready") log.success(line); + else if (record.status === "failed") log.warn(line); + else log.info(line); + if (record.failure) log.dim(` failure: ${record.failure.code}${record.failure.detail ? ` — ${record.failure.detail}` : ""}`); + for (const role of record.roles ?? []) log.dim(` ${role.role}: ${role.status}${role.url ? ` ${role.url}` : ""}`); + if (allowance) log.dim(` allowance: ${allowance.used} of ${allowance.limit} used, ${allowance.remaining} left${allowance.month ? ` (${allowance.month})` : ""}`); + log.out(next); +} diff --git a/src/cli/commands/extract.ts b/src/cli/commands/extract.ts index db972ed..0aa7170 100644 --- a/src/cli/commands/extract.ts +++ b/src/cli/commands/extract.ts @@ -4,7 +4,8 @@ * Default: `extract=auto` (site-tailored Extract). If Extract has not prepared * the domain yet (REQS007 / AUTH010), the error names the prepare step; nothing * is retried. `--autoparse` runs the deprecated Autoparse explicitly. Other - * methods: `--css`, `--outputs`, `--output`. + * methods: `--css`, `--outputs`, `--output`. `zenrows extract prepare` prepares a + * domain (see extract-prepare.ts). */ import { ensureApiKey } from "../../core/ensure-key.ts"; import { maybeNudgeClaim } from "../../core/nudge.ts"; @@ -18,15 +19,18 @@ import { runExtract, type ExtractMethod, type ExtractOptions } from "../../adapt import { formatRequestCost } from "../../core/http.ts"; import { parse, asString, asNumber, type Command, type RunContext } from "../command.ts"; import { printError, writeOut } from "../output.ts"; +import { PREPARE_HELP, PREPARE_USAGE, runPrepare } from "./extract-prepare.ts"; export const extract: Command = { name: "extract", summary: "Turn a protected page into structured data (Extract / Autoparse / CSS / Markdown).", - usage: "zenrows extract [--autoparse | --css | --outputs | --output md|text] [flags]", + usage: + "zenrows extract [--autoparse | --css | --outputs | --output md|text] [flags]\n " + + PREPARE_USAGE, help: [ "Methods:", " (default) extract=auto: site-tailored Extract on any prepared domain", - " (not prepared yet? prepare it: POST /v1/extract/prepared-domains)", + " (REQS007 = not prepared yet: run zenrows extract prepare, below)", " --autoparse deprecated general-purpose Autoparse (any domain)", " --css CSS selector map, e.g. '{\"title\":\"h1\",\"price\":\".price\"}'", " --outputs built-in output filters → JSON: emails, phone_numbers, headings,", @@ -42,8 +46,12 @@ export const extract: Command = { "", "Cost (credits per request): 1x normal · 5x --js-render · 10x --premium-proxy · 25x both.", "The exact charge is reported after every request (costCredits / X-Request-Credits).", + "", + "Prepare a domain:", + PREPARE_HELP.split("\n").map((l) => (l ? " " + l : l)).join("\n"), ].join("\n"), async run(argv: string[], ctx: RunContext): Promise { + if (argv[0] === "prepare") return runPrepare(argv.slice(1), ctx); const { values, positionals } = parse(argv, { autoparse: { type: "boolean" }, css: { type: "string" }, diff --git a/src/core/errors.ts b/src/core/errors.ts index 1c4da84..9b51c50 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -30,6 +30,13 @@ export type ErrorCode = | "EXTRACT_FAILED" | "EXTRACT_DOMAIN_NOT_ENABLED" | "EXTRACT_VALIDATION_FAILED" + | "EXTRACT_PREPARE_INVALID" + | "EXTRACT_PREPARE_NOT_FOUND" + | "EXTRACT_PREPARE_IN_PROGRESS" + | "EXTRACT_PREPARE_OUT_OF_ALLOWANCE" + | "EXTRACT_PREPARE_UNAVAILABLE" + | "EXTRACT_PREPARE_FAILED" + | "EXTRACT_PREPARE_TIMEOUT" | "BROWSER_UNAVAILABLE" | "BATCH_ACCESS_DENIED" | "BATCH_QUOTA_EXCEEDED" diff --git a/src/core/http.ts b/src/core/http.ts index 6be00cd..765d983 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -11,6 +11,7 @@ import { ENV_KEY, resolveApiKey } from "./auth.ts"; import { registerSecret } from "./logger.ts"; import { CLI_USER_AGENT } from "./config.ts"; import { agentClientHeader } from "./agent-client.ts"; +import { normalizeDomain, prepareCommand } from "./prepared-domains-api.ts"; export interface ScraperResult { status: number; @@ -306,33 +307,23 @@ export async function scrape( * the operator to check connectivity would send them at the wrong problem. The * elapsed time is included so the 90s server budget is visible in the output. */ -/** How to prepare a domain for Extract, shared by every surface that reports REQS007. */ -export const PREPARE_DOMAIN_STEP = - "Prepare it with POST /v1/extract/prepared-domains (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse."; - /** * REQS007 (or the legacy AUTH010) on extract=auto: Extract has not prepared the - * URL's domain yet. The message names the next step, because agents copy it. + * URL's domain yet. The message names the exact command to run, because agents + * copy it. */ export function extractDomainNotPrepared(url: string, detail: string | null): ToolkitError { - const domain = hostnameOf(url); + const domain = normalizeDomain(url); + const prepare = prepareCommand(url); return new ToolkitError({ code: "EXTRACT_DOMAIN_NOT_ENABLED", - message: `Extract hasn't prepared ${domain} yet. ${PREPARE_DOMAIN_STEP}`, + message: `Extract hasn't prepared ${domain} yet. Prepare it with \`${prepare}\` (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse.`, likely_cause: detail ?? `Extract has not prepared ${domain} yet.`, - next_action: `${PREPARE_DOMAIN_STEP.replace("Prepare it", `Prepare ${domain}`)} The body is {"domain":"${domain}","example_urls":["${url}"]} (1 to 5 URLs) with your key in the X-API-Key header; poll GET /v1/extract/prepared-domains/${domain} until status is ready.`, - suggested_commands: [`zenrows extract ${url} --autoparse`], + next_action: `Run \`${prepare} --wait\` (add up to 5 --url pages, one per kind of page you want), then retry \`zenrows extract ${url}\`. Without --wait, check progress with \`zenrows extract prepare status ${domain}\`.`, + suggested_commands: [`${prepare} --wait`, `zenrows extract prepare status ${domain}`, `zenrows extract ${url}`], }); } -function hostnameOf(url: string): string { - try { - return new URL(url).hostname.replace(/^www\./, ""); - } catch { - return url; - } -} - export function requestTimeout(url: string, timeoutMs: number, elapsedMs: number): ToolkitError { const suggestedMs = Math.max(timeoutMs + 60_000, SERVER_BUDGET_MS + 60_000); const atBudget = elapsedMs >= SERVER_BUDGET_MS; diff --git a/src/core/prepared-domains-api.ts b/src/core/prepared-domains-api.ts new file mode 100644 index 0000000..ba272f4 --- /dev/null +++ b/src/core/prepared-domains-api.ts @@ -0,0 +1,292 @@ +/** + * Client for Extract's prepared-domains API. + * + * `extract=auto` answers REQS007 on a domain Extract has not prepared yet. Preparing + * it is a call any key can make: `POST {apiBase}/extract/prepared-domains` with the + * domain and 1 to 5 example URLs answers 202 at once; the preparation itself takes a + * few minutes and costs no credits. `GET .../{domain}` reports progress and + * `GET .../allowance` the account's monthly allowance of new domains. Auth is the + * `X-API-Key` header; errors are problem+json and we branch on `status` + `code`. + * `fetchImpl` / `sleepImpl` / `now` are injectable for tests. + */ +import { ToolkitError } from "./errors.ts"; +import { registerSecret } from "./logger.ts"; +import { CLI_USER_AGENT, loadConfig } from "./config.ts"; +import { agentClientHeader } from "./agent-client.ts"; + +export const MAX_EXAMPLE_URLS = 5; +/** How often `--wait` polls, and for how long by default. */ +export const DEFAULT_POLL_INTERVAL_MS = 15_000; +export const DEFAULT_WAIT_TIMEOUT_MS = 15 * 60_000; + +/** status: preparing | ready | failed. state adds the step (received, discovering, preparing, verifying). */ +export interface PreparedDomain { + domain: string; + status: string; + state?: string; + roles?: { role: string; status: string; url?: string; detail?: string }[]; + failure?: { code: string; detail?: string; slot_consumed?: boolean }; + submitted_at?: string; + updated_at?: string; + [k: string]: unknown; +} + +export interface Allowance { + used: number; + limit: number; + remaining: number; + month?: string; + [k: string]: unknown; +} + +export interface PrepareRequestOpts { + apiKey: string; + apiBase?: string; + fetchImpl?: typeof fetch; +} + +/** The domain as the API wants it: host only, no scheme, path or leading `www.`. Accepts a URL. */ +export function normalizeDomain(input: string): string { + const trimmed = input.trim(); + let host = trimmed; + try { + host = new URL(/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`).hostname; + } catch { + // Left as given: the API answers invalid_argument with the reason. + } + return host.toLowerCase().replace(/^www\./, ""); +} + +/** The exact command an agent runs to prepare the domain of `url`. */ +export function prepareCommand(url: string): string { + return `zenrows extract prepare ${normalizeDomain(url)} --url ${url}`; +} + +function base(opts: PrepareRequestOpts): string { + return (opts.apiBase ?? loadConfig().apiBase).replace(/\/+$/, ""); +} + +async function request(method: "GET" | "POST", path: string, domain: string, opts: PrepareRequestOpts & { body?: unknown }): Promise { + registerSecret(opts.apiKey); + const headers: Record = { + "X-API-Key": opts.apiKey, + Accept: "application/json", + "User-Agent": CLI_USER_AGENT, + ...agentClientHeader(), + }; + if (opts.body !== undefined) headers["Content-Type"] = "application/json"; + let res: Response; + try { + res = await (opts.fetchImpl ?? fetch)(`${base(opts)}/extract/prepared-domains${path}`, { + method, + headers, + body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined, + }); + } catch (err) { + throw new ToolkitError({ + code: "BACKEND_UNAVAILABLE", + message: "Could not reach the Zenrows API.", + likely_cause: err instanceof Error ? err.message : String(err), + next_action: "Check connectivity and retry. Verify the api base in `zenrows config show`.", + suggested_commands: ["zenrows status", "zenrows config show"], + }); + } + const text = await res.text(); + if (res.status < 200 || res.status >= 300) throw prepareError(res.status, text, domain); + try { + return JSON.parse(text) as T; + } catch { + throw new ToolkitError({ + code: "EXTRACT_PREPARE_FAILED", + message: "The prepared-domains API returned a non-JSON answer.", + likely_cause: text.slice(0, 200), + next_action: "Retry in a minute.", + }); + } +} + +/** Map a non-2xx prepared-domains answer to an error that names the next step. */ +export function prepareError(status: number, body: string, domain: string): ToolkitError { + let p: Record = {}; + try { + p = (JSON.parse(body) as Record) ?? {}; + } catch { + // not problem+json + } + const code = typeof p.code === "string" ? p.code.toLowerCase() : undefined; + const detail = typeof p.detail === "string" ? p.detail : body.slice(0, 200) || `HTTP ${status}`; + const cause = code ? `(${code}) ${detail}` : detail; + if (code === "invalid_argument" || status === 400) { + return new ToolkitError({ + code: "EXTRACT_PREPARE_INVALID", + message: `The request to prepare ${domain} was rejected.`, + likely_cause: p.reason ? `${cause} [reason: ${String(p.reason)}]` : cause, + next_action: `Pass the domain without scheme or path, and 1 to ${MAX_EXAMPLE_URLS} full page URLs from that site with --url.`, + suggested_commands: [`zenrows extract prepare ${domain} --url https://${domain}/`], + server_code: code, + }); + } + if (code === "not_found" || status === 404) { + return new ToolkitError({ + code: "EXTRACT_PREPARE_NOT_FOUND", + message: `This account has not submitted ${domain} for preparation.`, + likely_cause: cause, + next_action: `Prepare it first: zenrows extract prepare ${domain} --url (1 to ${MAX_EXAMPLE_URLS} --url).`, + suggested_commands: [`zenrows extract prepare ${domain} --url https://${domain}/`], + server_code: code, + }); + } + if (code === "in_progress") { + return new ToolkitError({ + code: "EXTRACT_PREPARE_IN_PROGRESS", + message: `${domain} is already being prepared.`, + likely_cause: cause, + next_action: "Do not resubmit. Retry the extract in a few minutes.", + suggested_commands: [`zenrows extract prepare status ${domain}`], + server_code: code, + }); + } + if (code === "idempotency_conflict") { + return new ToolkitError({ + code: "EXTRACT_PREPARE_INVALID", + message: "This idempotency key was already used with a different request.", + likely_cause: cause, + next_action: "Retry without the key, or with a new one.", + server_code: code, + }); + } + if (code === "out_of_allowance" || status === 429) { + const used = p.used !== undefined && p.limit !== undefined ? ` (${String(p.used)} of ${String(p.limit)} used)` : ""; + return new ToolkitError({ + code: "EXTRACT_PREPARE_OUT_OF_ALLOWANCE", + message: `This account has used its monthly allowance of new domains to prepare${used}.`, + likely_cause: cause, + next_action: "Do not retry this month. Meanwhile extract this domain with --css and explicit selectors.", + suggested_commands: [`zenrows extract --css '{"title":"h1"}'`], + server_code: code, + }); + } + if (code === "overloaded" || code === "disabled" || status === 503) { + return new ToolkitError({ + code: "EXTRACT_PREPARE_UNAVAILABLE", + message: "Domain preparation is unavailable right now.", + likely_cause: cause, + next_action: "Retry in a few minutes; do not loop. Meanwhile extract with --css and explicit selectors.", + server_code: code, + }); + } + if (status === 401 || status === 403) { + return new ToolkitError({ + code: "AUTH_INVALID", + message: "Zenrows rejected the API key.", + likely_cause: cause, + next_action: "Re-check your key and log in again.", + suggested_commands: ["zenrows login --api-key "], + server_code: code, + }); + } + return new ToolkitError({ + code: "EXTRACT_PREPARE_FAILED", + message: `Preparing ${domain} failed with HTTP ${status}.`, + likely_cause: cause, + next_action: "Retry in a few minutes.", + server_code: code, + }); +} + +export function submitPreparedDomain( + domain: string, + exampleUrls: string[], + opts: PrepareRequestOpts, +): Promise { + return request("POST", "", domain, { ...opts, body: { domain, example_urls: exampleUrls } }); +} + +export function getPreparedDomain(domain: string, opts: PrepareRequestOpts): Promise { + return request("GET", `/${encodeURIComponent(domain)}`, domain, opts); +} + +export function getAllowance(opts: PrepareRequestOpts): Promise { + return request("GET", "/allowance", "", opts); +} + +/** + * Submit a domain. A 409 in_progress (already being prepared) answers with the + * current status when this account can read it, so the caller just follows it. + */ +export async function prepareDomain( + domain: string, + exampleUrls: string[], + opts: PrepareRequestOpts, +): Promise { + try { + return await submitPreparedDomain(domain, exampleUrls, opts); + } catch (err) { + if (!(err instanceof ToolkitError) || err.code !== "EXTRACT_PREPARE_IN_PROGRESS") throw err; + try { + return { ...(await getPreparedDomain(domain, opts)), already_in_progress: true }; + } catch { + throw err; + } + } +} + +export function isTerminal(record: PreparedDomain): boolean { + return record.status === "ready" || record.status === "failed"; +} + +/** Poll until ready or failed. Throws EXTRACT_PREPARE_TIMEOUT when the wait runs out. */ +export async function waitForPreparedDomain( + domain: string, + opts: PrepareRequestOpts & { + intervalMs?: number; + timeoutMs?: number; + sleepImpl?: (ms: number) => Promise; + now?: () => number; + onPoll?: (record: PreparedDomain) => void; + }, +): Promise { + const sleep = opts.sleepImpl ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const now = opts.now ?? Date.now; + const interval = opts.intervalMs ?? DEFAULT_POLL_INTERVAL_MS; + const deadline = now() + (opts.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS); + for (;;) { + const record = await getPreparedDomain(domain, opts); + opts.onPoll?.(record); + if (isTerminal(record)) return record; + if (now() + interval > deadline) { + throw new ToolkitError({ + code: "EXTRACT_PREPARE_TIMEOUT", + message: `${domain} is still being prepared (state: ${record.state ?? record.status}).`, + likely_cause: "Preparation usually takes a few minutes; this one is taking longer than the wait.", + next_action: "Check again later; it keeps running on the server.", + suggested_commands: [`zenrows extract prepare status ${domain}`], + }); + } + await sleep(interval); + } +} + +/** What to do after a failed preparation, by failure.code (docs.zenrows.com extract/endpoints). */ +export function failureNextStep(domain: string, code: string | undefined): string { + switch (code) { + case "interrupted": + case "preparation_exhausted": + return `Run the same zenrows extract prepare ${domain} command again. This did not use an allowance slot.`; + case "discovery_failed": + return `Run zenrows extract prepare ${domain} again with 1 to ${MAX_EXAMPLE_URLS} --url page URLs. This did not use an allowance slot.`; + case "unpreparable": + return `Run zenrows extract prepare ${domain} again with different --url pages from the same site; the same URLs fail again. This did not use an allowance slot.`; + case "preparation_failed": + return "Try other --url pages, or contact Zenrows support. This used an allowance slot."; + default: + return "Contact Zenrows support before preparing this domain again. Meanwhile extract with --css."; + } +} + +/** The next step for a status record. */ +export function statusNextStep(record: PreparedDomain, exampleUrl?: string): string { + if (record.status === "ready") return `Ready. Run the extract again${exampleUrl ? `: zenrows extract ${exampleUrl}` : "."}`; + if (record.status === "failed") return failureNextStep(record.domain, record.failure?.code); + return `Preparing (a few minutes). Check with: zenrows extract prepare status ${record.domain}`; +} diff --git a/templates/extract-pricing-pages/README.md b/templates/extract-pricing-pages/README.md index 59814bd..d7ae076 100644 --- a/templates/extract-pricing-pages/README.md +++ b/templates/extract-pricing-pages/README.md @@ -4,7 +4,7 @@ Extract structured pricing fields from product pages with Zenrows Extract. ```bash # Default: extract=auto on a prepared domain (site-tailored JSON). -# Not prepared yet? Prepare it: POST /v1/extract/prepared-domains (see the extract skill). +# Not prepared yet (REQS007)? zenrows extract prepare --url --wait zenrows extract https://www.owler.com/company/meltwater # A precise CSS selector map, on any domain diff --git a/tests/capabilities.test.ts b/tests/capabilities.test.ts index 03970e3..d6c33b2 100644 --- a/tests/capabilities.test.ts +++ b/tests/capabilities.test.ts @@ -16,6 +16,8 @@ test("capability matrix loads with the honest classifications", () => { test("extract notes name the prepare step and no automatic autoparse fallback", () => { const notes = loadCapabilities().extract?.notes ?? ""; assert.match(notes, /POST \/v1\/extract\/prepared-domains/); + assert.match(notes, /zenrows extract prepare --url /); + assert.match(notes, /zenrows extract prepare status /); assert.match(notes, /autoparse is deprecated/); assert.doesNotMatch(notes, /falls? back/i); }); diff --git a/tests/extract-outputs.test.ts b/tests/extract-outputs.test.ts index 15bc6d7..04faddd 100644 --- a/tests/extract-outputs.test.ts +++ b/tests/extract-outputs.test.ts @@ -188,9 +188,13 @@ test("runExtract reports REQS007 with the prepare step and does not retry", asyn assert.equal(e.code, "EXTRACT_DOMAIN_NOT_ENABLED"); assert.equal( e.message, - "Extract hasn't prepared x.com yet. Prepare it with POST /v1/extract/prepared-domains (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse.", + "Extract hasn't prepared x.com yet. Prepare it with `zenrows extract prepare x.com --url https://www.x.com/item/1` (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse.", ); - assert.match(e.toJSON().next_action, /"example_urls":\["https:\/\/www\.x\.com\/item\/1"\]/); + assert.deepEqual(e.toJSON().suggested_commands, [ + "zenrows extract prepare x.com --url https://www.x.com/item/1 --wait", + "zenrows extract prepare status x.com", + "zenrows extract https://www.x.com/item/1", + ]); return true; }, ); @@ -209,7 +213,7 @@ test("runExtract reports legacy AUTH010 like REQS007 and does not retry", async (e: unknown) => e instanceof ToolkitError && e.code === "EXTRACT_DOMAIN_NOT_ENABLED" && - e.message.startsWith("Extract hasn't prepared x.com yet. Prepare it with POST /v1/extract/prepared-domains"), + e.message.startsWith("Extract hasn't prepared x.com yet. Prepare it with `zenrows extract prepare x.com --url https://x.com`"), ); assert.equal(calls.length, 1, "AUTH010 must not be retried with autoparse"); }); diff --git a/tests/extract-prepare.test.ts b/tests/extract-prepare.test.ts new file mode 100644 index 0000000..494d30e --- /dev/null +++ b/tests/extract-prepare.test.ts @@ -0,0 +1,246 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { runPrepare } from "../src/cli/commands/extract-prepare.ts"; +import { extract } from "../src/cli/commands/extract.ts"; +import { failureNextStep, normalizeDomain, prepareCommand } from "../src/core/prepared-domains-api.ts"; + +const ctx = { json: true, yes: false }; +const BASE = "/v1/extract/prepared-domains"; +const RECORD = { + domain: "example.com", + status: "preparing", + state: "received", + submitted_at: "2026-10-15T09:00:00Z", + updated_at: "2026-10-15T09:00:00Z", +}; +const ALLOWANCE = { used: 3, limit: 10, remaining: 7, month: "2026-10" }; + +type Call = { method: string; path: string; headers: Record; body?: unknown }; + +function res(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); +} +const problem = (status: number, code: string, extra: Record = {}) => + res({ code, detail: `${code} detail`, status, title: code, ...extra }, status); + +/** A fetch stub answering by "METHOD path"; a route may be a list answered in order. */ +function stub(routes: Record Response) | (() => Response)[]>) { + const calls: Call[] = []; + const fetchImpl = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(String(input)); + const method = init?.method ?? "GET"; + calls.push({ + method, + path: url.pathname, + headers: (init?.headers ?? {}) as Record, + body: init?.body ? JSON.parse(String(init.body)) : undefined, + }); + const route = routes[`${method} ${url.pathname}`]; + if (!route) throw new Error(`unexpected ${method} ${url.pathname}`); + if (Array.isArray(route)) { + const next = route.length > 1 ? route.shift()! : route[0]!; + return next(); + } + return route(); + }) as typeof fetch; + return { calls, fetchImpl }; +} + +async function captureOut(fn: () => unknown): Promise { + const orig = process.stdout.write.bind(process.stdout); + let buf = ""; + process.stdout.write = ((s: string | Uint8Array, ...rest: unknown[]) => { + if (typeof s !== "string") return (orig as (...a: unknown[]) => boolean)(s, ...rest); + buf += s; + return true; + }) as typeof process.stdout.write; + try { + await fn(); + } finally { + process.stdout.write = orig; + } + return buf; +} + +async function run(args: string[], fetchImpl?: typeof fetch, extra: { now?: () => number } = {}) { + let code = -1; + const raw = await captureOut(async () => { + code = await runPrepare(args, ctx, { apiKey: "k", fetchImpl, sleepImpl: async () => {}, intervalMs: 1000, ...extra }); + }); + return { code, out: JSON.parse(raw) as Record }; +} + +test("normalizeDomain and prepareCommand", () => { + assert.equal(normalizeDomain("https://www.Example.com/p/1"), "example.com"); + assert.equal(normalizeDomain("shop.example.com"), "shop.example.com"); + assert.equal(prepareCommand("https://www.example.com/p/1"), "zenrows extract prepare example.com --url https://www.example.com/p/1"); +}); + +test("prepare submits the domain and example URLs, returns at once without --wait", async () => { + const { calls, fetchImpl } = stub({ [`POST ${BASE}`]: () => res(RECORD, 202) }); + const { code, out } = await run( + ["www.example.com", "--url", "https://example.com/p/1", "--url", "https://example.com/list"], + fetchImpl, + ); + assert.equal(code, 0); + assert.equal(out.status, "preparing"); + assert.match(out.next, /zenrows extract prepare status example\.com/); + assert.equal(calls.length, 1); + assert.deepEqual(calls[0]!.body, { domain: "example.com", example_urls: ["https://example.com/p/1", "https://example.com/list"] }); + assert.equal(calls[0]!.headers["X-API-Key"], "k"); +}); + +test("prepare --wait polls until ready", async () => { + const { calls, fetchImpl } = stub({ + [`POST ${BASE}`]: () => res(RECORD, 202), + [`GET ${BASE}/example.com`]: [ + () => res({ ...RECORD, state: "preparing" }), + () => res({ ...RECORD, state: "verifying" }), + () => res({ ...RECORD, status: "ready", state: "ready" }), + ], + }); + const { code, out } = await run(["example.com", "--url", "https://example.com/p/1", "--wait"], fetchImpl); + assert.equal(code, 0); + assert.equal(out.status, "ready"); + assert.match(out.next, /zenrows extract https:\/\/example\.com\/p\/1/); + assert.equal(calls.filter((c) => c.method === "GET").length, 3); +}); + +test("prepare --wait on failed exits 1 with the step for the failure code", async () => { + const { fetchImpl } = stub({ + [`POST ${BASE}`]: () => res(RECORD, 202), + [`GET ${BASE}/example.com`]: () => + res({ ...RECORD, status: "failed", state: "failed", failure: { code: "unpreparable", detail: "x", slot_consumed: false } }), + }); + const { code, out } = await run(["example.com", "--url", "https://example.com/p/1", "--wait"], fetchImpl); + assert.equal(code, 1); + assert.equal(out.error.code, "EXTRACT_PREPARE_FAILED"); + assert.equal(out.error.server_code, "unpreparable"); + assert.match(out.error.next_action, /different --url pages/); +}); + +test("prepare --wait gives up after --timeout with EXTRACT_PREPARE_TIMEOUT", async () => { + let t = 0; + const { fetchImpl } = stub({ + [`POST ${BASE}`]: () => res(RECORD, 202), + [`GET ${BASE}/example.com`]: () => res(RECORD), + }); + const { code, out } = await run(["example.com", "--url", "https://example.com/p/1", "--wait", "--timeout", "3"], fetchImpl, { + now: () => (t += 1000), + }); + assert.equal(code, 1); + assert.equal(out.error.code, "EXTRACT_PREPARE_TIMEOUT"); + assert.deepEqual(out.error.suggested_commands, ["zenrows extract prepare status example.com"]); +}); + +test("prepare on 409 in_progress follows the running preparation", async () => { + const { calls, fetchImpl } = stub({ + [`POST ${BASE}`]: () => problem(409, "in_progress"), + [`GET ${BASE}/example.com`]: () => res({ ...RECORD, state: "verifying" }), + }); + const { code, out } = await run(["example.com", "--url", "https://example.com/p/1"], fetchImpl); + assert.equal(code, 0); + assert.equal(out.already_in_progress, true); + assert.equal(out.state, "verifying"); + assert.equal(calls.length, 2); +}); + +test("prepare on 409 in_progress that this account cannot read reports EXTRACT_PREPARE_IN_PROGRESS", async () => { + const { fetchImpl } = stub({ + [`POST ${BASE}`]: () => problem(409, "in_progress"), + [`GET ${BASE}/example.com`]: () => problem(404, "not_found"), + }); + const { code, out } = await run(["example.com", "--url", "https://example.com/p/1"], fetchImpl); + assert.equal(code, 1); + assert.equal(out.error.code, "EXTRACT_PREPARE_IN_PROGRESS"); + assert.match(out.error.next_action, /Do not resubmit/); +}); + +for (const [status, serverCode, code, pattern] of [ + [400, "invalid_argument", "EXTRACT_PREPARE_INVALID", /1 to 5 full page URLs/], + [429, "out_of_allowance", "EXTRACT_PREPARE_OUT_OF_ALLOWANCE", /Do not retry this month/], + [503, "overloaded", "EXTRACT_PREPARE_UNAVAILABLE", /do not loop/], + [503, "disabled", "EXTRACT_PREPARE_UNAVAILABLE", /do not loop/], +] as const) { + test(`prepare maps ${status} ${serverCode} to ${code}`, async () => { + const { calls, fetchImpl } = stub({ + [`POST ${BASE}`]: () => problem(status, serverCode, serverCode === "out_of_allowance" ? { used: 10, limit: 10 } : {}), + }); + const { code: exit, out } = await run(["example.com", "--url", "https://example.com/p/1"], fetchImpl); + assert.equal(exit, 1); + assert.equal(out.error.code, code); + assert.equal(out.error.server_code, serverCode); + assert.match(out.error.next_action, pattern); + assert.equal(calls.length, 1); + if (serverCode === "out_of_allowance") assert.match(out.error.message, /10 of 10 used/); + }); +} + +test("prepare rejects zero, six or non-URL --url values before any request", async () => { + const { calls, fetchImpl } = stub({}); + for (const args of [ + ["example.com"], + ["example.com", ...Array.from({ length: 6 }, (_, i) => ["--url", `https://example.com/${i}`]).flat()], + ["example.com", "--url", "example.com/p/1"], + [], + ]) { + const { code, out } = await run(args, fetchImpl); + assert.equal(code, 1, args.join(" ")); + assert.equal(out.error.code, "INVALID_USAGE", args.join(" ")); + } + assert.equal(calls.length, 0); +}); + +test("prepare status shows the record, the allowance and the next step", async () => { + const { fetchImpl } = stub({ + [`GET ${BASE}/example.com`]: () => res({ ...RECORD, state: "preparing" }), + [`GET ${BASE}/allowance`]: () => res(ALLOWANCE), + }); + const { code, out } = await run(["status", "https://www.example.com/"], fetchImpl); + assert.equal(code, 0); + assert.equal(out.status, "preparing"); + assert.deepEqual(out.allowance, ALLOWANCE); + assert.match(out.next, /Preparing/); +}); + +test("prepare status keeps the record when the allowance call fails", async () => { + const { fetchImpl } = stub({ + [`GET ${BASE}/example.com`]: () => res({ ...RECORD, status: "ready", state: "ready" }), + [`GET ${BASE}/allowance`]: () => problem(503, "overloaded"), + }); + const { code, out } = await run(["status", "example.com"], fetchImpl); + assert.equal(code, 0); + assert.equal(out.status, "ready"); + assert.equal(out.allowance, undefined); + assert.match(out.next, /Run the extract again/); +}); + +test("prepare status on 404 says to prepare the domain first", async () => { + const { fetchImpl } = stub({ + [`GET ${BASE}/example.com`]: () => problem(404, "not_found"), + [`GET ${BASE}/allowance`]: () => res(ALLOWANCE), + }); + const { code, out } = await run(["status", "example.com"], fetchImpl); + assert.equal(code, 1); + assert.equal(out.error.code, "EXTRACT_PREPARE_NOT_FOUND"); + assert.match(out.error.next_action, /zenrows extract prepare example\.com --url/); +}); + +test("failureNextStep covers every documented failure code", () => { + assert.match(failureNextStep("x.com", "interrupted"), /same zenrows extract prepare/); + assert.match(failureNextStep("x.com", "preparation_exhausted"), /same zenrows extract prepare/); + assert.match(failureNextStep("x.com", "discovery_failed"), /--url page URLs/); + assert.match(failureNextStep("x.com", "unpreparable"), /different --url pages/); + assert.match(failureNextStep("x.com", "preparation_failed"), /other --url pages/); + assert.match(failureNextStep("x.com", "verification_failed"), /support/); + assert.match(failureNextStep("x.com", "unsupported"), /support/); +}); + +test("zenrows extract routes `prepare` to the prepare command", async () => { + let code = -1; + const raw = await captureOut(async () => { + code = await extract.run(["prepare", "example.com"], ctx); + }); + assert.equal(code, 1); + assert.equal(JSON.parse(raw).error.code, "INVALID_USAGE", "no --url: rejected locally, no extract of a URL named prepare"); +}); diff --git a/tests/http.test.ts b/tests/http.test.ts index 7bfea54..ae6a647 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -181,7 +181,7 @@ test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async assert.equal((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_ENABLED"); assert.equal( (err as { message: string }).message, - "Extract hasn't prepared example.net yet. Prepare it with POST /v1/extract/prepared-domains (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse.", + "Extract hasn't prepared example.net yet. Prepare it with `zenrows extract prepare example.net --url https://example.net` (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse.", ); return true; }, diff --git a/workflows/competitor-intelligence/WORKFLOW.md b/workflows/competitor-intelligence/WORKFLOW.md index 5bdb05f..99397ce 100644 --- a/workflows/competitor-intelligence/WORKFLOW.md +++ b/workflows/competitor-intelligence/WORKFLOW.md @@ -20,7 +20,7 @@ simple per-URL loop). ``` zenrows fetch https://competitor.example/products ``` -2. **Validate extraction** on one page (on a domain not prepared yet, prepare it first: see the extract skill). +2. **Validate extraction** on one page (on a domain not prepared yet, run `zenrows extract prepare --url --wait` first). ``` zenrows extract https://competitor.example/p/123 --validate ``` From 2f6faafb759908fadc7918a628de4961b319705a Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 13:46:42 +0000 Subject: [PATCH 3/8] fix(extract): stop the REQS007 prepare loop and drop autoparse from the error REQS007 has two causes. On a domain not prepared yet, preparing it is the fix. On a prepared domain, the gateway answers REQS007 with "Extract did not recognise this page" when it does not cover that kind of page, and resubmitting the domain only returns its existing ready record. The old wording sent agents round extract, prepare (ready at once), extract again. - A REQS007 whose body says the page was not recognised is now EXTRACT_PAGE_NOT_RECOGNISED: use --css or contact support with the URL; no prepare command. - `zenrows extract prepare` answering ready at once sets `already_prepared` and says that a REQS007 on that domain means the kind of page is not covered, instead of "run the extract again". - The REQS007 error no longer offers --autoparse; it offers --css. - failureNextStep says whether the attempt used an allowance slot from failure.slot_consumed, not from the failure code. - `zenrows extract prepare` never creates a Free plan account; the unused --no-signup flag is gone. - Each function in http.ts has its own doc comment again. - normalizeDomain keeps the full host: Extract prepares each host on its own (a prepared es.wikipedia.org does not cover pt.wikipedia.org). - Poll interval: 30 seconds, shared with the MCP server. --- agent-plugin/skills/extract/SKILL.md | 10 ++++- skills/extract/SKILL.md | 14 ++++++- src/cli/commands/extract-prepare.ts | 32 ++++++++------ src/core/errors.ts | 1 + src/core/http.ts | 58 +++++++++++++++++++------ src/core/prepared-domains-api.ts | 63 ++++++++++++++++++++++------ tests/extract-outputs.test.ts | 35 +++++++++++++++- tests/extract-prepare.test.ts | 44 +++++++++++++++---- tests/http.test.ts | 2 +- 9 files changed, 207 insertions(+), 52 deletions(-) diff --git a/agent-plugin/skills/extract/SKILL.md b/agent-plugin/skills/extract/SKILL.md index 76b9019..ec59061 100644 --- a/agent-plugin/skills/extract/SKILL.md +++ b/agent-plugin/skills/extract/SKILL.md @@ -53,7 +53,15 @@ extract({ url }) # 4. extract again (`used` / `limit` / `remaining`) and a `next` field with what to do now. - `out_of_allowance`: no slots left this month; use `mode=css`. `overloaded` / `disabled`: retry in a few minutes, do not loop. - A `failed` status carries `failure.code`; follow `next`. + A `failed` status carries `failure.code` and `failure.slot_consumed`; follow `next`. +- The domain is the host: `shop.example.com` and `example.com` are prepared + separately; a leading `www.` is dropped. +- **Do not loop.** If `prepare_domain` answers `ready` at once + (`already_prepared: true`), the domain was prepared before; an `extract` that + still returns REQS007 means that kind of page is not covered, and preparing + again changes nothing. The same goes for REQS007 with + `reason: "page_not_recognised"`. Use `mode=css`, or contact Zenrows support + with the URL. ## Fallback: `scrape` extraction params diff --git a/skills/extract/SKILL.md b/skills/extract/SKILL.md index c1bf980..e2f69cb 100644 --- a/skills/extract/SKILL.md +++ b/skills/extract/SKILL.md @@ -45,12 +45,22 @@ zenrows extract https://example.com/p/1 # 3. extract again - `--url`: 1 to 5 real pages you want to extract, one per kind of page. - Without `--wait` the command returns at once; then poll with - `zenrows extract prepare status example.com` until it says `ready`. + `zenrows extract prepare status example.com` (about every 30 seconds) until + it says `ready`. +- The domain is the host: `shop.example.com` and `example.com` are prepared + separately; a leading `www.` is dropped. - Preparing costs no credits. Each new domain uses one slot of the account's monthly allowance; `prepare status` shows `used` / `limit` / `remaining`. - `EXTRACT_PREPARE_OUT_OF_ALLOWANCE`: no slots left this month; use `--css`. `EXTRACT_PREPARE_UNAVAILABLE`: retry in a few minutes, do not loop. - A `failed` status prints the next step for its failure code. + A `failed` status prints the next step for its failure code and says whether + the attempt used an allowance slot. +- **Do not loop.** If `prepare` answers `ready` at once (`already_prepared`), + the domain was prepared before; an extract that still fails with REQS007 + means that kind of page is not covered, and preparing again changes nothing. + The same goes for `EXTRACT_PAGE_NOT_RECOGNISED` (REQS007 on a prepared domain: + Extract did not recognise this page). Use `--css`, or contact Zenrows support + with the URL. ## Rules - Validate on a single page before scaling across many URLs. diff --git a/src/cli/commands/extract-prepare.ts b/src/cli/commands/extract-prepare.ts index 38e0554..c5cad55 100644 --- a/src/cli/commands/extract-prepare.ts +++ b/src/cli/commands/extract-prepare.ts @@ -61,7 +61,6 @@ export async function runPrepare(argv: string[], ctx: RunContext, deps: PrepareD url: { type: "string", multiple: true }, wait: { type: "boolean" }, timeout: { type: "string" }, - "no-signup": { type: "boolean" }, json: { type: "boolean" }, help: { type: "boolean" }, }); @@ -84,12 +83,8 @@ export async function runPrepare(argv: string[], ctx: RunContext, deps: PrepareD const policy = loadPolicy(); const apiKey = deps.apiKey ?? - (await ensureApiKey(isStatus || values["no-signup"] ? { ...policy, auto_signup: false } : policy, { - onProvision: (a) => { - log.info("No API key found — created a Zenrows Free plan account for you."); - log.dim(`Claim it anytime (keeps your usage): ${a.claimUrl}`); - }, - })); + // Preparing never creates a Free plan account: it needs a key the user already has. + (await ensureApiKey({ ...policy, auto_signup: false })); const opts: PrepareRequestOpts = { apiKey, apiBase: loadConfig().apiBase, fetchImpl: deps.fetchImpl }; if (isStatus) { @@ -101,6 +96,8 @@ export async function runPrepare(argv: string[], ctx: RunContext, deps: PrepareD log.step(`Prepare ${domain} for Extract (${urls.length} example URL${urls.length === 1 ? "" : "s"})…`); let record: PreparedDomain = await prepareDomain(domain, urls, opts); if (record.already_in_progress) log.info(`${domain} was already being prepared; following that preparation.`); + // A submit that answers ready at once found the domain already prepared. + const alreadyReady = record.status === "ready" && !record.already_in_progress; if (values.wait === true && record.status !== "ready" && record.status !== "failed") { record = await waitForPreparedDomain(domain, { ...opts, @@ -116,11 +113,11 @@ export async function runPrepare(argv: string[], ctx: RunContext, deps: PrepareD code: "EXTRACT_PREPARE_FAILED", message: `Preparing ${domain} failed${record.failure?.code ? ` (${record.failure.code})` : ""}.`, likely_cause: record.failure?.detail ?? "The preparation did not finish.", - next_action: failureNextStep(domain, record.failure?.code), + next_action: failureNextStep(domain, record.failure), server_code: record.failure?.code, }); } - report(record, undefined, json, urls[0]); + report(record, undefined, json, { exampleUrl: urls[0], alreadyReady }); return 0; } catch (err) { printError(err, json); @@ -172,10 +169,21 @@ function parseTimeout(v?: string): number { return n * 1000; } -function report(record: PreparedDomain, allowance: Allowance | undefined, json: boolean, exampleUrl?: string): void { - const next = statusNextStep(record, exampleUrl); +function report( + record: PreparedDomain, + allowance: Allowance | undefined, + json: boolean, + opts: { exampleUrl?: string; alreadyReady?: boolean } = {}, +): void { + const next = statusNextStep(record, opts); if (json) { - log.out(JSON.stringify({ ok: true, ...record, ...(allowance ? { allowance } : {}), next }, null, 2)); + log.out( + JSON.stringify( + { ok: true, ...record, ...(opts.alreadyReady ? { already_prepared: true } : {}), ...(allowance ? { allowance } : {}), next }, + null, + 2, + ), + ); return; } const line = `${record.domain}: ${record.status}${record.state && record.state !== record.status ? ` (${record.state})` : ""}`; diff --git a/src/core/errors.ts b/src/core/errors.ts index 9b51c50..2e26441 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -29,6 +29,7 @@ export type ErrorCode = | "FETCH_EMPTY_RESPONSE" | "EXTRACT_FAILED" | "EXTRACT_DOMAIN_NOT_ENABLED" + | "EXTRACT_PAGE_NOT_RECOGNISED" | "EXTRACT_VALIDATION_FAILED" | "EXTRACT_PREPARE_INVALID" | "EXTRACT_PREPARE_NOT_FOUND" diff --git a/src/core/http.ts b/src/core/http.ts index 765d983..e3117fc 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -149,7 +149,7 @@ export async function scrape( // REQS007 on extract=auto: Extract has not prepared the domain yet. Reported, // never retried: the fix is to prepare the domain, which the error names. if (res.status === 403 && params.extract !== undefined && zrErrorCode(body) === "REQS007") { - throw extractDomainNotPrepared(params.url, zrErrorDetail(body)); + throw extractDomainNotPrepared(params.url, body); } // 422 no_data_extracted on an Extract request: the page was processed and had // nothing to extract. It is an answer, not a failure to reach the target, so @@ -210,7 +210,7 @@ export async function scrape( // AUTH010 on extract=auto is the legacy code for REQS007: the domain is // not prepared for Extract. Distinct from credit exhaustion (AUTH004). if (params.extract !== undefined && zrErrorCode(body) === "AUTH010") { - throw extractDomainNotPrepared(params.url, zrErrorDetail(body)); + throw extractDomainNotPrepared(params.url, body); } // AUTH014: this key hit one of its credit caps; the account still has credits. if (isKeyCapReached(zrErrorCode(body) ?? undefined)) { @@ -302,28 +302,60 @@ export async function scrape( } /** - * The CLI gave up waiting before the API answered. This is NOT unreachability: - * the connection was established and the gateway was still working, so telling - * the operator to check connectivity would send them at the wrong problem. The - * elapsed time is included so the 90s server budget is visible in the output. + * REQS007 has two causes, which the problem body tells apart: the domain is not + * prepared ("Extract is not prepared for the requested domain (...)"), or it is + * prepared and Extract did not recognise this page ("Extract did not recognise + * this page on ...", detail "Extract serves ..."). Only the first is fixed by + * preparing the domain; preparing a prepared domain returns its existing ready + * record, so pointing there would loop. */ +export function isPageNotRecognised(body: string): boolean { + try { + const j = JSON.parse(body) as { title?: unknown; detail?: unknown }; + const title = typeof j.title === "string" ? j.title : ""; + const detail = typeof j.detail === "string" ? j.detail : ""; + return /did not recogni[sz]e this page/i.test(title) || /^Extract serves /.test(detail); + } catch { + return false; + } +} + /** - * REQS007 (or the legacy AUTH010) on extract=auto: Extract has not prepared the - * URL's domain yet. The message names the exact command to run, because agents - * copy it. + * REQS007 (or the legacy AUTH010) on extract=auto. On a domain not prepared yet + * the message names the exact command to prepare it, because agents copy it; on + * a page a prepared domain does not recognise it names --css instead. Neither + * offers Autoparse. */ -export function extractDomainNotPrepared(url: string, detail: string | null): ToolkitError { +export function extractDomainNotPrepared(url: string, body: string): ToolkitError { const domain = normalizeDomain(url); + const detail = zrErrorDetail(body); + const css = `zenrows extract ${url} --css '{"title":"h1"}'`; + if (isPageNotRecognised(body)) { + return new ToolkitError({ + code: "EXTRACT_PAGE_NOT_RECOGNISED", + message: `Extract serves ${domain} but did not recognise this page. Preparing the domain again does not help; use --css, or contact Zenrows support with the URL.`, + likely_cause: detail ?? `Extract serves ${domain} but did not recognise this page.`, + next_action: "Extract this page with --css and explicit selectors, or contact Zenrows support with the URL.", + suggested_commands: [css], + server_code: "REQS007", + }); + } const prepare = prepareCommand(url); return new ToolkitError({ code: "EXTRACT_DOMAIN_NOT_ENABLED", - message: `Extract hasn't prepared ${domain} yet. Prepare it with \`${prepare}\` (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse.`, + message: `Extract hasn't prepared ${domain} yet. Prepare it with \`${prepare}\` (a few example URLs, a few minutes, no credits), then retry; meanwhile use --css.`, likely_cause: detail ?? `Extract has not prepared ${domain} yet.`, - next_action: `Run \`${prepare} --wait\` (add up to 5 --url pages, one per kind of page you want), then retry \`zenrows extract ${url}\`. Without --wait, check progress with \`zenrows extract prepare status ${domain}\`.`, - suggested_commands: [`${prepare} --wait`, `zenrows extract prepare status ${domain}`, `zenrows extract ${url}`], + next_action: `Run \`${prepare} --wait\` (add up to 5 --url pages, one per kind of page you want), then retry \`zenrows extract ${url}\`. Without --wait, check progress with \`zenrows extract prepare status ${domain}\`. Meanwhile extract with --css and explicit selectors.`, + suggested_commands: [`${prepare} --wait`, `zenrows extract prepare status ${domain}`, css], }); } +/** + * The CLI gave up waiting before the API answered. This is NOT unreachability: + * the connection was established and the gateway was still working, so telling + * the operator to check connectivity would send them at the wrong problem. The + * elapsed time is included so the 90s server budget is visible in the output. + */ export function requestTimeout(url: string, timeoutMs: number, elapsedMs: number): ToolkitError { const suggestedMs = Math.max(timeoutMs + 60_000, SERVER_BUDGET_MS + 60_000); const atBudget = elapsedMs >= SERVER_BUDGET_MS; diff --git a/src/core/prepared-domains-api.ts b/src/core/prepared-domains-api.ts index ba272f4..af99be6 100644 --- a/src/core/prepared-domains-api.ts +++ b/src/core/prepared-domains-api.ts @@ -16,7 +16,7 @@ import { agentClientHeader } from "./agent-client.ts"; export const MAX_EXAMPLE_URLS = 5; /** How often `--wait` polls, and for how long by default. */ -export const DEFAULT_POLL_INTERVAL_MS = 15_000; +export const DEFAULT_POLL_INTERVAL_MS = 30_000; export const DEFAULT_WAIT_TIMEOUT_MS = 15 * 60_000; /** status: preparing | ready | failed. state adds the step (received, discovering, preparing, verifying). */ @@ -45,7 +45,16 @@ export interface PrepareRequestOpts { fetchImpl?: typeof fetch; } -/** The domain as the API wants it: host only, no scheme, path or leading `www.`. Accepts a URL. */ +/** + * The domain as the API keys it: host only, no scheme, path or leading `www.`. + * Accepts a URL. + * + * Deliberately not the registrable domain: Extract prepares and serves each host + * on its own. A prepared `es.wikipedia.org` does not cover `pt.wikipedia.org` + * (REQS007 names `pt.wikipedia.org`), and `wikipedia.org` has no record of its + * own. The API itself folds `www.` into the bare host, so dropping it here + * matches what it stores. + */ export function normalizeDomain(input: string): string { const trimmed = input.trim(); let host = trimmed; @@ -267,26 +276,54 @@ export async function waitForPreparedDomain( } } +/** Whether a failed preparation used an allowance slot, from the record's failure.slot_consumed. */ +function slotSentence(slotConsumed: boolean | undefined): string { + if (slotConsumed === true) return " This attempt used an allowance slot."; + if (slotConsumed === false) return " This attempt did not use an allowance slot."; + return ""; +} + /** What to do after a failed preparation, by failure.code (docs.zenrows.com extract/endpoints). */ -export function failureNextStep(domain: string, code: string | undefined): string { - switch (code) { +export function failureNextStep(domain: string, failure: PreparedDomain["failure"]): string { + let step: string; + switch (failure?.code) { case "interrupted": case "preparation_exhausted": - return `Run the same zenrows extract prepare ${domain} command again. This did not use an allowance slot.`; + step = `Run the same zenrows extract prepare ${domain} command again.`; + break; case "discovery_failed": - return `Run zenrows extract prepare ${domain} again with 1 to ${MAX_EXAMPLE_URLS} --url page URLs. This did not use an allowance slot.`; + step = `Run zenrows extract prepare ${domain} again with 1 to ${MAX_EXAMPLE_URLS} --url page URLs.`; + break; case "unpreparable": - return `Run zenrows extract prepare ${domain} again with different --url pages from the same site; the same URLs fail again. This did not use an allowance slot.`; + step = `Run zenrows extract prepare ${domain} again with different --url pages from the same site; the same URLs fail again.`; + break; case "preparation_failed": - return "Try other --url pages, or contact Zenrows support. This used an allowance slot."; + step = "Try other --url pages, or contact Zenrows support."; + break; default: - return "Contact Zenrows support before preparing this domain again. Meanwhile extract with --css."; + step = "Contact Zenrows support before preparing this domain again. Meanwhile extract with --css."; } + return step + slotSentence(failure?.slot_consumed); } -/** The next step for a status record. */ -export function statusNextStep(record: PreparedDomain, exampleUrl?: string): string { - if (record.status === "ready") return `Ready. Run the extract again${exampleUrl ? `: zenrows extract ${exampleUrl}` : "."}`; - if (record.status === "failed") return failureNextStep(record.domain, record.failure?.code); +/** + * What an already-prepared domain means for an extract that answered REQS007: + * that kind of page is not covered, and preparing again does not change it (the + * API returns the existing ready record). Said instead of "run the extract + * again", which would loop. + */ +export const PAGE_KIND_NOT_COVERED = + "If the extract still fails with REQS007 on this domain, that kind of page is not covered and preparing again does not change it: use --css, or contact Zenrows support with the URL."; + +/** + * The next step for a status record. `alreadyReady` is set when a submit answered + * ready at once: the domain was prepared before this call. + */ +export function statusNextStep(record: PreparedDomain, opts: { exampleUrl?: string; alreadyReady?: boolean } = {}): string { + if (record.status === "ready") { + if (opts.alreadyReady) return `${record.domain} was already prepared, so nothing new was submitted. ${PAGE_KIND_NOT_COVERED}`; + return `Ready. Run the extract again${opts.exampleUrl ? `: zenrows extract ${opts.exampleUrl}` : ""}. ${PAGE_KIND_NOT_COVERED}`; + } + if (record.status === "failed") return failureNextStep(record.domain, record.failure); return `Preparing (a few minutes). Check with: zenrows extract prepare status ${record.domain}`; } diff --git a/tests/extract-outputs.test.ts b/tests/extract-outputs.test.ts index 04faddd..67bc83e 100644 --- a/tests/extract-outputs.test.ts +++ b/tests/extract-outputs.test.ts @@ -188,13 +188,14 @@ test("runExtract reports REQS007 with the prepare step and does not retry", asyn assert.equal(e.code, "EXTRACT_DOMAIN_NOT_ENABLED"); assert.equal( e.message, - "Extract hasn't prepared x.com yet. Prepare it with `zenrows extract prepare x.com --url https://www.x.com/item/1` (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse.", + "Extract hasn't prepared x.com yet. Prepare it with `zenrows extract prepare x.com --url https://www.x.com/item/1` (a few example URLs, a few minutes, no credits), then retry; meanwhile use --css.", ); assert.deepEqual(e.toJSON().suggested_commands, [ "zenrows extract prepare x.com --url https://www.x.com/item/1 --wait", "zenrows extract prepare status x.com", - "zenrows extract https://www.x.com/item/1", + `zenrows extract https://www.x.com/item/1 --css '{"title":"h1"}'`, ]); + assert.doesNotMatch(JSON.stringify(e.toJSON()), /autoparse/); return true; }, ); @@ -202,6 +203,36 @@ test("runExtract reports REQS007 with the prepare step and does not retry", asyn }); }); +test("runExtract on REQS007 for a page a prepared domain does not recognise points to --css, not prepare", async () => { + // The gateway's body for a prepared domain whose page it did not recognise. + const body = JSON.stringify({ + code: "REQS007", + title: "Extract did not recognise this page on x.com (REQS007)", + detail: "Extract serves x.com, but did not recognise this page as one it can extract.", + status: 403, + }); + let calls = 0; + await withFetchHandler(() => { + calls += 1; + return new Response(body, { status: 403, headers: { "content-type": "application/json" } }); + }, async () => { + await assert.rejects( + () => runExtract({ url: "https://x.com/search?q=a" }, cfg, defaultPolicy(), "testkey"), + (e: unknown) => { + assert.ok(e instanceof ToolkitError); + assert.equal(e.code, "EXTRACT_PAGE_NOT_RECOGNISED"); + assert.match(e.message, /Preparing the domain again does not help/); + const shape = JSON.stringify(e.toJSON()); + assert.doesNotMatch(shape, /extract prepare/, "no prepare command: it would answer ready and loop"); + assert.doesNotMatch(shape, /autoparse/); + assert.match(shape, /--css/); + return true; + }, + ); + assert.equal(calls, 1); + }); +}); + test("runExtract reports legacy AUTH010 like REQS007 and does not retry", async () => { const calls: string[] = []; await withFetchHandler((url) => { diff --git a/tests/extract-prepare.test.ts b/tests/extract-prepare.test.ts index 494d30e..7063912 100644 --- a/tests/extract-prepare.test.ts +++ b/tests/extract-prepare.test.ts @@ -226,14 +226,42 @@ test("prepare status on 404 says to prepare the domain first", async () => { assert.match(out.error.next_action, /zenrows extract prepare example\.com --url/); }); -test("failureNextStep covers every documented failure code", () => { - assert.match(failureNextStep("x.com", "interrupted"), /same zenrows extract prepare/); - assert.match(failureNextStep("x.com", "preparation_exhausted"), /same zenrows extract prepare/); - assert.match(failureNextStep("x.com", "discovery_failed"), /--url page URLs/); - assert.match(failureNextStep("x.com", "unpreparable"), /different --url pages/); - assert.match(failureNextStep("x.com", "preparation_failed"), /other --url pages/); - assert.match(failureNextStep("x.com", "verification_failed"), /support/); - assert.match(failureNextStep("x.com", "unsupported"), /support/); +test("failureNextStep: the step comes from failure.code, the slot sentence from failure.slot_consumed", () => { + assert.match(failureNextStep("x.com", { code: "interrupted" }), /same zenrows extract prepare/); + assert.match(failureNextStep("x.com", { code: "preparation_exhausted" }), /same zenrows extract prepare/); + assert.match(failureNextStep("x.com", { code: "discovery_failed" }), /--url page URLs/); + assert.match(failureNextStep("x.com", { code: "unpreparable" }), /different --url pages/); + assert.match(failureNextStep("x.com", { code: "preparation_failed" }), /other --url pages/); + assert.match(failureNextStep("x.com", { code: "verification_failed" }), /support/); + assert.match(failureNextStep("x.com", { code: "unsupported" }), /support/); + assert.match(failureNextStep("x.com", { code: "unpreparable", slot_consumed: false }), /did not use an allowance slot/); + assert.match(failureNextStep("x.com", { code: "unpreparable", slot_consumed: true }), /This attempt used an allowance slot/); + assert.doesNotMatch(failureNextStep("x.com", { code: "unpreparable" }), /slot/); +}); + +test("prepare answering ready at once says the domain was already prepared, not to run the extract again", async () => { + const { calls, fetchImpl } = stub({ [`POST ${BASE}`]: () => res({ ...RECORD, status: "ready", state: "ready" }, 202) }); + const { code, out } = await run(["example.com", "--url", "https://example.com/search", "--wait"], fetchImpl); + assert.equal(code, 0); + assert.equal(out.already_prepared, true); + assert.match(out.next, /already prepared/); + assert.match(out.next, /use --css/); + assert.doesNotMatch(out.next, /Run the extract again/); + assert.equal(calls.length, 1, "--wait does not poll a domain that is already ready"); +}); + +test("prepare never auto-creates a Free plan account", async () => { + const { calls, fetchImpl } = stub({}); + const saved = process.env.ZENROWS_API_KEY; + delete process.env.ZENROWS_API_KEY; + let code = -1; + const raw = await captureOut(async () => { + code = await runPrepare(["example.com", "--url", "https://example.com/p/1"], ctx, { fetchImpl }); + }); + if (saved !== undefined) process.env.ZENROWS_API_KEY = saved; + assert.equal(code, 1); + assert.equal(JSON.parse(raw).error.code, "AUTH_MISSING"); + assert.equal(calls.length, 0); }); test("zenrows extract routes `prepare` to the prepare command", async () => { diff --git a/tests/http.test.ts b/tests/http.test.ts index ae6a647..fabeea1 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -181,7 +181,7 @@ test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async assert.equal((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_ENABLED"); assert.equal( (err as { message: string }).message, - "Extract hasn't prepared example.net yet. Prepare it with `zenrows extract prepare example.net --url https://example.net` (a few example URLs, a few minutes, no credits), then retry; or run with --autoparse.", + "Extract hasn't prepared example.net yet. Prepare it with `zenrows extract prepare example.net --url https://example.net` (a few example URLs, a few minutes, no credits), then retry; meanwhile use --css.", ); return true; }, From 77e51fbf4db96d6708888251c0ab42bdaea9cc44 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 15:02:04 +0000 Subject: [PATCH 4/8] fix(extract): read the REQS007 reason, and give the API's own advice for an unrecognised page The API now says which REQS007 it is in a `reason` field: domain_not_prepared, page_not_recognised or domain_not_enabled. The error is classified by that field first; matching the API's sentences stays only as the fallback for an API that does not send it yet, and says so in a comment. page_not_recognised does not mean that a kind of page is missing: on a prepared domain Extract learns a new kind of page on its first request, so the page is not one Extract extracts (a home, search, login or category page) or a transient failure kept it from trying. The error now gives the API's advice in the API's order: try a product, article or listing page; if this page should work, retry it once in a few minutes, not in a loop; otherwise use css. It no longer says "not covered" or sends to support first, and it still says that preparing again changes nothing. domain_not_enabled says that preparing does not help, and offers css. The "ready" and "already prepared" next steps say the same thing for a later page_not_recognised. Skills, help, README, docs and tool descriptions now tell one story with one vocabulary: prepared, ready, page kind, allowance slot, a few minutes, about every 30 seconds. --- README.md | 11 +- agent-plugin/README.md | 2 +- agent-plugin/skills/extract/SKILL.md | 40 ++++--- agent-plugin/skills/zenrows/SKILL.md | 2 +- docs/capabilities.md | 2 +- registry/capabilities.json | 2 +- skills/extract/SKILL.md | 44 ++++---- skills/zenrows/SKILL.md | 5 +- src/cli/commands/extract-prepare.ts | 17 +-- src/cli/commands/extract.ts | 9 +- src/core/errors.ts | 1 + src/core/http.ts | 66 ++++++++---- src/core/prepared-domains-api.ts | 25 +++-- templates/extract-pricing-pages/README.md | 2 +- tests/capabilities.test.ts | 2 + tests/extract-outputs.test.ts | 100 ++++++++++++++---- tests/extract-prepare.test.ts | 11 +- workflows/competitor-intelligence/WORKFLOW.md | 2 +- 18 files changed, 230 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index 3674564..1e58878 100644 --- a/README.md +++ b/README.md @@ -123,9 +123,14 @@ zenrows extract --outputs emails,links # built- zenrows extract --output markdown ``` -On a domain Extract hasn't prepared yet, `zenrows extract` fails with REQS007 -and prints the command to prepare it. Preparing takes a few minutes and costs -no credits; each new domain uses one slot of the monthly allowance. +On a domain Extract hasn't prepared yet, `zenrows extract` fails with +`EXTRACT_DOMAIN_NOT_ENABLED` (API code REQS007) and prints the command to +prepare it. Preparing takes a few minutes and costs no credits; each new domain +uses one slot of the monthly allowance. `EXTRACT_PAGE_NOT_RECOGNISED` (also +REQS007) is different: the domain is prepared, but this page is not one Extract +extracts. Try a product, article or listing page instead, retry it once in a +few minutes if it should work, or use `--css`; preparing again does not change +it. ```bash zenrows extract prepare example.com --url https://example.com/p/1 --url https://example.com/list --wait diff --git a/agent-plugin/README.md b/agent-plugin/README.md index 0d96adc..b9fdc8a 100644 --- a/agent-plugin/README.md +++ b/agent-plugin/README.md @@ -62,7 +62,7 @@ Use MCP tool names — not CLI `zenrows fetch`: | --- | --- | | `zenrows` | Decision tree across all primitives | | `protected-fetch` | `scrape` | -| `extract` | `extract` (fallback: `scrape` + css_extractor); on REQS007 `prepare_domain` then `prepared_domain_status` | +| `extract` | `extract` (fallback: `scrape` + css_extractor); on REQS007 with reason `domain_not_prepared`, `prepare_domain` then `prepared_domain_status` | | `batch-jobs` | `batch_create`, `batch_status`, `batch_results`, `batch_cancel` | | `interact-browser` | `browser_*` (escalation only) | | `cost-control` | Cost multipliers + escalate-with-evidence | diff --git a/agent-plugin/skills/extract/SKILL.md b/agent-plugin/skills/extract/SKILL.md index ec59061..e2e7ccc 100644 --- a/agent-plugin/skills/extract/SKILL.md +++ b/agent-plugin/skills/extract/SKILL.md @@ -31,37 +31,43 @@ protected or dynamic sites. Set `js_render` or `premium_proxy` only to force a fixed configuration; that turns Adaptive Stealth Mode off. Check `empty` in the result: true means no field came back with a value. -## Domain not prepared yet (REQS007): prepare it, then extract again +## REQS007: prepare the domain, then extract again -`mode=auto` works on any prepared domain, and you can prepare one yourself. On -a domain Extract hasn't prepared yet, `extract` returns a tool error (`REQS007`; -legacy `AUTH010`) whose `next_tool` holds the `prepare_domain` arguments. It does -not retry with Autoparse unless you set `fallback_autoparse: true`. The flow: +`mode=auto` works on any prepared domain, and you can prepare one yourself. +When Extract refuses a page, `extract` returns a tool error with code +`REQS007` (legacy `AUTH010`) and a `reason`. It never retries with Autoparse +unless you set `fallback_autoparse: true`. + +| `reason` | Meaning | Next step | +| --- | --- | --- | +| `domain_not_prepared` | The domain is not prepared yet. | `prepare_domain` (the error's `next_tool` holds its arguments), then poll, then `extract` again. | +| `page_not_recognised` | The domain is prepared, but Extract did not recognise this page as one it can extract (a home, search, login or category page, or a transient failure). | Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use `mode=css`. Do not call `prepare_domain`: preparing again does not change this. | +| `domain_not_enabled` | Extract is not enabled for the site. | Use `mode=css`, or contact Zenrows support to have the site enabled. Preparing does not change this. | + +Preparing a domain: ``` -extract({ url }) # 1. error REQS007 +extract({ url }) # 1. REQS007, reason domain_not_prepared prepare_domain({ domain: "example.com", example_urls: ["https://example.com/p/1", "https://example.com/list"] }) # 2. returns at once, status "preparing" -prepared_domain_status({ domain: "example.com" }) # 3. every ~30 s until status "ready" +prepared_domain_status({ domain: "example.com" }) # 3. about every 30 seconds until "ready" extract({ url }) # 4. extract again ``` -- `example_urls`: 1 to 5 real pages you want to extract, one per kind of page. +- `example_urls`: 1 to 5 real pages you want to extract, one per page kind + (for example a product page and a listing page). +- The domain is the host: `shop.example.com` and `example.com` are prepared + separately, and a leading `www.` is dropped. - Preparing takes a few minutes and costs no credits. Each new domain uses one slot of the account's monthly allowance; `prepared_domain_status` returns it (`used` / `limit` / `remaining`) and a `next` field with what to do now. - `out_of_allowance`: no slots left this month; use `mode=css`. - `overloaded` / `disabled`: retry in a few minutes, do not loop. + `overloaded` / `disabled`: retry in a few minutes; do not loop. A `failed` status carries `failure.code` and `failure.slot_consumed`; follow `next`. -- The domain is the host: `shop.example.com` and `example.com` are prepared - separately; a leading `www.` is dropped. -- **Do not loop.** If `prepare_domain` answers `ready` at once - (`already_prepared: true`), the domain was prepared before; an `extract` that - still returns REQS007 means that kind of page is not covered, and preparing - again changes nothing. The same goes for REQS007 with - `reason: "page_not_recognised"`. Use `mode=css`, or contact Zenrows support - with the URL. +- A domain that is already prepared answers `ready` at once + (`already_prepared: true`) and uses no slot: call `extract`. If it then + returns `page_not_recognised`, follow that row of the table, not `prepare_domain`. ## Fallback: `scrape` extraction params diff --git a/agent-plugin/skills/zenrows/SKILL.md b/agent-plugin/skills/zenrows/SKILL.md index 2082d5c..ca73ae4 100644 --- a/agent-plugin/skills/zenrows/SKILL.md +++ b/agent-plugin/skills/zenrows/SKILL.md @@ -67,7 +67,7 @@ If the user needs login, clicks, forms, sessions, or persistent state: | --- | --- | --- | | Protected Fetch | `scrape` | Default markdown; HTML / plaintext / PDF / screenshot options | | Extract | `extract` | Structured JSON (`extract=auto` on any prepared domain, css_extractor; autoparse deprecated). `extract=auto` is currently free; billing may apply later | -| Prepare a domain | `prepare_domain`, `prepared_domain_status` | After an `extract` REQS007: prepare the domain with 1 to 5 example URLs, poll until ready, extract again. No credits; uses the monthly allowance | +| Prepare a domain | `prepare_domain`, `prepared_domain_status` | After an `extract` REQS007 with reason `domain_not_prepared`: prepare the domain with 1 to 5 example URLs, poll about every 30 seconds until ready, extract again. No credits; uses one monthly allowance slot. Not for `page_not_recognised`: that domain is already prepared | | Batch | `batch_create`, `batch_status`, `batch_results`, `batch_cancel` | Same credits as Fetch; a 403 maps to `BATCH_ACCESS_DENIED` | | Browser | `browser_*` (start with `browser_navigate`) | Escalation-only; same backend as Browser Sessions | diff --git a/docs/capabilities.md b/docs/capabilities.md index 62b1476..ff49e05 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -18,7 +18,7 @@ Classification is based on the public Zenrows documentation: | Capability | Backend evidence | Status | | --- | --- | --- | | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | -| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (any prepared domain; on one not prepared yet the CLI reports REQS007 with the command to prepare it: `zenrows extract prepare --url `, which calls `POST /v1/extract/prepared-domains`), plus deprecated `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | available | +| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (any prepared domain; on one not prepared yet the CLI reports REQS007 as `EXTRACT_DOMAIN_NOT_ENABLED` with the command to prepare it: `zenrows extract prepare --url `, which calls `POST /v1/extract/prepared-domains`; a page Extract did not recognise on a prepared domain is `EXTRACT_PAGE_NOT_RECOGNISED`, which preparing does not change), plus deprecated `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | available | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header). Local JSONL spec validation + credit estimation work with no key. | available | | `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL, up to `depth` when set (optional; no depth limit when omitted), on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Subcommands: `create`, `get`, `list`, `results`, `content`, `download`, `stop`, `wait`. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | diff --git a/registry/capabilities.json b/registry/capabilities.json index 9596ba4..3a27ebf 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -17,7 +17,7 @@ "command": "zenrows extract", "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, css_extractor, outputs, response_type; autoparse deprecated)", "requiresAuth": true, - "notes": "Default is extract=auto, which works on any prepared domain with any key. On a domain not prepared yet (REQS007, legacy AUTH010) the CLI reports the error and does not retry; prepare the domain with `zenrows extract prepare --url [--wait]` (POST /v1/extract/prepared-domains: 1 to 5 example URLs, a few minutes, no credits) and check it with `zenrows extract prepare status `. CSS / outputs / markdown work on any domain; autoparse is deprecated and runs only with --autoparse." + "notes": "Default is extract=auto, which works on any prepared domain with any key. A REQS007 (legacy AUTH010) is reported, never retried, with the next step for its reason: on a domain not prepared yet (EXTRACT_DOMAIN_NOT_ENABLED), prepare it with `zenrows extract prepare --url [--wait]` (POST /v1/extract/prepared-domains: 1 to 5 example URLs, a few minutes, no credits) and check it with `zenrows extract prepare status `; on a page Extract did not recognise on a prepared domain (EXTRACT_PAGE_NOT_RECOGNISED), try a product, article or listing page, retry it once in a few minutes, or use --css, because preparing again does not change it. CSS / outputs / markdown work on any domain; autoparse is deprecated and runs only with --autoparse." }, "batch": { "key": "batch", diff --git a/skills/extract/SKILL.md b/skills/extract/SKILL.md index e2f69cb..79f3019 100644 --- a/skills/extract/SKILL.md +++ b/skills/extract/SKILL.md @@ -28,39 +28,45 @@ zenrows extract --validate # fail if not valid JSON - **CSS**: you know the exact fields/selectors and want determinism. - **Markdown/plaintext**: feed clean content to an LLM yourself. -## Domain not prepared yet (REQS007): prepare it, then extract again +## REQS007: prepare the domain, then extract again `extract=auto` works on any prepared domain, and you can prepare one yourself. -On a domain Extract hasn't prepared yet, `zenrows extract` fails with -`EXTRACT_DOMAIN_NOT_ENABLED` (API code `403 REQS007`; legacy `402 AUTH010`) and -prints the command to run. It does not retry with Autoparse. The flow: +When Extract refuses a page with `403 REQS007`, `zenrows extract` says which +of three cases it is and prints the next step. It never retries with Autoparse. + +| Error code | Meaning | Next step | +| --- | --- | --- | +| `EXTRACT_DOMAIN_NOT_ENABLED` | The domain is not prepared yet (also the legacy `402 AUTH010`). | Prepare it (below), then extract again. | +| `EXTRACT_PAGE_NOT_RECOGNISED` | The domain is prepared, but Extract did not recognise this page as one it can extract (a home, search, login or category page, or a transient failure). | Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use `--css`. Preparing the domain again does not change this. | +| `EXTRACT_DOMAIN_NOT_ALLOWED` | Extract is not enabled for the site. | Use `--css`, or contact Zenrows support to have the site enabled. Preparing does not change this. | + +Preparing a domain: ``` -zenrows extract https://example.com/p/1 # 1. fails: Extract hasn't prepared example.com yet +zenrows extract https://example.com/p/1 # 1. fails: EXTRACT_DOMAIN_NOT_ENABLED zenrows extract prepare example.com \ --url https://example.com/p/1 --url https://example.com/list --wait # 2. prepare it and wait until ready (a few minutes) zenrows extract https://example.com/p/1 # 3. extract again ``` -- `--url`: 1 to 5 real pages you want to extract, one per kind of page. -- Without `--wait` the command returns at once; then poll with - `zenrows extract prepare status example.com` (about every 30 seconds) until - it says `ready`. +- `--url`: 1 to 5 real pages you want to extract, one per page kind (for + example a product page and a listing page). +- Without `--wait` the command returns at once; then check about every 30 + seconds with `zenrows extract prepare status example.com` until it says + `ready`. `--wait` polls at the same pace. - The domain is the host: `shop.example.com` and `example.com` are prepared - separately; a leading `www.` is dropped. -- Preparing costs no credits. Each new domain uses one slot of the account's - monthly allowance; `prepare status` shows `used` / `limit` / `remaining`. + separately, and a leading `www.` is dropped. +- Preparing takes a few minutes and costs no credits. Each new domain uses one + slot of the account's monthly allowance; `prepare status` shows `used` / + `limit` / `remaining`. - `EXTRACT_PREPARE_OUT_OF_ALLOWANCE`: no slots left this month; use `--css`. - `EXTRACT_PREPARE_UNAVAILABLE`: retry in a few minutes, do not loop. + `EXTRACT_PREPARE_UNAVAILABLE`: retry in a few minutes; do not loop. A `failed` status prints the next step for its failure code and says whether the attempt used an allowance slot. -- **Do not loop.** If `prepare` answers `ready` at once (`already_prepared`), - the domain was prepared before; an extract that still fails with REQS007 - means that kind of page is not covered, and preparing again changes nothing. - The same goes for `EXTRACT_PAGE_NOT_RECOGNISED` (REQS007 on a prepared domain: - Extract did not recognise this page). Use `--css`, or contact Zenrows support - with the URL. +- A domain that is already prepared answers `ready` at once + (`already_prepared`) and uses no slot: run the extract. If it then fails with + `EXTRACT_PAGE_NOT_RECOGNISED`, follow that row of the table, not `prepare`. ## Rules - Validate on a single page before scaling across many URLs. diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index 0e6ca80..ba13729 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -79,8 +79,9 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | MCP | `zenrows mcp` | **available**: `https://mcp.zenrows.com/mcp` | - Extract: `extract=auto` works on any prepared domain. On one not prepared yet - (REQS007), run `zenrows extract prepare --url --wait`, then - extract again; see [[extract]]. + (`EXTRACT_DOMAIN_NOT_ENABLED`), run `zenrows extract prepare --url + --wait`, then extract again. `EXTRACT_PAGE_NOT_RECOGNISED` means the + domain is prepared but not this page: do not prepare again; see [[extract]]. - Batch: the cheapest path at scale (create, status, results). Estimate specs locally with no key. - Crawl: collects the URLs behind one start page, optionally with each page's HTML. - Browser: escalation-only, for pages Fetch can't handle; same backend as the MCP `browser_*` tools. diff --git a/src/cli/commands/extract-prepare.ts b/src/cli/commands/extract-prepare.ts index c5cad55..7973535 100644 --- a/src/cli/commands/extract-prepare.ts +++ b/src/cli/commands/extract-prepare.ts @@ -34,16 +34,21 @@ export const PREPARE_USAGE = " zenrows extract prepare status "; export const PREPARE_HELP = [ - "Prepare a domain so `zenrows extract ` (extract=auto) works on it. Use it after", - "an extract fails with REQS007 (Extract hasn't prepared the domain yet).", + "Prepare a domain so `zenrows extract ` (extract=auto) works on it. Use it when an", + "extract fails with EXTRACT_DOMAIN_NOT_ENABLED (REQS007: the domain is not prepared yet).", + "Not for EXTRACT_PAGE_NOT_RECOGNISED: that domain is already prepared, and preparing it", + "again does not change that answer.", "", - ` --url 1 to ${MAX_EXAMPLE_URLS} real pages you want to extract, one per kind of page (repeat --url)`, - " --wait poll until the domain is ready or failed (a few minutes); off by default", + ` --url 1 to ${MAX_EXAMPLE_URLS} real pages you want to extract, one per page kind (repeat --url)`, + " --wait poll every 30 seconds until the domain is ready or failed; off by default", ` --timeout give up waiting after this long (default ${DEFAULT_WAIT_TIMEOUT_MS / 1000})`, " status show progress and the account's monthly allowance", " --json structured result", "", - "Costs no credits. Each new domain uses one slot of the account's monthly allowance.", + "Preparing takes a few minutes and costs no credits. Each new domain uses one slot of the", + "account's monthly allowance; a domain that is already prepared answers ready at once and", + "uses none. The domain is the host (shop.example.com and example.com are separate); a", + "leading www. is dropped.", ].join("\n"); /** Injectable for tests: network, sleep and clock. */ @@ -132,7 +137,7 @@ function validateUrls(raw: unknown, domain: string): string[] { code: "INVALID_USAGE", message: `Pass 1 to ${MAX_EXAMPLE_URLS} example page URLs with --url (got ${urls.length}).`, likely_cause: "Preparation learns the domain from real pages you want to extract.", - next_action: "Add one --url per kind of page, e.g. a product page and a listing page.", + next_action: "Add one --url per page kind, e.g. a product page and a listing page.", suggested_commands: [`zenrows extract prepare ${domain} --url https://${domain}/`], }); } diff --git a/src/cli/commands/extract.ts b/src/cli/commands/extract.ts index 0aa7170..2c6c6ae 100644 --- a/src/cli/commands/extract.ts +++ b/src/cli/commands/extract.ts @@ -1,9 +1,10 @@ /** * `zenrows extract [flags]` → Extract (structured extraction on `/v1/`). * - * Default: `extract=auto` (site-tailored Extract). If Extract has not prepared - * the domain yet (REQS007 / AUTH010), the error names the prepare step; nothing - * is retried. `--autoparse` runs the deprecated Autoparse explicitly. Other + * Default: `extract=auto` (site-tailored Extract). On a REQS007 / AUTH010 the + * error names the next step for its reason (prepare the domain, or try another + * page); nothing is retried. `--autoparse` runs the deprecated Autoparse + * explicitly. Other * methods: `--css`, `--outputs`, `--output`. `zenrows extract prepare` prepares a * domain (see extract-prepare.ts). */ @@ -30,7 +31,7 @@ export const extract: Command = { help: [ "Methods:", " (default) extract=auto: site-tailored Extract on any prepared domain", - " (REQS007 = not prepared yet: run zenrows extract prepare, below)", + " (EXTRACT_DOMAIN_NOT_ENABLED = not prepared yet: run zenrows extract prepare, below)", " --autoparse deprecated general-purpose Autoparse (any domain)", " --css CSS selector map, e.g. '{\"title\":\"h1\",\"price\":\".price\"}'", " --outputs built-in output filters → JSON: emails, phone_numbers, headings,", diff --git a/src/core/errors.ts b/src/core/errors.ts index 2e26441..1a473e6 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -30,6 +30,7 @@ export type ErrorCode = | "EXTRACT_FAILED" | "EXTRACT_DOMAIN_NOT_ENABLED" | "EXTRACT_PAGE_NOT_RECOGNISED" + | "EXTRACT_DOMAIN_NOT_ALLOWED" | "EXTRACT_VALIDATION_FAILED" | "EXTRACT_PREPARE_INVALID" | "EXTRACT_PREPARE_NOT_FOUND" diff --git a/src/core/http.ts b/src/core/http.ts index e3117fc..20939c6 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -11,7 +11,7 @@ import { ENV_KEY, resolveApiKey } from "./auth.ts"; import { registerSecret } from "./logger.ts"; import { CLI_USER_AGENT } from "./config.ts"; import { agentClientHeader } from "./agent-client.ts"; -import { normalizeDomain, prepareCommand } from "./prepared-domains-api.ts"; +import { normalizeDomain, PAGE_NOT_RECOGNISED_STEPS, prepareCommand } from "./prepared-domains-api.ts"; export interface ScraperResult { status: number; @@ -301,41 +301,64 @@ export async function scrape( return result; } +/** Why a REQS007 refused, as the gateway's `reason` field names it. */ +export type RefusalReason = "domain_not_prepared" | "page_not_recognised" | "domain_not_enabled"; + +const REFUSAL_REASONS: readonly string[] = ["domain_not_prepared", "page_not_recognised", "domain_not_enabled"]; + /** - * REQS007 has two causes, which the problem body tells apart: the domain is not - * prepared ("Extract is not prepared for the requested domain (...)"), or it is - * prepared and Extract did not recognise this page ("Extract did not recognise - * this page on ...", detail "Extract serves ..."). Only the first is fixed by - * preparing the domain; preparing a prepared domain returns its existing ready - * record, so pointing there would loop. + * Which REQS007 this is. Only domain_not_prepared is fixed by preparing the + * domain: on page_not_recognised the domain is already prepared, and + * resubmitting it returns the existing ready record, so pointing there would + * loop. + * + * The gateway says it in the problem's `reason` field, read first. The sentence + * match below is only the fallback for a gateway deployed before that field + * existed, and reads the titles that gateway sends; remove it once every gateway + * sends `reason`. */ -export function isPageNotRecognised(body: string): boolean { +export function refusalReason(body: string): RefusalReason { + let j: { reason?: unknown; title?: unknown; detail?: unknown }; try { - const j = JSON.parse(body) as { title?: unknown; detail?: unknown }; - const title = typeof j.title === "string" ? j.title : ""; - const detail = typeof j.detail === "string" ? j.detail : ""; - return /did not recogni[sz]e this page/i.test(title) || /^Extract serves /.test(detail); + j = JSON.parse(body) as typeof j; } catch { - return false; + return "domain_not_prepared"; } + if (typeof j.reason === "string" && REFUSAL_REASONS.includes(j.reason)) return j.reason as RefusalReason; + const title = typeof j.title === "string" ? j.title : ""; + const detail = typeof j.detail === "string" ? j.detail : ""; + if (/did not recogni[sz]e this page/i.test(title) || /^Extract serves /.test(detail)) return "page_not_recognised"; + if (/^Extract is not enabled/i.test(title)) return "domain_not_enabled"; + return "domain_not_prepared"; } /** - * REQS007 (or the legacy AUTH010) on extract=auto. On a domain not prepared yet - * the message names the exact command to prepare it, because agents copy it; on - * a page a prepared domain does not recognise it names --css instead. Neither - * offers Autoparse. + * REQS007 (or the legacy AUTH010) on extract=auto. Each reason names its own + * next step, because agents copy it: the exact prepare command for a domain not + * prepared yet, the gateway's advice in the gateway's order for a page it did not + * recognise. None offers Autoparse. */ export function extractDomainNotPrepared(url: string, body: string): ToolkitError { const domain = normalizeDomain(url); const detail = zrErrorDetail(body); const css = `zenrows extract ${url} --css '{"title":"h1"}'`; - if (isPageNotRecognised(body)) { + const reason = refusalReason(body); + if (reason === "page_not_recognised") { return new ToolkitError({ code: "EXTRACT_PAGE_NOT_RECOGNISED", - message: `Extract serves ${domain} but did not recognise this page. Preparing the domain again does not help; use --css, or contact Zenrows support with the URL.`, + message: `Extract serves ${domain} but did not recognise this page as one it can extract (it may be a home, search, login or category page). ${PAGE_NOT_RECOGNISED_STEPS} The request was not charged.`, likely_cause: detail ?? `Extract serves ${domain} but did not recognise this page.`, - next_action: "Extract this page with --css and explicit selectors, or contact Zenrows support with the URL.", + next_action: PAGE_NOT_RECOGNISED_STEPS, + suggested_commands: [css], + server_code: "REQS007", + }); + } + if (reason === "domain_not_enabled") { + return new ToolkitError({ + code: "EXTRACT_DOMAIN_NOT_ALLOWED", + message: `Extract is not enabled for ${domain}, and preparing it does not change this.`, + likely_cause: detail ?? `Extract is not enabled for ${domain}.`, + next_action: "Extract this page with --css and explicit selectors, or contact Zenrows support to have the site enabled.", suggested_commands: [css], server_code: "REQS007", }); @@ -345,8 +368,9 @@ export function extractDomainNotPrepared(url: string, body: string): ToolkitErro code: "EXTRACT_DOMAIN_NOT_ENABLED", message: `Extract hasn't prepared ${domain} yet. Prepare it with \`${prepare}\` (a few example URLs, a few minutes, no credits), then retry; meanwhile use --css.`, likely_cause: detail ?? `Extract has not prepared ${domain} yet.`, - next_action: `Run \`${prepare} --wait\` (add up to 5 --url pages, one per kind of page you want), then retry \`zenrows extract ${url}\`. Without --wait, check progress with \`zenrows extract prepare status ${domain}\`. Meanwhile extract with --css and explicit selectors.`, + next_action: `Run \`${prepare} --wait\` (up to 5 --url pages, one per page kind you want), then retry \`zenrows extract ${url}\`. Without --wait, check progress about every 30 seconds with \`zenrows extract prepare status ${domain}\`. Meanwhile extract with --css and explicit selectors.`, suggested_commands: [`${prepare} --wait`, `zenrows extract prepare status ${domain}`, css], + server_code: "REQS007", }); } diff --git a/src/core/prepared-domains-api.ts b/src/core/prepared-domains-api.ts index af99be6..9f1c9e3 100644 --- a/src/core/prepared-domains-api.ts +++ b/src/core/prepared-domains-api.ts @@ -307,13 +307,19 @@ export function failureNextStep(domain: string, failure: PreparedDomain["failure } /** - * What an already-prepared domain means for an extract that answered REQS007: - * that kind of page is not covered, and preparing again does not change it (the - * API returns the existing ready record). Said instead of "run the extract - * again", which would loop. + * What to do when a page on a prepared domain answers REQS007 with reason + * page_not_recognised, in the gateway's own order. On a prepared domain Extract + * learns a new kind of page on its first request, so that answer means this page + * is not one Extract extracts (a home, search, login or category page) or a + * transient failure kept it from trying, never that the domain needs preparing + * again: resubmitting returns the existing ready record, so pointing there would + * loop. */ -export const PAGE_KIND_NOT_COVERED = - "If the extract still fails with REQS007 on this domain, that kind of page is not covered and preparing again does not change it: use --css, or contact Zenrows support with the URL."; +export const PAGE_NOT_RECOGNISED_STEPS = + "Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use --css with explicit selectors. Preparing the domain again does not change this."; + +/** Said once a domain is ready, so a user who then meets EXTRACT_PAGE_NOT_RECOGNISED does not resubmit. */ +const IF_PAGE_NOT_RECOGNISED = `If the extract then fails with EXTRACT_PAGE_NOT_RECOGNISED, Extract did not recognise that page. ${PAGE_NOT_RECOGNISED_STEPS}`; /** * The next step for a status record. `alreadyReady` is set when a submit answered @@ -321,9 +327,10 @@ export const PAGE_KIND_NOT_COVERED = */ export function statusNextStep(record: PreparedDomain, opts: { exampleUrl?: string; alreadyReady?: boolean } = {}): string { if (record.status === "ready") { - if (opts.alreadyReady) return `${record.domain} was already prepared, so nothing new was submitted. ${PAGE_KIND_NOT_COVERED}`; - return `Ready. Run the extract again${opts.exampleUrl ? `: zenrows extract ${opts.exampleUrl}` : ""}. ${PAGE_KIND_NOT_COVERED}`; + const extract = opts.exampleUrl ? ` (\`zenrows extract ${opts.exampleUrl}\`)` : ""; + if (opts.alreadyReady) return `${record.domain} is already prepared, so nothing new was submitted: run the extract${extract}. ${IF_PAGE_NOT_RECOGNISED}`; + return `${record.domain} is ready: run the extract again${extract}. ${IF_PAGE_NOT_RECOGNISED}`; } if (record.status === "failed") return failureNextStep(record.domain, record.failure); - return `Preparing (a few minutes). Check with: zenrows extract prepare status ${record.domain}`; + return `Preparing; it takes a few minutes. Check about every 30 seconds with: zenrows extract prepare status ${record.domain}`; } diff --git a/templates/extract-pricing-pages/README.md b/templates/extract-pricing-pages/README.md index d7ae076..f7d93c4 100644 --- a/templates/extract-pricing-pages/README.md +++ b/templates/extract-pricing-pages/README.md @@ -4,7 +4,7 @@ Extract structured pricing fields from product pages with Zenrows Extract. ```bash # Default: extract=auto on a prepared domain (site-tailored JSON). -# Not prepared yet (REQS007)? zenrows extract prepare --url --wait +# Not prepared yet (EXTRACT_DOMAIN_NOT_ENABLED)? zenrows extract prepare --url --wait zenrows extract https://www.owler.com/company/meltwater # A precise CSS selector map, on any domain diff --git a/tests/capabilities.test.ts b/tests/capabilities.test.ts index d6c33b2..846fcf5 100644 --- a/tests/capabilities.test.ts +++ b/tests/capabilities.test.ts @@ -18,6 +18,8 @@ test("extract notes name the prepare step and no automatic autoparse fallback", assert.match(notes, /POST \/v1\/extract\/prepared-domains/); assert.match(notes, /zenrows extract prepare --url /); assert.match(notes, /zenrows extract prepare status /); + assert.match(notes, /EXTRACT_PAGE_NOT_RECOGNISED/); + assert.match(notes, /preparing again does not change it/); assert.match(notes, /autoparse is deprecated/); assert.doesNotMatch(notes, /falls? back/i); }); diff --git a/tests/extract-outputs.test.ts b/tests/extract-outputs.test.ts index 67bc83e..5d5f535 100644 --- a/tests/extract-outputs.test.ts +++ b/tests/extract-outputs.test.ts @@ -6,6 +6,7 @@ import { isEmptyData, runExtract } from "../src/adapters/extract.ts"; import { defaultConfig } from "../src/core/config.ts"; import { defaultPolicy } from "../src/core/policy.ts"; import { ToolkitError } from "../src/core/errors.ts"; +import { refusalReason } from "../src/core/http.ts"; const cfg = defaultConfig(); @@ -203,34 +204,91 @@ test("runExtract reports REQS007 with the prepare step and does not retry", asyn }); }); -test("runExtract on REQS007 for a page a prepared domain does not recognise points to --css, not prepare", async () => { - // The gateway's body for a prepared domain whose page it did not recognise. - const body = JSON.stringify({ - code: "REQS007", - title: "Extract did not recognise this page on x.com (REQS007)", - detail: "Extract serves x.com, but did not recognise this page as one it can extract.", - status: 403, - }); +/** A REQS007 the way the gateway sends it: with `reason`, or (an older gateway) without. */ +function reqs007(kind: "domain" | "page" | "allowlist", withReason: boolean): string { + const bodies = { + domain: { + title: "Extract is not prepared for the requested domain (x.com) (REQS007)", + detail: "Extract has not been prepared for x.com yet.", + reason: "domain_not_prepared", + }, + page: { + title: "Extract did not recognise this page on x.com (REQS007)", + detail: "Extract serves x.com, but did not recognise this page as one it can extract.", + reason: "page_not_recognised", + }, + allowlist: { + title: "Extract is not enabled for the requested domain (x.com) (REQS007)", + detail: "Extract is not enabled for the requested domain (x.com) yet.", + reason: "domain_not_enabled", + }, + } as const; + const { reason, ...rest } = bodies[kind]; + return JSON.stringify({ code: "REQS007", status: 403, ...rest, ...(withReason ? { reason } : {}) }); +} + +async function reqs007Error(body: string, url = "https://x.com/search?q=a"): Promise { let calls = 0; + let caught: unknown; await withFetchHandler(() => { calls += 1; return new Response(body, { status: 403, headers: { "content-type": "application/json" } }); }, async () => { - await assert.rejects( - () => runExtract({ url: "https://x.com/search?q=a" }, cfg, defaultPolicy(), "testkey"), - (e: unknown) => { - assert.ok(e instanceof ToolkitError); - assert.equal(e.code, "EXTRACT_PAGE_NOT_RECOGNISED"); - assert.match(e.message, /Preparing the domain again does not help/); - const shape = JSON.stringify(e.toJSON()); - assert.doesNotMatch(shape, /extract prepare/, "no prepare command: it would answer ready and loop"); - assert.doesNotMatch(shape, /autoparse/); - assert.match(shape, /--css/); - return true; - }, + caught = await runExtract({ url }, cfg, defaultPolicy(), "testkey").then( + () => undefined, + (e: unknown) => e, ); - assert.equal(calls, 1); }); + assert.equal(calls, 1, "a REQS007 is never retried"); + assert.ok(caught instanceof ToolkitError); + return caught; +} + +for (const withReason of [true, false]) { + const via = withReason ? "the reason field" : "the title (gateway without the reason field)"; + + test(`REQS007 page_not_recognised, read from ${via}: the gateway's advice in order, no prepare command`, async () => { + const e = await reqs007Error(reqs007("page", withReason)); + assert.equal(e.code, "EXTRACT_PAGE_NOT_RECOGNISED"); + let last = -1; + for (const step of [ + /product, article or listing page/, + /retry it once in a few minutes; do not retry in a loop/, + /--css with explicit selectors/, + /Preparing the domain again does not change this/, + ]) { + const at = e.message.search(step); + assert.ok(at > last, `${step} missing or out of order in: ${e.message}`); + last = at; + } + const shape = JSON.stringify(e.toJSON()); + assert.doesNotMatch(shape, /extract prepare/, "no prepare command: it would answer ready and loop"); + assert.doesNotMatch(shape, /autoparse|contact Zenrows support/); + }); + + test(`REQS007 domain_not_enabled, read from ${via}: no prepare command, --css`, async () => { + const e = await reqs007Error(reqs007("allowlist", withReason)); + assert.equal(e.code, "EXTRACT_DOMAIN_NOT_ALLOWED"); + assert.match(e.message, /preparing it does not change this/); + const shape = JSON.stringify(e.toJSON()); + assert.doesNotMatch(shape, /extract prepare|autoparse/); + assert.match(shape, /--css/); + }); + + test(`REQS007 domain_not_prepared, read from ${via}: the prepare command`, async () => { + const e = await reqs007Error(reqs007("domain", withReason), "https://www.x.com/p/1"); + assert.equal(e.code, "EXTRACT_DOMAIN_NOT_ENABLED"); + assert.equal(e.toJSON().suggested_commands?.[0], "zenrows extract prepare x.com --url https://www.x.com/p/1 --wait"); + }); +} + +test("refusalReason trusts the field over the sentence, and ignores a reason it does not know", () => { + assert.equal( + refusalReason(JSON.stringify({ title: "Extract did not recognise this page on x.com", reason: "domain_not_prepared" })), + "domain_not_prepared", + ); + assert.equal(refusalReason(JSON.stringify({ reason: "something_new", detail: "Extract serves x.com, but" })), "page_not_recognised"); + assert.equal(refusalReason("not json"), "domain_not_prepared"); }); test("runExtract reports legacy AUTH010 like REQS007 and does not retry", async () => { diff --git a/tests/extract-prepare.test.ts b/tests/extract-prepare.test.ts index 7063912..fe0e657 100644 --- a/tests/extract-prepare.test.ts +++ b/tests/extract-prepare.test.ts @@ -212,7 +212,8 @@ test("prepare status keeps the record when the allowance call fails", async () = assert.equal(code, 0); assert.equal(out.status, "ready"); assert.equal(out.allowance, undefined); - assert.match(out.next, /Run the extract again/); + assert.match(out.next, /example\.com is ready: run the extract again/); + assert.match(out.next, /EXTRACT_PAGE_NOT_RECOGNISED/); }); test("prepare status on 404 says to prepare the domain first", async () => { @@ -239,14 +240,14 @@ test("failureNextStep: the step comes from failure.code, the slot sentence from assert.doesNotMatch(failureNextStep("x.com", { code: "unpreparable" }), /slot/); }); -test("prepare answering ready at once says the domain was already prepared, not to run the extract again", async () => { +test("prepare answering ready at once says the domain is already prepared and what a page refusal then means", async () => { const { calls, fetchImpl } = stub({ [`POST ${BASE}`]: () => res({ ...RECORD, status: "ready", state: "ready" }, 202) }); const { code, out } = await run(["example.com", "--url", "https://example.com/search", "--wait"], fetchImpl); assert.equal(code, 0); assert.equal(out.already_prepared, true); - assert.match(out.next, /already prepared/); - assert.match(out.next, /use --css/); - assert.doesNotMatch(out.next, /Run the extract again/); + assert.match(out.next, /already prepared, so nothing new was submitted: run the extract \(`zenrows extract https:\/\/example\.com\/search`\)\./); + assert.match(out.next, /Preparing the domain again does not change this/); + assert.doesNotMatch(out.next, /not covered|contact Zenrows support|autoparse/); assert.equal(calls.length, 1, "--wait does not poll a domain that is already ready"); }); diff --git a/workflows/competitor-intelligence/WORKFLOW.md b/workflows/competitor-intelligence/WORKFLOW.md index 99397ce..f797a26 100644 --- a/workflows/competitor-intelligence/WORKFLOW.md +++ b/workflows/competitor-intelligence/WORKFLOW.md @@ -20,7 +20,7 @@ simple per-URL loop). ``` zenrows fetch https://competitor.example/products ``` -2. **Validate extraction** on one page (on a domain not prepared yet, run `zenrows extract prepare --url --wait` first). +2. **Validate extraction** on one page (on a domain not prepared yet, `EXTRACT_DOMAIN_NOT_ENABLED`, run `zenrows extract prepare --url --wait` first). ``` zenrows extract https://competitor.example/p/123 --validate ``` From 7fd8bc5311ddbcf990fcce09df9c3d9f48840bb9 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 15:11:42 +0000 Subject: [PATCH 5/8] fix(extract): describe an unrecognised page as one with nothing to extract, not a search or category page Production extracts search and category pages on prepared domains: a prepared domain learns any new kind of page on its first request. The example list "a home, search, login or category page" for a page Extract did not recognise was therefore wrong for two of its four entries, and it contradicted the advice to send a search results URL as an example. The page-not-recognised wording now says "a page with nothing to extract, such as a home page", everywhere it appears. --- agent-plugin/skills/extract/SKILL.md | 2 +- skills/extract/SKILL.md | 2 +- src/core/http.ts | 2 +- src/core/prepared-domains-api.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/agent-plugin/skills/extract/SKILL.md b/agent-plugin/skills/extract/SKILL.md index e2e7ccc..a3be1e6 100644 --- a/agent-plugin/skills/extract/SKILL.md +++ b/agent-plugin/skills/extract/SKILL.md @@ -41,7 +41,7 @@ unless you set `fallback_autoparse: true`. | `reason` | Meaning | Next step | | --- | --- | --- | | `domain_not_prepared` | The domain is not prepared yet. | `prepare_domain` (the error's `next_tool` holds its arguments), then poll, then `extract` again. | -| `page_not_recognised` | The domain is prepared, but Extract did not recognise this page as one it can extract (a home, search, login or category page, or a transient failure). | Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use `mode=css`. Do not call `prepare_domain`: preparing again does not change this. | +| `page_not_recognised` | The domain is prepared, but Extract did not recognise this page as one it can extract (a page with nothing to extract, such as a home page, or a transient failure). | Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use `mode=css`. Do not call `prepare_domain`: preparing again does not change this. | | `domain_not_enabled` | Extract is not enabled for the site. | Use `mode=css`, or contact Zenrows support to have the site enabled. Preparing does not change this. | Preparing a domain: diff --git a/skills/extract/SKILL.md b/skills/extract/SKILL.md index 79f3019..1128f6f 100644 --- a/skills/extract/SKILL.md +++ b/skills/extract/SKILL.md @@ -37,7 +37,7 @@ of three cases it is and prints the next step. It never retries with Autoparse. | Error code | Meaning | Next step | | --- | --- | --- | | `EXTRACT_DOMAIN_NOT_ENABLED` | The domain is not prepared yet (also the legacy `402 AUTH010`). | Prepare it (below), then extract again. | -| `EXTRACT_PAGE_NOT_RECOGNISED` | The domain is prepared, but Extract did not recognise this page as one it can extract (a home, search, login or category page, or a transient failure). | Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use `--css`. Preparing the domain again does not change this. | +| `EXTRACT_PAGE_NOT_RECOGNISED` | The domain is prepared, but Extract did not recognise this page as one it can extract (a page with nothing to extract, such as a home page, or a transient failure). | Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use `--css`. Preparing the domain again does not change this. | | `EXTRACT_DOMAIN_NOT_ALLOWED` | Extract is not enabled for the site. | Use `--css`, or contact Zenrows support to have the site enabled. Preparing does not change this. | Preparing a domain: diff --git a/src/core/http.ts b/src/core/http.ts index 20939c6..7466ab4 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -346,7 +346,7 @@ export function extractDomainNotPrepared(url: string, body: string): ToolkitErro if (reason === "page_not_recognised") { return new ToolkitError({ code: "EXTRACT_PAGE_NOT_RECOGNISED", - message: `Extract serves ${domain} but did not recognise this page as one it can extract (it may be a home, search, login or category page). ${PAGE_NOT_RECOGNISED_STEPS} The request was not charged.`, + message: `Extract serves ${domain} but did not recognise this page as one it can extract (it may be a page with nothing to extract, such as a home page). ${PAGE_NOT_RECOGNISED_STEPS} The request was not charged.`, likely_cause: detail ?? `Extract serves ${domain} but did not recognise this page.`, next_action: PAGE_NOT_RECOGNISED_STEPS, suggested_commands: [css], diff --git a/src/core/prepared-domains-api.ts b/src/core/prepared-domains-api.ts index 9f1c9e3..69bb5e0 100644 --- a/src/core/prepared-domains-api.ts +++ b/src/core/prepared-domains-api.ts @@ -310,7 +310,7 @@ export function failureNextStep(domain: string, failure: PreparedDomain["failure * What to do when a page on a prepared domain answers REQS007 with reason * page_not_recognised, in the gateway's own order. On a prepared domain Extract * learns a new kind of page on its first request, so that answer means this page - * is not one Extract extracts (a home, search, login or category page) or a + * is not one Extract extracts (a page with nothing to extract, such as a home page) or a * transient failure kept it from trying, never that the domain needs preparing * again: resubmitting returns the existing ready record, so pointing there would * loop. From 8c1bf18f8bb7d1b61822d8569b474af0b37a2405 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 16:15:59 +0000 Subject: [PATCH 6/8] feat(extract): name the REQS007 codes for what they mean A domain Extract has not prepared yet is now EXTRACT_DOMAIN_NOT_PREPARED (it was EXTRACT_DOMAIN_NOT_ENABLED), and a domain Extract is not enabled for is now EXTRACT_DOMAIN_NOT_ENABLED (it was EXTRACT_DOMAIN_NOT_ALLOWED). Help, skills, docs, the capability registry and tests use the new names. Code that compared against the old EXTRACT_DOMAIN_NOT_ENABLED did so to fall back to autoparse, and that fallback no longer exists. --- README.md | 2 +- docs/capabilities.md | 2 +- registry/capabilities.json | 2 +- skills/extract/SKILL.md | 6 +++--- skills/zenrows/SKILL.md | 2 +- src/adapters/extract.ts | 2 +- src/cli/commands/extract-prepare.ts | 2 +- src/cli/commands/extract.ts | 2 +- src/core/errors.ts | 4 ++-- src/core/http.ts | 4 ++-- templates/extract-pricing-pages/README.md | 2 +- tests/extract-outputs.test.ts | 8 ++++---- tests/http.test.ts | 14 +++++++------- workflows/competitor-intelligence/WORKFLOW.md | 2 +- 14 files changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 1e58878..68a2074 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ zenrows extract --output markdown ``` On a domain Extract hasn't prepared yet, `zenrows extract` fails with -`EXTRACT_DOMAIN_NOT_ENABLED` (API code REQS007) and prints the command to +`EXTRACT_DOMAIN_NOT_PREPARED` (API code REQS007) and prints the command to prepare it. Preparing takes a few minutes and costs no credits; each new domain uses one slot of the monthly allowance. `EXTRACT_PAGE_NOT_RECOGNISED` (also REQS007) is different: the domain is prepared, but this page is not one Extract diff --git a/docs/capabilities.md b/docs/capabilities.md index ff49e05..84bf420 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -18,7 +18,7 @@ Classification is based on the public Zenrows documentation: | Capability | Backend evidence | Status | | --- | --- | --- | | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | -| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (any prepared domain; on one not prepared yet the CLI reports REQS007 as `EXTRACT_DOMAIN_NOT_ENABLED` with the command to prepare it: `zenrows extract prepare --url `, which calls `POST /v1/extract/prepared-domains`; a page Extract did not recognise on a prepared domain is `EXTRACT_PAGE_NOT_RECOGNISED`, which preparing does not change), plus deprecated `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | available | +| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (any prepared domain; on one not prepared yet the CLI reports REQS007 as `EXTRACT_DOMAIN_NOT_PREPARED` with the command to prepare it: `zenrows extract prepare --url `, which calls `POST /v1/extract/prepared-domains`; a page Extract did not recognise on a prepared domain is `EXTRACT_PAGE_NOT_RECOGNISED`, which preparing does not change), plus deprecated `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | available | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header). Local JSONL spec validation + credit estimation work with no key. | available | | `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL, up to `depth` when set (optional; no depth limit when omitted), on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Subcommands: `create`, `get`, `list`, `results`, `content`, `download`, `stop`, `wait`. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | diff --git a/registry/capabilities.json b/registry/capabilities.json index 3a27ebf..13b2f71 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -17,7 +17,7 @@ "command": "zenrows extract", "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, css_extractor, outputs, response_type; autoparse deprecated)", "requiresAuth": true, - "notes": "Default is extract=auto, which works on any prepared domain with any key. A REQS007 (legacy AUTH010) is reported, never retried, with the next step for its reason: on a domain not prepared yet (EXTRACT_DOMAIN_NOT_ENABLED), prepare it with `zenrows extract prepare --url [--wait]` (POST /v1/extract/prepared-domains: 1 to 5 example URLs, a few minutes, no credits) and check it with `zenrows extract prepare status `; on a page Extract did not recognise on a prepared domain (EXTRACT_PAGE_NOT_RECOGNISED), try a product, article or listing page, retry it once in a few minutes, or use --css, because preparing again does not change it. CSS / outputs / markdown work on any domain; autoparse is deprecated and runs only with --autoparse." + "notes": "Default is extract=auto, which works on any prepared domain with any key. A REQS007 (legacy AUTH010) is reported, never retried, with the next step for its reason: on a domain not prepared yet (EXTRACT_DOMAIN_NOT_PREPARED), prepare it with `zenrows extract prepare --url [--wait]` (POST /v1/extract/prepared-domains: 1 to 5 example URLs, a few minutes, no credits) and check it with `zenrows extract prepare status `; on a page Extract did not recognise on a prepared domain (EXTRACT_PAGE_NOT_RECOGNISED), try a product, article or listing page, retry it once in a few minutes, or use --css, because preparing again does not change it. CSS / outputs / markdown work on any domain; autoparse is deprecated and runs only with --autoparse." }, "batch": { "key": "batch", diff --git a/skills/extract/SKILL.md b/skills/extract/SKILL.md index 1128f6f..00d4b96 100644 --- a/skills/extract/SKILL.md +++ b/skills/extract/SKILL.md @@ -36,14 +36,14 @@ of three cases it is and prints the next step. It never retries with Autoparse. | Error code | Meaning | Next step | | --- | --- | --- | -| `EXTRACT_DOMAIN_NOT_ENABLED` | The domain is not prepared yet (also the legacy `402 AUTH010`). | Prepare it (below), then extract again. | +| `EXTRACT_DOMAIN_NOT_PREPARED` | The domain is not prepared yet (also the legacy `402 AUTH010`). | Prepare it (below), then extract again. | | `EXTRACT_PAGE_NOT_RECOGNISED` | The domain is prepared, but Extract did not recognise this page as one it can extract (a page with nothing to extract, such as a home page, or a transient failure). | Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use `--css`. Preparing the domain again does not change this. | -| `EXTRACT_DOMAIN_NOT_ALLOWED` | Extract is not enabled for the site. | Use `--css`, or contact Zenrows support to have the site enabled. Preparing does not change this. | +| `EXTRACT_DOMAIN_NOT_ENABLED` | Extract is not enabled for the site. | Use `--css`, or contact Zenrows support to have the site enabled. Preparing does not change this. | Preparing a domain: ``` -zenrows extract https://example.com/p/1 # 1. fails: EXTRACT_DOMAIN_NOT_ENABLED +zenrows extract https://example.com/p/1 # 1. fails: EXTRACT_DOMAIN_NOT_PREPARED zenrows extract prepare example.com \ --url https://example.com/p/1 --url https://example.com/list --wait # 2. prepare it and wait until ready (a few minutes) diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index ba13729..ea23d6d 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -79,7 +79,7 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | MCP | `zenrows mcp` | **available**: `https://mcp.zenrows.com/mcp` | - Extract: `extract=auto` works on any prepared domain. On one not prepared yet - (`EXTRACT_DOMAIN_NOT_ENABLED`), run `zenrows extract prepare --url + (`EXTRACT_DOMAIN_NOT_PREPARED`), run `zenrows extract prepare --url --wait`, then extract again. `EXTRACT_PAGE_NOT_RECOGNISED` means the domain is prepared but not this page: do not prepare again; see [[extract]]. - Batch: the cheapest path at scale (create, status, results). Estimate specs locally with no key. diff --git a/src/adapters/extract.ts b/src/adapters/extract.ts index f673521..8f72dd1 100644 --- a/src/adapters/extract.ts +++ b/src/adapters/extract.ts @@ -100,7 +100,7 @@ export async function runExtract( try { return await runExtractOnce({ ...opts, method: "extract" }, config, policy, apiKey); } catch (err) { - if (err instanceof ToolkitError && err.code === "EXTRACT_DOMAIN_NOT_ENABLED") { + if (err instanceof ToolkitError && err.code === "EXTRACT_DOMAIN_NOT_PREPARED") { const outcome = await runExtractOnce({ ...opts, method: "autoparse" }, config, policy, apiKey); return { ...outcome, fellBackToAutoparse: true }; } diff --git a/src/cli/commands/extract-prepare.ts b/src/cli/commands/extract-prepare.ts index 7973535..fb3d3b6 100644 --- a/src/cli/commands/extract-prepare.ts +++ b/src/cli/commands/extract-prepare.ts @@ -35,7 +35,7 @@ export const PREPARE_USAGE = export const PREPARE_HELP = [ "Prepare a domain so `zenrows extract ` (extract=auto) works on it. Use it when an", - "extract fails with EXTRACT_DOMAIN_NOT_ENABLED (REQS007: the domain is not prepared yet).", + "extract fails with EXTRACT_DOMAIN_NOT_PREPARED (REQS007: the domain is not prepared yet).", "Not for EXTRACT_PAGE_NOT_RECOGNISED: that domain is already prepared, and preparing it", "again does not change that answer.", "", diff --git a/src/cli/commands/extract.ts b/src/cli/commands/extract.ts index 2c6c6ae..cd08dff 100644 --- a/src/cli/commands/extract.ts +++ b/src/cli/commands/extract.ts @@ -31,7 +31,7 @@ export const extract: Command = { help: [ "Methods:", " (default) extract=auto: site-tailored Extract on any prepared domain", - " (EXTRACT_DOMAIN_NOT_ENABLED = not prepared yet: run zenrows extract prepare, below)", + " (EXTRACT_DOMAIN_NOT_PREPARED = not prepared yet: run zenrows extract prepare, below)", " --autoparse deprecated general-purpose Autoparse (any domain)", " --css CSS selector map, e.g. '{\"title\":\"h1\",\"price\":\".price\"}'", " --outputs built-in output filters → JSON: emails, phone_numbers, headings,", diff --git a/src/core/errors.ts b/src/core/errors.ts index 1a473e6..0ecc0bf 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -28,9 +28,9 @@ export type ErrorCode = | "FETCH_FAILED" | "FETCH_EMPTY_RESPONSE" | "EXTRACT_FAILED" - | "EXTRACT_DOMAIN_NOT_ENABLED" + | "EXTRACT_DOMAIN_NOT_PREPARED" | "EXTRACT_PAGE_NOT_RECOGNISED" - | "EXTRACT_DOMAIN_NOT_ALLOWED" + | "EXTRACT_DOMAIN_NOT_ENABLED" | "EXTRACT_VALIDATION_FAILED" | "EXTRACT_PREPARE_INVALID" | "EXTRACT_PREPARE_NOT_FOUND" diff --git a/src/core/http.ts b/src/core/http.ts index 7466ab4..0806193 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -355,7 +355,7 @@ export function extractDomainNotPrepared(url: string, body: string): ToolkitErro } if (reason === "domain_not_enabled") { return new ToolkitError({ - code: "EXTRACT_DOMAIN_NOT_ALLOWED", + code: "EXTRACT_DOMAIN_NOT_ENABLED", message: `Extract is not enabled for ${domain}, and preparing it does not change this.`, likely_cause: detail ?? `Extract is not enabled for ${domain}.`, next_action: "Extract this page with --css and explicit selectors, or contact Zenrows support to have the site enabled.", @@ -365,7 +365,7 @@ export function extractDomainNotPrepared(url: string, body: string): ToolkitErro } const prepare = prepareCommand(url); return new ToolkitError({ - code: "EXTRACT_DOMAIN_NOT_ENABLED", + code: "EXTRACT_DOMAIN_NOT_PREPARED", message: `Extract hasn't prepared ${domain} yet. Prepare it with \`${prepare}\` (a few example URLs, a few minutes, no credits), then retry; meanwhile use --css.`, likely_cause: detail ?? `Extract has not prepared ${domain} yet.`, next_action: `Run \`${prepare} --wait\` (up to 5 --url pages, one per page kind you want), then retry \`zenrows extract ${url}\`. Without --wait, check progress about every 30 seconds with \`zenrows extract prepare status ${domain}\`. Meanwhile extract with --css and explicit selectors.`, diff --git a/templates/extract-pricing-pages/README.md b/templates/extract-pricing-pages/README.md index f7d93c4..408e1e3 100644 --- a/templates/extract-pricing-pages/README.md +++ b/templates/extract-pricing-pages/README.md @@ -4,7 +4,7 @@ Extract structured pricing fields from product pages with Zenrows Extract. ```bash # Default: extract=auto on a prepared domain (site-tailored JSON). -# Not prepared yet (EXTRACT_DOMAIN_NOT_ENABLED)? zenrows extract prepare --url --wait +# Not prepared yet (EXTRACT_DOMAIN_NOT_PREPARED)? zenrows extract prepare --url --wait zenrows extract https://www.owler.com/company/meltwater # A precise CSS selector map, on any domain diff --git a/tests/extract-outputs.test.ts b/tests/extract-outputs.test.ts index 5d5f535..7629ac9 100644 --- a/tests/extract-outputs.test.ts +++ b/tests/extract-outputs.test.ts @@ -186,7 +186,7 @@ test("runExtract reports REQS007 with the prepare step and does not retry", asyn () => runExtract({ url: "https://www.x.com/item/1" }, cfg, defaultPolicy(), "testkey"), (e: unknown) => { assert.ok(e instanceof ToolkitError); - assert.equal(e.code, "EXTRACT_DOMAIN_NOT_ENABLED"); + assert.equal(e.code, "EXTRACT_DOMAIN_NOT_PREPARED"); assert.equal( e.message, "Extract hasn't prepared x.com yet. Prepare it with `zenrows extract prepare x.com --url https://www.x.com/item/1` (a few example URLs, a few minutes, no credits), then retry; meanwhile use --css.", @@ -268,7 +268,7 @@ for (const withReason of [true, false]) { test(`REQS007 domain_not_enabled, read from ${via}: no prepare command, --css`, async () => { const e = await reqs007Error(reqs007("allowlist", withReason)); - assert.equal(e.code, "EXTRACT_DOMAIN_NOT_ALLOWED"); + assert.equal(e.code, "EXTRACT_DOMAIN_NOT_ENABLED"); assert.match(e.message, /preparing it does not change this/); const shape = JSON.stringify(e.toJSON()); assert.doesNotMatch(shape, /extract prepare|autoparse/); @@ -277,7 +277,7 @@ for (const withReason of [true, false]) { test(`REQS007 domain_not_prepared, read from ${via}: the prepare command`, async () => { const e = await reqs007Error(reqs007("domain", withReason), "https://www.x.com/p/1"); - assert.equal(e.code, "EXTRACT_DOMAIN_NOT_ENABLED"); + assert.equal(e.code, "EXTRACT_DOMAIN_NOT_PREPARED"); assert.equal(e.toJSON().suggested_commands?.[0], "zenrows extract prepare x.com --url https://www.x.com/p/1 --wait"); }); } @@ -301,7 +301,7 @@ test("runExtract reports legacy AUTH010 like REQS007 and does not retry", async () => runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"), (e: unknown) => e instanceof ToolkitError && - e.code === "EXTRACT_DOMAIN_NOT_ENABLED" && + e.code === "EXTRACT_DOMAIN_NOT_PREPARED" && e.message.startsWith("Extract hasn't prepared x.com yet. Prepare it with `zenrows extract prepare x.com --url https://x.com`"), ); assert.equal(calls.length, 1, "AUTH010 must not be retried with autoparse"); diff --git a/tests/http.test.ts b/tests/http.test.ts index fabeea1..3923ed0 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -138,7 +138,7 @@ test("scrape does NOT advise a (billed) retry when the target cannot be resolved ); }); -test("scrape maps AUTH010 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async () => { +test("scrape maps AUTH010 on extract=auto to EXTRACT_DOMAIN_NOT_PREPARED", async () => { const AUTH010 = JSON.stringify({ code: "AUTH010", detail: "This domain is not enabled for Extract.", @@ -157,7 +157,7 @@ test("scrape maps AUTH010 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async }), (err: unknown) => { assert.ok(err && typeof err === "object" && "code" in err); - assert.equal((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_ENABLED"); + assert.equal((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_PREPARED"); return true; }, ); @@ -165,7 +165,7 @@ test("scrape maps AUTH010 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async ); }); -test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async () => { +test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_PREPARED", async () => { const REQS007 = JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for example.net yet.", @@ -178,7 +178,7 @@ test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async await assert.rejects( () => scrape("https://api.zenrows.com/v1/", "test-key", { url: "https://example.net", extract: "auto" }), (err: unknown) => { - assert.equal((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_ENABLED"); + assert.equal((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_PREPARED"); assert.equal( (err as { message: string }).message, "Extract hasn't prepared example.net yet. Prepare it with `zenrows extract prepare example.net --url https://example.net` (a few example URLs, a few minutes, no credits), then retry; meanwhile use --css.", @@ -190,7 +190,7 @@ test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async ); }); -test("scrape does not map REQS007 to EXTRACT_DOMAIN_NOT_ENABLED without extract=auto", async () => { +test("scrape does not map REQS007 to EXTRACT_DOMAIN_NOT_PREPARED without extract=auto", async () => { const REQS007 = JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for example.net yet.", status: 403 }); await withFetch( () => new Response(REQS007, { status: 403, headers: { "content-type": "application/json" } }), @@ -198,7 +198,7 @@ test("scrape does not map REQS007 to EXTRACT_DOMAIN_NOT_ENABLED without extract= await assert.rejects( () => scrape("https://api.zenrows.com/v1/", "test-key", { url: "https://example.net", autoparse: true }), (err: unknown) => { - assert.notEqual((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_ENABLED"); + assert.notEqual((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_PREPARED"); return true; }, ); @@ -208,7 +208,7 @@ test("scrape does not map REQS007 to EXTRACT_DOMAIN_NOT_ENABLED without extract= test("scrape treats AUTH010 without extract as a credits/plan error, not EXTRACT_DOMAIN", async () => { // AUTH010 is also used for other plan-gated features; only map it to - // EXTRACT_DOMAIN_NOT_ENABLED when the request asked for extract=auto. + // EXTRACT_DOMAIN_NOT_PREPARED when the request asked for extract=auto. const AUTH010 = JSON.stringify({ code: "AUTH010", detail: "Feature is not included in plan.", diff --git a/workflows/competitor-intelligence/WORKFLOW.md b/workflows/competitor-intelligence/WORKFLOW.md index f797a26..122762b 100644 --- a/workflows/competitor-intelligence/WORKFLOW.md +++ b/workflows/competitor-intelligence/WORKFLOW.md @@ -20,7 +20,7 @@ simple per-URL loop). ``` zenrows fetch https://competitor.example/products ``` -2. **Validate extraction** on one page (on a domain not prepared yet, `EXTRACT_DOMAIN_NOT_ENABLED`, run `zenrows extract prepare --url --wait` first). +2. **Validate extraction** on one page (on a domain not prepared yet, `EXTRACT_DOMAIN_NOT_PREPARED`, run `zenrows extract prepare --url --wait` first). ``` zenrows extract https://competitor.example/p/123 --validate ``` From 4d8ec61cf18b047aec2da1c2176436678cf03a9f Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 16:32:37 +0000 Subject: [PATCH 7/8] docs(evals): call the extract-smoke target a prepared domain --- evals/extract-smoke/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/extract-smoke/README.md b/evals/extract-smoke/README.md index 7f63c7e..3d13957 100644 --- a/evals/extract-smoke/README.md +++ b/evals/extract-smoke/README.md @@ -2,7 +2,7 @@ Reproducible smoke test for Extract (`extract=auto`). -- **Target:** `https://www.owler.com/company/meltwater` (enabled domain, real company page) +- **Target:** `https://www.owler.com/company/meltwater` (prepared domain, real company page) - **Config:** `extract=auto` - **Success criteria:** HTTP 2xx and a non-empty body - **Cost:** ~1× basic request From 2ba31ad8dad5b0a17783838c8f39ed27baa8bbd9 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 16 Oct 2026 07:03:25 +0000 Subject: [PATCH 8/8] feat(extract): remove the autoparse fallback zenrows extract no longer retries a refused Extract request with Autoparse, on any REQS007 or AUTH010, whatever the caller passes. The adapter option fallbackAutoparse is still accepted so existing callers keep working, but it is deprecated and has no effect: when it is true, the CLI prints one deprecation warning on stderr. fellBackToAutoparse is gone from the outcome, the run artifact and the --json output, since it can never be true. The explicit --autoparse flag is unchanged. --- README.md | 3 +- agent-plugin/skills/extract/SKILL.md | 4 +- src/adapters/extract.ts | 32 ++++------ src/cli/commands/extract.ts | 9 +-- tests/extract-outputs.test.ts | 91 ++++++++++++++++------------ 5 files changed, 71 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 68a2074..7b83931 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,8 @@ uses one slot of the monthly allowance. `EXTRACT_PAGE_NOT_RECOGNISED` (also REQS007) is different: the domain is prepared, but this page is not one Extract extracts. Try a product, article or listing page instead, retry it once in a few minutes if it should work, or use `--css`; preparing again does not change -it. +it. `zenrows extract` never falls back to Autoparse. The `fallbackAutoparse` +adapter option is deprecated and has no effect. ```bash zenrows extract prepare example.com --url https://example.com/p/1 --url https://example.com/list --wait diff --git a/agent-plugin/skills/extract/SKILL.md b/agent-plugin/skills/extract/SKILL.md index a3be1e6..084b508 100644 --- a/agent-plugin/skills/extract/SKILL.md +++ b/agent-plugin/skills/extract/SKILL.md @@ -35,8 +35,8 @@ result: true means no field came back with a value. `mode=auto` works on any prepared domain, and you can prepare one yourself. When Extract refuses a page, `extract` returns a tool error with code -`REQS007` (legacy `AUTH010`) and a `reason`. It never retries with Autoparse -unless you set `fallback_autoparse: true`. +`REQS007` (legacy `AUTH010`) and a `reason`. It never retries with Autoparse. +`fallback_autoparse` is deprecated and has no effect; don't send it. | `reason` | Meaning | Next step | | --- | --- | --- | diff --git a/src/adapters/extract.ts b/src/adapters/extract.ts index 8f72dd1..d526db3 100644 --- a/src/adapters/extract.ts +++ b/src/adapters/extract.ts @@ -9,12 +9,13 @@ * - response_type=markdown|plaintext * * Default method is `extract`. REQS007 (or the legacy AUTH010: Extract has not - * prepared the domain yet) is reported with the prepare step, not retried. - * Autoparse is deprecated and runs only when asked for: `--autoparse`, or - * `fallbackAutoparse: true`. + * prepared the domain yet) is reported with the prepare step, never retried: + * there is no fallback to Autoparse. Autoparse is deprecated and runs only when + * asked for directly, with `--autoparse` (method `autoparse`). */ import type { Policy, ToolkitConfig } from "../types/index.ts"; import { ToolkitError } from "../core/errors.ts"; +import { log } from "../core/logger.ts"; import { runFetch, type FetchOptions, type FetchOutcome } from "./protected-fetch.ts"; export type ExtractMethod = "extract" | "autoparse" | "css" | "outputs" | "markdown" | "plaintext"; @@ -28,9 +29,9 @@ export interface ExtractOptions extends Omit --url ` instead."; + export async function runExtract( opts: ExtractOptions, config: ToolkitConfig, @@ -96,17 +100,7 @@ export async function runExtract( }); } - if (method === "extract" && opts.fallbackAutoparse === true) { - try { - return await runExtractOnce({ ...opts, method: "extract" }, config, policy, apiKey); - } catch (err) { - if (err instanceof ToolkitError && err.code === "EXTRACT_DOMAIN_NOT_PREPARED") { - const outcome = await runExtractOnce({ ...opts, method: "autoparse" }, config, policy, apiKey); - return { ...outcome, fellBackToAutoparse: true }; - } - throw err; - } - } + if (opts.fallbackAutoparse === true) log.warn(FALLBACK_AUTOPARSE_DEPRECATION); return runExtractOnce({ ...opts, method }, config, policy, apiKey); } diff --git a/src/cli/commands/extract.ts b/src/cli/commands/extract.ts index cd08dff..cc0b443 100644 --- a/src/cli/commands/extract.ts +++ b/src/cli/commands/extract.ts @@ -143,10 +143,6 @@ export const extract: Command = { const safeParams = { ...outcome.params }; delete (safeParams as Record).apikey; - if (outcome.fellBackToAutoparse) { - log.info("Extract not available for this domain yet — fell back to Autoparse."); - } - const runDir = writeRun( { runId, @@ -159,7 +155,6 @@ export const extract: Command = { request: { ...safeParams, method: outcome.method, - fellBackToAutoparse: outcome.fellBackToAutoparse ?? false, }, result: { httpStatus: outcome.result.status, bytes: outcome.result.body.length, parsed: outcome.data !== undefined }, costUsd: outcome.result.costUsd, @@ -174,9 +169,8 @@ export const extract: Command = { writeOut(values.out as string, payload); log.success(`Wrote output → ${values.out}`); } - const via = outcome.fellBackToAutoparse ? `${outcome.method} (fallback)` : outcome.method; log.success( - `Extracted via ${via} · ${outcome.result.body.length} bytes · ${formatRequestCost(outcome.result.costUsd, outcome.result.costCredits)} · run ${runId}`, + `Extracted via ${outcome.method} · ${outcome.result.body.length} bytes · ${formatRequestCost(outcome.result.costUsd, outcome.result.costCredits)} · run ${runId}`, ); if (runDir) log.dim(` artifact: ${runDir}`); if (outcome.noDataExtracted) { @@ -196,7 +190,6 @@ export const extract: Command = { ok: true, runId, method: outcome.method, - fellBackToAutoparse: outcome.fellBackToAutoparse ?? false, empty: outcome.empty ?? false, noDataExtracted: outcome.noDataExtracted ?? false, httpStatus: outcome.result.status, diff --git a/tests/extract-outputs.test.ts b/tests/extract-outputs.test.ts index 7629ac9..cf0884f 100644 --- a/tests/extract-outputs.test.ts +++ b/tests/extract-outputs.test.ts @@ -2,7 +2,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { normalizeOutputs } from "../src/cli/commands/extract.ts"; import { buildParams, type FetchOptions } from "../src/adapters/protected-fetch.ts"; -import { isEmptyData, runExtract } from "../src/adapters/extract.ts"; +import { FALLBACK_AUTOPARSE_DEPRECATION, isEmptyData, runExtract } from "../src/adapters/extract.ts"; import { defaultConfig } from "../src/core/config.ts"; import { defaultPolicy } from "../src/core/policy.ts"; import { ToolkitError } from "../src/core/errors.ts"; @@ -143,7 +143,6 @@ test("runExtract default method is extract=auto and unwraps {parsed,html}", asyn assert.equal(outcome.params.autoparse, undefined); assert.deepEqual(outcome.data, { title: "X" }); assert.equal(outcome.html, "x"); - assert.equal(outcome.fellBackToAutoparse, undefined); }, ); }); @@ -308,28 +307,59 @@ test("runExtract reports legacy AUTH010 like REQS007 and does not retry", async }); }); -test("runExtract retries with autoparse on REQS007 only when fallbackAutoparse is true", async () => { - const REQS007 = JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for x.com yet.", status: 403 }); - const calls: string[] = []; - await withFetchHandler((url) => { - calls.push(url); - if (url.includes("extract=auto")) { - return new Response(REQS007, { status: 403, headers: { "content-type": "application/json" } }); - } - return new Response('{"title":"fallback"}', { - status: 200, - headers: { "content-type": "application/json", "x-request-credits": "1" }, - }); - }, async () => { - const outcome = await runExtract({ url: "https://x.com", fallbackAutoparse: true }, cfg, defaultPolicy(), "testkey"); - assert.equal(outcome.method, "autoparse"); - assert.equal(outcome.fellBackToAutoparse, true); - assert.deepEqual(outcome.data, { title: "fallback" }); - assert.equal(calls.length, 2); - assert.match(calls[0]!, /extract=auto/); - assert.match(calls[1]!, /autoparse=true/); - assert.doesNotMatch(calls[1]!, /extract=auto/); +/** Runs fn with stderr captured, and returns what was written to it. */ +async function captureStderr(fn: () => Promise): Promise { + const original = process.stderr.write.bind(process.stderr); + let written = ""; + process.stderr.write = ((chunk: string | Uint8Array) => { + written += typeof chunk === "string" ? chunk : Buffer.from(chunk).toString(); + return true; + }) as typeof process.stderr.write; + try { + await fn(); + } finally { + process.stderr.write = original; + } + return written; +} + +// fallbackAutoparse is deprecated and has no effect: the refusal is reported in +// one request, with the prepare step, and the caller is told once. +for (const [name, body, status] of [ + ["REQS007", JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for x.com yet.", status: 403 }), 403], + ["AUTH010", AUTH010, 402], +] as const) { + test(`runExtract ignores fallbackAutoparse: true on ${name}, in one request, with one deprecation warning`, async () => { + const calls: string[] = []; + const stderr = await captureStderr(() => + withFetchHandler((url) => { + calls.push(url); + return new Response(body, { status, headers: { "content-type": "application/json" } }); + }, async () => { + await assert.rejects( + () => runExtract({ url: "https://x.com", fallbackAutoparse: true }, cfg, defaultPolicy(), "testkey"), + (e: unknown) => + e instanceof ToolkitError && + e.code === "EXTRACT_DOMAIN_NOT_PREPARED" && + e.suggested_commands?.includes("zenrows extract prepare x.com --url https://x.com --wait") === true, + ); + }), + ); + assert.equal(calls.length, 1, "fallbackAutoparse must not retry with autoparse"); + assert.doesNotMatch(calls[0]!, /autoparse=true/); + assert.equal(stderr.split(FALLBACK_AUTOPARSE_DEPRECATION).length - 1, 1, `want one deprecation warning, got: ${stderr}`); }); +} + +test("runExtract prints no deprecation warning without fallbackAutoparse, or with it false", async () => { + for (const fallbackAutoparse of [undefined, false]) { + const stderr = await captureStderr(() => + withFetch(JSON.stringify({ parsed: { title: "X" } }), async () => { + await runExtract({ url: "https://x.com", fallbackAutoparse }, cfg, defaultPolicy(), "testkey"); + }), + ); + assert.doesNotMatch(stderr, /fallbackAutoparse/); + } }); test("runExtract does not fall back on AUTH004 (credits exhausted)", async () => { @@ -346,20 +376,6 @@ test("runExtract does not fall back on AUTH004 (credits exhausted)", async () => }); }); -test("runExtract surfaces autoparse failure after an opted-in AUTH010 fallback", async () => { - await withFetchHandler((url) => { - if (url.includes("extract=auto")) { - return new Response(AUTH010, { status: 402, headers: { "content-type": "application/json" } }); - } - return new Response("upstream boom", { status: 500, headers: { "content-type": "text/plain" } }); - }, async () => { - await assert.rejects( - () => runExtract({ url: "https://x.com", fallbackAutoparse: true }, cfg, defaultPolicy(), "testkey"), - (e: unknown) => e instanceof ToolkitError && e.code === "FETCH_FAILED", - ); - }); -}); - test("runExtract --validate fails when body is not JSON", async () => { await withFetch("not json", async () => { await assert.rejects( @@ -465,7 +481,6 @@ test("runExtract reports 422 no_data_extracted as an empty result, in one reques assert.equal(outcome.method, "extract"); assert.equal(outcome.empty, true); assert.equal(outcome.noDataExtracted, true); - assert.equal(outcome.fellBackToAutoparse, undefined); assert.equal(outcome.data, undefined); assert.equal(calls.length, 1); assert.match(calls[0]!, /extract=auto/);