diff --git a/README.md b/README.md index 1f7180f..7b83931 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ that endpoint with extraction parameters, not a separate product. | Command | What it does | Status (this build) | | --- | --- | --- | | `zenrows fetch` | Fetch: retrieve a protected page | **available**: `GET https://api.zenrows.com/v1/` | -| `zenrows extract` | Extract: turn a page into structured data with `extract=auto` on any prepared domain (falls back to Autoparse on one not prepared yet), CSS selectors, or Markdown | **available**: `GET https://api.zenrows.com/v1/?extract=auto` | +| `zenrows extract` | Extract: turn a page into structured data with `extract=auto` on any prepared domain (prepare one yourself with `zenrows extract prepare`), CSS selectors, or Markdown | **available**: `GET https://api.zenrows.com/v1/?extract=auto` | | `zenrows batch` | Batch: run Fetch and Extract over many URLs in one job; specs validate and estimate locally with no key | **available**: `POST https://async.api.zenrows.com/v1/jobs` | | `zenrows crawl` | Crawl (new): collect the URLs behind one start page, optionally with each page's HTML | **available**: `POST https://api.zenrows.com/v1/crawls` | | `zenrows browser` | Browser Sessions: a real cloud browser over REST (same backend as MCP `browser_*`); escalation-only, billed by bandwidth + time | **available**: `POST https://mcp.zenrows.com/browser/sessions` | @@ -56,7 +56,7 @@ npm install -g @zenrows/cli # puts `zenrows` on PATH (466 ms, zero deps) zenrows init zenrows fetch https://httpbin.io/html # auto-provisions a Free plan account on first use zenrows extract https://www.owler.com/company/meltwater # extract=auto on a prepared domain -zenrows extract https://www.scrapingcourse.com/ecommerce/ --autoparse # Autoparse (any domain) +zenrows extract https://www.scrapingcourse.com/ecommerce/ --css '{"title":"h1"}' # CSS selectors (any domain) ``` > Prefer no global install? Prefix each command with `npx -y @zenrows/cli` @@ -117,12 +117,28 @@ zenrows fetch --proxy-country us --wait-for ".price" ```bash zenrows extract https://www.owler.com/company/meltwater # extract=auto (prepared domain) -zenrows extract https://www.scrapingcourse.com/ecommerce/ --autoparse # Autoparse (any domain) +zenrows extract https://www.scrapingcourse.com/ecommerce/ --autoparse # deprecated Autoparse (any domain) zenrows extract --css '{"title":"h1","price":".price"}' --validate zenrows extract --outputs emails,links # built-in output filters → JSON zenrows extract --output markdown ``` +On a domain Extract hasn't prepared yet, `zenrows extract` fails with +`EXTRACT_DOMAIN_NOT_PREPARED` (API code REQS007) and prints the command to +prepare it. Preparing takes a few minutes and costs no credits; each new domain +uses one slot of the monthly allowance. `EXTRACT_PAGE_NOT_RECOGNISED` (also +REQS007) is different: the domain is prepared, but this page is not one Extract +extracts. Try a product, article or listing page instead, retry it once in a +few minutes if it should work, or use `--css`; preparing again does not change +it. `zenrows extract` never falls back to Autoparse. The `fallbackAutoparse` +adapter option is deprecated and has no effect. + +```bash +zenrows extract prepare example.com --url https://example.com/p/1 --url https://example.com/list --wait +zenrows extract prepare status example.com # progress + monthly allowance +zenrows extract https://example.com/p/1 # then extract again +``` + ## 9. Batch Zenrows **Batch** (`https://async.api.zenrows.com/v1`) fans a diff --git a/agent-plugin/README.md b/agent-plugin/README.md index 351d5c6..b9fdc8a 100644 --- a/agent-plugin/README.md +++ b/agent-plugin/README.md @@ -62,7 +62,7 @@ Use MCP tool names — not CLI `zenrows fetch`: | --- | --- | | `zenrows` | Decision tree across all primitives | | `protected-fetch` | `scrape` | -| `extract` | `extract` (fallback: `scrape` + autoparse / css_extractor) | +| `extract` | `extract` (fallback: `scrape` + css_extractor); on REQS007 with reason `domain_not_prepared`, `prepare_domain` then `prepared_domain_status` | | `batch-jobs` | `batch_create`, `batch_status`, `batch_results`, `batch_cancel` | | `interact-browser` | `browser_*` (escalation only) | | `cost-control` | Cost multipliers + escalate-with-evidence | diff --git a/agent-plugin/skills/cost-control/SKILL.md b/agent-plugin/skills/cost-control/SKILL.md index d6a76f0..4adf57f 100644 --- a/agent-plugin/skills/cost-control/SKILL.md +++ b/agent-plugin/skills/cost-control/SKILL.md @@ -19,8 +19,8 @@ These multipliers apply when you enable the flags (on `scrape` or `extract`): - Both: **25×** `extract` `mode=auto` is currently free; billing may apply -later. Do not promise permanent free extract pricing. Autoparse / CSS still -follow the multipliers above when stealth flags are set. +later. Do not promise permanent free extract pricing. CSS (and the deprecated +Autoparse) still follow the multipliers above when stealth flags are set. ## Browser sessions (`browser_*`) diff --git a/agent-plugin/skills/extract/SKILL.md b/agent-plugin/skills/extract/SKILL.md index 4d08cc9..084b508 100644 --- a/agent-plugin/skills/extract/SKILL.md +++ b/agent-plugin/skills/extract/SKILL.md @@ -14,28 +14,67 @@ Convert protected pages into structured data. Value is **protected page access Use **`extract`** when you need structured JSON fields rather than a full page: ``` -extract({ url }) # mode=auto (default) -extract({ url, mode: "autoparse" }) # general Autoparse (any domain) +extract({ url }) # mode=auto (default), returns `parsed` only +extract({ url, include_html: true }) # also return the full page HTML +extract({ url, mode: "autoparse" }) # deprecated Autoparse (any domain) extract({ url, mode: "css", css_extractor: '{"title":"h1","price":".price"}' }) extract({ url, proxy_country: "US" }) # geo-targeted ``` -`mode=auto` (`extract=auto`) works on any domain Zenrows has prepared, with any -key, and is currently free; billing may apply later. If the -domain is not enabled (AUTH010) or not prepared yet (REQS007), the tool retries -once with Autoparse by default (`fallback_autoparse`). +`mode=auto` (`extract=auto`) works on any prepared domain, with any key, and +is currently free; billing may apply later. The result carries `parsed` only; +the page HTML is most of the response size, so ask for it with +`include_html: true` only when you need it. Every mode uses Adaptive Stealth Mode by default: pass just the URL, even for protected or dynamic sites. Set `js_render` or `premium_proxy` only to force a fixed configuration; that turns Adaptive Stealth Mode off. Check `empty` in the result: true means no field came back with a value. +## REQS007: prepare the domain, then extract again + +`mode=auto` works on any prepared domain, and you can prepare one yourself. +When Extract refuses a page, `extract` returns a tool error with code +`REQS007` (legacy `AUTH010`) and a `reason`. It never retries with Autoparse. +`fallback_autoparse` is deprecated and has no effect; don't send it. + +| `reason` | Meaning | Next step | +| --- | --- | --- | +| `domain_not_prepared` | The domain is not prepared yet. | `prepare_domain` (the error's `next_tool` holds its arguments), then poll, then `extract` again. | +| `page_not_recognised` | The domain is prepared, but Extract did not recognise this page as one it can extract (a page with nothing to extract, such as a home page, or a transient failure). | Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use `mode=css`. Do not call `prepare_domain`: preparing again does not change this. | +| `domain_not_enabled` | Extract is not enabled for the site. | Use `mode=css`, or contact Zenrows support to have the site enabled. Preparing does not change this. | + +Preparing a domain: + +``` +extract({ url }) # 1. REQS007, reason domain_not_prepared +prepare_domain({ domain: "example.com", + example_urls: ["https://example.com/p/1", "https://example.com/list"] }) + # 2. returns at once, status "preparing" +prepared_domain_status({ domain: "example.com" }) # 3. about every 30 seconds until "ready" +extract({ url }) # 4. extract again +``` + +- `example_urls`: 1 to 5 real pages you want to extract, one per page kind + (for example a product page and a listing page). +- The domain is the host: `shop.example.com` and `example.com` are prepared + separately, and a leading `www.` is dropped. +- Preparing takes a few minutes and costs no credits. Each new domain uses one + slot of the account's monthly allowance; `prepared_domain_status` returns it + (`used` / `limit` / `remaining`) and a `next` field with what to do now. +- `out_of_allowance`: no slots left this month; use `mode=css`. + `overloaded` / `disabled`: retry in a few minutes; do not loop. + A `failed` status carries `failure.code` and `failure.slot_consumed`; follow `next`. +- A domain that is already prepared answers `ready` at once + (`already_prepared: true`) and uses no slot: call `extract`. If it then + returns `page_not_recognised`, follow that row of the table, not `prepare_domain`. + ## Fallback: `scrape` extraction params If `extract` is unavailable, similar outcomes are available on **`scrape`**: ``` -scrape({ url, autoparse: true }) +scrape({ url, autoparse: true }) # deprecated scrape({ url, css_extractor: '{"title":"h1","price":".price"}' }) scrape({ url, outputs: "emails,links" }) # or "*" for all built-ins ``` @@ -46,7 +85,8 @@ scrape({ url, outputs: "emails,links" }) # or "*" for all built-ins ## When to use which - **`mode=auto`**: site-tailored fields on any prepared domain. -- **`mode=autoparse`**: any website; also the automatic AUTH010 / REQS007 fallback. +- **`mode=autoparse`** (deprecated): any website, only when asked for. Do not + build new work on it. - **`mode=css`**: known fields/selectors; deterministic. - **Markdown via `scrape`**: feed clean content to an LLM yourself. @@ -57,7 +97,7 @@ Stealth flags (`js_render`, `premium_proxy`) still apply the usual multipliers ## Rules - Validate on a single page before `batch_create` ([[batch-jobs]]). -- If Autoparse misses fields, switch to `css_extractor`, or add `js_render` +- If a field is missing, switch to `css_extractor`, or add `js_render` for JS-heavy pages. - Prefer MCP `extract` / `scrape` over CLI `zenrows extract` when the MCP server is connected. diff --git a/agent-plugin/skills/protected-fetch/SKILL.md b/agent-plugin/skills/protected-fetch/SKILL.md index c6cc969..6b1c60c 100644 --- a/agent-plugin/skills/protected-fetch/SKILL.md +++ b/agent-plugin/skills/protected-fetch/SKILL.md @@ -34,7 +34,7 @@ scrape({ url, js_render: true, wait_for: ".price" }) - Prefer `response_type: "markdown"` for LLM consumption; use `html` only when you need raw markup. - For **structured** fields, prefer the `extract` tool (or `scrape` with - `autoparse` / `css_extractor` / `outputs`) — see [[extract]]. + `css_extractor` / `outputs`) — see [[extract]]. - On failure, escalate with evidence before `browser_*` ([[interact-browser]]). - Cost model: [[cost-control]]. diff --git a/agent-plugin/skills/zenrows/SKILL.md b/agent-plugin/skills/zenrows/SKILL.md index 1cae7d3..ca73ae4 100644 --- a/agent-plugin/skills/zenrows/SKILL.md +++ b/agent-plugin/skills/zenrows/SKILL.md @@ -36,7 +36,7 @@ If the user has a known URL and wants page content: If the user has a known URL and wants structured data: → Use Extract. (MCP tool: extract) - Fallback: scrape with autoparse / css_extractor / outputs + Fallback: scrape with css_extractor / outputs If the user has many URLs (bulk): → Use Batch. (validate one page with scrape/extract first, @@ -66,7 +66,8 @@ If the user needs login, clicks, forms, sessions, or persistent state: | Primitive | MCP tools | Notes | | --- | --- | --- | | Protected Fetch | `scrape` | Default markdown; HTML / plaintext / PDF / screenshot options | -| Extract | `extract` | Structured JSON (`extract=auto`, autoparse, css_extractor). `extract=auto` is currently free; billing may apply later | +| Extract | `extract` | Structured JSON (`extract=auto` on any prepared domain, css_extractor; autoparse deprecated). `extract=auto` is currently free; billing may apply later | +| Prepare a domain | `prepare_domain`, `prepared_domain_status` | After an `extract` REQS007 with reason `domain_not_prepared`: prepare the domain with 1 to 5 example URLs, poll about every 30 seconds until ready, extract again. No credits; uses one monthly allowance slot. Not for `page_not_recognised`: that domain is already prepared | | Batch | `batch_create`, `batch_status`, `batch_results`, `batch_cancel` | Same credits as Fetch; a 403 maps to `BATCH_ACCESS_DENIED` | | Browser | `browser_*` (start with `browser_navigate`) | Escalation-only; same backend as Browser Sessions | diff --git a/docs/capabilities.md b/docs/capabilities.md index 36c2409..84bf420 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -18,7 +18,7 @@ Classification is based on the public Zenrows documentation: | Capability | Backend evidence | Status | | --- | --- | --- | | `protected_fetch` | Fetch `GET https://api.zenrows.com/v1/` with `mode`, `js_render`, `premium_proxy`, `proxy_country`, `wait`/`wait_for`, `js_instructions`, `response_type`, `screenshot`, `original_status`, … | available | -| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (any prepared domain; CLI falls back to `autoparse` on one not prepared yet), plus `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | available | +| `extract` | Extract `GET https://api.zenrows.com/v1/` via `extract=auto` (any prepared domain; on one not prepared yet the CLI reports REQS007 as `EXTRACT_DOMAIN_NOT_PREPARED` with the command to prepare it: `zenrows extract prepare --url `, which calls `POST /v1/extract/prepared-domains`; a page Extract did not recognise on a prepared domain is `EXTRACT_PAGE_NOT_RECOGNISED`, which preparing does not change), plus deprecated `autoparse`, `css_extractor`, `outputs`, `response_type=markdown\|plaintext` | available | | `batch` | Batch `https://async.api.zenrows.com/v1` (separate host, `X-API-Key` header). Local JSONL spec validation + credit estimation work with no key. | available | | `crawl` | Crawl `https://api.zenrows.com/v1/crawls` (`X-API-Key` header; same host as Fetch/Extract). Follows the links on each page from one start URL, up to `depth` when set (optional; no depth limit when omitted), on the start URL's registrable domain (subdomains count); returns URLs, and with `output_format: "html"` each page's HTML. Subcommands: `create`, `get`, `list`, `results`, `content`, `download`, `stop`, `wait`. Without Crawl access the API returns 403 `REQS008` → `CRAWL_NOT_ENABLED`. | available | | `browser` | Browser Sessions REST API `https://mcp.zenrows.com/browser/sessions/*` (Bearer; same backend as `@zenrows/mcp` `browser_*`); CDP via `zenrows browser connect`. Escalation-only (prefer fetch/extract); on by default, opt out via `policy.allow_browser`; bills by bandwidth + session time (15-min max) | available | diff --git a/evals/extract-smoke/README.md b/evals/extract-smoke/README.md index 7f63c7e..3d13957 100644 --- a/evals/extract-smoke/README.md +++ b/evals/extract-smoke/README.md @@ -2,7 +2,7 @@ Reproducible smoke test for Extract (`extract=auto`). -- **Target:** `https://www.owler.com/company/meltwater` (enabled domain, real company page) +- **Target:** `https://www.owler.com/company/meltwater` (prepared domain, real company page) - **Config:** `extract=auto` - **Success criteria:** HTTP 2xx and a non-empty body - **Cost:** ~1× basic request diff --git a/registry/capabilities.json b/registry/capabilities.json index d81b872..13b2f71 100644 --- a/registry/capabilities.json +++ b/registry/capabilities.json @@ -12,12 +12,12 @@ }, "extract": { "key": "extract", - "label": "Extract (extract=auto / Autoparse / CSS / Markdown)", + "label": "Extract (extract=auto / CSS / Markdown)", "status": "available", "command": "zenrows extract", - "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, autoparse, css_extractor, outputs, response_type)", + "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, css_extractor, outputs, response_type; autoparse deprecated)", "requiresAuth": true, - "notes": "Default is extract=auto, which works on any prepared domain with any key; CLI falls back to autoparse on REQS007 (domain not prepared yet) or AUTH010. Autoparse / CSS / outputs / markdown work on any domain." + "notes": "Default is extract=auto, which works on any prepared domain with any key. A REQS007 (legacy AUTH010) is reported, never retried, with the next step for its reason: on a domain not prepared yet (EXTRACT_DOMAIN_NOT_PREPARED), prepare it with `zenrows extract prepare --url [--wait]` (POST /v1/extract/prepared-domains: 1 to 5 example URLs, a few minutes, no credits) and check it with `zenrows extract prepare status `; on a page Extract did not recognise on a prepared domain (EXTRACT_PAGE_NOT_RECOGNISED), try a product, article or listing page, retry it once in a few minutes, or use --css, because preparing again does not change it. CSS / outputs / markdown work on any domain; autoparse is deprecated and runs only with --autoparse." }, "batch": { "key": "batch", diff --git a/registry/skills.json b/registry/skills.json index cdc806c..2169684 100644 --- a/registry/skills.json +++ b/registry/skills.json @@ -25,13 +25,13 @@ { "name": "extract", "type": "skill", - "description": "Turn protected pages into structured data with Autoparse / CSS / Markdown.", + "description": "Turn protected pages into structured data with Extract / CSS / Markdown (Autoparse is deprecated).", "status": "available", "requires_backend_capabilities": ["extract"], "requires_auth": true, "version": "0.1.0", "path": "skills/extract", - "tags": ["extract", "autoparse", "structured-data"] + "tags": ["extract", "structured-data"] }, { "name": "batch-jobs", diff --git a/registry/templates.json b/registry/templates.json index 39f25f6..8686d6e 100644 --- a/registry/templates.json +++ b/registry/templates.json @@ -14,7 +14,7 @@ { "name": "extract-pricing-pages", "type": "template", - "description": "Extract structured pricing fields from product pages (Autoparse / CSS).", + "description": "Extract structured pricing fields from product pages (extract=auto / CSS).", "status": "available", "requires_backend_capabilities": ["extract"], "requires_auth": true, diff --git a/skills/extract/SKILL.md b/skills/extract/SKILL.md index 597d49c..00d4b96 100644 --- a/skills/extract/SKILL.md +++ b/skills/extract/SKILL.md @@ -1,6 +1,6 @@ --- name: extract -description: Turn protected pages into structured data with Extract / Autoparse / CSS / Markdown. +description: Turn protected pages into structured data with Extract / CSS / Markdown (Autoparse is deprecated). version: 0.1.0 requires_backend_capabilities: [extract] --- @@ -12,8 +12,8 @@ access + extraction**, not generic LLM parsing. ## Methods ``` -zenrows extract # extract=auto (default); falls back to Autoparse if the domain is not prepared yet -zenrows extract --autoparse # general-purpose Autoparse (any domain) +zenrows extract # extract=auto (default), any prepared domain +zenrows extract --autoparse # deprecated general-purpose Autoparse (any domain) zenrows extract --css '{"title":"h1","price":".price"}' # selector map zenrows extract --outputs emails,links # built-in output filters → JSON zenrows extract --output markdown # Markdown conversion @@ -21,14 +21,58 @@ zenrows extract --validate # fail if not valid JSON ``` ## When to use which -- **Extract** (`extract=auto`, default): richest fields; works on any domain Zenrows has prepared. -- **Autoparse**: any website; also the automatic fallback when the domain is not prepared for Extract yet. +- **Extract** (`extract=auto`, default): richest fields; works on any prepared + domain (you can prepare one yourself, see below). +- **Autoparse** (deprecated): only with `--autoparse`. Never automatic; do not + build new work on it. - **CSS**: you know the exact fields/selectors and want determinism. - **Markdown/plaintext**: feed clean content to an LLM yourself. +## REQS007: prepare the domain, then extract again + +`extract=auto` works on any prepared domain, and you can prepare one yourself. +When Extract refuses a page with `403 REQS007`, `zenrows extract` says which +of three cases it is and prints the next step. It never retries with Autoparse. + +| Error code | Meaning | Next step | +| --- | --- | --- | +| `EXTRACT_DOMAIN_NOT_PREPARED` | The domain is not prepared yet (also the legacy `402 AUTH010`). | Prepare it (below), then extract again. | +| `EXTRACT_PAGE_NOT_RECOGNISED` | The domain is prepared, but Extract did not recognise this page as one it can extract (a page with nothing to extract, such as a home page, or a transient failure). | Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use `--css`. Preparing the domain again does not change this. | +| `EXTRACT_DOMAIN_NOT_ENABLED` | Extract is not enabled for the site. | Use `--css`, or contact Zenrows support to have the site enabled. Preparing does not change this. | + +Preparing a domain: + +``` +zenrows extract https://example.com/p/1 # 1. fails: EXTRACT_DOMAIN_NOT_PREPARED +zenrows extract prepare example.com \ + --url https://example.com/p/1 --url https://example.com/list --wait + # 2. prepare it and wait until ready (a few minutes) +zenrows extract https://example.com/p/1 # 3. extract again +``` + +- `--url`: 1 to 5 real pages you want to extract, one per page kind (for + example a product page and a listing page). +- Without `--wait` the command returns at once; then check about every 30 + seconds with `zenrows extract prepare status example.com` until it says + `ready`. `--wait` polls at the same pace. +- The domain is the host: `shop.example.com` and `example.com` are prepared + separately, and a leading `www.` is dropped. +- Preparing takes a few minutes and costs no credits. Each new domain uses one + slot of the account's monthly allowance; `prepare status` shows `used` / + `limit` / `remaining`. +- `EXTRACT_PREPARE_OUT_OF_ALLOWANCE`: no slots left this month; use `--css`. + `EXTRACT_PREPARE_UNAVAILABLE`: retry in a few minutes; do not loop. + A `failed` status prints the next step for its failure code and says whether + the attempt used an allowance slot. +- A domain that is already prepared answers `ready` at once + (`already_prepared`) and uses no slot: run the extract. If it then fails with + `EXTRACT_PAGE_NOT_RECOGNISED`, follow that row of the table, not `prepare`. + ## Rules - Validate on a single page before scaling across many URLs. -- If Autoparse misses fields, switch to `--css` with explicit selectors. Auto +- Check `empty` in `--json` output: true means every field came back null or + empty, so the result is not data. +- If a field is missing, switch to `--css` with explicit selectors. Auto mode already renders JS when the target needs it, so reach for `--manual --js-render` only after auto mode has returned an incomplete page: it costs several times a basic call ([[cost-control]]) and it makes the diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index c782a36..ea23d6d 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -32,7 +32,7 @@ If the user has a known URL and wants page content: → Use Protected Fetch. (zenrows fetch ) If the user has a known URL and wants structured data: - → Use Extract. (zenrows extract | --autoparse | --css) + → Use Extract. (zenrows extract | --css) If the user has a start page (listing, index, section) but not the item URLs: → Use Crawl. (zenrows crawl create --depth 1 --follow, @@ -72,13 +72,16 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | Primitive | Command | Status | | --- | --- | --- | | Protected Fetch | `zenrows fetch` | **available**: `GET https://api.zenrows.com/v1/` | -| Extract (`extract=auto`, Autoparse, CSS, Markdown) | `zenrows extract` | **available**: `GET https://api.zenrows.com/v1/?extract=auto` | +| Extract (`extract=auto`, CSS, Markdown; Autoparse deprecated) | `zenrows extract` | **available**: `GET https://api.zenrows.com/v1/?extract=auto` | | Batch | `zenrows batch` | **available**: `POST https://async.api.zenrows.com/v1/jobs` | | Crawl (new) | `zenrows crawl` | **available**: `POST https://api.zenrows.com/v1/crawls` | | Browser | `zenrows browser` | **available**: `POST https://mcp.zenrows.com/browser/sessions` | | MCP | `zenrows mcp` | **available**: `https://mcp.zenrows.com/mcp` | -- Extract: `extract=auto` falls back to Autoparse on a domain not prepared yet. +- Extract: `extract=auto` works on any prepared domain. On one not prepared yet + (`EXTRACT_DOMAIN_NOT_PREPARED`), run `zenrows extract prepare --url + --wait`, then extract again. `EXTRACT_PAGE_NOT_RECOGNISED` means the + domain is prepared but not this page: do not prepare again; see [[extract]]. - Batch: the cheapest path at scale (create, status, results). Estimate specs locally with no key. - Crawl: collects the URLs behind one start page, optionally with each page's HTML. - Browser: escalation-only, for pages Fetch can't handle; same backend as the MCP `browser_*` tools. diff --git a/src/adapters/extract.ts b/src/adapters/extract.ts index 0876e96..d526db3 100644 --- a/src/adapters/extract.ts +++ b/src/adapters/extract.ts @@ -8,13 +8,14 @@ * - outputs= → built-in output filters → JSON * - response_type=markdown|plaintext * - * Default method is `extract`. On AUTH010 / REQS007 (domain not enabled or not - * prepared for Extract) we - * automatically retry once with Autoparse unless the caller opted into - * `--autoparse` (or another explicit method) or disabled the fallback. + * Default method is `extract`. REQS007 (or the legacy AUTH010: Extract has not + * prepared the domain yet) is reported with the prepare step, never retried: + * there is no fallback to Autoparse. Autoparse is deprecated and runs only when + * asked for directly, with `--autoparse` (method `autoparse`). */ import type { Policy, ToolkitConfig } from "../types/index.ts"; import { ToolkitError } from "../core/errors.ts"; +import { log } from "../core/logger.ts"; import { runFetch, type FetchOptions, type FetchOutcome } from "./protected-fetch.ts"; export type ExtractMethod = "extract" | "autoparse" | "css" | "outputs" | "markdown" | "plaintext"; @@ -28,8 +29,9 @@ export interface ExtractOptions extends Omit --url ` instead."; + export async function runExtract( opts: ExtractOptions, config: ToolkitConfig, @@ -90,17 +100,7 @@ export async function runExtract( }); } - if (method === "extract" && opts.fallbackAutoparse !== false) { - try { - return await runExtractOnce({ ...opts, method: "extract" }, config, policy, apiKey); - } catch (err) { - if (err instanceof ToolkitError && err.code === "EXTRACT_DOMAIN_NOT_ENABLED") { - const outcome = await runExtractOnce({ ...opts, method: "autoparse" }, config, policy, apiKey); - return { ...outcome, fellBackToAutoparse: true }; - } - throw err; - } - } + if (opts.fallbackAutoparse === true) log.warn(FALLBACK_AUTOPARSE_DEPRECATION); return runExtractOnce({ ...opts, method }, config, policy, apiKey); } @@ -179,7 +179,7 @@ function parseExtractBody( } const empty = isEmptyData(data); - // ACT-1514: an empty result must not read as a silent success. Under + // An empty result must not read as a silent success. Under // --validate it's an error; otherwise the command surfaces a warning. if (opts.validate && empty) { throw new ToolkitError({ diff --git a/src/cli/commands/extract-prepare.ts b/src/cli/commands/extract-prepare.ts new file mode 100644 index 0000000..fb3d3b6 --- /dev/null +++ b/src/cli/commands/extract-prepare.ts @@ -0,0 +1,202 @@ +/** + * `zenrows extract prepare --url [--url ...] [--wait]` and + * `zenrows extract prepare status `. + * + * The way out of REQS007: prepare the domain so `extract=auto` works on it. + * Submitting returns at once; `--wait` polls until the preparation is ready or + * failed (a few minutes). Costs no credits; each new domain uses one slot of the + * account's monthly allowance. + */ +import { ensureApiKey } from "../../core/ensure-key.ts"; +import { loadConfig } from "../../core/config.ts"; +import { loadPolicy } from "../../core/policy.ts"; +import { log } from "../../core/logger.ts"; +import { ToolkitError } from "../../core/errors.ts"; +import { + DEFAULT_WAIT_TIMEOUT_MS, + MAX_EXAMPLE_URLS, + failureNextStep, + getAllowance, + getPreparedDomain, + normalizeDomain, + prepareDomain, + statusNextStep, + waitForPreparedDomain, + type Allowance, + type PreparedDomain, + type PrepareRequestOpts, +} from "../../core/prepared-domains-api.ts"; +import { parse, asString, type RunContext } from "../command.ts"; +import { printError } from "../output.ts"; + +export const PREPARE_USAGE = + "zenrows extract prepare --url [--url ...] [--wait [--timeout ]]\n" + + " zenrows extract prepare status "; + +export const PREPARE_HELP = [ + "Prepare a domain so `zenrows extract ` (extract=auto) works on it. Use it when an", + "extract fails with EXTRACT_DOMAIN_NOT_PREPARED (REQS007: the domain is not prepared yet).", + "Not for EXTRACT_PAGE_NOT_RECOGNISED: that domain is already prepared, and preparing it", + "again does not change that answer.", + "", + ` --url 1 to ${MAX_EXAMPLE_URLS} real pages you want to extract, one per page kind (repeat --url)`, + " --wait poll every 30 seconds until the domain is ready or failed; off by default", + ` --timeout give up waiting after this long (default ${DEFAULT_WAIT_TIMEOUT_MS / 1000})`, + " status show progress and the account's monthly allowance", + " --json structured result", + "", + "Preparing takes a few minutes and costs no credits. Each new domain uses one slot of the", + "account's monthly allowance; a domain that is already prepared answers ready at once and", + "uses none. The domain is the host (shop.example.com and example.com are separate); a", + "leading www. is dropped.", +].join("\n"); + +/** Injectable for tests: network, sleep and clock. */ +export interface PrepareDeps { + fetchImpl?: typeof fetch; + sleepImpl?: (ms: number) => Promise; + now?: () => number; + intervalMs?: number; + apiKey?: string; +} + +export async function runPrepare(argv: string[], ctx: RunContext, deps: PrepareDeps = {}): Promise { + const isStatus = argv[0] === "status"; + const { values, positionals } = parse(isStatus ? argv.slice(1) : argv, { + url: { type: "string", multiple: true }, + wait: { type: "boolean" }, + timeout: { type: "string" }, + json: { type: "boolean" }, + help: { type: "boolean" }, + }); + const json = ctx.json || values.json === true; + try { + const raw = positionals[0]; + if (!raw) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: "Missing domain.", + likely_cause: "No was given.", + next_action: `Usage: ${PREPARE_USAGE}`, + suggested_commands: ["zenrows extract prepare example.com --url https://example.com/products/1"], + }); + } + const domain = normalizeDomain(raw); + const urls = isStatus ? [] : validateUrls(values.url, domain); + const timeoutMs = parseTimeout(asString(values.timeout)); + + const policy = loadPolicy(); + const apiKey = + deps.apiKey ?? + // Preparing never creates a Free plan account: it needs a key the user already has. + (await ensureApiKey({ ...policy, auto_signup: false })); + const opts: PrepareRequestOpts = { apiKey, apiBase: loadConfig().apiBase, fetchImpl: deps.fetchImpl }; + + if (isStatus) { + const [record, allowance] = await Promise.all([getPreparedDomain(domain, opts), getAllowance(opts).catch(() => undefined)]); + report(record, allowance, json); + return 0; + } + + log.step(`Prepare ${domain} for Extract (${urls.length} example URL${urls.length === 1 ? "" : "s"})…`); + let record: PreparedDomain = await prepareDomain(domain, urls, opts); + if (record.already_in_progress) log.info(`${domain} was already being prepared; following that preparation.`); + // A submit that answers ready at once found the domain already prepared. + const alreadyReady = record.status === "ready" && !record.already_in_progress; + if (values.wait === true && record.status !== "ready" && record.status !== "failed") { + record = await waitForPreparedDomain(domain, { + ...opts, + timeoutMs, + intervalMs: deps.intervalMs, + sleepImpl: deps.sleepImpl, + now: deps.now, + onPoll: (r) => log.dim(` ${r.domain}: ${r.status} (${r.state ?? r.status})`), + }); + } + if (record.status === "failed") { + throw new ToolkitError({ + code: "EXTRACT_PREPARE_FAILED", + message: `Preparing ${domain} failed${record.failure?.code ? ` (${record.failure.code})` : ""}.`, + likely_cause: record.failure?.detail ?? "The preparation did not finish.", + next_action: failureNextStep(domain, record.failure), + server_code: record.failure?.code, + }); + } + report(record, undefined, json, { exampleUrl: urls[0], alreadyReady }); + return 0; + } catch (err) { + printError(err, json); + return 1; + } +} + +function validateUrls(raw: unknown, domain: string): string[] { + const urls = (Array.isArray(raw) ? raw : raw === undefined ? [] : [raw]).map(String); + if (urls.length < 1 || urls.length > MAX_EXAMPLE_URLS) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Pass 1 to ${MAX_EXAMPLE_URLS} example page URLs with --url (got ${urls.length}).`, + likely_cause: "Preparation learns the domain from real pages you want to extract.", + next_action: "Add one --url per page kind, e.g. a product page and a listing page.", + suggested_commands: [`zenrows extract prepare ${domain} --url https://${domain}/`], + }); + } + for (const u of urls) { + let parsed: URL | undefined; + try { + parsed = new URL(u); + } catch { + // reported below + } + if (!parsed || (parsed.protocol !== "http:" && parsed.protocol !== "https:")) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `--url ${u} is not a full http(s) URL.`, + likely_cause: "Each --url must be a page URL including its scheme.", + next_action: `Use full URLs, e.g. --url https://${domain}/.`, + }); + } + } + return urls; +} + +function parseTimeout(v?: string): number { + if (v === undefined) return DEFAULT_WAIT_TIMEOUT_MS; + const n = Number(v); + if (!Number.isFinite(n) || n <= 0) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `--timeout must be a positive number of seconds (got ${v}).`, + likely_cause: "The value is not a number.", + next_action: "Pass seconds, e.g. --timeout 600.", + }); + } + return n * 1000; +} + +function report( + record: PreparedDomain, + allowance: Allowance | undefined, + json: boolean, + opts: { exampleUrl?: string; alreadyReady?: boolean } = {}, +): void { + const next = statusNextStep(record, opts); + if (json) { + log.out( + JSON.stringify( + { ok: true, ...record, ...(opts.alreadyReady ? { already_prepared: true } : {}), ...(allowance ? { allowance } : {}), next }, + null, + 2, + ), + ); + return; + } + const line = `${record.domain}: ${record.status}${record.state && record.state !== record.status ? ` (${record.state})` : ""}`; + if (record.status === "ready") log.success(line); + else if (record.status === "failed") log.warn(line); + else log.info(line); + if (record.failure) log.dim(` failure: ${record.failure.code}${record.failure.detail ? ` — ${record.failure.detail}` : ""}`); + for (const role of record.roles ?? []) log.dim(` ${role.role}: ${role.status}${role.url ? ` ${role.url}` : ""}`); + if (allowance) log.dim(` allowance: ${allowance.used} of ${allowance.limit} used, ${allowance.remaining} left${allowance.month ? ` (${allowance.month})` : ""}`); + log.out(next); +} diff --git a/src/cli/commands/extract.ts b/src/cli/commands/extract.ts index 3128720..cc0b443 100644 --- a/src/cli/commands/extract.ts +++ b/src/cli/commands/extract.ts @@ -1,9 +1,12 @@ /** * `zenrows extract [flags]` → Extract (structured extraction on `/v1/`). * - * Default: `extract=auto` (site-tailored Extract). If the domain is - * not enabled or prepared (AUTH010 / REQS007), falls back once to Autoparse. Opt into Autoparse - * directly with `--autoparse`. Other methods: `--css`, `--outputs`, `--output`. + * Default: `extract=auto` (site-tailored Extract). On a REQS007 / AUTH010 the + * error names the next step for its reason (prepare the domain, or try another + * page); nothing is retried. `--autoparse` runs the deprecated Autoparse + * explicitly. Other + * methods: `--css`, `--outputs`, `--output`. `zenrows extract prepare` prepares a + * domain (see extract-prepare.ts). */ import { ensureApiKey } from "../../core/ensure-key.ts"; import { maybeNudgeClaim } from "../../core/nudge.ts"; @@ -17,15 +20,19 @@ import { runExtract, type ExtractMethod, type ExtractOptions } from "../../adapt import { formatRequestCost } from "../../core/http.ts"; import { parse, asString, asNumber, type Command, type RunContext } from "../command.ts"; import { printError, writeOut } from "../output.ts"; +import { PREPARE_HELP, PREPARE_USAGE, runPrepare } from "./extract-prepare.ts"; export const extract: Command = { name: "extract", summary: "Turn a protected page into structured data (Extract / Autoparse / CSS / Markdown).", - usage: "zenrows extract [--autoparse | --css | --outputs | --output md|text] [flags]", + usage: + "zenrows extract [--autoparse | --css | --outputs | --output md|text] [flags]\n " + + PREPARE_USAGE, help: [ "Methods:", - " (default) extract=auto: site-tailored Extract; falls back to Autoparse if the domain is not enabled or prepared", - " --autoparse general-purpose Autoparse (any domain, no Extract fallback)", + " (default) extract=auto: site-tailored Extract on any prepared domain", + " (EXTRACT_DOMAIN_NOT_PREPARED = not prepared yet: run zenrows extract prepare, below)", + " --autoparse deprecated general-purpose Autoparse (any domain)", " --css CSS selector map, e.g. '{\"title\":\"h1\",\"price\":\".price\"}'", " --outputs built-in output filters → JSON: emails, phone_numbers, headings,", " images, audios, videos, links, menus, hashtags, metadata, tables,", @@ -40,8 +47,12 @@ export const extract: Command = { "", "Cost (credits per request): 1x normal · 5x --js-render · 10x --premium-proxy · 25x both.", "The exact charge is reported after every request (costCredits / X-Request-Credits).", + "", + "Prepare a domain:", + PREPARE_HELP.split("\n").map((l) => (l ? " " + l : l)).join("\n"), ].join("\n"), async run(argv: string[], ctx: RunContext): Promise { + if (argv[0] === "prepare") return runPrepare(argv.slice(1), ctx); const { values, positionals } = parse(argv, { autoparse: { type: "boolean" }, css: { type: "string" }, @@ -122,6 +133,9 @@ export const extract: Command = { const runId = newRunId(); const startedAt = new Date().toISOString(); log.step(`Extract ${url} (method=${opts.method ?? "extract"})…`); + if (opts.method === "autoparse") { + log.warn("Autoparse is deprecated. Prefer the default extract=auto on a prepared domain, or --css."); + } try { const outcome = await runExtract(opts, config, policy, apiKey); @@ -129,10 +143,6 @@ export const extract: Command = { const safeParams = { ...outcome.params }; delete (safeParams as Record).apikey; - if (outcome.fellBackToAutoparse) { - log.info("Extract not available for this domain yet — fell back to Autoparse."); - } - const runDir = writeRun( { runId, @@ -145,7 +155,6 @@ export const extract: Command = { request: { ...safeParams, method: outcome.method, - fellBackToAutoparse: outcome.fellBackToAutoparse ?? false, }, result: { httpStatus: outcome.result.status, bytes: outcome.result.body.length, parsed: outcome.data !== undefined }, costUsd: outcome.result.costUsd, @@ -160,9 +169,8 @@ export const extract: Command = { writeOut(values.out as string, payload); log.success(`Wrote output → ${values.out}`); } - const via = outcome.fellBackToAutoparse ? `${outcome.method} (fallback)` : outcome.method; log.success( - `Extracted via ${via} · ${outcome.result.body.length} bytes · ${formatRequestCost(outcome.result.costUsd, outcome.result.costCredits)} · run ${runId}`, + `Extracted via ${outcome.method} · ${outcome.result.body.length} bytes · ${formatRequestCost(outcome.result.costUsd, outcome.result.costCredits)} · run ${runId}`, ); if (runDir) log.dim(` artifact: ${runDir}`); if (outcome.noDataExtracted) { @@ -182,7 +190,6 @@ export const extract: Command = { ok: true, runId, method: outcome.method, - fellBackToAutoparse: outcome.fellBackToAutoparse ?? false, empty: outcome.empty ?? false, noDataExtracted: outcome.noDataExtracted ?? false, httpStatus: outcome.result.status, diff --git a/src/core/errors.ts b/src/core/errors.ts index 1c4da84..0ecc0bf 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -28,8 +28,17 @@ export type ErrorCode = | "FETCH_FAILED" | "FETCH_EMPTY_RESPONSE" | "EXTRACT_FAILED" + | "EXTRACT_DOMAIN_NOT_PREPARED" + | "EXTRACT_PAGE_NOT_RECOGNISED" | "EXTRACT_DOMAIN_NOT_ENABLED" | "EXTRACT_VALIDATION_FAILED" + | "EXTRACT_PREPARE_INVALID" + | "EXTRACT_PREPARE_NOT_FOUND" + | "EXTRACT_PREPARE_IN_PROGRESS" + | "EXTRACT_PREPARE_OUT_OF_ALLOWANCE" + | "EXTRACT_PREPARE_UNAVAILABLE" + | "EXTRACT_PREPARE_FAILED" + | "EXTRACT_PREPARE_TIMEOUT" | "BROWSER_UNAVAILABLE" | "BATCH_ACCESS_DENIED" | "BATCH_QUOTA_EXCEEDED" diff --git a/src/core/http.ts b/src/core/http.ts index 8619ccb..0806193 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -11,6 +11,7 @@ import { ENV_KEY, resolveApiKey } from "./auth.ts"; import { registerSecret } from "./logger.ts"; import { CLI_USER_AGENT } from "./config.ts"; import { agentClientHeader } from "./agent-client.ts"; +import { normalizeDomain, PAGE_NOT_RECOGNISED_STEPS, prepareCommand } from "./prepared-domains-api.ts"; export interface ScraperResult { status: number; @@ -145,18 +146,10 @@ export async function scrape( redactedUrl: redacted, }; - // REQS007 on extract=auto: the domain hasn't been prepared for Extract yet. - // Same recovery as AUTH010 (the extract adapter falls back to autoparse), so it - // maps to the same code. Checked before the 403 auth branch below. + // REQS007 on extract=auto: Extract has not prepared the domain yet. Reported, + // never retried: the fix is to prepare the domain, which the error names. if (res.status === 403 && params.extract !== undefined && zrErrorCode(body) === "REQS007") { - throw new ToolkitError({ - code: "EXTRACT_DOMAIN_NOT_ENABLED", - message: "Extract has not been prepared for this domain yet.", - likely_cause: zrErrorDetail(body) ?? "This domain is not prepared for Extract.", - next_action: - "Retry with --autoparse for general-purpose extraction on any site, or get the domain prepared for Extract (POST /v1/extract/prepared-domains).", - suggested_commands: [`zenrows extract ${params.url} --autoparse`], - }); + throw extractDomainNotPrepared(params.url, body); } // 422 no_data_extracted on an Extract request: the page was processed and had // nothing to extract. It is an answer, not a failure to reach the target, so @@ -214,18 +207,10 @@ export async function scrape( } } if (res.status === 402 && isZenrowsErrorEnvelope(body)) { - // AUTH010 on extract=auto means the target domain is not enabled for - // Extract, distinct from credit exhaustion (AUTH004). Let the extract - // adapter fall back to autoparse. + // AUTH010 on extract=auto is the legacy code for REQS007: the domain is + // not prepared for Extract. Distinct from credit exhaustion (AUTH004). if (params.extract !== undefined && zrErrorCode(body) === "AUTH010") { - throw new ToolkitError({ - code: "EXTRACT_DOMAIN_NOT_ENABLED", - message: "Extract is not enabled for this domain yet.", - likely_cause: zrErrorDetail(body) ?? "This domain is not enabled for Extract.", - next_action: - "Retry with --autoparse for general-purpose extraction on any site, or get the domain prepared for Extract (POST /v1/extract/prepared-domains).", - suggested_commands: [`zenrows extract ${params.url} --autoparse`], - }); + throw extractDomainNotPrepared(params.url, body); } // AUTH014: this key hit one of its credit caps; the account still has credits. if (isKeyCapReached(zrErrorCode(body) ?? undefined)) { @@ -316,6 +301,79 @@ export async function scrape( return result; } +/** Why a REQS007 refused, as the gateway's `reason` field names it. */ +export type RefusalReason = "domain_not_prepared" | "page_not_recognised" | "domain_not_enabled"; + +const REFUSAL_REASONS: readonly string[] = ["domain_not_prepared", "page_not_recognised", "domain_not_enabled"]; + +/** + * Which REQS007 this is. Only domain_not_prepared is fixed by preparing the + * domain: on page_not_recognised the domain is already prepared, and + * resubmitting it returns the existing ready record, so pointing there would + * loop. + * + * The gateway says it in the problem's `reason` field, read first. The sentence + * match below is only the fallback for a gateway deployed before that field + * existed, and reads the titles that gateway sends; remove it once every gateway + * sends `reason`. + */ +export function refusalReason(body: string): RefusalReason { + let j: { reason?: unknown; title?: unknown; detail?: unknown }; + try { + j = JSON.parse(body) as typeof j; + } catch { + return "domain_not_prepared"; + } + if (typeof j.reason === "string" && REFUSAL_REASONS.includes(j.reason)) return j.reason as RefusalReason; + const title = typeof j.title === "string" ? j.title : ""; + const detail = typeof j.detail === "string" ? j.detail : ""; + if (/did not recogni[sz]e this page/i.test(title) || /^Extract serves /.test(detail)) return "page_not_recognised"; + if (/^Extract is not enabled/i.test(title)) return "domain_not_enabled"; + return "domain_not_prepared"; +} + +/** + * REQS007 (or the legacy AUTH010) on extract=auto. Each reason names its own + * next step, because agents copy it: the exact prepare command for a domain not + * prepared yet, the gateway's advice in the gateway's order for a page it did not + * recognise. None offers Autoparse. + */ +export function extractDomainNotPrepared(url: string, body: string): ToolkitError { + const domain = normalizeDomain(url); + const detail = zrErrorDetail(body); + const css = `zenrows extract ${url} --css '{"title":"h1"}'`; + const reason = refusalReason(body); + if (reason === "page_not_recognised") { + return new ToolkitError({ + code: "EXTRACT_PAGE_NOT_RECOGNISED", + message: `Extract serves ${domain} but did not recognise this page as one it can extract (it may be a page with nothing to extract, such as a home page). ${PAGE_NOT_RECOGNISED_STEPS} The request was not charged.`, + likely_cause: detail ?? `Extract serves ${domain} but did not recognise this page.`, + next_action: PAGE_NOT_RECOGNISED_STEPS, + suggested_commands: [css], + server_code: "REQS007", + }); + } + if (reason === "domain_not_enabled") { + return new ToolkitError({ + code: "EXTRACT_DOMAIN_NOT_ENABLED", + message: `Extract is not enabled for ${domain}, and preparing it does not change this.`, + likely_cause: detail ?? `Extract is not enabled for ${domain}.`, + next_action: "Extract this page with --css and explicit selectors, or contact Zenrows support to have the site enabled.", + suggested_commands: [css], + server_code: "REQS007", + }); + } + const prepare = prepareCommand(url); + return new ToolkitError({ + code: "EXTRACT_DOMAIN_NOT_PREPARED", + message: `Extract hasn't prepared ${domain} yet. Prepare it with \`${prepare}\` (a few example URLs, a few minutes, no credits), then retry; meanwhile use --css.`, + likely_cause: detail ?? `Extract has not prepared ${domain} yet.`, + next_action: `Run \`${prepare} --wait\` (up to 5 --url pages, one per page kind you want), then retry \`zenrows extract ${url}\`. Without --wait, check progress about every 30 seconds with \`zenrows extract prepare status ${domain}\`. Meanwhile extract with --css and explicit selectors.`, + suggested_commands: [`${prepare} --wait`, `zenrows extract prepare status ${domain}`, css], + server_code: "REQS007", + }); +} + /** * The CLI gave up waiting before the API answered. This is NOT unreachability: * the connection was established and the gateway was still working, so telling diff --git a/src/core/prepared-domains-api.ts b/src/core/prepared-domains-api.ts new file mode 100644 index 0000000..69bb5e0 --- /dev/null +++ b/src/core/prepared-domains-api.ts @@ -0,0 +1,336 @@ +/** + * Client for Extract's prepared-domains API. + * + * `extract=auto` answers REQS007 on a domain Extract has not prepared yet. Preparing + * it is a call any key can make: `POST {apiBase}/extract/prepared-domains` with the + * domain and 1 to 5 example URLs answers 202 at once; the preparation itself takes a + * few minutes and costs no credits. `GET .../{domain}` reports progress and + * `GET .../allowance` the account's monthly allowance of new domains. Auth is the + * `X-API-Key` header; errors are problem+json and we branch on `status` + `code`. + * `fetchImpl` / `sleepImpl` / `now` are injectable for tests. + */ +import { ToolkitError } from "./errors.ts"; +import { registerSecret } from "./logger.ts"; +import { CLI_USER_AGENT, loadConfig } from "./config.ts"; +import { agentClientHeader } from "./agent-client.ts"; + +export const MAX_EXAMPLE_URLS = 5; +/** How often `--wait` polls, and for how long by default. */ +export const DEFAULT_POLL_INTERVAL_MS = 30_000; +export const DEFAULT_WAIT_TIMEOUT_MS = 15 * 60_000; + +/** status: preparing | ready | failed. state adds the step (received, discovering, preparing, verifying). */ +export interface PreparedDomain { + domain: string; + status: string; + state?: string; + roles?: { role: string; status: string; url?: string; detail?: string }[]; + failure?: { code: string; detail?: string; slot_consumed?: boolean }; + submitted_at?: string; + updated_at?: string; + [k: string]: unknown; +} + +export interface Allowance { + used: number; + limit: number; + remaining: number; + month?: string; + [k: string]: unknown; +} + +export interface PrepareRequestOpts { + apiKey: string; + apiBase?: string; + fetchImpl?: typeof fetch; +} + +/** + * The domain as the API keys it: host only, no scheme, path or leading `www.`. + * Accepts a URL. + * + * Deliberately not the registrable domain: Extract prepares and serves each host + * on its own. A prepared `es.wikipedia.org` does not cover `pt.wikipedia.org` + * (REQS007 names `pt.wikipedia.org`), and `wikipedia.org` has no record of its + * own. The API itself folds `www.` into the bare host, so dropping it here + * matches what it stores. + */ +export function normalizeDomain(input: string): string { + const trimmed = input.trim(); + let host = trimmed; + try { + host = new URL(/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`).hostname; + } catch { + // Left as given: the API answers invalid_argument with the reason. + } + return host.toLowerCase().replace(/^www\./, ""); +} + +/** The exact command an agent runs to prepare the domain of `url`. */ +export function prepareCommand(url: string): string { + return `zenrows extract prepare ${normalizeDomain(url)} --url ${url}`; +} + +function base(opts: PrepareRequestOpts): string { + return (opts.apiBase ?? loadConfig().apiBase).replace(/\/+$/, ""); +} + +async function request(method: "GET" | "POST", path: string, domain: string, opts: PrepareRequestOpts & { body?: unknown }): Promise { + registerSecret(opts.apiKey); + const headers: Record = { + "X-API-Key": opts.apiKey, + Accept: "application/json", + "User-Agent": CLI_USER_AGENT, + ...agentClientHeader(), + }; + if (opts.body !== undefined) headers["Content-Type"] = "application/json"; + let res: Response; + try { + res = await (opts.fetchImpl ?? fetch)(`${base(opts)}/extract/prepared-domains${path}`, { + method, + headers, + body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined, + }); + } catch (err) { + throw new ToolkitError({ + code: "BACKEND_UNAVAILABLE", + message: "Could not reach the Zenrows API.", + likely_cause: err instanceof Error ? err.message : String(err), + next_action: "Check connectivity and retry. Verify the api base in `zenrows config show`.", + suggested_commands: ["zenrows status", "zenrows config show"], + }); + } + const text = await res.text(); + if (res.status < 200 || res.status >= 300) throw prepareError(res.status, text, domain); + try { + return JSON.parse(text) as T; + } catch { + throw new ToolkitError({ + code: "EXTRACT_PREPARE_FAILED", + message: "The prepared-domains API returned a non-JSON answer.", + likely_cause: text.slice(0, 200), + next_action: "Retry in a minute.", + }); + } +} + +/** Map a non-2xx prepared-domains answer to an error that names the next step. */ +export function prepareError(status: number, body: string, domain: string): ToolkitError { + let p: Record = {}; + try { + p = (JSON.parse(body) as Record) ?? {}; + } catch { + // not problem+json + } + const code = typeof p.code === "string" ? p.code.toLowerCase() : undefined; + const detail = typeof p.detail === "string" ? p.detail : body.slice(0, 200) || `HTTP ${status}`; + const cause = code ? `(${code}) ${detail}` : detail; + if (code === "invalid_argument" || status === 400) { + return new ToolkitError({ + code: "EXTRACT_PREPARE_INVALID", + message: `The request to prepare ${domain} was rejected.`, + likely_cause: p.reason ? `${cause} [reason: ${String(p.reason)}]` : cause, + next_action: `Pass the domain without scheme or path, and 1 to ${MAX_EXAMPLE_URLS} full page URLs from that site with --url.`, + suggested_commands: [`zenrows extract prepare ${domain} --url https://${domain}/`], + server_code: code, + }); + } + if (code === "not_found" || status === 404) { + return new ToolkitError({ + code: "EXTRACT_PREPARE_NOT_FOUND", + message: `This account has not submitted ${domain} for preparation.`, + likely_cause: cause, + next_action: `Prepare it first: zenrows extract prepare ${domain} --url (1 to ${MAX_EXAMPLE_URLS} --url).`, + suggested_commands: [`zenrows extract prepare ${domain} --url https://${domain}/`], + server_code: code, + }); + } + if (code === "in_progress") { + return new ToolkitError({ + code: "EXTRACT_PREPARE_IN_PROGRESS", + message: `${domain} is already being prepared.`, + likely_cause: cause, + next_action: "Do not resubmit. Retry the extract in a few minutes.", + suggested_commands: [`zenrows extract prepare status ${domain}`], + server_code: code, + }); + } + if (code === "idempotency_conflict") { + return new ToolkitError({ + code: "EXTRACT_PREPARE_INVALID", + message: "This idempotency key was already used with a different request.", + likely_cause: cause, + next_action: "Retry without the key, or with a new one.", + server_code: code, + }); + } + if (code === "out_of_allowance" || status === 429) { + const used = p.used !== undefined && p.limit !== undefined ? ` (${String(p.used)} of ${String(p.limit)} used)` : ""; + return new ToolkitError({ + code: "EXTRACT_PREPARE_OUT_OF_ALLOWANCE", + message: `This account has used its monthly allowance of new domains to prepare${used}.`, + likely_cause: cause, + next_action: "Do not retry this month. Meanwhile extract this domain with --css and explicit selectors.", + suggested_commands: [`zenrows extract --css '{"title":"h1"}'`], + server_code: code, + }); + } + if (code === "overloaded" || code === "disabled" || status === 503) { + return new ToolkitError({ + code: "EXTRACT_PREPARE_UNAVAILABLE", + message: "Domain preparation is unavailable right now.", + likely_cause: cause, + next_action: "Retry in a few minutes; do not loop. Meanwhile extract with --css and explicit selectors.", + server_code: code, + }); + } + if (status === 401 || status === 403) { + return new ToolkitError({ + code: "AUTH_INVALID", + message: "Zenrows rejected the API key.", + likely_cause: cause, + next_action: "Re-check your key and log in again.", + suggested_commands: ["zenrows login --api-key "], + server_code: code, + }); + } + return new ToolkitError({ + code: "EXTRACT_PREPARE_FAILED", + message: `Preparing ${domain} failed with HTTP ${status}.`, + likely_cause: cause, + next_action: "Retry in a few minutes.", + server_code: code, + }); +} + +export function submitPreparedDomain( + domain: string, + exampleUrls: string[], + opts: PrepareRequestOpts, +): Promise { + return request("POST", "", domain, { ...opts, body: { domain, example_urls: exampleUrls } }); +} + +export function getPreparedDomain(domain: string, opts: PrepareRequestOpts): Promise { + return request("GET", `/${encodeURIComponent(domain)}`, domain, opts); +} + +export function getAllowance(opts: PrepareRequestOpts): Promise { + return request("GET", "/allowance", "", opts); +} + +/** + * Submit a domain. A 409 in_progress (already being prepared) answers with the + * current status when this account can read it, so the caller just follows it. + */ +export async function prepareDomain( + domain: string, + exampleUrls: string[], + opts: PrepareRequestOpts, +): Promise { + try { + return await submitPreparedDomain(domain, exampleUrls, opts); + } catch (err) { + if (!(err instanceof ToolkitError) || err.code !== "EXTRACT_PREPARE_IN_PROGRESS") throw err; + try { + return { ...(await getPreparedDomain(domain, opts)), already_in_progress: true }; + } catch { + throw err; + } + } +} + +export function isTerminal(record: PreparedDomain): boolean { + return record.status === "ready" || record.status === "failed"; +} + +/** Poll until ready or failed. Throws EXTRACT_PREPARE_TIMEOUT when the wait runs out. */ +export async function waitForPreparedDomain( + domain: string, + opts: PrepareRequestOpts & { + intervalMs?: number; + timeoutMs?: number; + sleepImpl?: (ms: number) => Promise; + now?: () => number; + onPoll?: (record: PreparedDomain) => void; + }, +): Promise { + const sleep = opts.sleepImpl ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const now = opts.now ?? Date.now; + const interval = opts.intervalMs ?? DEFAULT_POLL_INTERVAL_MS; + const deadline = now() + (opts.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS); + for (;;) { + const record = await getPreparedDomain(domain, opts); + opts.onPoll?.(record); + if (isTerminal(record)) return record; + if (now() + interval > deadline) { + throw new ToolkitError({ + code: "EXTRACT_PREPARE_TIMEOUT", + message: `${domain} is still being prepared (state: ${record.state ?? record.status}).`, + likely_cause: "Preparation usually takes a few minutes; this one is taking longer than the wait.", + next_action: "Check again later; it keeps running on the server.", + suggested_commands: [`zenrows extract prepare status ${domain}`], + }); + } + await sleep(interval); + } +} + +/** Whether a failed preparation used an allowance slot, from the record's failure.slot_consumed. */ +function slotSentence(slotConsumed: boolean | undefined): string { + if (slotConsumed === true) return " This attempt used an allowance slot."; + if (slotConsumed === false) return " This attempt did not use an allowance slot."; + return ""; +} + +/** What to do after a failed preparation, by failure.code (docs.zenrows.com extract/endpoints). */ +export function failureNextStep(domain: string, failure: PreparedDomain["failure"]): string { + let step: string; + switch (failure?.code) { + case "interrupted": + case "preparation_exhausted": + step = `Run the same zenrows extract prepare ${domain} command again.`; + break; + case "discovery_failed": + step = `Run zenrows extract prepare ${domain} again with 1 to ${MAX_EXAMPLE_URLS} --url page URLs.`; + break; + case "unpreparable": + step = `Run zenrows extract prepare ${domain} again with different --url pages from the same site; the same URLs fail again.`; + break; + case "preparation_failed": + step = "Try other --url pages, or contact Zenrows support."; + break; + default: + step = "Contact Zenrows support before preparing this domain again. Meanwhile extract with --css."; + } + return step + slotSentence(failure?.slot_consumed); +} + +/** + * What to do when a page on a prepared domain answers REQS007 with reason + * page_not_recognised, in the gateway's own order. On a prepared domain Extract + * learns a new kind of page on its first request, so that answer means this page + * is not one Extract extracts (a page with nothing to extract, such as a home page) or a + * transient failure kept it from trying, never that the domain needs preparing + * again: resubmitting returns the existing ready record, so pointing there would + * loop. + */ +export const PAGE_NOT_RECOGNISED_STEPS = + "Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use --css with explicit selectors. Preparing the domain again does not change this."; + +/** Said once a domain is ready, so a user who then meets EXTRACT_PAGE_NOT_RECOGNISED does not resubmit. */ +const IF_PAGE_NOT_RECOGNISED = `If the extract then fails with EXTRACT_PAGE_NOT_RECOGNISED, Extract did not recognise that page. ${PAGE_NOT_RECOGNISED_STEPS}`; + +/** + * The next step for a status record. `alreadyReady` is set when a submit answered + * ready at once: the domain was prepared before this call. + */ +export function statusNextStep(record: PreparedDomain, opts: { exampleUrl?: string; alreadyReady?: boolean } = {}): string { + if (record.status === "ready") { + const extract = opts.exampleUrl ? ` (\`zenrows extract ${opts.exampleUrl}\`)` : ""; + if (opts.alreadyReady) return `${record.domain} is already prepared, so nothing new was submitted: run the extract${extract}. ${IF_PAGE_NOT_RECOGNISED}`; + return `${record.domain} is ready: run the extract again${extract}. ${IF_PAGE_NOT_RECOGNISED}`; + } + if (record.status === "failed") return failureNextStep(record.domain, record.failure); + return `Preparing; it takes a few minutes. Check about every 30 seconds with: zenrows extract prepare status ${record.domain}`; +} diff --git a/templates/extract-pricing-pages/README.md b/templates/extract-pricing-pages/README.md index 81e4001..408e1e3 100644 --- a/templates/extract-pricing-pages/README.md +++ b/templates/extract-pricing-pages/README.md @@ -3,13 +3,11 @@ Extract structured pricing fields from product pages with Zenrows Extract. ```bash -# Default: extract=auto on an enabled domain (site-tailored JSON) +# Default: extract=auto on a prepared domain (site-tailored JSON). +# Not prepared yet (EXTRACT_DOMAIN_NOT_PREPARED)? zenrows extract prepare --url --wait zenrows extract https://www.owler.com/company/meltwater -# Autoparse — general-purpose JSON on any domain -zenrows extract https://www.scrapingcourse.com/ecommerce/ --autoparse - -# Or a precise CSS selector map +# A precise CSS selector map, on any domain zenrows extract --css '{"title":"h1","price":".price"}' --validate ``` diff --git a/tests/capabilities.test.ts b/tests/capabilities.test.ts index 1ea23fd..846fcf5 100644 --- a/tests/capabilities.test.ts +++ b/tests/capabilities.test.ts @@ -13,6 +13,17 @@ test("capability matrix loads with the honest classifications", () => { assert.equal(caps.crawl?.status, "available"); }); +test("extract notes name the prepare step and no automatic autoparse fallback", () => { + const notes = loadCapabilities().extract?.notes ?? ""; + assert.match(notes, /POST \/v1\/extract\/prepared-domains/); + assert.match(notes, /zenrows extract prepare --url /); + assert.match(notes, /zenrows extract prepare status /); + assert.match(notes, /EXTRACT_PAGE_NOT_RECOGNISED/); + assert.match(notes, /preparing again does not change it/); + assert.match(notes, /autoparse is deprecated/); + assert.doesNotMatch(notes, /falls? back/i); +}); + test("isUsable true for available + open beta, false for absent capabilities", () => { assert.equal(isUsable("protected_fetch"), true); assert.equal(isUsable("extract"), true); diff --git a/tests/extract-outputs.test.ts b/tests/extract-outputs.test.ts index 2a4158e..cf0884f 100644 --- a/tests/extract-outputs.test.ts +++ b/tests/extract-outputs.test.ts @@ -2,10 +2,11 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { normalizeOutputs } from "../src/cli/commands/extract.ts"; import { buildParams, type FetchOptions } from "../src/adapters/protected-fetch.ts"; -import { runExtract } from "../src/adapters/extract.ts"; +import { FALLBACK_AUTOPARSE_DEPRECATION, isEmptyData, runExtract } from "../src/adapters/extract.ts"; import { defaultConfig } from "../src/core/config.ts"; import { defaultPolicy } from "../src/core/policy.ts"; import { ToolkitError } from "../src/core/errors.ts"; +import { refusalReason } from "../src/core/http.ts"; const cfg = defaultConfig(); @@ -142,7 +143,6 @@ test("runExtract default method is extract=auto and unwraps {parsed,html}", asyn assert.equal(outcome.params.autoparse, undefined); assert.deepEqual(outcome.data, { title: "X" }); assert.equal(outcome.html, "x"); - assert.equal(outcome.fellBackToAutoparse, undefined); }, ); }); @@ -174,66 +174,192 @@ test("runExtract --autoparse skips extract and does not unwrap envelope", async }); }); -test("runExtract falls back to autoparse on AUTH010 for extract=auto", async () => { +test("runExtract reports REQS007 with the prepare step and does not retry", async () => { + const REQS007 = JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for x.com yet.", status: 403 }); const calls: string[] = []; await withFetchHandler((url) => { calls.push(url); - if (url.includes("extract=auto")) { - return new Response(AUTH010, { status: 402, headers: { "content-type": "application/json" } }); - } - return new Response('{"title":"fallback"}', { - status: 200, - headers: { - "content-type": "application/json", - "x-request-cost": "0.0003625", - "x-request-credits": "1", + return new Response(REQS007, { status: 403, headers: { "content-type": "application/json" } }); + }, async () => { + await assert.rejects( + () => runExtract({ url: "https://www.x.com/item/1" }, cfg, defaultPolicy(), "testkey"), + (e: unknown) => { + assert.ok(e instanceof ToolkitError); + assert.equal(e.code, "EXTRACT_DOMAIN_NOT_PREPARED"); + assert.equal( + e.message, + "Extract hasn't prepared x.com yet. Prepare it with `zenrows extract prepare x.com --url https://www.x.com/item/1` (a few example URLs, a few minutes, no credits), then retry; meanwhile use --css.", + ); + assert.deepEqual(e.toJSON().suggested_commands, [ + "zenrows extract prepare x.com --url https://www.x.com/item/1 --wait", + "zenrows extract prepare status x.com", + `zenrows extract https://www.x.com/item/1 --css '{"title":"h1"}'`, + ]); + assert.doesNotMatch(JSON.stringify(e.toJSON()), /autoparse/); + return true; }, - }); + ); + assert.equal(calls.length, 1, "REQS007 must not be retried with autoparse"); + }); +}); + +/** A REQS007 the way the gateway sends it: with `reason`, or (an older gateway) without. */ +function reqs007(kind: "domain" | "page" | "allowlist", withReason: boolean): string { + const bodies = { + domain: { + title: "Extract is not prepared for the requested domain (x.com) (REQS007)", + detail: "Extract has not been prepared for x.com yet.", + reason: "domain_not_prepared", + }, + page: { + title: "Extract did not recognise this page on x.com (REQS007)", + detail: "Extract serves x.com, but did not recognise this page as one it can extract.", + reason: "page_not_recognised", + }, + allowlist: { + title: "Extract is not enabled for the requested domain (x.com) (REQS007)", + detail: "Extract is not enabled for the requested domain (x.com) yet.", + reason: "domain_not_enabled", + }, + } as const; + const { reason, ...rest } = bodies[kind]; + return JSON.stringify({ code: "REQS007", status: 403, ...rest, ...(withReason ? { reason } : {}) }); +} + +async function reqs007Error(body: string, url = "https://x.com/search?q=a"): Promise { + let calls = 0; + let caught: unknown; + await withFetchHandler(() => { + calls += 1; + return new Response(body, { status: 403, headers: { "content-type": "application/json" } }); }, async () => { - const outcome = await runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"); - assert.equal(outcome.method, "autoparse"); - assert.equal(outcome.fellBackToAutoparse, true); - assert.equal(outcome.params.autoparse, true); - assert.deepEqual(outcome.data, { title: "fallback" }); - assert.equal(outcome.result.costCredits, 1); - assert.equal(calls.length, 2); - assert.match(calls[0]!, /extract=auto/); - assert.match(calls[1]!, /autoparse=true/); - assert.doesNotMatch(calls[1]!, /extract=auto/); + caught = await runExtract({ url }, cfg, defaultPolicy(), "testkey").then( + () => undefined, + (e: unknown) => e, + ); + }); + assert.equal(calls, 1, "a REQS007 is never retried"); + assert.ok(caught instanceof ToolkitError); + return caught; +} + +for (const withReason of [true, false]) { + const via = withReason ? "the reason field" : "the title (gateway without the reason field)"; + + test(`REQS007 page_not_recognised, read from ${via}: the gateway's advice in order, no prepare command`, async () => { + const e = await reqs007Error(reqs007("page", withReason)); + assert.equal(e.code, "EXTRACT_PAGE_NOT_RECOGNISED"); + let last = -1; + for (const step of [ + /product, article or listing page/, + /retry it once in a few minutes; do not retry in a loop/, + /--css with explicit selectors/, + /Preparing the domain again does not change this/, + ]) { + const at = e.message.search(step); + assert.ok(at > last, `${step} missing or out of order in: ${e.message}`); + last = at; + } + const shape = JSON.stringify(e.toJSON()); + assert.doesNotMatch(shape, /extract prepare/, "no prepare command: it would answer ready and loop"); + assert.doesNotMatch(shape, /autoparse|contact Zenrows support/); + }); + + test(`REQS007 domain_not_enabled, read from ${via}: no prepare command, --css`, async () => { + const e = await reqs007Error(reqs007("allowlist", withReason)); + assert.equal(e.code, "EXTRACT_DOMAIN_NOT_ENABLED"); + assert.match(e.message, /preparing it does not change this/); + const shape = JSON.stringify(e.toJSON()); + assert.doesNotMatch(shape, /extract prepare|autoparse/); + assert.match(shape, /--css/); }); + + test(`REQS007 domain_not_prepared, read from ${via}: the prepare command`, async () => { + const e = await reqs007Error(reqs007("domain", withReason), "https://www.x.com/p/1"); + assert.equal(e.code, "EXTRACT_DOMAIN_NOT_PREPARED"); + assert.equal(e.toJSON().suggested_commands?.[0], "zenrows extract prepare x.com --url https://www.x.com/p/1 --wait"); + }); +} + +test("refusalReason trusts the field over the sentence, and ignores a reason it does not know", () => { + assert.equal( + refusalReason(JSON.stringify({ title: "Extract did not recognise this page on x.com", reason: "domain_not_prepared" })), + "domain_not_prepared", + ); + assert.equal(refusalReason(JSON.stringify({ reason: "something_new", detail: "Extract serves x.com, but" })), "page_not_recognised"); + assert.equal(refusalReason("not json"), "domain_not_prepared"); }); -test("runExtract falls back to autoparse on REQS007 (domain not prepared) for extract=auto", async () => { - const REQS007 = JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for x.com yet.", status: 403 }); +test("runExtract reports legacy AUTH010 like REQS007 and does not retry", async () => { const calls: string[] = []; await withFetchHandler((url) => { calls.push(url); - if (url.includes("extract=auto")) { - return new Response(REQS007, { status: 403, headers: { "content-type": "application/json" } }); - } - return new Response('{"title":"fallback"}', { status: 200, headers: { "content-type": "application/json" } }); + return new Response(AUTH010, { status: 402, headers: { "content-type": "application/json" } }); }, async () => { - const outcome = await runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"); - assert.equal(outcome.method, "autoparse"); - assert.equal(outcome.fellBackToAutoparse, true); - assert.equal(calls.length, 2); - assert.match(calls[1]!, /autoparse=true/); + await assert.rejects( + () => runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"), + (e: unknown) => + e instanceof ToolkitError && + e.code === "EXTRACT_DOMAIN_NOT_PREPARED" && + e.message.startsWith("Extract hasn't prepared x.com yet. Prepare it with `zenrows extract prepare x.com --url https://x.com`"), + ); + assert.equal(calls.length, 1, "AUTH010 must not be retried with autoparse"); }); }); -test("runExtract does not fall back when fallbackAutoparse is false", async () => { - await withFetch(AUTH010, async () => { - await assert.rejects( - () => - runExtract( - { url: "https://x.com", method: "extract", fallbackAutoparse: false }, - cfg, - defaultPolicy(), - "testkey", - ), - (e: unknown) => e instanceof ToolkitError && e.code === "EXTRACT_DOMAIN_NOT_ENABLED", +/** Runs fn with stderr captured, and returns what was written to it. */ +async function captureStderr(fn: () => Promise): Promise { + const original = process.stderr.write.bind(process.stderr); + let written = ""; + process.stderr.write = ((chunk: string | Uint8Array) => { + written += typeof chunk === "string" ? chunk : Buffer.from(chunk).toString(); + return true; + }) as typeof process.stderr.write; + try { + await fn(); + } finally { + process.stderr.write = original; + } + return written; +} + +// fallbackAutoparse is deprecated and has no effect: the refusal is reported in +// one request, with the prepare step, and the caller is told once. +for (const [name, body, status] of [ + ["REQS007", JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for x.com yet.", status: 403 }), 403], + ["AUTH010", AUTH010, 402], +] as const) { + test(`runExtract ignores fallbackAutoparse: true on ${name}, in one request, with one deprecation warning`, async () => { + const calls: string[] = []; + const stderr = await captureStderr(() => + withFetchHandler((url) => { + calls.push(url); + return new Response(body, { status, headers: { "content-type": "application/json" } }); + }, async () => { + await assert.rejects( + () => runExtract({ url: "https://x.com", fallbackAutoparse: true }, cfg, defaultPolicy(), "testkey"), + (e: unknown) => + e instanceof ToolkitError && + e.code === "EXTRACT_DOMAIN_NOT_PREPARED" && + e.suggested_commands?.includes("zenrows extract prepare x.com --url https://x.com --wait") === true, + ); + }), + ); + assert.equal(calls.length, 1, "fallbackAutoparse must not retry with autoparse"); + assert.doesNotMatch(calls[0]!, /autoparse=true/); + assert.equal(stderr.split(FALLBACK_AUTOPARSE_DEPRECATION).length - 1, 1, `want one deprecation warning, got: ${stderr}`); + }); +} + +test("runExtract prints no deprecation warning without fallbackAutoparse, or with it false", async () => { + for (const fallbackAutoparse of [undefined, false]) { + const stderr = await captureStderr(() => + withFetch(JSON.stringify({ parsed: { title: "X" } }), async () => { + await runExtract({ url: "https://x.com", fallbackAutoparse }, cfg, defaultPolicy(), "testkey"); + }), ); - }, { status: 402 }); + assert.doesNotMatch(stderr, /fallbackAutoparse/); + } }); test("runExtract does not fall back on AUTH004 (credits exhausted)", async () => { @@ -250,20 +376,6 @@ test("runExtract does not fall back on AUTH004 (credits exhausted)", async () => }); }); -test("runExtract surfaces autoparse failure after AUTH010 fallback", async () => { - await withFetchHandler((url) => { - if (url.includes("extract=auto")) { - return new Response(AUTH010, { status: 402, headers: { "content-type": "application/json" } }); - } - return new Response("upstream boom", { status: 500, headers: { "content-type": "text/plain" } }); - }, async () => { - await assert.rejects( - () => runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"), - (e: unknown) => e instanceof ToolkitError && e.code === "FETCH_FAILED", - ); - }); -}); - test("runExtract --validate fails when body is not JSON", async () => { await withFetch("not json", async () => { await assert.rejects( @@ -286,7 +398,7 @@ test("runExtract --css without a selector map fails locally (no network)", async ); }); -// --- ACT-1514: empty extraction must not be a silent success ---------------- +// --- empty extraction must not be a silent success -------------------------- test("runExtract flags an empty extract=auto result (empty=true)", async () => { await withFetch(JSON.stringify({ parsed: {}, html: "" }), async () => { const outcome = await runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"); @@ -308,6 +420,27 @@ test("runExtract flags an empty autoparse result (empty=true)", async () => { }); }); +test("runExtract flags a result whose fields are all null or empty (empty=true)", async () => { + const scaffolding = { + title: null, + author: { name: "", url: null }, + tags: [], + comments: [{ body: null, date: " " }], + }; + await withFetch(JSON.stringify({ parsed: scaffolding, html: "" }), async () => { + const outcome = await runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"); + assert.deepEqual(outcome.data, scaffolding); + assert.equal(outcome.empty, true); + }); +}); + +test("isEmptyData keeps 0, false and one filled nested field as data", () => { + assert.equal(isEmptyData({ count: 0 }), false); + assert.equal(isEmptyData({ in_stock: false }), false); + assert.equal(isEmptyData({ title: null, items: [{ name: null }, { name: "A" }] }), false); + assert.equal(isEmptyData([[], {}, null, ""]), true); +}); + test("runExtract does not flag a non-empty result", async () => { await withFetch(JSON.stringify({ parsed: { title: "X" }, html: "" }), async () => { const outcome = await runExtract({ url: "https://x.com" }, cfg, defaultPolicy(), "testkey"); @@ -348,7 +481,6 @@ test("runExtract reports 422 no_data_extracted as an empty result, in one reques assert.equal(outcome.method, "extract"); assert.equal(outcome.empty, true); assert.equal(outcome.noDataExtracted, true); - assert.equal(outcome.fellBackToAutoparse, undefined); assert.equal(outcome.data, undefined); assert.equal(calls.length, 1); assert.match(calls[0]!, /extract=auto/); diff --git a/tests/extract-prepare.test.ts b/tests/extract-prepare.test.ts new file mode 100644 index 0000000..fe0e657 --- /dev/null +++ b/tests/extract-prepare.test.ts @@ -0,0 +1,275 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { runPrepare } from "../src/cli/commands/extract-prepare.ts"; +import { extract } from "../src/cli/commands/extract.ts"; +import { failureNextStep, normalizeDomain, prepareCommand } from "../src/core/prepared-domains-api.ts"; + +const ctx = { json: true, yes: false }; +const BASE = "/v1/extract/prepared-domains"; +const RECORD = { + domain: "example.com", + status: "preparing", + state: "received", + submitted_at: "2026-10-15T09:00:00Z", + updated_at: "2026-10-15T09:00:00Z", +}; +const ALLOWANCE = { used: 3, limit: 10, remaining: 7, month: "2026-10" }; + +type Call = { method: string; path: string; headers: Record; body?: unknown }; + +function res(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); +} +const problem = (status: number, code: string, extra: Record = {}) => + res({ code, detail: `${code} detail`, status, title: code, ...extra }, status); + +/** A fetch stub answering by "METHOD path"; a route may be a list answered in order. */ +function stub(routes: Record Response) | (() => Response)[]>) { + const calls: Call[] = []; + const fetchImpl = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(String(input)); + const method = init?.method ?? "GET"; + calls.push({ + method, + path: url.pathname, + headers: (init?.headers ?? {}) as Record, + body: init?.body ? JSON.parse(String(init.body)) : undefined, + }); + const route = routes[`${method} ${url.pathname}`]; + if (!route) throw new Error(`unexpected ${method} ${url.pathname}`); + if (Array.isArray(route)) { + const next = route.length > 1 ? route.shift()! : route[0]!; + return next(); + } + return route(); + }) as typeof fetch; + return { calls, fetchImpl }; +} + +async function captureOut(fn: () => unknown): Promise { + const orig = process.stdout.write.bind(process.stdout); + let buf = ""; + process.stdout.write = ((s: string | Uint8Array, ...rest: unknown[]) => { + if (typeof s !== "string") return (orig as (...a: unknown[]) => boolean)(s, ...rest); + buf += s; + return true; + }) as typeof process.stdout.write; + try { + await fn(); + } finally { + process.stdout.write = orig; + } + return buf; +} + +async function run(args: string[], fetchImpl?: typeof fetch, extra: { now?: () => number } = {}) { + let code = -1; + const raw = await captureOut(async () => { + code = await runPrepare(args, ctx, { apiKey: "k", fetchImpl, sleepImpl: async () => {}, intervalMs: 1000, ...extra }); + }); + return { code, out: JSON.parse(raw) as Record }; +} + +test("normalizeDomain and prepareCommand", () => { + assert.equal(normalizeDomain("https://www.Example.com/p/1"), "example.com"); + assert.equal(normalizeDomain("shop.example.com"), "shop.example.com"); + assert.equal(prepareCommand("https://www.example.com/p/1"), "zenrows extract prepare example.com --url https://www.example.com/p/1"); +}); + +test("prepare submits the domain and example URLs, returns at once without --wait", async () => { + const { calls, fetchImpl } = stub({ [`POST ${BASE}`]: () => res(RECORD, 202) }); + const { code, out } = await run( + ["www.example.com", "--url", "https://example.com/p/1", "--url", "https://example.com/list"], + fetchImpl, + ); + assert.equal(code, 0); + assert.equal(out.status, "preparing"); + assert.match(out.next, /zenrows extract prepare status example\.com/); + assert.equal(calls.length, 1); + assert.deepEqual(calls[0]!.body, { domain: "example.com", example_urls: ["https://example.com/p/1", "https://example.com/list"] }); + assert.equal(calls[0]!.headers["X-API-Key"], "k"); +}); + +test("prepare --wait polls until ready", async () => { + const { calls, fetchImpl } = stub({ + [`POST ${BASE}`]: () => res(RECORD, 202), + [`GET ${BASE}/example.com`]: [ + () => res({ ...RECORD, state: "preparing" }), + () => res({ ...RECORD, state: "verifying" }), + () => res({ ...RECORD, status: "ready", state: "ready" }), + ], + }); + const { code, out } = await run(["example.com", "--url", "https://example.com/p/1", "--wait"], fetchImpl); + assert.equal(code, 0); + assert.equal(out.status, "ready"); + assert.match(out.next, /zenrows extract https:\/\/example\.com\/p\/1/); + assert.equal(calls.filter((c) => c.method === "GET").length, 3); +}); + +test("prepare --wait on failed exits 1 with the step for the failure code", async () => { + const { fetchImpl } = stub({ + [`POST ${BASE}`]: () => res(RECORD, 202), + [`GET ${BASE}/example.com`]: () => + res({ ...RECORD, status: "failed", state: "failed", failure: { code: "unpreparable", detail: "x", slot_consumed: false } }), + }); + const { code, out } = await run(["example.com", "--url", "https://example.com/p/1", "--wait"], fetchImpl); + assert.equal(code, 1); + assert.equal(out.error.code, "EXTRACT_PREPARE_FAILED"); + assert.equal(out.error.server_code, "unpreparable"); + assert.match(out.error.next_action, /different --url pages/); +}); + +test("prepare --wait gives up after --timeout with EXTRACT_PREPARE_TIMEOUT", async () => { + let t = 0; + const { fetchImpl } = stub({ + [`POST ${BASE}`]: () => res(RECORD, 202), + [`GET ${BASE}/example.com`]: () => res(RECORD), + }); + const { code, out } = await run(["example.com", "--url", "https://example.com/p/1", "--wait", "--timeout", "3"], fetchImpl, { + now: () => (t += 1000), + }); + assert.equal(code, 1); + assert.equal(out.error.code, "EXTRACT_PREPARE_TIMEOUT"); + assert.deepEqual(out.error.suggested_commands, ["zenrows extract prepare status example.com"]); +}); + +test("prepare on 409 in_progress follows the running preparation", async () => { + const { calls, fetchImpl } = stub({ + [`POST ${BASE}`]: () => problem(409, "in_progress"), + [`GET ${BASE}/example.com`]: () => res({ ...RECORD, state: "verifying" }), + }); + const { code, out } = await run(["example.com", "--url", "https://example.com/p/1"], fetchImpl); + assert.equal(code, 0); + assert.equal(out.already_in_progress, true); + assert.equal(out.state, "verifying"); + assert.equal(calls.length, 2); +}); + +test("prepare on 409 in_progress that this account cannot read reports EXTRACT_PREPARE_IN_PROGRESS", async () => { + const { fetchImpl } = stub({ + [`POST ${BASE}`]: () => problem(409, "in_progress"), + [`GET ${BASE}/example.com`]: () => problem(404, "not_found"), + }); + const { code, out } = await run(["example.com", "--url", "https://example.com/p/1"], fetchImpl); + assert.equal(code, 1); + assert.equal(out.error.code, "EXTRACT_PREPARE_IN_PROGRESS"); + assert.match(out.error.next_action, /Do not resubmit/); +}); + +for (const [status, serverCode, code, pattern] of [ + [400, "invalid_argument", "EXTRACT_PREPARE_INVALID", /1 to 5 full page URLs/], + [429, "out_of_allowance", "EXTRACT_PREPARE_OUT_OF_ALLOWANCE", /Do not retry this month/], + [503, "overloaded", "EXTRACT_PREPARE_UNAVAILABLE", /do not loop/], + [503, "disabled", "EXTRACT_PREPARE_UNAVAILABLE", /do not loop/], +] as const) { + test(`prepare maps ${status} ${serverCode} to ${code}`, async () => { + const { calls, fetchImpl } = stub({ + [`POST ${BASE}`]: () => problem(status, serverCode, serverCode === "out_of_allowance" ? { used: 10, limit: 10 } : {}), + }); + const { code: exit, out } = await run(["example.com", "--url", "https://example.com/p/1"], fetchImpl); + assert.equal(exit, 1); + assert.equal(out.error.code, code); + assert.equal(out.error.server_code, serverCode); + assert.match(out.error.next_action, pattern); + assert.equal(calls.length, 1); + if (serverCode === "out_of_allowance") assert.match(out.error.message, /10 of 10 used/); + }); +} + +test("prepare rejects zero, six or non-URL --url values before any request", async () => { + const { calls, fetchImpl } = stub({}); + for (const args of [ + ["example.com"], + ["example.com", ...Array.from({ length: 6 }, (_, i) => ["--url", `https://example.com/${i}`]).flat()], + ["example.com", "--url", "example.com/p/1"], + [], + ]) { + const { code, out } = await run(args, fetchImpl); + assert.equal(code, 1, args.join(" ")); + assert.equal(out.error.code, "INVALID_USAGE", args.join(" ")); + } + assert.equal(calls.length, 0); +}); + +test("prepare status shows the record, the allowance and the next step", async () => { + const { fetchImpl } = stub({ + [`GET ${BASE}/example.com`]: () => res({ ...RECORD, state: "preparing" }), + [`GET ${BASE}/allowance`]: () => res(ALLOWANCE), + }); + const { code, out } = await run(["status", "https://www.example.com/"], fetchImpl); + assert.equal(code, 0); + assert.equal(out.status, "preparing"); + assert.deepEqual(out.allowance, ALLOWANCE); + assert.match(out.next, /Preparing/); +}); + +test("prepare status keeps the record when the allowance call fails", async () => { + const { fetchImpl } = stub({ + [`GET ${BASE}/example.com`]: () => res({ ...RECORD, status: "ready", state: "ready" }), + [`GET ${BASE}/allowance`]: () => problem(503, "overloaded"), + }); + const { code, out } = await run(["status", "example.com"], fetchImpl); + assert.equal(code, 0); + assert.equal(out.status, "ready"); + assert.equal(out.allowance, undefined); + assert.match(out.next, /example\.com is ready: run the extract again/); + assert.match(out.next, /EXTRACT_PAGE_NOT_RECOGNISED/); +}); + +test("prepare status on 404 says to prepare the domain first", async () => { + const { fetchImpl } = stub({ + [`GET ${BASE}/example.com`]: () => problem(404, "not_found"), + [`GET ${BASE}/allowance`]: () => res(ALLOWANCE), + }); + const { code, out } = await run(["status", "example.com"], fetchImpl); + assert.equal(code, 1); + assert.equal(out.error.code, "EXTRACT_PREPARE_NOT_FOUND"); + assert.match(out.error.next_action, /zenrows extract prepare example\.com --url/); +}); + +test("failureNextStep: the step comes from failure.code, the slot sentence from failure.slot_consumed", () => { + assert.match(failureNextStep("x.com", { code: "interrupted" }), /same zenrows extract prepare/); + assert.match(failureNextStep("x.com", { code: "preparation_exhausted" }), /same zenrows extract prepare/); + assert.match(failureNextStep("x.com", { code: "discovery_failed" }), /--url page URLs/); + assert.match(failureNextStep("x.com", { code: "unpreparable" }), /different --url pages/); + assert.match(failureNextStep("x.com", { code: "preparation_failed" }), /other --url pages/); + assert.match(failureNextStep("x.com", { code: "verification_failed" }), /support/); + assert.match(failureNextStep("x.com", { code: "unsupported" }), /support/); + assert.match(failureNextStep("x.com", { code: "unpreparable", slot_consumed: false }), /did not use an allowance slot/); + assert.match(failureNextStep("x.com", { code: "unpreparable", slot_consumed: true }), /This attempt used an allowance slot/); + assert.doesNotMatch(failureNextStep("x.com", { code: "unpreparable" }), /slot/); +}); + +test("prepare answering ready at once says the domain is already prepared and what a page refusal then means", async () => { + const { calls, fetchImpl } = stub({ [`POST ${BASE}`]: () => res({ ...RECORD, status: "ready", state: "ready" }, 202) }); + const { code, out } = await run(["example.com", "--url", "https://example.com/search", "--wait"], fetchImpl); + assert.equal(code, 0); + assert.equal(out.already_prepared, true); + assert.match(out.next, /already prepared, so nothing new was submitted: run the extract \(`zenrows extract https:\/\/example\.com\/search`\)\./); + assert.match(out.next, /Preparing the domain again does not change this/); + assert.doesNotMatch(out.next, /not covered|contact Zenrows support|autoparse/); + assert.equal(calls.length, 1, "--wait does not poll a domain that is already ready"); +}); + +test("prepare never auto-creates a Free plan account", async () => { + const { calls, fetchImpl } = stub({}); + const saved = process.env.ZENROWS_API_KEY; + delete process.env.ZENROWS_API_KEY; + let code = -1; + const raw = await captureOut(async () => { + code = await runPrepare(["example.com", "--url", "https://example.com/p/1"], ctx, { fetchImpl }); + }); + if (saved !== undefined) process.env.ZENROWS_API_KEY = saved; + assert.equal(code, 1); + assert.equal(JSON.parse(raw).error.code, "AUTH_MISSING"); + assert.equal(calls.length, 0); +}); + +test("zenrows extract routes `prepare` to the prepare command", async () => { + let code = -1; + const raw = await captureOut(async () => { + code = await extract.run(["prepare", "example.com"], ctx); + }); + assert.equal(code, 1); + assert.equal(JSON.parse(raw).error.code, "INVALID_USAGE", "no --url: rejected locally, no extract of a URL named prepare"); +}); diff --git a/tests/http.test.ts b/tests/http.test.ts index c40bcdf..3923ed0 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -138,7 +138,7 @@ test("scrape does NOT advise a (billed) retry when the target cannot be resolved ); }); -test("scrape maps AUTH010 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async () => { +test("scrape maps AUTH010 on extract=auto to EXTRACT_DOMAIN_NOT_PREPARED", async () => { const AUTH010 = JSON.stringify({ code: "AUTH010", detail: "This domain is not enabled for Extract.", @@ -157,7 +157,7 @@ test("scrape maps AUTH010 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async }), (err: unknown) => { assert.ok(err && typeof err === "object" && "code" in err); - assert.equal((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_ENABLED"); + assert.equal((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_PREPARED"); return true; }, ); @@ -165,7 +165,7 @@ test("scrape maps AUTH010 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async ); }); -test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async () => { +test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_PREPARED", async () => { const REQS007 = JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for example.net yet.", @@ -178,7 +178,11 @@ test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async await assert.rejects( () => scrape("https://api.zenrows.com/v1/", "test-key", { url: "https://example.net", extract: "auto" }), (err: unknown) => { - assert.equal((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_ENABLED"); + assert.equal((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_PREPARED"); + assert.equal( + (err as { message: string }).message, + "Extract hasn't prepared example.net yet. Prepare it with `zenrows extract prepare example.net --url https://example.net` (a few example URLs, a few minutes, no credits), then retry; meanwhile use --css.", + ); return true; }, ); @@ -186,7 +190,7 @@ test("scrape maps REQS007 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async ); }); -test("scrape does not map REQS007 to EXTRACT_DOMAIN_NOT_ENABLED without extract=auto", async () => { +test("scrape does not map REQS007 to EXTRACT_DOMAIN_NOT_PREPARED without extract=auto", async () => { const REQS007 = JSON.stringify({ code: "REQS007", detail: "Extract has not been prepared for example.net yet.", status: 403 }); await withFetch( () => new Response(REQS007, { status: 403, headers: { "content-type": "application/json" } }), @@ -194,7 +198,7 @@ test("scrape does not map REQS007 to EXTRACT_DOMAIN_NOT_ENABLED without extract= await assert.rejects( () => scrape("https://api.zenrows.com/v1/", "test-key", { url: "https://example.net", autoparse: true }), (err: unknown) => { - assert.notEqual((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_ENABLED"); + assert.notEqual((err as { code: string }).code, "EXTRACT_DOMAIN_NOT_PREPARED"); return true; }, ); @@ -204,7 +208,7 @@ test("scrape does not map REQS007 to EXTRACT_DOMAIN_NOT_ENABLED without extract= test("scrape treats AUTH010 without extract as a credits/plan error, not EXTRACT_DOMAIN", async () => { // AUTH010 is also used for other plan-gated features; only map it to - // EXTRACT_DOMAIN_NOT_ENABLED when the request asked for extract=auto. + // EXTRACT_DOMAIN_NOT_PREPARED when the request asked for extract=auto. const AUTH010 = JSON.stringify({ code: "AUTH010", detail: "Feature is not included in plan.", diff --git a/workflows/competitor-intelligence/WORKFLOW.md b/workflows/competitor-intelligence/WORKFLOW.md index d0618c0..122762b 100644 --- a/workflows/competitor-intelligence/WORKFLOW.md +++ b/workflows/competitor-intelligence/WORKFLOW.md @@ -9,7 +9,7 @@ status: available A higher-level process built on Zenrows **Fetch and Extract**: retrieve protected pages with `zenrows fetch`, then turn them into structured data with -`zenrows extract` (the same API's Autoparse / CSS / Markdown — not a separate +`zenrows extract` (the same API's `extract=auto` / CSS / Markdown — not a separate product). Scheduling and scaling use the tools you already have (cron/CI, a simple per-URL loop). @@ -20,7 +20,7 @@ simple per-URL loop). ``` zenrows fetch https://competitor.example/products ``` -2. **Validate extraction** on one page (extract=auto falls back to autoparse on a domain not prepared yet). +2. **Validate extraction** on one page (on a domain not prepared yet, `EXTRACT_DOMAIN_NOT_PREPARED`, run `zenrows extract prepare --url --wait` first). ``` zenrows extract https://competitor.example/p/123 --validate ```