From 6605d873864763cae398446851e957bb75818b3c Mon Sep 17 00:00:00 2001 From: Emirlan Rasulov Date: Fri, 16 Oct 2026 16:41:47 +0600 Subject: [PATCH 1/3] feat(batch): agents could not recover, inspect or manage batch jobs from the CLI batch cancel misread the stopped run and always reported status unknown, runs that ran out of credits got advice to fix their tasks, retry-failed could not reach the tasks a stop left pending, the default policy blocked any Adaptive Stealth batch over 400 URLs, and the estimate ignored the job-level flags it would submit. Fix those, and follow the API and the crawl commands for errors, waits and retries. - cancel, pause and resume read the run the API answers with; cancel on an ended run prints the job as it ended - a failed run gets the code and advice of the matching refused request (out of credits, inactive subscription, key cap) with server_code and job_id, and the command that reruns its unfinished tasks - rerun with --status pending,processing,failed,successful or a full rerun; retry-failed --include-pending - each lifecycle 409, the 410 and the content 422 get their own code and advice; every error carries server_code and job_id - create and rerun send an Idempotency-Key and retry transient failures under it; reads retry 429/5xx; an unknown outcome prints the safe repeat - POST tasks send method and body at task level; metadata, external_id, url and body are checked against the API's limits before submitting - the estimate prices job-level flags, nested zenrows_params and "false", as a range for Adaptive Stealth Mode; estimate takes create's flags - default policy caps raised to 10,000 URLs and 250,000 estimated credits - --timeout takes 30s, 5m, 1h (a bare number stays milliseconds) and is validated; create --follow takes it; a wait that runs out or meets a paused run exits 0 with a note; Ctrl-C stops the wait and exits 130 - JSON output keeps job_id, run_id, paused, ingest_status, accepted_tasks and rerun counts, and the full job, alongside the existing keys - new: batch list, content, pause, resume, rerun, delete; create takes --name, --external-id and --metadata --- README.md | 70 +- agent-plugin/skills/batch-jobs/SKILL.md | 19 +- recipes/batch-estimate/RECIPE.md | 9 +- skills/batch-jobs/SKILL.md | 198 ++++-- skills/crawl/SKILL.md | 2 +- skills/trace-debug/SKILL.md | 19 +- skills/zenrows/SKILL.md | 5 +- src/adapters/batch.ts | 309 +++++++-- src/cli/commands/batch.ts | 784 +++++++++++++++++++---- src/cli/commands/crawl.ts | 2 +- src/core/batch-api.ts | 718 ++++++++++++++++----- src/core/errors.ts | 22 + src/core/policy.ts | 19 +- templates/batch-jsonl-pipeline/README.md | 15 +- tests/batch-api.test.ts | 22 +- tests/batch-consistency.test.ts | 650 +++++++++++++++++++ 16 files changed, 2417 insertions(+), 446 deletions(-) create mode 100644 tests/batch-consistency.test.ts diff --git a/README.md b/README.md index 1f7180f..67ae77e 100644 --- a/README.md +++ b/README.md @@ -127,20 +127,72 @@ zenrows extract --output markdown Zenrows **Batch** (`https://async.api.zenrows.com/v1`) fans a protected fetch/extract out over many URLs. The cloud subcommands work -with any active API key and spend the same credits as Fetch. Local spec -validation + credit estimation always work (no key needed). +with any active API key and cost the same credits as Fetch: Batch is the +reliable path at scale, at Fetch's prices. Local spec validation + credit +estimation always work (no key needed). ```bash -zenrows batch estimate jobs.jsonl # validate the spec + estimate credits (local) -zenrows batch create jobs.jsonl --wait # submit a job and wait for it to finish -zenrows batch status # run status + progress stats -zenrows batch results --out results.jsonl # collect results (paginated) -zenrows batch cancel # stop an in-flight run +zenrows batch estimate jobs.jsonl # validate the spec + estimate credits (local) +zenrows batch create jobs.jsonl --follow # submit a job and wait for it to finish +zenrows batch list # your jobs, newest first +zenrows batch status # run status, progress stats, pause state +zenrows batch wait --timeout 30m # wait for the run to end +zenrows batch results --out results.jsonl # collect results (paginated) +zenrows batch content # one task's result body +zenrows batch pause # pause the run (resume continues it) +zenrows batch cancel # stop an in-flight run +zenrows batch retry-failed --include-pending # rerun every task that did not succeed +zenrows batch rerun # rerun every task +zenrows batch delete --yes # delete the job, its runs and results ``` `jobs.jsonl` is one JSON object per line, each with a `url` (plus optional per-line overrides like `js_render`, `premium_proxy`, `proxy_country`, -`extract`). Validate and estimate locally first, then submit with `create`. +`extract`, nested `zenrows_params`, an `external_id`, string `metadata`, and +`"method": "POST"` with a `body` of up to 16 KiB). `estimate` and `create` +check every line against the API's limits before anything is sent: `url` up +to 2048 characters, `external_id` up to 128 letters, digits, `.`, `_` or `-`, +and `metadata` up to 20 string entries (4 KiB). Validate and estimate locally +first, then submit with `create`. `estimate` takes `create`'s +`--manual` / `--js-render` / `--premium-proxy` / `--proxy-country` flags and +prices the same request, as a range (Adaptive Stealth Mode costs 1 to 25). + +`create` also takes `--name`, `--external-id` and `--metadata key=value` +(repeatable) for the job itself. It sends an `Idempotency-Key` (generated, or +`--idempotency-key`) and retries network errors and 502/503/504 under it; an +outcome that stays unknown prints the command that repeats the request with +that key, which cannot create a second job. + +The local policy caps a run at 10,000 URLs and 250,000 estimated credits by +default (an Adaptive Stealth Mode task counts 25), so a default run of up to +10,000 URLs passes in any mode. Raise them for larger jobs (the API takes up to +100,000 tasks per job): `zenrows policy set max_pages_per_run 100000` and +`zenrows policy set max_credits_per_run 2500000`. A workspace created by an +earlier version keeps its caps (1,000 and 10,000) in `policy.json`. + +`wait`, `create --follow` and `rerun --follow` poll until the run ends, the run +is paused, or `--timeout` runs out (default 10 minutes; `30s`, `5m`, `1h`; a +bare number is milliseconds, as before). When the time runs out, or the run is +paused, the command exits 0 and prints the run with a `note` and the next +command; the job keeps running. Ctrl-C stops the wait, not the job, and exits +130. A run that ends `failed` exits 1: out of credits is +`POLICY_MAX_CREDITS_EXCEEDED`, an inactive subscription `AUTH_INVALID`, a key +cap `KEY_CREDIT_CAP_REACHED` (the same codes a refused request gets), anything +else `BATCH_FAILED`, each with the run's `failure_reason` as `server_code` and +the `job_id`. Then run the unfinished tasks with +`zenrows batch rerun --status failed,pending,processing`. A rerun with +`--status` works for 10 days after the run started; a full rerun always does. + +With `--json`, `create`, `status`, `wait`, `cancel`, `pause`, `resume` and +`rerun` print `{ok, job_id, run_id, status, stats, job}` (`jobId` is kept for +older scripts), plus `paused`, `ingest_status`, `accepted_tasks`, `rerun_of`, +`retried_tasks` and `inherited_tasks` when the API sends them. Results are +kept 14 days by default and then read `expired`; a `result_url` is valid 2 +hours. Lifecycle errors get their own codes (`BATCH_RUN_IN_PROGRESS`, +`BATCH_NO_MATCHING_TASKS`, `BATCH_RERUN_WINDOW_EXPIRED`, +`BATCH_REQUEST_IN_FLIGHT`, `BATCH_RESULT_EXPIRED`, ...) and carry the API's code +as `server_code`. Schedules, webhooks, ZIP exports, CSV inputs and open jobs +are available through the Batch API and the SDKs, not this CLI. **Extract in Batch.** Set `"extract": "auto"` on a line to run that URL through Extract — structured data instead of raw HTML: @@ -185,7 +237,7 @@ zenrows crawl stop # stop a running crawl follows links until `--max-items` or `--max-pages` stops it, or nothing is left to open. `--max-items` and `--max-pages` cap the crawl (API default 100 each), and `--max-pages` bounds the cost, since each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default -1000). `--output-format html` uses up the page budget: each kept page is one +10000). `--output-format html` uses up the page budget: each kept page is one more fetch, and `--max-pages` counts it. `--include-patterns` / `--exclude-patterns` are repeatable substring filters on the URL. `get` and `list` read one page and take `--cursor` and `--limit`. `results` follows every diff --git a/agent-plugin/skills/batch-jobs/SKILL.md b/agent-plugin/skills/batch-jobs/SKILL.md index db3ef13..878e533 100644 --- a/agent-plugin/skills/batch-jobs/SKILL.md +++ b/agent-plugin/skills/batch-jobs/SKILL.md @@ -7,7 +7,8 @@ version: 0.1.0 # Batch Jobs (MCP) Process large workloads asynchronously. Batch is where Zenrows' high-scale -anti-bot advantage becomes obvious — thousands to millions of URLs. +anti-bot advantage becomes obvious: thousands to 100,000 URLs per job, at +Fetch's prices. > Cloud Batch lives on `async.api.zenrows.com/v1`. A 403 maps to > `BATCH_ACCESS_DENIED`. Do not fake a successful @@ -33,10 +34,22 @@ Optional if exposed by the server: `batch_wait` to block until terminal status. `autoparse`, `extract`, `external_id`, `metadata`). `extract: "auto"` returns structured data instead of raw HTML, at the same credit cost as a regular task; the result carries `html` and `parsed` keys. -3. Poll `batch_status` until terminal (`completed` / `stopped` / …). -4. Collect with `batch_results`. +3. Poll `batch_status` until terminal (`completed`, `stopped`, `failed` or + `deleted`). A paused run (`pause_state: paused`) is not terminal and picks + up no new task until resumed: report it instead of polling on. +4. Collect with `batch_results`. Results are kept 14 days by default (then a + row reads `expired`); a `result_url` is valid 2 hours. 5. `batch_cancel` if the user aborts or the run should stop early. +A run ends `failed` when an account problem stops it: `failure_reason` +`insufficient_credits` (add credits), `subscription_inactive` (fix it in +Billing) or `api_key_cap_reached` (the key's own cap; the account still has +credits). The tasks not yet run stay `pending`: once fixed, rerun them with +status `failed,pending,processing` rather than resubmitting the job. + +Schedules, webhooks, ZIP exports and CSV inputs are available through the +Batch API and the SDKs. + ## Rules - **Never batch before validating** on a single page ([[extract]] / diff --git a/recipes/batch-estimate/RECIPE.md b/recipes/batch-estimate/RECIPE.md index 84d4bd4..60270e8 100644 --- a/recipes/batch-estimate/RECIPE.md +++ b/recipes/batch-estimate/RECIPE.md @@ -23,11 +23,14 @@ zenrows batch estimate jobs.jsonl ``` Swap `jobs.jsonl` for your own spec. Per-line keys (`js_render`, -`premium_proxy`, `proxy_country`, `mode`, `autoparse`, `external_id`) tune each -URL; cost multipliers are basic 1x / JS 5x / proxy 10x / both 25x. +`premium_proxy`, `proxy_country`, `mode`, `autoparse`, `external_id`, +`metadata`, `method`, `body`) tune each URL; cost multipliers are basic 1x / JS +5x / proxy 10x / both 25x, and Adaptive Stealth Mode costs 1x to 25x. Pass +`create`'s job-level flags (`--premium-proxy`, `--js-render`, `--manual`) to +`estimate` to price the same request. Once the spec is clean, submit it: ``` -zenrows batch create jobs.jsonl --wait +zenrows batch create jobs.jsonl --follow ``` See [[batch-jobs]] for the full cloud flow. diff --git a/skills/batch-jobs/SKILL.md b/skills/batch-jobs/SKILL.md index 79d1fa2..e8ef5f1 100644 --- a/skills/batch-jobs/SKILL.md +++ b/skills/batch-jobs/SKILL.md @@ -1,6 +1,6 @@ --- name: batch-jobs -description: Scale protected fetch/extract over many URLs via Batch. Cloud create/status/results/cancel/wait/retry-failed work with an API key; estimate/validate run locally with no key. +description: "Scale protected fetch/extract over many URLs via Batch. Cloud create/list/status/wait/results/content/cancel/pause/resume/rerun/delete work with an API key; estimate/validate run locally with no key." version: 0.1.0 requires_backend_capabilities: [batch] --- @@ -8,22 +8,40 @@ requires_backend_capabilities: [batch] # Batch Jobs Process large workloads reliably and asynchronously. Batch is where Zenrows' -high-scale anti-bot advantage becomes obvious — Zenrows wins when the workflow -runs over thousands, millions, or recurring sets of URLs. +high-scale anti-bot advantage becomes obvious: thousands to 100,000 URLs per +job, at Fetch's prices. > Zenrows **Batch** runs on a separate host (`async.api.zenrows.com/v1`) and -> spends the same credits as Fetch. The cloud subcommands work with any active -> API key; a 403 maps to `BATCH_ACCESS_DENIED`. The toolkit never fakes a -> cloud run. +> costs the same credits as Fetch (1 basic, 5 js_render, 10 premium_proxy, 25 +> both; Adaptive Stealth Mode 1 to 25). Failed tasks are not billed, except +> target 404/410 answers, as on Fetch. The cloud subcommands work with any +> active API key; a 403 maps to `BATCH_ACCESS_DENIED`. The toolkit never fakes +> a cloud run. + +## Spec and estimate (local, no key) -## Local (no key, always works) ``` -zenrows batch estimate jobs.jsonl # validate the spec + estimate credits +zenrows batch estimate jobs.jsonl [--premium-proxy] [--js-render] [--manual] ``` -`jobs.jsonl` is one JSON object per line, each with a `url` (plus optional -per-task keys: `external_id`, `metadata`, and scrape params like `js_render`, -`premium_proxy`, `proxy_country`, `mode`, `autoparse`, `extract`). +`jobs.jsonl` is one JSON object per line, each with a `url` (http or https, +up to 2048 characters). Optional per-task keys: + +- `external_id`: up to 128 letters, digits, `.`, `_` or `-`; echoed on each + result row (it does not have to be unique). +- `metadata`: up to 20 string-to-string entries; keys 1 to 64 of the same + characters, values up to 1024 characters, 4 KiB in total. +- `method` and `body`: `"method": "POST"` sends `body` (up to 16 KiB): a JSON + string goes as a form body, any other JSON value as JSON. A POST task cannot + use `js_render`, `js_instructions` or `json_response`. +- scrape params, flat (`js_render`, `premium_proxy`, `proxy_country`, `mode`, + `autoparse`, `extract`, ...) or nested under `zenrows_params`. Booleans are + `true`/`false` (or `"true"`/`"false"`). + +`estimate` checks every line against those limits and prices the run the way +`create` would submit it, job-level flags included, as a range: Adaptive +Stealth Mode tasks count 1 at the low end and 25 at the upper bound. It exits 1 +on any invalid line. ## Extract in Batch @@ -39,42 +57,136 @@ correctly. The result carries two keys: `html` (the raw page) and `parsed` (the structured data). Validate on a single URL with `zenrows extract ` before running the full batch. -## Cloud (needs a key) +## Create, wait, read + ``` -zenrows batch create jobs.jsonl [--proxy-country cc] [--wait] -zenrows batch create jobs.jsonl --manual [--js-render] [--premium-proxy] -zenrows batch status # run status + stats -zenrows batch results [--status successful|failed|all] [--out results.jsonl] -zenrows batch wait [--timeout ms] -zenrows batch cancel # stop an in-flight run -zenrows batch retry-failed # rerun only the failed tasks (new run) +zenrows batch create jobs.jsonl --follow --json +zenrows batch create jobs.jsonl --manual --premium-proxy --proxy-country us --output markdown +zenrows batch create jobs.jsonl --name nightly --external-id run-42 --metadata team=growth +zenrows batch results --out results.jsonl # every row, as JSONL +zenrows batch results --download out/ # each result body + _manifest.jsonl +zenrows batch content # one task's body ``` -Every task runs in Adaptive Stealth Mode (`mode=auto`) by default, like `fetch` -and `extract`: Zenrows enables JS rendering and premium proxies only on the pages -that need them, and bills only the configuration that succeeds. A task that sets -`js_render`, `premium_proxy` or `mode` itself keeps its own settings, and -`--js-render` / `--premium-proxy` force that configuration on every task instead. -`--manual` turns Adaptive Stealth Mode off. `batch estimate` counts auto tasks at -the 25x upper bound, so it's the most the run can cost, not the expected cost. - -`create` validates the spec first, then submits. Outside Adaptive Stealth Mode, -`--proxy-country` needs `--premium-proxy` and is rejected before any request. -Job-level flags apply to every task; per-task keys in the JSONL override them. +- Every GET task runs in Adaptive Stealth Mode (`mode=auto`) by default, like + `fetch` and `extract`: Zenrows enables JS rendering and premium proxies only + on the pages that need them, and bills only the configuration that + succeeds. A task that turns on `js_render` or `premium_proxy`, or sets + `mode`, itself keeps its own settings; `--js-render` / `--premium-proxy` force that + configuration on every task; `--manual` turns Adaptive Stealth Mode off. + Outside Adaptive Stealth Mode, `--proxy-country` needs `--premium-proxy`. +- Job-level flags apply to every task; per-task keys override them. +- `--follow` (alias `--wait`) waits until the run ends, the run is paused, or + `--timeout` runs out (default 10 minutes; `30s`, `5m`, `1h`; a bare number + is milliseconds). A wait that runs out exits 0 and prints the run with a + `note` and `next: zenrows batch wait `; the job keeps running. Ctrl-C + stops the wait, not the job, and exits 130. +- A submit of 10,000 tasks or more answers 202 with `ingest_status: pending` + while the API still receives the tasks; results stay partial until `done`. +- `create` and `rerun` send an `Idempotency-Key` (generated, or + `--idempotency-key`) and retry network errors and 502/503/504 under it. + When the outcome stays unknown, the error carries `idempotency_key` and the + exact command that repeats the request safely (the API keeps a key 24 hours + and answers a repeat with the original job). Never repeat with a new key. +- Results are kept 14 days by default; after that a row reads `expired`, has no + `result_url`, and `content` answers `BATCH_RESULT_EXPIRED`. `--download` + skips expired rows and counts them. A row's `result_url` is valid 2 hours: + list the results again for a fresh one. +- `results --status successful|failed|processing|all`, `--run ` for an + earlier run. On a run still in progress the output has `partial: true`. + +With `--json`, `create`, `status`, `wait`, `cancel`, `pause`, `resume` and +`rerun` print `{ok, job_id, run_id, status, stats, job}` (plus `jobId`, kept +for older scripts), with `paused`, `ingest_status`, `accepted_tasks`, +`rerun_of`, `retried_tasks`, `inherited_tasks`, `failure_reason` and +`failure_detail` when the API sends them, and `note` when a wait ends early. + +## While it runs, and after -A run ends in one of `completed`, `failed`, `stopped`, or `deleted`. `status`, -`wait`, and `create --wait` exit non-zero (`ok: false` under `--json`) only for -`failed`, and print the run's `failure_reason` and `failure_detail`. When -`failure_reason` is `api_key_cap_reached`, the key hit its own credit cap -(`KEY_CREDIT_CAP_REACHED`): the account still has credits, other keys keep -working, and the tasks not yet run stay pending. Wait for the cap to reset -(`zenrows usage`) or raise the cap; do not resubmit the job with the same key. - -Deferred (documented, not yet in the CLI): CSV upload, open/queue jobs, -scheduled jobs, webhooks/HMAC, and ZIP export. +``` +zenrows batch list [--status open|closed] [--type regular|scheduled] +zenrows batch status +zenrows batch wait --timeout 30m +zenrows batch pause # no new task is picked up +zenrows batch resume +zenrows batch cancel # stop the run; an ended run answers as it ended +zenrows batch retry-failed # rerun the failed tasks +zenrows batch retry-failed --include-pending # also the unfinished ones +zenrows batch rerun [--status failed,pending,processing] +zenrows batch delete --yes # job, runs and stored results; frees its slot +``` -Validate locally and fan out with `zenrows fetch` per URL on a small sample -before running a full batch. +A run ends `completed`, `stopped`, `failed` or `deleted`. A paused run is not +ended: `status` shows `paused: true`, and `wait` returns at once with a note +to `zenrows batch resume `. + +`status`, `wait`, `create --follow` and `rerun --follow` exit 1 (`ok: false`) +only for `failed`. A run fails when an account problem stops it, and the +error carries the same code and advice as the matching refused request, with +the run's `failure_reason` as `server_code` and the `job_id`: + +- `insufficient_credits` → `POLICY_MAX_CREDITS_EXCEEDED`: add credits or claim + the account (`zenrows usage`). +- `subscription_inactive` → `AUTH_INVALID`: fix the subscription in Billing. +- `api_key_cap_reached` → `KEY_CREDIT_CAP_REACHED`: the account still has + credits and other keys keep working; wait for the cap to reset or raise it. +- anything else → `BATCH_FAILED`. + +After any of these, and after a `cancel`, the tasks not yet run stay `pending` +(some can stay `processing`): fix the cause, then run them with +`zenrows batch rerun --status failed,pending,processing`. Do not resubmit +the job. A rerun with `--status` works for 10 days after the run started; +after that only a full rerun (`zenrows batch rerun `) does. Every rerun is +a new run that answers 202 with `ingest_status: pending`. + +## Limits + +- 100,000 tasks per job; 3 active jobs per account, shared with Crawl. A + closed job frees its slot when its run ends; an open or scheduled job only + when deleted. +- The CLI's local policy caps a run at 10,000 URLs and 250,000 estimated + credits by default (an Adaptive Stealth Mode task counts 25), so a default + run of up to 10,000 URLs passes in any mode. For more, raise both: + `zenrows policy set max_pages_per_run 100000` and + `zenrows policy set max_credits_per_run 2500000`. A workspace created by an + earlier CLI version keeps the caps in its `policy.json` (1,000 and 10,000); + `POLICY_LIMIT_EXCEEDED` prints the exact command to raise them. +- Results 14 days; `result_url` 2 hours; partial reruns 10 days; a task that + fails transiently is retried automatically up to 3 times. + +Schedules, webhooks, ZIP exports, CSV inputs, open jobs (add tasks, close), +the runs list and task history are available through the Batch API and the +SDKs (Python, Node.js, Go), not this CLI. + +## Errors + +Every Batch error carries the API's code as `server_code` (when it sent one) +and `job_id` on any call about one job. Reads retry 429/502/503/504 and network +errors up to 3 times; `cancel`, `pause` and `resume` retry 502/503/504; `create` +and `rerun` retry the same under their Idempotency-Key. The "Retry?" column is +for errors that remain after these retries. + +| Code | Meaning | Retry? | +| --- | --- | --- | +| `BATCH_ACCESS_DENIED` | 403: Batch is not enabled for this account or key | no | +| `AUTH_INVALID` | 401: the key was rejected; or the subscription is not active (401 `subscription_inactive`, 402 "Subscription has expired."; fix it in Billing) | no | +| `POLICY_MAX_CREDITS_EXCEEDED` | 402 `payment_required`: the account is out of credits | after credits renew or are added | +| `KEY_CREDIT_CAP_REACHED` | 402 `api_key_cap_reached`: this key reached its credit cap | after the cap resets or is raised | +| `BATCH_QUOTA_EXCEEDED` | 429 `quota_exceeded`: 3 active jobs (shared with Crawl), 5 scheduled jobs, or an open job's 100,000 tasks | after a slot frees (`zenrows batch list`) | +| `BATCH_NOT_FOUND` | 404: wrong id, deleted, or another account's job | no | +| `BATCH_RUN_IN_PROGRESS` | 409 `run_not_terminal`: the run is still going (or still receiving tasks) | after `wait` or `cancel` | +| `BATCH_RUN_ENDED` | 409 `run_not_stoppable`: the run already ended (pause/resume) | no; `rerun` starts a new run | +| `BATCH_NO_MATCHING_TASKS` | 409 `no_matching_tasks`: no task has the statuses asked for | with `--status failed,pending,processing`, or a full rerun | +| `BATCH_RERUN_WINDOW_EXPIRED` | 409 `partial_rerun_window_expired`: `--status` is over 10 days late | with a full rerun | +| `BATCH_REQUEST_IN_FLIGHT` | 409 `idempotency_key_conflict`: the key is in use or was used for another body | same command, same key, a moment later | +| `BATCH_CONFLICT` | any other 409 (job state, key no longer valid, concurrent change) | read the cause | +| `BATCH_TASK_NOT_READY` | 409 `task_not_ready` on `content`: the task has not finished | after `wait` | +| `BATCH_TASK_FAILED` | 422 on `content`: the task failed | after `retry-failed` | +| `BATCH_RESULT_EXPIRED` | 410 `result_expired`: past the 14-day visibility window | after a full rerun | +| `BATCH_FAILED` | the run ended `failed` for another reason; a 400 (fix the flagged tasks) or 5xx | 5xx: yes | +| `POLICY_LIMIT_EXCEEDED` | the local policy caps (see Limits) | after raising the cap | +| `BACKEND_UNAVAILABLE` | the API could not be reached | yes | Rules: **never batch before validating** the workflow on a single page -([[extract]]). Mind the cost multipliers ([[cost-control]]). +([[extract]]). Mind the cost multipliers ([[cost-control]]). Read errors with +[[trace-debug]]. diff --git a/skills/crawl/SKILL.md b/skills/crawl/SKILL.md index f5b1db2..6877ab1 100644 --- a/skills/crawl/SKILL.md +++ b/skills/crawl/SKILL.md @@ -34,7 +34,7 @@ zenrows crawl results --out urls.jsonl # same, as JSONL a depth has no `depth` field in its output. - `--max-items` (kept URLs) and `--max-pages` (fetched pages) cap the crawl; both default to 100. `--max-pages` bounds the cost: each page is one fetch. - The local policy `max_pages_per_run` caps `--max-pages` (default 1000); + The local policy `max_pages_per_run` caps `--max-pages` (default 10000); above it the CLI refuses with `POLICY_LIMIT_EXCEEDED` before any request. - `--include-patterns` / `--exclude-patterns` are substring matches on the normalized URL and repeat: diff --git a/skills/trace-debug/SKILL.md b/skills/trace-debug/SKILL.md index 0948509..1afd8b5 100644 --- a/skills/trace-debug/SKILL.md +++ b/skills/trace-debug/SKILL.md @@ -43,8 +43,23 @@ zenrows trace export # JSON for sharing local policy credit limit was hit). Do not retry-loop; `zenrows usage` shows when credits renew. Top up or upgrade, or claim the account if it is an auto-created Free plan. -- `BATCH_FAILED` → the batch run ended `failed`; `failure_reason` and - `failure_detail` in `zenrows batch status --json` say why. +- `BATCH_FAILED` → the batch run ended `failed` for a reason other than + credits, subscription or key cap (those get `POLICY_MAX_CREDITS_EXCEEDED`, + `AUTH_INVALID` or `KEY_CREDIT_CAP_REACHED`, with the run's `failure_reason` + as `server_code` and its `job_id`); `failure_detail` in + `zenrows batch status --json` says why. Once fixed, run the unfinished + tasks with `zenrows batch rerun --status failed,pending,processing`; + do not resubmit. Also a 400 (fix the flagged tasks) or a 5xx (retry). +- `BATCH_QUOTA_EXCEEDED` → 429: 3 active jobs (shared with Crawl), 5 scheduled + jobs, or an open job's task cap. Nothing was created; free a slot (see + `zenrows batch list` and `zenrows crawl list`). +- `BATCH_ACCESS_DENIED` / `BATCH_NOT_FOUND` → 403 Batch not enabled, or 404 + wrong or deleted job id. Do not retry. +- `BATCH_RUN_IN_PROGRESS`, `BATCH_RUN_ENDED`, `BATCH_NO_MATCHING_TASKS`, + `BATCH_RERUN_WINDOW_EXPIRED`, `BATCH_REQUEST_IN_FLIGHT`, `BATCH_CONFLICT`, + `BATCH_TASK_NOT_READY`, `BATCH_TASK_FAILED`, `BATCH_RESULT_EXPIRED` → the + Batch API's 409/410/422 answers; `server_code` names the case and + `next_action` the fix. See [[batch-jobs]]. - `CRAWL_NOT_ENABLED` → the API answered 403 `REQS008`: Crawl is not enabled for this account. Do not retry; ask Zenrows support for access. - `CRAWL_INSUFFICIENT_CREDITS` / `CRAWL_KEY_CAP_REACHED` → Crawl's 402: the diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index c782a36..76fd8cf 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -41,7 +41,8 @@ If the user has a start page (listing, index, section) but not the item URLs: If the user has many URLs (bulk): → Use Batch. (validate one page with fetch/extract first, then `zenrows batch create `) - Batch is the cheap path at scale — do NOT + Batch is the reliable path at scale, at + Fetch's prices. Do NOT fan out fetch/extract per URL for bulk work. If the user needs login, clicks, forms, sessions, or persistent state: @@ -79,7 +80,7 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | MCP | `zenrows mcp` | **available**: `https://mcp.zenrows.com/mcp` | - Extract: `extract=auto` falls back to Autoparse on a domain not prepared yet. -- Batch: the cheapest path at scale (create, status, results). Estimate specs locally with no key. +- Batch: the reliable path at scale, at Fetch's prices (create, list, status, wait, results, content, pause, resume, rerun, delete). Estimate specs locally with no key. - Crawl: collects the URLs behind one start page, optionally with each page's HTML. - Browser: escalation-only, for pages Fetch can't handle; same backend as the MCP `browser_*` tools. diff --git a/src/adapters/batch.ts b/src/adapters/batch.ts index b32bf7d..470bc58 100644 --- a/src/adapters/batch.ts +++ b/src/adapters/batch.ts @@ -3,29 +3,49 @@ * * Status: `available`. The cloud subcommands call the Batch API with an API key * (see `core/batch-api.ts`); a 403 maps to BATCH_ACCESS_DENIED. This adapter owns - * the local, no-network pieces: validating a JSONL job spec, estimating credit - * cost, and mapping the validated spec to the Batch API request body. + * the local, no-network pieces: validating a JSONL job spec against the API's + * limits, estimating credit cost, and mapping the validated spec to the Batch + * API request body. */ import { existsSync, readFileSync } from "node:fs"; import { ToolkitError } from "../core/errors.ts"; -/** Per-task keys handled explicitly; everything else on a line is a scrape param. */ -const TASK_RESERVED = new Set(["url", "external_id", "metadata", "zenrows_params"]); +/** Per-task keys sent at task level; everything else on a line is a scrape param. */ +const TASK_RESERVED = new Set(["url", "external_id", "metadata", "zenrows_params", "method", "body"]); + +/** The API's limits on a task and on job-level fields. */ +export const LIMITS = { + urlLength: 2048, + externalIdLength: 128, + metadataEntries: 20, + metadataKeyLength: 64, + metadataValueLength: 1024, + metadataTotalBytes: 4 * 1024, + bodyBytes: 16 * 1024, + nameLength: 100, +} as const; + +/** `external_id` and metadata keys: letters, digits, `.`, `_` and `-`. */ +const SAFE_ID = /^[A-Za-z0-9._-]+$/; export interface BatchJob { url: string; - /** Optional stable id echoed back on each result row. */ + /** Optional id echoed back on each result row (not required to be unique). */ external_id?: string; - /** Opaque per-task metadata carried through to results. */ - metadata?: unknown; + /** Per-task string-to-string metadata carried through to results. */ + metadata?: Record; /** Optional nested per-task scrape params (merged with the flat keys below). */ zenrows_params?: Record; - /** Optional per-job overrides mirroring fetch options. */ - js_render?: boolean; - premium_proxy?: boolean; + /** `GET` (default) or `POST`. */ + method?: string; + /** POST body: a JSON string is sent verbatim as a form body, any other JSON value as JSON. */ + body?: unknown; + /** Optional per-task overrides mirroring fetch options. */ + js_render?: boolean | string; + premium_proxy?: boolean | string; proxy_country?: string; mode?: string; - autoparse?: boolean; + autoparse?: boolean | string; [k: string]: unknown; } @@ -36,7 +56,7 @@ export interface BatchValidation { jobs: BatchJob[]; } -/** Parse + validate a JSONL job spec locally. */ +/** Parse + validate a JSONL job spec locally, against the API's per-task limits. */ export function validateJsonl(file: string): BatchValidation { if (!existsSync(file)) { throw new ToolkitError({ @@ -53,41 +73,158 @@ export function validateJsonl(file: string): BatchValidation { const trimmed = line.trim(); if (!trimmed) return; result.totalLines++; + let obj: BatchJob; try { - const obj = JSON.parse(trimmed) as BatchJob; - if (!obj.url || typeof obj.url !== "string") { - result.errors.push({ line: idx + 1, reason: "missing string `url`" }); - return; - } - try { - new URL(obj.url); - } catch { - result.errors.push({ line: idx + 1, reason: `invalid url: ${obj.url}` }); - return; - } - result.validJobs++; - result.jobs.push(obj); + obj = JSON.parse(trimmed) as BatchJob; } catch { result.errors.push({ line: idx + 1, reason: "invalid JSON" }); + return; + } + const reason = taskError(obj); + if (reason) { + result.errors.push({ line: idx + 1, reason }); + return; } + result.validJobs++; + result.jobs.push(obj); }); return result; } +/** Why one JSONL line would be rejected by the API, or null when it is valid. */ +function taskError(obj: unknown): string | null { + if (typeof obj !== "object" || obj === null || Array.isArray(obj)) return "a line must be a JSON object"; + const job = obj as BatchJob; + if (!job.url || typeof job.url !== "string") return "missing string `url`"; + let u: URL; + try { + u = new URL(job.url); + } catch { + return `invalid url: ${job.url}`; + } + if (u.protocol !== "http:" && u.protocol !== "https:") return `url must be http or https: ${job.url}`; + if (job.url.length > LIMITS.urlLength) return `url is longer than ${LIMITS.urlLength} characters`; + if (job.external_id !== undefined && job.external_id !== null) { + const e = externalIdError(job.external_id); + if (e) return e; + } + const meta = metadataError(job.metadata); + if (meta) return meta; + if (job.zenrows_params !== undefined && (typeof job.zenrows_params !== "object" || job.zenrows_params === null || Array.isArray(job.zenrows_params))) { + return "`zenrows_params` must be an object"; + } + return methodBodyError(job.method, job.body); +} + +/** Why an `external_id` is invalid, or null. Numbers are sent as their string form. */ +export function externalIdError(v: unknown): string | null { + if (typeof v !== "string" && typeof v !== "number") return "`external_id` must be a string"; + const s = String(v); + if (s.length === 0 || s.length > LIMITS.externalIdLength) return `\`external_id\` must be 1 to ${LIMITS.externalIdLength} characters`; + if (!SAFE_ID.test(s)) return "`external_id` may only use letters, digits, `.`, `_` and `-`"; + return null; +} + +/** + * Why `metadata` is invalid, or null: the API takes at most 20 string-to-string + * entries, keys of 1 to 64 safe characters, values up to 1024 characters, 4 KiB + * in total. Any other shape fails the whole submit. + */ +export function metadataError(metadata: unknown): string | null { + if (metadata === undefined || metadata === null) return null; + if (typeof metadata !== "object" || Array.isArray(metadata)) return "`metadata` must be an object of string values"; + const entries = Object.entries(metadata as Record); + if (entries.length > LIMITS.metadataEntries) return `\`metadata\` has more than ${LIMITS.metadataEntries} entries`; + let total = 0; + for (const [k, v] of entries) { + if (k.length === 0 || k.length > LIMITS.metadataKeyLength || !SAFE_ID.test(k)) { + return `\`metadata\` key "${k}" must be 1 to ${LIMITS.metadataKeyLength} letters, digits, \`.\`, \`_\` or \`-\``; + } + if (typeof v !== "string") return `\`metadata.${k}\` must be a string`; + if (v.length > LIMITS.metadataValueLength) return `\`metadata.${k}\` is longer than ${LIMITS.metadataValueLength} characters`; + total += Buffer.byteLength(k) + Buffer.byteLength(v); + } + if (total > LIMITS.metadataTotalBytes) return "`metadata` is larger than 4 KiB"; + return null; +} + +/** Why `method` / `body` are invalid, or null. Only GET and POST exist; a body needs POST. */ +function methodBodyError(method: unknown, body: unknown): string | null { + if (method !== undefined && method !== null) { + if (typeof method !== "string" || !/^(get|post)$/i.test(method)) return "`method` must be GET or POST"; + } + if (body === undefined || body === null) return null; + if (!isPost(method)) return "`body` needs `\"method\": \"POST\"`"; + const bytes = typeof body === "string" ? Buffer.byteLength(body) : Buffer.byteLength(JSON.stringify(body)); + if (bytes > LIMITS.bodyBytes) return `\`body\` is larger than ${LIMITS.bodyBytes / 1024} KiB`; + return null; +} + +function isPost(method: unknown): boolean { + return typeof method === "string" && method.toUpperCase() === "POST"; +} + +/** + * Scrape params the API reads as booleans. It accepts JSON `true`/`false` or the + * strings "true"/"false" in any case, and rejects anything else with a 400. + */ +const BOOLEAN_PARAMS = new Set(["js_render", "premium_proxy", "original_status", "json_response", "autoparse"]); + +/** JSON `true` and "true" in any case become "true"; likewise "false". Any other value is unchanged. */ +function canonicalBool(v: unknown): unknown { + if (v === true) return "true"; + if (v === false) return "false"; + if (typeof v === "string") { + const s = v.trim().toLowerCase(); + if (s === "true" || s === "false") return s; + } + return v; +} + +/** A task's own params: its nested `zenrows_params`, then its flat keys (later wins). */ +function taskParamsOf(job: BatchJob): Record { + const flat: Record = {}; + for (const [k, v] of Object.entries(job)) { + if (!TASK_RESERVED.has(k)) flat[k] = v; + } + return { ...(job.zenrows_params ?? {}), ...flat }; +} + /** - * Rough local credit estimate using documented multipliers - * (basic 1x, js 5x, proxies 10x, both 25x). Counts in abstract "credits". + * The params a task actually runs with: the job-level params and the task's + * own, each canonicalized exactly as submitted, merged the way the API merges + * them (the task wins on a shared key). A task key set to null is not + * submitted, so the job's value applies. */ -export function estimateCredits(jobs: BatchJob[]): { credits: number; perJob: number[] } { +export function effectiveParams(job: BatchJob, jobParams: Record = {}): Record { + return { ...normalizeParams(jobParams), ...normalizeParams(taskParamsOf(job)) }; +} + +/** + * Local credit estimate from the rate card: basic 1, js_render 5, + * premium_proxy 10, both 25. An Adaptive Stealth Mode task (`mode=auto`) + * costs 1 to 25 depending on what each page needs, so it counts 1 in `min` + * and 25 in `credits` (the upper bound). Each task is priced on the params + * it is submitted with, job-level flags included. + */ +export function estimateCredits( + jobs: BatchJob[], + jobParams: Record = {}, +): { credits: number; min: number; perJob: number[] } { + let min = 0; const perJob = jobs.map((j) => { - const js = j.js_render || j.mode === "auto"; - const proxy = j.premium_proxy || j.mode === "auto"; - if (js && proxy) return 25; - if (proxy) return 10; - if (js) return 5; - return 1; + const p = effectiveParams(j, jobParams); + if (p.mode === "auto") { + min += 1; + return 25; + } + const js = p.js_render === "true"; + const proxy = p.premium_proxy === "true"; + const cost = js && proxy ? 25 : proxy ? 10 : js ? 5 : 1; + min += cost; + return cost; }); - return { credits: perJob.reduce((a, b) => a + b, 0), perJob }; + return { credits: perJob.reduce((a, b) => a + b, 0), min, perJob }; } /** @@ -96,27 +233,38 @@ export function estimateCredits(jobs: BatchJob[]): { credits: number; perJob: nu * `premium_proxy` or set `mode` itself, and none at all when the job-level params * force either flag. Applied per task rather than job-level because the API rejects * a task that combines a job-level mode=auto with its own js_render or - * premium_proxy (REQS004). + * premium_proxy (REQS004). A POST task is left alone: Adaptive Stealth Mode may + * turn on JavaScript rendering, which a POST cannot use. */ export function withAdaptiveStealth(jobs: BatchJob[], jobParams: Record = {}): BatchJob[] { const forced = (p: Record) => isTruthyFlag(p.js_render) || isTruthyFlag(p.premium_proxy); if (forced(jobParams) || jobParams.mode != null) return jobs; return jobs.map((job) => { - const own = { ...(job.zenrows_params ?? {}), ...job }; + if (isPost(job.method)) return job; + const own = taskParamsOf(job); if (forced(own) || own.mode != null) return job; return { ...job, mode: "auto" }; }); } +/** Job-level fields of a submit other than its params. */ +export interface JobFields { + name?: string; + external_id?: string; + metadata?: Record; +} + /** Body for `POST /jobs` (closed, all-URLs-up-front job). */ -export interface JobBody { +export interface JobBody extends JobFields { type: "regular"; status: "closed"; zenrows_params?: Record; tasks: Array<{ url: string; external_id?: string; - metadata?: unknown; + metadata?: Record; + method?: string; + body?: unknown; zenrows_params?: Record; }>; } @@ -126,36 +274,40 @@ export interface JobBody { * The flat per-line options (`js_render`, `premium_proxy`, `proxy_country`, * `autoparse`, `mode`, …) plus any nested `zenrows_params` become that task's * `zenrows_params`; `jobParams` (from CLI flags) becomes the job-level - * `zenrows_params`. `url`/`external_id`/`metadata` map straight through. + * `zenrows_params`. `url`, `external_id`, `metadata`, `method` and `body` are + * task fields and map straight through. * - * The API rejects `proxy_country` without `premium_proxy` (unless mode=auto), - * so we surface PARAM_PROXY_COUNTRY_REQUIRES_PREMIUM here — before submitting — - * for the effective (job + task) params, mirroring the manual-mode guard in - * `protected-fetch.ts::validateAutoManual`. + * Checked here, before submitting, on the effective (job + task) params: + * boolean params hold true or false; `proxy_country` needs `premium_proxy` + * unless mode=auto (PARAM_PROXY_COUNTRY_REQUIRES_PREMIUM, mirroring + * `protected-fetch.ts::validateAutoManual`); a POST task does not turn on + * `js_render`, `js_instructions` or `json_response`, which the API rejects. */ -export function toJobBody(jobs: BatchJob[], jobParams: Record = {}): JobBody { - // No separate job-level check: a job-level proxy_country is valid for each task - // in Adaptive Stealth Mode, and the per-task check below sees job ⊕ task params. +export function toJobBody(jobs: BatchJob[], jobParams: Record = {}, fields: JobFields = {}): JobBody { const tasks = jobs.map((job, idx) => { - const { url, external_id, metadata, zenrows_params } = job; - const flat: Record = {}; - for (const [k, v] of Object.entries(job)) { - if (!TASK_RESERVED.has(k)) flat[k] = v; - } - const taskParams: Record = { ...(zenrows_params ?? {}), ...flat }; - // Validate on the effective (job ⊕ task) params so a job-level premium_proxy - // covers a per-task proxy_country, matching how the API merges them. - assertProxyCountryPremium({ ...jobParams, ...taskParams }, `line ${idx + 1} (${url})`); + const { url, external_id, metadata } = job; + const where = `line ${idx + 1} (${url})`; + const effective = effectiveParams(job, jobParams); + assertBooleanParams(effective, where); + assertProxyCountryPremium(effective, where); const task: JobBody["tasks"][number] = { url }; - if (external_id !== undefined) task.external_id = String(external_id); - if (metadata !== undefined) task.metadata = metadata; - const zp = normalizeParams(taskParams); + if (external_id !== undefined && external_id !== null) task.external_id = String(external_id); + if (metadata !== undefined && metadata !== null) task.metadata = metadata; + if (isPost(job.method)) { + assertPostParams(effective, where); + task.method = "POST"; + if (job.body !== undefined && job.body !== null) task.body = job.body; + } + const zp = normalizeParams(taskParamsOf(job)); if (Object.keys(zp).length) task.zenrows_params = zp; return task; }); const body: JobBody = { type: "regular", status: "closed", tasks }; + if (fields.name) body.name = fields.name; + if (fields.external_id) body.external_id = fields.external_id; + if (fields.metadata && Object.keys(fields.metadata).length) body.metadata = fields.metadata; const jobZp = normalizeParams(jobParams); if (Object.keys(jobZp).length) body.zenrows_params = jobZp; return body; @@ -163,13 +315,15 @@ export function toJobBody(jobs: BatchJob[], jobParams: Record = /** * Coerce a flat options object into the string-valued map the Batch API expects - * (its docs pass `zenrows_params` values as strings, e.g. `"true"`, `"us"`). - * Booleans/numbers are stringified; objects are JSON-encoded; nullish is dropped. + * (`zenrows_params` values are strings, e.g. `"true"`, `"us"`). Boolean params + * are canonicalized to "true"/"false"; other booleans/numbers are stringified; + * objects are JSON-encoded; nullish is dropped. */ function normalizeParams(obj: Record): Record { const out: Record = {}; - for (const [k, v] of Object.entries(obj)) { - if (v === undefined || v === null) continue; + for (const [k, raw] of Object.entries(obj)) { + if (raw === undefined || raw === null) continue; + const v = BOOLEAN_PARAMS.has(k) ? canonicalBool(raw) : raw; if (typeof v === "string") out[k] = v; else if (typeof v === "boolean" || typeof v === "number") out[k] = String(v); else out[k] = JSON.stringify(v); @@ -182,7 +336,34 @@ function isAutoMode(v: unknown): boolean { return v === "auto" || v === true; } function isTruthyFlag(v: unknown): boolean { - return v === true || v === "true"; + return canonicalBool(v) === "true"; +} + +/** Throw before submitting if a boolean param holds a value the API rejects. */ +function assertBooleanParams(params: Record, where: string): void { + for (const k of BOOLEAN_PARAMS) { + const v = params[k]; + if (v !== undefined && v !== "true" && v !== "false") { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `${k} must be true or false (got ${JSON.stringify(v)}) in ${where}.`, + likely_cause: `The Batch API accepts only true or false for ${k}; any other value fails the whole job.`, + next_action: `Set ${k} to true or false on that line, then retry.`, + }); + } + } +} + +/** Throw if a POST task would turn on a param the API refuses with POST (method_param_conflict). */ +function assertPostParams(params: Record, where: string): void { + const conflict = params.js_render === "true" ? "js_render" : params.json_response === "true" ? "json_response" : params.js_instructions ? "js_instructions" : undefined; + if (!conflict) return; + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `A POST task cannot use ${conflict}: ${where}.`, + likely_cause: "POST tasks run without a browser, so the Batch API rejects js_render, js_instructions and json_response on them (method_param_conflict).", + next_action: "Drop that param from the line (or the job-level --js-render flag), or send the task as GET.", + }); } /** Throw if proxy_country is set without premium_proxy and outside auto mode. */ diff --git a/src/cli/commands/batch.ts b/src/cli/commands/batch.ts index d109cf9..3d688dd 100644 --- a/src/cli/commands/batch.ts +++ b/src/cli/commands/batch.ts @@ -1,52 +1,119 @@ /** * `zenrows batch`: Batch API (status: available). * - * Local (no key, always works): `estimate`/`create --dry-run`-style spec - * validation + credit estimate. Cloud (needs a key): - * `create`, `status`, `results`, `cancel`, `wait`, `retry-failed`. A 403 maps to - * BATCH_ACCESS_DENIED. + * Local (no key, always works): `estimate` validates a JSONL spec and prices + * it. Cloud (needs a key): `create`, `list`, `status`, `wait`, `results`, + * `content`, `cancel`, `pause`, `resume`, `rerun`, `retry-failed`, `delete`. + * A 403 maps to BATCH_ACCESS_DENIED. Schedules, webhooks, exports, CSV inputs, + * open jobs, runs and task history are API/SDK-only. */ -import { statSync } from "node:fs"; import { join } from "node:path"; import { log, ANSI, c } from "../../core/logger.ts"; -import { estimateCredits, toJobBody, validateJsonl, withAdaptiveStealth, type BatchJob } from "../../adapters/batch.ts"; +import { + estimateCredits, + externalIdError, + LIMITS, + metadataError, + toJobBody, + validateJsonl, + withAdaptiveStealth, + type BatchJob, + type JobFields, +} from "../../adapters/batch.ts"; import { loadConfig } from "../../core/config.ts"; import { requireApiKey } from "../../core/auth.ts"; import { ensureApiKey } from "../../core/ensure-key.ts"; import { assertUsable } from "../../core/capabilities.ts"; import { assertDomainAllowed, assertWithinLimits, loadPolicy } from "../../core/policy.ts"; import { newRunId, writeRun } from "../../core/artifacts.ts"; -import { createJob, downloadResults, getJob, listResults, rerunJob, stopJob, waitForJob, type Job } from "../../core/batch-api.ts"; +import { + RERUN_STATUSES, + assertIdempotencyKey, + createJob, + deleteJob, + downloadResults, + getJob, + getRun, + getTaskContent, + isPaused, + isTerminal, + listJobs, + listResults, + newIdempotencyKey, + pauseJob, + rerunJob, + resumeJob, + stopJob, + subscriptionInactive, + waitForJob, + type Job, + type JobRun, + type ResultStatus, +} from "../../core/batch-api.ts"; +import { readAccount } from "../../core/agent-account.ts"; import { asNumber, asString, parse, type Command, type RunContext } from "../command.ts"; -import { ToolkitError, isKeyCapReached, keyCapReached } from "../../core/errors.ts"; +import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "../../core/errors.ts"; import { printError, writeOut } from "../output.ts"; +/** The task statuses that recover every task a run did not finish successfully. */ +const UNFINISHED = "failed,pending,processing"; + export const batch: Command = { name: "batch", summary: "Run JSONL batch jobs on Zenrows Batch.", - usage: "zenrows batch |status |results |cancel |wait |retry-failed >", + usage: + "zenrows batch |create |list|status |wait |results |content |cancel |pause |resume |rerun |retry-failed |delete >", help: [ "Local (no key):", - " estimate [--manual] validate the spec + estimate credits (upper bound)", + " estimate [flags] validate the spec and price it (credits: min and upper bound);", + " takes the same --manual / --js-render / --premium-proxy /", + " --proxy-country flags as create, so both price the same request", "Cloud (needs a key):", " create [flags] validate, then submit the job", - " (default) Adaptive Stealth Mode (mode=auto) on every task that", - " doesn't force js_render / premium_proxy itself", + " (default) Adaptive Stealth Mode (mode=auto) on every GET task that", + " doesn't force js_render / premium_proxy or set mode itself", " --manual no Adaptive Stealth Mode; plain requests unless flags below", " --js-render job-level: force JavaScript rendering (turns auto off)", " --premium-proxy job-level: force residential IPs (turns auto off)", " --proxy-country job-level: geo-target (auto mode, or with --premium-proxy)", " --output job-level response_type (markdown|plaintext|pdf|html)", - " --follow poll until the run finishes (alias: --wait)", + " --name job name (up to 100 characters)", + " --external-id job-level external id (letters, digits, . _ -; up to 128)", + " --metadata job-level metadata entry (repeatable, up to 20)", + " --idempotency-key so a repeated create starts no second job (one is generated", + " otherwise; the API keeps it 24 hours)", + " --follow wait until the run ends (alias: --wait; Ctrl-C stops the", + " wait, not the job)", + " --timeout with --follow: stop waiting after this long (10s, 5m, 1h;", + " a bare number is milliseconds; default 10m)", " --no-signup do not auto-create a Free plan account if no key exists", - " status show run status + stats", - " results [--status s] list results (successful|failed|all); paginated", + " list [--status open|closed] [--type regular|scheduled] [--limit n] [--cursor c]", + " your jobs, newest first (limit up to 100)", + " status show the job's latest run: status, stats, pause state", + " wait [--timeout ] poll until the run ends, is paused, or the time runs out", + " (default 10m); a run still going then is printed, not stopped", + " results [--status s] list results (successful|failed|processing|all); partial", + " while the run is in progress", + " --run read an earlier run instead of the latest", " --out write results as JSONL instead of printing", " --download fetch each result body into (+ _manifest.jsonl)", - " cancel stop an in-flight run", - " wait [--timeout ] poll until the run finishes", - " retry-failed rerun only the failed tasks (new run)", - " --json print structured output", + " content [--run ] [--out ]", + " print or save one task's result body", + " cancel stop the in-flight run (an ended run answers as it ended)", + " pause / resume pause the in-flight run, or resume it", + " rerun [--status ] start a new run: every task, or only tasks with these", + " statuses (pending,processing,failed,successful); takes", + " --idempotency-key, --follow and --timeout like create", + " retry-failed rerun only the failed tasks (rerun --status failed); takes", + " --idempotency-key, --follow and --timeout like rerun", + " --include-pending also the tasks a stop or a credit failure left unfinished", + " (rerun --status failed,pending,processing)", + " delete --yes delete the job, its runs and stored results; frees its slot", + " --json print structured output: create/status/wait/cancel/pause/", + " resume/rerun print {ok, job_id, run_id, status, stats, job};", + " a wait that runs out or meets a paused run adds note", + "Schedules, webhooks, exports, CSV inputs and open jobs are available through the Batch API", + "and the SDKs: https://docs.zenrows.com", ].join("\n"), async run(argv: string[], ctx: RunContext): Promise { const [sub, ...rest] = argv; @@ -56,22 +123,37 @@ export const batch: Command = { return estimateCmd(rest, ctx); case "create": return await createCmd(rest, ctx); + case "list": + return await listCmd(rest, ctx); case "status": return await statusCmd(rest, ctx); + case "wait": + return await waitCmd(rest, ctx); case "results": return await resultsCmd(rest, ctx); + case "content": + return await contentCmd(rest, ctx); case "cancel": + case "stop": return await cancelCmd(rest, ctx); - case "wait": - return await waitCmd(rest, ctx); + case "pause": + return await pauseCmd(rest, ctx, "pause"); + case "resume": + return await pauseCmd(rest, ctx, "resume"); + case "rerun": + return await rerunCmd(rest, ctx, "rerun"); case "retry-failed": - return await retryCmd(rest, ctx); + return await rerunCmd(rest, ctx, "retry-failed"); + case "delete": + return await deleteCmd(rest, ctx); default: throw new ToolkitError({ code: "INVALID_USAGE", message: `Unknown batch subcommand: ${sub ?? "(none)"}`, likely_cause: "Subcommand not recognized.", - next_action: "Use estimate | create | status | results | cancel | wait | retry-failed .", + next_action: + "Use estimate | create | list | status | wait | results | content | cancel | pause | resume | rerun | retry-failed | delete .", + suggested_commands: ["zenrows batch --help"], }); } } catch (err) { @@ -81,23 +163,61 @@ export const batch: Command = { }, }; +/** Flags that shape the request each task sends; shared by `estimate` and `create`. */ +const PARAM_FLAGS = { + "js-render": { type: "boolean" }, + "premium-proxy": { type: "boolean" }, + "proxy-country": { type: "string" }, + manual: { type: "boolean" }, + output: { type: "string" }, +} as const; + +/** Job-level `zenrows_params` from the CLI flags. */ +function jobParamsOf(values: Record): Record { + const jobParams: Record = {}; + if (values["js-render"] === true) jobParams.js_render = true; + if (values["premium-proxy"] === true) jobParams.premium_proxy = true; + const proxyCountry = asString(values["proxy-country"]); + if (proxyCountry) jobParams.proxy_country = proxyCountry; + const responseType = normalizeOutput(asString(values.output)); + if (responseType) jobParams.response_type = responseType; + return jobParams; +} + /** Jobs as they will be sent: Adaptive Stealth Mode applied unless manual or disabled in config. */ function effectiveJobs(jobs: BatchJob[], jobParams: Record, manual: boolean): BatchJob[] { return manual || loadConfig().defaultMode !== "auto" ? jobs : withAdaptiveStealth(jobs, jobParams); } function estimateCmd(rest: string[], ctx: RunContext): number { - const { values, positionals } = parse(rest, { manual: { type: "boolean" } }); + const { values, positionals } = parse(rest, { ...PARAM_FLAGS, json: { type: "boolean" } }); + const json = ctx.json || values.json === true; const file = positionals[0]; if (!file) throw needFile(); const v = validateJsonl(file); - const est = estimateCredits(effectiveJobs(v.jobs, {}, values.manual === true)); - if (ctx.json) { - log.out(JSON.stringify({ ok: v.errors.length === 0, ...v, estimatedCredits: est.credits }, null, 2)); + const jobParams = jobParamsOf(values); + const jobs = effectiveJobs(v.jobs, jobParams, values.manual === true); + // Price exactly what create would submit, and catch what it would refuse. + if (!v.errors.length) toJobBody(jobs, jobParams); + const est = estimateCredits(jobs, jobParams); + if (json) { + log.out( + JSON.stringify( + { + ok: v.errors.length === 0, + ...v, + estimatedCredits: est.credits, + estimated_credits: est.credits, + estimated_credits_min: est.min, + }, + null, + 2, + ), + ); } else { log.info(c(ANSI.bold, `Job spec: ${file}`)); log.info(`valid jobs: ${v.validJobs}/${v.totalLines}`); - log.info(`estimated credits, upper bound (1x basic / 5x js / 10x proxy / 25x both or auto): ${est.credits}`); + log.info(`estimated credits: ${est.min} to ${est.credits} (1 basic / 5 js / 10 proxy / 25 both; auto costs 1 to 25)`); if (v.errors.length) { log.warn(`${v.errors.length} invalid line(s):`); v.errors.slice(0, 10).forEach((e) => log.dim(` line ${e.line}: ${e.reason}`)); @@ -108,15 +228,48 @@ function estimateCmd(rest: string[], ctx: RunContext): number { return v.errors.length ? 1 : 0; } +/** `--name`, `--external-id` and `--metadata key=value`, checked against the API's limits. */ +function jobFieldsOf(values: Record): JobFields { + const fields: JobFields = {}; + const name = asString(values.name); + if (name !== undefined) { + if (!name.trim() || name.length > LIMITS.nameLength) { + throw usageError(`Invalid --name: it must be 1 to ${LIMITS.nameLength} characters.`, "Pass a shorter name."); + } + fields.name = name; + } + const externalId = asString(values["external-id"]); + if (externalId !== undefined) { + const e = externalIdError(externalId); + if (e) throw usageError(`Invalid --external-id: ${e}.`, "Pass e.g. --external-id nightly-2026-10-16."); + fields.external_id = externalId; + } + const pairs = Array.isArray(values.metadata) ? (values.metadata as unknown[]) : []; + if (pairs.length) { + const metadata: Record = {}; + for (const p of pairs) { + const s = String(p); + const eq = s.indexOf("="); + if (eq <= 0) throw usageError(`Invalid --metadata '${s}': use key=value.`, "Pass e.g. --metadata source=catalog."); + metadata[s.slice(0, eq)] = s.slice(eq + 1); + } + const e = metadataError(metadata); + if (e) throw usageError(`Invalid --metadata: ${e}.`, "Fix the entry, then retry."); + fields.metadata = metadata; + } + return fields; +} + async function createCmd(rest: string[], ctx: RunContext): Promise { const { values, positionals } = parse(rest, { - "js-render": { type: "boolean" }, - "premium-proxy": { type: "boolean" }, - "proxy-country": { type: "string" }, - manual: { type: "boolean" }, - output: { type: "string" }, + ...PARAM_FLAGS, + name: { type: "string" }, + "external-id": { type: "string" }, + metadata: { type: "string", multiple: true }, + "idempotency-key": { type: "string" }, follow: { type: "boolean" }, wait: { type: "boolean" }, // back-compat alias for --follow + timeout: { type: "string" }, "no-signup": { type: "boolean" }, json: { type: "boolean" }, }); @@ -126,59 +279,59 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { const json = ctx.json || values.json === true; const file = positionals[0]; if (!file) throw needFile(); + const waitMs = timeoutFlag(values.timeout); + const fields = jobFieldsOf(values); + const givenKey = asString(values["idempotency-key"]); + if (givenKey !== undefined) assertIdempotencyKey(givenKey); - // Capability gate first — a beta-disabled backend never gets a spec read. + // Capability gate first: a disabled backend never gets a spec read. assertUsable("batch"); const v = validateJsonl(file); if (v.errors.length) { - log.warn(`${v.errors.length} invalid line(s) — fix them before submitting:`); + log.warn(`${v.errors.length} invalid line(s), fix them before submitting:`); v.errors.slice(0, 10).forEach((e) => log.dim(` line ${e.line}: ${e.reason}`)); throw new ToolkitError({ code: "INVALID_USAGE", message: `Job spec has ${v.errors.length} invalid line(s).`, - likely_cause: "One or more JSONL lines are not valid JSON or lack a valid `url`.", + likely_cause: `One or more JSONL lines are not valid JSON or break the API's task limits (first: line ${v.errors[0]!.line}: ${v.errors[0]!.reason}).`, next_action: "Fix the flagged lines (see `zenrows batch estimate`), then retry.", suggested_commands: [`zenrows batch estimate ${file}`], }); } - // Pre-flight governance — all local, before any network call: + // Pre-flight governance, all local, before any network call: // 1. every task URL must pass the allow/deny domain policy, - // 2. the run must fit the batch-only page/credit caps (a batch is the one - // primitive that fans out into many requests, so the caps bind here). - const jobParams: Record = {}; - if (values["js-render"] === true) jobParams.js_render = true; - if (values["premium-proxy"] === true) jobParams.premium_proxy = true; - const proxyCountry = asString(values["proxy-country"]); - if (proxyCountry) jobParams.proxy_country = proxyCountry; - const responseType = normalizeOutput(asString(values.output)); - if (responseType) jobParams.response_type = responseType; + // 2. the run must fit the page/credit caps, priced on the params as submitted. + const jobParams = jobParamsOf(values); const jobs = effectiveJobs(v.jobs, jobParams, values.manual === true); - const policy = loadPolicy(); for (const job of jobs) assertDomainAllowed(job.url, policy); - const est = estimateCredits(jobs); + const est = estimateCredits(jobs, jobParams); assertWithinLimits({ pages: v.validJobs, credits: est.credits }, policy, "batch"); - // toJobBody validates proxy_country/premium_proxy BEFORE any HTTP call. - const body = toJobBody(jobs, jobParams); - const apiKey = await ensureApiKey( - values["no-signup"] ? { ...policy, auto_signup: false } : policy, - { - onProvision: (a) => { - log.info("No API key found — created a Zenrows Free plan account for you."); - log.dim(`Claim it anytime (keeps your usage): ${a.claimUrl}`); - }, + // toJobBody validates params (proxy_country/premium_proxy, booleans, POST) BEFORE any HTTP call. + const body = toJobBody(jobs, jobParams, fields); + const apiKey = await ensureApiKey(values["no-signup"] ? { ...policy, auto_signup: false } : policy, { + onProvision: (a) => { + log.info("No API key found: created a Zenrows Free plan account for you."); + log.dim(`Claim it anytime (keeps your usage): ${a.claimUrl}`); }, - ); + }); + const idempotencyKey = givenKey ?? newIdempotencyKey(); const runId = newRunId(); const startedAt = new Date().toISOString(); - log.step(`Submitting batch job (${body.tasks.length} tasks, ~${est.credits} credits)…`); + const request = { file, tasks: body.tasks.length, estimatedCredits: est.credits, jobParams, ...fields }; + log.step(`Submitting batch job (${body.tasks.length} tasks, ${est.min} to ${est.credits} credits)…`); try { - const job = await createJob(body, { apiKey }); - const finished = follow ? await waitForJob(job.job_id, { apiKey }) : job; + let job: Job; + try { + job = await createJob(body, { apiKey, idempotencyKey }); + } catch (err) { + throw withRepeatCommand(err, "create", rest); + } + const finished = follow ? await followJob(job.job_id, apiKey, waitMs, job) : job; const runError = runFailure(finished); const runDir = writeRun({ runId, @@ -187,11 +340,19 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { startedAt, finishedAt: new Date().toISOString(), status: runError ? "error" : "ok", - request: { file, tasks: body.tasks.length, estimatedCredits: est.credits, jobParams }, - result: { jobId: job.job_id, status: finished.latest_run?.status ?? "unknown", ...failureFields(finished) }, + request, + result: { + jobId: job.job_id, + job_id: job.job_id, + run_id: finished.latest_run?.run_id, + status: finished.latest_run?.status ?? "unknown", + ...failureFields(finished), + }, ...(runError ? { error: runError.toJSON() } : {}), }); - const code = printJob(finished, json, `Submitted job ${job.job_id}`); + const code = follow + ? printJob(finished, json, waitHeadline(finished), { note: waitNote(finished) }) + : printJob(finished, json, `Submitted job ${job.job_id}`); if (runDir && !json) log.dim(` artifact: ${runDir}`); return code; } catch (err) { @@ -202,50 +363,171 @@ async function createCmd(rest: string[], ctx: RunContext): Promise { startedAt, finishedAt: new Date().toISOString(), status: "error", - request: { file, tasks: body.tasks.length, estimatedCredits: est.credits, jobParams }, + request, error: err instanceof ToolkitError ? err.toJSON() : { message: String(err) }, }); throw err; } } +/** + * A write whose outcome is unknown carries its Idempotency-Key: suggest the + * exact command that repeats it under that key, which cannot create a duplicate. + */ +function withRepeatCommand(err: unknown, sub: string, rest: string[]): unknown { + if (!(err instanceof ToolkitError) || !err.idempotency_key) return err; + const args: string[] = []; + for (let i = 0; i < rest.length; i++) { + const a = rest[i]!; + if (a === "--idempotency-key") { + i++; + continue; + } + if (a.startsWith("--idempotency-key=")) continue; + args.push(/^[A-Za-z0-9._\-/:=,@%+]+$/.test(a) ? a : `'${a.replace(/'/g, "'\\''")}'`); + } + const repeat = `zenrows batch ${sub} ${args.join(" ")} --idempotency-key ${err.idempotency_key}`.replace(/\s+/g, " "); + return new ToolkitError({ ...err.toJSON(), suggested_commands: [repeat, "zenrows batch list"] }); +} + +async function listCmd(rest: string[], ctx: RunContext): Promise { + const { values } = parse(rest, { + status: { type: "string" }, + type: { type: "string" }, + limit: { type: "string" }, + cursor: { type: "string" }, + json: { type: "boolean" }, + }); + const json = ctx.json || values.json === true; + const status = oneOf("status", asString(values.status), ["open", "closed"]); + const type = oneOf("type", asString(values.type), ["regular", "scheduled"]); + const limit = pageLimit(values.limit, 100); + const cursor = asString(values.cursor); + assertUsable("batch"); + const apiKey = requireApiKey(); + const page = await listJobs({ apiKey, limit, cursor, status, type }); + if (json) { + log.out(JSON.stringify({ ok: true, count: page.jobs.length, jobs: page.jobs, next_cursor: page.next_cursor }, null, 2)); + return 0; + } + if (!page.jobs.length) log.info("No jobs."); + for (const j of page.jobs) { + const run = j.latest_run; + const runState = run ? `${run.status}${isPaused(run) ? " (paused)" : ""}` : "no run yet"; + const progress = run?.stats ? `${run.stats.completed}/${run.stats.total}` : ""; + const created = typeof j.created_at === "string" ? j.created_at : ""; + const name = typeof j.name === "string" ? j.name : ""; + log.out(`${j.job_id} ${String(j.type ?? "")} ${String(j.status ?? "")} ${runState} ${progress} ${created} ${name}`.trimEnd()); + } + if (page.next_cursor) log.dim(` more: zenrows batch list --cursor ${page.next_cursor}`); + return 0; +} + async function statusCmd(rest: string[], ctx: RunContext): Promise { - const { positionals } = parse(rest, {}); - const id = requireId(positionals[0]); + const { values, positionals } = parse(rest, { json: { type: "boolean" } }); + const id = requireId(positionals[0], "status"); assertUsable("batch"); const apiKey = requireApiKey(); const job = await getJob(id, { apiKey }); - return printJob(job, ctx.json, `Job ${id}`); + return printJob(job, ctx.json || values.json === true, `Job ${id}`, { note: isPaused(job.latest_run) ? pausedNote(id) : undefined }); +} + +async function waitCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { timeout: { type: "string" }, json: { type: "boolean" } }); + const json = ctx.json || values.json === true; + const id = requireId(positionals[0], "wait"); + const waitMs = timeoutFlag(values.timeout); + assertUsable("batch"); + const apiKey = requireApiKey(); + const job = await followJob(id, apiKey, waitMs); + return printJob(job, json, waitHeadline(job), { note: waitNote(job) }); +} + +/** + * Wait for a job's run to end. Ctrl-C ends the wait, not the job: say so, give + * the command that resumes the wait, and exit 130 as shells do on SIGINT. A + * submit's own answer is printed instead of a read when the wait ends before + * the first poll, so its `accepted_tasks` survive. + */ +async function followJob(id: string, apiKey: string, waitMs: number | undefined, created?: Job): Promise { + const onSigint = () => { + log.warn(`Stopped waiting. Job ${id} is still running; it was not stopped.`); + log.dim(` resume: zenrows batch wait ${id}`); + process.exit(130); + }; + process.once("SIGINT", onSigint); + try { + const job = await waitForJob(id, { apiKey, waitMs }); + // Keep the submit/rerun answer's own fields (accepted_tasks, rerun_of, ...). + return created ? { ...pickAnswerFields(created), ...job } : job; + } finally { + process.off("SIGINT", onSigint); + } +} + +/** The fields only a submit or rerun answer carries. */ +function pickAnswerFields(job: Job): Partial { + const out: Partial = {}; + for (const k of ["accepted_tasks", "rerun_of", "retried_tasks", "inherited_tasks"] as const) { + if (job[k] !== undefined) (out as Record)[k] = job[k]; + } + return out; +} + +function waitHeadline(job: Job): string { + if (isTerminal(job.latest_run)) return `Job ${job.job_id} finished`; + if (isPaused(job.latest_run)) return `Stopped waiting: job ${job.job_id} is paused`; + return `Stopped waiting: job ${job.job_id} is still running`; +} + +/** Set when a wait ended without a terminal run: it ran out, or the run is paused. */ +function waitNote(job: Job): string | undefined { + if (isTerminal(job.latest_run)) return undefined; + if (isPaused(job.latest_run)) return pausedNote(job.job_id); + return `The wait ran out and the run is still in progress. Run zenrows batch wait ${job.job_id}.`; +} + +function pausedNote(id: string): string { + return `The run is paused: it picks up no new task until it is resumed. Run zenrows batch resume ${id}.`; } async function resultsCmd(rest: string[], ctx: RunContext): Promise { const { values, positionals } = parse(rest, { status: { type: "string" }, + run: { type: "string" }, out: { type: "string" }, download: { type: "string" }, json: { type: "boolean" }, }); const json = ctx.json || values.json === true; - const id = requireId(positionals[0]); + const id = requireId(positionals[0], "results"); const status = normalizeResultStatus(asString(values.status)); + const runId = asString(values.run); assertUsable("batch"); const apiKey = requireApiKey(); - const rows = await listResults(id, { apiKey, status }); + const rows = await listResults(id, { apiKey, status, runId }); + const state = await runState(id, runId, apiKey); + const note = state.partial ? partialNote(state.status, id) : undefined; + const meta = { jobId: id, job_id: id, ...(runId ? { run_id: runId } : {}), ...(state.status ? { status: state.status } : {}) }; const downloadDir = asString(values.download); if (downloadDir) { const out = await downloadResults(rows, downloadDir); + const expired = out.skipped.filter((s) => s.reason.startsWith("result expired")).length; if (json) { log.out( JSON.stringify( { ok: out.failed.length === 0, - jobId: id, + ...meta, dir: out.dir, downloaded: out.downloaded.length, failed: out.failed.length, skipped: out.skipped.length, + expired, + partial: state.partial, + ...(note ? { note } : {}), results: out, }, null, @@ -255,11 +537,15 @@ async function resultsCmd(rest: string[], ctx: RunContext): Promise { } else { log.success(`Downloaded ${out.downloaded.length}/${rows.length} result(s) → ${out.dir}`); log.dim(` index: ${join(out.dir, "_manifest.jsonl")}`); - if (out.skipped.length) log.dim(` skipped ${out.skipped.length} (no body — e.g. failed tasks)`); + if (out.skipped.length) { + log.dim(` skipped ${out.skipped.length} with no body (failed, expired or unfinished tasks${expired ? `; ${expired} expired` : ""})`); + } + if (expired) log.dim(` expired results are gone after 14 days by default; rerun the job in full: zenrows batch rerun ${id}`); if (out.failed.length) { log.warn(` ${out.failed.length} download(s) failed:`); out.failed.slice(0, 10).forEach((f) => log.dim(` ${f.external_id ?? f.task_id}: ${f.reason}`)); } + if (note) log.warn(` ${note}`); } return out.failed.length ? 1 : 0; } @@ -267,131 +553,331 @@ async function resultsCmd(rest: string[], ctx: RunContext): Promise { const jsonl = rows.map((r) => JSON.stringify(r)).join("\n"); const outFile = asString(values.out); if (outFile) { - if (statSync(outFile, { throwIfNoEntry: false })?.isDirectory()) { - throw new ToolkitError({ - code: "INVALID_USAGE", - message: `--out expects a file, but '${outFile}' is a directory.`, - likely_cause: "--out writes the results listing to a single JSONL file; you passed a directory.", - next_action: - "Use a file path (e.g. --out results.jsonl), or download each result body into a directory with --download .", - suggested_commands: [`zenrows batch results ${id} --download ${outFile}`], - }); - } writeOut(outFile, jsonl + (jsonl ? "\n" : "")); - log.success(`Wrote ${rows.length} result(s) → ${outFile}`); + if (json) { + log.out(JSON.stringify({ ok: true, ...meta, count: rows.length, file: outFile, partial: state.partial, ...(note ? { note } : {}) }, null, 2)); + } else { + log.success(`Wrote ${rows.length} result(s) → ${outFile}`); + } } else if (json) { - log.out(JSON.stringify({ ok: true, jobId: id, count: rows.length, results: rows }, null, 2)); + log.out(JSON.stringify({ ok: true, ...meta, count: rows.length, results: rows, partial: state.partial, ...(note ? { note } : {}) }, null, 2)); } else { - log.info(`${rows.length} result(s) for job ${id}:`); + log.info(`${rows.length} result(s) for job ${id}${state.status ? ` (status: ${state.status})` : ""}:`); log.out(jsonl); } + if (note && !json) log.warn(` ${note}`); + return 0; +} + +/** + * Whether the run whose results were read can still change: in progress, or + * still receiving its tasks. An error here never fails the listing. + */ +async function runState(id: string, runId: string | undefined, apiKey: string): Promise<{ status?: string; partial: boolean }> { + try { + const run: JobRun | undefined = runId ? await getRun(id, runId, { apiKey }) : (await getJob(id, { apiKey })).latest_run; + if (!run) return { partial: false }; + return { status: run.status, partial: !isTerminal(run) || run.ingest_status === "pending" }; + } catch { + return { partial: false }; + } +} + +function partialNote(status: string | undefined, id: string): string { + return status && isTerminal({ status } as JobRun) + ? `The run is still receiving its tasks: these results are partial. Read them again once zenrows batch status ${id} shows ingest_status done.` + : `The run is still in progress: these results are partial. Wait with zenrows batch wait ${id}, then read them again.`; +} + +async function contentCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { run: { type: "string" }, out: { type: "string" }, json: { type: "boolean" } }); + const json = ctx.json || values.json === true; + const id = requireId(positionals[0], "content"); + const taskId = positionals[1]; + if (!taskId) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: "Missing task id.", + likely_cause: "No positional was provided.", + next_action: `Usage: zenrows batch content ${id} . Each result row carries its task_id.`, + suggested_commands: [`zenrows batch results ${id}`], + }); + } + const runId = asString(values.run); + assertUsable("batch"); + const apiKey = requireApiKey(); + const { contentType, body } = await getTaskContent(id, taskId, { apiKey, runId }); + const ids = { job_id: id, task_id: taskId, ...(runId ? { run_id: runId } : {}) }; + const outFile = asString(values.out); + if (outFile) { + writeOut(outFile, body); + if (json) { + log.out(JSON.stringify({ ok: true, ...ids, content_type: contentType, bytes: body.length, file: outFile }, null, 2)); + } else { + log.success(`Wrote ${body.length} bytes (${contentType || "unknown type"}) → ${outFile}`); + } + } else if (json) { + log.out(JSON.stringify({ ok: true, ...ids, content_type: contentType, bytes: body.length, content: body.toString("utf8") }, null, 2)); + } else { + process.stdout.write(body); + if (body.length && body[body.length - 1] !== 0x0a && isText(contentType)) process.stdout.write("\n"); + } return 0; } +function isText(contentType: string): boolean { + return /^(text\/|application\/(json|xml))/i.test(contentType); +} + async function cancelCmd(rest: string[], ctx: RunContext): Promise { - const { positionals } = parse(rest, {}); - const id = requireId(positionals[0]); + const { values, positionals } = parse(rest, { json: { type: "boolean" } }); + const json = ctx.json || values.json === true; + const id = requireId(positionals[0], "cancel"); assertUsable("batch"); const apiKey = requireApiKey(); - const job = await stopJob(id, { apiKey }); - return printJob(job, ctx.json, `Stopped job ${id}`); + let run: JobRun; + try { + run = await stopJob(id, { apiKey }); + } catch (err) { + // An ended run cannot be stopped: answer with the job as it ended, like crawl's stop. + if (err instanceof ToolkitError && err.server_code === "run_not_stoppable") { + const job = await getJob(id, { apiKey }); + return printJob(job, json, `Nothing to stop: job ${id} had already ended`); + } + throw err; + } + return printJob(asJob(id, run), json, `Stopped job ${id}`); } -async function waitCmd(rest: string[], ctx: RunContext): Promise { - const { values, positionals } = parse(rest, { timeout: { type: "string" }, json: { type: "boolean" } }); +async function pauseCmd(rest: string[], ctx: RunContext, op: "pause" | "resume"): Promise { + const { values, positionals } = parse(rest, { json: { type: "boolean" } }); + const json = ctx.json || values.json === true; + const id = requireId(positionals[0], op); + assertUsable("batch"); + const apiKey = requireApiKey(); + const run = op === "pause" ? await pauseJob(id, { apiKey }) : await resumeJob(id, { apiKey }); + const job = asJob(id, run); + return op === "pause" + ? printJob(job, json, `Paused job ${id}`, { note: isPaused(run) ? pausedNote(id) : undefined }) + : printJob(job, json, `Resumed job ${id}`); +} + +/** stop, pause and resume answer with the run, not the job. */ +function asJob(id: string, run: JobRun): Job { + return { job_id: run.job_id ?? id, latest_run: run }; +} + +async function rerunCmd(rest: string[], ctx: RunContext, sub: "rerun" | "retry-failed"): Promise { + const { values, positionals } = parse(rest, { + ...(sub === "rerun" ? { status: { type: "string" } } : { "include-pending": { type: "boolean" } }), + "idempotency-key": { type: "string" }, + follow: { type: "boolean" }, + timeout: { type: "string" }, + json: { type: "boolean" }, + }); const json = ctx.json || values.json === true; - const id = requireId(positionals[0]); + const id = requireId(positionals[0], sub); + const status = + sub === "rerun" ? rerunStatus(asString(values.status)) : values["include-pending"] === true ? UNFINISHED : "failed"; + const waitMs = timeoutFlag(values.timeout); + const givenKey = asString(values["idempotency-key"]); + if (givenKey !== undefined) assertIdempotencyKey(givenKey); assertUsable("batch"); const apiKey = requireApiKey(); - const job = await waitForJob(id, { apiKey, timeoutMs: asNumber(values.timeout) }); - return printJob(job, json, `Job ${id} finished`); + let job: Job; + try { + job = await rerunJob(id, { apiKey, status, idempotencyKey: givenKey ?? newIdempotencyKey() }); + } catch (err) { + throw withRepeatCommand(err, sub, rest); + } + const what = status ? `tasks with status ${status}` : "every task"; + if (values.follow === true) { + const finished = await followJob(job.job_id ?? id, apiKey, waitMs, job); + return printJob(finished, json, waitHeadline(finished), { note: waitNote(finished) }); + } + return printJob(job, json, `Started a new run of job ${id} (${what})`); } -async function retryCmd(rest: string[], ctx: RunContext): Promise { - const { positionals } = parse(rest, {}); - const id = requireId(positionals[0]); +/** `--status` for a rerun: a comma-separated list of task statuses; undefined (unset) means every task. */ +function rerunStatus(v: string | undefined): string | undefined { + if (v === undefined) return undefined; + const parts = [...new Set(v.split(",").map((s) => s.trim().toLowerCase()).filter(Boolean))]; + const bad = parts.find((p) => !RERUN_STATUSES.includes(p)); + if (!parts.length || bad !== undefined) { + throw usageError( + `Invalid --status '${v}'.`, + `Pass a comma-separated list of ${RERUN_STATUSES.join(", ")} (e.g. --status ${UNFINISHED}), or leave --status out to rerun every task.`, + `A rerun selects tasks of the previous run by status: ${RERUN_STATUSES.join(", ")}.`, + ); + } + return parts.join(","); +} + +async function deleteCmd(rest: string[], ctx: RunContext): Promise { + const { values, positionals } = parse(rest, { json: { type: "boolean" } }); + const json = ctx.json || values.json === true; + const id = requireId(positionals[0], "delete"); + if (!ctx.yes) { + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Deleting job ${id} needs --yes.`, + likely_cause: "Delete stops the job's run and removes the job, its runs and its stored results for good.", + next_action: "Re-run with --yes to confirm.", + suggested_commands: [`zenrows batch delete ${id} --yes`], + }); + } assertUsable("batch"); const apiKey = requireApiKey(); - // "Reruns and retrying failures": POST /jobs/{id}/rerun?status=failed replays - // only the failures; already-successful tasks carry over. - const job = await rerunJob(id, { apiKey, status: "failed" }); - return printJob(job, ctx.json, `Reran failed tasks for job ${id}`); + await deleteJob(id, { apiKey }); + if (json) log.out(JSON.stringify({ ok: true, job_id: id, deleted: true }, null, 2)); + else log.success(`Deleted job ${id}. Its stored results are removed in the background.`); + return 0; } /** The run's failure_reason / failure_detail, when the API reports them. */ function failureFields(job: Job): { failure_reason?: string; failure_detail?: string } { - const run = job.latest_run ?? ({} as Job["latest_run"]); + const run = job.latest_run; const out: { failure_reason?: string; failure_detail?: string } = {}; - if (typeof run.failure_reason === "string" && run.failure_reason) out.failure_reason = run.failure_reason; - if (typeof run.failure_detail === "string" && run.failure_detail) out.failure_detail = run.failure_detail; + if (typeof run?.failure_reason === "string" && run.failure_reason) out.failure_reason = run.failure_reason; + if (typeof run?.failure_detail === "string" && run.failure_detail) out.failure_detail = run.failure_detail; return out; } /** - * The error for a run that ended `failed`, or null for any other state. A run the - * key's credit cap stopped (`api_key_cap_reached`) gets the cap guidance: the - * account still has credits and its other keys keep working. + * The error for a run that ended `failed`, or null for any other state. A run + * the account's credits, its subscription or the key's credit cap stopped gets + * the code and advice the matching HTTP refusal gets (POLICY_MAX_CREDITS_EXCEEDED, + * AUTH_INVALID, KEY_CREDIT_CAP_REACHED); any other reason is BATCH_FAILED. All + * carry `server_code` (the failure_reason) and `job_id`, and say how to run the + * tasks the failure left unfinished. */ export function runFailure(job: Job): ToolkitError | null { if (job.latest_run?.status !== "failed") return null; const { failure_reason, failure_detail } = failureFields(job); - const where = `batch job ${job.job_id}`; + const id = job.job_id; + const where = `batch job ${id}`; + const rerun = `zenrows batch rerun ${id} --status ${UNFINISHED}`; + const ids = { ...(failure_reason ? { server_code: failure_reason } : {}), job_id: id }; + const detail = failure_detail ? `${failure_detail.replace(/[.\s]+$/, "")}. ` : ""; if (isKeyCapReached(failure_reason)) { - return keyCapReached(where, { status: null, detail: failure_detail }); + const cap = keyCapReached(where, { status: null, detail: failure_detail }).toJSON(); + return new ToolkitError({ + ...cap, + ...ids, + next_action: thenRerun(cap.next_action, rerun), + suggested_commands: [...(cap.suggested_commands ?? []), rerun], + }); + } + if (failure_reason === "insufficient_credits") { + const acct = readAccount(); + const quota = quotaExhausted(where, acct?.unclaimed ? acct.claimUrl : undefined, { detail: failure_detail }).toJSON(); + return new ToolkitError({ + ...quota, + ...ids, + likely_cause: `${detail}Stopped ${where}: the account ran out of credits.`, + next_action: thenRerun(quota.next_action, rerun), + suggested_commands: [...(quota.suggested_commands ?? []), rerun], + }); + } + if (failure_reason === "subscription_inactive") { + const sub = subscriptionInactive(`${detail}Stopped ${where}`).toJSON(); + return new ToolkitError({ + ...sub, + ...ids, + next_action: thenRerun(sub.next_action, rerun), + suggested_commands: [rerun], + }); } const why = [failure_reason, failure_detail].filter(Boolean).join(": "); return new ToolkitError({ code: "BATCH_FAILED", - message: `Batch job ${job.job_id} failed.`, + message: `Batch job ${id} stopped before it finished.`, likely_cause: why || "The run ended in status failed without a reason.", - next_action: "Inspect the per-task results, fix the cause, then rerun the failed tasks.", - suggested_commands: [`zenrows batch results ${job.job_id} --status failed`, `zenrows batch retry-failed ${job.job_id}`], + next_action: `Check the run with \`zenrows batch status ${id}\`. Once the cause is fixed, run the unfinished tasks: \`${rerun}\`.`, + suggested_commands: [`zenrows batch status ${id}`, rerun], + ...ids, }); } +/** `next` followed by the command that runs the tasks a failure left unfinished. */ +function thenRerun(next: string, rerun: string): string { + return `${next}${/[.!?]$/.test(next) ? "" : "."} Then run the unfinished tasks: \`${rerun}\`.`; +} + /** - * Print a job's status + stats, structured under --json, and return the exit - * code: 1 when the run ended `failed`, else 0. `stopped` and `deleted` are - * deliberate outcomes (someone cancelled or removed the run), so they exit 0 - * but print as a warning, not a success. + * Print a job's latest run, structured under --json, and return the exit code: + * 1 when the run ended `failed`, else 0. `stopped` and `deleted` are deliberate + * outcomes (someone cancelled or removed the run), so they exit 0 but print as + * a warning. `extra.note` says why a wait ended without a terminal run. */ -function printJob(job: Job, json: boolean, headline: string): number { - const run = job.latest_run ?? ({} as Job["latest_run"]); - const stats = run.stats; +function printJob(job: Job, json: boolean, headline: string, extra: { note?: string } = {}): number { + const run = job.latest_run; + const stats = run?.stats; const failure = failureFields(job); const err = runFailure(job); + const paused = isPaused(run); + const answer = pickAnswerFields(job); + const ingest = run?.ingest_status; if (json) { log.out( JSON.stringify( - { ok: !err, jobId: job.job_id, status: run.status, stats, ...failure, ...(err ? { error: err.toJSON() } : {}) }, + { + ok: !err, + job_id: job.job_id, + jobId: job.job_id, + ...(run?.run_id ? { run_id: run.run_id } : {}), + status: run?.status ?? null, + ...(paused ? { paused: true } : {}), + stats, + ...failure, + ...(ingest ? { ingest_status: ingest } : {}), + ...answer, + job, + ...(extra.note ? { note: extra.note } : {}), + ...(err ? { error: err.toJSON() } : {}), + }, null, 2, ), ); return err ? 1 : 0; } - const line = `${headline} · status: ${run.status ?? "unknown"}`; + const line = `${headline} · status: ${run?.status ?? "no run yet"}${paused ? " (paused)" : ""}`; if (err) log.error(line); - else if (run.status === "stopped" || run.status === "deleted") log.warn(line); + else if (run?.status === "stopped" || run?.status === "deleted" || paused || extra.note) log.warn(line); else log.success(line); + if (run?.run_id) log.info(` run: ${run.run_id}`); if (stats) { log.info(` ${stats.completed}/${stats.total} completed · ${stats.successful} successful · ${stats.failed} failed`); } + if (answer.accepted_tasks !== undefined) log.info(` accepted_tasks: ${answer.accepted_tasks}`); + if (answer.rerun_of) { + log.info(` rerun of ${answer.rerun_of}: ${answer.retried_tasks ?? 0} task(s) retried, ${answer.inherited_tasks ?? 0} carried over`); + } + if (ingest === "pending") log.dim(" ingest_status: pending (the API is still receiving the tasks; results stay partial until done)"); if (failure.failure_reason) log.info(` failure_reason: ${failure.failure_reason}`); if (failure.failure_detail) log.info(` failure_detail: ${failure.failure_detail}`); + if (!err && run?.status === "stopped" && stats && stats.completed < stats.total) { + log.dim(` ${stats.total - stats.completed} task(s) did not finish: zenrows batch rerun ${job.job_id} --status ${UNFINISHED}`); + } + if (extra.note) { + log.dim(` ${extra.note}`); + log.dim(` next: zenrows batch ${paused ? "resume" : "wait"} ${job.job_id}`); + } if (err) printError(err, false); return err ? 1 : 0; } -function normalizeResultStatus(v?: string): "successful" | "failed" | "all" | undefined { +function normalizeResultStatus(v?: string): ResultStatus | undefined { if (!v) return undefined; const s = v.toLowerCase(); - if (s === "successful" || s === "failed" || s === "all") return s; + if (s === "successful" || s === "failed" || s === "processing" || s === "all") return s; throw new ToolkitError({ code: "INVALID_USAGE", message: `Invalid --status: ${v}`, - likely_cause: "Only successful | failed | all are supported.", - next_action: "Use --status successful | failed | all (default: all).", + likely_cause: "Only successful | failed | processing | all are supported.", + next_action: + "Use --status successful | failed | processing | all (default: all). processing lists the tasks not finished yet; expired rows come back with successful.", }); } @@ -421,14 +907,54 @@ function normalizeOutput(v?: string): string | undefined { return mapped ? mapped : undefined; // html → undefined (no response_type) } -function requireId(id: string | undefined): string { +/** + * `--timeout` as a duration, returned in milliseconds; undefined when unset. + * Takes `500ms`, `30s`, `5m` or `1h`; a bare number keeps its earlier meaning, + * milliseconds. + */ +export function timeoutFlag(v: unknown): number | undefined { + if (v === undefined) return undefined; + const m = typeof v === "string" ? /^\s*(\d+(?:\.\d+)?)\s*(ms|s|m|h)?\s*$/i.exec(v) : null; + const unit = { ms: 1, s: 1000, m: 60_000, h: 3_600_000 }[(m?.[2] ?? "ms").toLowerCase() as "ms" | "s" | "m" | "h"]; + const ms = m ? Number(m[1]) * unit : NaN; + if (!Number.isFinite(ms) || ms <= 0) { + throw usageError( + `Invalid --timeout value '${String(v)}'.`, + "Pass a duration such as --timeout 30s, 5m or 1h (a bare number is milliseconds).", + "--timeout takes a positive duration.", + ); + } + return Math.round(ms); +} + +/** `--limit` for a paged read: a whole number from 1 to `max`. */ +function pageLimit(v: unknown, max: number): number | undefined { + if (v === undefined) return undefined; + const n = asNumber(v); + if (n === undefined || !Number.isInteger(n) || n < 1 || n > max) { + throw usageError(`Invalid --limit value '${String(v)}'.`, `Pass e.g. --limit ${Math.min(max, 50)}.`, `--limit takes a whole number from 1 to ${max}.`); + } + return n; +} + +function oneOf(flag: string, v: string | undefined, allowed: string[]): string | undefined { + if (v === undefined) return undefined; + if (allowed.includes(v)) return v; + throw usageError(`Invalid --${flag} '${v}'.`, `Pass --${flag} ${allowed.join(" or ")}, or leave it out.`); +} + +function usageError(message: string, next_action: string, likely_cause = "A flag value is outside what the Batch API accepts."): ToolkitError { + return new ToolkitError({ code: "INVALID_USAGE", message, likely_cause, next_action }); +} + +function requireId(id: string | undefined, sub: string): string { if (id) return id; throw new ToolkitError({ code: "INVALID_USAGE", message: "Missing job id.", likely_cause: "No positional was provided.", - next_action: "Usage: zenrows batch status ", - suggested_commands: ["zenrows batch status "], + next_action: `Usage: zenrows batch ${sub} . List your jobs with \`zenrows batch list\`.`, + suggested_commands: ["zenrows batch list"], }); } diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index 47a5a90..e0c7fd8 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -50,7 +50,7 @@ export const crawl: Command = { " leave it out for no depth limit (the caps below stop the crawl)", " --max-items stop after keeping n URLs (API default 100)", " --max-pages stop after fetching n pages (API default 100; bounds cost);", - " the local policy max_pages_per_run caps it (default 1000)", + " the local policy max_pages_per_run caps it (default 10000)", " --include-patterns

keep only URLs containing this substring (repeatable)", " --exclude-patterns

drop URLs containing this substring (repeatable)", " --output-format also fetch each kept URL's page, as HTML or Markdown", diff --git a/src/core/batch-api.ts b/src/core/batch-api.ts index 2b5cf28..bd4ca5d 100644 --- a/src/core/batch-api.ts +++ b/src/core/batch-api.ts @@ -4,19 +4,28 @@ * A SEPARATE host from Fetch and Extract (`api.zenrows.com/v1`): batch * jobs live at `https://async.api.zenrows.com/v1`. Auth is the `X-API-Key` * header (same key as Fetch and Extract), never the `apikey` query param. Bodies - * are JSON; errors come back as `application/problem+json` (RFC 7807) with a - * stable `code` we branch on. The key is registered as a secret so it is - * redacted from any logged output. `fetchImpl` is injectable for tests, mirroring - * `usage.ts` / `agent-account.ts`. + * are JSON; errors come back as `application/problem+json` (RFC 9457) with a + * stable `code` we branch on and carry as `server_code`. The key is registered + * as a secret so it is redacted from any logged output. `fetchImpl` is + * injectable for tests, mirroring `usage.ts` / `agent-account.ts`. * - * Deferred (documented, not wired here): CSV upload (`/job_inputs` + presigned - * PUT), open/queue jobs (`/tasks`, `last_batch`), scheduled jobs - * (`schedule` / `PUT /schedule` / `schedule/state`), webhooks + HMAC, ZIP - * archive export, and the `Idempotency-Key` header. + * Retries: reads (GET, DELETE) retry 429/502/503/504 and connection errors; + * stop, pause and resume retry 502/503/504 and connection errors, since + * repeating them is harmless; a submit or rerun retries the same way only + * under its `Idempotency-Key`, which `createJob` and `rerunJob` always send and + * reuse across the retries of one call. The API keeps a key for 24 hours and + * replays the original answer to a repeat with the same body, so a retry never + * starts a second job or run. A keyed write that still has no answer after its + * retries fails with `idempotency_key` set: repeating it with that key is safe. + * + * Not in the CLI (the API and the SDKs have them): CSV inputs, open jobs and + * add-tasks, scheduled jobs, webhooks and HMAC keys, ZIP exports, the runs + * list, and task history. */ import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; -import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.ts"; +import { randomUUID } from "node:crypto"; +import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted, type ErrorCode } from "./errors.ts"; import { readAccount } from "./agent-account.ts"; import { registerSecret } from "./logger.ts"; import { CLI_USER_AGENT } from "./config.ts"; @@ -40,25 +49,45 @@ export interface JobStats { completed: number; successful: number; failed: number; + [k: string]: unknown; } export interface JobRun { status: string; stats: JobStats; run_id?: string; - /** Why a `failed` run stopped, e.g. `api_key_cap_reached`. */ + job_id?: string; + /** Why a `failed` run stopped, e.g. `insufficient_credits` or `api_key_cap_reached`. */ failure_reason?: string; /** Human-readable detail for `failure_reason`. */ failure_detail?: string; + /** `paused` while the run is paused; omitted while it is active. */ + pause_state?: string; + /** `pending` while a large submit or a rerun is still receiving its tasks, then `done`. */ + ingest_status?: string; [k: string]: unknown; } export interface Job { job_id: string; - latest_run: JobRun; + /** Absent on a scheduled job that has not fired yet. */ + latest_run?: JobRun; + /** `open | closed` (the job, not its run). */ + status?: string; + /** Submit answers: how many tasks the API accepted. */ + accepted_tasks?: number; + /** Rerun answers: the run this one reruns, and how its tasks were split. */ + rerun_of?: string; + retried_tasks?: number; + inherited_tasks?: number; [k: string]: unknown; } +export interface JobsPage { + jobs: Job[]; + next_cursor: string | null; +} + export interface ResultRow { external_id?: string; task_id: string; @@ -72,230 +101,561 @@ export interface ResultsPage { next_cursor: string | null; } -/** RFC 7807 problem body returned by the Batch API on non-2xx. */ +/** RFC 9457 problem body returned by the Batch API on non-2xx. */ interface ProblemJson { type?: string; title?: string; status?: number; detail?: string; code?: string; - invalid_tasks?: Array<{ index: number; reason: string }>; + invalid_tasks?: Array<{ index: number; reason: string; value?: string }>; } -/** Terminal run states — a run stops progressing once it reaches one of these. */ +/** Terminal run states: a run stops progressing once it reaches one of these. */ export const TERMINAL_STATUSES: ReadonlySet = new Set(["completed", "stopped", "failed", "deleted"]); -interface RequestOpts { +/** Result filters the API accepts on the results listing. */ +export type ResultStatus = "successful" | "failed" | "processing" | "all"; + +/** Task statuses a rerun can select (`?status=` on `/rerun`). */ +export const RERUN_STATUSES: readonly string[] = ["pending", "processing", "failed", "successful"]; + +/** True when the run reached a terminal state. */ +export function isTerminal(run: JobRun | undefined): boolean { + return !!run && TERMINAL_STATUSES.has(run.status); +} + +/** True when a live run is paused: it picks up no new task until resumed. */ +export function isPaused(run: JobRun | undefined): boolean { + return run?.pause_state === "paused" && !isTerminal(run); +} + +type RetryKind = "read" | "write"; + +interface CallOpts { apiKey: string; - body?: unknown; - query?: Record; fetchImpl?: typeof fetch; + /** Per HTTP attempt. */ timeoutMs?: number; + sleepImpl?: (ms: number) => Promise; } +interface RequestOpts extends CallOpts { + body?: unknown; + query?: Record; + /** Which failures to retry; unset means the request runs once. */ + retry?: RetryKind; + /** Sent as `Idempotency-Key`. */ + idempotencyKey?: string; + /** The job the request is about, carried on its errors as `job_id`. */ + jobId?: string; +} + +interface RawResponse { + status: number; + headers: Headers; + body: Buffer; +} + +const MAX_RETRIES = 3; +/** The longest a Retry-After may make one retry wait, so a large value cannot hang a command. */ +const MAX_RETRY_AFTER_SECONDS = 30; +const READ_RETRY_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]); +// 429 on a write is an account limit (active jobs): a retry only waits for a slot that may never free. +const WRITE_RETRY_STATUSES: ReadonlySet = new Set([502, 503, 504]); + +/** 250 ms x 2^attempt, capped at 10 s, +/-20% jitter. */ +function backoffMs(attempt: number): number { + return Math.min(250 * 2 ** attempt, 10_000) * (0.8 + Math.random() * 0.4); +} + +/** `Retry-After` in whole seconds, as milliseconds, at most `MAX_RETRY_AFTER_SECONDS`. */ +function retryAfterMs(value: string | null): number | undefined { + return value && /^\d+$/.test(value.trim()) ? Math.min(Number(value), MAX_RETRY_AFTER_SECONDS) * 1000 : undefined; +} + +const defaultSleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +/** A fresh Idempotency-Key for one logical submit or rerun. */ +export function newIdempotencyKey(): string { + return randomUUID(); +} + +/** The longest Idempotency-Key the Batch API accepts. */ +export const MAX_IDEMPOTENCY_KEY_LENGTH = 128; + /** - * Perform a Batch API request. Sets `X-API-Key`, adds a JSON content-type when a - * body is present, parses the JSON response, and on a non-2xx parses the - * problem+json body into a normalized ToolkitError. Returns `null` for empty - * (204 / no-body) responses. + * Throw before any request if `key` cannot work as an Idempotency-Key: HTTP + * strips a header's surrounding whitespace, so a blank or padded key would + * reach the API as a different key, and control characters cannot be sent. */ -export async function batchRequest(method: string, path: string, opts: RequestOpts): Promise { +export function assertIdempotencyKey(key: string): void { + let reason: string | undefined; + if (key.trim() === "") reason = "it is empty"; + else if (key !== key.trim()) reason = "it starts or ends with whitespace"; + else if (/[\u0000-\u001f\u007f]/.test(key)) reason = "it contains a control character"; + else if (key.length > MAX_IDEMPOTENCY_KEY_LENGTH) reason = `it is longer than ${MAX_IDEMPOTENCY_KEY_LENGTH} characters`; + if (!reason) return; + throw new ToolkitError({ + code: "INVALID_USAGE", + message: `Invalid --idempotency-key: ${reason}.`, + likely_cause: `An Idempotency-Key is 1 to ${MAX_IDEMPOTENCY_KEY_LENGTH} characters, with no surrounding whitespace and no control characters.`, + next_action: "Pass a key such as a UUID, or leave --idempotency-key out to have one generated.", + }); +} + +/** Errors a poll may ride out: the API was unreachable or answered 429/5xx. */ +const transientErrors = new WeakSet(); + +/** True for an error worth polling past (a connection failure, a 429 or a 5xx). */ +export function isTransient(err: unknown): boolean { + return err instanceof ToolkitError && transientErrors.has(err); +} + +function transient(err: ToolkitError): ToolkitError { + transientErrors.add(err); + return err; +} + +/** One HTTP attempt with its own timeout; the body is read inside it. */ +async function attempt(url: string, init: RequestInit, opts: RequestOpts): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000); + try { + const res = await (opts.fetchImpl ?? fetch)(url, { ...init, signal: controller.signal }); + return { status: res.status, headers: res.headers, body: Buffer.from(await res.arrayBuffer()) }; + } finally { + clearTimeout(timeout); + } +} + +/** + * Perform a Batch API request and return the raw response. Sets `X-API-Key`, + * adds a JSON content-type when a body is present, retries transient failures + * as `opts.retry` allows, and on a non-2xx parses the problem+json body into a + * normalized ToolkitError. + */ +async function batchFetch(method: string, path: string, opts: RequestOpts): Promise { registerSecret(opts.apiKey); const url = new URL(batchBase() + path); for (const [k, v] of Object.entries(opts.query ?? {})) { - if (v !== undefined && v !== null) url.searchParams.set(k, v); + if (v !== undefined && v !== null) url.searchParams.set(k, String(v)); } - const doFetch = opts.fetchImpl ?? fetch; - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000); - const headers: Record = { "X-API-Key": opts.apiKey, Accept: "application/json", "User-Agent": CLI_USER_AGENT, ...agentClientHeader(), }; + if (opts.idempotencyKey) headers["Idempotency-Key"] = opts.idempotencyKey; if (opts.body !== undefined) headers["Content-Type"] = "application/json"; - - let res: Response; - try { - res = await doFetch(url.toString(), { - method, - headers, - body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined, - signal: controller.signal, - }); - } catch (err) { - clearTimeout(timeout); - throw new ToolkitError({ - code: "BACKEND_UNAVAILABLE", - message: "Could not reach the Zenrows Batch API.", - likely_cause: err instanceof Error ? err.message : String(err), - next_action: - "Check connectivity and retry. Override the host with ZENROWS_BATCH_API_BASE if you are testing against staging.", - suggested_commands: ["zenrows status"], - }); + const init: RequestInit = { method, headers, body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined }; + const retryOn = opts.retry === "read" ? READ_RETRY_STATUSES : opts.retry === "write" ? WRITE_RETRY_STATUSES : undefined; + const sleep = opts.sleepImpl ?? defaultSleep; + const key = opts.idempotencyKey; + // Set once an attempt of a keyed write got no answer: the API may have done the work. + let mayHaveLanded = false; + + for (let n = 0; ; n++) { + const canRetry = retryOn !== undefined && n < MAX_RETRIES; + let res: RawResponse; + try { + res = await attempt(url.toString(), init, opts); + } catch (err) { + if (canRetry) { + mayHaveLanded = true; + await sleep(backoffMs(n)); + continue; + } + const failure = transient( + new ToolkitError({ + code: "BACKEND_UNAVAILABLE", + message: "Could not reach the Zenrows Batch API.", + likely_cause: err instanceof Error ? err.message : String(err), + next_action: + "Check connectivity and retry. Override the host with ZENROWS_BATCH_API_BASE if you are testing against staging.", + suggested_commands: ["zenrows status"], + ...(opts.jobId ? { job_id: opts.jobId } : {}), + }), + ); + throw key ? outcomeUnknown(failure, key) : failure; + } + if (canRetry && retryOn.has(res.status)) { + if (res.status >= 500) mayHaveLanded = true; + await sleep(retryAfterMs(res.headers.get("retry-after")) ?? backoffMs(n)); + continue; + } + if (res.status < 200 || res.status >= 300) { + let failure = problemToError(res.status, res.body.toString("utf8"), method, path, opts.jobId); + if (res.status === 429 || res.status >= 500) failure = transient(failure); + if (key && (res.status >= 500 || (mayHaveLanded && failure.server_code === "idempotency_key_conflict"))) { + throw outcomeUnknown(failure, key); + } + throw failure; + } + return res; } - clearTimeout(timeout); +} - const text = await res.text(); - if (res.status < 200 || res.status >= 300) { - throw problemToError(res.status, text, method, path); - } +/** `batchFetch` + JSON parse. Returns `null` for an empty body. */ +export async function batchRequest(method: string, path: string, opts: RequestOpts): Promise { + return batchJson(method, path, opts); +} + +async function batchJson(method: string, path: string, opts: RequestOpts): Promise { + const { status, body } = await batchFetch(method, path, opts); + const text = body.toString("utf8"); if (!text) return null as T; try { return JSON.parse(text) as T; } catch { - throw new ToolkitError({ + const failure = new ToolkitError({ code: "BATCH_FAILED", - message: "The Batch API response was not valid JSON.", + message: `The Batch API answered ${method} ${path} (HTTP ${status}) with a body that is not JSON.`, likely_cause: text.slice(0, 240), - next_action: "Retry, or inspect the job in the Zenrows dashboard.", + next_action: "Retry, or check the job with `zenrows batch list`.", + ...(opts.jobId ? { job_id: opts.jobId } : {}), }); + throw opts.idempotencyKey ? outcomeUnknown(failure, opts.idempotencyKey) : failure; } } -/** Map an RFC 7807 problem body + HTTP status to a normalized ToolkitError. */ -function problemToError(status: number, body: string, method: string, path: string): ToolkitError { +/** + * A keyed write (submit or rerun) whose outcome is unknown: the API may have + * done the work although no usable answer arrived. Carry the key and say what + * is safe; the command layer adds the exact command to repeat. + */ +function outcomeUnknown(err: ToolkitError, key: string): ToolkitError { + const out = new ToolkitError({ + ...err.toJSON(), + idempotency_key: key, + next_action: + `The request may have been accepted even though no usable answer arrived. Repeat it with the same Idempotency-Key (${key}) ` + + "within 24 hours: the API then answers with the original job or run instead of creating a second one. " + + "Do not repeat it with a new key, which can create a duplicate. Check `zenrows batch list` first.", + }); + return isTransient(err) ? transient(out) : out; +} + +/** `s` without a trailing period, so sentences can be joined. */ +function trimDot(s: string): string { + return s.replace(/[.\s]+$/, ""); +} + +/** The error for an account whose subscription is not active (401 subscription_inactive, 402 expired). */ +export function subscriptionInactive(cause: string): ToolkitError { + // Same AUTH_INVALID code as any 401 so callers that branch on it keep + // working; only the guidance changes, since re-entering the key won't help. + return new ToolkitError({ + code: "AUTH_INVALID", + message: "Zenrows refused the Batch API request because the subscription is not active.", + likely_cause: `${trimDot(cause)}. The API key is valid, but the subscription is expired, past due, unpaid, incomplete or paused.`, + next_action: `Fix the subscription in Billing (${BILLING_URL}), or in Azure Marketplace if you subscribed there, then retry.`, + }); +} + +/** Map a problem+json body + HTTP status to a normalized ToolkitError carrying `server_code` and `job_id`. */ +function problemToError(status: number, text: string, method: string, path: string, jobId?: string): ToolkitError { let problem: ProblemJson = {}; try { - problem = JSON.parse(body) as ProblemJson; + problem = JSON.parse(text) as ProblemJson; } catch { - // non-JSON error body — fall through with an empty problem + // non-JSON error body: fall through with an empty problem } - const serverCode = problem.code ?? ""; - const detail = problem.detail || problem.title || body.slice(0, 240) || `HTTP ${status}`; - const cause = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${method} ${path}: ${detail}`; + const serverCode = problem.code || undefined; + const detail = trimDot(problem.detail || problem.title || text.slice(0, 240) || `HTTP ${status}`); + const cause = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${method} ${path}: ${detail}.`; + const err = classifyProblem(status, problem, detail, cause, serverCode, method, path, jobId ?? ""); + return new ToolkitError({ + ...err.toJSON(), + ...(serverCode ? { server_code: serverCode } : {}), + ...(jobId ? { job_id: jobId } : {}), + }); +} + +function classifyProblem( + status: number, + problem: ProblemJson, + detail: string, + cause: string, + serverCode: string | undefined, + method: string, + path: string, + id: string, +): ToolkitError { + const err = (code: ErrorCode, message: string, next_action: string, suggested_commands?: string[], likely = cause) => + new ToolkitError({ code, message, likely_cause: likely, next_action, suggested_commands }); if (status === 403) { - return new ToolkitError({ - code: "BATCH_ACCESS_DENIED", - message: "The Batch API rejected this request (access denied).", - likely_cause: `${cause}. Batch is not enabled for this account or API key.`, - next_action: - "Check that your Zenrows API key is active, and contact Zenrows support if the 403 persists. Meanwhile validate/estimate specs locally and fan out with `zenrows fetch` per URL.", - suggested_commands: ["zenrows batch estimate jobs.jsonl"], - }); - } - if (status === 401 && serverCode === "subscription_inactive") { - // The key is fine; the subscription needs a billing action. Same - // AUTH_INVALID code as any 401 so callers that branch on it keep - // working; only the guidance changes, since re-entering the key won't help. - return new ToolkitError({ - code: "AUTH_INVALID", - message: "Zenrows refused the Batch API request because the subscription is not active.", - likely_cause: `${cause.replace(/\.?$/, ".")} The API key is valid, but the subscription is past due, unpaid, incomplete or paused.`, - next_action: `Fix the subscription in Billing (${BILLING_URL}), or in Azure Marketplace if you subscribed there, then retry.`, - }); + return err( + "BATCH_ACCESS_DENIED", + "The Batch API rejected this request (access denied).", + "Check that your Zenrows API key is active, and contact Zenrows support if the 403 persists. Meanwhile validate/estimate specs locally and fan out with `zenrows fetch` per URL.", + ["zenrows batch estimate jobs.jsonl"], + `${cause} Batch is not enabled for this account or API key.`, + ); } + if (status === 401 && serverCode === "subscription_inactive") return subscriptionInactive(cause); if (status === 401) { - return new ToolkitError({ - code: "AUTH_INVALID", - message: "Zenrows rejected the API key for the Batch API.", - likely_cause: cause, - next_action: "Re-check your key and log in again.", - suggested_commands: ["zenrows login --api-key "], - }); + return err("AUTH_INVALID", "Zenrows rejected the API key for the Batch API.", "Re-check your key and log in again.", [ + "zenrows login --api-key ", + ]); } if (status === 404) { - return new ToolkitError({ - code: "BATCH_NOT_FOUND", - message: "Batch job, run, or task not found.", - likely_cause: `${cause}. The id may be wrong or the job is not owned by this account.`, - next_action: "Check the job id, or list your recent jobs in the Zenrows dashboard.", - }); + const task = /\/tasks\//.test(path); + return err( + "BATCH_NOT_FOUND", + task ? "Batch job, run or task not found." : "Batch job or run not found.", + task + ? `Check the ids. List the job's task ids with \`zenrows batch results ${id}\`.` + : "Check the job id, or list your jobs with `zenrows batch list`.", + [task ? `zenrows batch results ${id}` : "zenrows batch list"], + `${cause} The id may be wrong, the job deleted, or not owned by this account.`, + ); } if (status === 429) { - return new ToolkitError({ - code: "BATCH_QUOTA_EXCEEDED", - message: "Batch quota exceeded.", - likely_cause: `${cause}. An account limit was reached (e.g. the maximum of 3 concurrent active jobs).`, - next_action: - "Wait for an in-flight job to finish (or stop one with `zenrows batch cancel `), then retry.", - suggested_commands: ["zenrows batch status "], - }); + if (/scheduled job/i.test(detail)) { + return err( + "BATCH_QUOTA_EXCEEDED", + "The account has reached its limit of scheduled jobs.", + "Nothing was created. A scheduled job holds its slot until it is deleted, whatever its runs do: delete one you no longer need, then retry.", + ["zenrows batch list --type scheduled", "zenrows batch delete --yes"], + `${cause} The account holds 5 scheduled jobs at most by default.`, + ); + } + if (/task cap/i.test(detail)) { + return err( + "BATCH_QUOTA_EXCEEDED", + "The open job has reached its task cap.", + "Nothing was added. An open job holds 100,000 tasks at most: close it and start a new job for the rest.", + ); + } + return err( + "BATCH_QUOTA_EXCEEDED", + "The account has reached its limit of active jobs.", + "Nothing was created. Active jobs are shared with Crawl. A closed job frees its slot when its run ends: wait for one (`zenrows batch wait `) or stop it (`zenrows batch cancel `). An open or scheduled job frees it only when deleted (`zenrows batch delete --yes`). Then retry.", + ["zenrows batch list", "zenrows crawl list"], + `${cause} The account holds 3 active jobs at most by default, Batch and Crawl together.`, + ); } if (status === 402 && isKeyCapReached(serverCode)) { return keyCapReached(`${method} ${path}`, { status: 402, detail: problem.detail || problem.title || undefined }); } + if (status === 402 && /subscription has expired/i.test(detail)) return subscriptionInactive(cause); if (status === 402) { - // Out of credits ("Subscription has no credit available") — the same - // exhausted state as the scraper/usage 402. Surface the credits error with - // a claim link (unclaimed Free plan) or dashboard/upgrade pointer, not a - // generic "fix the problem and retry". + // Out of credits ("Subscription has no credit available"): the same + // exhausted state as the scraper/usage 402, with a claim link (unclaimed + // Free plan) or dashboard/upgrade pointer. const acct = readAccount(); return quotaExhausted(`${method} ${path}`, acct?.unclaimed ? acct.claimUrl : undefined, { status: 402, detail: problem.detail || problem.title || undefined, }); } + if (status === 409) return conflictError(serverCode, detail, cause, id); + if (status === 410) { + return err( + "BATCH_RESULT_EXPIRED", + "This result has expired.", + `Results are kept 14 days by default; after that a task reads as expired and its body is gone. Scrape it again with a full rerun: \`zenrows batch rerun ${id}\`.`, + [`zenrows batch rerun ${id}`], + ); + } + if (status === 422 && /\/content$/.test(path)) { + return err( + "BATCH_TASK_FAILED", + "This task failed, so it has no content.", + `Read why in its result row (\`zenrows batch results ${id} --status failed\`), then rerun the failures with \`zenrows batch retry-failed ${id}\`.`, + [`zenrows batch results ${id} --status failed`], + ); + } - // 400 invalid_argument, 409 conflict, 5xx, etc. + // 400 invalid_argument, 501, 5xx, etc. const invalid = problem.invalid_tasks?.length ? ` invalid_tasks: ${problem.invalid_tasks .slice(0, 10) - .map((t) => `#${t.index}: ${t.reason}`) + .map((t) => `#${t.index}: ${t.reason}${t.value ? ` (${t.value})` : ""}`) .join("; ")}` : ""; - return new ToolkitError({ - code: "BATCH_FAILED", - message: `Batch request failed (HTTP ${status}).`, - likely_cause: `${cause}.${invalid}`, - next_action: - status >= 500 - ? "This is transient — retry with a short backoff." - : "Fix the reported problem and retry. For validation errors, correct the flagged tasks.", - }); + return err( + "BATCH_FAILED", + `Batch request failed (HTTP ${status}).`, + status === 501 + ? "This operation is not available for this job." + : status >= 500 + ? "This is transient: retry with a short backoff." + : "Fix the reported problem and retry. For validation errors, correct the flagged tasks (#n is the task's 0-based position in the submit).", + undefined, + `${cause}${invalid}`, + ); } -interface CallOpts { - apiKey: string; - fetchImpl?: typeof fetch; - timeoutMs?: number; +/** One code and next action per 409 the Batch API answers. */ +function conflictError(serverCode: string | undefined, detail: string, cause: string, id: string): ToolkitError { + const e = (code: ErrorCode, message: string, next_action: string, suggested_commands: string[] = [`zenrows batch status ${id}`]) => + new ToolkitError({ code, message, likely_cause: cause, next_action, suggested_commands }); + switch (serverCode) { + case "run_not_terminal": + if (/ingestion did not complete/i.test(detail)) { + return e( + "BATCH_RUN_IN_PROGRESS", + "The job's last run never finished receiving its tasks.", + `It cannot be rerun. Delete the job (\`zenrows batch delete ${id} --yes\`) and submit the URLs again as a new job.`, + ); + } + return e( + "BATCH_RUN_IN_PROGRESS", + "The job's current run is still in progress.", + `Wait for it with \`zenrows batch wait ${id}\`, or stop it with \`zenrows batch cancel ${id}\`, then retry. A large submit or a rerun also reads as in progress while it is still receiving its tasks.`, + [`zenrows batch wait ${id}`], + ); + case "run_not_stoppable": + return e( + "BATCH_RUN_ENDED", + "The job's run already ended.", + `Only a pending or running run can be stopped, paused or resumed. Start a new run with \`zenrows batch rerun ${id}\`.`, + ); + case "no_matching_tasks": + return e( + "BATCH_NO_MATCHING_TASKS", + "No task of the last run has the statuses you asked to rerun.", + `Tasks a stop or a credit failure left unfinished are pending or processing: rerun them with \`zenrows batch rerun ${id} --status failed,pending,processing\`, or rerun every task with \`zenrows batch rerun ${id}\`.`, + [`zenrows batch rerun ${id} --status failed,pending,processing`], + ); + case "partial_rerun_window_expired": + return e( + "BATCH_RERUN_WINDOW_EXPIRED", + "Rerunning only some tasks is no longer allowed for this run.", + `A rerun with --status works for 10 days after the run started. Rerun every task with \`zenrows batch rerun ${id}\` instead.`, + [`zenrows batch rerun ${id}`], + ); + case "idempotency_key_conflict": + return e( + "BATCH_REQUEST_IN_FLIGHT", + "This Idempotency-Key is already in use.", + "Either the first request with this key is still being processed, or the key was used for a different request body. Wait a moment and repeat the same command with the same key; use a new key only for a deliberately different request.", + ["zenrows batch list"], + ); + case "task_not_ready": + return e( + "BATCH_TASK_NOT_READY", + "This task has not finished yet.", + `Wait for the run with \`zenrows batch wait ${id}\`, then read the task again.`, + [`zenrows batch wait ${id}`], + ); + case "conflict": + if (/no longer valid/i.test(detail)) { + return e( + "BATCH_CONFLICT", + "The API key that created this job no longer works.", + "The job cannot run again. Submit its URLs as a new job.", + [], + ); + } + return e("BATCH_CONFLICT", "The job changed while the request ran.", "Retry the same command."); + default: + return e( + "BATCH_CONFLICT", + "The job's state does not allow this request.", + `Check the job with \`zenrows batch status ${id}\`; the cause above says what blocked it.`, + ); + } } -/** Create a job. Accepts both 201 (<10k tasks) and 202 (≥10k) as success. */ -export function createJob(body: unknown, opts: CallOpts): Promise { - return batchRequest("POST", "/jobs", { apiKey: opts.apiKey, body, fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs }); +const jobPath = (id: string, rest = "") => `/jobs/${encodeURIComponent(id)}${rest}`; + +/** + * Create a job. Both 201 (tasks stored) and 202 (a large submit, still + * receiving its tasks: `latest_run.ingest_status` is `pending`) are success. + */ +export function createJob(body: unknown, opts: CallOpts & { idempotencyKey?: string }): Promise { + const idempotencyKey = opts.idempotencyKey ?? newIdempotencyKey(); + assertIdempotencyKey(idempotencyKey); + return batchJson("POST", "/jobs", { ...opts, body, idempotencyKey, retry: "write" }); } /** Read a job (`latest_run.status` + `latest_run.stats`). */ export function getJob(id: string, opts: CallOpts): Promise { - return batchRequest("GET", `/jobs/${encodeURIComponent(id)}`, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs }); + return batchJson("GET", jobPath(id), { ...opts, retry: "read", jobId: id }); +} + +/** Read one run of a job. */ +export function getRun(id: string, runId: string, opts: CallOpts): Promise { + return batchJson("GET", jobPath(id, `/runs/${encodeURIComponent(runId)}`), { ...opts, retry: "read", jobId: id }); +} + +/** One page of the account's jobs, newest first. */ +export async function listJobs( + opts: CallOpts & { limit?: number; cursor?: string; status?: string; type?: string }, +): Promise { + const page = await batchJson | null>("GET", "/jobs", { + ...opts, + retry: "read", + query: { limit: opts.limit, cursor: opts.cursor, status: opts.status, type: opts.type }, + }); + return { jobs: page?.jobs ?? [], next_cursor: page?.next_cursor || null }; +} + +/** Stop the job's in-flight run. The API answers with the refreshed run, not the job. */ +export function stopJob(id: string, opts: CallOpts): Promise { + return batchJson("POST", jobPath(id, "/stop"), { ...opts, retry: "write", jobId: id }); +} + +/** Pause the job's in-flight run: no new task is picked up until it is resumed. Answers with the run. */ +export function pauseJob(id: string, opts: CallOpts): Promise { + return batchJson("POST", jobPath(id, "/pause"), { ...opts, retry: "write", jobId: id }); +} + +/** Resume a paused run. Needs credit on the job's API key. Answers with the run. */ +export function resumeJob(id: string, opts: CallOpts): Promise { + return batchJson("POST", jobPath(id, "/resume"), { ...opts, retry: "write", jobId: id }); } -/** Abort an in-flight run. */ -export function stopJob(id: string, opts: CallOpts): Promise { - return batchRequest("POST", `/jobs/${encodeURIComponent(id)}/stop`, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs }); +/** + * Delete a job, its runs and its stored results (202, asynchronous). Stops an + * in-flight run and frees the job's slot. Repeating it is harmless. + */ +export async function deleteJob(id: string, opts: CallOpts): Promise { + await batchFetch("DELETE", jobPath(id), { ...opts, retry: "read", jobId: id }); } /** - * Replay a job as a new run. `status="failed"` re-runs only the failures - * (append `,pending` to also include tasks that never started). Already- - * successful tasks carry over, so you only re-scrape what failed. + * Start a new run of a job. With no `status`, every task runs again (a full + * rerun). `status` is a comma-separated list of task statuses to rerun + * (`pending,processing,failed,successful`); the rest carry over from the + * previous run. After a stop or a credit failure, unstarted tasks stay + * `pending` and tasks cut off mid-flight can stay `processing`, so + * `failed,pending,processing` recovers everything that did not succeed. */ -export function rerunJob(id: string, opts: CallOpts & { status?: string }): Promise { - return batchRequest("POST", `/jobs/${encodeURIComponent(id)}/rerun`, { - apiKey: opts.apiKey, +export function rerunJob(id: string, opts: CallOpts & { status?: string; idempotencyKey?: string }): Promise { + const idempotencyKey = opts.idempotencyKey ?? newIdempotencyKey(); + assertIdempotencyKey(idempotencyKey); + return batchJson("POST", jobPath(id, "/rerun"), { + ...opts, query: opts.status ? { status: opts.status } : undefined, - fetchImpl: opts.fetchImpl, - timeoutMs: opts.timeoutMs, + idempotencyKey, + retry: "write", + jobId: id, }); } -/** List all result rows for a run, paginating over the `cursor` until `next_cursor` is null. */ +/** + * List all result rows of the latest run (or `runId`), following `next_cursor` + * until it is null. + */ export async function listResults( id: string, - opts: CallOpts & { status?: "successful" | "failed" | "all" }, + opts: CallOpts & { status?: ResultStatus; runId?: string }, ): Promise { + const path = opts.runId ? jobPath(id, `/runs/${encodeURIComponent(opts.runId)}/results`) : jobPath(id, "/results"); const all: ResultRow[] = []; let cursor: string | undefined; do { - const page = await batchRequest("GET", `/jobs/${encodeURIComponent(id)}/results`, { - apiKey: opts.apiKey, + const page = await batchJson("GET", path, { + ...opts, + retry: "read", + jobId: id, query: { status: opts.status, cursor }, - fetchImpl: opts.fetchImpl, - timeoutMs: opts.timeoutMs, }); if (page?.results) all.push(...page.results); cursor = page?.next_cursor ?? undefined; @@ -303,6 +663,18 @@ export async function listResults( return all; } +/** One task's result body (latest run, or `runId`), as returned by the content endpoint. */ +export async function getTaskContent( + id: string, + taskId: string, + opts: CallOpts & { runId?: string }, +): Promise<{ contentType: string; body: Buffer }> { + const task = `/tasks/${encodeURIComponent(taskId)}/content`; + const path = opts.runId ? jobPath(id, `/runs/${encodeURIComponent(opts.runId)}${task}`) : jobPath(id, task); + const res = await batchFetch("GET", path, { ...opts, retry: "read", jobId: id }); + return { contentType: res.headers.get("content-type") ?? "", body: res.body }; +} + /** Map a result `type` to a file extension for downloaded bodies. */ const EXT_BY_TYPE: Record = { json: "json", @@ -340,15 +712,23 @@ export interface DownloadOutcome { skipped: Array<{ task_id: string; external_id?: string; reason: string }>; } +/** Why a row has no body to download. */ +function skipReason(row: ResultRow): string { + if (row.status === "failed") return "task failed: no result body"; + if (row.status === "expired") return "result expired (kept 14 days by default): a full rerun scrapes it again"; + if (row.status === "pending" || row.status === "processing") return "task not finished yet"; + return "no result_url"; +} + /** * Stream each result body into its own file under `dir` (created if needed), - * plus a self-describing `_manifest.jsonl` index (result_url dropped — it is a + * plus a self-describing `_manifest.jsonl` index (result_url dropped: it is a * short-lived presigned link). Fetches run `concurrency` at a time and a single * failure never aborts the rest: per-task outcomes come back in the result. * - * Only tasks with a `result_url` have a body to fetch; failed tasks carry an - * `error` instead and are reported as `skipped`. Because the presigned links - * expire, download soon after `listResults`. Deliberately opt-in (not the + * Only tasks with a `result_url` have a body to fetch; failed, expired and + * unfinished tasks are reported as `skipped`. The presigned links are valid for + * 2 hours, so download soon after `listResults`. Deliberately opt-in (not the * `results` default): a large job's bodies can be many GB, and the URL manifest * is the right primitive to pull selectively. */ @@ -375,10 +755,7 @@ export async function downloadResults( const { row, file } = planned[next++]!; const ref = { task_id: row.task_id, external_id: row.external_id }; if (!file) { - outcome.skipped.push({ - ...ref, - reason: row.status === "failed" ? "task failed — no result body" : "no result_url", - }); + outcome.skipped.push({ ...ref, reason: skipReason(row) }); continue; } try { @@ -399,7 +776,7 @@ export async function downloadResults( const fileByTask = new Map(outcome.downloaded.map((d) => [d.task_id, d.file])); const manifest = rows.map((r) => { const rest = { ...r }; - delete rest.result_url; // expires — the local `file` is the durable pointer + delete rest.result_url; // expires: the local `file` is the durable pointer return { ...rest, file: fileByTask.get(r.task_id) ?? null }; }); writeFileSync( @@ -410,28 +787,31 @@ export async function downloadResults( } /** - * Poll `GET /jobs/{id}` until `latest_run.status` is terminal, backing off - * 2s → ×1.5 → capped at 15s. Throws BATCH_FAILED on timeout. `sleepImpl` is - * injectable for tests. + * Poll `GET /jobs/{id}` until the latest run is terminal, the run is paused (a + * paused run never progresses, so the wait returns at once), or `waitMs` + * (default 10 minutes) runs out, backing off 2s, x1.5, capped at 15s. Returns + * the last job read in every case; the caller tells them apart with + * `isTerminal` / `isPaused`. A transient error (connection, 429, 5xx) is + * ridden out until the deadline. `sleepImpl` is injectable for tests. */ -export async function waitForJob( - id: string, - opts: CallOpts & { sleepImpl?: (ms: number) => Promise }, -): Promise { - const sleep = opts.sleepImpl ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); - const deadline = Date.now() + (opts.timeoutMs ?? 600_000); +export async function waitForJob(id: string, opts: CallOpts & { waitMs?: number }): Promise { + const sleep = opts.sleepImpl ?? defaultSleep; + const deadline = Date.now() + (opts.waitMs ?? 600_000); let delay = 2000; + let last: Job | undefined; + let lastErr: unknown; for (;;) { - const job = await getJob(id, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl }); - if (job.latest_run && TERMINAL_STATUSES.has(job.latest_run.status)) return job; - if (Date.now() > deadline) { - throw new ToolkitError({ - code: "BATCH_FAILED", - message: `Timed out waiting for batch job ${id} to finish.`, - likely_cause: `The run did not reach a terminal state within ${Math.round((opts.timeoutMs ?? 600_000) / 1000)}s.`, - next_action: "Re-check progress with `zenrows batch status `, or raise --timeout.", - suggested_commands: [`zenrows batch status ${id}`], - }); + try { + last = await getJob(id, opts); + if (isTerminal(last.latest_run) || isPaused(last.latest_run)) return last; + } catch (err) { + if (!isTransient(err)) throw err; + lastErr = err; + } + if (Date.now() + delay > deadline) { + // Out of time: report the last state read, or the error if none was. + if (!last) throw lastErr; + return last; } await sleep(delay); delay = Math.min(delay * 1.5, 15_000); diff --git a/src/core/errors.ts b/src/core/errors.ts index 1c4da84..405fdfc 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -35,6 +35,15 @@ export type ErrorCode = | "BATCH_QUOTA_EXCEEDED" | "BATCH_NOT_FOUND" | "BATCH_FAILED" + | "BATCH_RUN_IN_PROGRESS" + | "BATCH_RUN_ENDED" + | "BATCH_NO_MATCHING_TASKS" + | "BATCH_RERUN_WINDOW_EXPIRED" + | "BATCH_REQUEST_IN_FLIGHT" + | "BATCH_CONFLICT" + | "BATCH_TASK_NOT_READY" + | "BATCH_TASK_FAILED" + | "BATCH_RESULT_EXPIRED" | "CRAWL_NOT_ENABLED" | "CRAWL_INSUFFICIENT_CREDITS" | "CRAWL_KEY_CAP_REACHED" @@ -63,6 +72,13 @@ export interface ToolkitErrorShape { server_code?: string; /** The crawl the error is about, once it exists. */ crawl_id?: string; + /** The Batch job the error is about, once it exists. */ + job_id?: string; + /** + * The Idempotency-Key a Batch write was sent with, when its outcome is + * unknown. Repeating the same request with this key cannot create a duplicate. + */ + idempotency_key?: string; /** Seconds to wait before a retry (the `Retry-After` header). */ retry_after?: number; } @@ -74,6 +90,8 @@ export class ToolkitError extends Error { readonly suggested_commands: string[]; readonly server_code?: string; readonly crawl_id?: string; + readonly job_id?: string; + readonly idempotency_key?: string; readonly retry_after?: number; constructor(shape: ToolkitErrorShape) { @@ -85,6 +103,8 @@ export class ToolkitError extends Error { this.suggested_commands = shape.suggested_commands ?? []; this.server_code = shape.server_code; this.crawl_id = shape.crawl_id; + this.job_id = shape.job_id; + this.idempotency_key = shape.idempotency_key; this.retry_after = shape.retry_after; } @@ -97,6 +117,8 @@ export class ToolkitError extends Error { suggested_commands: this.suggested_commands, ...(this.server_code ? { server_code: this.server_code } : {}), ...(this.crawl_id ? { crawl_id: this.crawl_id } : {}), + ...(this.job_id ? { job_id: this.job_id } : {}), + ...(this.idempotency_key ? { idempotency_key: this.idempotency_key } : {}), ...(this.retry_after !== undefined ? { retry_after: this.retry_after } : {}), }; } diff --git a/src/core/policy.ts b/src/core/policy.ts index fd6badd..0f1c92e 100644 --- a/src/core/policy.ts +++ b/src/core/policy.ts @@ -5,10 +5,17 @@ import type { Policy } from "../types/index.ts"; import { ToolkitError } from "./errors.ts"; import { findWorkspace, readJson, workspacePaths, writeJson } from "./workspace.ts"; +/** + * Safe defaults. The per-run caps let a default run submit up to 10,000 URLs + * in any mode: an Adaptive Stealth Mode task is estimated at its 25-credit + * upper bound, so 10,000 of them estimate at 250,000 credits. Larger runs + * (the Batch API takes up to 100,000 tasks per job) need the caps raised + * explicitly with `zenrows policy set`. + */ export function defaultPolicy(): Policy { return { - max_credits_per_run: 10000, - max_pages_per_run: 1000, + max_credits_per_run: 250_000, + max_pages_per_run: 10_000, max_concurrency: 20, allow_browser: true, allow_experimental: false, @@ -111,8 +118,8 @@ export function assertWithinLimits( code: "POLICY_LIMIT_EXCEEDED", message: `This ${context} submits ${requested.pages} page(s), over the policy cap of ${policy.max_pages_per_run}.`, likely_cause: "policy.max_pages_per_run bounds how many URLs a single run may submit.", - next_action: `Split the job into runs of ≤${policy.max_pages_per_run} page(s), or raise max_pages_per_run in policy.json.`, - suggested_commands: ["zenrows policy show"], + next_action: `Split the job into runs of at most ${policy.max_pages_per_run} page(s), or raise the cap: \`zenrows policy set max_pages_per_run ${requested.pages}\` (default ${defaultPolicy().max_pages_per_run}).`, + suggested_commands: [`zenrows policy set max_pages_per_run ${requested.pages}`, "zenrows policy show"], }); } if (requested.credits !== undefined && requested.credits > policy.max_credits_per_run) { @@ -120,8 +127,8 @@ export function assertWithinLimits( code: "POLICY_LIMIT_EXCEEDED", message: `This ${context} is estimated at ${requested.credits} credit(s), over the policy cap of ${policy.max_credits_per_run}.`, likely_cause: "policy.max_credits_per_run bounds the estimated credit cost of a single run.", - next_action: "Reduce the job size or per-task cost (e.g. drop premium_proxy/js_render), or raise max_credits_per_run in policy.json.", - suggested_commands: ["zenrows policy show"], + next_action: `Reduce the job size or per-task cost (e.g. drop premium_proxy/js_render), or raise the cap: \`zenrows policy set max_credits_per_run ${requested.credits}\` (default ${defaultPolicy().max_credits_per_run}). The estimate is an upper bound: an Adaptive Stealth Mode task counts 25 credits, and is billed 1 to 25.`, + suggested_commands: [`zenrows policy set max_credits_per_run ${requested.credits}`, "zenrows policy show"], }); } } diff --git a/templates/batch-jsonl-pipeline/README.md b/templates/batch-jsonl-pipeline/README.md index cc98586..6b81a86 100644 --- a/templates/batch-jsonl-pipeline/README.md +++ b/templates/batch-jsonl-pipeline/README.md @@ -3,8 +3,9 @@ Scaffold for a high-scale workload expressed as a JSONL job spec, run on the Zenrows **Batch**. One JSON object per line, each with a `url` (plus optional per-line overrides like `js_render`, `premium_proxy`, `proxy_country`, -`mode`, `autoparse`, `extract`, and an `external_id` echoed back on each -result). +`mode`, `autoparse`, `extract`, an `external_id` echoed back on each result, +string `metadata`, and `"method": "POST"` with a `body`). `batch estimate` +checks every line against the API's limits. Set `"extract": "auto"` on a line to run that URL through Extract — structured data instead of raw HTML, at the same credit cost as a regular task. Extract @@ -18,17 +19,21 @@ zenrows batch estimate jobs.example.jsonl # validate the spec + estimate cred ## Cloud (needs a key) -The cloud subcommands run with any active API key and spend the same credits +The cloud subcommands run with any active API key and cost the same credits as Fetch. ```bash -zenrows batch create jobs.example.jsonl --wait # submit + wait for the run to finish +zenrows batch create jobs.example.jsonl --follow # submit + wait for the run to finish zenrows batch status # run status + progress stats zenrows batch results --out results.jsonl # collect results (paginated) -zenrows batch retry-failed # rerun only the failed tasks +zenrows batch retry-failed --include-pending # rerun what did not succeed zenrows batch cancel # stop an in-flight run ``` +The local policy caps a run at 10,000 URLs and 250,000 estimated credits by +default; raise them with `zenrows policy set max_pages_per_run ` and +`zenrows policy set max_credits_per_run `. + Tasks use Adaptive Stealth Mode (`mode=auto`) by default. Job-level scrape options come from flags (`--js-render`, `--premium-proxy`, `--proxy-country`, `--output`, or `--manual` to turn Adaptive Stealth Mode off); per-line keys diff --git a/tests/batch-api.test.ts b/tests/batch-api.test.ts index 39977d5..9a29c03 100644 --- a/tests/batch-api.test.ts +++ b/tests/batch-api.test.ts @@ -62,9 +62,9 @@ test("getJob parses latest_run stats", async () => { latest_run: { status: "completed", stats: { total: 10, completed: 10, successful: 8, failed: 2 } }, }); const job = await getJob("job_1", { apiKey: "k", fetchImpl: impl }); - assert.equal(job.latest_run.status, "completed"); - assert.equal(job.latest_run.stats.successful, 8); - assert.equal(job.latest_run.stats.failed, 2); + assert.equal(job.latest_run?.status, "completed"); + assert.equal(job.latest_run?.stats.successful, 8); + assert.equal(job.latest_run?.stats.failed, 2); assert.match(calls[0]!.url, /\/jobs\/job_1$/); }); @@ -90,13 +90,17 @@ test("listResults paginates across >=2 cursor pages until next_cursor is null", assert.deepEqual(seenCursors, [null, "c1", "c2"]); }); -test("stopJob posts /jobs/{id}/stop", async () => { +test("stopJob posts /jobs/{id}/stop and reads the Run it answers with", async () => { + // The API answers /stop with the refreshed Run, not the Job. const { impl, calls } = jsonFetch(200, { + run_id: "run_1", job_id: "job_1", - latest_run: { status: "stopped", stats: { total: 5, completed: 3, successful: 3, failed: 0 } }, + status: "stopped", + stats: { total: 5, completed: 3, successful: 3, failed: 0 }, }); - const job = await stopJob("job_1", { apiKey: "k", fetchImpl: impl }); - assert.equal(job.latest_run.status, "stopped"); + const run = await stopJob("job_1", { apiKey: "k", fetchImpl: impl }); + assert.equal(run.status, "stopped"); + assert.equal(run.stats.completed, 3); assert.equal(calls[0]!.init?.method, "POST"); assert.match(calls[0]!.url, /\/jobs\/job_1\/stop$/); }); @@ -251,7 +255,7 @@ test("waitForJob polls until a terminal status", async () => { }) as unknown as typeof fetch; const job = await waitForJob("j", { apiKey: "k", fetchImpl: impl, sleepImpl: async () => {} }); - assert.equal(job.latest_run.status, "completed"); + assert.equal(job.latest_run?.status, "completed"); assert.equal(i, 3); }); @@ -267,6 +271,6 @@ test("waitForJob treats failed as terminal", async () => { }) as unknown as typeof fetch; const job = await waitForJob("j", { apiKey: "k", fetchImpl: impl, sleepImpl: async () => {} }); - assert.equal(job.latest_run.status, "failed"); + assert.equal(job.latest_run?.status, "failed"); assert.equal(i, 2); }); diff --git a/tests/batch-consistency.test.ts b/tests/batch-consistency.test.ts new file mode 100644 index 0000000..0b3a835 --- /dev/null +++ b/tests/batch-consistency.test.ts @@ -0,0 +1,650 @@ +/** + * `zenrows batch` against the Batch API's real wire shapes: the run that stop, + * pause and resume answer with, failed runs per failure_reason, reruns by task + * status, lifecycle 409s, Idempotency-Key and retries, POST tasks, the API's + * metadata and external_id limits, durations, paused runs, and the new list, + * content and delete commands. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { batch, runFailure, timeoutFlag } from "../src/cli/commands/batch.ts"; +import { estimateCredits, toJobBody, validateJsonl, withAdaptiveStealth } from "../src/adapters/batch.ts"; +import { waitForJob } from "../src/core/batch-api.ts"; +import { createWorkspace } from "../src/core/workspace.ts"; +import { savePolicy, defaultPolicy } from "../src/core/policy.ts"; +import { saveApiKey } from "../src/core/auth.ts"; +import { tempRoot } from "./helpers.ts"; + +type Handler = (url: URL, init?: RequestInit) => Response | Promise; +type Call = { url: URL; init?: RequestInit }; + +const stats = { total: 5, completed: 3, successful: 3, failed: 0 }; +const run = (o: Record = {}) => ({ + run_id: "r1", + job_id: "j1", + run_sequence: 1, + status: "running", + trigger: "manual", + stats, + created_at: "2026-10-16T09:00:00Z", + ...o, +}); +const job = (r: Record = {}, o: Record = {}) => ({ + job_id: "j1", + type: "regular", + status: "closed", + latest_run: run(r), + ...o, +}); + +function json(body: unknown, status = 200, headers: Record = {}): Response { + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json", ...headers } }); +} +function problem(status: number, code: string, detail: string): Response { + return new Response(JSON.stringify({ type: `https://docs.zenrows.com/api-error-codes#${code}`, title: code, status, code, detail }), { + status, + headers: { "content-type": "application/problem+json" }, + }); +} + +/** Run `fn` in a fresh workspace with a saved key; every network call goes to `handler` and is recorded. */ +async function withWorkspace( + handler: Handler, + fn: (calls: Call[], root: string) => Promise, + policy: Partial> = {}, +): Promise { + const { root, cleanup } = tempRoot(); + const cwd = process.cwd(); + createWorkspace(root); + savePolicy({ ...defaultPolicy(), ...policy }, root); + saveApiKey("0".repeat(41), root); + process.chdir(root); + const calls: Call[] = []; + const orig = globalThis.fetch; + globalThis.fetch = (async (u: string, init?: RequestInit) => { + const url = new URL(u); + calls.push({ url, init }); + return handler(url, init); + }) as unknown as typeof fetch; + try { + await fn(calls, root); + } finally { + globalThis.fetch = orig; + process.chdir(cwd); + cleanup(); + } +} + +async function captureOut(fn: () => unknown): Promise { + const orig = process.stdout.write.bind(process.stdout); + let buf = ""; + process.stdout.write = ((s: string | Uint8Array, ...rest: unknown[]) => { + if (typeof s !== "string") return (orig as (...a: unknown[]) => boolean)(s, ...rest); + buf += s; + return true; + }) as typeof process.stdout.write; + try { + await fn(); + } finally { + process.stdout.write = orig; + } + return buf; +} + +async function run_(args: string[], yes = false): Promise<{ code: number; out: Record }> { + let code = -1; + const raw = await captureOut(async () => { + code = await batch.run(args, { json: true, yes }); + }); + return { code, out: JSON.parse(raw) }; +} + +function spec(root: string, lines: unknown[]): string { + const file = join(root, "jobs.jsonl"); + writeFileSync(file, lines.map((l) => JSON.stringify(l)).join("\n") + "\n"); + return file; +} + +const sentBody = (c: Call) => JSON.parse(String(c.init?.body)); +const header = (c: Call, name: string) => (c.init?.headers as Record)[name]; + +// ---------- cancel / pause / resume read the Run ---------- + +test("cancel reads the stopped Run: status and stats, not 'unknown'", async () => { + await withWorkspace( + () => json(run({ status: "stopped" })), + async () => { + const { code, out } = await run_(["cancel", "j1"]); + assert.equal(code, 0); + assert.equal(out.ok, true); + assert.equal(out.status, "stopped"); + assert.equal(out.job_id, "j1"); + assert.equal(out.jobId, "j1"); + assert.equal(out.run_id, "r1"); + assert.deepEqual(out.stats, stats); + }, + ); +}); + +test("cancel on an ended run says nothing was stopped and prints the job as it ended", async () => { + await withWorkspace( + (url) => + url.pathname.endsWith("/stop") + ? problem(409, "run_not_stoppable", "Run is not in a stoppable state.") + : json(job({ status: "completed", stats: { total: 5, completed: 5, successful: 5, failed: 0 } })), + async () => { + const { code, out } = await run_(["cancel", "j1"]); + assert.equal(code, 0); + assert.equal(out.status, "completed"); + }, + ); +}); + +test("pause and resume read the Run they answer with; a paused run says so", async () => { + await withWorkspace( + (url) => json(run(url.pathname.endsWith("/pause") ? { pause_state: "paused" } : {})), + async (calls) => { + const paused = await run_(["pause", "j1"]); + assert.equal(paused.code, 0); + assert.equal(paused.out.paused, true); + assert.match(paused.out.note, /zenrows batch resume j1/); + const resumed = await run_(["resume", "j1"]); + assert.equal(resumed.code, 0); + assert.equal(resumed.out.paused, undefined); + assert.deepEqual( + calls.map((c) => `${c.init?.method} ${c.url.pathname}`), + ["POST /v1/jobs/j1/pause", "POST /v1/jobs/j1/resume"], + ); + }, + ); +}); + +test("pause on an ended run is BATCH_RUN_ENDED with server_code and job_id", async () => { + await withWorkspace( + () => problem(409, "run_not_stoppable", "Latest run is terminal; pause/resume only apply to in-flight runs."), + async () => { + const { code, out } = await run_(["pause", "j1"]); + assert.equal(code, 1); + assert.equal(out.error.code, "BATCH_RUN_ENDED"); + assert.equal(out.error.server_code, "run_not_stoppable"); + assert.equal(out.error.job_id, "j1"); + }, + ); +}); + +// ---------- failed runs map like the matching HTTP refusal ---------- + +test("a run that ran out of credits is POLICY_MAX_CREDITS_EXCEEDED, exit 1, with server_code, job_id and the rerun", async () => { + await withWorkspace( + () => json(job({ status: "failed", failure_reason: "insufficient_credits", failure_detail: "Subscription has no credit available." })), + async () => { + const { code, out } = await run_(["status", "j1"]); + assert.equal(code, 1); + assert.equal(out.ok, false); + assert.equal(out.error.code, "POLICY_MAX_CREDITS_EXCEEDED"); + assert.equal(out.error.server_code, "insufficient_credits"); + assert.equal(out.error.job_id, "j1"); + assert.match(out.error.likely_cause, /Stopped batch job j1/); + assert.doesNotMatch(out.error.likely_cause, /HTTP/); + assert.ok(out.error.suggested_commands.includes("zenrows batch rerun j1 --status failed,pending,processing")); + assert.ok(out.error.suggested_commands.includes("zenrows usage")); + }, + ); +}); + +test("a run whose subscription lapsed is AUTH_INVALID with Billing advice", () => { + const err = runFailure(job({ status: "failed", failure_reason: "subscription_inactive", failure_detail: "Subscription has expired." }) as never)!; + assert.equal(err.code, "AUTH_INVALID"); + assert.match(err.next_action, /app\.zenrows\.com\/billing/); + assert.equal(err.server_code, "subscription_inactive"); + assert.equal(err.job_id, "j1"); +}); + +test("a run the key cap stopped is KEY_CREDIT_CAP_REACHED with server_code", () => { + const err = runFailure(job({ status: "failed", failure_reason: "api_key_cap_reached" }) as never)!; + assert.equal(err.code, "KEY_CREDIT_CAP_REACHED"); + assert.equal(err.server_code, "api_key_cap_reached"); + assert.equal(err.job_id, "j1"); +}); + +test("any other failure stays BATCH_FAILED, without 'fix the tasks' advice", () => { + const err = runFailure(job({ status: "failed", failure_reason: "something_new" }) as never)!; + assert.equal(err.code, "BATCH_FAILED"); + assert.match(err.message, /stopped before it finished/); + assert.equal(err.server_code, "something_new"); + assert.equal(runFailure(job({ status: "stopped" }) as never), null); +}); + +// ---------- rerun ---------- + +test("rerun with no --status is a full rerun; output keeps rerun_of, retried_tasks, inherited_tasks", async () => { + const answer = { job_id: "j1", status: "closed", latest_run: run({ run_id: "r2", ingest_status: "pending" }), rerun_of: "r1", retried_tasks: 5, inherited_tasks: 0 }; + await withWorkspace( + () => json(answer, 202), + async (calls) => { + const { code, out } = await run_(["rerun", "j1"]); + assert.equal(code, 0); + assert.equal(calls[0]!.url.searchParams.get("status"), null); + assert.ok(header(calls[0]!, "Idempotency-Key")); + assert.equal(out.run_id, "r2"); + assert.equal(out.rerun_of, "r1"); + assert.equal(out.retried_tasks, 5); + assert.equal(out.inherited_tasks, 0); + assert.equal(out.ingest_status, "pending"); + }, + ); +}); + +test("rerun --status takes pending,processing,failed,successful and rejects anything else before any request", async () => { + await withWorkspace( + () => json(job()), + async (calls) => { + const ok = await run_(["rerun", "j1", "--status", "Failed, pending,processing,failed"]); + assert.equal(ok.code, 0); + assert.equal(calls[0]!.url.searchParams.get("status"), "failed,pending,processing"); + const all = await run_(["rerun", "j1", "--status", "successful"]); + assert.equal(all.code, 0); + const bad = await run_(["rerun", "j1", "--status", "expired"]); + assert.equal(bad.code, 1); + assert.equal(bad.out.error.code, "INVALID_USAGE"); + assert.equal(calls.length, 2); + }, + ); +}); + +test("retry-failed sends status=failed, and --include-pending recovers unfinished tasks", async () => { + await withWorkspace( + () => json(job()), + async (calls) => { + await run_(["retry-failed", "j1"]); + await run_(["retry-failed", "j1", "--include-pending"]); + assert.equal(calls[0]!.url.searchParams.get("status"), "failed"); + assert.equal(calls[1]!.url.searchParams.get("status"), "failed,pending,processing"); + }, + ); +}); + +test("each lifecycle 409 gets its own code and advice, carrying server_code and job_id", async () => { + const cases: Array<[string, string, string, RegExp]> = [ + ["no_matching_tasks", "No tasks match the requested statuses.", "BATCH_NO_MATCHING_TASKS", /--status failed,pending,processing/], + ["partial_rerun_window_expired", "Partial rerun window expired.", "BATCH_RERUN_WINDOW_EXPIRED", /zenrows batch rerun j1`/], + ["run_not_terminal", "Previous run is still in flight; call /stop before /rerun.", "BATCH_RUN_IN_PROGRESS", /batch wait j1/], + ["idempotency_key_conflict", "Idempotency key conflict.", "BATCH_REQUEST_IN_FLIGHT", /same key/], + ["run_has_descendants", "Run has descendants.", "BATCH_CONFLICT", /batch status j1/], + ]; + for (const [server, detail, code, advice] of cases) { + await withWorkspace( + () => problem(409, server, detail), + async () => { + const { out } = await run_(["rerun", "j1", "--status", "failed"]); + assert.equal(out.error.code, code, server); + assert.equal(out.error.server_code, server); + assert.equal(out.error.job_id, "j1"); + assert.match(out.error.next_action, advice); + assert.doesNotMatch(out.error.likely_cause, /\.\./, "no doubled period"); + }, + ); + } +}); + +test("402 'Subscription has expired.' uses subscription wording, not out of credits", async () => { + await withWorkspace( + () => problem(402, "payment_required", "Subscription has expired."), + async () => { + const { out } = await run_(["rerun", "j1"]); + assert.equal(out.error.code, "AUTH_INVALID"); + assert.equal(out.error.server_code, "payment_required"); + assert.match(out.error.next_action, /Billing/); + }, + ); +}); + +test("429 names the slot rules and real commands, not a placeholder status", async () => { + await withWorkspace( + () => problem(429, "quota_exceeded", "Account already has 3 active jobs; max 3 allowed concurrently. Wait for one to finish or DELETE one before submitting a new one."), + async (_calls, root) => { + const file = spec(root, [{ url: "https://example.com/a" }]); + const { out } = await run_(["create", file]); + assert.equal(out.error.code, "BATCH_QUOTA_EXCEEDED"); + assert.deepEqual(out.error.suggested_commands, ["zenrows batch list", "zenrows crawl list"]); + assert.match(out.error.next_action, /shared with Crawl/); + }, + ); +}); + +// ---------- create: idempotency, retries, output ---------- + +test("create sends an Idempotency-Key and reuses it when a 503 is retried", async () => { + let n = 0; + await withWorkspace( + () => (n++ === 0 ? problem(503, "internal", "boom") : json({ job_id: "j1", status: "closed", latest_run: run(), accepted_tasks: 2 }, 201)), + async (calls, root) => { + const file = spec(root, [{ url: "https://example.com/a" }, { url: "https://example.com/b" }]); + const { code, out } = await run_(["create", file]); + assert.equal(code, 0); + assert.equal(calls.length, 2); + const k1 = header(calls[0]!, "Idempotency-Key"); + assert.ok(k1); + assert.equal(header(calls[1]!, "Idempotency-Key"), k1); + assert.equal(out.accepted_tasks, 2); + assert.equal(out.run_id, "r1"); + assert.equal(out.job.status, "closed"); + }, + ); +}); + +test("create whose outcome is unknown names the key and the exact command that repeats it safely", async () => { + await withWorkspace( + () => problem(503, "internal", "boom"), + async (calls, root) => { + const file = spec(root, [{ url: "https://example.com/a" }]); + const { code, out } = await run_(["create", file, "--idempotency-key", "abc-123"]); + assert.equal(code, 1); + assert.equal(calls.length, 4, "the first try plus 3 retries"); + assert.equal(out.error.idempotency_key, "abc-123"); + assert.equal(out.error.suggested_commands[0], `zenrows batch create ${file} --idempotency-key abc-123`); + }, + ); +}); + +test("create keeps accepted_tasks and ingest_status of a 202 submit", async () => { + await withWorkspace( + () => json({ job_id: "j1", status: "closed", latest_run: run({ ingest_status: "pending" }), accepted_tasks: 20000 }, 202), + async (_calls, root) => { + const file = spec(root, [{ url: "https://example.com/a" }]); + const { out } = await run_(["create", file]); + assert.equal(out.accepted_tasks, 20000); + assert.equal(out.ingest_status, "pending"); + }, + ); +}); + +test("create sends job-level --name, --external-id and --metadata, and rejects invalid ones locally", async () => { + await withWorkspace( + () => json(job()), + async (calls, root) => { + const file = spec(root, [{ url: "https://example.com/a" }]); + const ok = await run_(["create", file, "--name", "Nightly", "--external-id", "n-1", "--metadata", "team=growth", "--metadata", "src=cli"]); + assert.equal(ok.code, 0); + const body = sentBody(calls[0]!); + assert.equal(body.name, "Nightly"); + assert.equal(body.external_id, "n-1"); + assert.deepEqual(body.metadata, { team: "growth", src: "cli" }); + const bad = await run_(["create", file, "--external-id", "has space"]); + assert.equal(bad.code, 1); + const badMeta = await run_(["create", file, "--metadata", "novalue"]); + assert.equal(badMeta.code, 1); + assert.equal(calls.length, 1); + }, + ); +}); + +test("create --follow honours --timeout and exits 0 with a note when the wait runs out", async () => { + await withWorkspace( + (url, init) => (init?.method === "POST" ? json({ job_id: "j1", latest_run: run(), accepted_tasks: 1 }, 201) : json(job())), + async (_calls, root) => { + const file = spec(root, [{ url: "https://example.com/a" }]); + const { code, out } = await run_(["create", file, "--follow", "--timeout", "1ms"]); + assert.equal(code, 0); + assert.equal(out.status, "running"); + assert.equal(out.accepted_tasks, 1); + assert.match(out.note, /zenrows batch wait j1/); + }, + ); +}); + +// ---------- the default policy and the estimate ---------- + +test("the default policy lets an Adaptive Stealth Mode batch of 1,000 URLs through", async () => { + await withWorkspace( + () => json(job()), + async (calls, root) => { + const file = spec(root, Array.from({ length: 1000 }, (_, i) => ({ url: `https://example.com/${i}` }))); + const { code } = await run_(["create", file]); + assert.equal(code, 0); + assert.equal(calls.length, 1); + }, + ); + assert.equal(defaultPolicy().max_pages_per_run, 10_000); + assert.equal(defaultPolicy().max_credits_per_run, 250_000); +}); + +test("a policy cap error prints the exact command to raise it", async () => { + await withWorkspace( + () => json(job()), + async (_calls, root) => { + const file = spec(root, [{ url: "https://example.com/a" }, { url: "https://example.com/b" }]); + const { out } = await run_(["create", file], false); + assert.equal(out.ok, true); + const capped = await (async () => { + savePolicy({ ...defaultPolicy(), max_credits_per_run: 10 }, root); + return run_(["create", file]); + })(); + assert.equal(capped.out.error.code, "POLICY_LIMIT_EXCEEDED"); + assert.equal(capped.out.error.suggested_commands[0], "zenrows policy set max_credits_per_run 50"); + }, + ); +}); + +test("estimate prices job-level flags, nested zenrows_params and \"false\" the way create submits them", () => { + const jobs = [ + { url: "https://a.example/" }, + { url: "https://b.example/", zenrows_params: { js_render: "true" } }, + { url: "https://c.example/", js_render: "false" }, + ]; + assert.deepEqual(estimateCredits(jobs).perJob, [1, 5, 1]); + assert.deepEqual(estimateCredits(jobs, { premium_proxy: true }).perJob, [10, 25, 10]); + const auto = estimateCredits(withAdaptiveStealth([{ url: "https://a.example/" }])); + assert.equal(auto.credits, 25); + assert.equal(auto.min, 1); +}); + +test("batch estimate takes create's flags and reports the min and the upper bound", async () => { + const dir = mkdtempSync(join(tmpdir(), "zr-est-")); + const file = spec(dir, [{ url: "https://a.example/" }, { url: "https://b.example/" }]); + const { code, out } = await run_(["estimate", file, "--premium-proxy"]); + assert.equal(code, 0); + assert.equal(out.estimated_credits, 20); + assert.equal(out.estimatedCredits, 20); + assert.equal(out.estimated_credits_min, 20); +}); + +// ---------- POST tasks, metadata and external_id ---------- + +test("method and body are task fields, not zenrows_params; a POST task gets no Adaptive Stealth Mode", () => { + const jobs = withAdaptiveStealth([ + { url: "https://api.example.com/graphql", method: "POST", body: { query: "{a}" }, external_id: "g1" }, + { url: "https://example.com/b" }, + ]); + const body = toJobBody(jobs); + assert.deepEqual(body.tasks[0], { url: "https://api.example.com/graphql", external_id: "g1", method: "POST", body: { query: "{a}" } }); + assert.deepEqual(body.tasks[1]!.zenrows_params, { mode: "auto" }); + assert.throws(() => toJobBody([{ url: "https://x.example/", method: "post" }], { js_render: true }), /POST task cannot use js_render/); +}); + +test("validateJsonl enforces the API's metadata, external_id, url and method/body limits", () => { + const dir = mkdtempSync(join(tmpdir(), "zr-val-")); + const file = spec(dir, [ + { url: "https://ok.example/", metadata: { a: "1" }, external_id: "ok-1" }, + { url: "https://b.example/", metadata: { price: 3 } }, + { url: "https://c.example/", external_id: "has space" }, + { url: "https://d.example/", external_id: "x".repeat(129) }, + { url: "https://e.example/", metadata: { "bad key": "v" } }, + { url: "https://f.example/", metadata: Object.fromEntries(Array.from({ length: 21 }, (_, i) => [`k${i}`, "v"])) }, + { url: "ftp://g.example/" }, + { url: "https://h.example/", body: "x=1" }, + { url: "https://i.example/", method: "PUT" }, + { url: "https://j.example/", method: "POST", body: "x".repeat(16 * 1024 + 1) }, + ]); + const v = validateJsonl(file); + assert.equal(v.validJobs, 1); + assert.deepEqual( + v.errors.map((e) => e.line), + [2, 3, 4, 5, 6, 7, 8, 9, 10], + ); +}); + +// ---------- durations, paused runs, waits ---------- + +test("--timeout takes 10s, 5m, 1h and 500ms; a bare number stays milliseconds; junk is rejected", () => { + assert.equal(timeoutFlag("10s"), 10_000); + assert.equal(timeoutFlag("5m"), 300_000); + assert.equal(timeoutFlag("1h"), 3_600_000); + assert.equal(timeoutFlag("500ms"), 500); + assert.equal(timeoutFlag("600000"), 600_000); + assert.equal(timeoutFlag(undefined), undefined); + assert.throws(() => timeoutFlag("10x"), /Invalid --timeout/); + assert.throws(() => timeoutFlag("0"), /Invalid --timeout/); +}); + +test("wait returns at once on a paused run, exit 0, with paused and a resume note", async () => { + await withWorkspace( + () => json(job({ pause_state: "paused" })), + async (calls) => { + const { code, out } = await run_(["wait", "j1", "--timeout", "10m"]); + assert.equal(code, 0); + assert.equal(calls.length, 1); + assert.equal(out.paused, true); + assert.match(out.note, /paused/); + }, + ); +}); + +test("wait that runs out exits 0 with the run's state and stats, like crawl wait", async () => { + await withWorkspace( + () => json(job()), + async () => { + const { code, out } = await run_(["wait", "j1", "--timeout", "1"]); + assert.equal(code, 0); + assert.equal(out.status, "running"); + assert.deepEqual(out.stats, stats); + assert.match(out.note, /wait ran out/); + }, + ); +}); + +test("waitForJob rides out transient errors that outlast the per-request retries", async () => { + let n = 0; + const impl = (async () => { + n++; + if (n >= 2 && n <= 6) return problem(503, "internal", "boom"); + return json(job({ status: n > 6 ? "completed" : "running" })); + }) as unknown as typeof fetch; + const done = await waitForJob("j1", { apiKey: "k", fetchImpl: impl, sleepImpl: async () => {}, waitMs: 60_000 }); + assert.equal(done.latest_run?.status, "completed"); + assert.ok(n > 6); + // A non-transient error still ends the wait. + const gone = (async () => problem(404, "not_found", "Job not found.")) as unknown as typeof fetch; + await assert.rejects(() => waitForJob("j1", { apiKey: "k", fetchImpl: gone, sleepImpl: async () => {} }), /not found/i); +}); + +// ---------- list, results, content, delete ---------- + +test("list reads GET /jobs with its filters and prints {ok, count, jobs, next_cursor}", async () => { + await withWorkspace( + () => json({ jobs: [job()], next_cursor: "c2" }), + async (calls) => { + const { code, out } = await run_(["list", "--status", "open", "--type", "scheduled", "--limit", "10"]); + assert.equal(code, 0); + assert.equal(out.count, 1); + assert.equal(out.next_cursor, "c2"); + const q = calls[0]!.url.searchParams; + assert.equal(q.get("status"), "open"); + assert.equal(q.get("type"), "scheduled"); + assert.equal(q.get("limit"), "10"); + const bad = await run_(["list", "--limit", "101"]); + assert.equal(bad.code, 1); + }, + ); +}); + +test("results accepts processing and --run, and flags a run still in progress as partial", async () => { + await withWorkspace( + (url) => + url.pathname.endsWith("/results") + ? json({ results: [{ task_id: "t1", status: "processing" }], next_cursor: null }) + : json(run()), + async (calls) => { + const { code, out } = await run_(["results", "j1", "--status", "processing", "--run", "r1"]); + assert.equal(code, 0); + assert.equal(calls[0]!.url.pathname, "/v1/jobs/j1/runs/r1/results"); + assert.equal(calls[0]!.url.searchParams.get("status"), "processing"); + assert.equal(out.job_id, "j1"); + assert.equal(out.partial, true); + assert.match(out.note, /partial/); + }, + ); +}); + +test("results --download skips expired rows with the reason and counts them", async () => { + await withWorkspace( + (url) => { + if (url.pathname.endsWith("/results")) { + return json({ + results: [ + { task_id: "t1", status: "expired", type: "html" }, + { task_id: "t2", status: "successful", type: "html", result_url: "https://s3.example/t2" }, + ], + next_cursor: null, + }); + } + if (url.hostname === "s3.example") return new Response(""); + return json(job({ status: "completed" })); + }, + async (_calls, root) => { + const { out } = await run_(["results", "j1", "--download", join(root, "dl")]); + assert.equal(out.downloaded, 1); + assert.equal(out.expired, 1); + assert.match(out.results.skipped[0].reason, /expired/); + assert.equal(out.partial, false); + }, + ); +}); + +test("content prints one task's body, and maps 409/410/422 to specific codes", async () => { + let mode = "ok"; + await withWorkspace( + () => { + if (mode === "ok") return new Response("hi", { headers: { "content-type": "text/html" } }); + if (mode === "notready") return problem(409, "task_not_ready", "Task is still processing."); + if (mode === "expired") return problem(410, "result_expired", "Result has expired and is no longer available."); + return problem(422, "AUTH006", "Could not get content."); + }, + async (calls, root) => { + const ok = await run_(["content", "j1", "t1"]); + assert.equal(ok.code, 0); + assert.equal(ok.out.content, "hi"); + assert.equal(ok.out.content_type, "text/html"); + assert.equal(calls[0]!.url.pathname, "/v1/jobs/j1/tasks/t1/content"); + const file = join(root, "t1.html"); + await run_(["content", "j1", "t1", "--run", "r1", "--out", file]); + assert.equal(readFileSync(file, "utf8"), "hi"); + assert.equal(calls[1]!.url.pathname, "/v1/jobs/j1/runs/r1/tasks/t1/content"); + for (const [m, code] of [["notready", "BATCH_TASK_NOT_READY"], ["expired", "BATCH_RESULT_EXPIRED"], ["failed", "BATCH_TASK_FAILED"]]) { + mode = m!; + const r = await run_(["content", "j1", "t1"]); + assert.equal(r.code, 1); + assert.equal(r.out.error.code, code); + } + }, + ); +}); + +test("delete needs --yes, then sends DELETE /jobs/{id}", async () => { + await withWorkspace( + () => new Response(null, { status: 202 }), + async (calls) => { + const refused = await run_(["delete", "j1"]); + assert.equal(refused.code, 1); + assert.equal(calls.length, 0); + const { code, out } = await run_(["delete", "j1"], true); + assert.equal(code, 0); + assert.deepEqual(out, { ok: true, job_id: "j1", deleted: true }); + assert.equal(calls[0]!.init?.method, "DELETE"); + assert.equal(calls[0]!.url.pathname, "/v1/jobs/j1"); + }, + ); +}); From e19e20d8def380fe8fead8a701699f02015c94fb Mon Sep 17 00:00:00 2001 From: Emirlan Rasulov Date: Fri, 16 Oct 2026 16:44:51 +0600 Subject: [PATCH 2/3] fix(batch): a bare rerun could re-bill tasks that had already succeeded zenrows batch rerun with no flag now reruns the failed and pending tasks, matching the MCP batch_rerun default. --status picks the statuses and --all reruns every task; the two cannot be combined. --- README.md | 9 ++++++--- skills/batch-jobs/SKILL.md | 16 ++++++++++------ src/cli/commands/batch.ts | 34 +++++++++++++++++++++++++-------- src/core/batch-api.ts | 12 ++++++------ tests/batch-consistency.test.ts | 21 +++++++++++++++++--- 5 files changed, 66 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 67ae77e..4be610e 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ zenrows batch content # one task's result body zenrows batch pause # pause the run (resume continues it) zenrows batch cancel # stop an in-flight run zenrows batch retry-failed --include-pending # rerun every task that did not succeed -zenrows batch rerun # rerun every task +zenrows batch rerun # rerun failed + pending tasks (--all: every task) zenrows batch delete --yes # delete the job, its runs and results ``` @@ -180,8 +180,11 @@ command; the job keeps running. Ctrl-C stops the wait, not the job, and exits cap `KEY_CREDIT_CAP_REACHED` (the same codes a refused request gets), anything else `BATCH_FAILED`, each with the run's `failure_reason` as `server_code` and the `job_id`. Then run the unfinished tasks with -`zenrows batch rerun --status failed,pending,processing`. A rerun with -`--status` works for 10 days after the run started; a full rerun always does. +`zenrows batch rerun --status failed,pending,processing`. With no +flag, `rerun` reruns the `failed` and `pending` tasks, so successful tasks are +never billed again by surprise; `--status ` picks the statuses and +`--all` reruns every task (not with `--status`). A partial rerun works for 10 +days after the run started; `--all` always does. With `--json`, `create`, `status`, `wait`, `cancel`, `pause`, `resume` and `rerun` print `{ok, job_id, run_id, status, stats, job}` (`jobId` is kept for diff --git a/skills/batch-jobs/SKILL.md b/skills/batch-jobs/SKILL.md index e8ef5f1..7b703cd 100644 --- a/skills/batch-jobs/SKILL.md +++ b/skills/batch-jobs/SKILL.md @@ -112,7 +112,9 @@ zenrows batch resume zenrows batch cancel # stop the run; an ended run answers as it ended zenrows batch retry-failed # rerun the failed tasks zenrows batch retry-failed --include-pending # also the unfinished ones -zenrows batch rerun [--status failed,pending,processing] +zenrows batch rerun # failed + pending tasks; successful ones carry over +zenrows batch rerun --status failed,pending,processing +zenrows batch rerun --all # every task (not with --status) zenrows batch delete --yes # job, runs and stored results; frees its slot ``` @@ -135,8 +137,10 @@ the run's `failure_reason` as `server_code` and the `job_id`: After any of these, and after a `cancel`, the tasks not yet run stay `pending` (some can stay `processing`): fix the cause, then run them with `zenrows batch rerun --status failed,pending,processing`. Do not resubmit -the job. A rerun with `--status` works for 10 days after the run started; -after that only a full rerun (`zenrows batch rerun `) does. Every rerun is +the job. `rerun` with no flag reruns `failed,pending`, never the successful +tasks. A partial rerun (the default, or `--status`) works for 10 days after the +run started; after that only a full rerun (`zenrows batch rerun --all`) +does. Every rerun is a new run that answers 202 with `ingest_status: pending`. ## Limits @@ -176,13 +180,13 @@ for errors that remain after these retries. | `BATCH_NOT_FOUND` | 404: wrong id, deleted, or another account's job | no | | `BATCH_RUN_IN_PROGRESS` | 409 `run_not_terminal`: the run is still going (or still receiving tasks) | after `wait` or `cancel` | | `BATCH_RUN_ENDED` | 409 `run_not_stoppable`: the run already ended (pause/resume) | no; `rerun` starts a new run | -| `BATCH_NO_MATCHING_TASKS` | 409 `no_matching_tasks`: no task has the statuses asked for | with `--status failed,pending,processing`, or a full rerun | -| `BATCH_RERUN_WINDOW_EXPIRED` | 409 `partial_rerun_window_expired`: `--status` is over 10 days late | with a full rerun | +| `BATCH_NO_MATCHING_TASKS` | 409 `no_matching_tasks`: no task has the statuses asked for | with `--status failed,pending,processing`, or `rerun --all` | +| `BATCH_RERUN_WINDOW_EXPIRED` | 409 `partial_rerun_window_expired`: a partial rerun is over 10 days late | with `rerun --all` | | `BATCH_REQUEST_IN_FLIGHT` | 409 `idempotency_key_conflict`: the key is in use or was used for another body | same command, same key, a moment later | | `BATCH_CONFLICT` | any other 409 (job state, key no longer valid, concurrent change) | read the cause | | `BATCH_TASK_NOT_READY` | 409 `task_not_ready` on `content`: the task has not finished | after `wait` | | `BATCH_TASK_FAILED` | 422 on `content`: the task failed | after `retry-failed` | -| `BATCH_RESULT_EXPIRED` | 410 `result_expired`: past the 14-day visibility window | after a full rerun | +| `BATCH_RESULT_EXPIRED` | 410 `result_expired`: past the 14-day visibility window | after `rerun --all` | | `BATCH_FAILED` | the run ended `failed` for another reason; a 400 (fix the flagged tasks) or 5xx | 5xx: yes | | `POLICY_LIMIT_EXCEEDED` | the local policy caps (see Limits) | after raising the cap | | `BACKEND_UNAVAILABLE` | the API could not be reached | yes | diff --git a/src/cli/commands/batch.ts b/src/cli/commands/batch.ts index 3d688dd..f0e8c3d 100644 --- a/src/cli/commands/batch.ts +++ b/src/cli/commands/batch.ts @@ -55,6 +55,8 @@ import { asNumber, asString, parse, type Command, type RunContext } from "../com import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "../../core/errors.ts"; import { printError, writeOut } from "../output.ts"; +/** What `rerun` selects with no --status: the unfinished tasks, never the successful ones. */ +const DEFAULT_RERUN = "failed,pending"; /** The task statuses that recover every task a run did not finish successfully. */ const UNFINISHED = "failed,pending,processing"; @@ -101,9 +103,12 @@ export const batch: Command = { " print or save one task's result body", " cancel stop the in-flight run (an ended run answers as it ended)", " pause / resume pause the in-flight run, or resume it", - " rerun [--status ] start a new run: every task, or only tasks with these", - " statuses (pending,processing,failed,successful); takes", - " --idempotency-key, --follow and --timeout like create", + " rerun start a new run of the failed and pending tasks", + " (successful tasks carry over and are not billed again)", + " --status rerun only tasks with these statuses", + " (pending,processing,failed,successful)", + " --all rerun every task (a full rerun; not with --status)", + " rerun also takes --idempotency-key, --follow and --timeout", " retry-failed rerun only the failed tasks (rerun --status failed); takes", " --idempotency-key, --follow and --timeout like rerun", " --include-pending also the tasks a stop or a credit failure left unfinished", @@ -540,7 +545,7 @@ async function resultsCmd(rest: string[], ctx: RunContext): Promise { if (out.skipped.length) { log.dim(` skipped ${out.skipped.length} with no body (failed, expired or unfinished tasks${expired ? `; ${expired} expired` : ""})`); } - if (expired) log.dim(` expired results are gone after 14 days by default; rerun the job in full: zenrows batch rerun ${id}`); + if (expired) log.dim(` expired results are gone after 14 days by default; rerun the job in full: zenrows batch rerun ${id} --all`); if (out.failed.length) { log.warn(` ${out.failed.length} download(s) failed:`); out.failed.slice(0, 10).forEach((f) => log.dim(` ${f.external_id ?? f.task_id}: ${f.reason}`)); @@ -669,7 +674,7 @@ function asJob(id: string, run: JobRun): Job { async function rerunCmd(rest: string[], ctx: RunContext, sub: "rerun" | "retry-failed"): Promise { const { values, positionals } = parse(rest, { - ...(sub === "rerun" ? { status: { type: "string" } } : { "include-pending": { type: "boolean" } }), + ...(sub === "rerun" ? { status: { type: "string" }, all: { type: "boolean" } } : { "include-pending": { type: "boolean" } }), "idempotency-key": { type: "string" }, follow: { type: "boolean" }, timeout: { type: "string" }, @@ -677,8 +682,21 @@ async function rerunCmd(rest: string[], ctx: RunContext, sub: "rerun" | "retry-f }); const json = ctx.json || values.json === true; const id = requireId(positionals[0], sub); + if (values.all === true && values.status !== undefined) { + throw usageError( + "--all and --status cannot be combined.", + "Pass --all to rerun every task, or --status to pick statuses, not both.", + "--all reruns every task; --status reruns only some.", + ); + } const status = - sub === "rerun" ? rerunStatus(asString(values.status)) : values["include-pending"] === true ? UNFINISHED : "failed"; + sub === "rerun" + ? values.all === true + ? undefined + : (rerunStatus(asString(values.status)) ?? DEFAULT_RERUN) + : values["include-pending"] === true + ? UNFINISHED + : "failed"; const waitMs = timeoutFlag(values.timeout); const givenKey = asString(values["idempotency-key"]); if (givenKey !== undefined) assertIdempotencyKey(givenKey); @@ -698,7 +716,7 @@ async function rerunCmd(rest: string[], ctx: RunContext, sub: "rerun" | "retry-f return printJob(job, json, `Started a new run of job ${id} (${what})`); } -/** `--status` for a rerun: a comma-separated list of task statuses; undefined (unset) means every task. */ +/** `--status` for a rerun: a comma-separated list of task statuses; undefined when unset. */ function rerunStatus(v: string | undefined): string | undefined { if (v === undefined) return undefined; const parts = [...new Set(v.split(",").map((s) => s.trim().toLowerCase()).filter(Boolean))]; @@ -706,7 +724,7 @@ function rerunStatus(v: string | undefined): string | undefined { if (!parts.length || bad !== undefined) { throw usageError( `Invalid --status '${v}'.`, - `Pass a comma-separated list of ${RERUN_STATUSES.join(", ")} (e.g. --status ${UNFINISHED}), or leave --status out to rerun every task.`, + `Pass a comma-separated list of ${RERUN_STATUSES.join(", ")} (e.g. --status ${UNFINISHED}), leave it out to rerun failed and pending tasks, or pass --all to rerun every task.`, `A rerun selects tasks of the previous run by status: ${RERUN_STATUSES.join(", ")}.`, ); } diff --git a/src/core/batch-api.ts b/src/core/batch-api.ts index bd4ca5d..68522f4 100644 --- a/src/core/batch-api.ts +++ b/src/core/batch-api.ts @@ -457,8 +457,8 @@ function classifyProblem( return err( "BATCH_RESULT_EXPIRED", "This result has expired.", - `Results are kept 14 days by default; after that a task reads as expired and its body is gone. Scrape it again with a full rerun: \`zenrows batch rerun ${id}\`.`, - [`zenrows batch rerun ${id}`], + `Results are kept 14 days by default; after that a task reads as expired and its body is gone. Scrape it again with a full rerun: \`zenrows batch rerun ${id} --all\`.`, + [`zenrows batch rerun ${id} --all`], ); } if (status === 422 && /\/content$/.test(path)) { @@ -519,15 +519,15 @@ function conflictError(serverCode: string | undefined, detail: string, cause: st return e( "BATCH_NO_MATCHING_TASKS", "No task of the last run has the statuses you asked to rerun.", - `Tasks a stop or a credit failure left unfinished are pending or processing: rerun them with \`zenrows batch rerun ${id} --status failed,pending,processing\`, or rerun every task with \`zenrows batch rerun ${id}\`.`, + `Tasks a stop or a credit failure left unfinished are pending or processing: rerun them with \`zenrows batch rerun ${id} --status failed,pending,processing\`, or rerun every task with \`zenrows batch rerun ${id} --all\`.`, [`zenrows batch rerun ${id} --status failed,pending,processing`], ); case "partial_rerun_window_expired": return e( "BATCH_RERUN_WINDOW_EXPIRED", "Rerunning only some tasks is no longer allowed for this run.", - `A rerun with --status works for 10 days after the run started. Rerun every task with \`zenrows batch rerun ${id}\` instead.`, - [`zenrows batch rerun ${id}`], + `A partial rerun (the default, or --status) works for 10 days after the run started. Rerun every task with \`zenrows batch rerun ${id} --all\` instead.`, + [`zenrows batch rerun ${id} --all`], ); case "idempotency_key_conflict": return e( @@ -715,7 +715,7 @@ export interface DownloadOutcome { /** Why a row has no body to download. */ function skipReason(row: ResultRow): string { if (row.status === "failed") return "task failed: no result body"; - if (row.status === "expired") return "result expired (kept 14 days by default): a full rerun scrapes it again"; + if (row.status === "expired") return "result expired (kept 14 days by default): a full rerun (--all) scrapes it again"; if (row.status === "pending" || row.status === "processing") return "task not finished yet"; return "no result_url"; } diff --git a/tests/batch-consistency.test.ts b/tests/batch-consistency.test.ts index 0b3a835..5bf4177 100644 --- a/tests/batch-consistency.test.ts +++ b/tests/batch-consistency.test.ts @@ -220,12 +220,12 @@ test("any other failure stays BATCH_FAILED, without 'fix the tasks' advice", () // ---------- rerun ---------- -test("rerun with no --status is a full rerun; output keeps rerun_of, retried_tasks, inherited_tasks", async () => { +test("rerun --all is a full rerun; output keeps rerun_of, retried_tasks, inherited_tasks", async () => { const answer = { job_id: "j1", status: "closed", latest_run: run({ run_id: "r2", ingest_status: "pending" }), rerun_of: "r1", retried_tasks: 5, inherited_tasks: 0 }; await withWorkspace( () => json(answer, 202), async (calls) => { - const { code, out } = await run_(["rerun", "j1"]); + const { code, out } = await run_(["rerun", "j1", "--all"]); assert.equal(code, 0); assert.equal(calls[0]!.url.searchParams.get("status"), null); assert.ok(header(calls[0]!, "Idempotency-Key")); @@ -238,6 +238,21 @@ test("rerun with no --status is a full rerun; output keeps rerun_of, retried_tas ); }); +test("rerun with no flags reruns failed,pending only; --all and --status cannot be combined", async () => { + await withWorkspace( + () => json(job()), + async (calls) => { + const { code } = await run_(["rerun", "j1"]); + assert.equal(code, 0); + assert.equal(calls[0]!.url.searchParams.get("status"), "failed,pending"); + const both = await run_(["rerun", "j1", "--all", "--status", "failed"]); + assert.equal(both.code, 1); + assert.equal(both.out.error.code, "INVALID_USAGE"); + assert.equal(calls.length, 1); + }, + ); +}); + test("rerun --status takes pending,processing,failed,successful and rejects anything else before any request", async () => { await withWorkspace( () => json(job()), @@ -270,7 +285,7 @@ test("retry-failed sends status=failed, and --include-pending recovers unfinishe test("each lifecycle 409 gets its own code and advice, carrying server_code and job_id", async () => { const cases: Array<[string, string, string, RegExp]> = [ ["no_matching_tasks", "No tasks match the requested statuses.", "BATCH_NO_MATCHING_TASKS", /--status failed,pending,processing/], - ["partial_rerun_window_expired", "Partial rerun window expired.", "BATCH_RERUN_WINDOW_EXPIRED", /zenrows batch rerun j1`/], + ["partial_rerun_window_expired", "Partial rerun window expired.", "BATCH_RERUN_WINDOW_EXPIRED", /zenrows batch rerun j1 --all`/], ["run_not_terminal", "Previous run is still in flight; call /stop before /rerun.", "BATCH_RUN_IN_PROGRESS", /batch wait j1/], ["idempotency_key_conflict", "Idempotency key conflict.", "BATCH_REQUEST_IN_FLIGHT", /same key/], ["run_has_descendants", "Run has descendants.", "BATCH_CONFLICT", /batch status j1/], From b5de23f0b0cb34f437a5ba490766b28154a5df05 Mon Sep 17 00:00:00 2001 From: Emirlan Rasulov Date: Fri, 16 Oct 2026 18:08:15 +0600 Subject: [PATCH 3/3] fix(batch): a retried submit could still end in a duplicate job or an endless wait - once an attempt of a keyed create or rerun got no answer, any later refusal (a 402 included, which the API checks before the key) keeps the Idempotency-Key and the safe-repeat advice - the repeat command shell-quotes every argument and the key and keeps their whitespace - a wait passes its deadline to each request: attempts are cut off at it and no retry sleeps past it, so --timeout holds - results whose run state cannot be read are reported as possibly partial - url and metadata value limits count UTF-8 bytes, and the job name counts code points, as the API does --- README.md | 2 +- skills/batch-jobs/SKILL.md | 7 ++- src/adapters/batch.ts | 7 ++- src/cli/commands/batch.ts | 27 +++++++-- src/core/batch-api.ts | 45 ++++++++++---- tests/batch-consistency.test.ts | 100 +++++++++++++++++++++++++++++++- 6 files changed, 161 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 4be610e..6290f63 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ per-line overrides like `js_render`, `premium_proxy`, `proxy_country`, `extract`, nested `zenrows_params`, an `external_id`, string `metadata`, and `"method": "POST"` with a `body` of up to 16 KiB). `estimate` and `create` check every line against the API's limits before anything is sent: `url` up -to 2048 characters, `external_id` up to 128 letters, digits, `.`, `_` or `-`, +to 2048 bytes, `external_id` up to 128 letters, digits, `.`, `_` or `-`, and `metadata` up to 20 string entries (4 KiB). Validate and estimate locally first, then submit with `create`. `estimate` takes `create`'s `--manual` / `--js-render` / `--premium-proxy` / `--proxy-country` flags and diff --git a/skills/batch-jobs/SKILL.md b/skills/batch-jobs/SKILL.md index 7b703cd..db67f09 100644 --- a/skills/batch-jobs/SKILL.md +++ b/skills/batch-jobs/SKILL.md @@ -25,14 +25,15 @@ zenrows batch estimate jobs.jsonl [--premium-proxy] [--js-render] [--manual] ``` `jobs.jsonl` is one JSON object per line, each with a `url` (http or https, -up to 2048 characters). Optional per-task keys: +up to 2048 bytes). Optional per-task keys: - `external_id`: up to 128 letters, digits, `.`, `_` or `-`; echoed on each result row (it does not have to be unique). - `metadata`: up to 20 string-to-string entries; keys 1 to 64 of the same - characters, values up to 1024 characters, 4 KiB in total. + characters, values up to 1024 bytes (UTF-8), 4 KiB in total. - `method` and `body`: `"method": "POST"` sends `body` (up to 16 KiB): a JSON - string goes as a form body, any other JSON value as JSON. A POST task cannot + string goes as a form body, any other JSON value as JSON (`"body": null` means + no body). A POST task cannot use `js_render`, `js_instructions` or `json_response`. - scrape params, flat (`js_render`, `premium_proxy`, `proxy_country`, `mode`, `autoparse`, `extract`, ...) or nested under `zenrows_params`. Booleans are diff --git a/src/adapters/batch.ts b/src/adapters/batch.ts index 470bc58..f4c3c8d 100644 --- a/src/adapters/batch.ts +++ b/src/adapters/batch.ts @@ -103,7 +103,8 @@ function taskError(obj: unknown): string | null { return `invalid url: ${job.url}`; } if (u.protocol !== "http:" && u.protocol !== "https:") return `url must be http or https: ${job.url}`; - if (job.url.length > LIMITS.urlLength) return `url is longer than ${LIMITS.urlLength} characters`; + // The API counts these limits in UTF-8 bytes, not characters. + if (Buffer.byteLength(job.url) > LIMITS.urlLength) return `url is longer than ${LIMITS.urlLength} bytes`; if (job.external_id !== undefined && job.external_id !== null) { const e = externalIdError(job.external_id); if (e) return e; @@ -127,7 +128,7 @@ export function externalIdError(v: unknown): string | null { /** * Why `metadata` is invalid, or null: the API takes at most 20 string-to-string - * entries, keys of 1 to 64 safe characters, values up to 1024 characters, 4 KiB + * entries, keys of 1 to 64 safe characters, values up to 1024 bytes, 4 KiB * in total. Any other shape fails the whole submit. */ export function metadataError(metadata: unknown): string | null { @@ -141,7 +142,7 @@ export function metadataError(metadata: unknown): string | null { return `\`metadata\` key "${k}" must be 1 to ${LIMITS.metadataKeyLength} letters, digits, \`.\`, \`_\` or \`-\``; } if (typeof v !== "string") return `\`metadata.${k}\` must be a string`; - if (v.length > LIMITS.metadataValueLength) return `\`metadata.${k}\` is longer than ${LIMITS.metadataValueLength} characters`; + if (Buffer.byteLength(v) > LIMITS.metadataValueLength) return `\`metadata.${k}\` is longer than ${LIMITS.metadataValueLength} bytes`; total += Buffer.byteLength(k) + Buffer.byteLength(v); } if (total > LIMITS.metadataTotalBytes) return "`metadata` is larger than 4 KiB"; diff --git a/src/cli/commands/batch.ts b/src/cli/commands/batch.ts index f0e8c3d..137c80e 100644 --- a/src/cli/commands/batch.ts +++ b/src/cli/commands/batch.ts @@ -238,7 +238,8 @@ function jobFieldsOf(values: Record): JobFields { const fields: JobFields = {}; const name = asString(values.name); if (name !== undefined) { - if (!name.trim() || name.length > LIMITS.nameLength) { + // The API counts the name in characters (code points), not UTF-16 units. + if (!name.trim() || [...name].length > LIMITS.nameLength) { throw usageError(`Invalid --name: it must be 1 to ${LIMITS.nameLength} characters.`, "Pass a shorter name."); } fields.name = name; @@ -389,12 +390,17 @@ function withRepeatCommand(err: unknown, sub: string, rest: string[]): unknown { continue; } if (a.startsWith("--idempotency-key=")) continue; - args.push(/^[A-Za-z0-9._\-/:=,@%+]+$/.test(a) ? a : `'${a.replace(/'/g, "'\\''")}'`); + args.push(shellQuote(a)); } - const repeat = `zenrows batch ${sub} ${args.join(" ")} --idempotency-key ${err.idempotency_key}`.replace(/\s+/g, " "); + const repeat = ["zenrows", "batch", sub, ...args, "--idempotency-key", shellQuote(err.idempotency_key)].join(" "); return new ToolkitError({ ...err.toJSON(), suggested_commands: [repeat, "zenrows batch list"] }); } +/** `s` as one POSIX shell word: bare when it is plainly safe, else single-quoted. */ +export function shellQuote(s: string): string { + return /^[A-Za-z0-9._\-/:=,@%+]+$/.test(s) ? s : `'${s.replace(/'/g, "'\\''")}'`; +} + async function listCmd(rest: string[], ctx: RunContext): Promise { const { values } = parse(rest, { status: { type: "string" }, @@ -513,7 +519,11 @@ async function resultsCmd(rest: string[], ctx: RunContext): Promise { const rows = await listResults(id, { apiKey, status, runId }); const state = await runState(id, runId, apiKey); - const note = state.partial ? partialNote(state.status, id) : undefined; + const note = state.unknown + ? `The run's state could not be read, so these results may be partial. Check it with zenrows batch status ${id}, then read them again.` + : state.partial + ? partialNote(state.status, id) + : undefined; const meta = { jobId: id, job_id: id, ...(runId ? { run_id: runId } : {}), ...(state.status ? { status: state.status } : {}) }; const downloadDir = asString(values.download); @@ -578,13 +588,18 @@ async function resultsCmd(rest: string[], ctx: RunContext): Promise { * Whether the run whose results were read can still change: in progress, or * still receiving its tasks. An error here never fails the listing. */ -async function runState(id: string, runId: string | undefined, apiKey: string): Promise<{ status?: string; partial: boolean }> { +async function runState( + id: string, + runId: string | undefined, + apiKey: string, +): Promise<{ status?: string; partial: boolean; unknown?: boolean }> { try { const run: JobRun | undefined = runId ? await getRun(id, runId, { apiKey }) : (await getJob(id, { apiKey })).latest_run; if (!run) return { partial: false }; return { status: run.status, partial: !isTerminal(run) || run.ingest_status === "pending" }; } catch { - return { partial: false }; + // Unknown is not complete: report the rows as possibly partial. + return { partial: true, unknown: true }; } } diff --git a/src/core/batch-api.ts b/src/core/batch-api.ts index 68522f4..78df40f 100644 --- a/src/core/batch-api.ts +++ b/src/core/batch-api.ts @@ -138,6 +138,11 @@ interface CallOpts { /** Per HTTP attempt. */ timeoutMs?: number; sleepImpl?: (ms: number) => Promise; + /** + * Epoch ms the whole call must end by, retries included: each attempt is cut + * off at it and no retry starts that would sleep past it. + */ + deadline?: number; } interface RequestOpts extends CallOpts { @@ -220,7 +225,9 @@ function transient(err: ToolkitError): ToolkitError { /** One HTTP attempt with its own timeout; the body is read inside it. */ async function attempt(url: string, init: RequestInit, opts: RequestOpts): Promise { const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000); + const perAttempt = opts.timeoutMs ?? 60_000; + const ms = opts.deadline !== undefined ? Math.max(1, Math.min(perAttempt, opts.deadline - Date.now())) : perAttempt; + const timeout = setTimeout(() => controller.abort(), ms); try { const res = await (opts.fetchImpl ?? fetch)(url, { ...init, signal: controller.signal }); return { status: res.status, headers: res.headers, body: Buffer.from(await res.arrayBuffer()) }; @@ -256,17 +263,22 @@ async function batchFetch(method: string, path: string, opts: RequestOpts): Prom // Set once an attempt of a keyed write got no answer: the API may have done the work. let mayHaveLanded = false; + // A retry that would sleep past the call's deadline is not started. + const fits = (waitMs: number) => opts.deadline === undefined || Date.now() + waitMs < opts.deadline; + for (let n = 0; ; n++) { const canRetry = retryOn !== undefined && n < MAX_RETRIES; let res: RawResponse; try { res = await attempt(url.toString(), init, opts); } catch (err) { - if (canRetry) { + const wait = backoffMs(n); + if (canRetry && fits(wait)) { mayHaveLanded = true; - await sleep(backoffMs(n)); + await sleep(wait); continue; } + if (key) mayHaveLanded = true; const failure = transient( new ToolkitError({ code: "BACKEND_UNAVAILABLE", @@ -281,16 +293,21 @@ async function batchFetch(method: string, path: string, opts: RequestOpts): Prom throw key ? outcomeUnknown(failure, key) : failure; } if (canRetry && retryOn.has(res.status)) { - if (res.status >= 500) mayHaveLanded = true; - await sleep(retryAfterMs(res.headers.get("retry-after")) ?? backoffMs(n)); - continue; + const wait = retryAfterMs(res.headers.get("retry-after")) ?? backoffMs(n); + if (fits(wait)) { + if (res.status >= 500) mayHaveLanded = true; + await sleep(wait); + continue; + } } if (res.status < 200 || res.status >= 300) { let failure = problemToError(res.status, res.body.toString("utf8"), method, path, opts.jobId); if (res.status === 429 || res.status >= 500) failure = transient(failure); - if (key && (res.status >= 500 || (mayHaveLanded && failure.server_code === "idempotency_key_conflict"))) { - throw outcomeUnknown(failure, key); - } + if (key && res.status >= 500) throw outcomeUnknown(failure, key); + // An earlier attempt may have been accepted, and the API checks credit + // (402) and other gates before it looks at the key: whatever this attempt + // answers, the safe next step is still to repeat with the same key. + if (key && mayHaveLanded) throw outcomeUnknown(failure, key, { keepAdvice: true }); throw failure; } return res; @@ -325,12 +342,15 @@ async function batchJson(method: string, path: string, opts: RequestOpts): Pr * done the work although no usable answer arrived. Carry the key and say what * is safe; the command layer adds the exact command to repeat. */ -function outcomeUnknown(err: ToolkitError, key: string): ToolkitError { +function outcomeUnknown(err: ToolkitError, key: string, opts: { keepAdvice?: boolean } = {}): ToolkitError { + const lead = opts.keepAdvice + ? `${err.next_action.replace(/[.\s]*$/, ".")} An earlier attempt of this request got no answer and may have been accepted. Once that is resolved, repeat it with the same Idempotency-Key (${key}) ` + : `The request may have been accepted even though no usable answer arrived. Repeat it with the same Idempotency-Key (${key}) `; const out = new ToolkitError({ ...err.toJSON(), idempotency_key: key, next_action: - `The request may have been accepted even though no usable answer arrived. Repeat it with the same Idempotency-Key (${key}) ` + + lead + "within 24 hours: the API then answers with the original job or run instead of creating a second one. " + "Do not repeat it with a new key, which can create a duplicate. Check `zenrows batch list` first.", }); @@ -798,11 +818,12 @@ export async function waitForJob(id: string, opts: CallOpts & { waitMs?: number const sleep = opts.sleepImpl ?? defaultSleep; const deadline = Date.now() + (opts.waitMs ?? 600_000); let delay = 2000; + const read = { ...opts, deadline }; let last: Job | undefined; let lastErr: unknown; for (;;) { try { - last = await getJob(id, opts); + last = await getJob(id, read); if (isTerminal(last.latest_run) || isPaused(last.latest_run)) return last; } catch (err) { if (!isTransient(err)) throw err; diff --git a/tests/batch-consistency.test.ts b/tests/batch-consistency.test.ts index 5bf4177..f9dd3d7 100644 --- a/tests/batch-consistency.test.ts +++ b/tests/batch-consistency.test.ts @@ -10,9 +10,10 @@ import assert from "node:assert/strict"; import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { batch, runFailure, timeoutFlag } from "../src/cli/commands/batch.ts"; +import { batch, runFailure, shellQuote, timeoutFlag } from "../src/cli/commands/batch.ts"; import { estimateCredits, toJobBody, validateJsonl, withAdaptiveStealth } from "../src/adapters/batch.ts"; -import { waitForJob } from "../src/core/batch-api.ts"; +import { createJob, waitForJob } from "../src/core/batch-api.ts"; +import { ToolkitError } from "../src/core/errors.ts"; import { createWorkspace } from "../src/core/workspace.ts"; import { savePolicy, defaultPolicy } from "../src/core/policy.ts"; import { saveApiKey } from "../src/core/auth.ts"; @@ -663,3 +664,98 @@ test("delete needs --yes, then sends DELETE /jobs/{id}", async () => { }, ); }); + +// ---------- review follow-ups ---------- + +test("a 402 after a lost attempt keeps the Idempotency-Key and the safe-repeat advice", async () => { + let n = 0; + const impl = (async () => { + if (n++ === 0) throw new TypeError("socket hang up"); + return problem(402, "payment_required", "Subscription has no credit available."); + }) as unknown as typeof fetch; + await assert.rejects( + () => createJob({ type: "regular", status: "closed", tasks: [] }, { apiKey: "k", fetchImpl: impl, sleepImpl: async () => {}, idempotencyKey: "key-1" }), + (e: unknown) => + e instanceof ToolkitError && + e.code === "POLICY_MAX_CREDITS_EXCEEDED" && + e.idempotency_key === "key-1" && + /may have been accepted/.test(e.next_action) && + /same Idempotency-Key \(key-1\)/.test(e.next_action), + ); + // Without a lost attempt, a 402 is a plain refusal. + const plain = (async () => problem(402, "payment_required", "Subscription has no credit available.")) as unknown as typeof fetch; + await assert.rejects( + () => createJob({ type: "regular", status: "closed", tasks: [] }, { apiKey: "k", fetchImpl: plain, idempotencyKey: "key-2" }), + (e: unknown) => e instanceof ToolkitError && e.idempotency_key === undefined, + ); +}); + +test("the repeat command shell-quotes every argument and the key, keeping inner whitespace", async () => { + assert.equal(shellQuote("plain-1.jsonl"), "plain-1.jsonl"); + assert.equal(shellQuote("key with space"), "'key with space'"); + assert.equal(shellQuote("it's;$(rm)"), "'it'\\''s;$(rm)'"); + assert.equal(shellQuote(""), "''"); + await withWorkspace( + () => problem(503, "internal", "boom"), + async (_calls, root) => { + const file = spec(root, [{ url: "https://example.com/a" }]); + const { out } = await run_(["create", file, "--name", "two spaces", "--idempotency-key", "k $(x) y"]); + assert.equal(out.error.suggested_commands[0], `zenrows batch create ${file} --name 'two spaces' --idempotency-key 'k $(x) y'`); + }, + ); +}); + +test("a wait never sleeps or retries past its --timeout", async () => { + const sleeps: number[] = []; + let n = 0; + const impl = (async () => { + if (n++ === 0) return json(job()); + return new Response(JSON.stringify({ code: "internal", detail: "boom" }), { status: 503, headers: { "retry-after": "30" } }); + }) as unknown as typeof fetch; + const started = Date.now(); + const done = await waitForJob("j1", { + apiKey: "k", + fetchImpl: impl, + waitMs: 3000, + sleepImpl: async (ms) => { + sleeps.push(ms); + }, + }); + assert.equal(done.latest_run?.status, "running", "returns the last state read"); + assert.ok(sleeps.every((ms) => ms < 3000), `no sleep reaches past the deadline: ${sleeps}`); + assert.ok(!sleeps.includes(30_000), "a 30 s Retry-After is not slept on a 3 s wait"); + assert.ok(Date.now() - started < 3000); + + // A request that hangs is cut off at the deadline, not after the 60 s per-attempt timeout. + const hang = ((_u: string, init?: RequestInit) => + new Promise((_r, reject) => init?.signal?.addEventListener("abort", () => reject(new Error("aborted"))))) as unknown as typeof fetch; + const t0 = Date.now(); + await assert.rejects(() => waitForJob("j1", { apiKey: "k", fetchImpl: hang, waitMs: 300, sleepImpl: async () => {} })); + assert.ok(Date.now() - t0 < 2000, `took ${Date.now() - t0} ms`); +}); + +test("results whose run state cannot be read are reported as possibly partial", async () => { + await withWorkspace( + (url) => + url.pathname.endsWith("/results") + ? json({ results: [{ task_id: "t1", status: "pending" }], next_cursor: null }) + : problem(404, "not_found", "Job not found."), + async () => { + const { code, out } = await run_(["results", "j1"]); + assert.equal(code, 0); + assert.equal(out.partial, true); + assert.match(out.note, /could not be read/); + }, + ); +}); + +test("url and metadata limits count UTF-8 bytes, as the API does", () => { + const dir = mkdtempSync(join(tmpdir(), "zr-bytes-")); + const file = spec(dir, [ + { url: "https://ok.example/", metadata: { v: "é".repeat(512) } }, + { url: "https://b.example/", metadata: { v: "é".repeat(600) } }, + { url: `https://c.example/${"é".repeat(1100)}` }, + ]); + const v = validateJsonl(file); + assert.deepEqual(v.errors.map((e) => e.line), [2, 3]); +});