From 6828b512a4c00c239e5b71e21519cb2973f3f098 Mon Sep 17 00:00:00 2001 From: Paulius Krutkis Date: Thu, 11 Jun 2026 09:57:10 +0300 Subject: [PATCH 1/3] Add global --timeout flag for per-request timeout Expose the SDK's request timeout as a global --timeout flag (default 180000, the SDK default). Applies to every command and to each item in a batch run; a timed-out request surfaces as TimeoutError (exit 6) and, in batch mode, as a per-item error record without aborting the run. Invalid values are rejected as a usage error (exit 2). --- README.md | 12 +++++++ src/cli/services/global-opts.ts | 1 + src/cli/services/parse-timeout.ts | 11 ++++++ src/index.ts | 6 ++++ src/scrape/services/client.ts | 4 ++- src/scrape/services/run-target-scrape.ts | 10 ++++-- src/scrape/types/run-target-scrape.ts | 1 + tests/cli/services/parse-timeout.test.ts | 13 +++++++ tests/index.test.ts | 4 ++- .../scrape/services/run-target-scrape.test.ts | 35 ++++++++++++++++++- 10 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 src/cli/services/parse-timeout.ts create mode 100644 tests/cli/services/parse-timeout.test.ts diff --git a/README.md b/README.md index 94ef4b6..1036d69 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,18 @@ decodo google-search "shoes" --geo de --parse Use `decodo --help` for all geo, locale, and target-specific options from the API schema. +### Request timeouts + +Requests use the SDK default timeout of 180s. Override it globally with `--timeout ` — useful to fail fast on slow targets or batch runs. A timed-out request exits with code `6`. + +```bash +# Fail fast: give up after 30s +decodo scrape https://example.com --timeout 30000 + +# Allow longer for a heavy page render +decodo screenshot https://example.com --timeout 120000 -o shot.png +``` + ## Agent tooling Coding agents (Cursor, Claude Code, Codex, Gemini CLI, Windsurf) should invoke the CLI as a **shell subprocess**, not embed scraping logic. diff --git a/src/cli/services/global-opts.ts b/src/cli/services/global-opts.ts index 3867460..f99a774 100644 --- a/src/cli/services/global-opts.ts +++ b/src/cli/services/global-opts.ts @@ -1,6 +1,7 @@ import type { Command } from "commander"; export interface RootOptions { + timeout?: number; token?: string; verbose?: boolean; } diff --git a/src/cli/services/parse-timeout.ts b/src/cli/services/parse-timeout.ts new file mode 100644 index 0000000..85bd60b --- /dev/null +++ b/src/cli/services/parse-timeout.ts @@ -0,0 +1,11 @@ +import { CliUsageError } from "../../platform/services/handle-cli-error.js"; + +export function parseTimeout(value: string): number { + const parsed = Number.parseInt(value, 10); + if (Number.isNaN(parsed) || parsed < 1) { + throw new CliUsageError( + "--timeout must be a positive integer (milliseconds)." + ); + } + return parsed; +} diff --git a/src/index.ts b/src/index.ts index 537f2cf..b6c1b8c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url"; import { Command } from "commander"; import { createCommands } from "./cli/register.js"; import { configureCommanderExit } from "./cli/services/configure-commander-exit.js"; +import { parseTimeout } from "./cli/services/parse-timeout.js"; import { handleCliError } from "./platform/services/handle-cli-error.js"; function readVersion(): string { @@ -27,6 +28,11 @@ const program = new Command() .option( "--token ", "Basic auth token (overrides DECODO_AUTH_TOKEN and saved config)" + ) + .option( + "--timeout ", + "Per-request timeout in milliseconds (default: 180000)", + parseTimeout ); async function main(): Promise { diff --git a/src/scrape/services/client.ts b/src/scrape/services/client.ts index 5870be2..2729b90 100644 --- a/src/scrape/services/client.ts +++ b/src/scrape/services/client.ts @@ -2,10 +2,12 @@ import { DecodoClient, type DecodoSchema } from "@decodo/sdk-ts"; export function createDecodoClient( token: string, - schema?: DecodoSchema + schema?: DecodoSchema, + timeoutMs?: number ): DecodoClient { return new DecodoClient({ webScrapingApi: { token }, schema, + timeoutMs, }); } diff --git a/src/scrape/services/run-target-scrape.ts b/src/scrape/services/run-target-scrape.ts index 5ef7aae..fe0b13b 100644 --- a/src/scrape/services/run-target-scrape.ts +++ b/src/scrape/services/run-target-scrape.ts @@ -32,8 +32,9 @@ async function executeScrape({ outputContext, input, verbose = false, + timeoutMs, }: ExecuteScrapeOptions): Promise { - const client = createDecodoClient(token, schema); + const client = createDecodoClient(token, schema, timeoutMs); const startedAt = Date.now(); const response = await client.webScrapingApi.scrape( body as unknown as ScrapeRequest @@ -52,6 +53,7 @@ interface ExecuteBatchOptions { options: Record; resolveBody: ScrapeBodyBuilder; schema: DecodoSchema; + timeoutMs?: number; token: string; verbose: boolean; } @@ -63,8 +65,9 @@ async function executeBatch({ resolveBody, binary, verbose, + timeoutMs, }: ExecuteBatchOptions): Promise { - const client = createDecodoClient(token, schema); + const client = createDecodoClient(token, schema, timeoutMs); const batch = options as BatchFlags & OutputOptions; const full = batch.full === true; @@ -107,6 +110,7 @@ export function createTargetAction( ): Promise => { const rootOpts = getRootOpts(command); const verbose = rootOpts.verbose === true; + const timeoutMs = rootOpts.timeout; try { const batchMode = (options as BatchFlags).inputFile !== undefined; @@ -134,6 +138,7 @@ export function createTargetAction( resolveBody, binary: outputContext?.binary?.kind === "png", verbose, + timeoutMs, }); return; } @@ -149,6 +154,7 @@ export function createTargetAction( outputContext, input, verbose, + timeoutMs, }); } catch (err) { handleCliError(err, { fallbackMessage: "Scrape failed." }); diff --git a/src/scrape/types/run-target-scrape.ts b/src/scrape/types/run-target-scrape.ts index b974b5f..e8ebc99 100644 --- a/src/scrape/types/run-target-scrape.ts +++ b/src/scrape/types/run-target-scrape.ts @@ -7,6 +7,7 @@ export interface ExecuteScrapeOptions { options: Record; outputContext?: Partial; schema: DecodoSchema; + timeoutMs?: number; token: string; verbose?: boolean; } diff --git a/tests/cli/services/parse-timeout.test.ts b/tests/cli/services/parse-timeout.test.ts new file mode 100644 index 0000000..95d5c85 --- /dev/null +++ b/tests/cli/services/parse-timeout.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { parseTimeout } from "../../../src/cli/services/parse-timeout.js"; +import { CliUsageError } from "../../../src/platform/services/handle-cli-error.js"; + +describe("parseTimeout", () => { + it("parses a positive integer of milliseconds", () => { + expect(parseTimeout("5000")).toBe(5000); + }); + + it.each(["0", "-1", "abc", ""])("rejects %p as a usage error", (value) => { + expect(() => parseTimeout(value)).toThrow(CliUsageError); + }); +}); diff --git a/tests/index.test.ts b/tests/index.test.ts index 27f3849..99c172f 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -32,12 +32,13 @@ describe("cli", () => { expect(output).toBe(packageJson.version); }); - it("shows verbose flag in root help", () => { + it("shows verbose and timeout flags in root help", () => { const output = execFileSync(process.execPath, [cliPath, "--help"], { encoding: "utf8", }); expect(output).toContain("-v, --verbose"); + expect(output).toContain("--timeout "); }); it.each([ @@ -45,6 +46,7 @@ describe("cli", () => { ["unknown command", ["nosuchcmd"], 2], ["missing required arg", ["search"], 2], ["invalid choice", ["search", "q", "--engine", "yahoo"], 2], + ["invalid timeout", ["--timeout", "0", "scrape", "https://x.com"], 2], ])("exits with code 2 on %s", (_label, args, expectedExit) => { const { exitCode } = runCli(args); expect(exitCode).toBe(expectedExit); diff --git a/tests/scrape/services/run-target-scrape.test.ts b/tests/scrape/services/run-target-scrape.test.ts index 178a921..cd0031b 100644 --- a/tests/scrape/services/run-target-scrape.test.ts +++ b/tests/scrape/services/run-target-scrape.test.ts @@ -85,11 +85,44 @@ describe("createTargetAction", () => { }); expect(createDecodoClient).toHaveBeenCalledWith( "test-token", - BundledSchema.shared + BundledSchema.shared, + undefined ); expect(stdout).toBe('{"ok":true}\n'); }); + it("passes the global --timeout through to the client", async () => { + const scrape = vi.fn().mockResolvedValue({ + results: [{ content: { ok: true } }], + }); + vi.mocked(createDecodoClient).mockReturnValue({ + webScrapingApi: { scrape }, + } as never); + + const googleSearch = new Command("google-search") + .argument("") + .action(createTargetAction("google_search", BundledSchema.shared)); + attachScrapeOutputOptions(googleSearch); + + const program = new Command() + .option("--token ") + .option("--timeout ", "", (value: string) => + Number.parseInt(value, 10) + ) + .addCommand(googleSearch); + + await program.parseAsync( + ["google-search", "coffee", "--token", "test-token", "--timeout", "5000"], + { from: "user" } + ); + + expect(createDecodoClient).toHaveBeenCalledWith( + "test-token", + BundledSchema.shared, + 5000 + ); + }); + it("prints verbose logs to stderr when --verbose is set", async () => { const scrape = vi.fn().mockResolvedValue({ results: [{ content: { ok: true } }], From a192b63b512c8b54d522dc8674cc4aacd44a0f65 Mon Sep 17 00:00:00 2001 From: Paulius Krutkis Date: Thu, 11 Jun 2026 10:59:50 +0300 Subject: [PATCH 2/3] Update @decodo/sdk-ts to 2.1.2 and wire integrationHeader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Bump @decodo/sdk-ts ^2.1.1 → ^2.1.2 - Pass integrationHeader: INTEGRATION_HEADER ("cli") to DecodoClient - Remove TODO(SCR-3150) comment — SDK now supports integrationHeader - Update auth-validation test to expect x-integration: cli Co-Authored-By: Oz --- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- src/scrape/constants.ts | 1 - src/scrape/services/client.ts | 6 +++++- tests/scrape/services/auth-validation.test.ts | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index d3c1f3d..422d699 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ }, "packageManager": "pnpm@10.33.3", "dependencies": { - "@decodo/sdk-ts": "^2.1.1", + "@decodo/sdk-ts": "^2.1.2", "commander": "^14.0.0", "env-paths": "^3.0.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 348e7b7..4c43f13 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@decodo/sdk-ts': - specifier: ^2.1.1 - version: 2.1.1 + specifier: ^2.1.2 + version: 2.1.2 commander: specifier: ^14.0.0 version: 14.0.3 @@ -104,8 +104,8 @@ packages: resolution: {integrity: sha512-S0My7XPGIgpRWMDG8uRqalbgT+a6FmCUdOW+HaIOVVpUPHOb7RrpvjTjiODadKp06fsrVDJZlIzc6yCTp4AnxA==} engines: {node: '>= 20.12.0'} - '@decodo/sdk-ts@2.1.1': - resolution: {integrity: sha512-1pSgG4BiPzjQEObSKahnF35wgU9Gck/aD466JwXwiNS0iRtEtcZlYpjSOuTvKV50CWuNAFZk0Ak7NHuWc4+Q7A==} + '@decodo/sdk-ts@2.1.2': + resolution: {integrity: sha512-V/SdHS0DV9L8gBJc5ljQbTmbgV7C8Cn7V91qn3JfnHKgvSmuUq2kLH27UPgCbAa0T2ZVXOkBzbIog5ZnOzAUMg==} engines: {node: '>=18.0.0'} '@esbuild/aix-ppc64@0.27.7': @@ -966,7 +966,7 @@ snapshots: fast-wrap-ansi: 0.2.2 sisteransi: 1.0.5 - '@decodo/sdk-ts@2.1.1': + '@decodo/sdk-ts@2.1.2': dependencies: zod: 4.4.3 diff --git a/src/scrape/constants.ts b/src/scrape/constants.ts index 27c9f1f..5e424a1 100644 --- a/src/scrape/constants.ts +++ b/src/scrape/constants.ts @@ -1,4 +1,3 @@ -// TODO(SCR-3150): switch to cli when sdk task lands export const INTEGRATION_HEADER = "cli"; export const SCHEMA_TTL_MS = 3_600_000; diff --git a/src/scrape/services/client.ts b/src/scrape/services/client.ts index 2729b90..8e5d3e7 100644 --- a/src/scrape/services/client.ts +++ b/src/scrape/services/client.ts @@ -1,4 +1,5 @@ import { DecodoClient, type DecodoSchema } from "@decodo/sdk-ts"; +import { INTEGRATION_HEADER } from "../constants.js"; export function createDecodoClient( token: string, @@ -6,7 +7,10 @@ export function createDecodoClient( timeoutMs?: number ): DecodoClient { return new DecodoClient({ - webScrapingApi: { token }, + webScrapingApi: { + token, + integrationHeader: INTEGRATION_HEADER, + }, schema, timeoutMs, }); diff --git a/tests/scrape/services/auth-validation.test.ts b/tests/scrape/services/auth-validation.test.ts index 8c8ca5f..3bd9080 100644 --- a/tests/scrape/services/auth-validation.test.ts +++ b/tests/scrape/services/auth-validation.test.ts @@ -29,7 +29,7 @@ describe("validateAuthToken", () => { }); expect(init.headers).toMatchObject({ Authorization: "Basic test-token", - "x-integration": "sdk-ts", // TODO: switch to cli when sdk task lands + "x-integration": "cli", }); }); }); From c102524653f11f40d25f546bcc89eeec1d2a49f2 Mon Sep 17 00:00:00 2001 From: Paulius Krutkis Date: Thu, 11 Jun 2026 11:16:25 +0300 Subject: [PATCH 3/3] Revert "Update @decodo/sdk-ts to 2.1.2 and wire integrationHeader" This reverts commit a192b63b512c8b54d522dc8674cc4aacd44a0f65. --- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- src/scrape/constants.ts | 1 + src/scrape/services/client.ts | 6 +----- tests/scrape/services/auth-validation.test.ts | 2 +- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index 422d699..d3c1f3d 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ }, "packageManager": "pnpm@10.33.3", "dependencies": { - "@decodo/sdk-ts": "^2.1.2", + "@decodo/sdk-ts": "^2.1.1", "commander": "^14.0.0", "env-paths": "^3.0.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4c43f13..348e7b7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@decodo/sdk-ts': - specifier: ^2.1.2 - version: 2.1.2 + specifier: ^2.1.1 + version: 2.1.1 commander: specifier: ^14.0.0 version: 14.0.3 @@ -104,8 +104,8 @@ packages: resolution: {integrity: sha512-S0My7XPGIgpRWMDG8uRqalbgT+a6FmCUdOW+HaIOVVpUPHOb7RrpvjTjiODadKp06fsrVDJZlIzc6yCTp4AnxA==} engines: {node: '>= 20.12.0'} - '@decodo/sdk-ts@2.1.2': - resolution: {integrity: sha512-V/SdHS0DV9L8gBJc5ljQbTmbgV7C8Cn7V91qn3JfnHKgvSmuUq2kLH27UPgCbAa0T2ZVXOkBzbIog5ZnOzAUMg==} + '@decodo/sdk-ts@2.1.1': + resolution: {integrity: sha512-1pSgG4BiPzjQEObSKahnF35wgU9Gck/aD466JwXwiNS0iRtEtcZlYpjSOuTvKV50CWuNAFZk0Ak7NHuWc4+Q7A==} engines: {node: '>=18.0.0'} '@esbuild/aix-ppc64@0.27.7': @@ -966,7 +966,7 @@ snapshots: fast-wrap-ansi: 0.2.2 sisteransi: 1.0.5 - '@decodo/sdk-ts@2.1.2': + '@decodo/sdk-ts@2.1.1': dependencies: zod: 4.4.3 diff --git a/src/scrape/constants.ts b/src/scrape/constants.ts index 5e424a1..27c9f1f 100644 --- a/src/scrape/constants.ts +++ b/src/scrape/constants.ts @@ -1,3 +1,4 @@ +// TODO(SCR-3150): switch to cli when sdk task lands export const INTEGRATION_HEADER = "cli"; export const SCHEMA_TTL_MS = 3_600_000; diff --git a/src/scrape/services/client.ts b/src/scrape/services/client.ts index 8e5d3e7..2729b90 100644 --- a/src/scrape/services/client.ts +++ b/src/scrape/services/client.ts @@ -1,5 +1,4 @@ import { DecodoClient, type DecodoSchema } from "@decodo/sdk-ts"; -import { INTEGRATION_HEADER } from "../constants.js"; export function createDecodoClient( token: string, @@ -7,10 +6,7 @@ export function createDecodoClient( timeoutMs?: number ): DecodoClient { return new DecodoClient({ - webScrapingApi: { - token, - integrationHeader: INTEGRATION_HEADER, - }, + webScrapingApi: { token }, schema, timeoutMs, }); diff --git a/tests/scrape/services/auth-validation.test.ts b/tests/scrape/services/auth-validation.test.ts index 3bd9080..8c8ca5f 100644 --- a/tests/scrape/services/auth-validation.test.ts +++ b/tests/scrape/services/auth-validation.test.ts @@ -29,7 +29,7 @@ describe("validateAuthToken", () => { }); expect(init.headers).toMatchObject({ Authorization: "Basic test-token", - "x-integration": "cli", + "x-integration": "sdk-ts", // TODO: switch to cli when sdk task lands }); }); });