Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,18 @@ decodo google-search "shoes" --geo de --parse

Use `decodo <target> --help` for all geo, locale, and target-specific options from the API schema.

### Request timeouts

Requests use the SDK default timeout of 180s. Override it globally with `--timeout <ms>` — useful to fail fast on slow targets or batch runs. A timed-out request exits with code `6`.

```bash
# Fail fast: give up after 30s
decodo scrape https://example.com --timeout 30000

# Allow longer for a heavy page render
decodo screenshot https://example.com --timeout 120000 -o shot.png
```

## Agent tooling

Coding agents (Cursor, Claude Code, Codex, Gemini CLI, Windsurf) should invoke the CLI as a **shell subprocess**, not embed scraping logic.
Expand Down
1 change: 1 addition & 0 deletions src/cli/services/global-opts.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import type { Command } from "commander";

export interface RootOptions {
timeout?: number;
token?: string;
verbose?: boolean;
}
Expand Down
11 changes: 11 additions & 0 deletions src/cli/services/parse-timeout.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import { CliUsageError } from "../../platform/services/handle-cli-error.js";

export function parseTimeout(value: string): number {
const parsed = Number.parseInt(value, 10);
if (Number.isNaN(parsed) || parsed < 1) {
throw new CliUsageError(
"--timeout must be a positive integer (milliseconds)."
);
}
return parsed;
}
6 changes: 6 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url";
import { Command } from "commander";
import { createCommands } from "./cli/register.js";
import { configureCommanderExit } from "./cli/services/configure-commander-exit.js";
import { parseTimeout } from "./cli/services/parse-timeout.js";
import { handleCliError } from "./platform/services/handle-cli-error.js";

function readVersion(): string {
Expand All @@ -27,6 +28,11 @@ const program = new Command()
.option(
"--token <token>",
"Basic auth token (overrides DECODO_AUTH_TOKEN and saved config)"
)
.option(
"--timeout <ms>",
"Per-request timeout in milliseconds (default: 180000)",
parseTimeout
);

async function main(): Promise<void> {
Expand Down
4 changes: 3 additions & 1 deletion src/scrape/services/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ import { DecodoClient, type DecodoSchema } from "@decodo/sdk-ts";

export function createDecodoClient(
token: string,
schema?: DecodoSchema
schema?: DecodoSchema,
timeoutMs?: number
): DecodoClient {
return new DecodoClient({
webScrapingApi: { token },
schema,
timeoutMs,
});
}
10 changes: 8 additions & 2 deletions src/scrape/services/run-target-scrape.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,9 @@ async function executeScrape({
outputContext,
input,
verbose = false,
timeoutMs,
}: ExecuteScrapeOptions): Promise<void> {
const client = createDecodoClient(token, schema);
const client = createDecodoClient(token, schema, timeoutMs);
const startedAt = Date.now();
const response = await client.webScrapingApi.scrape(
body as unknown as ScrapeRequest
Expand All @@ -52,6 +53,7 @@ interface ExecuteBatchOptions {
options: Record<string, unknown>;
resolveBody: ScrapeBodyBuilder;
schema: DecodoSchema;
timeoutMs?: number;
token: string;
verbose: boolean;
}
Expand All @@ -63,8 +65,9 @@ async function executeBatch({
resolveBody,
binary,
verbose,
timeoutMs,
}: ExecuteBatchOptions): Promise<void> {
const client = createDecodoClient(token, schema);
const client = createDecodoClient(token, schema, timeoutMs);
const batch = options as BatchFlags & OutputOptions;
const full = batch.full === true;

Expand Down Expand Up @@ -107,6 +110,7 @@ export function createTargetAction(
): Promise<void> => {
const rootOpts = getRootOpts(command);
const verbose = rootOpts.verbose === true;
const timeoutMs = rootOpts.timeout;

try {
const batchMode = (options as BatchFlags).inputFile !== undefined;
Expand Down Expand Up @@ -134,6 +138,7 @@ export function createTargetAction(
resolveBody,
binary: outputContext?.binary?.kind === "png",
verbose,
timeoutMs,
});
return;
}
Expand All @@ -149,6 +154,7 @@ export function createTargetAction(
outputContext,
input,
verbose,
timeoutMs,
});
} catch (err) {
handleCliError(err, { fallbackMessage: "Scrape failed." });
Expand Down
1 change: 1 addition & 0 deletions src/scrape/types/run-target-scrape.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ export interface ExecuteScrapeOptions {
options: Record<string, unknown>;
outputContext?: Partial<WriteScrapeResponseContext>;
schema: DecodoSchema;
timeoutMs?: number;
token: string;
verbose?: boolean;
}
Expand Down
13 changes: 13 additions & 0 deletions tests/cli/services/parse-timeout.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import { describe, expect, it } from "vitest";
import { parseTimeout } from "../../../src/cli/services/parse-timeout.js";
import { CliUsageError } from "../../../src/platform/services/handle-cli-error.js";

describe("parseTimeout", () => {
it("parses a positive integer of milliseconds", () => {
expect(parseTimeout("5000")).toBe(5000);
});

it.each(["0", "-1", "abc", ""])("rejects %p as a usage error", (value) => {
expect(() => parseTimeout(value)).toThrow(CliUsageError);
});
});
4 changes: 3 additions & 1 deletion tests/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,21 @@ describe("cli", () => {
expect(output).toBe(packageJson.version);
});

it("shows verbose flag in root help", () => {
it("shows verbose and timeout flags in root help", () => {
const output = execFileSync(process.execPath, [cliPath, "--help"], {
encoding: "utf8",
});

expect(output).toContain("-v, --verbose");
expect(output).toContain("--timeout <ms>");
});

it.each([
["unknown flag", ["--bad-flag"], 2],
["unknown command", ["nosuchcmd"], 2],
["missing required arg", ["search"], 2],
["invalid choice", ["search", "q", "--engine", "yahoo"], 2],
["invalid timeout", ["--timeout", "0", "scrape", "https://x.com"], 2],
])("exits with code 2 on %s", (_label, args, expectedExit) => {
const { exitCode } = runCli(args);
expect(exitCode).toBe(expectedExit);
Expand Down
35 changes: 34 additions & 1 deletion tests/scrape/services/run-target-scrape.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,11 +85,44 @@ describe("createTargetAction", () => {
});
expect(createDecodoClient).toHaveBeenCalledWith(
"test-token",
BundledSchema.shared
BundledSchema.shared,
undefined
);
expect(stdout).toBe('{"ok":true}\n');
});

it("passes the global --timeout through to the client", async () => {
const scrape = vi.fn().mockResolvedValue({
results: [{ content: { ok: true } }],
});
vi.mocked(createDecodoClient).mockReturnValue({
webScrapingApi: { scrape },
} as never);

const googleSearch = new Command("google-search")
.argument("<input>")
.action(createTargetAction("google_search", BundledSchema.shared));
attachScrapeOutputOptions(googleSearch);

const program = new Command()
.option("--token <token>")
.option("--timeout <ms>", "", (value: string) =>
Number.parseInt(value, 10)
)
.addCommand(googleSearch);

await program.parseAsync(
["google-search", "coffee", "--token", "test-token", "--timeout", "5000"],
{ from: "user" }
);

expect(createDecodoClient).toHaveBeenCalledWith(
"test-token",
BundledSchema.shared,
5000
);
});

it("prints verbose logs to stderr when --verbose is set", async () => {
const scrape = vi.fn().mockResolvedValue({
results: [{ content: { ok: true } }],
Expand Down
Loading