From fca74c1bbae4bb7a51006360aeddb1a23be4ea9b Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:53:47 +0000 Subject: [PATCH 01/10] computer: support callable backends in the exec tool The exec tool routed every call through the runtime's command path, which only carried a command string and returned drained stdout and stderr. The JavaScript backend accepts a structured input value and returns a structured result, and both command and module backends now take per-run environment variables. The tool could express none of this. Add env and input arguments to the exec tool's input schema and forward them to the runtime. Carry the backend's structured return value out on a result field, present only when the backend produced one. Mark backends callable through ExecBackendDescription so the tool can reject input for a non-callable backend before it reaches the wire, returning the same message the runtime would raise. Surface the callable backends in the tool description so the model knows which ones run their command as module source and read a value back. --- packages/computer/src/tools/ai.test.ts | 171 +++++++++++++++++++++++++ packages/computer/src/tools/exec.ts | 72 ++++++++++- 2 files changed, 236 insertions(+), 7 deletions(-) diff --git a/packages/computer/src/tools/ai.test.ts b/packages/computer/src/tools/ai.test.ts index bb60d70c..41f0ce9a 100644 --- a/packages/computer/src/tools/ai.test.ts +++ b/packages/computer/src/tools/ai.test.ts @@ -481,6 +481,177 @@ describe("createAITools exec tool", () => { }); }); +describe("createAITools callable exec", () => { + it("forwards env and input to the runtime and returns the result value", async () => { + const calls: Array<{ + command: string; + env: Record | undefined; + input: unknown; + backend: string | undefined; + }> = []; + const workspace = { + runtime: { + async exec( + command: string, + options: { + cwd?: string; + encoding: "utf8"; + backend?: string; + env?: Record; + input?: unknown; + }, + ) { + calls.push({ + command, + env: options.env, + input: options.input, + backend: options.backend, + }); + return { + result: async () => ({ + exitCode: 0, + stdout: "ran", + stderr: "", + value: { doubled: 84 }, + }), + }; + }, + }, + }; + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "js", + backends: { + js: { description: "JavaScript module runtime", callable: true }, + }, + }, + }); + + await expect( + executeTool(tools.exec, { + command: "export default (input) => ({ doubled: input.value * 2 })", + env: { API_KEY: "secret" }, + input: { value: 42 }, + }), + ).resolves.toEqual({ + command: "export default (input) => ({ doubled: input.value * 2 })", + cwd: null, + backend: "js", + exitCode: 0, + stdout: "ran", + stderr: "", + result: { doubled: 84 }, + }); + expect(calls).toEqual([ + { + command: "export default (input) => ({ doubled: input.value * 2 })", + env: { API_KEY: "secret" }, + input: { value: 42 }, + backend: "js", + }, + ]); + }); + + it("omits the result field when the backend returns no value", async () => { + const workspace = { + runtime: { + async exec() { + return { + result: async () => ({ exitCode: 0, stdout: "ok", stderr: "" }), + }; + }, + }, + }; + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "js", + backends: { js: { description: "JavaScript module runtime", callable: true } }, + }, + }); + + const output = (await executeTool(tools.exec, { command: "noop" })) as Record; + expect(output).not.toHaveProperty("result"); + expect(output).toMatchObject({ backend: "js", exitCode: 0, stdout: "ok" }); + }); + + it("errors quickly without calling the backend when input targets a non-callable backend", async () => { + let called = false; + const workspace = { + runtime: { + async exec() { + called = true; + return { result: async () => ({ exitCode: 0, stdout: "", stderr: "" }) }; + }, + }, + }; + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "shell", + backends: { + shell: { description: "fast shell" }, + js: { description: "JavaScript module runtime", callable: true }, + }, + }, + }); + + await expect( + executeTool(tools.exec, { command: "echo hi", input: { value: 1 }, backend: "shell" }), + ).resolves.toEqual({ + command: "echo hi", + cwd: null, + backend: "shell", + error: 'Backend "shell" is not callable; it does not accept structured input.', + }); + expect(called).toBe(false); + }); + + it("allows env on non-callable backends", async () => { + const calls: Array<{ env: Record | undefined; input: unknown }> = []; + const workspace = { + runtime: { + async exec(_command: string, options: { env?: Record; input?: unknown }) { + calls.push({ env: options.env, input: options.input }); + return { result: async () => ({ exitCode: 0, stdout: "", stderr: "" }) }; + }, + }, + }; + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "shell", + backends: { shell: { description: "fast shell" } }, + }, + }); + + await expect( + executeTool(tools.exec, { command: "env", env: { FOO: "bar" } }), + ).resolves.toMatchObject({ backend: "shell", exitCode: 0 }); + expect(calls).toEqual([{ env: { FOO: "bar" }, input: undefined }]); + }); + + it("describes callable backends in the tool description", () => { + const workspace = { + runtime: { + async exec() { + throw new Error("not used"); + }, + }, + }; + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "js", + backends: { js: { description: "JavaScript module runtime", callable: true } }, + }, + }); + + expect(toolDescription(tools.exec)).toContain("callable"); + }); +}); + describe("createAITools publish tool", () => { it("adds publish by default when assets are configured", async () => { const calls: Array<{ path: string; expiresAfter: number; prefix?: string }> = []; diff --git a/packages/computer/src/tools/exec.ts b/packages/computer/src/tools/exec.ts index efbb93c1..72fbc9bd 100644 --- a/packages/computer/src/tools/exec.ts +++ b/packages/computer/src/tools/exec.ts @@ -1,16 +1,25 @@ import { type Tool, tool } from "ai"; import { z } from "zod"; +import type { WorkspaceRuntimeValue } from "../runtime/types.js"; + export interface ExecWorkspaceLike { runtime: { exec( command: string, - options: { cwd?: string; encoding: "utf8"; backend?: string }, + options: { + cwd?: string; + encoding: "utf8"; + backend?: string; + env?: Record; + input?: WorkspaceRuntimeValue; + }, ): Promise<{ result(): Promise<{ exitCode: number; stdout: string; stderr: string; + value?: unknown; }>; }>; }; @@ -18,6 +27,10 @@ export interface ExecWorkspaceLike { export interface ExecBackendDescription { description: string; + // Whether the backend accepts a structured `input` value and + // returns a structured `result` value. When false or omitted the + // tool rejects `input` for this backend before touching the wire. + callable?: boolean; } export interface ExecToolOptions { @@ -29,9 +42,13 @@ export interface ExecToolOptions { const DEFAULT_MAX_BYTES = 64 * 1024; -export function createExecTool( - options: ExecToolOptions, -): Tool<{ command: string; cwd?: string; backend?: string }> { +export function createExecTool(options: ExecToolOptions): Tool<{ + command: string; + cwd?: string; + backend?: string; + env?: Record; + input?: unknown; +}> { const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; const backendIds = Object.keys(options.backends); if (backendIds.length === 0) { @@ -43,9 +60,22 @@ export function createExecTool( ); } + const callableBackendIds = new Set( + backendIds.filter((id) => options.backends[id].callable === true), + ); const backendGuidance = backendIds - .map((id) => `- ${JSON.stringify(id)}: ${options.backends[id].description}`) + .map((id) => { + const suffix = callableBackendIds.has(id) ? " (callable)" : ""; + return `- ${JSON.stringify(id)}${suffix}: ${options.backends[id].description}`; + }) .join("\n"); + const callableGuidance = + callableBackendIds.size > 0 + ? [ + "", + `Callable backends (${[...callableBackendIds].map((id) => JSON.stringify(id)).join(", ")}) run \`command\` as module source rather than a shell command. Pass \`input\` to hand the module a structured value, and read the module's returned value back from the \`result\` field. Other backends reject \`input\`.`, + ].join("\n") + : ""; const description = [ "Run a shell command in the workspace. The workspace exposes multiple backends, each with different capabilities.", "Pick the cheapest backend that can run the command; fall back to a heavier one only when the lighter backend's command set doesn't cover what you need.", @@ -55,6 +85,7 @@ export function createExecTool( "", `Default backend: ${JSON.stringify(options.defaultBackend)}. Try this first for any command you're not sure about; if it fails with a "command not found" or a similar capability error, retry on a backend whose description covers the missing tool.`, "Use for builds, test runs, typechecks, formatters, and git plumbing. Prefer the dedicated read, write, and edit tools for file operations. Long output is truncated to keep tool replies small.", + callableGuidance, ].join("\n"); const backendSchema = z @@ -71,17 +102,43 @@ export function createExecTool( return tool({ description, inputSchema: z.object({ - command: z.string().describe("Shell command, e.g. 'npm test' or 'git diff HEAD'."), + command: z + .string() + .describe( + "Shell command, e.g. 'npm test' or 'git diff HEAD'. For a callable backend this is the module source to run.", + ), cwd: z.string().optional().describe("Working directory. Defaults to the workspace root."), backend: backendSchema, + env: z + .record(z.string(), z.string()) + .optional() + .describe( + "Environment variables for this run only. Values override the backend's base environment without affecting later runs.", + ), + input: z + .unknown() + .optional() + .describe( + "Structured value handed to a callable backend's module. Only callable backends accept it; other backends reject it.", + ), }), - execute: async ({ command, cwd, backend }) => { + execute: async ({ command, cwd, backend, env, input }) => { const selectedBackend = backend ?? options.defaultBackend; + if (input !== undefined && !callableBackendIds.has(selectedBackend)) { + return { + command, + cwd: cwd ?? null, + backend: selectedBackend, + error: `Backend ${JSON.stringify(selectedBackend)} is not callable; it does not accept structured input.`, + }; + } try { const handle = await options.workspace.runtime.exec(command, { cwd, encoding: "utf8", backend: selectedBackend, + env, + input: input as WorkspaceRuntimeValue, }); const result = await handle.result(); return { @@ -91,6 +148,7 @@ export function createExecTool( exitCode: result.exitCode, stdout: truncate(result.stdout, maxBytes), stderr: truncate(result.stderr, maxBytes), + ...(result.value === undefined ? {} : { result: result.value }), }; } catch (err) { return { From e6b16b53c1e66003880933e5bafe27bd179e5e06 Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:01:07 +0000 Subject: [PATCH 02/10] computer: stream exec output from the tool as it arrives The exec tool drained the runtime handle to a single aggregate and returned one object, so the model saw a command's output only after it finished. The runtime already exposes a live event stream, and the AI SDK lets a tool's execute function yield a sequence of results. Turn execute into an async generator. When the runtime handle is async-iterable, iterate it and yield a running snapshot on each stdout or stderr chunk, then a terminal snapshot once the exit event lands; a callable backend's return value rides the final snapshot. Fall back to draining result() when the handle is not iterable, which keeps non-streaming callers working. Running snapshots carry a null exit code so consumers can tell progress from completion. Export ExecStreamEvent, ExecRuntimeHandle, and ExecToolOutput for consumers that build handles or handle the streamed output. --- packages/computer/src/tools/ai.test.ts | 186 ++++++++++++++++++++++++- packages/computer/src/tools/exec.ts | 145 ++++++++++++++----- packages/computer/src/tools/index.ts | 9 +- 3 files changed, 307 insertions(+), 33 deletions(-) diff --git a/packages/computer/src/tools/ai.test.ts b/packages/computer/src/tools/ai.test.ts index 41f0ce9a..45c60c28 100644 --- a/packages/computer/src/tools/ai.test.ts +++ b/packages/computer/src/tools/ai.test.ts @@ -17,7 +17,43 @@ async function executeTool(tool: unknown, input: unknown): Promise { const execute = (tool as { execute?: (input: unknown, options: typeof toolOptions) => unknown }) .execute; if (!execute) throw new Error("tool has no execute function"); - return await execute(input, toolOptions); + const output = await execute(input, toolOptions); + if (output && typeof output === "object" && Symbol.asyncIterator in output) { + let last: unknown; + for await (const chunk of output as AsyncIterable) last = chunk; + return last; + } + return output; +} + +async function collectTool(tool: unknown, input: unknown): Promise { + const execute = (tool as { execute?: (input: unknown, options: typeof toolOptions) => unknown }) + .execute; + if (!execute) throw new Error("tool has no execute function"); + const output = await execute(input, toolOptions); + if (!output || typeof output !== "object" || !(Symbol.asyncIterator in output)) { + return [output]; + } + const chunks: unknown[] = []; + for await (const chunk of output as AsyncIterable) chunks.push(chunk); + return chunks; +} + +type ExecStreamEvent = + | { name: "stdout"; value: string } + | { name: "stderr"; value: string } + | { name: "result"; value: unknown } + | { name: "exit"; value: number }; + +function streamingHandle(events: ExecStreamEvent[]) { + return { + async *[Symbol.asyncIterator]() { + for (const event of events) yield event; + }, + result: async () => { + throw new Error("result() must not be called on a streamed handle"); + }, + }; } function toolDescription(tool: unknown): string { @@ -652,6 +688,154 @@ describe("createAITools callable exec", () => { }); }); +describe("createAITools exec streaming", () => { + it("streams stdout and stderr chunks and yields a final aggregate", async () => { + const workspace = { + runtime: { + async exec() { + return streamingHandle([ + { name: "stdout", value: "one\n" }, + { name: "stderr", value: "warn\n" }, + { name: "stdout", value: "two\n" }, + { name: "exit", value: 0 }, + ]); + }, + }, + }; + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "shell", + backends: { shell: { description: "fast shell" } }, + }, + }); + + await expect(collectTool(tools.exec, { command: "run" })).resolves.toEqual([ + { command: "run", cwd: null, backend: "shell", exitCode: null, stdout: "one\n", stderr: "" }, + { + command: "run", + cwd: null, + backend: "shell", + exitCode: null, + stdout: "one\n", + stderr: "warn\n", + }, + { + command: "run", + cwd: null, + backend: "shell", + exitCode: null, + stdout: "one\ntwo\n", + stderr: "warn\n", + }, + { + command: "run", + cwd: null, + backend: "shell", + exitCode: 0, + stdout: "one\ntwo\n", + stderr: "warn\n", + }, + ]); + }); + + it("streams a callable backend's result value on the final chunk", async () => { + const workspace = { + runtime: { + async exec() { + return streamingHandle([ + { name: "stdout", value: "working\n" }, + { name: "result", value: { ok: true } }, + { name: "exit", value: 0 }, + ]); + }, + }, + }; + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "js", + backends: { js: { description: "JavaScript module runtime", callable: true } }, + }, + }); + + const chunks = await collectTool(tools.exec, { + command: "export default () => ({ ok: true })", + input: {}, + }); + expect(chunks).toHaveLength(2); + expect(chunks.at(-1)).toEqual({ + command: "export default () => ({ ok: true })", + cwd: null, + backend: "js", + exitCode: 0, + stdout: "working\n", + stderr: "", + result: { ok: true }, + }); + }); + + it("truncates streamed output on UTF-8 byte boundaries", async () => { + const workspace = { + runtime: { + async exec() { + return streamingHandle([ + { name: "stdout", value: "a\u{1f642}b" }, + { name: "exit", value: 0 }, + ]); + }, + }, + }; + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "shell", + backends: { shell: { description: "fast shell" } }, + maxBytes: 5, + }, + }); + + const chunks = await collectTool(tools.exec, { command: "echo emoji" }); + expect(chunks.at(-1)).toMatchObject({ + exitCode: 0, + stdout: "a\u{1f642}\n\n[truncated, 1 more bytes]", + }); + }); + + it("yields a structured error when the stream fails mid-run", async () => { + const workspace = { + runtime: { + async exec() { + return { + async *[Symbol.asyncIterator]() { + yield { name: "stdout", value: "partial\n" } as ExecStreamEvent; + throw new Error("stream broke"); + }, + result: async () => { + throw new Error("result() must not be called on a streamed handle"); + }, + }; + }, + }, + }; + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "shell", + backends: { shell: { description: "fast shell" } }, + }, + }); + + const chunks = await collectTool(tools.exec, { command: "run" }); + expect(chunks.at(-1)).toEqual({ + command: "run", + cwd: null, + backend: "shell", + error: "stream broke", + }); + }); +}); + describe("createAITools publish tool", () => { it("adds publish by default when assets are configured", async () => { const calls: Array<{ path: string; expiresAfter: number; prefix?: string }> = []; diff --git a/packages/computer/src/tools/exec.ts b/packages/computer/src/tools/exec.ts index 72fbc9bd..33c760ad 100644 --- a/packages/computer/src/tools/exec.ts +++ b/packages/computer/src/tools/exec.ts @@ -3,6 +3,27 @@ import { z } from "zod"; import type { WorkspaceRuntimeValue } from "../runtime/types.js"; +// One event drained from a running execution. stdout / stderr carry +// output chunks as they arrive; result carries a callable backend's +// structured return value; exit carries the process exit code. +export type ExecStreamEvent = + | { name: "stdout"; value: string } + | { name: "stderr"; value: string } + | { name: "result"; value: unknown } + | { name: "exit"; value: number }; + +// A detached execution handle. The tool streams stdout / stderr +// chunks by iterating the handle when it is async-iterable, and +// falls back to draining result() when it is not. +export interface ExecRuntimeHandle extends Partial> { + result(): Promise<{ + exitCode: number; + stdout: string; + stderr: string; + value?: unknown; + }>; +} + export interface ExecWorkspaceLike { runtime: { exec( @@ -14,14 +35,7 @@ export interface ExecWorkspaceLike { env?: Record; input?: WorkspaceRuntimeValue; }, - ): Promise<{ - result(): Promise<{ - exitCode: number; - stdout: string; - stderr: string; - value?: unknown; - }>; - }>; + ): Promise; }; } @@ -42,13 +56,32 @@ export interface ExecToolOptions { const DEFAULT_MAX_BYTES = 64 * 1024; -export function createExecTool(options: ExecToolOptions): Tool<{ - command: string; - cwd?: string; - backend?: string; - env?: Record; - input?: unknown; -}> { +// Progressive snapshot emitted while a command streams (exitCode +// null until the run ends), and the terminal snapshot once the exit +// code lands. `result` appears only when a callable backend returned +// a value; `error` replaces the run fields when the exec fails. +export type ExecToolOutput = + | { + command: string; + cwd: string | null; + backend: string; + exitCode: number | null; + stdout: string; + stderr: string; + result?: unknown; + } + | { command: string; cwd: string | null; backend: string; error: string }; + +export function createExecTool(options: ExecToolOptions): Tool< + { + command: string; + cwd?: string; + backend?: string; + env?: Record; + input?: unknown; + }, + ExecToolOutput +> { const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; const backendIds = Object.keys(options.backends); if (backendIds.length === 0) { @@ -122,46 +155,96 @@ export function createExecTool(options: ExecToolOptions): Tool<{ "Structured value handed to a callable backend's module. Only callable backends accept it; other backends reject it.", ), }), - execute: async ({ command, cwd, backend, env, input }) => { + execute: async function* ({ command, cwd, backend, env, input }) { const selectedBackend = backend ?? options.defaultBackend; + const base = { command, cwd: cwd ?? null, backend: selectedBackend }; if (input !== undefined && !callableBackendIds.has(selectedBackend)) { - return { - command, - cwd: cwd ?? null, - backend: selectedBackend, + yield { + ...base, error: `Backend ${JSON.stringify(selectedBackend)} is not callable; it does not accept structured input.`, }; + return; } + let handle: ExecRuntimeHandle; try { - const handle = await options.workspace.runtime.exec(command, { + handle = await options.workspace.runtime.exec(command, { cwd, encoding: "utf8", backend: selectedBackend, env, input: input as WorkspaceRuntimeValue, }); + } catch (err) { + yield { ...base, error: errorMessage(err) }; + return; + } + + // Stream stdout / stderr chunks as they arrive when the handle + // is iterable. Each chunk yields a fresh snapshot with the + // running output so the model sees progress before the run + // ends; the exit event settles the terminal snapshot. + if (typeof handle[Symbol.asyncIterator] === "function") { + let stdout = ""; + let stderr = ""; + let exitCode: number | null = null; + let value: unknown; + let hasValue = false; + try { + for await (const event of handle as AsyncIterable) { + if (event.name === "stdout") stdout += event.value; + else if (event.name === "stderr") stderr += event.value; + else if (event.name === "result") { + value = event.value; + hasValue = true; + continue; + } else { + exitCode = event.value; + continue; + } + // A stdout / stderr chunk arrived: emit a running snapshot + // so the model sees output before the run ends. + yield { + ...base, + exitCode: null, + stdout: truncate(stdout, maxBytes), + stderr: truncate(stderr, maxBytes), + }; + } + } catch (err) { + yield { ...base, error: errorMessage(err) }; + return; + } + yield { + ...base, + exitCode, + stdout: truncate(stdout, maxBytes), + stderr: truncate(stderr, maxBytes), + ...(hasValue ? { result: value } : {}), + }; + return; + } + + // Non-streaming handle: drain the aggregate result. + try { const result = await handle.result(); - return { - command, - cwd: cwd ?? null, - backend: selectedBackend, + yield { + ...base, exitCode: result.exitCode, stdout: truncate(result.stdout, maxBytes), stderr: truncate(result.stderr, maxBytes), ...(result.value === undefined ? {} : { result: result.value }), }; } catch (err) { - return { - command, - cwd: cwd ?? null, - backend: selectedBackend, - error: err instanceof Error ? err.message : String(err), - }; + yield { ...base, error: errorMessage(err) }; } }, }); } +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + const encoder = new TextEncoder(); function truncate(value: string, maxBytes: number): string { diff --git a/packages/computer/src/tools/index.ts b/packages/computer/src/tools/index.ts index 45dc297c..b2686a55 100644 --- a/packages/computer/src/tools/index.ts +++ b/packages/computer/src/tools/index.ts @@ -1,5 +1,12 @@ export { type CreateAIToolsOptions, createAITools } from "./ai.js"; -export { createExecTool, type ExecBackendDescription, type ExecToolOptions } from "./exec.js"; +export { + createExecTool, + type ExecBackendDescription, + type ExecRuntimeHandle, + type ExecStreamEvent, + type ExecToolOptions, + type ExecToolOutput, +} from "./exec.js"; export { createEditTool, type EditToolOptions } from "./fs/edit.js"; export { createListTool, type ListToolOptions } from "./fs/list.js"; export { createReadTool, type ReadToolOptions } from "./fs/read.js"; From d27d9d101195eae233ecede7f03a769088a51322 Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:18:02 +0000 Subject: [PATCH 03/10] computer: fold the callable result onto the exit event The streamed event model carried a callable backend's return value on its own result event, separate from the exit event that reports the process exit code. The two settle at the same instant: the runtime only emits a result when the run exits zero, and it emits the two together as one terminal batch. Splitting them across two events forced a consumer to collect both and correlate them to learn the outcome of a single run. Fold the value onto the exit event, where it belongs alongside the exit code. The tool still accepts the standalone result event so it keeps working against the current wire, which has not yet been consolidated; consuming both settles the value the same way. Once the wire carries the value on its exit frame the standalone event can go. --- packages/computer/src/tools/ai.test.ts | 37 +++++++++++++++++++++++++- packages/computer/src/tools/exec.ts | 16 ++++++++--- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/packages/computer/src/tools/ai.test.ts b/packages/computer/src/tools/ai.test.ts index 45c60c28..2127c031 100644 --- a/packages/computer/src/tools/ai.test.ts +++ b/packages/computer/src/tools/ai.test.ts @@ -43,7 +43,7 @@ type ExecStreamEvent = | { name: "stdout"; value: string } | { name: "stderr"; value: string } | { name: "result"; value: unknown } - | { name: "exit"; value: number }; + | { name: "exit"; value: number; result?: unknown }; function streamingHandle(events: ExecStreamEvent[]) { return { @@ -775,6 +775,41 @@ describe("createAITools exec streaming", () => { }); }); + it("reads a callable backend's result folded onto the exit event", async () => { + const workspace = { + runtime: { + async exec() { + return streamingHandle([ + { name: "stdout", value: "working\n" }, + { name: "exit", value: 0, result: { ok: true } }, + ]); + }, + }, + }; + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "js", + backends: { js: { description: "JavaScript module runtime", callable: true } }, + }, + }); + + const chunks = await collectTool(tools.exec, { + command: "export default () => ({ ok: true })", + input: {}, + }); + expect(chunks).toHaveLength(2); + expect(chunks.at(-1)).toEqual({ + command: "export default () => ({ ok: true })", + cwd: null, + backend: "js", + exitCode: 0, + stdout: "working\n", + stderr: "", + result: { ok: true }, + }); + }); + it("truncates streamed output on UTF-8 byte boundaries", async () => { const workspace = { runtime: { diff --git a/packages/computer/src/tools/exec.ts b/packages/computer/src/tools/exec.ts index 33c760ad..251d942a 100644 --- a/packages/computer/src/tools/exec.ts +++ b/packages/computer/src/tools/exec.ts @@ -4,13 +4,19 @@ import { z } from "zod"; import type { WorkspaceRuntimeValue } from "../runtime/types.js"; // One event drained from a running execution. stdout / stderr carry -// output chunks as they arrive; result carries a callable backend's -// structured return value; exit carries the process exit code. +// output chunks as they arrive; exit carries the process exit code and, +// for a callable backend, the structured return value on `result`. The +// value settles at the same instant as the exit code, so it rides the +// same terminal event rather than a separate one. +// +// The standalone `result` event is the shape the runtime wire still +// emits today. The tool accepts it so it keeps working until the wire is +// consolidated, but callers should read the value from the exit event. export type ExecStreamEvent = | { name: "stdout"; value: string } | { name: "stderr"; value: string } | { name: "result"; value: unknown } - | { name: "exit"; value: number }; + | { name: "exit"; value: number; result?: unknown }; // A detached execution handle. The tool streams stdout / stderr // chunks by iterating the handle when it is async-iterable, and @@ -199,6 +205,10 @@ export function createExecTool(options: ExecToolOptions): Tool< continue; } else { exitCode = event.value; + if ("result" in event) { + value = event.result; + hasValue = true; + } continue; } // A stdout / stderr chunk arrived: emit a running snapshot From bf1cd3eb46630e80ef3088c0441e687c0caf2290 Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:00:06 +0000 Subject: [PATCH 04/10] computer: read callable state from the runtime, not tool options The exec tool learned which backends were callable from a `callable` flag on each backend description the caller passed to createAITools. That duplicated a fact the Workspace already holds: it derives the callable set from each backend's own `callable` flag at construction. Declaring it a second time on the tool let the two drift. Expose `isCallable(id)` on the runtime and have the tool ask it, dropping `callable` from the backend description. The runtime answers from the same set that guards its own structured-input check, so the tool and the runtime can no longer disagree about which backends accept input. --- packages/computer/src/runtime/runtime.ts | 10 +++++++++- packages/computer/src/tools/ai.test.ts | 18 ++++++++++++------ packages/computer/src/tools/exec.ts | 14 +++++++------- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/packages/computer/src/runtime/runtime.ts b/packages/computer/src/runtime/runtime.ts index 9086b2c9..3a7a61f8 100644 --- a/packages/computer/src/runtime/runtime.ts +++ b/packages/computer/src/runtime/runtime.ts @@ -34,6 +34,14 @@ export class WorkspaceRuntime { this.#options = options; } + // Whether the named backend accepts a structured `input` value and + // returns a structured result. Consumers such as the exec tool ask + // this to know whether a backend is callable without the caller + // having to declare it a second time. + isCallable(id: string): boolean { + return this.#options.callableBackendIds.has(id); + } + exec(source: string): Promise>; exec( source: string, @@ -49,7 +57,7 @@ export class WorkspaceRuntime { ): Promise> { if (options.id !== undefined) assertExecutionId(options.id); const backend = this.#backend(options.backend); - if (options.input !== undefined && !this.#options.callableBackendIds.has(backend)) { + if (options.input !== undefined && !this.isCallable(backend)) { throw new Error(notCallableMessage(backend)); } const runtime = await this.#options.backendHandle(backend); diff --git a/packages/computer/src/tools/ai.test.ts b/packages/computer/src/tools/ai.test.ts index 2127c031..1a0cb7ab 100644 --- a/packages/computer/src/tools/ai.test.ts +++ b/packages/computer/src/tools/ai.test.ts @@ -552,6 +552,7 @@ describe("createAITools callable exec", () => { }), }; }, + isCallable: (id: string) => id === "js", }, }; const tools = createAITools({ @@ -559,7 +560,7 @@ describe("createAITools callable exec", () => { shell: { defaultBackend: "js", backends: { - js: { description: "JavaScript module runtime", callable: true }, + js: { description: "JavaScript module runtime" }, }, }, }); @@ -597,13 +598,14 @@ describe("createAITools callable exec", () => { result: async () => ({ exitCode: 0, stdout: "ok", stderr: "" }), }; }, + isCallable: (id: string) => id === "js", }, }; const tools = createAITools({ workspace, shell: { defaultBackend: "js", - backends: { js: { description: "JavaScript module runtime", callable: true } }, + backends: { js: { description: "JavaScript module runtime" } }, }, }); @@ -620,6 +622,7 @@ describe("createAITools callable exec", () => { called = true; return { result: async () => ({ exitCode: 0, stdout: "", stderr: "" }) }; }, + isCallable: (id: string) => id === "js", }, }; const tools = createAITools({ @@ -628,7 +631,7 @@ describe("createAITools callable exec", () => { defaultBackend: "shell", backends: { shell: { description: "fast shell" }, - js: { description: "JavaScript module runtime", callable: true }, + js: { description: "JavaScript module runtime" }, }, }, }); @@ -674,13 +677,14 @@ describe("createAITools callable exec", () => { async exec() { throw new Error("not used"); }, + isCallable: (id: string) => id === "js", }, }; const tools = createAITools({ workspace, shell: { defaultBackend: "js", - backends: { js: { description: "JavaScript module runtime", callable: true } }, + backends: { js: { description: "JavaScript module runtime" } }, }, }); @@ -749,13 +753,14 @@ describe("createAITools exec streaming", () => { { name: "exit", value: 0 }, ]); }, + isCallable: (id: string) => id === "js", }, }; const tools = createAITools({ workspace, shell: { defaultBackend: "js", - backends: { js: { description: "JavaScript module runtime", callable: true } }, + backends: { js: { description: "JavaScript module runtime" } }, }, }); @@ -784,13 +789,14 @@ describe("createAITools exec streaming", () => { { name: "exit", value: 0, result: { ok: true } }, ]); }, + isCallable: (id: string) => id === "js", }, }; const tools = createAITools({ workspace, shell: { defaultBackend: "js", - backends: { js: { description: "JavaScript module runtime", callable: true } }, + backends: { js: { description: "JavaScript module runtime" } }, }, }); diff --git a/packages/computer/src/tools/exec.ts b/packages/computer/src/tools/exec.ts index 251d942a..22db305f 100644 --- a/packages/computer/src/tools/exec.ts +++ b/packages/computer/src/tools/exec.ts @@ -42,15 +42,16 @@ export interface ExecWorkspaceLike { input?: WorkspaceRuntimeValue; }, ): Promise; + // Whether a backend accepts a structured `input` value and returns + // a structured result. The tool asks this to know which backends + // are callable; the runtime derives it from each backend's + // `callable` flag. Omit when no backend is callable. + isCallable?(id: string): boolean; }; } export interface ExecBackendDescription { description: string; - // Whether the backend accepts a structured `input` value and - // returns a structured `result` value. When false or omitted the - // tool rejects `input` for this backend before touching the wire. - callable?: boolean; } export interface ExecToolOptions { @@ -99,9 +100,8 @@ export function createExecTool(options: ExecToolOptions): Tool< ); } - const callableBackendIds = new Set( - backendIds.filter((id) => options.backends[id].callable === true), - ); + const isCallable = options.workspace.runtime.isCallable?.bind(options.workspace.runtime); + const callableBackendIds = new Set(backendIds.filter((id) => isCallable?.(id) === true)); const backendGuidance = backendIds .map((id) => { const suffix = callableBackendIds.has(id) ? " (callable)" : ""; From e8da151e37dd55a6d1c74cf8609eda322d930767 Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:22:58 +0000 Subject: [PATCH 05/10] computer: drop the standalone result stream event The exec tool's ExecStreamEvent carried a standalone result variant alongside the exit event. The runtime folds a callable backend's return value onto its exit event, so no handle the tool consumes emits a separate result event; the variant and the loop branch that read it were dead while still exported for callers to build against. Drop the variant, read the value only from the exit event, and fold the two result tests into one that reflects the wire. --- packages/computer/src/tools/ai.test.ts | 40 +------------------------- packages/computer/src/tools/exec.ts | 11 +------ 2 files changed, 2 insertions(+), 49 deletions(-) diff --git a/packages/computer/src/tools/ai.test.ts b/packages/computer/src/tools/ai.test.ts index 1a0cb7ab..1c02784f 100644 --- a/packages/computer/src/tools/ai.test.ts +++ b/packages/computer/src/tools/ai.test.ts @@ -42,7 +42,6 @@ async function collectTool(tool: unknown, input: unknown): Promise { type ExecStreamEvent = | { name: "stdout"; value: string } | { name: "stderr"; value: string } - | { name: "result"; value: unknown } | { name: "exit"; value: number; result?: unknown }; function streamingHandle(events: ExecStreamEvent[]) { @@ -743,44 +742,7 @@ describe("createAITools exec streaming", () => { ]); }); - it("streams a callable backend's result value on the final chunk", async () => { - const workspace = { - runtime: { - async exec() { - return streamingHandle([ - { name: "stdout", value: "working\n" }, - { name: "result", value: { ok: true } }, - { name: "exit", value: 0 }, - ]); - }, - isCallable: (id: string) => id === "js", - }, - }; - const tools = createAITools({ - workspace, - shell: { - defaultBackend: "js", - backends: { js: { description: "JavaScript module runtime" } }, - }, - }); - - const chunks = await collectTool(tools.exec, { - command: "export default () => ({ ok: true })", - input: {}, - }); - expect(chunks).toHaveLength(2); - expect(chunks.at(-1)).toEqual({ - command: "export default () => ({ ok: true })", - cwd: null, - backend: "js", - exitCode: 0, - stdout: "working\n", - stderr: "", - result: { ok: true }, - }); - }); - - it("reads a callable backend's result folded onto the exit event", async () => { + it("streams a callable backend's result folded onto the exit event", async () => { const workspace = { runtime: { async exec() { diff --git a/packages/computer/src/tools/exec.ts b/packages/computer/src/tools/exec.ts index 22db305f..e781946c 100644 --- a/packages/computer/src/tools/exec.ts +++ b/packages/computer/src/tools/exec.ts @@ -8,14 +8,9 @@ import type { WorkspaceRuntimeValue } from "../runtime/types.js"; // for a callable backend, the structured return value on `result`. The // value settles at the same instant as the exit code, so it rides the // same terminal event rather than a separate one. -// -// The standalone `result` event is the shape the runtime wire still -// emits today. The tool accepts it so it keeps working until the wire is -// consolidated, but callers should read the value from the exit event. export type ExecStreamEvent = | { name: "stdout"; value: string } | { name: "stderr"; value: string } - | { name: "result"; value: unknown } | { name: "exit"; value: number; result?: unknown }; // A detached execution handle. The tool streams stdout / stderr @@ -199,11 +194,7 @@ export function createExecTool(options: ExecToolOptions): Tool< for await (const event of handle as AsyncIterable) { if (event.name === "stdout") stdout += event.value; else if (event.name === "stderr") stderr += event.value; - else if (event.name === "result") { - value = event.value; - hasValue = true; - continue; - } else { + else { exitCode = event.value; if ("result" in event) { value = event.result; From 4797e682316205b134b6f8f0ebc072d149fa6502 Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:36:14 +0000 Subject: [PATCH 06/10] computer: bound and coalesce the exec tool's streamed output Streaming yielded a snapshot of the whole accumulated output on every chunk, and each snapshot re-encoded that whole buffer to enforce the display cap. A chatty command was quadratic in bytes encoded and in bytes pushed into the model stream, and the accumulator grew without bound regardless of the cap. Coalesce running snapshots to at most one per 100ms, so a chunk-heavy command yields on a wall-clock floor instead of once per chunk; the terminal snapshot always fires. Accumulate each stream through a bounded buffer that counts total bytes incrementally and retains at most streamMaxBytes (512 KiB) of head text, so the truncation marker still reports every byte seen while memory stays bounded and the per-snapshot render no longer re-encodes the entire buffer. The coalescing clock is injectable for deterministic tests. --- packages/computer/src/tools/ai.test.ts | 75 ++++++++++++++++++++ packages/computer/src/tools/exec.ts | 95 +++++++++++++++++++++++--- 2 files changed, 160 insertions(+), 10 deletions(-) diff --git a/packages/computer/src/tools/ai.test.ts b/packages/computer/src/tools/ai.test.ts index 1c02784f..1734c474 100644 --- a/packages/computer/src/tools/ai.test.ts +++ b/packages/computer/src/tools/ai.test.ts @@ -55,6 +55,17 @@ function streamingHandle(events: ExecStreamEvent[]) { }; } +// A clock that advances 200ms per read, past the 100ms coalescing +// floor, so every chunk produces its own running snapshot. Streaming +// tests that assert per-chunk output inject this to defeat coalescing. +function steppingClock(stepMs = 200): () => number { + let t = 0; + return () => { + t += stepMs; + return t; + }; +} + function toolDescription(tool: unknown): string { const description = (tool as { description?: unknown }).description; if (typeof description !== "string") throw new Error("tool has no description"); @@ -710,6 +721,7 @@ describe("createAITools exec streaming", () => { shell: { defaultBackend: "shell", backends: { shell: { description: "fast shell" } }, + now: steppingClock(), }, }); @@ -837,6 +849,69 @@ describe("createAITools exec streaming", () => { error: "stream broke", }); }); + + it("coalesces running snapshots to at most one per interval", async () => { + const workspace = { + runtime: { + async exec() { + return streamingHandle([ + { name: "stdout", value: "a" }, + { name: "stdout", value: "b" }, + { name: "stdout", value: "c" }, + { name: "exit", value: 0 }, + ]); + }, + }, + }; + // A frozen clock never advances past the coalescing floor. The + // first chunk still yields a running snapshot for responsiveness; + // the later chunks coalesce into the terminal snapshot. + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "shell", + backends: { shell: { description: "fast shell" } }, + now: () => 1000, + }, + }); + + const chunks = await collectTool(tools.exec, { command: "run" }); + expect(chunks).toEqual([ + { command: "run", cwd: null, backend: "shell", exitCode: null, stdout: "a", stderr: "" }, + { command: "run", cwd: null, backend: "shell", exitCode: 0, stdout: "abc", stderr: "" }, + ]); + }); + + it("caps streamed output in memory at streamMaxBytes", async () => { + const workspace = { + runtime: { + async exec() { + return streamingHandle([ + { name: "stdout", value: "a".repeat(10) }, + { name: "stdout", value: "b".repeat(10) }, + { name: "exit", value: 0 }, + ]); + }, + }, + }; + // Hold at most 12 bytes; show at most 8. The marker counts every + // byte seen (20), not just the 12 retained. + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "shell", + backends: { shell: { description: "fast shell" } }, + maxBytes: 8, + streamMaxBytes: 12, + }, + }); + + const chunks = await collectTool(tools.exec, { command: "run" }); + expect(chunks.at(-1)).toMatchObject({ + exitCode: 0, + stdout: "aaaaaaaa\n\n[truncated, 12 more bytes]", + }); + }); }); describe("createAITools publish tool", () => { diff --git a/packages/computer/src/tools/exec.ts b/packages/computer/src/tools/exec.ts index e781946c..96fa03f1 100644 --- a/packages/computer/src/tools/exec.ts +++ b/packages/computer/src/tools/exec.ts @@ -53,10 +53,24 @@ export interface ExecToolOptions { workspace: ExecWorkspaceLike; backends: Record; defaultBackend: string; + // Per-snapshot display cap for each of stdout and stderr, in bytes. + // Output past it is shown as a truncation marker. Defaults to 64 KiB. maxBytes?: number; + // In-memory cap per stream while streaming, in bytes. Output past it + // is counted toward the truncation marker but not retained, so a long + // run does not grow the buffer without bound. Defaults to 512 KiB. + streamMaxBytes?: number; + // Clock backing the running-snapshot coalescing floor. Defaults to + // Date.now; injectable so tests can drive the interval deterministically. + now?: () => number; } const DEFAULT_MAX_BYTES = 64 * 1024; +const DEFAULT_STREAM_MAX_BYTES = 512 * 1024; +// Minimum wall-clock gap between running snapshots. A chatty command +// yields at most one snapshot per interval instead of one per chunk; +// the terminal snapshot always fires regardless. +const STREAM_COALESCE_MS = 100; // Progressive snapshot emitted while a command streams (exitCode // null until the run ends), and the terminal snapshot once the exit @@ -85,6 +99,8 @@ export function createExecTool(options: ExecToolOptions): Tool< ExecToolOutput > { const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; + const streamMaxBytes = options.streamMaxBytes ?? DEFAULT_STREAM_MAX_BYTES; + const now = options.now ?? Date.now; const backendIds = Object.keys(options.backends); if (backendIds.length === 0) { throw new Error("createExecTool: pass at least one backend in `backends`"); @@ -185,15 +201,19 @@ export function createExecTool(options: ExecToolOptions): Tool< // running output so the model sees progress before the run // ends; the exit event settles the terminal snapshot. if (typeof handle[Symbol.asyncIterator] === "function") { - let stdout = ""; - let stderr = ""; + const stdout = new StreamBuffer(streamMaxBytes); + const stderr = new StreamBuffer(streamMaxBytes); let exitCode: number | null = null; let value: unknown; let hasValue = false; + // Coalesce running snapshots to at most one per interval. A + // chatty command would otherwise yield a full-buffer snapshot + // per chunk; the terminal snapshot below always fires. + let lastSnapshot = 0; try { for await (const event of handle as AsyncIterable) { - if (event.name === "stdout") stdout += event.value; - else if (event.name === "stderr") stderr += event.value; + if (event.name === "stdout") stdout.push(event.value); + else if (event.name === "stderr") stderr.push(event.value); else { exitCode = event.value; if ("result" in event) { @@ -202,13 +222,14 @@ export function createExecTool(options: ExecToolOptions): Tool< } continue; } - // A stdout / stderr chunk arrived: emit a running snapshot - // so the model sees output before the run ends. + const at = now(); + if (at - lastSnapshot < STREAM_COALESCE_MS) continue; + lastSnapshot = at; yield { ...base, exitCode: null, - stdout: truncate(stdout, maxBytes), - stderr: truncate(stderr, maxBytes), + stdout: stdout.render(maxBytes), + stderr: stderr.render(maxBytes), }; } } catch (err) { @@ -218,8 +239,8 @@ export function createExecTool(options: ExecToolOptions): Tool< yield { ...base, exitCode, - stdout: truncate(stdout, maxBytes), - stderr: truncate(stderr, maxBytes), + stdout: stdout.render(maxBytes), + stderr: stderr.render(maxBytes), ...(hasValue ? { result: value } : {}), }; return; @@ -248,6 +269,60 @@ function errorMessage(err: unknown): string { const encoder = new TextEncoder(); +// A bounded, incrementally-counted accumulator for one output stream. +// It keeps at most `cap` bytes of head text in memory while tracking +// the total bytes seen, so a long run neither grows without bound nor +// re-encodes the whole buffer on every snapshot. `render` returns the +// display-capped view with a marker for the bytes not shown. +class StreamBuffer { + #head = ""; + #headBytes = 0; + #totalBytes = 0; + readonly #cap: number; + + constructor(cap: number) { + this.#cap = cap; + } + + push(chunk: string): void { + const chunkBytes = encoder.encode(chunk).byteLength; + this.#totalBytes += chunkBytes; + if (this.#headBytes >= this.#cap) return; + if (this.#headBytes + chunkBytes <= this.#cap) { + this.#head += chunk; + this.#headBytes += chunkBytes; + return; + } + // The chunk crosses the cap: keep the largest whole-character + // prefix that fits, then stop growing the head. + let used = this.#headBytes; + let end = 0; + for (const char of chunk) { + const charBytes = encoder.encode(char).byteLength; + if (used + charBytes > this.#cap) break; + used += charBytes; + end += char.length; + } + this.#head += chunk.slice(0, end); + this.#headBytes = used; + } + + render(maxBytes: number): string { + if (this.#totalBytes <= maxBytes && this.#totalBytes === this.#headBytes) { + return this.#head; + } + let used = 0; + let end = 0; + for (const char of this.#head) { + const charBytes = encoder.encode(char).byteLength; + if (used + charBytes > maxBytes) break; + used += charBytes; + end += char.length; + } + return `${this.#head.slice(0, end)}\n\n[truncated, ${this.#totalBytes - used} more bytes]`; + } +} + function truncate(value: string, maxBytes: number): string { if (!value) return value; const totalBytes = encoder.encode(value).byteLength; From fd46bb503efada2d7a92e0fcd8bbf92281b8e754 Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:08:58 +0000 Subject: [PATCH 07/10] computer: name the exit code field in the exec tool's stream event The runtime's exit event carries its process exit code on `code`. The tool's ExecStreamEvent still read it from `value`, so a streamed exit resolved its code to undefined. Rename the exit variant's payload to `code` and read it there, matching the event the tool consumes. --- packages/computer/src/tools/ai.test.ts | 12 ++++++------ packages/computer/src/tools/exec.ts | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/computer/src/tools/ai.test.ts b/packages/computer/src/tools/ai.test.ts index 1734c474..f16530d8 100644 --- a/packages/computer/src/tools/ai.test.ts +++ b/packages/computer/src/tools/ai.test.ts @@ -42,7 +42,7 @@ async function collectTool(tool: unknown, input: unknown): Promise { type ExecStreamEvent = | { name: "stdout"; value: string } | { name: "stderr"; value: string } - | { name: "exit"; value: number; result?: unknown }; + | { name: "exit"; code: number; result?: unknown }; function streamingHandle(events: ExecStreamEvent[]) { return { @@ -711,7 +711,7 @@ describe("createAITools exec streaming", () => { { name: "stdout", value: "one\n" }, { name: "stderr", value: "warn\n" }, { name: "stdout", value: "two\n" }, - { name: "exit", value: 0 }, + { name: "exit", code: 0 }, ]); }, }, @@ -760,7 +760,7 @@ describe("createAITools exec streaming", () => { async exec() { return streamingHandle([ { name: "stdout", value: "working\n" }, - { name: "exit", value: 0, result: { ok: true } }, + { name: "exit", code: 0, result: { ok: true } }, ]); }, isCallable: (id: string) => id === "js", @@ -796,7 +796,7 @@ describe("createAITools exec streaming", () => { async exec() { return streamingHandle([ { name: "stdout", value: "a\u{1f642}b" }, - { name: "exit", value: 0 }, + { name: "exit", code: 0 }, ]); }, }, @@ -858,7 +858,7 @@ describe("createAITools exec streaming", () => { { name: "stdout", value: "a" }, { name: "stdout", value: "b" }, { name: "stdout", value: "c" }, - { name: "exit", value: 0 }, + { name: "exit", code: 0 }, ]); }, }, @@ -889,7 +889,7 @@ describe("createAITools exec streaming", () => { return streamingHandle([ { name: "stdout", value: "a".repeat(10) }, { name: "stdout", value: "b".repeat(10) }, - { name: "exit", value: 0 }, + { name: "exit", code: 0 }, ]); }, }, diff --git a/packages/computer/src/tools/exec.ts b/packages/computer/src/tools/exec.ts index 96fa03f1..514b2653 100644 --- a/packages/computer/src/tools/exec.ts +++ b/packages/computer/src/tools/exec.ts @@ -11,7 +11,7 @@ import type { WorkspaceRuntimeValue } from "../runtime/types.js"; export type ExecStreamEvent = | { name: "stdout"; value: string } | { name: "stderr"; value: string } - | { name: "exit"; value: number; result?: unknown }; + | { name: "exit"; code: number; result?: unknown }; // A detached execution handle. The tool streams stdout / stderr // chunks by iterating the handle when it is async-iterable, and @@ -215,7 +215,7 @@ export function createExecTool(options: ExecToolOptions): Tool< if (event.name === "stdout") stdout.push(event.value); else if (event.name === "stderr") stderr.push(event.value); else { - exitCode = event.value; + exitCode = event.code; if ("result" in event) { value = event.result; hasValue = true; From b8542d493c5636021b920c1625031254d0300dd9 Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:17:37 +0000 Subject: [PATCH 08/10] computer: validate the exec tool input with a JSON-value schema The tool declared its structured input as z.unknown(), which serializes to an empty JSON Schema that some providers reject under strict function calling, and pushed a non-serializable input past the tool boundary to fail later inside the backend. The runtime call then needed a cast because unknown is wider than the backend's value type. Describe input with a recursive JSON-value schema so it validates at the tool boundary, generates a real schema, and types as the backend's value shape, dropping the cast. --- packages/computer/src/tools/exec.ts | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/computer/src/tools/exec.ts b/packages/computer/src/tools/exec.ts index 514b2653..81e405ea 100644 --- a/packages/computer/src/tools/exec.ts +++ b/packages/computer/src/tools/exec.ts @@ -3,6 +3,23 @@ import { z } from "zod"; import type { WorkspaceRuntimeValue } from "../runtime/types.js"; +// A finite JSON value: what a callable backend accepts as `input` and +// returns as `result`. Declared as a concrete recursive schema rather +// than z.unknown() so the tool validates `input` at its own boundary +// and the generated JSON Schema describes a real shape (z.unknown() +// serializes to an empty schema that some providers reject under +// strict function calling). +const jsonValueSchema: z.ZodType = z.lazy(() => + z.union([ + z.string(), + z.number(), + z.boolean(), + z.null(), + z.array(jsonValueSchema), + z.record(z.string(), jsonValueSchema), + ]), +); + // One event drained from a running execution. stdout / stderr carry // output chunks as they arrive; exit carries the process exit code and, // for a callable backend, the structured return value on `result`. The @@ -94,7 +111,7 @@ export function createExecTool(options: ExecToolOptions): Tool< cwd?: string; backend?: string; env?: Record; - input?: unknown; + input?: WorkspaceRuntimeValue; }, ExecToolOutput > { @@ -165,8 +182,7 @@ export function createExecTool(options: ExecToolOptions): Tool< .describe( "Environment variables for this run only. Values override the backend's base environment without affecting later runs.", ), - input: z - .unknown() + input: jsonValueSchema .optional() .describe( "Structured value handed to a callable backend's module. Only callable backends accept it; other backends reject it.", @@ -189,7 +205,7 @@ export function createExecTool(options: ExecToolOptions): Tool< encoding: "utf8", backend: selectedBackend, env, - input: input as WorkspaceRuntimeValue, + input, }); } catch (err) { yield { ...base, error: errorMessage(err) }; From 92028ecbf754d876d188a0682dcee3f2237a8182 Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:20:47 +0000 Subject: [PATCH 09/10] computer: exercise the exec tool against a real Workspace Every exec tool test bound a hand-shaped workspace literal, so the ExecWorkspaceLike contract was only ever checked against fakes written to satisfy it. A real Workspace binding lived only in the examples, which is why a type-level break in that contract surfaced as an example failure rather than a package test failure. Add a test that registers an in-process command backend on a real Workspace and runs the exec tool through it. It covers the binding and the assumption underneath the streaming path: that the real runtime handle is async-iterable. --- packages/computer/src/tools/ai.test.ts | 76 ++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/packages/computer/src/tools/ai.test.ts b/packages/computer/src/tools/ai.test.ts index f16530d8..9da31885 100644 --- a/packages/computer/src/tools/ai.test.ts +++ b/packages/computer/src/tools/ai.test.ts @@ -76,6 +76,50 @@ function makeWorkspace(): Workspace { return new Workspace({ storage: new SQLiteTestStorage(), now: () => 1_700_000_000_000 }); } +// An in-process command backend that streams a fixed event sequence. +// Registered on a real Workspace so the exec tool runs against the +// genuine WorkspaceRuntime handle rather than a hand-shaped fake: +// this pins the ExecWorkspaceLike binding and the assumption that the +// real handle is async-iterable. +function streamingCommandBackend(events: import("@cloudflare/computer-rpc").ExecEvent[]): { + id: string; + type: string; + connect(): Promise<{ + rpc: import("@cloudflare/computer-rpc").WorkspaceRPC; + sync: "none"; + close(): Promise; + }>; +} { + const shell: import("@cloudflare/computer-rpc").ShellRPC = { + async exec(input) { + const id = input.id ?? "cmd-1"; + return { + id, + events: new ReadableStream({ + start(controller) { + for (const event of events) controller.enqueue({ ...event, id }); + controller.close(); + }, + }), + }; + }, + getExec: () => Promise.reject(new Error("not used")), + killExec: () => Promise.resolve(), + disposeExec: () => Promise.resolve(), + }; + const noopSync = new Proxy( + {}, + { get: () => () => Promise.reject(new Error("sync: none")) }, + ) as import("@cloudflare/computer-rpc").SyncRPC; + return { + id: "shell", + type: "fake-command", + async connect() { + return { rpc: { sync: noopSync, shell }, sync: "none", close: async () => {} }; + }, + }; +} + function bytes(text: string): Uint8Array { return new TextEncoder().encode(text); } @@ -702,6 +746,38 @@ describe("createAITools callable exec", () => { }); }); +describe("createAITools exec against a real Workspace", () => { + it("streams a real runtime handle end to end", async () => { + const workspace = new Workspace({ + storage: new SQLiteTestStorage(), + backends: [ + streamingCommandBackend([ + { id: "cmd-1", seq: 1, name: "stdout", value: new TextEncoder().encode("hello\n") }, + { id: "cmd-1", seq: 2, name: "exit", code: 0 }, + ]) as never, + ], + }); + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "shell", + backends: { shell: { description: "fast shell" } }, + }, + }); + + const chunks = await collectTool(tools.exec, { command: "echo hello" }); + expect(chunks.at(-1)).toEqual({ + command: "echo hello", + cwd: null, + backend: "shell", + exitCode: 0, + stdout: "hello\n", + stderr: "", + }); + await workspace.close(); + }); +}); + describe("createAITools exec streaming", () => { it("streams stdout and stderr chunks and yields a final aggregate", async () => { const workspace = { From 154bbc6a6905049c60f3d39dee32acb15deda94d Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:28:24 +0000 Subject: [PATCH 10/10] computer: kill the backend when the exec tool turn aborts The streaming exec tool ignored the turn's abort signal: aborting the model turn abandoned the iteration but left the backend execution running. It also built the not-callable rejection from its own string copy. Wire the abort signal to the handle's kill so an aborted turn stops the backend, removing the listener once the run settles, and reject a non-callable input with the runtime's exported message. Cover the abort path with a test that asserts kill fires. --- packages/computer/src/tools/ai.test.ts | 51 +++++++++ packages/computer/src/tools/exec.ts | 140 ++++++++++++++----------- 2 files changed, 130 insertions(+), 61 deletions(-) diff --git a/packages/computer/src/tools/ai.test.ts b/packages/computer/src/tools/ai.test.ts index 9da31885..b2636f94 100644 --- a/packages/computer/src/tools/ai.test.ts +++ b/packages/computer/src/tools/ai.test.ts @@ -988,6 +988,57 @@ describe("createAITools exec streaming", () => { stdout: "aaaaaaaa\n\n[truncated, 12 more bytes]", }); }); + + it("kills the backend execution when the turn aborts", async () => { + let killed = 0; + const controller = new AbortController(); + const workspace = { + runtime: { + async exec() { + return { + async *[Symbol.asyncIterator]() { + yield { name: "stdout", value: "working\n" } as ExecStreamEvent; + controller.abort(); + // The abort listener calls kill(); yield once more so + // the iteration observes the signal before the stream + // ends on its own. + yield { name: "exit", code: 130 } as ExecStreamEvent; + }, + result: async () => { + throw new Error("result() must not be called on a streamed handle"); + }, + kill: async () => { + killed += 1; + }, + }; + }, + }, + }; + const tools = createAITools({ + workspace, + shell: { + defaultBackend: "shell", + backends: { shell: { description: "fast shell" } }, + }, + }); + + const execute = ( + tools.exec as { + execute: ( + input: unknown, + options: { toolCallId: string; messages: []; abortSignal: AbortSignal }, + ) => AsyncIterable; + } + ).execute; + const output = execute( + { command: "sleep" }, + { toolCallId: "t", messages: [], abortSignal: controller.signal }, + ); + for await (const _chunk of output) { + // Drain to completion. + } + expect(killed).toBe(1); + }); }); describe("createAITools publish tool", () => { diff --git a/packages/computer/src/tools/exec.ts b/packages/computer/src/tools/exec.ts index 81e405ea..5a58fb36 100644 --- a/packages/computer/src/tools/exec.ts +++ b/packages/computer/src/tools/exec.ts @@ -1,6 +1,7 @@ import { type Tool, tool } from "ai"; import { z } from "zod"; +import { notCallableMessage } from "../runtime/runtime.js"; import type { WorkspaceRuntimeValue } from "../runtime/types.js"; // A finite JSON value: what a callable backend accepts as `input` and @@ -40,6 +41,10 @@ export interface ExecRuntimeHandle extends Partial; + // Signal the running execution. The tool calls it when the model + // turn aborts, so the backend stops rather than running on after + // the tool stops iterating. + kill?(): Promise; } export interface ExecWorkspaceLike { @@ -188,14 +193,11 @@ export function createExecTool(options: ExecToolOptions): Tool< "Structured value handed to a callable backend's module. Only callable backends accept it; other backends reject it.", ), }), - execute: async function* ({ command, cwd, backend, env, input }) { + execute: async function* ({ command, cwd, backend, env, input }, { abortSignal }) { const selectedBackend = backend ?? options.defaultBackend; const base = { command, cwd: cwd ?? null, backend: selectedBackend }; if (input !== undefined && !callableBackendIds.has(selectedBackend)) { - yield { - ...base, - error: `Backend ${JSON.stringify(selectedBackend)} is not callable; it does not accept structured input.`, - }; + yield { ...base, error: notCallableMessage(selectedBackend) }; return; } let handle: ExecRuntimeHandle; @@ -212,68 +214,84 @@ export function createExecTool(options: ExecToolOptions): Tool< return; } - // Stream stdout / stderr chunks as they arrive when the handle - // is iterable. Each chunk yields a fresh snapshot with the - // running output so the model sees progress before the run - // ends; the exit event settles the terminal snapshot. - if (typeof handle[Symbol.asyncIterator] === "function") { - const stdout = new StreamBuffer(streamMaxBytes); - const stderr = new StreamBuffer(streamMaxBytes); - let exitCode: number | null = null; - let value: unknown; - let hasValue = false; - // Coalesce running snapshots to at most one per interval. A - // chatty command would otherwise yield a full-buffer snapshot - // per chunk; the terminal snapshot below always fires. - let lastSnapshot = 0; - try { - for await (const event of handle as AsyncIterable) { - if (event.name === "stdout") stdout.push(event.value); - else if (event.name === "stderr") stderr.push(event.value); - else { - exitCode = event.code; - if ("result" in event) { - value = event.result; - hasValue = true; + // Aborting the model turn kills the backend execution so it does + // not run on unobserved after the tool stops iterating. The run + // then emits its terminal event and the stream closes normally. + const onAbort = () => void handle.kill?.().catch(() => undefined); + if (abortSignal?.aborted) onAbort(); + else abortSignal?.addEventListener("abort", onAbort, { once: true }); + try { + yield* runExecution(); + } finally { + abortSignal?.removeEventListener("abort", onAbort); + } + + // Produce the run's snapshots. Streams the raw events when the + // handle is iterable; otherwise drains the aggregate result. + async function* runExecution(): AsyncGenerator { + // Stream stdout / stderr chunks as they arrive when the handle + // is iterable. Each chunk yields a fresh snapshot with the + // running output so the model sees progress before the run + // ends; the exit event settles the terminal snapshot. + if (typeof handle[Symbol.asyncIterator] === "function") { + const stdout = new StreamBuffer(streamMaxBytes); + const stderr = new StreamBuffer(streamMaxBytes); + let exitCode: number | null = null; + let value: unknown; + let hasValue = false; + // Coalesce running snapshots to at most one per interval. A + // chatty command would otherwise yield a full-buffer snapshot + // per chunk; the terminal snapshot below always fires. + let lastSnapshot = 0; + try { + for await (const event of handle as AsyncIterable) { + if (event.name === "stdout") stdout.push(event.value); + else if (event.name === "stderr") stderr.push(event.value); + else { + exitCode = event.code; + if ("result" in event) { + value = event.result; + hasValue = true; + } + continue; } - continue; + const at = now(); + if (at - lastSnapshot < STREAM_COALESCE_MS) continue; + lastSnapshot = at; + yield { + ...base, + exitCode: null, + stdout: stdout.render(maxBytes), + stderr: stderr.render(maxBytes), + }; } - const at = now(); - if (at - lastSnapshot < STREAM_COALESCE_MS) continue; - lastSnapshot = at; - yield { - ...base, - exitCode: null, - stdout: stdout.render(maxBytes), - stderr: stderr.render(maxBytes), - }; + } catch (err) { + yield { ...base, error: errorMessage(err) }; + return; } - } catch (err) { - yield { ...base, error: errorMessage(err) }; + yield { + ...base, + exitCode, + stdout: stdout.render(maxBytes), + stderr: stderr.render(maxBytes), + ...(hasValue ? { result: value } : {}), + }; return; } - yield { - ...base, - exitCode, - stdout: stdout.render(maxBytes), - stderr: stderr.render(maxBytes), - ...(hasValue ? { result: value } : {}), - }; - return; - } - // Non-streaming handle: drain the aggregate result. - try { - const result = await handle.result(); - yield { - ...base, - exitCode: result.exitCode, - stdout: truncate(result.stdout, maxBytes), - stderr: truncate(result.stderr, maxBytes), - ...(result.value === undefined ? {} : { result: result.value }), - }; - } catch (err) { - yield { ...base, error: errorMessage(err) }; + // Non-streaming handle: drain the aggregate result. + try { + const result = await handle.result(); + yield { + ...base, + exitCode: result.exitCode, + stdout: truncate(result.stdout, maxBytes), + stderr: truncate(result.stderr, maxBytes), + ...(result.value === undefined ? {} : { result: result.value }), + }; + } catch (err) { + yield { ...base, error: errorMessage(err) }; + } } }, });