From 62a700852811ba0a2da876f05b56f79f51457f0a Mon Sep 17 00:00:00 2001 From: kolaworld Date: Sun, 23 Aug 2026 22:54:49 -0400 Subject: [PATCH 1/4] fix(ai): preserve client tool output errors --- .changeset/fix-client-tool-error-resume.md | 7 + docs/tools/client-tools.md | 3 +- packages/ai-client/src/chat-client.ts | 44 +++-- packages/ai-client/src/interrupt-manager.ts | 95 ++++++++++- .../tests/chat-client-context.test.ts | 54 +++++- .../tests/chat-client-interrupts.test.ts | 116 ++++++++++++- .../chat-client-join-run-client-tool.test.ts | 25 ++- .../tests/chat-client-resume.test.ts | 159 +++++++++++++++++- packages/ai-persistence/src/middleware.ts | 2 + .../ai-persistence/tests/interrupts.test.ts | 90 +++++++++- packages/ai/src/activities/chat/index.ts | 34 +++- .../src/activities/chat/middleware/types.ts | 1 + .../src/activities/chat/tools/tool-calls.ts | 44 ++++- packages/ai/src/interrupt-resume.ts | 53 +++++- packages/ai/tests/chat.test.ts | 20 ++- packages/ai/tests/interrupt-resume.test.ts | 134 +++++++++++++++ packages/ai/tests/tool-call-manager.test.ts | 58 +++++++ testing/e2e/src/lib/tools-test-tools.ts | 15 ++ testing/e2e/src/routes/api.tools-test.ts | 38 +++-- testing/e2e/src/routes/tools-test.tsx | 14 ++ testing/e2e/tests/tool-error.spec.ts | 26 +++ 21 files changed, 960 insertions(+), 72 deletions(-) create mode 100644 .changeset/fix-client-tool-error-resume.md diff --git a/.changeset/fix-client-tool-error-resume.md b/.changeset/fix-client-tool-error-resume.md new file mode 100644 index 0000000000..48b6a62b35 --- /dev/null +++ b/.changeset/fix-client-tool-error-resume.md @@ -0,0 +1,7 @@ +--- +'@tanstack/ai': patch +'@tanstack/ai-client': patch +'@tanstack/ai-persistence': patch +--- + +Preserve failed client tool results across native interrupt resumes and await asynchronous client output validation diff --git a/docs/tools/client-tools.md b/docs/tools/client-tools.md index ffcee5c2ee..b59b3170a0 100644 --- a/docs/tools/client-tools.md +++ b/docs/tools/client-tools.md @@ -58,7 +58,8 @@ sequenceDiagram 4. **Client Execution**: The browser finds the registered `.client()` implementation by tool name and runs it with the parsed input 5. **Result Return**: Client auto-submits the result via the resume batch -6. **Server Update**: Result is validated and added to the conversation +6. **Server Update**: Successful output is validated. Execution and output + validation failures are added as failed tool results. 7. **LLM Continuation**: LLM receives the result and continues the conversation Native client-tool execution shares the atomic interrupt **batch** lifecycle diff --git a/packages/ai-client/src/chat-client.ts b/packages/ai-client/src/chat-client.ts index 2ef86aec57..03e59e67f0 100644 --- a/packages/ai-client/src/chat-client.ts +++ b/packages/ai-client/src/chat-client.ts @@ -1,13 +1,14 @@ import { StreamProcessor, + cloneAndDeepFreezeJson, convertSchemaToJsonSchema, generateMessageId, isStandardSchema, mergeMetadata, normalizeToUIMessage, - parseWithStandardSchema, restoreInboundChunk, tanstackMetadata, + validateWithStandardSchema, } from '@tanstack/ai/client' import { ByokBlockedError, @@ -2576,11 +2577,15 @@ export class ChatClient< clientTool: AnyClientTool | undefined, context?: ChatClientRunEventContext, ): Promise { - if (clientTool && result.state !== 'output-error') { + if (result.state !== 'output-error') { try { result = { ...result, - output: this.validateClientToolOutput(clientTool, result.output), + output: + clientTool?.outputSchema && + isStandardSchema(clientTool.outputSchema) + ? await this.validateClientToolOutput(clientTool, result.output) + : cloneAndDeepFreezeJson(result.output), } } catch (error: any) { result = { @@ -2611,12 +2616,16 @@ export class ChatClient< ) this.devtoolsBridge.emitSnapshot() - const resolvedViaInterrupt = this.interruptManager.resolveClientToolOutput( - result.toolCallId, + const resolvedViaInterrupt = result.state === 'output-error' - ? { error: result.errorText || 'Tool execution failed' } - : result.output, - ) + ? this.interruptManager.resolveClientToolError( + result.toolCallId, + result.errorText || 'Tool execution failed', + ) + : this.interruptManager.resolveClientToolOutput( + result.toolCallId, + result.output, + ) if (resolvedViaInterrupt) { // Interrupt manager stages/submits the resume batch (deferred until the // parent stream settles when still loading). Skip legacy continuation. @@ -2632,15 +2641,20 @@ export class ChatClient< await this.checkForContinuation() } - private validateClientToolOutput( + private async validateClientToolOutput( clientTool: AnyClientTool, - output: any, - ): any { - if (clientTool.outputSchema && isStandardSchema(clientTool.outputSchema)) { - return parseWithStandardSchema(clientTool.outputSchema, output) + output: unknown, + ): Promise { + const validation = await validateWithStandardSchema( + clientTool.outputSchema, + output, + ) + if (!validation.success) { + throw new Error( + validation.issues.map((issue) => issue.message).join(', '), + ) } - - return output + return cloneAndDeepFreezeJson(validation.data) } /** diff --git a/packages/ai-client/src/interrupt-manager.ts b/packages/ai-client/src/interrupt-manager.ts index 5d4c675663..e2ed9069b0 100644 --- a/packages/ai-client/src/interrupt-manager.ts +++ b/packages/ai-client/src/interrupt-manager.ts @@ -120,11 +120,13 @@ function resolutionWithContinuation( const continuation = genericInterruptContinuationFromDescriptor( item.descriptor, ) - if (!continuation) return resolution - return { - ...resolution, - metadata: wrapGenericInterruptContinuation(continuation), + if (continuation) { + return { + ...resolution, + metadata: wrapGenericInterruptContinuation(continuation), + } } + return resolution } function isRootResolvableInterrupt< @@ -677,6 +679,25 @@ export class InterruptManager< } resolveClientToolOutput(toolCallId: string, output: unknown): boolean { + return this.resolveClientToolResult(toolCallId, { + state: 'output-available', + output, + }) + } + + resolveClientToolError(toolCallId: string, errorText: string): boolean { + return this.resolveClientToolResult(toolCallId, { + state: 'output-error', + errorText, + }) + } + + private resolveClientToolResult( + toolCallId: string, + result: + | { state: 'output-available'; output: unknown } + | { state: 'output-error'; errorText: string }, + ): boolean { const item = this.items.find( (candidate) => (candidate.kind === 'client-tool-execution' && @@ -688,7 +709,14 @@ export class InterruptManager< isLegacyClientToolMetadata(candidate.descriptor.metadata)), ) if (!item) return false - this.resolveItem(item.descriptor.id, output) + this.resolveItem( + item.descriptor.id, + item.kind === 'client-tool-execution' + ? result + : result.state === 'output-error' + ? { error: result.errorText } + : result.output, + ) return true } @@ -714,7 +742,9 @@ export class InterruptManager< const interrupt = cloneAndDeepFreezeJson(descriptor) const candidate = getDescriptorBinding(interrupt) const legacyResumable = - candidate === undefined && isLegacyInterruptMetadata(interrupt) + candidate === undefined && + !hasReservedFirstPartyBindingMarker(interrupt) && + isLegacyInterruptMetadata(interrupt) // No binding we understand, and nothing else identifying the descriptor as // ours, means this interrupt was not produced by this package's resume @@ -1272,11 +1302,60 @@ export class InterruptManager< : preserveInput(validation) } if (item.kind === 'client-tool-execution') { - return validateWithSchema( + if (!isUnknownObject(payload)) { + return { + code: 'invalid-tool-output', + message: 'Client tool results require a result state.', + } + } + if ( + payload['state'] === 'output-error' && + typeof payload['errorText'] === 'string' && + Object.keys(payload).length === 2 + ) { + return { + valid: true, + payload: cloneAndDeepFreezeJson({ + state: 'output-error', + errorText: payload['errorText'], + }), + } + } + if ( + payload['state'] !== 'output-available' || + !Object.hasOwn(payload, 'output') || + Object.keys(payload).length !== 2 + ) { + return { + code: 'invalid-tool-output', + message: 'Client tool results require output or errorText.', + } + } + const validation = validateWithSchema( item.tool?.outputSchema, - payload, + payload['output'], 'invalid-tool-output', ) + const canonicalize = (result: ValidationResult): ValidationResult => { + if (!('valid' in result)) return result + try { + return { + valid: true, + payload: cloneAndDeepFreezeJson({ + state: 'output-available', + output: result.payload, + }), + } + } catch (error) { + return { + code: 'invalid-tool-output', + message: error instanceof Error ? error.message : String(error), + } + } + } + return isPromiseLike(validation) + ? Promise.resolve(validation).then(canonicalize) + : canonicalize(validation) } return this.validateApprovalCandidate(item, payload) } diff --git a/packages/ai-client/tests/chat-client-context.test.ts b/packages/ai-client/tests/chat-client-context.test.ts index 95dfe5a7be..471607c7e3 100644 --- a/packages/ai-client/tests/chat-client-context.test.ts +++ b/packages/ai-client/tests/chat-client-context.test.ts @@ -5,6 +5,10 @@ import { ChatClient } from '../src/chat-client' import { createTextChunks, createToolCallChunks } from './test-utils' import type { ConnectConnectionAdapter } from '../src/connection-adapters' +const asyncCountSchema = z + .object({ count: z.number() }) + .refine(async ({ count }) => count > 0, 'expected positive count') + function findToolCallPart(client: ChatClient, toolCallId: string) { for (const message of client.getMessages()) { if (message.role !== 'assistant') { @@ -321,6 +325,48 @@ describe('ChatClient runtime context', () => { }) }) + it('awaits asynchronous outputSchema validation for executable client tools', async () => { + const firstChunks = createToolCallChunks([ + { + id: 'tc-async-executable-output', + name: 'async_output_tool', + arguments: '{}', + }, + ]) + const secondChunks = createTextChunks('done', 'msg-async-output') + let callIndex = 0 + + const adapter: ConnectConnectionAdapter = { + async *connect(_messages, _data, abortSignal) { + const chunks = callIndex === 0 ? firstChunks : secondChunks + callIndex++ + for (const chunk of chunks) { + if (abortSignal?.aborted) return + yield chunk + } + }, + } + + const tool = toolDefinition({ + name: 'async_output_tool', + description: 'Returns asynchronously validated output', + outputSchema: asyncCountSchema, + }).client(() => ({ count: 1 })) + + const client = new ChatClient({ connection: adapter, tools: [tool] }) + await client.sendMessage('call async output tool') + + expect( + findToolCallPart(client, 'tc-async-executable-output'), + ).toMatchObject({ + state: 'input-complete', + output: { count: 1 }, + }) + expect( + findToolResultPart(client, 'tc-async-executable-output'), + ).toMatchObject({ state: 'complete' }) + }) + it('renders a client tool that throws an empty-message error as terminal "error" (issue #718)', async () => { const firstChunks = createToolCallChunks([ { @@ -369,11 +415,11 @@ describe('ChatClient runtime context', () => { }) }) - it('validates manual client tool results against outputSchema', async () => { + it('awaits asynchronous outputSchema validation for manual client tool results', async () => { const tool = toolDefinition({ name: 'manual_invalid_output_tool', description: 'Validates manual output', - outputSchema: z.object({ count: z.number() }), + outputSchema: asyncCountSchema, }).client(() => ({ count: 1 })) const client = new ChatClient({ @@ -403,13 +449,13 @@ describe('ChatClient runtime context', () => { await client.addToolResult({ toolCallId: 'tc-manual-invalid-output', tool: 'manual_invalid_output_tool', - output: JSON.parse('{"count":"not-a-number"}'), + output: { count: 0 }, }) expect(findToolCallPart(client, 'tc-manual-invalid-output')).toMatchObject({ state: 'error', output: { - error: expect.stringContaining('expected number'), + error: 'expected positive count', }, }) expect( diff --git a/packages/ai-client/tests/chat-client-interrupts.test.ts b/packages/ai-client/tests/chat-client-interrupts.test.ts index ea92e9da85..49cd99bbde 100644 --- a/packages/ai-client/tests/chat-client-interrupts.test.ts +++ b/packages/ai-client/tests/chat-client-interrupts.test.ts @@ -2,6 +2,7 @@ import { chat, createInterruptBinding, defineInterrupt } from '@tanstack/ai' import { EventType, + INTERRUPT_BINDING_VERSION, canonicalInterruptJson, convertSchemaToJsonSchema, digestInterruptJson, @@ -10,7 +11,6 @@ import { toolDefinition, } from '@tanstack/ai/client' import { z } from 'zod' -import { INTERRUPT_BINDING_VERSION } from '@tanstack/ai/client' import { InterruptManager } from '../src/interrupt-manager' import { ChatClient } from '../src/chat-client' import type { @@ -25,12 +25,11 @@ import type { } from '@tanstack/ai/client' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { InterruptManagerSubmission } from '../src/interrupt-manager' -import type { ResolvableChatInterrupt } from '../src/types' +import type { ResolvableChatInterrupt, UIMessage } from '../src/types' import type { ConnectConnectionAdapter, RunAgentInputContext, } from '../src/connection-adapters' -import type { UIMessage } from '../src/types' const transferDefinition = toolDefinition({ name: 'transfer', @@ -381,8 +380,29 @@ describe('InterruptManager hydration', () => { const expectedOutputSchemaHash = hashSchemaInput( lookupDefinition.outputSchema, ) - const expectedResponseSchema = - convertSchemaToJsonSchema(lookupDefinition.outputSchema) ?? {} + const expectedResponseSchema = { + oneOf: [ + { + type: 'object', + properties: { + state: { const: 'output-available' }, + output: + convertSchemaToJsonSchema(lookupDefinition.outputSchema) ?? {}, + }, + required: ['state', 'output'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + state: { const: 'output-error' }, + errorText: { type: 'string' }, + }, + required: ['state', 'errorText'], + additionalProperties: false, + }, + ], + } const expectedResponseSchemaHash = digestInterruptJson( canonicalInterruptJson(expectedResponseSchema), ) @@ -1037,7 +1057,83 @@ describe('InterruptManager transactions', () => { ).toBe(true) }) - it('resolves client-tool generic fallback for native reason string', () => { + it('stages client-tool errors without validating the success schema', async () => { + const { manager, submit } = createManager() + const outputSchemaHash = hashSchemaInput(lookupDefinition.outputSchema) + const binding: InterruptBinding = { + v: INTERRUPT_BINDING_VERSION, + kind: 'client-tool-execution', + interruptId: 'client-1', + interruptedRunId: 'run-1', + generation: 1, + toolName: 'lookup', + toolCallId: 'call-1', + outputSchemaHash, + responseSchemaHash: outputSchemaHash, + } + manager.hydrate({ + threadId: 'thread-1', + interruptedRunId: 'run-1', + generation: 1, + interrupts: [descriptor(binding)], + }) + + expect(manager.resolveClientToolError('call-1', 'Lookup failed')).toBe(true) + await settle() + + expect(submit.mock.calls[0]?.[0].resolutions).toEqual([ + { + interruptId: 'client-1', + status: 'resolved', + payload: { state: 'output-error', errorText: 'Lookup failed' }, + }, + ]) + }) + + it('reports client-tool canonicalization failures as invalid output', async () => { + const canonicalTool = toolDefinition({ + name: 'canonical', + description: 'Return canonical output', + }).client() + const outputSchemaHash = hashSchemaInput(canonicalTool.outputSchema) + const submit = vi.fn( + async (_submission: InterruptManagerSubmission) => undefined, + ) + const manager = new InterruptManager({ tools: [canonicalTool], submit }) + const binding: InterruptBinding = { + v: INTERRUPT_BINDING_VERSION, + kind: 'client-tool-execution', + interruptId: 'client-1', + interruptedRunId: 'run-1', + generation: 1, + toolName: 'canonical', + toolCallId: 'call-1', + outputSchemaHash, + responseSchemaHash: outputSchemaHash, + } + manager.hydrate({ + threadId: 'thread-1', + interruptedRunId: 'run-1', + generation: 1, + interrupts: [descriptor(binding)], + }) + + expect( + manager.resolveClientToolOutput('call-1', { + value: undefined, + }), + ).toBe(true) + await settle() + + expect(submit).not.toHaveBeenCalled() + expect( + manager + .getInterruptErrors() + .some((error) => error.code === 'item-validation-failed'), + ).toBe(true) + }) + + it('resolves pre-binding client-tool fallback with raw output', () => { const { manager, submit } = createManager() manager.hydrate({ threadId: 'thread-1', @@ -1061,7 +1157,13 @@ describe('InterruptManager transactions', () => { expect( manager.resolveClientToolOutput('call-degraded', { accountId: 'a' }), ).toBe(true) - expect(submit).toHaveBeenCalled() + expect(submit.mock.calls[0]?.[0].resolutions).toEqual([ + { + interruptId: 'client-degraded', + status: 'resolved', + payload: { accountId: 'a' }, + }, + ]) }) it('supersedes a server batch error set without dropping local client, transport, or item errors', async () => { diff --git a/packages/ai-client/tests/chat-client-join-run-client-tool.test.ts b/packages/ai-client/tests/chat-client-join-run-client-tool.test.ts index f75f9183e4..2dc31a6559 100644 --- a/packages/ai-client/tests/chat-client-join-run-client-tool.test.ts +++ b/packages/ai-client/tests/chat-client-join-run-client-tool.test.ts @@ -68,7 +68,28 @@ describe('joinRun client-tool continuation (issue #1058)', () => { return { answer: 42 } }) const outputSchemaHash = hashSchemaInput(outputSchema) - const responseSchema = convertSchemaToJsonSchema(outputSchema) ?? {} + const responseSchema = { + oneOf: [ + { + type: 'object', + properties: { + state: { const: 'output-available' }, + output: convertSchemaToJsonSchema(outputSchema) ?? {}, + }, + required: ['state', 'output'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + state: { const: 'output-error' }, + errorText: { type: 'string' }, + }, + required: ['state', 'errorText'], + additionalProperties: false, + }, + ], + } const responseSchemaHash = digestInterruptJson( canonicalInterruptJson(responseSchema), ) @@ -200,7 +221,7 @@ describe('joinRun client-tool continuation (issue #1058)', () => { { interruptId: 'client_tool_tool-call-1', status: 'resolved', - payload: { answer: 42 }, + payload: { state: 'output-available', output: { answer: 42 } }, }, ]) expect(client.getInterruptState().interruptErrors).toEqual([]) diff --git a/packages/ai-client/tests/chat-client-resume.test.ts b/packages/ai-client/tests/chat-client-resume.test.ts index 5267126699..48e6ff7089 100644 --- a/packages/ai-client/tests/chat-client-resume.test.ts +++ b/packages/ai-client/tests/chat-client-resume.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest' import { EventType, + INTERRUPT_BINDING_VERSION, convertSchemaToJsonSchema, digestInterruptJson, canonicalInterruptJson, @@ -116,6 +117,31 @@ const runStarted: StreamChunk = { timestamp: Date.now(), } +function clientToolResponseSchema(outputSchema: Record) { + return { + oneOf: [ + { + type: 'object', + properties: { + state: { const: 'output-available' }, + output: outputSchema, + }, + required: ['state', 'output'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + state: { const: 'output-error' }, + errorText: { type: 'string' }, + }, + required: ['state', 'errorText'], + additionalProperties: false, + }, + ], + } +} + describe('ChatClient resume', () => { it('tracks the run/thread of an interrupted run', async () => { const { adapter, contexts } = recordingAdapter([ @@ -1089,7 +1115,9 @@ describe('ChatClient resume', () => { outputSchema, }).client(async () => ({ answer: 42 })) const outputSchemaHash = hashSchemaInput(outputSchema) - const responseSchema = convertSchemaToJsonSchema(outputSchema) ?? {} + const responseSchema = clientToolResponseSchema( + convertSchemaToJsonSchema(outputSchema) ?? {}, + ) const responseSchemaHash = digestInterruptJson( canonicalInterruptJson(responseSchema), ) @@ -1196,12 +1224,131 @@ describe('ChatClient resume', () => { { interruptId: 'client_tool_tool-call-1', status: 'resolved', - payload: { answer: 42 }, + payload: { state: 'output-available', output: { answer: 42 } }, }, ]) expect(client.getInterruptState().interruptErrors).toEqual([]) }) + it('resumes canonicalization failures as client-tool output errors', async () => { + const lookup = toolDefinition({ + name: 'lookup', + description: 'Look up', + inputSchema: z.object({ query: z.string() }), + }).client(async () => ({ value: undefined })) + const outputSchemaHash = hashSchemaInput(lookup.outputSchema) + const responseSchema = clientToolResponseSchema({}) + const responseSchemaHash = digestInterruptJson( + canonicalInterruptJson(responseSchema), + ) + const { adapter, contexts } = recordingAdapter([ + (ctx) => { + const runId = ctx?.runId ?? 'run-1' + const threadId = ctx?.threadId ?? 'thread-1' + return [ + { + type: EventType.RUN_STARTED, + runId, + threadId, + timestamp: Date.now(), + }, + { + type: EventType.TOOL_CALL_START, + toolCallId: 'tool-call-1', + toolCallName: 'lookup', + toolName: 'lookup', + timestamp: Date.now(), + }, + { + type: EventType.TOOL_CALL_ARGS, + toolCallId: 'tool-call-1', + delta: '{"query":"first"}', + timestamp: Date.now(), + }, + { + type: EventType.RUN_FINISHED, + runId, + threadId, + timestamp: Date.now(), + outcome: { + type: 'interrupt', + interrupts: [ + { + id: 'client_tool_tool-call-1', + reason: 'tanstack:client_tool_execution', + toolCallId: 'tool-call-1', + responseSchema, + metadata: { + kind: 'client_tool', + toolName: 'lookup', + input: { query: 'first' }, + 'tanstack:interruptBinding': { + v: INTERRUPT_BINDING_VERSION, + kind: 'client-tool-execution', + interruptId: 'client_tool_tool-call-1', + interruptedRunId: runId, + generation: 0, + toolName: 'lookup', + toolCallId: 'tool-call-1', + outputSchemaHash, + responseSchemaHash, + }, + }, + }, + ], + }, + }, + ] + }, + (ctx) => [ + { + type: EventType.RUN_STARTED, + runId: ctx?.runId ?? 'run-2', + threadId: ctx?.threadId ?? 'thread-1', + timestamp: Date.now(), + }, + text('continued'), + { + type: EventType.RUN_FINISHED, + runId: ctx?.runId ?? 'run-2', + threadId: ctx?.threadId ?? 'thread-1', + timestamp: Date.now(), + metadata: { tanstack: { finishReason: 'stop' } }, + }, + ], + ]) + const client = new ChatClient({ + connection: adapter, + threadId: 'thread-1', + tools: [lookup], + }) + + await client.sendMessage('hi') + + await vi.waitFor(() => { + expect(contexts).toHaveLength(2) + expect( + client + .getMessages() + .some((message) => + message.parts.some( + (part) => part.type === 'text' && part.content === 'continued', + ), + ), + ).toBe(true) + }) + expect(contexts[1]?.resume).toEqual([ + { + interruptId: 'client_tool_tool-call-1', + status: 'resolved', + payload: { + state: 'output-error', + errorText: expect.any(String), + }, + }, + ]) + }) + it('continues a legacy client tool emitted by a native resume', async () => { const lookup = toolDefinition({ name: 'lookup', @@ -1337,7 +1484,9 @@ describe('ChatClient resume', () => { outputSchema, }).client(async ({ query }) => ({ answer: query === 'first' ? 42 : 43 })) const outputSchemaHash = hashSchemaInput(outputSchema) - const responseSchema = convertSchemaToJsonSchema(outputSchema) ?? {} + const responseSchema = clientToolResponseSchema( + convertSchemaToJsonSchema(outputSchema) ?? {}, + ) const responseSchemaHash = digestInterruptJson( canonicalInterruptJson(responseSchema), ) @@ -1429,7 +1578,7 @@ describe('ChatClient resume', () => { { interruptId: 'client_tool_tool-call-1', status: 'resolved', - payload: { answer: 42 }, + payload: { state: 'output-available', output: { answer: 42 } }, }, ]) expect(contexts[2]?.parentRunId).toBe(contexts[1]?.runId) @@ -1437,7 +1586,7 @@ describe('ChatClient resume', () => { { interruptId: 'client_tool_tool-call-2', status: 'resolved', - payload: { answer: 43 }, + payload: { state: 'output-available', output: { answer: 43 } }, }, ]) }) diff --git a/packages/ai-persistence/src/middleware.ts b/packages/ai-persistence/src/middleware.ts index 7e8eb184e2..650b8ccb47 100644 --- a/packages/ai-persistence/src/middleware.ts +++ b/packages/ai-persistence/src/middleware.ts @@ -327,6 +327,7 @@ function mergeResumeToolState( left.clientToolResults, right.clientToolResults, ), + clientToolErrors: mergeMaps(left.clientToolErrors, right.clientToolErrors), genericInterrupts: mergeMaps( left.genericInterrupts, right.genericInterrupts, @@ -850,6 +851,7 @@ function resumeToolStateFromPending( for (const interrupt of pending) { const entry = resumeByInterruptId.get(interrupt.interruptId) if (!entry) continue + if (hasReservedInterruptBinding(interrupt.payload)) continue const kind = interruptKind(interrupt) const reason = stringField(interrupt.payload, 'reason') diff --git a/packages/ai-persistence/tests/interrupts.test.ts b/packages/ai-persistence/tests/interrupts.test.ts index f828dc0737..251a5f27c1 100644 --- a/packages/ai-persistence/tests/interrupts.test.ts +++ b/packages/ai-persistence/tests/interrupts.test.ts @@ -564,7 +564,10 @@ describe('interrupt persistence', () => { { interruptId: 'client_tool_tool-call-1', status: 'resolved', - payload: { answer: 42 }, + payload: { + state: 'output-available', + output: { answer: 42 }, + }, }, ], middleware: [withPersistence(persistence)], @@ -587,6 +590,49 @@ describe('interrupt persistence', () => { expect(await persistence.stores.interrupts!.listPending('t1')).toEqual([]) }) + it('preserves failed client-tool resumes from persisted state', async () => { + const persistence = memoryPersistence() + await persistClientToolTurn(persistence, [clientTool('clientSearch')]) + + const continuation = mockAdapter([ + [runStarted(), text('recovered'), runFinished('r1')], + ]) + const chunks = await collect( + chat({ + adapter: continuation.adapter, + messages: [], + tools: [clientTool('clientSearch')], + runId: 'r1', + threadId: 't1', + resume: [ + { + interruptId: 'client_tool_tool-call-1', + status: 'resolved', + payload: { + state: 'output-error', + errorText: 'Client tool failed', + }, + }, + ], + middleware: [withPersistence(persistence)], + }) as AsyncIterable, + ) + + expect(continuation.calls).toHaveLength(1) + expect(chunks).toContainEqual( + expect.objectContaining({ + type: EventType.TOOL_CALL_RESULT, + toolCallId: 'tool-call-1', + content: JSON.stringify({ error: 'Client tool failed' }), + metadata: { tanstack: { state: 'output-error' } }, + }), + ) + expect(chunks).toContainEqual( + expect.objectContaining({ delta: 'recovered' }), + ) + expect(await persistence.stores.interrupts!.listPending('t1')).toEqual([]) + }) + // Issue #1088: cancelling a hydrated client-tool interrupt under // withPersistence must complete the turn. Persistence clears `config.resume` // and must therefore put the cancelled toolCallId on `cancelledToolCallIds`. @@ -1128,6 +1174,48 @@ describe('interrupt persistence', () => { ) }) + it('translates raw legacy client-tool resume payloads', async () => { + const persistence = memoryPersistence() + await persistence.stores.interrupts!.create({ + interruptId: 'client-1', + runId: 'r1', + threadId: 't1', + requestedAt: 1, + payload: { toolCallId: 'tc1', metadata: { kind: 'client_tool' } }, + }) + + const resumeStates: Array = [] + await collect( + chat({ + adapter: mockAdapter([[runStarted(), text('ok'), runFinished('r1')]]) + .adapter, + messages: [], + runId: 'r1', + threadId: 't1', + resume: [ + { + interruptId: 'client-1', + status: 'resolved', + payload: { answer: 42 }, + }, + ], + middleware: [ + withPersistence(persistence), + defineChatMiddleware({ + name: 'observe-resume-state', + onConfig(_ctx, config) { + resumeStates.push(config.resumeToolState) + }, + }), + ], + }) as AsyncIterable, + ) + + expect(resumeStates[0]?.clientToolResults?.get('tc1')).toEqual({ + answer: 42, + }) + }) + it('tolerates malformed persisted interrupt payloads without crashing', async () => { const persistence = memoryPersistence() // Payload with the wrong shapes for the defensive parsers: metadata is a diff --git a/packages/ai/src/activities/chat/index.ts b/packages/ai/src/activities/chat/index.ts index 91f84d3490..9fdca46b47 100644 --- a/packages/ai/src/activities/chat/index.ts +++ b/packages/ai/src/activities/chat/index.ts @@ -823,6 +823,7 @@ class TextEngine< private readonly initialClientToolResults: Map private readonly resumeApprovals = new Map() private readonly resumeClientToolResults = new Map() + private readonly resumeClientToolErrors = new Map() private readonly resumeDeniedToolResults = new Map() private readonly resumeCancelledToolCallIds = new Set() private readonly resumeGenericInterrupts = new Map< @@ -2031,6 +2032,7 @@ class TextEngine< this.middlewareCtx.context, this.toolAbortSignal, { + clientToolErrors: this.resumeClientToolErrors, deniedToolResults: this.resumeDeniedToolResults, cancelledToolCallIds: this.resumeCancelledToolCallIds, }, @@ -2209,6 +2211,7 @@ class TextEngine< this.middlewareCtx.context, this.toolAbortSignal, { + clientToolErrors: this.resumeClientToolErrors, deniedToolResults: this.resumeDeniedToolResults, cancelledToolCallIds: this.resumeCancelledToolCallIds, }, @@ -2635,7 +2638,29 @@ class TextEngine< const tool = this.tools.find( (candidate) => candidate.name === clientTool.toolName, ) - const responseSchema = convertSchemaToJsonSchema(tool?.outputSchema) ?? {} + const outputSchema = convertSchemaToJsonSchema(tool?.outputSchema) ?? {} + const responseSchema = { + oneOf: [ + { + type: 'object', + properties: { + state: { const: 'output-available' }, + output: outputSchema, + }, + required: ['state', 'output'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + state: { const: 'output-error' }, + errorText: { type: 'string' }, + }, + required: ['state', 'errorText'], + additionalProperties: false, + }, + ], + } interrupts.push({ id: `client_tool_${clientTool.toolCallId}`, reason: 'tanstack:client_tool_execution', @@ -2999,6 +3024,7 @@ class TextEngine< } else if ( !tool.execute && !clientToolResults.has(toolCall.id) && + !this.resumeClientToolErrors.has(toolCall.id) && !this.resumeCancelledToolCallIds.has(toolCall.id) ) { clientRequests.push({ @@ -3956,6 +3982,7 @@ class TextEngine< resumeToolState: { approvals: this.resumeApprovals, clientToolResults: this.resumeClientToolResults, + clientToolErrors: this.resumeClientToolErrors, deniedToolResults: this.resumeDeniedToolResults, cancelledToolCallIds: this.resumeCancelledToolCallIds, }, @@ -4301,6 +4328,11 @@ class TextEngine< this.resumeClientToolResults.set(toolCallId, result) } } + if (state?.clientToolErrors) { + for (const [toolCallId, errorText] of state.clientToolErrors) { + this.resumeClientToolErrors.set(toolCallId, errorText) + } + } if (state?.deniedToolResults) { for (const [toolCallId, result] of state.deniedToolResults) { this.resumeDeniedToolResults.set(toolCallId, result) diff --git a/packages/ai/src/activities/chat/middleware/types.ts b/packages/ai/src/activities/chat/middleware/types.ts index 6cd5de10fb..2516bb9f04 100644 --- a/packages/ai/src/activities/chat/middleware/types.ts +++ b/packages/ai/src/activities/chat/middleware/types.ts @@ -322,6 +322,7 @@ export interface ChatMiddlewareConfig { export interface ChatResumeToolState { approvals?: ReadonlyMap | undefined clientToolResults?: ReadonlyMap | undefined + clientToolErrors?: ReadonlyMap | undefined genericInterrupts?: | ReadonlyMap | undefined diff --git a/packages/ai/src/activities/chat/tools/tool-calls.ts b/packages/ai/src/activities/chat/tools/tool-calls.ts index 78637427ac..0b28ed8ddc 100644 --- a/packages/ai/src/activities/chat/tools/tool-calls.ts +++ b/packages/ai/src/activities/chat/tools/tool-calls.ts @@ -1,7 +1,12 @@ import { normalizeToolResult } from '../../../utilities/tool-result' import { tanstackMetadata } from '../../../utilities/merge-metadata' import type { AdapterYieldChunk } from '../../../utilities/adapter-yield-chunk' -import { isStandardSchema, parseWithStandardSchema } from './schema-converter' +import { + StandardSchemaValidationError, + isStandardSchema, + parseWithStandardSchema, + validateWithStandardSchema, +} from './schema-converter' import type { ToolApprovalResolution } from '../../../interrupts' import type { AnyTool, @@ -459,6 +464,7 @@ export interface ClientToolRequest { } export interface ToolResumeExecutionState { + clientToolErrors?: ReadonlyMap deniedToolResults?: ReadonlyMap cancelledToolCallIds?: ReadonlySet } @@ -694,17 +700,35 @@ export async function* executeServerTool( } } -function buildClientToolResult( +async function buildClientToolResult( toolCallId: string, toolName: string, tool: AnyTool, rawResult: unknown, input?: unknown, -): ToolResult { + errorText?: string, +): Promise { + if (errorText !== undefined) { + return { + toolCallId, + toolName, + result: { error: errorText }, + input, + state: 'output-error', + } + } + try { let result = rawResult if (tool.outputSchema && isStandardSchema(tool.outputSchema)) { - result = parseWithStandardSchema(tool.outputSchema, result) + const validation = await validateWithStandardSchema( + tool.outputSchema, + result, + ) + if (!validation.success) { + throw new StandardSchemaValidationError(validation.issues) + } + result = validation.data } const parsed = @@ -891,14 +915,16 @@ export async function* executeToolCalls( if (approved) { input = editedApprovalArgs(resolution) ?? input // Approved - check if client has executed - if (clientResults.has(toolCall.id)) { + const clientError = resumeState?.clientToolErrors?.get(toolCall.id) + if (clientResults.has(toolCall.id) || clientError !== undefined) { results.push( - buildClientToolResult( + await buildClientToolResult( toolCall.id, toolName, tool, clientResults.get(toolCall.id), input, + clientError, ), ) } else { @@ -932,14 +958,16 @@ export async function* executeToolCalls( } } else { // No approval needed - check if client has executed - if (clientResults.has(toolCall.id)) { + const clientError = resumeState?.clientToolErrors?.get(toolCall.id) + if (clientResults.has(toolCall.id) || clientError !== undefined) { results.push( - buildClientToolResult( + await buildClientToolResult( toolCall.id, toolName, tool, clientResults.get(toolCall.id), input, + clientError, ), ) } else { diff --git a/packages/ai/src/interrupt-resume.ts b/packages/ai/src/interrupt-resume.ts index 6202c3d1bf..bc2c2708b6 100644 --- a/packages/ai/src/interrupt-resume.ts +++ b/packages/ai/src/interrupt-resume.ts @@ -95,6 +95,30 @@ function stringField( return typeof value[key] === 'string' ? value[key] : undefined } +type ClientToolResumeResult = + | { state: 'output-available'; output: unknown } + | { state: 'output-error'; errorText: string } + +function clientToolResult( + entry: RunAgentResumeItem, +): ClientToolResumeResult | null { + const result = objectValue(entry.payload) + if (!result || Object.keys(result).length !== 2) return null + if (result.state === 'output-available' && Object.hasOwn(result, 'output')) { + return { + state: 'output-available', + output: result.output, + } + } + if (result.state === 'output-error' && typeof result.errorText === 'string') { + return { + state: 'output-error', + errorText: result.errorText, + } + } + return null +} + function normalizeIssuePath( path: ReadonlyArray | undefined, ): ReadonlyArray | undefined { @@ -527,6 +551,18 @@ export async function validateInterruptResumeBatch( if (schemaDrifted) continue if (binding.kind === 'client-tool-execution') { + const result = clientToolResult(entry) + if (!result) { + errors.push( + interruptItemError( + input, + record.interruptId, + 'invalid-tool-output', + `Tool ${binding.toolName} result is invalid.`, + ), + ) + continue + } if (responseSchema !== undefined) { await pushSchemaIssues({ request: input, @@ -538,13 +574,16 @@ export async function validateInterruptResumeBatch( label: `Tool ${binding.toolName} output is invalid`, }) } - if (tool.outputSchema !== undefined) { + if ( + result.state === 'output-available' && + tool.outputSchema !== undefined + ) { await pushSchemaIssues({ request: input, errors, interruptId: record.interruptId, schema: tool.outputSchema, - value: entry.payload, + value: result.output, code: 'invalid-tool-output', label: `Tool ${binding.toolName} output is invalid`, }) @@ -681,6 +720,7 @@ export async function validateInterruptResumeBatch( const canonical = canonicalizeInterruptResolutions(input.resume ?? []) const approvals = new Map() const clientToolResults = new Map() + const clientToolErrors = new Map() const genericInterrupts = new Map< string, | { interruptId: string; status: 'resolved'; payload: unknown } @@ -737,7 +777,13 @@ export async function validateInterruptResumeBatch( continue } if (binding.kind === 'client-tool-execution') { - clientToolResults.set(binding.toolCallId, entry.payload) + const result = clientToolResult(entry) + if (!result) continue + if (result.state === 'output-error') { + clientToolErrors.set(binding.toolCallId, result.errorText) + } else { + clientToolResults.set(binding.toolCallId, result.output) + } continue } const envelope = objectValue(entry.payload) @@ -780,6 +826,7 @@ export async function validateInterruptResumeBatch( resumeToolState: { approvals, clientToolResults, + clientToolErrors, genericInterrupts, deniedToolResults, cancelledToolCallIds, diff --git a/packages/ai/tests/chat.test.ts b/packages/ai/tests/chat.test.ts index ce55e484c1..f963364503 100644 --- a/packages/ai/tests/chat.test.ts +++ b/packages/ai/tests/chat.test.ts @@ -1704,7 +1704,10 @@ describe('chat()', () => { { interruptId: 'client_tool_call_search', status: 'resolved', - payload: { result: 'found' }, + payload: { + state: 'output-available', + output: { result: 'found' }, + }, }, ], }) as AsyncIterable, @@ -1779,7 +1782,10 @@ describe('chat()', () => { { interruptId: 'client_tool_call_1', status: 'resolved', - payload: { browserValue: 'sensor-reading-for-manual-lab' }, + payload: { + state: 'output-available', + output: { browserValue: 'sensor-reading-for-manual-lab' }, + }, }, ], }) as AsyncIterable, @@ -1861,7 +1867,10 @@ describe('chat()', () => { { interruptId: 'client_tool_call_client', status: 'resolved', - payload: { result: 'done' }, + payload: { + state: 'output-available', + output: { result: 'done' }, + }, }, ], }) as AsyncIterable, @@ -1930,7 +1939,10 @@ describe('chat()', () => { { interruptId: 'client_tool_call_client', status: 'resolved', - payload: { result: 'done' }, + payload: { + state: 'output-available', + output: { result: 'done' }, + }, }, ], }) as AsyncIterable, diff --git a/packages/ai/tests/interrupt-resume.test.ts b/packages/ai/tests/interrupt-resume.test.ts index 625c687590..f318689d8e 100644 --- a/packages/ai/tests/interrupt-resume.test.ts +++ b/packages/ai/tests/interrupt-resume.test.ts @@ -2,7 +2,10 @@ import { describe, expect, it } from 'vitest' import { z } from 'zod' import { defineInterrupt, + digestInterruptJson, hashSchemaInput, + canonicalInterruptJson, + convertSchemaToJsonSchema, normalizeApprovalSchema, toolDefinition, validateInterruptResumeBatch, @@ -24,6 +27,64 @@ const transferDef = toolDefinition({ const transfer = transferDef.server(async () => ({ receipt: 'ok' })) +const lookupDef = toolDefinition({ + name: 'lookup', + description: 'Look up an account', + outputSchema: z.object({ accountId: z.string() }), +}) + +const lookup = lookupDef.client() + +function clientToolFixture() { + const outputSchema = convertSchemaToJsonSchema(lookupDef.outputSchema) ?? {} + const responseSchema = { + oneOf: [ + { + type: 'object', + properties: { + state: { const: 'output-available' }, + output: outputSchema, + }, + required: ['state', 'output'], + additionalProperties: false, + }, + { + type: 'object', + properties: { + state: { const: 'output-error' }, + errorText: { type: 'string' }, + }, + required: ['state', 'errorText'], + additionalProperties: false, + }, + ], + } + const binding: Extract = + { + v: INTERRUPT_BINDING_VERSION, + kind: 'client-tool-execution', + interruptId: 'client_tool_call-1', + interruptedRunId: 'run-1', + generation: 0, + toolName: 'lookup', + toolCallId: 'call-1', + outputSchemaHash: hashSchemaInput(lookupDef.outputSchema), + responseSchemaHash: digestInterruptJson( + canonicalInterruptJson(responseSchema), + ), + } + return { + binding, + pending: [ + { + interruptId: binding.interruptId, + payload: { responseSchema }, + binding, + }, + ], + } +} + function approvalFixture( overrides: Partial> = {}, ) { @@ -90,6 +151,79 @@ function pendingOf(fixture: ReturnType) { } describe('validateInterruptResumeBatch', () => { + it('preserves successful client-tool resume state', async () => { + const fixture = clientToolFixture() + const result = await validateInterruptResumeBatch({ + threadId: 'thread-1', + interruptedRunId: 'run-1', + generation: 0, + pending: fixture.pending, + resume: [ + { + interruptId: fixture.binding.interruptId, + status: 'resolved', + payload: { + state: 'output-available', + output: { accountId: 'account-1' }, + }, + }, + ], + tools: [lookup], + }) + + expect(result.errors).toEqual([]) + expect(result.resumeToolState?.clientToolResults?.get('call-1')).toEqual({ + accountId: 'account-1', + }) + }) + + it('preserves failed client-tool resume state without validating the success schema', async () => { + const fixture = clientToolFixture() + const result = await validateInterruptResumeBatch({ + threadId: 'thread-1', + interruptedRunId: 'run-1', + generation: 0, + pending: fixture.pending, + resume: [ + { + interruptId: fixture.binding.interruptId, + status: 'resolved', + payload: { state: 'output-error', errorText: 'Lookup failed' }, + }, + ], + tools: [lookup], + }) + + expect(result.errors).toEqual([]) + expect(result.resumeToolState?.clientToolResults?.has('call-1')).toBe(false) + expect(result.resumeToolState?.clientToolErrors?.get('call-1')).toBe( + 'Lookup failed', + ) + }) + + it('rejects malformed client-tool result envelopes', async () => { + const fixture = clientToolFixture() + const result = await validateInterruptResumeBatch({ + threadId: 'thread-1', + interruptedRunId: 'run-1', + generation: 0, + pending: fixture.pending, + resume: [ + { + interruptId: fixture.binding.interruptId, + status: 'resolved', + payload: { error: 'Lookup failed' }, + }, + ], + tools: [lookup], + }) + + expect( + result.errors.some((error) => error.code === 'invalid-tool-output'), + ).toBe(true) + expect(result.resumeToolState).toBeUndefined() + }) + it('accepts a complete payload-bearing approval batch', async () => { const fixture = approvalFixture() const result = await validateInterruptResumeBatch( diff --git a/packages/ai/tests/tool-call-manager.test.ts b/packages/ai/tests/tool-call-manager.test.ts index f1e45c691e..ed89015cda 100644 --- a/packages/ai/tests/tool-call-manager.test.ts +++ b/packages/ai/tests/tool-call-manager.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import { z } from 'zod' +import type { StandardSchemaV1 } from '@standard-schema/spec' import { ToolCallManager, executeToolCalls, @@ -732,6 +733,33 @@ describe('executeToolCalls', () => { }) }) + it('supports async output schemas for client tool results', async () => { + const outputSchema: StandardSchemaV1 = { + '~standard': { + version: 1, + vendor: 'test', + validate: async () => ({ value: { count: 2 } }), + }, + } + const tool: Tool = { + name: 'get_count', + description: 'Get a count', + inputSchema: z.object({}), + outputSchema, + } + const toolCalls = [makeToolCall('call_1', 'get_count')] + const clientResults = new Map([['call_1', { count: '2' }]]) + + const result = await drainExecuteToolCalls( + toolCalls, + [tool], + new Map(), + clientResults, + ) + + expect(result.results[0]?.result).toEqual({ count: 2 }) + }) + it('validates null client tool results against outputSchema', async () => { const tool: Tool = { name: 'get_count', @@ -800,6 +828,36 @@ describe('executeToolCalls', () => { expect(result.results).toHaveLength(1) expect(result.results[0]?.result).toBe(false) }) + + it('preserves failed client tool results', async () => { + const tool: Tool = { + name: 'get_count', + description: 'Get a count', + inputSchema: z.object({}), + outputSchema: z.object({ count: z.number() }), + } + const toolCalls = [makeToolCall('call_1', 'get_count')] + const result = await drainExecuteToolCalls( + toolCalls, + [tool], + new Map(), + new Map(), + undefined, + undefined, + undefined, + undefined, + { + clientToolErrors: new Map([['call_1', 'Client execution failed']]), + }, + ) + + expect(result.results).toEqual([ + expect.objectContaining({ + result: { error: 'Client execution failed' }, + state: 'output-error', + }), + ]) + }) }) describe('server tool with approval', () => { diff --git a/testing/e2e/src/lib/tools-test-tools.ts b/testing/e2e/src/lib/tools-test-tools.ts index 9ae4c0eb4d..6c8399cbd3 100644 --- a/testing/e2e/src/lib/tools-test-tools.ts +++ b/testing/e2e/src/lib/tools-test-tools.ts @@ -130,6 +130,13 @@ export const clientToolDefinitions = { data: z.array(z.number()), }), }).client(), + + fail_client_tool: toolDefinition({ + name: 'fail_client_tool', + description: 'Fail while executing in the client', + inputSchema: z.object({}), + outputSchema: z.object({ completed: z.boolean() }), + }).client(), } export const searchInventory = toolDefinition({ @@ -226,6 +233,11 @@ export const SCENARIO_LIST = [ label: 'Tool Throws Error', category: 'basic', }, + { + id: 'client-tool-error', + label: 'Client Tool Throws Error', + category: 'basic', + }, { id: 'null-tool-input', label: 'Null Tool Input (Regression #265)', @@ -300,6 +312,9 @@ export function getToolsForScenario(scenario: string) { case 'client-tool-reasoning': return [clientToolDefinitions.show_notification] + case 'client-tool-error': + return [clientToolDefinitions.fail_client_tool] + case 'server-context': return [serverTools.read_server_context] diff --git a/testing/e2e/src/routes/api.tools-test.ts b/testing/e2e/src/routes/api.tools-test.ts index 7977ec689d..b25f1230b1 100644 --- a/testing/e2e/src/routes/api.tools-test.ts +++ b/testing/e2e/src/routes/api.tools-test.ts @@ -14,6 +14,7 @@ import { getToolsForScenario } from '@/lib/tools-test-tools' const providerFreeScenarios = new Set([ 'server-context', 'client-context', + 'client-tool-error', 'client-server-context', 'malformed-tool-arguments', 'provider-rejected-tool-call', @@ -43,19 +44,30 @@ function createProviderFreeAdapter(scenario: string): AnyTextAdapter { state: undefined, toolName: 'check_status', } - : { - arguments: '{}', - initialText: 'Reading runtime context.', - input: {}, - name: 'runtime-context-test', - responseText: 'Runtime context was read.', - result: undefined, - state: undefined, - toolName: - scenario === 'client-context' - ? 'read_client_context' - : 'read_server_context', - } + : scenario === 'client-tool-error' + ? { + arguments: '{}', + initialText: 'Running the client tool.', + input: {}, + name: 'client-tool-error-test', + responseText: 'Recovered from client tool failure.', + result: undefined, + state: undefined, + toolName: 'fail_client_tool', + } + : { + arguments: '{}', + initialText: 'Reading runtime context.', + input: {}, + name: 'runtime-context-test', + responseText: 'Runtime context was read.', + result: undefined, + state: undefined, + toolName: + scenario === 'client-context' + ? 'read_client_context' + : 'read_server_context', + } return { kind: 'text', name: config.name, diff --git a/testing/e2e/src/routes/tools-test.tsx b/testing/e2e/src/routes/tools-test.tsx index 99c6a4970b..5d67f7d7d6 100644 --- a/testing/e2e/src/routes/tools-test.tsx +++ b/testing/e2e/src/routes/tools-test.tsx @@ -139,6 +139,19 @@ function createTrackedTools( } }) + const failClientTool = toolDefinition({ + name: 'fail_client_tool', + description: 'Fail while executing in the client', + inputSchema: z.object({}), + outputSchema: z.object({ completed: z.boolean() }), + }).client(async () => { + addEvent({ + type: 'execution-start', + toolName: 'fail_client_tool', + }) + throw new Error('Client tool failed') + }) + // Client-side stub for the server `delete_file` approval tool. Must be // registered with `needsApproval: true` so InterruptManager hydrates the // pause as `kind: 'tool-approval'` (matching schema hashes) instead of @@ -157,6 +170,7 @@ function createTrackedTools( readClientContextTool, showNotificationTool, displayChartTool, + failClientTool, deleteFileApprovalTool, ] } diff --git a/testing/e2e/tests/tool-error.spec.ts b/testing/e2e/tests/tool-error.spec.ts index f2d846afd0..5b9d26bcbb 100644 --- a/testing/e2e/tests/tool-error.spec.ts +++ b/testing/e2e/tests/tool-error.spec.ts @@ -36,6 +36,32 @@ test.describe('Tool Error Handling', () => { expect(failingCall?.state).toBe('error') }) + test('client tool that throws produces an error result and chat continues', async ({ + page, + testId, + aimockPort, + }) => { + await selectScenario(page, 'client-tool-error', testId, aimockPort) + await runTest(page) + await waitForTestComplete(page, 15000, 1) + + const metadata = await getMetadata(page) + expect(metadata.hasError).toBe('false') + + const toolCalls = await getToolCalls(page) + expect(toolCalls).toContainEqual( + expect.objectContaining({ name: 'fail_client_tool', state: 'error' }), + ) + + const messages = await getMessages(page) + const responseText = messages + .flatMap((message) => message.parts) + .filter((part) => part.type === 'text') + .map((part) => part.content) + .join(' ') + expect(responseText).toContain('Recovered from client tool failure.') + }) + test('malformed tool arguments produce an error result and chat continues', async ({ page, testId, From a8e25aa7627119d390a54506fae77e04b76c39d0 Mon Sep 17 00:00:00 2001 From: kolaworld Date: Mon, 24 Aug 2026 11:13:47 -0400 Subject: [PATCH 2/4] fix(ai-client): fail closed on stale client tool bindings --- packages/ai-client/src/interrupt-manager.ts | 14 ++++++++++++++ .../tests/chat-client-interrupts.test.ts | 19 ++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/packages/ai-client/src/interrupt-manager.ts b/packages/ai-client/src/interrupt-manager.ts index e2ed9069b0..d09362b5e9 100644 --- a/packages/ai-client/src/interrupt-manager.ts +++ b/packages/ai-client/src/interrupt-manager.ts @@ -889,6 +889,20 @@ export class InterruptManager< validationGeneration: 0, } } + return { + descriptor: interrupt, + binding: genericBinding(interrupt, hydration, candidate), + kind: 'generic', + status: 'error', + canResolve: false, + resumable: false, + error: this.itemError( + interrupt.id, + 'stale', + 'The client tool interrupt no longer matches the registered tool.', + ), + validationGeneration: 0, + } } if ( diff --git a/packages/ai-client/tests/chat-client-interrupts.test.ts b/packages/ai-client/tests/chat-client-interrupts.test.ts index 49cd99bbde..dcd238c1d5 100644 --- a/packages/ai-client/tests/chat-client-interrupts.test.ts +++ b/packages/ai-client/tests/chat-client-interrupts.test.ts @@ -290,7 +290,7 @@ describe('InterruptManager hydration', () => { expect(Object.isFrozen(snapshot[0]?.binding)).toBe(true) }) - it('hydrates a real core client-tool terminal with distinct schema identity hashes', async () => { + it('hydrates a real client-tool terminal with distinct schema hashes and rejects drift', async () => { const coreChunks = [ { type: EventType.RUN_STARTED, @@ -410,7 +410,7 @@ describe('InterruptManager hydration', () => { expect(binding.responseSchemaHash).toBe(expectedResponseSchemaHash) expect(binding.outputSchemaHash).not.toBe(binding.responseSchemaHash) - const { manager } = createManager() + const { manager, submit } = createManager() manager.hydrate({ threadId: 'core-thread', interruptedRunId: 'core-run', @@ -437,7 +437,20 @@ describe('InterruptManager hydration', () => { }, ], }) - expect(manager.getInterrupts()[0]?.kind).toBe('generic') + const stale = manager.getInterrupts()[0] + expect(stale).toMatchObject({ + kind: 'generic', + status: 'error', + canResolve: false, + errors: [{ code: 'stale' }], + }) + expect( + manager.resolveClientToolOutput('core-call', { + accountId: 'account-1', + }), + ).toBe(true) + await settle() + expect(submit).not.toHaveBeenCalled() }) it('keeps deprecated approval and client-tool reason aliases compatible', () => { From a1f7490ebea652151fa92064f91171e7024befde Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:23:17 +0000 Subject: [PATCH 3/4] ci: apply automated fixes --- testing/e2e/src/routes/api.tools-test.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/testing/e2e/src/routes/api.tools-test.ts b/testing/e2e/src/routes/api.tools-test.ts index 6bfb614968..670ffa7b6d 100644 --- a/testing/e2e/src/routes/api.tools-test.ts +++ b/testing/e2e/src/routes/api.tools-test.ts @@ -68,18 +68,18 @@ function createProviderFreeAdapter(scenario: string): AnyTextAdapter { toolName: 'show_notification', } : { - arguments: '{}', - initialText: 'Reading runtime context.', - input: {}, - name: 'runtime-context-test', - responseText: 'Runtime context was read.', - result: undefined, - state: undefined, - toolName: - scenario === 'client-context' - ? 'read_client_context' - : 'read_server_context', - } + arguments: '{}', + initialText: 'Reading runtime context.', + input: {}, + name: 'runtime-context-test', + responseText: 'Runtime context was read.', + result: undefined, + state: undefined, + toolName: + scenario === 'client-context' + ? 'read_client_context' + : 'read_server_context', + } return { kind: 'text', name: config.name, From 60b4f3aa80b191f797cfd7011c4961fdb4736035 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:22:54 +0000 Subject: [PATCH 4/4] ci: apply automated fixes --- testing/e2e/src/routes/api.tools-test.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/testing/e2e/src/routes/api.tools-test.ts b/testing/e2e/src/routes/api.tools-test.ts index 9d10d27ae7..34f1e48c5f 100644 --- a/testing/e2e/src/routes/api.tools-test.ts +++ b/testing/e2e/src/routes/api.tools-test.ts @@ -93,12 +93,11 @@ function createProviderFreeAdapter(scenario: string): AnyTextAdapter { : 'Runtime context was read.', result: undefined, state: undefined, - toolName: - stopsPendingTool - ? 'show_notification' - : scenario === 'client-context' - ? 'read_client_context' - : 'read_server_context', + toolName: stopsPendingTool + ? 'show_notification' + : scenario === 'client-context' + ? 'read_client_context' + : 'read_server_context', } return { kind: 'text',