diff --git a/packages/typescript/ai-client/src/chat-client.ts b/packages/typescript/ai-client/src/chat-client.ts index 1d1ba091..36571cb9 100644 --- a/packages/typescript/ai-client/src/chat-client.ts +++ b/packages/typescript/ai-client/src/chat-client.ts @@ -27,6 +27,10 @@ export class ChatClient { private currentStreamId: string | null = null private currentMessageId: string | null = null private postStreamActions: Array<() => Promise> = [] + // Track pending client tool executions to await them before stream finalization + private pendingToolExecutions: Map> = new Map() + // Flag to deduplicate continuation checks during action draining + private continuationPending = false private callbacksRef: { current: { @@ -127,31 +131,41 @@ export class ChatClient { ) } }, - onToolCall: async (args: { + onToolCall: (args: { toolCallId: string toolName: string input: any }) => { // Handle client-side tool execution automatically const clientTool = this.clientToolsRef.current.get(args.toolName) - if (clientTool?.execute) { - try { - const output = await clientTool.execute(args.input) - await this.addToolResult({ - toolCallId: args.toolCallId, - tool: args.toolName, - output, - state: 'output-available', - }) - } catch (error: any) { - await this.addToolResult({ - toolCallId: args.toolCallId, - tool: args.toolName, - output: null, - state: 'output-error', - errorText: error.message, - }) - } + const executeFunc = clientTool?.execute + if (executeFunc) { + // Create and track the execution promise + const executionPromise = (async () => { + try { + const output = await executeFunc(args.input) + await this.addToolResult({ + toolCallId: args.toolCallId, + tool: args.toolName, + output, + state: 'output-available', + }) + } catch (error: any) { + await this.addToolResult({ + toolCallId: args.toolCallId, + tool: args.toolName, + output: null, + state: 'output-error', + errorText: error.message, + }) + } finally { + // Remove from pending when complete + this.pendingToolExecutions.delete(args.toolCallId) + } + })() + + // Track the pending execution + this.pendingToolExecutions.set(args.toolCallId, executionPromise) } }, onApprovalRequest: (args: { @@ -221,6 +235,12 @@ export class ChatClient { await new Promise((resolve) => setTimeout(resolve, 0)) } + // Wait for all pending tool executions to complete before finalizing + // This ensures client tools finish before we check for continuation + if (this.pendingToolExecutions.size > 0) { + await Promise.all(this.pendingToolExecutions.values()) + } + // Finalize the stream this.processor.finalizeStream() @@ -288,9 +308,17 @@ export class ChatClient { * Stream a response from the LLM */ private async streamResponse(): Promise { + // Guard against concurrent streams - if already loading, skip + if (this.isLoading) { + return + } + this.setIsLoading(true) this.setError(undefined) this.abortController = new AbortController() + // Reset pending tool executions for the new stream + this.pendingToolExecutions.clear() + let streamCompletedSuccessfully = false try { // Get model messages for the LLM @@ -313,6 +341,7 @@ export class ChatClient { ) await this.processStream(stream) + streamCompletedSuccessfully = true } catch (err) { if (err instanceof Error) { if (err.name === 'AbortError') { @@ -327,6 +356,20 @@ export class ChatClient { // Drain any actions that were queued while the stream was in progress await this.drainPostStreamActions() + + // Continue conversation if the stream ended with a tool result (server tool completed) + if (streamCompletedSuccessfully) { + const messages = this.processor.getMessages() + const lastPart = messages.at(-1)?.parts.at(-1) + + if (lastPart?.type === 'tool-result' && this.shouldAutoSend()) { + try { + await this.checkForContinuation() + } catch (error) { + console.error('Failed to continue flow after tool result:', error) + } + } + } } } @@ -470,8 +513,18 @@ export class ChatClient { * Check if we should continue the flow and do so if needed */ private async checkForContinuation(): Promise { + // Prevent duplicate continuation attempts + if (this.continuationPending || this.isLoading) { + return + } + if (this.shouldAutoSend()) { - await this.streamResponse() + this.continuationPending = true + try { + await this.streamResponse() + } finally { + this.continuationPending = false + } } } diff --git a/packages/typescript/ai/src/activities/chat/index.ts b/packages/typescript/ai/src/activities/chat/index.ts index 95c01374..19e39cd5 100644 --- a/packages/typescript/ai/src/activities/chat/index.ts +++ b/packages/typescript/ai/src/activities/chat/index.ts @@ -712,7 +712,7 @@ class TextEngine< const clientToolResults = new Map() for (const message of this.messages) { - // todo remove any and fix this + // Check for UIMessage format (parts array) if (message.role === 'assistant' && (message as any).parts) { const parts = (message as any).parts for (const part of parts) { @@ -733,6 +733,15 @@ class TextEngine< } } } + + // Check for ModelMessage format (toolCalls array with approval info) + if (message.role === 'assistant' && message.toolCalls) { + for (const toolCall of message.toolCalls) { + if (toolCall.approval) { + approvals.set(toolCall.approval.id, toolCall.approval.approved) + } + } + } } return { approvals, clientToolResults } diff --git a/packages/typescript/ai/src/activities/chat/messages.ts b/packages/typescript/ai/src/activities/chat/messages.ts index 14c8dc62..c0ba1667 100644 --- a/packages/typescript/ai/src/activities/chat/messages.ts +++ b/packages/typescript/ai/src/activities/chat/messages.ts @@ -104,6 +104,14 @@ export function uiMessageToModelMessages( name: p.name, arguments: p.arguments, }, + // Include approval info if tool was approved/denied (for server to know the decision) + ...(p.state === 'approval-responded' && + p.approval?.approved !== undefined && { + approval: { + id: p.approval.id, + approved: p.approval.approved, + }, + }), })) : undefined diff --git a/packages/typescript/ai/src/types.ts b/packages/typescript/ai/src/types.ts index 9df621c6..6646ff4b 100644 --- a/packages/typescript/ai/src/types.ts +++ b/packages/typescript/ai/src/types.ts @@ -91,6 +91,11 @@ export interface ToolCall { name: string arguments: string // JSON string } + /** Approval info for tools requiring user approval (included in messages sent back to server) */ + approval?: { + id: string + approved: boolean + } } // ============================================================================ diff --git a/packages/typescript/smoke-tests/adapters/package.json b/packages/typescript/smoke-tests/adapters/package.json index 17f686eb..3ca2985c 100644 --- a/packages/typescript/smoke-tests/adapters/package.json +++ b/packages/typescript/smoke-tests/adapters/package.json @@ -6,8 +6,16 @@ "author": "", "license": "MIT", "type": "module", + "exports": { + ".": { + "import": "./src/llm-simulator.ts", + "types": "./src/llm-simulator.ts" + } + }, "scripts": { "start": "tsx src/cli.ts", + "test": "vitest run", + "test:watch": "vitest", "typecheck": "tsc --noEmit" }, "dependencies": { @@ -24,6 +32,7 @@ "dotenv": "^17.2.3", "tsx": "^4.20.6", "typescript": "5.9.3", + "vitest": "^4.0.14", "zod": "^4.2.0" } } diff --git a/packages/typescript/smoke-tests/adapters/src/llm-simulator.ts b/packages/typescript/smoke-tests/adapters/src/llm-simulator.ts new file mode 100644 index 00000000..5d81fa5c --- /dev/null +++ b/packages/typescript/smoke-tests/adapters/src/llm-simulator.ts @@ -0,0 +1,500 @@ +import type { + StreamChunk, + ChatOptions, + StructuredOutputOptions, + StructuredOutputResult, +} from '@tanstack/ai' + +/** + * Defines a tool call in the simulator script + */ +export interface SimulatorToolCall { + /** Tool name to call */ + name: string + /** Arguments to pass to the tool (will be JSON stringified) */ + arguments: Record + /** Optional custom tool call ID (auto-generated if not provided) */ + id?: string +} + +/** + * Defines a single iteration (LLM turn) in the simulator script + */ +export interface SimulatorIteration { + /** Text content to stream (optional) */ + content?: string + /** Tool calls to make (optional) */ + toolCalls?: Array + /** Finish reason - defaults to 'stop' if no tool calls, 'tool_calls' if has tool calls */ + finishReason?: 'stop' | 'tool_calls' | 'length' | null +} + +/** + * Complete script defining the LLM behavior + */ +export interface SimulatorScript { + /** Array of iterations (LLM turns) */ + iterations: Array + /** Model name to report in chunks (default: 'simulator-model') */ + model?: string +} + +/** + * LLM Simulator Adapter + * + * A deterministic mock adapter that yields predictable responses + * based on a pre-defined script. Useful for testing tool execution + * flows without depending on actual LLM behavior. + * + * @example + * ```typescript + * const script: SimulatorScript = { + * iterations: [ + * { + * content: "Let me check the temperature", + * toolCalls: [{ name: "get_temperature", arguments: { location: "Paris" } }] + * }, + * { + * content: "The temperature in Paris is 70 degrees" + * } + * ] + * } + * + * const adapter = createLLMSimulator(script) + * const stream = chat({ adapter, model: 'simulator', messages, tools }) + * ``` + */ +export class LLMSimulatorAdapter { + readonly kind = 'text' as const + readonly name = 'llm-simulator' + readonly models = ['simulator-model'] as const + + private script: SimulatorScript + private iterationIndex = 0 + private toolCallCounter = 0 + + constructor(script: SimulatorScript) { + this.script = script + } + + /** + * Reset the simulator to start from the first iteration + */ + reset(): void { + this.iterationIndex = 0 + this.toolCallCounter = 0 + } + + /** + * Get the current iteration index + */ + getCurrentIteration(): number { + return this.iterationIndex + } + + async *chatStream( + options: ChatOptions>, + ): AsyncIterable { + // Determine iteration based on message history for stateless operation across requests. + // This is primarily for E2E tests where each HTTP request creates a new adapter instance. + // + // Only apply message-based iteration when: + // 1. We're at index 0 (fresh adapter instance) + // 2. The script contains the tool calls we see in messages (full conversation script) + // + // For "continuation scripts" (unit tests) that only contain remaining iterations, + // we rely on the stateful iterationIndex. + if (this.iterationIndex === 0) { + const iterationFromMessages = this.determineIterationFromMessages( + options.messages, + ) + if (iterationFromMessages !== null && iterationFromMessages > 0) { + // Check if this script is a "full script" by seeing if iteration 0 + // has a tool call that matches one in the messages + const firstIterationToolCalls = this.script.iterations[0]?.toolCalls + const messagesHaveMatchingToolCall = + firstIterationToolCalls?.some((tc) => + this.isToolCallInMessages(tc.name, options.messages), + ) ?? false + + if ( + messagesHaveMatchingToolCall && + iterationFromMessages < this.script.iterations.length + ) { + // Full script mode: use message-based iteration + this.iterationIndex = iterationFromMessages + } + // Otherwise: continuation script mode, keep iterationIndex at 0 + } + } + + const iteration = this.script.iterations[this.iterationIndex] + + if (!iteration) { + // No more iterations - just return done + yield { + type: 'done', + id: this.generateId(), + model: this.script.model || 'simulator-model', + timestamp: Date.now(), + finishReason: 'stop', + } + return + } + + const baseId = this.generateId() + const model = this.script.model || 'simulator-model' + + // Yield content chunks if content is provided + if (iteration.content) { + // Split content into chunks for more realistic streaming + const words = iteration.content.split(' ') + let accumulated = '' + + for (let i = 0; i < words.length; i++) { + const word = words[i] + const delta = i === 0 ? word : ` ${word}` + accumulated += delta + + yield { + type: 'content', + id: baseId, + model, + timestamp: Date.now(), + delta, + content: accumulated, + role: 'assistant', + } + } + } + + // Yield tool call chunks if tool calls are provided + if (iteration.toolCalls && iteration.toolCalls.length > 0) { + for (let i = 0; i < iteration.toolCalls.length; i++) { + const toolCall = iteration.toolCalls[i]! + const toolCallId = + toolCall.id || `call-${++this.toolCallCounter}-${Date.now()}` + + yield { + type: 'tool_call', + id: baseId, + model, + timestamp: Date.now(), + toolCall: { + id: toolCallId, + type: 'function', + function: { + name: toolCall.name, + arguments: JSON.stringify(toolCall.arguments), + }, + }, + index: i, + } + } + } + + // Determine finish reason + let finishReason = iteration.finishReason + if (finishReason === undefined) { + finishReason = + iteration.toolCalls && iteration.toolCalls.length > 0 + ? 'tool_calls' + : 'stop' + } + + // Yield done chunk + yield { + type: 'done', + id: baseId, + model, + timestamp: Date.now(), + finishReason, + usage: { + promptTokens: 10, + completionTokens: 20, + totalTokens: 30, + }, + } + + // Advance to next iteration for next call + this.iterationIndex++ + } + + async structuredOutput( + _options: StructuredOutputOptions>, + ): Promise> { + // Simple mock implementation + return { + data: {}, + rawText: '{}', + } + } + + private generateId(): string { + return `sim-${Date.now()}-${Math.random().toString(36).substring(7)}` + } + + /** + * Check if a tool with the given name appears in the messages + */ + private isToolCallInMessages( + toolName: string, + messages: Array<{ + role: string + toolCalls?: Array<{ function: { name: string } }> + }>, + ): boolean { + for (const msg of messages) { + if (msg.role === 'assistant' && msg.toolCalls) { + for (const tc of msg.toolCalls) { + if (tc.function.name === toolName) { + return true + } + } + } + } + return false + } + + /** + * Determine which iteration we should be on based on message history. + * This enables stateless operation across requests - each request can + * determine the correct iteration based on how many tool call rounds + * have been completed. + * + * Logic: + * - Count assistant messages that have tool calls + * - For each such message, check if there are corresponding tool results + * - Tool results can be in: + * 1. Separate `role: 'tool'` messages with `toolCallId` + * 2. The `parts` array of assistant messages with `output` set + * - Completed tool call rounds = iterations we've already processed + */ + private determineIterationFromMessages( + messages: Array<{ + role: string + toolCalls?: Array<{ + id: string + approval?: { id: string; approved: boolean } + }> + toolCallId?: string + parts?: Array<{ + type: string + id?: string + output?: any + approval?: { approved?: boolean } + }> + }>, + ): number | null { + if (!messages || messages.length === 0) { + return 0 // Fresh conversation, start at iteration 0 + } + + // Find all assistant messages with tool calls + const assistantToolCallMessages = messages.filter( + (m) => m.role === 'assistant' && m.toolCalls && m.toolCalls.length > 0, + ) + + if (assistantToolCallMessages.length === 0) { + // No tool calls in history, might be first iteration or continuation + // Check if there's a user message (fresh start) + const hasUserMessage = messages.some((m) => m.role === 'user') + return hasUserMessage ? 0 : null + } + + // Get all completed tool call IDs from: + // 1. Separate tool result messages (role: 'tool') + // 2. Parts array with output set (client tool results) - UIMessage format + // 3. Parts array with approval.approved set (approval responses) - UIMessage format + // 4. toolCalls array with approval.approved set (approval responses) - ModelMessage format + const completedToolIds = new Set() + + for (const msg of messages) { + // Check for role: 'tool' messages (server tool results) + if (msg.role === 'tool' && msg.toolCallId) { + completedToolIds.add(msg.toolCallId) + } + + // Check for UIMessage format: parts with output or approval responses + if (msg.parts) { + for (const part of msg.parts) { + if (part.type === 'tool-call' && part.id) { + // Client tool results have output set + if (part.output !== undefined) { + completedToolIds.add(part.id) + } + // Approval tools are complete when approval.approved is set (true or false) + if (part.approval?.approved !== undefined) { + completedToolIds.add(part.id) + } + } + } + } + + // Check for ModelMessage format: toolCalls with approval info + if (msg.role === 'assistant' && msg.toolCalls) { + for (const tc of msg.toolCalls) { + // Approval tools are complete when approval.approved is set + if (tc.approval?.approved !== undefined) { + completedToolIds.add(tc.id) + } + } + } + } + + // Count how many complete tool call rounds we have + let completedRounds = 0 + for (const assistantMsg of assistantToolCallMessages) { + const toolCalls = assistantMsg.toolCalls as Array<{ id: string }> + const allToolsComplete = toolCalls.every((tc) => + completedToolIds.has(tc.id), + ) + if (allToolsComplete) { + completedRounds++ + } + } + + // The next iteration is completedRounds (0-indexed) + // e.g., if we've completed 1 round, we're on iteration 1 + return completedRounds + } +} + +/** + * Create a new LLM Simulator adapter with the given script + */ +export function createLLMSimulator( + script: SimulatorScript, +): LLMSimulatorAdapter { + return new LLMSimulatorAdapter(script) +} + +// ============================================================================ +// Pre-built Scripts for Common Scenarios +// ============================================================================ + +/** + * Script builders for common test scenarios + */ +export const SimulatorScripts = { + /** + * Script for a single server tool call + */ + singleServerTool( + toolName: string, + toolArgs: Record, + responseContent: string, + ): SimulatorScript { + return { + iterations: [ + { + content: `I'll use the ${toolName} tool.`, + toolCalls: [{ name: toolName, arguments: toolArgs }], + }, + { + content: responseContent, + }, + ], + } + }, + + /** + * Script for a single client tool call (no server execute) + */ + singleClientTool( + toolName: string, + toolArgs: Record, + responseContent: string, + ): SimulatorScript { + return { + iterations: [ + { + content: `I'll use the ${toolName} tool.`, + toolCalls: [{ name: toolName, arguments: toolArgs }], + }, + { + content: responseContent, + }, + ], + } + }, + + /** + * Script for a tool that requires approval + */ + approvalTool( + toolName: string, + toolArgs: Record, + responseAfterApproval: string, + ): SimulatorScript { + return { + iterations: [ + { + content: `I need to use ${toolName}, which requires your approval.`, + toolCalls: [{ name: toolName, arguments: toolArgs }], + }, + { + content: responseAfterApproval, + }, + ], + } + }, + + /** + * Script for sequential tool calls (tool A then tool B) + */ + sequentialTools( + tool1: { name: string; args: Record }, + tool2: { name: string; args: Record }, + finalResponse: string, + ): SimulatorScript { + return { + iterations: [ + { + content: `First, I'll use ${tool1.name}.`, + toolCalls: [{ name: tool1.name, arguments: tool1.args }], + }, + { + content: `Now I'll use ${tool2.name}.`, + toolCalls: [{ name: tool2.name, arguments: tool2.args }], + }, + { + content: finalResponse, + }, + ], + } + }, + + /** + * Script for multiple tools in the same turn + */ + parallelTools( + tools: Array<{ name: string; args: Record }>, + responseContent: string, + ): SimulatorScript { + return { + iterations: [ + { + content: `I'll use multiple tools at once.`, + toolCalls: tools.map((t) => ({ name: t.name, arguments: t.args })), + }, + { + content: responseContent, + }, + ], + } + }, + + /** + * Script for a simple text response (no tools) + */ + textOnly(content: string): SimulatorScript { + return { + iterations: [ + { + content, + }, + ], + } + }, +} diff --git a/packages/typescript/smoke-tests/adapters/src/tests/tools/approval.test.ts b/packages/typescript/smoke-tests/adapters/src/tests/tools/approval.test.ts new file mode 100644 index 00000000..157a15e3 --- /dev/null +++ b/packages/typescript/smoke-tests/adapters/src/tests/tools/approval.test.ts @@ -0,0 +1,380 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat, maxIterations, toolDefinition } from '@tanstack/ai' +import { z } from 'zod' +import { SimulatorScripts, createLLMSimulator } from '../../llm-simulator' +import type { SimulatorScript } from '../../llm-simulator' + +/** + * Helper to collect all chunks from a stream + */ +async function collectChunks(stream: AsyncIterable): Promise> { + const chunks: Array = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + return chunks +} + +describe('Approval Flow Tests', () => { + describe('Approval Requested', () => { + it('should emit approval-requested for tools with needsApproval', async () => { + const script = SimulatorScripts.approvalTool( + 'delete_file', + { path: '/tmp/important.txt' }, + 'The file has been deleted.', + ) + const adapter = createLLMSimulator(script) + + const executeFn = vi.fn(async (args: { path: string }) => { + return JSON.stringify({ deleted: true, path: args.path }) + }) + + const deleteTool = toolDefinition({ + name: 'delete_file', + description: 'Delete a file from the filesystem', + inputSchema: z.object({ + path: z.string().describe('The file path to delete'), + }), + needsApproval: true, + }).server(executeFn) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Delete /tmp/important.txt' }], + tools: [deleteTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Should have approval-requested chunk + const approvalChunks = chunks.filter( + (c) => c.type === 'approval-requested', + ) + expect(approvalChunks.length).toBe(1) + + const approvalChunk = approvalChunks[0] as any + expect(approvalChunk.toolName).toBe('delete_file') + expect(approvalChunk.input).toEqual({ path: '/tmp/important.txt' }) + expect(approvalChunk.approval.needsApproval).toBe(true) + + // Tool should NOT be executed yet (waiting for approval) + expect(executeFn).not.toHaveBeenCalled() + }) + + it('should stop iteration when approval is needed', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'I will delete the file for you.', + toolCalls: [ + { name: 'dangerous_action', arguments: { confirm: true } }, + ], + }, + { + content: 'Action completed.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const executeFn = vi.fn(async () => 'done') + + const tool = toolDefinition({ + name: 'dangerous_action', + description: 'A dangerous action requiring approval', + inputSchema: z.object({ confirm: z.boolean() }), + needsApproval: true, + }).server(executeFn) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Do the dangerous thing' }], + tools: [tool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Should stop after first iteration + const doneChunks = chunks.filter((c) => c.type === 'done') + expect(doneChunks.length).toBe(1) + + // Tool should not be executed + expect(executeFn).not.toHaveBeenCalled() + + // Simulator should be waiting at iteration 1 + expect(adapter.getCurrentIteration()).toBe(1) + }) + }) + + describe('Approval Accepted', () => { + it('should execute tool when approval is granted via messages', async () => { + const script: SimulatorScript = { + iterations: [ + { + // After receiving approval, LLM responds + content: 'The file has been successfully deleted.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const executeFn = vi.fn(async (args: { path: string }) => { + return JSON.stringify({ deleted: true, path: args.path }) + }) + + const deleteTool = toolDefinition({ + name: 'delete_file', + description: 'Delete a file', + inputSchema: z.object({ path: z.string() }), + needsApproval: true, + }).server(executeFn) + + // Messages with approval already granted + const messagesWithApproval = [ + { role: 'user' as const, content: 'Delete /tmp/test.txt' }, + { + role: 'assistant' as const, + content: 'I will delete the file.', + toolCalls: [ + { + id: 'call-1', + type: 'function' as const, + function: { + name: 'delete_file', + arguments: '{"path":"/tmp/test.txt"}', + }, + }, + ], + parts: [ + { + type: 'tool-call' as const, + id: 'call-1', + name: 'delete_file', + arguments: '{"path":"/tmp/test.txt"}', + state: 'approval-responded' as const, + approval: { + id: 'approval_call-1', + needsApproval: true, + approved: true, // User approved + }, + }, + ], + }, + ] + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: messagesWithApproval, + tools: [deleteTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Tool should have been executed because approval was granted + expect(executeFn).toHaveBeenCalledTimes(1) + expect(executeFn).toHaveBeenCalledWith({ path: '/tmp/test.txt' }) + + // Should have tool_result chunk + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(1) + }) + }) + + describe('Approval Denied', () => { + it('should not execute tool when approval is denied', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'I understand. I will not delete the file.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const executeFn = vi.fn(async () => 'deleted') + + const deleteTool = toolDefinition({ + name: 'delete_file', + description: 'Delete a file', + inputSchema: z.object({ path: z.string() }), + needsApproval: true, + }).server(executeFn) + + // Messages with approval denied + const messagesWithDenial = [ + { role: 'user' as const, content: 'Delete /tmp/test.txt' }, + { + role: 'assistant' as const, + content: 'I will delete the file.', + toolCalls: [ + { + id: 'call-1', + type: 'function' as const, + function: { + name: 'delete_file', + arguments: '{"path":"/tmp/test.txt"}', + }, + }, + ], + parts: [ + { + type: 'tool-call' as const, + id: 'call-1', + name: 'delete_file', + arguments: '{"path":"/tmp/test.txt"}', + state: 'approval-responded' as const, + approval: { + id: 'approval_call-1', + needsApproval: true, + approved: false, // User denied + }, + }, + ], + }, + ] + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: messagesWithDenial, + tools: [deleteTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Tool should NOT have been executed + expect(executeFn).not.toHaveBeenCalled() + + // Should have content response + const contentChunks = chunks.filter((c) => c.type === 'content') + expect(contentChunks.length).toBeGreaterThan(0) + }) + }) + + describe('Multiple Approval Tools', () => { + it('should handle multiple tools requiring approval', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'I need to perform two dangerous operations.', + toolCalls: [ + { name: 'tool_a', arguments: { value: 'A' } }, + { name: 'tool_b', arguments: { value: 'B' } }, + ], + }, + { + content: 'Both operations completed.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const executeFnA = vi.fn(async () => 'A done') + const executeFnB = vi.fn(async () => 'B done') + + const toolA = toolDefinition({ + name: 'tool_a', + description: 'Tool A', + inputSchema: z.object({ value: z.string() }), + needsApproval: true, + }).server(executeFnA) + + const toolB = toolDefinition({ + name: 'tool_b', + description: 'Tool B', + inputSchema: z.object({ value: z.string() }), + needsApproval: true, + }).server(executeFnB) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Do both operations' }], + tools: [toolA, toolB], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Should have approval-requested for both tools + const approvalChunks = chunks.filter( + (c) => c.type === 'approval-requested', + ) + expect(approvalChunks.length).toBe(2) + + // Neither tool should be executed + expect(executeFnA).not.toHaveBeenCalled() + expect(executeFnB).not.toHaveBeenCalled() + }) + }) + + describe('Mixed Approval and Non-Approval Tools', () => { + it('should execute non-approval tools and request approval for approval tools', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'I will check status and then delete.', + toolCalls: [ + { name: 'check_status', arguments: { id: '123' } }, + { name: 'delete_item', arguments: { id: '123' } }, + ], + }, + { + content: 'Done.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const checkExecute = vi.fn(async () => ({ status: 'active' })) + const deleteExecute = vi.fn(async () => ({ deleted: true })) + + const checkTool = toolDefinition({ + name: 'check_status', + description: 'Check status', + inputSchema: z.object({ id: z.string() }), + // No needsApproval - will execute immediately + }).server(checkExecute) + + const deleteTool = toolDefinition({ + name: 'delete_item', + description: 'Delete item', + inputSchema: z.object({ id: z.string() }), + needsApproval: true, // Needs approval + }).server(deleteExecute) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Check and delete 123' }], + tools: [checkTool, deleteTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Non-approval tool should execute + expect(checkExecute).toHaveBeenCalledTimes(1) + + // Approval tool should NOT execute (waiting for approval) + expect(deleteExecute).not.toHaveBeenCalled() + + // Should have approval request for delete tool + const approvalChunks = chunks.filter( + (c) => c.type === 'approval-requested', + ) + expect(approvalChunks.length).toBe(1) + expect((approvalChunks[0] as any).toolName).toBe('delete_item') + + // Check tool should have been executed (verify via mock call) + expect(checkExecute).toHaveBeenCalledWith({ id: '123' }) + }) + }) +}) diff --git a/packages/typescript/smoke-tests/adapters/src/tests/tools/client-tool.test.ts b/packages/typescript/smoke-tests/adapters/src/tests/tools/client-tool.test.ts new file mode 100644 index 00000000..53e5eb0d --- /dev/null +++ b/packages/typescript/smoke-tests/adapters/src/tests/tools/client-tool.test.ts @@ -0,0 +1,279 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat, maxIterations, toolDefinition } from '@tanstack/ai' +import { z } from 'zod' +import { SimulatorScripts, createLLMSimulator } from '../../llm-simulator' +import type { SimulatorScript } from '../../llm-simulator' + +/** + * Helper to collect all chunks from a stream + */ +async function collectChunks(stream: AsyncIterable): Promise> { + const chunks: Array = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + return chunks +} + +describe('Client Tool Tests', () => { + describe('Client Tool Without Execute (Definition Only)', () => { + it('should emit tool-input-available for client tool without execute', async () => { + const script = SimulatorScripts.singleClientTool( + 'show_notification', + { message: 'Hello World', type: 'info' }, + 'I have shown the notification.', + ) + const adapter = createLLMSimulator(script) + + // Client tool definition without execute function + const notificationTool = toolDefinition({ + name: 'show_notification', + description: 'Show a notification to the user', + inputSchema: z.object({ + message: z.string(), + type: z.enum(['info', 'warning', 'error']), + }), + }).client() // No execute function - client will handle it + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Show me a hello notification' }], + tools: [notificationTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Should have tool_call chunks + const toolCallChunks = chunks.filter((c) => c.type === 'tool_call') + expect(toolCallChunks.length).toBeGreaterThan(0) + + // Should have tool-input-available chunks (for client-side handling) + const inputAvailableChunks = chunks.filter( + (c) => c.type === 'tool-input-available', + ) + expect(inputAvailableChunks.length).toBe(1) + + const inputChunk = inputAvailableChunks[0] as any + expect(inputChunk.toolName).toBe('show_notification') + expect(inputChunk.input).toEqual({ message: 'Hello World', type: 'info' }) + }) + + it('should stop iteration when client tool needs input', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'I need to show something on screen.', + toolCalls: [ + { name: 'render_component', arguments: { component: 'Chart' } }, + ], + }, + { + // This iteration should NOT be reached until client provides result + content: 'The component has been rendered.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const clientTool = toolDefinition({ + name: 'render_component', + description: 'Render a UI component', + inputSchema: z.object({ component: z.string() }), + }).client() // No execute - waits for client + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Show me a chart' }], + tools: [clientTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // The stream should stop after first iteration (waiting for client) + const doneChunks = chunks.filter((c) => c.type === 'done') + expect(doneChunks.length).toBe(1) + + // Should have tool-input-available + const inputChunks = chunks.filter( + (c) => c.type === 'tool-input-available', + ) + expect(inputChunks.length).toBe(1) + + // Simulator should still be on iteration 1 (not advanced) + expect(adapter.getCurrentIteration()).toBe(1) + }) + }) + + describe('Client Tool With Execute', () => { + it('should execute client tool with execute function', async () => { + const script = SimulatorScripts.singleClientTool( + 'get_location', + {}, + 'You are in New York.', + ) + const adapter = createLLMSimulator(script) + + const executeFn = vi.fn(async () => { + return { latitude: 40.7128, longitude: -74.006, city: 'New York' } + }) + + const locationTool = toolDefinition({ + name: 'get_location', + description: 'Get current location', + inputSchema: z.object({}), + outputSchema: z.object({ + latitude: z.number(), + longitude: z.number(), + city: z.string(), + }), + }).client(executeFn) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Where am I?' }], + tools: [locationTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Client tool with execute should behave like server tool + expect(executeFn).toHaveBeenCalledTimes(1) + + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(1) + + const result = JSON.parse((toolResultChunks[0] as any).content) + expect(result.city).toBe('New York') + }) + }) + + describe('Simulating Client Tool Results (Message Injection)', () => { + it('should continue when client tool result is provided via messages', async () => { + // This simulates what happens when client sends back tool result + const script: SimulatorScript = { + iterations: [ + { + // LLM will receive the tool result and respond + content: + 'Based on the uploaded file, I can see it contains 100 lines.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const uploadTool = toolDefinition({ + name: 'upload_file', + description: 'Upload a file', + inputSchema: z.object({ filename: z.string() }), + }).client() // No execute - client handles + + // Simulate messages with tool result already present + // (as if client had previously provided the result) + const messagesWithToolResult = [ + { role: 'user' as const, content: 'Upload my file' }, + { + role: 'assistant' as const, + content: 'I will upload the file for you.', + toolCalls: [ + { + id: 'call-1', + type: 'function' as const, + function: { + name: 'upload_file', + arguments: '{"filename":"test.txt"}', + }, + }, + ], + parts: [ + { + type: 'tool-call' as const, + id: 'call-1', + name: 'upload_file', + arguments: '{"filename":"test.txt"}', + state: 'complete' as const, + output: { success: true, lines: 100 }, + }, + ], + }, + ] + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: messagesWithToolResult, + tools: [uploadTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Should get the response content + const contentChunks = chunks.filter((c) => c.type === 'content') + expect(contentChunks.length).toBeGreaterThan(0) + + const fullContent = contentChunks.map((c) => (c as any).content).join('') + expect(fullContent).toContain('100 lines') + }) + }) + + describe('Mixed Client Tools', () => { + it('should handle multiple client tools with different states', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'Let me help with both tasks.', + toolCalls: [ + { name: 'client_tool_a', arguments: { value: 'A' } }, + { name: 'client_tool_b', arguments: { value: 'B' } }, + ], + }, + { + content: 'Both tasks completed.', + }, + ], + } + const adapter = createLLMSimulator(script) + + // One client tool with execute, one without + const toolA = toolDefinition({ + name: 'client_tool_a', + description: 'Tool A', + inputSchema: z.object({ value: z.string() }), + }).client(async (args) => ({ processed: args.value })) + + const toolB = toolDefinition({ + name: 'client_tool_b', + description: 'Tool B', + inputSchema: z.object({ value: z.string() }), + }).client() // No execute + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Do both' }], + tools: [toolA, toolB], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Tool B should have tool-input-available (no execute) + const inputChunks = chunks.filter( + (c) => c.type === 'tool-input-available', + ) + expect(inputChunks.length).toBeGreaterThanOrEqual(1) + + // At least one should be for tool_b + const toolBInputs = inputChunks.filter( + (c) => (c as any).toolName === 'client_tool_b', + ) + expect(toolBInputs.length).toBe(1) + }) + }) +}) diff --git a/packages/typescript/smoke-tests/adapters/src/tests/tools/error-handling.test.ts b/packages/typescript/smoke-tests/adapters/src/tests/tools/error-handling.test.ts new file mode 100644 index 00000000..3aeb7f43 --- /dev/null +++ b/packages/typescript/smoke-tests/adapters/src/tests/tools/error-handling.test.ts @@ -0,0 +1,350 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat, maxIterations, toolDefinition } from '@tanstack/ai' +import { z } from 'zod' +import { createLLMSimulator } from '../../llm-simulator' +import type { SimulatorScript } from '../../llm-simulator' + +/** + * Helper to collect all chunks from a stream + */ +async function collectChunks(stream: AsyncIterable): Promise> { + const chunks: Array = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + return chunks +} + +describe('Error Handling Tests', () => { + describe('Tool Execution Errors', () => { + it('should handle tool that throws an error', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'Let me try that operation.', + toolCalls: [{ name: 'failing_tool', arguments: { input: 'test' } }], + }, + { + content: 'I encountered an error.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const failingExecute = vi.fn(async () => { + throw new Error('Tool execution failed: database connection error') + }) + + const failingTool = toolDefinition({ + name: 'failing_tool', + description: 'A tool that fails', + inputSchema: z.object({ input: z.string() }), + }).server(failingExecute) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Run the failing tool' }], + tools: [failingTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Tool should have been called + expect(failingExecute).toHaveBeenCalledTimes(1) + + // Should have a tool result with error + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(1) + + const result = JSON.parse((toolResultChunks[0] as any).content) + expect(result.error).toContain('database connection error') + }) + + it('should handle async rejection in tool', async () => { + const script: SimulatorScript = { + iterations: [ + { + toolCalls: [{ name: 'async_fail', arguments: {} }], + }, + { + content: 'Error handled.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const asyncFailExecute = vi.fn(async () => { + return Promise.reject(new Error('Async rejection')) + }) + + const tool = toolDefinition({ + name: 'async_fail', + description: 'Async failing tool', + inputSchema: z.object({}), + }).server(asyncFailExecute) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'test' }], + tools: [tool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(1) + + const result = JSON.parse((toolResultChunks[0] as any).content) + expect(result.error).toContain('Async rejection') + }) + }) + + describe('Unknown Tool', () => { + it('should handle call to unknown tool gracefully', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'Using the tool.', + toolCalls: [{ name: 'unknown_tool', arguments: { x: 1 } }], + }, + { + content: 'Done.', + }, + ], + } + const adapter = createLLMSimulator(script) + + // Only register a different tool + const knownTool = toolDefinition({ + name: 'known_tool', + description: 'A known tool', + inputSchema: z.object({}), + }).server(async () => 'result') + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'test' }], + tools: [knownTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Should have a tool result with error about unknown tool + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(1) + + const result = JSON.parse((toolResultChunks[0] as any).content) + expect(result.error).toContain('Unknown tool') + }) + }) + + describe('Tool With No Execute', () => { + it('should emit tool-input-available for tool definition without execute', async () => { + const script: SimulatorScript = { + iterations: [ + { + toolCalls: [ + { name: 'no_execute_tool', arguments: { data: 'test' } }, + ], + }, + ], + } + const adapter = createLLMSimulator(script) + + // Tool definition only (no .server() or .client() with execute) + const toolDef = toolDefinition({ + name: 'no_execute_tool', + description: 'Tool without execute', + inputSchema: z.object({ data: z.string() }), + }) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'test' }], + tools: [toolDef], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Should emit tool-input-available since there's no execute + const inputChunks = chunks.filter( + (c) => c.type === 'tool-input-available', + ) + expect(inputChunks.length).toBe(1) + }) + }) + + describe('Empty Tool Calls', () => { + it('should handle iteration with no tool calls gracefully', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'I will just respond without tools.', + // No toolCalls + }, + ], + } + const adapter = createLLMSimulator(script) + + const tool = toolDefinition({ + name: 'unused_tool', + description: 'Tool', + inputSchema: z.object({}), + }).server(vi.fn()) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'test' }], + tools: [tool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Should have content but no tool calls or results + const contentChunks = chunks.filter((c) => c.type === 'content') + const toolCallChunks = chunks.filter((c) => c.type === 'tool_call') + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + + expect(contentChunks.length).toBeGreaterThan(0) + expect(toolCallChunks.length).toBe(0) + expect(toolResultChunks.length).toBe(0) + }) + }) + + describe('Max Iterations', () => { + it('should stop after max iterations are reached', async () => { + // Script that would loop forever + const script: SimulatorScript = { + iterations: [ + { toolCalls: [{ name: 'loop_tool', arguments: {} }] }, + { toolCalls: [{ name: 'loop_tool', arguments: {} }] }, + { toolCalls: [{ name: 'loop_tool', arguments: {} }] }, + { toolCalls: [{ name: 'loop_tool', arguments: {} }] }, + { toolCalls: [{ name: 'loop_tool', arguments: {} }] }, + { toolCalls: [{ name: 'loop_tool', arguments: {} }] }, + { toolCalls: [{ name: 'loop_tool', arguments: {} }] }, + { toolCalls: [{ name: 'loop_tool', arguments: {} }] }, + { toolCalls: [{ name: 'loop_tool', arguments: {} }] }, + { toolCalls: [{ name: 'loop_tool', arguments: {} }] }, + ], + } + const adapter = createLLMSimulator(script) + + const execute = vi.fn(async () => 'continue') + + const tool = toolDefinition({ + name: 'loop_tool', + description: 'Looping tool', + inputSchema: z.object({}), + }).server(execute) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'loop' }], + tools: [tool], + agentLoopStrategy: maxIterations(3), // Limit to 3 iterations + }) + + const chunks = await collectChunks(stream) + + // Should stop at max iterations + expect(execute.mock.calls.length).toBeLessThanOrEqual(3) + + // Should have done chunks + const doneChunks = chunks.filter((c) => c.type === 'done') + expect(doneChunks.length).toBeGreaterThan(0) + }) + }) + + describe('Tool Returns Non-String', () => { + it('should handle tool returning object (auto-stringify)', async () => { + const script: SimulatorScript = { + iterations: [ + { + toolCalls: [{ name: 'object_tool', arguments: {} }], + }, + { + content: 'Got the object.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const tool = toolDefinition({ + name: 'object_tool', + description: 'Returns object', + inputSchema: z.object({}), + }).server(async () => { + // Return object directly (should be stringified) + return { key: 'value', nested: { a: 1 } } + }) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'test' }], + tools: [tool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(1) + + // Should be valid JSON + const content = (toolResultChunks[0] as any).content + const parsed = JSON.parse(content) + expect(parsed.key).toBe('value') + expect(parsed.nested.a).toBe(1) + }) + + it('should handle tool returning number', async () => { + const script: SimulatorScript = { + iterations: [ + { + toolCalls: [{ name: 'number_tool', arguments: {} }], + }, + { + content: 'Got the number.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const tool = toolDefinition({ + name: 'number_tool', + description: 'Returns number', + inputSchema: z.object({}), + }).server(async () => 42) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'test' }], + tools: [tool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(1) + + // Number should be stringified + const content = (toolResultChunks[0] as any).content + expect(content).toBe('42') + }) + }) +}) diff --git a/packages/typescript/smoke-tests/adapters/src/tests/tools/multi-tool.test.ts b/packages/typescript/smoke-tests/adapters/src/tests/tools/multi-tool.test.ts new file mode 100644 index 00000000..d6869ed7 --- /dev/null +++ b/packages/typescript/smoke-tests/adapters/src/tests/tools/multi-tool.test.ts @@ -0,0 +1,268 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat, maxIterations, toolDefinition } from '@tanstack/ai' +import { z } from 'zod' +import { SimulatorScripts, createLLMSimulator } from '../../llm-simulator' +import type { SimulatorScript } from '../../llm-simulator' + +/** + * Helper to collect all chunks from a stream + */ +async function collectChunks(stream: AsyncIterable): Promise> { + const chunks: Array = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + return chunks +} + +describe('Multi-Tool Tests', () => { + describe('Parallel Tool Execution', () => { + it('should execute multiple tools in the same iteration', async () => { + const script = SimulatorScripts.parallelTools( + [ + { name: 'get_weather', args: { city: 'NYC' } }, + { name: 'get_time', args: { timezone: 'EST' } }, + { name: 'get_news', args: { category: 'tech' } }, + ], + 'Here is the weather, time, and news.', + ) + const adapter = createLLMSimulator(script) + + const weatherExecute = vi.fn(async () => + JSON.stringify({ temp: 72, condition: 'sunny' }), + ) + const timeExecute = vi.fn(async () => + JSON.stringify({ time: '14:30', timezone: 'EST' }), + ) + const newsExecute = vi.fn(async () => + JSON.stringify({ headlines: ['AI advances'] }), + ) + + const weatherTool = toolDefinition({ + name: 'get_weather', + description: 'Get weather', + inputSchema: z.object({ city: z.string() }), + }).server(weatherExecute) + + const timeTool = toolDefinition({ + name: 'get_time', + description: 'Get time', + inputSchema: z.object({ timezone: z.string() }), + }).server(timeExecute) + + const newsTool = toolDefinition({ + name: 'get_news', + description: 'Get news', + inputSchema: z.object({ category: z.string() }), + }).server(newsExecute) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [ + { role: 'user', content: 'Give me weather, time, and news' }, + ], + tools: [weatherTool, timeTool, newsTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // All three tools should be executed + expect(weatherExecute).toHaveBeenCalledTimes(1) + expect(timeExecute).toHaveBeenCalledTimes(1) + expect(newsExecute).toHaveBeenCalledTimes(1) + + // Should have 3 tool results + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(3) + + // Should have 3 tool calls + const toolCallChunks = chunks.filter((c) => c.type === 'tool_call') + expect(toolCallChunks.length).toBe(3) + }) + + it('should handle different tool types in parallel', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'Executing multiple operations.', + toolCalls: [ + { name: 'server_tool', arguments: { value: 1 } }, + { name: 'approval_tool', arguments: { action: 'delete' } }, + { name: 'client_tool', arguments: { display: 'chart' } }, + ], + }, + { + content: 'Operations initiated.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const serverExecute = vi.fn(async () => ({ result: 'server done' })) + const approvalExecute = vi.fn(async () => ({ + result: 'approved action done', + })) + + const serverTool = toolDefinition({ + name: 'server_tool', + description: 'Server tool', + inputSchema: z.object({ value: z.number() }), + }).server(serverExecute) + + const approvalTool = toolDefinition({ + name: 'approval_tool', + description: 'Approval tool', + inputSchema: z.object({ action: z.string() }), + needsApproval: true, + }).server(approvalExecute) + + const clientTool = toolDefinition({ + name: 'client_tool', + description: 'Client tool', + inputSchema: z.object({ display: z.string() }), + }).client() // No execute + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Do all three' }], + tools: [serverTool, approvalTool, clientTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Server tool should execute + expect(serverExecute).toHaveBeenCalledTimes(1) + + // Approval tool should NOT execute (waiting for approval) + expect(approvalExecute).not.toHaveBeenCalled() + + // Should have approval-requested for approval tool + const approvalChunks = chunks.filter( + (c) => c.type === 'approval-requested', + ) + expect(approvalChunks.length).toBe(1) + + // Should have tool-input-available for client tool + const inputChunks = chunks.filter( + (c) => c.type === 'tool-input-available', + ) + expect(inputChunks.length).toBe(1) + }) + }) + + describe('Same Tool Called Multiple Times', () => { + it('should handle the same tool called multiple times with different args', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'Checking multiple cities.', + toolCalls: [ + { name: 'get_weather', arguments: { city: 'NYC' }, id: 'call-1' }, + { name: 'get_weather', arguments: { city: 'LA' }, id: 'call-2' }, + { + name: 'get_weather', + arguments: { city: 'Chicago' }, + id: 'call-3', + }, + ], + }, + { + content: 'Here is the weather for all three cities.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const weatherExecute = vi.fn(async (args: { city: string }) => { + const temps: Record = { NYC: 70, LA: 85, Chicago: 60 } + return JSON.stringify({ city: args.city, temp: temps[args.city] || 0 }) + }) + + const weatherTool = toolDefinition({ + name: 'get_weather', + description: 'Get weather', + inputSchema: z.object({ city: z.string() }), + }).server(weatherExecute) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [ + { role: 'user', content: 'Weather in NYC, LA, and Chicago' }, + ], + tools: [weatherTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Should be called 3 times + expect(weatherExecute).toHaveBeenCalledTimes(3) + expect(weatherExecute).toHaveBeenCalledWith({ city: 'NYC' }) + expect(weatherExecute).toHaveBeenCalledWith({ city: 'LA' }) + expect(weatherExecute).toHaveBeenCalledWith({ city: 'Chicago' }) + + // Should have 3 tool results + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(3) + }) + }) + + describe('Tool Selection', () => { + it('should only execute tools that are called', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'I only need tool B.', + toolCalls: [{ name: 'tool_b', arguments: {} }], + }, + { + content: 'Done with B.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const executeA = vi.fn(async () => 'A') + const executeB = vi.fn(async () => 'B') + const executeC = vi.fn(async () => 'C') + + const toolA = toolDefinition({ + name: 'tool_a', + description: 'Tool A', + inputSchema: z.object({}), + }).server(executeA) + + const toolB = toolDefinition({ + name: 'tool_b', + description: 'Tool B', + inputSchema: z.object({}), + }).server(executeB) + + const toolC = toolDefinition({ + name: 'tool_c', + description: 'Tool C', + inputSchema: z.object({}), + }).server(executeC) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Just use B' }], + tools: [toolA, toolB, toolC], + agentLoopStrategy: maxIterations(10), + }) + + await collectChunks(stream) + + // Only B should be executed + expect(executeA).not.toHaveBeenCalled() + expect(executeB).toHaveBeenCalledTimes(1) + expect(executeC).not.toHaveBeenCalled() + }) + }) +}) diff --git a/packages/typescript/smoke-tests/adapters/src/tests/tools/sequences.test.ts b/packages/typescript/smoke-tests/adapters/src/tests/tools/sequences.test.ts new file mode 100644 index 00000000..7bdc2ac5 --- /dev/null +++ b/packages/typescript/smoke-tests/adapters/src/tests/tools/sequences.test.ts @@ -0,0 +1,419 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat, maxIterations, toolDefinition } from '@tanstack/ai' +import { z } from 'zod' +import { SimulatorScripts, createLLMSimulator } from '../../llm-simulator' +import type { SimulatorScript } from '../../llm-simulator' + +/** + * Helper to collect all chunks from a stream + */ +async function collectChunks(stream: AsyncIterable): Promise> { + const chunks: Array = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + return chunks +} + +describe('Tool Sequence Tests', () => { + describe('Server Tool -> Server Tool', () => { + it('should execute sequential server tools', async () => { + const script = SimulatorScripts.sequentialTools( + { name: 'get_user', args: { userId: '123' } }, + { name: 'get_orders', args: { userId: '123' } }, + 'User has 5 orders.', + ) + const adapter = createLLMSimulator(script) + + const getUserExecute = vi.fn(async (args: { userId: string }) => { + return JSON.stringify({ id: args.userId, name: 'John' }) + }) + + const getOrdersExecute = vi.fn(async (args: { userId: string }) => { + return JSON.stringify({ orders: [1, 2, 3, 4, 5], count: 5 }) + }) + + const getUserTool = toolDefinition({ + name: 'get_user', + description: 'Get user by ID', + inputSchema: z.object({ userId: z.string() }), + }).server(getUserExecute) + + const getOrdersTool = toolDefinition({ + name: 'get_orders', + description: 'Get orders for user', + inputSchema: z.object({ userId: z.string() }), + }).server(getOrdersExecute) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Get orders for user 123' }], + tools: [getUserTool, getOrdersTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Both tools should be executed in sequence + expect(getUserExecute).toHaveBeenCalledTimes(1) + expect(getOrdersExecute).toHaveBeenCalledTimes(1) + + // Verify call order + expect(getUserExecute.mock.invocationCallOrder[0]).toBeLessThan( + getOrdersExecute.mock.invocationCallOrder[0]!, + ) + + // Should have 2 tool results + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(2) + }) + + it('should pass first tool result to context for second tool', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'Let me fetch the data.', + toolCalls: [{ name: 'fetch_data', arguments: { source: 'api' } }], + }, + { + content: 'Now I will process it.', + toolCalls: [ + { name: 'process_data', arguments: { format: 'json' } }, + ], + }, + { + content: 'Data processed successfully.', + }, + ], + } + const adapter = createLLMSimulator(script) + + let fetchResult = '' + const fetchExecute = vi.fn(async () => { + fetchResult = JSON.stringify({ raw: 'data123' }) + return fetchResult + }) + + const processExecute = vi.fn(async () => { + // In a real scenario, this would use the fetch result + return JSON.stringify({ processed: true }) + }) + + const fetchTool = toolDefinition({ + name: 'fetch_data', + description: 'Fetch data', + inputSchema: z.object({ source: z.string() }), + }).server(fetchExecute) + + const processTool = toolDefinition({ + name: 'process_data', + description: 'Process data', + inputSchema: z.object({ format: z.string() }), + }).server(processExecute) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Fetch and process data' }], + tools: [fetchTool, processTool], + agentLoopStrategy: maxIterations(10), + }) + + await collectChunks(stream) + + expect(fetchExecute).toHaveBeenCalledTimes(1) + expect(processExecute).toHaveBeenCalledTimes(1) + + // Process should be called after fetch + expect(fetchExecute.mock.invocationCallOrder[0]).toBeLessThan( + processExecute.mock.invocationCallOrder[0]!, + ) + }) + }) + + describe('Server Tool -> Client Tool', () => { + it('should execute server tool then request client tool input', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'First, let me check the data.', + toolCalls: [{ name: 'server_check', arguments: { id: 'abc' } }], + }, + { + content: 'Now please confirm on screen.', + toolCalls: [ + { name: 'client_confirm', arguments: { message: 'Proceed?' } }, + ], + }, + { + content: 'Great, all done!', + }, + ], + } + const adapter = createLLMSimulator(script) + + const serverExecute = vi.fn(async () => JSON.stringify({ valid: true })) + + const serverTool = toolDefinition({ + name: 'server_check', + description: 'Server-side check', + inputSchema: z.object({ id: z.string() }), + }).server(serverExecute) + + const clientTool = toolDefinition({ + name: 'client_confirm', + description: 'Client-side confirmation', + inputSchema: z.object({ message: z.string() }), + }).client() // No execute - handled by client + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Check and confirm' }], + tools: [serverTool, clientTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Server tool should execute + expect(serverExecute).toHaveBeenCalledTimes(1) + + // Should have tool result for server tool + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(1) + + // Should have tool-input-available for client tool + const inputChunks = chunks.filter( + (c) => c.type === 'tool-input-available', + ) + expect(inputChunks.length).toBe(1) + expect((inputChunks[0] as any).toolName).toBe('client_confirm') + }) + }) + + describe('Client Tool -> Server Tool', () => { + it('should execute client tool result then continue to server tool', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'Now I will process on server.', + toolCalls: [ + { name: 'server_process', arguments: { data: 'processed' } }, + ], + }, + { + content: 'Processing complete.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const serverExecute = vi.fn(async (args: any) => { + return { result: args.data + '_done' } + }) + + const clientTool = toolDefinition({ + name: 'client_collect', + description: 'Collect input from client', + inputSchema: z.object({}), + }).client() + + const serverTool = toolDefinition({ + name: 'server_process', + description: 'Process on server', + inputSchema: z.object({ data: z.string() }), + }).server(serverExecute) + + // Simulate that client tool already completed + const messagesWithClientResult = [ + { role: 'user' as const, content: 'Collect and process' }, + { + role: 'assistant' as const, + content: 'Let me collect your input.', + toolCalls: [ + { + id: 'call-1', + type: 'function' as const, + function: { + name: 'client_collect', + arguments: '{}', + }, + }, + ], + parts: [ + { + type: 'tool-call' as const, + id: 'call-1', + name: 'client_collect', + arguments: '{}', + state: 'complete' as const, + output: { userInput: 'client_data' }, + }, + ], + }, + ] + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: messagesWithClientResult, + tools: [clientTool, serverTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Server tool should execute + expect(serverExecute).toHaveBeenCalledTimes(1) + expect(serverExecute).toHaveBeenCalledWith({ data: 'processed' }) + + // Should have tool results (may include the client tool result that was injected) + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBeGreaterThanOrEqual(1) + }) + }) + + describe('Three Tool Sequence', () => { + it('should handle A -> B -> C tool sequence', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'Step 1', + toolCalls: [{ name: 'tool_a', arguments: { step: 1 } }], + }, + { + content: 'Step 2', + toolCalls: [{ name: 'tool_b', arguments: { step: 2 } }], + }, + { + content: 'Step 3', + toolCalls: [{ name: 'tool_c', arguments: { step: 3 } }], + }, + { + content: 'All three steps completed.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const callOrder: string[] = [] + + const toolA = toolDefinition({ + name: 'tool_a', + description: 'Tool A', + inputSchema: z.object({ step: z.number() }), + }).server(async () => { + callOrder.push('A') + return 'A done' + }) + + const toolB = toolDefinition({ + name: 'tool_b', + description: 'Tool B', + inputSchema: z.object({ step: z.number() }), + }).server(async () => { + callOrder.push('B') + return 'B done' + }) + + const toolC = toolDefinition({ + name: 'tool_c', + description: 'Tool C', + inputSchema: z.object({ step: z.number() }), + }).server(async () => { + callOrder.push('C') + return 'C done' + }) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Do A, B, C' }], + tools: [toolA, toolB, toolC], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // All tools should execute in order + expect(callOrder).toEqual(['A', 'B', 'C']) + + // Should have 3 tool results + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(3) + }) + }) + + describe('Parallel Tools in Sequence', () => { + it('should handle parallel tools followed by another tool', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'First, getting data from two sources.', + toolCalls: [ + { name: 'source_a', arguments: {} }, + { name: 'source_b', arguments: {} }, + ], + }, + { + content: 'Now combining results.', + toolCalls: [{ name: 'combine', arguments: {} }], + }, + { + content: 'Here are the combined results.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const callOrder: string[] = [] + + const sourceA = toolDefinition({ + name: 'source_a', + description: 'Source A', + inputSchema: z.object({}), + }).server(async () => { + callOrder.push('A') + return JSON.stringify({ source: 'A', data: [1, 2] }) + }) + + const sourceB = toolDefinition({ + name: 'source_b', + description: 'Source B', + inputSchema: z.object({}), + }).server(async () => { + callOrder.push('B') + return JSON.stringify({ source: 'B', data: [3, 4] }) + }) + + const combine = toolDefinition({ + name: 'combine', + description: 'Combine data', + inputSchema: z.object({}), + }).server(async () => { + callOrder.push('combine') + return JSON.stringify({ combined: [1, 2, 3, 4] }) + }) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Get and combine data' }], + tools: [sourceA, sourceB, combine], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // A and B should be called before combine + expect(callOrder.indexOf('A')).toBeLessThan(callOrder.indexOf('combine')) + expect(callOrder.indexOf('B')).toBeLessThan(callOrder.indexOf('combine')) + + // Should have 3 tool results + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(3) + }) + }) +}) diff --git a/packages/typescript/smoke-tests/adapters/src/tests/tools/server-tool.test.ts b/packages/typescript/smoke-tests/adapters/src/tests/tools/server-tool.test.ts new file mode 100644 index 00000000..cfc682be --- /dev/null +++ b/packages/typescript/smoke-tests/adapters/src/tests/tools/server-tool.test.ts @@ -0,0 +1,343 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat, maxIterations, toolDefinition } from '@tanstack/ai' +import { z } from 'zod' +import { SimulatorScripts, createLLMSimulator } from '../../llm-simulator' +import type { SimulatorScript } from '../../llm-simulator' + +/** + * Helper to collect all chunks from a stream + */ +async function collectChunks(stream: AsyncIterable): Promise> { + const chunks: Array = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + return chunks +} + +describe('Server Tool Tests', () => { + describe('Single Server Tool Execution', () => { + it('should execute a server tool and return the result', async () => { + const script = SimulatorScripts.singleServerTool( + 'get_temperature', + { location: 'San Francisco' }, + 'The temperature in San Francisco is 70 degrees.', + ) + const adapter = createLLMSimulator(script) + + const executeFn = vi.fn(async (args: { location: string }) => { + return `${args.location}: 70°F` + }) + + const temperatureTool = toolDefinition({ + name: 'get_temperature', + description: 'Get the current temperature for a location', + inputSchema: z.object({ + location: z.string().describe('The city name'), + }), + }).server(executeFn) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [ + { + role: 'user', + content: 'What is the temperature in San Francisco?', + }, + ], + tools: [temperatureTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + // Verify the tool was called + expect(executeFn).toHaveBeenCalledTimes(1) + expect(executeFn).toHaveBeenCalledWith({ location: 'San Francisco' }) + + // Verify we got tool call and tool result chunks + const toolCallChunks = chunks.filter((c) => c.type === 'tool_call') + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + + expect(toolCallChunks.length).toBeGreaterThan(0) + expect(toolResultChunks.length).toBe(1) + + // Verify the tool result content + const resultChunk = toolResultChunks[0] as any + expect(resultChunk.content).toContain('San Francisco') + expect(resultChunk.content).toContain('70') + }) + + it('should handle a tool with complex nested arguments', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'Let me search for that.', + toolCalls: [ + { + name: 'search_products', + arguments: { + query: 'laptop', + filters: { + minPrice: 500, + maxPrice: 2000, + brands: ['Apple', 'Dell'], + }, + limit: 10, + }, + }, + ], + }, + { + content: 'I found 5 laptops matching your criteria.', + }, + ], + } + const adapter = createLLMSimulator(script) + + const executeFn = vi.fn(async (args: any) => { + return JSON.stringify({ + products: [{ name: 'MacBook Pro', price: 1999 }], + total: 5, + }) + }) + + const searchTool = toolDefinition({ + name: 'search_products', + description: 'Search for products', + inputSchema: z.object({ + query: z.string(), + filters: z.object({ + minPrice: z.number(), + maxPrice: z.number(), + brands: z.array(z.string()), + }), + limit: z.number(), + }), + }).server(executeFn) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Find me a laptop' }], + tools: [searchTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + expect(executeFn).toHaveBeenCalledTimes(1) + expect(executeFn).toHaveBeenCalledWith({ + query: 'laptop', + filters: { + minPrice: 500, + maxPrice: 2000, + brands: ['Apple', 'Dell'], + }, + limit: 10, + }) + + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + expect(toolResultChunks.length).toBe(1) + }) + + it('should handle a tool that returns JSON', async () => { + const script = SimulatorScripts.singleServerTool( + 'get_user', + { userId: '123' }, + 'Here is the user information.', + ) + const adapter = createLLMSimulator(script) + + // Return an object (will be JSON.stringified by the framework) + const executeFn = vi.fn(async (args: { userId: string }) => { + return { + id: args.userId, + name: 'John Doe', + email: 'john@example.com', + } + }) + + const getUserTool = toolDefinition({ + name: 'get_user', + description: 'Get user by ID', + inputSchema: z.object({ + userId: z.string(), + }), + outputSchema: z.object({ + id: z.string(), + name: z.string(), + email: z.string(), + }), + }).server(executeFn) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Get user 123' }], + tools: [getUserTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + + expect(toolResultChunks.length).toBe(1) + // The content is the JSON-stringified result + const content = (toolResultChunks[0] as any).content + expect(content).toContain('123') + expect(content).toContain('John Doe') + }) + + it('should handle tool that returns an object result', async () => { + const script = SimulatorScripts.singleServerTool( + 'echo', + { message: 'Hello' }, + 'Echo complete.', + ) + const adapter = createLLMSimulator(script) + + // Return an object (framework handles stringification) + const executeFn = vi.fn(async (args: { message: string }) => { + return { echoed: args.message.toUpperCase() } + }) + + const echoTool = toolDefinition({ + name: 'echo', + description: 'Echo a message', + inputSchema: z.object({ message: z.string() }), + }).server(executeFn) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Echo hello' }], + tools: [echoTool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + + expect(executeFn).toHaveBeenCalledWith({ message: 'Hello' }) + expect(toolResultChunks.length).toBe(1) + expect((toolResultChunks[0] as any).content).toContain('HELLO') + }) + }) + + describe('Tool Execution Tracking', () => { + it('should track tool call ID correctly', async () => { + const script: SimulatorScript = { + iterations: [ + { + toolCalls: [ + { name: 'test_tool', arguments: {}, id: 'custom-call-id-123' }, + ], + }, + { content: 'Done' }, + ], + } + const adapter = createLLMSimulator(script) + + const executeFn = vi.fn(async () => 'result') + + const tool = toolDefinition({ + name: 'test_tool', + description: 'Test tool', + inputSchema: z.object({}), + }).server(executeFn) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'test' }], + tools: [tool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + const toolCallChunks = chunks.filter((c) => c.type === 'tool_call') + const toolResultChunks = chunks.filter((c) => c.type === 'tool_result') + + expect(toolCallChunks.length).toBeGreaterThan(0) + expect((toolCallChunks[0] as any).toolCall.id).toBe('custom-call-id-123') + expect((toolResultChunks[0] as any).toolCallId).toBe('custom-call-id-123') + }) + + it('should generate tool call ID if not provided', async () => { + const script: SimulatorScript = { + iterations: [ + { + toolCalls: [{ name: 'test_tool', arguments: {} }], + }, + { content: 'Done' }, + ], + } + const adapter = createLLMSimulator(script) + + const tool = toolDefinition({ + name: 'test_tool', + description: 'Test tool', + inputSchema: z.object({}), + }).server(async () => 'result') + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'test' }], + tools: [tool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + const toolCallChunks = chunks.filter((c) => c.type === 'tool_call') + + expect(toolCallChunks.length).toBeGreaterThan(0) + expect((toolCallChunks[0] as any).toolCall.id).toMatch(/^call-\d+-\d+$/) + }) + }) + + describe('Content and Tool Call Together', () => { + it('should handle content followed by tool call in same iteration', async () => { + const script: SimulatorScript = { + iterations: [ + { + content: 'Let me check that for you.', + toolCalls: [ + { name: 'check_status', arguments: { id: 'order-123' } }, + ], + }, + { content: 'Your order is on its way!' }, + ], + } + const adapter = createLLMSimulator(script) + + const executeFn = vi.fn(async () => JSON.stringify({ status: 'shipped' })) + + const tool = toolDefinition({ + name: 'check_status', + description: 'Check order status', + inputSchema: z.object({ id: z.string() }), + }).server(executeFn) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages: [{ role: 'user', content: 'Check my order' }], + tools: [tool], + agentLoopStrategy: maxIterations(10), + }) + + const chunks = await collectChunks(stream) + + const contentChunks = chunks.filter((c) => c.type === 'content') + const toolCallChunks = chunks.filter((c) => c.type === 'tool_call') + + // Should have content chunks from both iterations + expect(contentChunks.length).toBeGreaterThan(0) + expect(toolCallChunks.length).toBeGreaterThan(0) + expect(executeFn).toHaveBeenCalledWith({ id: 'order-123' }) + }) + }) +}) diff --git a/packages/typescript/smoke-tests/adapters/vitest.config.ts b/packages/typescript/smoke-tests/adapters/vitest.config.ts new file mode 100644 index 00000000..fbb124cd --- /dev/null +++ b/packages/typescript/smoke-tests/adapters/vitest.config.ts @@ -0,0 +1,26 @@ +import { defineConfig } from 'vitest/config' +import path from 'path' + +export default defineConfig({ + resolve: { + alias: { + '@tanstack/ai': path.resolve(__dirname, '../../ai/src/index.ts'), + }, + }, + test: { + globals: true, + environment: 'node', + include: ['src/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'dist/', + 'output/', + '**/*.test.ts', + '**/*.config.ts', + ], + }, + }, +}) diff --git a/packages/typescript/smoke-tests/e2e/package.json b/packages/typescript/smoke-tests/e2e/package.json index 3eefd342..2dcc5239 100644 --- a/packages/typescript/smoke-tests/e2e/package.json +++ b/packages/typescript/smoke-tests/e2e/package.json @@ -22,10 +22,12 @@ "@tanstack/react-router": "^1.141.1", "@tanstack/react-start": "^1.141.1", "@tanstack/router-plugin": "^1.139.7", + "@tanstack/tests-adapters": "workspace:*", "react": "^19.2.3", "react-dom": "^19.2.3", "tailwindcss": "^4.1.18", - "vite-tsconfig-paths": "^5.1.4" + "vite-tsconfig-paths": "^5.1.4", + "zod": "^4.2.0" }, "devDependencies": { "@playwright/test": "^1.57.0", diff --git a/packages/typescript/smoke-tests/e2e/src/routes/api.tools-test.ts b/packages/typescript/smoke-tests/e2e/src/routes/api.tools-test.ts new file mode 100644 index 00000000..1db5d5a8 --- /dev/null +++ b/packages/typescript/smoke-tests/e2e/src/routes/api.tools-test.ts @@ -0,0 +1,553 @@ +import { createFileRoute } from '@tanstack/react-router' +import { + chat, + maxIterations, + toStreamResponse, + toolDefinition, +} from '@tanstack/ai' +import { z } from 'zod' +import { + createLLMSimulator, + type SimulatorScript, +} from '@tanstack/tests-adapters' + +/** + * Pre-defined test scenarios for tool testing + */ +const SCENARIOS: Record = { + // Simple text response (no tools) + 'text-only': { + iterations: [ + { + content: 'This is a simple text response without any tools.', + }, + ], + }, + + // Single server tool + 'server-tool-single': { + iterations: [ + { + content: 'Let me get the weather for you.', + toolCalls: [ + { name: 'get_weather', arguments: { city: 'San Francisco' } }, + ], + }, + { + content: 'The weather in San Francisco is 72°F and sunny.', + }, + ], + }, + + // Single client tool + 'client-tool-single': { + iterations: [ + { + content: 'I need to show you a notification.', + toolCalls: [ + { + name: 'show_notification', + arguments: { message: 'Hello from the AI!', type: 'info' }, + }, + ], + }, + { + content: 'The notification has been shown.', + }, + ], + }, + + // Approval tool + 'approval-tool': { + iterations: [ + { + content: 'I need your permission to delete this file.', + toolCalls: [ + { + name: 'delete_file', + arguments: { path: '/tmp/test.txt' }, + }, + ], + }, + { + content: 'The file has been deleted.', + }, + ], + }, + + // Server tool -> Client tool sequence + 'sequence-server-client': { + iterations: [ + { + content: 'First, let me fetch the data.', + toolCalls: [{ name: 'fetch_data', arguments: { source: 'api' } }], + }, + { + content: 'Now let me display it on screen.', + toolCalls: [ + { + name: 'display_chart', + arguments: { type: 'bar', data: [1, 2, 3] }, + }, + ], + }, + { + content: 'The chart is now displayed.', + }, + ], + }, + + // Multiple tools in parallel + 'parallel-tools': { + iterations: [ + { + content: 'Let me gather all the information at once.', + toolCalls: [ + { name: 'get_weather', arguments: { city: 'NYC' } }, + { name: 'get_time', arguments: { timezone: 'EST' } }, + ], + }, + { + content: 'Here is the weather and time for NYC.', + }, + ], + }, + + // ========================================================================= + // RACE CONDITION / EVENT FLOW SCENARIOS + // These test the client-side event handling and continuation logic + // ========================================================================= + + // Two client tools in sequence - tests continuation after first client tool completes + 'sequential-client-tools': { + iterations: [ + { + content: 'First notification coming.', + toolCalls: [ + { + name: 'show_notification', + arguments: { message: 'First notification', type: 'info' }, + }, + ], + }, + { + content: 'Second notification coming.', + toolCalls: [ + { + name: 'show_notification', + arguments: { message: 'Second notification', type: 'warning' }, + }, + ], + }, + { + content: 'Both notifications have been shown.', + }, + ], + }, + + // Multiple client tools in parallel (same turn) - tests handling of concurrent client executions + 'parallel-client-tools': { + iterations: [ + { + content: 'Showing multiple things at once.', + toolCalls: [ + { + name: 'show_notification', + arguments: { message: 'Parallel 1', type: 'info' }, + }, + { + name: 'display_chart', + arguments: { type: 'bar', data: [1, 2, 3] }, + }, + ], + }, + { + content: 'All displayed.', + }, + ], + }, + + // Two approvals in sequence - tests approval flow continuation + 'sequential-approvals': { + iterations: [ + { + content: 'First I need to delete file A.', + toolCalls: [ + { + name: 'delete_file', + arguments: { path: '/tmp/a.txt' }, + }, + ], + }, + { + content: 'Now I need to delete file B.', + toolCalls: [ + { + name: 'delete_file', + arguments: { path: '/tmp/b.txt' }, + }, + ], + }, + { + content: 'Both files have been processed.', + }, + ], + }, + + // Multiple approvals in parallel (same turn) - tests handling of concurrent approvals + 'parallel-approvals': { + iterations: [ + { + content: 'I need to delete multiple files at once.', + toolCalls: [ + { + name: 'delete_file', + arguments: { path: '/tmp/parallel-a.txt' }, + }, + { + name: 'delete_file', + arguments: { path: '/tmp/parallel-b.txt' }, + }, + ], + }, + { + content: 'All files have been processed.', + }, + ], + }, + + // Client tool followed by approval - tests mixed flow + 'client-then-approval': { + iterations: [ + { + content: 'First a notification.', + toolCalls: [ + { + name: 'show_notification', + arguments: { message: 'Before approval', type: 'info' }, + }, + ], + }, + { + content: 'Now I need approval to delete.', + toolCalls: [ + { + name: 'delete_file', + arguments: { path: '/tmp/after-notify.txt' }, + }, + ], + }, + { + content: 'Complete.', + }, + ], + }, + + // Approval followed by client tool - tests that approval doesn't block subsequent client tools + 'approval-then-client': { + iterations: [ + { + content: 'First I need approval.', + toolCalls: [ + { + name: 'delete_file', + arguments: { path: '/tmp/before-notify.txt' }, + }, + ], + }, + { + content: 'Now showing notification.', + toolCalls: [ + { + name: 'show_notification', + arguments: { message: 'After approval', type: 'info' }, + }, + ], + }, + { + content: 'Complete.', + }, + ], + }, + + // Server tool followed by two client tools - tests complex continuation + 'server-then-two-clients': { + iterations: [ + { + content: 'Fetching data first.', + toolCalls: [{ name: 'fetch_data', arguments: { source: 'db' } }], + }, + { + content: 'First client action.', + toolCalls: [ + { + name: 'show_notification', + arguments: { message: 'Data fetched', type: 'info' }, + }, + ], + }, + { + content: 'Second client action.', + toolCalls: [ + { + name: 'display_chart', + arguments: { type: 'line', data: [10, 20, 30] }, + }, + ], + }, + { + content: 'All done.', + }, + ], + }, + + // Three client tools in sequence - stress test continuation logic + 'triple-client-sequence': { + iterations: [ + { + content: 'First step.', + toolCalls: [ + { + name: 'show_notification', + arguments: { message: 'Step 1', type: 'info' }, + }, + ], + }, + { + content: 'Second step.', + toolCalls: [ + { + name: 'display_chart', + arguments: { type: 'pie', data: [25, 25, 50] }, + }, + ], + }, + { + content: 'Third step.', + toolCalls: [ + { + name: 'show_notification', + arguments: { message: 'Step 3', type: 'warning' }, + }, + ], + }, + { + content: 'All three steps complete.', + }, + ], + }, +} + +/** + * Server-side tool definitions (for tools that execute on the server) + */ +const serverTools = { + get_weather: toolDefinition({ + name: 'get_weather', + description: 'Get weather for a city', + inputSchema: z.object({ + city: z.string(), + }), + }).server(async (args) => { + return JSON.stringify({ + city: args.city, + temperature: 72, + condition: 'sunny', + }) + }), + + fetch_data: toolDefinition({ + name: 'fetch_data', + description: 'Fetch data from a source', + inputSchema: z.object({ + source: z.string(), + }), + }).server(async (args) => { + return JSON.stringify({ + source: args.source, + data: [1, 2, 3, 4, 5], + }) + }), + + get_time: toolDefinition({ + name: 'get_time', + description: 'Get current time in timezone', + inputSchema: z.object({ + timezone: z.string(), + }), + }).server(async (args) => { + return JSON.stringify({ + timezone: args.timezone, + time: '14:30:00', + }) + }), + + delete_file: toolDefinition({ + name: 'delete_file', + description: 'Delete a file (requires approval)', + inputSchema: z.object({ + path: z.string(), + }), + needsApproval: true, + }).server(async (args) => { + return JSON.stringify({ + deleted: true, + path: args.path, + }) + }), +} + +/** + * Client-side tool definitions (tools that execute on the client) + * These use .client() without an execute function - execution happens on client side + */ +const clientToolDefinitions = { + show_notification: toolDefinition({ + name: 'show_notification', + description: 'Show a notification to the user', + inputSchema: z.object({ + message: z.string(), + type: z.enum(['info', 'warning', 'error']), + }), + }).client(), + + display_chart: toolDefinition({ + name: 'display_chart', + description: 'Display a chart on the screen', + inputSchema: z.object({ + type: z.enum(['bar', 'line', 'pie']), + data: z.array(z.number()), + }), + }).client(), +} + +export const Route = createFileRoute('/api/tools-test')({ + server: { + handlers: { + POST: async ({ request }) => { + const requestSignal = request.signal + + if (requestSignal?.aborted) { + return new Response(null, { status: 499 }) + } + + const abortController = new AbortController() + + try { + const body = await request.json() + // scenario is in body.data (from useChat body option) or body directly (legacy) + const messages = body.messages + const scenario = body.data?.scenario || body.scenario || 'text-only' + + // Get the script for this scenario + const script = SCENARIOS[scenario] + if (!script) { + return new Response( + JSON.stringify({ + error: `Unknown scenario: ${scenario}. Available: ${Object.keys(SCENARIOS).join(', ')}`, + }), + { + status: 400, + headers: { 'Content-Type': 'application/json' }, + }, + ) + } + + // Create simulator with the script + const adapter = createLLMSimulator(script) + + // Determine which tools to include based on the scenario + const tools = getToolsForScenario(scenario) + + const stream = chat({ + adapter, + model: 'simulator-model', + messages, + tools, + agentLoopStrategy: maxIterations(20), + abortController, + }) + + return toStreamResponse(stream, { abortController }) + } catch (error: any) { + console.error('[Tools Test API] Error:', error) + if (error.name === 'AbortError' || abortController.signal.aborted) { + return new Response(null, { status: 499 }) + } + return new Response( + JSON.stringify({ + error: error.message || 'An error occurred', + }), + { + status: 500, + headers: { 'Content-Type': 'application/json' }, + }, + ) + } + }, + }, + }, +}) + +/** + * Get the tools needed for a specific scenario + */ +function getToolsForScenario(scenario: string) { + switch (scenario) { + case 'text-only': + return [] + + case 'server-tool-single': + return [serverTools.get_weather] + + case 'client-tool-single': + return [clientToolDefinitions.show_notification] + + case 'approval-tool': + return [serverTools.delete_file] + + case 'sequence-server-client': + return [serverTools.fetch_data, clientToolDefinitions.display_chart] + + case 'parallel-tools': + return [serverTools.get_weather, serverTools.get_time] + + // Race condition / event flow scenarios + case 'sequential-client-tools': + return [clientToolDefinitions.show_notification] + + case 'parallel-client-tools': + return [ + clientToolDefinitions.show_notification, + clientToolDefinitions.display_chart, + ] + + case 'sequential-approvals': + return [serverTools.delete_file] + + case 'parallel-approvals': + return [serverTools.delete_file] + + case 'client-then-approval': + return [clientToolDefinitions.show_notification, serverTools.delete_file] + + case 'approval-then-client': + return [serverTools.delete_file, clientToolDefinitions.show_notification] + + case 'server-then-two-clients': + return [ + serverTools.fetch_data, + clientToolDefinitions.show_notification, + clientToolDefinitions.display_chart, + ] + + case 'triple-client-sequence': + return [ + clientToolDefinitions.show_notification, + clientToolDefinitions.display_chart, + ] + + default: + return [] + } +} diff --git a/packages/typescript/smoke-tests/e2e/src/routes/tools-test.tsx b/packages/typescript/smoke-tests/e2e/src/routes/tools-test.tsx new file mode 100644 index 00000000..d9c57238 --- /dev/null +++ b/packages/typescript/smoke-tests/e2e/src/routes/tools-test.tsx @@ -0,0 +1,616 @@ +import { useState, useCallback, useRef, useEffect } from 'react' +import { createFileRoute } from '@tanstack/react-router' +import { useChat, fetchServerSentEvents } from '@tanstack/ai-react' +import { toolDefinition } from '@tanstack/ai' +import { z } from 'zod' + +/** + * Event log entry for tracking tool execution flow + */ +interface ToolEvent { + timestamp: number + type: + | 'execution-start' + | 'execution-complete' + | 'approval-granted' + | 'approval-denied' + | 'error' + toolName: string + toolCallId?: string + details?: string +} + +/** + * Client-side tool definitions with execute functions + * These track execution for testing purposes + */ +function createTrackedTools( + addEvent: (event: Omit) => void, +) { + const showNotificationTool = toolDefinition({ + name: 'show_notification', + description: 'Show a notification to the user', + inputSchema: z.object({ + message: z.string(), + type: z.enum(['info', 'warning', 'error']), + }), + outputSchema: z.object({ + displayed: z.boolean(), + timestamp: z.number(), + }), + }).client(async (args) => { + addEvent({ + type: 'execution-start', + toolName: 'show_notification', + details: args.message, + }) + + // Simulate async work + await new Promise((r) => setTimeout(r, 50)) + + addEvent({ + type: 'execution-complete', + toolName: 'show_notification', + details: args.message, + }) + + return { + displayed: true, + timestamp: Date.now(), + } + }) + + const displayChartTool = toolDefinition({ + name: 'display_chart', + description: 'Display a chart on the screen', + inputSchema: z.object({ + type: z.enum(['bar', 'line', 'pie']), + data: z.array(z.number()), + }), + outputSchema: z.object({ + rendered: z.boolean(), + chartId: z.string(), + }), + }).client(async (args) => { + addEvent({ + type: 'execution-start', + toolName: 'display_chart', + details: args.type, + }) + + // Simulate async work + await new Promise((r) => setTimeout(r, 50)) + + addEvent({ + type: 'execution-complete', + toolName: 'display_chart', + details: args.type, + }) + + return { + rendered: true, + chartId: `chart-${Date.now()}`, + } + }) + + return [showNotificationTool, displayChartTool] +} + +// Available test scenarios +const SCENARIOS = [ + { id: 'text-only', label: 'Text Only (No Tools)', category: 'basic' }, + { id: 'server-tool-single', label: 'Single Server Tool', category: 'basic' }, + { id: 'client-tool-single', label: 'Single Client Tool', category: 'basic' }, + { id: 'approval-tool', label: 'Approval Required Tool', category: 'basic' }, + { + id: 'sequence-server-client', + label: 'Server → Client Sequence', + category: 'basic', + }, + { id: 'parallel-tools', label: 'Parallel Tools', category: 'basic' }, + // Race condition / event flow scenarios + { + id: 'sequential-client-tools', + label: 'Sequential Client Tools (2)', + category: 'race', + }, + { + id: 'parallel-client-tools', + label: 'Parallel Client Tools', + category: 'race', + }, + { + id: 'sequential-approvals', + label: 'Sequential Approvals (2)', + category: 'race', + }, + { id: 'parallel-approvals', label: 'Parallel Approvals', category: 'race' }, + { id: 'client-then-approval', label: 'Client → Approval', category: 'race' }, + { id: 'approval-then-client', label: 'Approval → Client', category: 'race' }, + { + id: 'server-then-two-clients', + label: 'Server → 2 Clients', + category: 'race', + }, + { + id: 'triple-client-sequence', + label: 'Triple Client Sequence', + category: 'race', + }, +] + +function ToolsTestPage() { + const [scenario, setScenario] = useState('text-only') + const [toolEvents, setToolEvents] = useState>([]) + const [testStartTime, setTestStartTime] = useState(null) + const [testComplete, setTestComplete] = useState(false) + + // Track approvals we've responded to (to avoid duplicate responses) + const respondedApprovals = useRef>(new Set()) + + // Create event logger + const addEvent = useCallback((event: Omit) => { + setToolEvents((prev) => [...prev, { ...event, timestamp: Date.now() }]) + }, []) + + // Create tracked tools (memoized since addEvent is stable) + const clientTools = useRef(createTrackedTools(addEvent)).current + + const { messages, sendMessage, isLoading, stop, addToolApprovalResponse } = + useChat({ + // Include scenario in ID so client is recreated when scenario changes + id: `tools-test-${scenario}`, + connection: fetchServerSentEvents('/api/tools-test'), + body: { scenario }, + tools: clientTools, + onFinish: () => { + setTestComplete(true) + }, + }) + + // Track when test completes (all tool calls are complete and not loading) + useEffect(() => { + if (!isLoading && testStartTime && messages.length > 1) { + // Get all tool results (for server tools) + const resultIds = new Set( + messages.flatMap((msg) => + msg.parts + .filter((p) => p.type === 'tool-result') + .map((p) => (p as { toolCallId: string }).toolCallId), + ), + ) + + // Check if any tool calls are still pending + const allToolCalls = messages.flatMap((msg) => + msg.parts.filter((p) => p.type === 'tool-call'), + ) + const pendingCalls = allToolCalls.filter( + (tc) => + tc.state !== 'complete' && + tc.state !== 'output-available' && + tc.output === undefined && + !resultIds.has(tc.id), + ) + if (pendingCalls.length === 0 && allToolCalls.length > 0) { + setTestComplete(true) + } + } + }, [isLoading, messages, testStartTime]) + + const handleSendMessage = useCallback(() => { + // Reset test state + setToolEvents([]) + setTestComplete(false) + setTestStartTime(Date.now()) + respondedApprovals.current.clear() + sendMessage('Run the test scenario') + }, [sendMessage]) + + // Extract tool call parts from messages for display + const toolCalls = messages.flatMap((msg) => + msg.parts + .filter((p) => p.type === 'tool-call') + .map((p) => ({ + messageId: msg.id, + ...p, + })), + ) + + // Extract tool result parts (for server tools) + const toolResultIds = new Set( + messages.flatMap((msg) => + msg.parts + .filter((p) => p.type === 'tool-result') + .map((p) => (p as { toolCallId: string }).toolCallId), + ), + ) + + // Extract approval requests + const pendingApprovals = toolCalls.filter( + (tc) => tc.approval?.needsApproval && tc.state === 'approval-requested', + ) + + return ( +
+

Tool Testing Page

+ + {/* Scenario Selector */} +
+ + + {testComplete && ( + + ✓ Test Complete + + )} +
+ + {/* Controls */} +
+ + {isLoading && ( + + )} +
+ + {/* Pending Approvals */} + {pendingApprovals.length > 0 && ( +
+

+ Pending Approvals ( + {pendingApprovals.length}) +

+ {pendingApprovals.map((tc) => ( +
+ + {tc.name}: {JSON.stringify(tc.arguments)} + + + +
+ ))} +
+ )} + + {/* Event Log - tracks execution flow for testing */} +
+

+ Event Log ({toolEvents.length}) +

+ {toolEvents.length === 0 ? ( +

No events yet

+ ) : ( +
+ {toolEvents.map((event, i) => ( +
+ [ + {new Date(event.timestamp) + .toISOString() + .split('T')[1] + ?.slice(0, 12)} + ] {event.type}: {event.toolName} + {event.details ? ` - ${event.details}` : ''} +
+ ))} +
+ )} +
+ + {/* Tool Calls Display */} +
+

Tool Calls

+ {toolCalls.length === 0 ? ( +

No tool calls yet

+ ) : ( + + + + + + + + + + + {toolCalls.map((tc) => ( + + + + + + + ))} + +
ToolStateArgumentsOutput
{tc.name} + + {tc.state} + + + {typeof tc.arguments === 'string' + ? tc.arguments + : JSON.stringify(tc.arguments)} + + {tc.output ? JSON.stringify(tc.output) : '-'} +
+ )} +
+ + {/* Messages JSON Display */} +
+ 
+          {JSON.stringify(messages, null, 2)}
+
+
+ + {/* Test metadata for assertions */} +
+ tc.state === 'complete' || + tc.state === 'output-available' || + tc.output !== undefined || + toolResultIds.has(tc.id), + ).length + } + data-event-count={toolEvents.length} + data-execution-start-count={ + toolEvents.filter((e) => e.type === 'execution-start').length + } + data-execution-complete-count={ + toolEvents.filter((e) => e.type === 'execution-complete').length + } + data-approval-granted-count={ + toolEvents.filter((e) => e.type === 'approval-granted').length + } + data-approval-denied-count={ + toolEvents.filter((e) => e.type === 'approval-denied').length + } + /> + + {/* Event log as JSON for easy parsing in tests */} +