From 1e43375cc85a774afdd3faad92ec4be4f8ca73cc Mon Sep 17 00:00:00 2001 From: Matt Apperson Date: Tue, 14 Apr 2026 15:52:25 -0400 Subject: [PATCH] feat(tools): add toModelOutput for custom tool result formatting Adds optional toModelOutput function to tool definitions that converts execution output to model-facing content blocks (Responses API format). This decouples what calling code receives from what the model sees. --- src/index.ts | 3 + src/lib/model-result.ts | 26 +++- src/lib/tool-types.ts | 44 ++++++ src/lib/tool.ts | 17 +++ tests/unit/create-tool.test.ts | 248 +++++++++++++++++++++++++++++++++ 5 files changed, 333 insertions(+), 5 deletions(-) diff --git a/src/index.ts b/src/index.ts index 04d6060..49ef829 100644 --- a/src/index.ts +++ b/src/index.ts @@ -137,12 +137,15 @@ export type { StepResult, StopCondition, StopWhen, + ToModelOutputFunction, + ToModelOutputResult, Tool, ToolApprovalCheck, ToolCallOutputEvent, ToolExecutionResult, ToolExecutionResultUnion, ToolHasApproval, + ToolOutputContentItem, ToolPreliminaryResultEvent, ToolResultEvent, ToolStreamEvent, diff --git a/src/lib/model-result.ts b/src/lib/model-result.ts index 6ff046e..56f42fa 100644 --- a/src/lib/model-result.ts +++ b/src/lib/model-result.ts @@ -841,15 +841,31 @@ export class ModelResult< value.preliminaryResultsForCall.length > 0 ? value.preliminaryResultsForCall : undefined, ); + let outputForModel: string | models.FunctionCallOutputItemOutputUnion1[]; + + if (value.result.error) { + outputForModel = JSON.stringify({ + error: value.result.error.message, + }); + } else if (value.tool.function.toModelOutput) { + // toModelOutput exists - call it (may throw, which surfaces the error) + const modelOutputResult = await value.tool.function.toModelOutput({ + output: value.result.result, + input: value.toolCall.arguments, + }); + outputForModel = + modelOutputResult.type === 'content' + ? modelOutputResult.value + : JSON.stringify(value.result.result); + } else { + outputForModel = JSON.stringify(value.result.result); + } + const executedOutput: models.FunctionCallOutputItem = { type: 'function_call_output' as const, id: `output_${value.toolCall.id}`, callId: value.toolCall.id, - output: value.result.error - ? JSON.stringify({ - error: value.result.error.message, - }) - : JSON.stringify(value.result.result), + output: outputForModel, }; toolResults.push(executedOutput); this.turnBroadcaster?.push({ diff --git a/src/lib/tool-types.ts b/src/lib/tool-types.ts index 0ea81ef..6860e16 100644 --- a/src/lib/tool-types.ts +++ b/src/lib/tool-types.ts @@ -151,6 +151,46 @@ export type ToolApprovalCheck = ( context: TurnContext, ) => boolean | Promise; +/** + * Content item types for tool output to model. + * These match the Responses API format for multimodal tool outputs. + */ +export type ToolOutputContentItem = + | { + type: 'input_text'; + text: string; + } + | { + type: 'input_image'; + detail: 'auto' | 'low' | 'high'; + imageUrl: string; + } + | { + type: 'input_file'; + fileId: string; + filename?: string; + }; + +/** + * Result of toModelOutput function. + * The 'content' type passes value array directly as tool output. + */ +export type ToModelOutputResult = { + type: 'content'; + value: ToolOutputContentItem[]; +}; + +/** + * Function to convert tool execution output to model-facing output. + * Receives the execute result and input arguments for full context. + * @template TInput - The tool's input type + * @template TOutput - The tool's output type + */ +export type ToModelOutputFunction = (params: { + output: TOutput; + input: TInput; +}) => ToModelOutputResult | Promise; + /** * Base tool function interface with inputSchema * @template TInput - Zod schema for tool input @@ -185,6 +225,8 @@ export interface ToolFunctionWithExecute< params: zodInfer, context?: ToolExecuteContext, ) => Promise> | zodInfer; + /** Convert tool execution output to model-facing output */ + toModelOutput?: ToModelOutputFunction, zodInfer>; } /** @@ -222,6 +264,8 @@ export interface ToolFunctionWithGenerator< params: zodInfer, context?: ToolExecuteContext, ) => AsyncGenerator | zodInfer, zodInfer | undefined>; + /** Convert tool execution output to model-facing output */ + toModelOutput?: ToModelOutputFunction, zodInfer>; } /** diff --git a/src/lib/tool.ts b/src/lib/tool.ts index 382d297..1c40b2d 100644 --- a/src/lib/tool.ts +++ b/src/lib/tool.ts @@ -2,6 +2,7 @@ import type { $ZodObject, $ZodShape, $ZodType, infer as zodInfer } from 'zod/v4/ import type { ManualTool, NextTurnParamsFunctions, + ToModelOutputFunction, Tool, ToolApprovalCheck, ToolExecuteContext, @@ -34,6 +35,8 @@ type RegularToolConfigWithOutput< params: zodInfer, context?: ToolExecuteContext, ) => Promise> | zodInfer; + /** Convert tool execution output to model-facing output */ + toModelOutput?: ToModelOutputFunction, zodInfer>; }; /** @@ -58,6 +61,8 @@ type RegularToolConfigWithoutOutput< params: zodInfer, context?: ToolExecuteContext, ) => Promise | TReturn; + /** Convert tool execution output to model-facing output */ + toModelOutput?: ToModelOutputFunction, TReturn>; }; /** @@ -83,6 +88,8 @@ type GeneratorToolConfig< params: zodInfer, context?: ToolExecuteContext, ) => AsyncGenerator | zodInfer>; + /** Convert tool execution output to model-facing output */ + toModelOutput?: ToModelOutputFunction, zodInfer>; }; /** @@ -122,6 +129,8 @@ type ToolConfigWithSharedContext> = { context?: ToolExecuteContext, TShared>, ) => AsyncGenerator) | false; + /** Convert tool execution output to model-facing output */ + toModelOutput?: ToModelOutputFunction, unknown>; }; //#endregion @@ -285,6 +294,10 @@ export function tool( fn.requireApproval = config.requireApproval; } + if ('toModelOutput' in config && config.toModelOutput !== undefined) { + fn.toModelOutput = config.toModelOutput; + } + return { type: ToolType.Function, function: fn, @@ -311,6 +324,10 @@ export function tool( ...(config.requireApproval !== undefined && { requireApproval: config.requireApproval, }), + ...('toModelOutput' in config && + config.toModelOutput !== undefined && { + toModelOutput: config.toModelOutput, + }), }; return { diff --git a/tests/unit/create-tool.test.ts b/tests/unit/create-tool.test.ts index 321c521..4bf8775 100644 --- a/tests/unit/create-tool.test.ts +++ b/tests/unit/create-tool.test.ts @@ -234,4 +234,252 @@ describe('tool', () => { expect(manualTool.function).not.toHaveProperty('execute'); }); }); + + describe('tool - toModelOutput', () => { + it('should create a tool with toModelOutput function', () => { + const imageGenTool = tool({ + name: 'image_gen', + inputSchema: z.object({ + prompt: z.string(), + }), + outputSchema: z.object({ + status: z.string(), + imageUrl: z.string(), + }), + execute: async (params) => { + return { + status: 'ok', + imageUrl: `https://example.com/${params.prompt}.png`, + }; + }, + toModelOutput: ({ output }) => ({ + type: 'content', + value: [ + { + type: 'input_text', + text: 'Image generated successfully.', + }, + { + type: 'input_image', + detail: 'auto', + imageUrl: output.imageUrl, + }, + ], + }), + }); + + expect(imageGenTool.type).toBe(ToolType.Function); + expect(imageGenTool.function.name).toBe('image_gen'); + expect(imageGenTool.function.toModelOutput).toBeDefined(); + expect(typeof imageGenTool.function.toModelOutput).toBe('function'); + }); + + it('toModelOutput receives both output and input', async () => { + let receivedOutput: unknown; + let receivedInput: unknown; + + const testTool = tool({ + name: 'test_tool', + inputSchema: z.object({ + prompt: z.string(), + style: z.string().optional(), + }), + execute: async (params) => { + return { + result: `Processed: ${params.prompt}`, + }; + }, + toModelOutput: ({ output, input }) => { + receivedOutput = output; + receivedInput = input; + return { + type: 'content', + value: [ + { + type: 'input_text', + text: 'Done', + }, + ], + }; + }, + }); + + // Execute the tool first + const output = await testTool.function.execute({ + prompt: 'hello', + style: 'modern', + }); + + // Then call toModelOutput manually to test it receives correct params + const modelOutput = testTool.function.toModelOutput!({ + output, + input: { + prompt: 'hello', + style: 'modern', + }, + }); + + expect(receivedOutput).toEqual({ + result: 'Processed: hello', + }); + expect(receivedInput).toEqual({ + prompt: 'hello', + style: 'modern', + }); + expect(modelOutput).toEqual({ + type: 'content', + value: [ + { + type: 'input_text', + text: 'Done', + }, + ], + }); + }); + + it('should support async toModelOutput function', async () => { + const asyncTool = tool({ + name: 'async_tool', + inputSchema: z.object({ + data: z.string(), + }), + execute: async () => { + return { + processed: true, + }; + }, + toModelOutput: async ({ output }) => { + // Simulate async work (e.g., fetching additional data) + await Promise.resolve(); + return { + type: 'content', + value: [ + { + type: 'input_text', + text: `Processed: ${output.processed}`, + }, + ], + }; + }, + }); + + const output = await asyncTool.function.execute({ + data: 'test', + }); + const modelOutput = await asyncTool.function.toModelOutput!({ + output, + input: { + data: 'test', + }, + }); + + expect(modelOutput).toEqual({ + type: 'content', + value: [ + { + type: 'input_text', + text: 'Processed: true', + }, + ], + }); + }); + + it('should support toModelOutput on tools without outputSchema', () => { + const noSchemaTool = tool({ + name: 'no_schema_tool', + inputSchema: z.object({ + input: z.string(), + }), + execute: (params) => { + return { + raw: params.input, + }; + }, + toModelOutput: ({ output }) => ({ + type: 'content', + value: [ + { + type: 'input_text', + text: `Output: ${output.raw}`, + }, + ], + }), + }); + + expect(noSchemaTool.function.toModelOutput).toBeDefined(); + + const output = noSchemaTool.function.execute({ + input: 'test', + }); + const modelOutput = noSchemaTool.function.toModelOutput!({ + output, + input: { + input: 'test', + }, + }); + + expect(modelOutput).toEqual({ + type: 'content', + value: [ + { + type: 'input_text', + text: 'Output: test', + }, + ], + }); + }); + + it('should support toModelOutput on generator tools', () => { + const generatorTool = tool({ + name: 'generator_tool', + inputSchema: z.object({ + query: z.string(), + }), + eventSchema: z.object({ + progress: z.number(), + }), + outputSchema: z.object({ + result: z.string(), + }), + execute: async function* (_params) { + yield { + progress: 50, + }; + yield { + result: 'done', + }; + }, + toModelOutput: ({ output }) => ({ + type: 'content', + value: [ + { + type: 'input_text', + text: `Final result: ${output.result}`, + }, + ], + }), + }); + + expect(generatorTool.function.toModelOutput).toBeDefined(); + + const modelOutput = generatorTool.function.toModelOutput!({ + output: { + result: 'completed', + }, + input: { + query: 'test', + }, + }); + + expect(modelOutput).toEqual({ + type: 'content', + value: [ + { + type: 'input_text', + text: 'Final result: completed', + }, + ], + }); + }); + }); });