From c121c24905a63823267e051274a6ceca0608da08 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 12 Nov 2025 21:17:04 -0500 Subject: [PATCH 1/7] effective logger --- packages/core/lib/v3/llm/aisdk.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/core/lib/v3/llm/aisdk.ts b/packages/core/lib/v3/llm/aisdk.ts index ecf0d716c..dd317bf77 100644 --- a/packages/core/lib/v3/llm/aisdk.ts +++ b/packages/core/lib/v3/llm/aisdk.ts @@ -40,8 +40,10 @@ export class AISdkClient extends LLMClient { async createChatCompletion({ options, + logger, }: CreateChatCompletionOptions): Promise { - this.logger?.({ + const effectiveLogger = logger ?? this.logger; + effectiveLogger?.({ category: "aisdk", message: "creating chat completion", level: 2, @@ -145,7 +147,7 @@ export class AISdkClient extends LLMClient { }); } catch (err) { if (NoObjectGeneratedError.isInstance(err)) { - this.logger?.({ + effectiveLogger?.({ category: "AISDK error", message: err.message, level: 0, @@ -191,7 +193,7 @@ export class AISdkClient extends LLMClient { }, } as T; - this.logger?.({ + effectiveLogger?.({ category: "aisdk", message: "response", level: 1, @@ -277,7 +279,7 @@ export class AISdkClient extends LLMClient { }, } as T; - this.logger?.({ + effectiveLogger?.({ category: "aisdk", message: "response", level: 2, From 761e2e05b39261fd87cd2d9d74c2b2c8e39b3e5e Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 12 Nov 2025 21:43:32 -0500 Subject: [PATCH 2/7] update console logger to logger --- packages/core/lib/inferenceLogUtils.ts | 7 +- .../core/lib/v3/agent/AnthropicCUAClient.ts | 36 ++++++++-- packages/core/lib/v3/agent/GoogleCUAClient.ts | 1 - packages/core/lib/v3/agent/OpenAICUAClient.ts | 1 - .../v3/agent/utils/googleCustomToolHandler.ts | 6 +- packages/core/lib/v3/dom/piercer.runtime.ts | 3 +- .../core/lib/v3/understudy/consoleMessage.ts | 15 +++- .../lib/v3/understudy/selectorResolver.ts | 68 +++++++++++++++++-- packages/evals/index.eval.ts | 3 +- .../evals/tasks/agent/github_react_version.ts | 10 ++- packages/evals/tasks/agent/hugging_face.ts | 10 ++- packages/evals/tasks/amazon_add_to_cart.ts | 10 ++- 12 files changed, 140 insertions(+), 30 deletions(-) diff --git a/packages/core/lib/inferenceLogUtils.ts b/packages/core/lib/inferenceLogUtils.ts index a6a60ca61..cdae1ef7d 100644 --- a/packages/core/lib/inferenceLogUtils.ts +++ b/packages/core/lib/inferenceLogUtils.ts @@ -107,8 +107,13 @@ function readSummaryFile(inferenceType: string): Record { ) { return parsed; } - } catch { + } catch (error) { // If we fail to parse for any reason, fall back to empty array + // This is a debug utility file - using console.warn for visibility + console.warn( + `Failed to parse inference summary file ${summaryPath}:`, + error instanceof Error ? error.message : String(error), + ); } return { [arrayKey]: [] }; } diff --git a/packages/core/lib/v3/agent/AnthropicCUAClient.ts b/packages/core/lib/v3/agent/AnthropicCUAClient.ts index d30905e03..7f40c5947 100644 --- a/packages/core/lib/v3/agent/AnthropicCUAClient.ts +++ b/packages/core/lib/v3/agent/AnthropicCUAClient.ts @@ -269,7 +269,7 @@ export class AnthropicCUAClient extends AgentClient { }); // Convert tool use to action and add to actions list - const action = this.convertToolUseToAction(toolUseItem); + const action = this.convertToolUseToAction(toolUseItem, logger); if (action) { logger({ category: "agent", @@ -501,7 +501,6 @@ export class AnthropicCUAClient extends AgentClient { usage, }; } catch (error) { - console.error("Error getting action from Anthropic:", error); throw error; } } @@ -736,7 +735,10 @@ export class AnthropicCUAClient extends AgentClient { return toolResults; } - private convertToolUseToAction(item: ToolUseItem): AgentAction | null { + private convertToolUseToAction( + item: ToolUseItem, + logger: (message: LogLine) => void, + ): AgentAction | null { try { const { name, input } = item; @@ -745,7 +747,14 @@ export class AnthropicCUAClient extends AgentClient { const action = input.action as string; if (!action) { - console.warn("Missing action in tool use item:", item); + logger({ + category: "agent", + message: "Missing action in tool use item", + level: 0, + auxiliary: { + item: { value: JSON.stringify(item), type: "object" }, + }, + }); return null; } @@ -901,10 +910,24 @@ export class AnthropicCUAClient extends AgentClient { return null; } - console.warn(`Unknown tool name: ${name}`); + logger({ + category: "agent", + message: `Unknown tool name: ${name}`, + level: 0, + }); return null; } catch (error) { - console.error("Error converting tool use to action:", error); + logger({ + category: "agent", + message: "Error converting tool use to action", + level: 0, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + }, + }); return null; } } @@ -924,7 +947,6 @@ export class AnthropicCUAClient extends AgentClient { const base64Image = await this.screenshotProvider(); return `data:image/png;base64,${base64Image}`; } catch (error) { - console.error("Error capturing screenshot:", error); throw error; } } diff --git a/packages/core/lib/v3/agent/GoogleCUAClient.ts b/packages/core/lib/v3/agent/GoogleCUAClient.ts index fb7462960..fa3aa7d29 100644 --- a/packages/core/lib/v3/agent/GoogleCUAClient.ts +++ b/packages/core/lib/v3/agent/GoogleCUAClient.ts @@ -894,7 +894,6 @@ export class GoogleCUAClient extends AgentClient { const base64Image = await this.screenshotProvider(); return `data:image/png;base64,${base64Image}`; } catch (error) { - console.error("Error capturing screenshot:", error); throw error; } } diff --git a/packages/core/lib/v3/agent/OpenAICUAClient.ts b/packages/core/lib/v3/agent/OpenAICUAClient.ts index 573c18a0e..3fcda32a0 100644 --- a/packages/core/lib/v3/agent/OpenAICUAClient.ts +++ b/packages/core/lib/v3/agent/OpenAICUAClient.ts @@ -432,7 +432,6 @@ export class OpenAICUAClient extends AgentClient { usage, }; } catch (error) { - console.error("Error getting action from OpenAI:", error); throw error; } } diff --git a/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts b/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts index 3343c775b..8ce13a008 100644 --- a/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts +++ b/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts @@ -136,10 +136,8 @@ function convertToolToFunctionDeclaration( parameters, }; } catch (error) { - console.error( - `Error converting tool ${name} to function declaration:`, - error, - ); + // Tool conversion failed - return null to filter this tool out + // This typically indicates an invalid tool schema definition return null; } } diff --git a/packages/core/lib/v3/dom/piercer.runtime.ts b/packages/core/lib/v3/dom/piercer.runtime.ts index e84d09cb2..5802201db 100644 --- a/packages/core/lib/v3/dom/piercer.runtime.ts +++ b/packages/core/lib/v3/dom/piercer.runtime.ts @@ -33,8 +33,7 @@ declare global { } export function installV3ShadowPiercer(opts: V3ShadowPatchOptions = {}): void { - // hardcoded debug (remove later if desired) - const DEBUG = true; + const DEBUG = opts.debug ?? false; type PatchedFn = Element["attachShadow"] & { __v3Patched?: boolean; diff --git a/packages/core/lib/v3/understudy/consoleMessage.ts b/packages/core/lib/v3/understudy/consoleMessage.ts index 2a8e22074..a53f3cc35 100644 --- a/packages/core/lib/v3/understudy/consoleMessage.ts +++ b/packages/core/lib/v3/understudy/consoleMessage.ts @@ -1,5 +1,6 @@ import type { Protocol } from "devtools-protocol"; import type { Page } from "./page"; +import { v3Logger } from "../logger"; type RemoteObject = Protocol.Runtime.RemoteObject; @@ -14,7 +15,19 @@ function formatRemoteObject(obj: RemoteObject | undefined): string { if (typeof value === "string") return value; try { return JSON.stringify(value); - } catch { + } catch (error) { + v3Logger({ + category: "ctx", + message: "Failed to stringify console message value", + level: 2, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + valueType: { value: typeof value, type: "string" }, + }, + }); return String(value); } } diff --git a/packages/core/lib/v3/understudy/selectorResolver.ts b/packages/core/lib/v3/understudy/selectorResolver.ts index 06b68301d..3b6d47865 100644 --- a/packages/core/lib/v3/understudy/selectorResolver.ts +++ b/packages/core/lib/v3/understudy/selectorResolver.ts @@ -331,7 +331,19 @@ export class FrameSelectorResolver { typeof data.count === "number" ? data.count : Number(data.count); if (!Number.isFinite(num)) return 0; return Math.max(0, Math.floor(num)); - } catch { + } catch (error) { + v3Logger({ + category: "locator", + message: "Failed to count CSS selector matches", + level: 2, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + selector: { value, type: "string" }, + }, + }); return 0; } } @@ -370,7 +382,19 @@ export class FrameSelectorResolver { : Number(evalRes.result.value); if (!Number.isFinite(num)) return 0; return Math.max(0, Math.floor(num)); - } catch { + } catch (error) { + v3Logger({ + category: "locator", + message: "Failed to count XPath matches", + level: 2, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + xpath: { value, type: "string" }, + }, + }); return 0; } } @@ -386,7 +410,19 @@ export class FrameSelectorResolver { { objectId }, ); nodeId = rn.nodeId ?? null; - } catch { + } catch (error) { + v3Logger({ + category: "locator", + message: "Failed to resolve node from object ID", + level: 2, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + objectId: { value: objectId, type: "string" }, + }, + }); nodeId = null; } @@ -418,7 +454,18 @@ export class FrameSelectorResolver { const num = typeof value === "number" ? value : Number(value); if (!Number.isFinite(num)) return 0; return Math.max(0, Math.floor(num)); - } catch { + } catch (error) { + v3Logger({ + category: "locator", + message: "Failed to evaluate count expression", + level: 2, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + }, + }); return 0; } } @@ -445,7 +492,18 @@ export class FrameSelectorResolver { } return this.resolveFromObjectId(evalRes.result.objectId); - } catch { + } catch (error) { + v3Logger({ + category: "locator", + message: "Failed to evaluate element expression", + level: 2, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + }, + }); return null; } } diff --git a/packages/evals/index.eval.ts b/packages/evals/index.eval.ts index 709fd2274..e34c7d38b 100644 --- a/packages/evals/index.eval.ts +++ b/packages/evals/index.eval.ts @@ -384,9 +384,8 @@ const generateFilteredTestcases = (): Testcase[] => { return result; } catch (error) { // Log any errors that occur during task execution - console.error(`❌ ${input.name}: Error - ${error}`); logger.error({ - message: `Error in task ${input.name}`, + message: `❌ ${input.name}: Error - ${error}`, level: 0, auxiliary: { error: { diff --git a/packages/evals/tasks/agent/github_react_version.ts b/packages/evals/tasks/agent/github_react_version.ts index ddf732723..a5f980dff 100644 --- a/packages/evals/tasks/agent/github_react_version.ts +++ b/packages/evals/tasks/agent/github_react_version.ts @@ -21,8 +21,14 @@ export const github_react_version: EvalFunction = async ({ question: "Does the page show the latest version of react and the date it was published", }); - console.log(`evaluation: ${evaluation}`); - console.log(`reasoning: ${reasoning}`); + logger.log({ + message: "Evaluation results", + level: 1, + auxiliary: { + evaluation: { value: evaluation, type: "string" }, + reasoning: { value: reasoning, type: "string" }, + }, + }); // only use url check for now, as using extract on the version is prone to breaking in future const success = evaluation === "YES"; if (!success) { diff --git a/packages/evals/tasks/agent/hugging_face.ts b/packages/evals/tasks/agent/hugging_face.ts index e70259de8..12b6063f0 100644 --- a/packages/evals/tasks/agent/hugging_face.ts +++ b/packages/evals/tasks/agent/hugging_face.ts @@ -17,7 +17,10 @@ export const hugging_face: EvalFunction = async ({ "Search for a model on Hugging Face with an Apache-2.0 license that has received the highest number of likes.", maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 20, }); - console.log(`agentResult: ${agentResult.message}`); + logger.log({ + message: `Agent result: ${agentResult.message}`, + level: 1, + }); const { evaluation, reasoning } = await evaluator.ask({ question: "Does the message mention 'kokoro-82m' or 'hexgrad/Kokoro-82M'?", @@ -27,7 +30,10 @@ export const hugging_face: EvalFunction = async ({ const success = evaluation === "YES"; - console.log(`reasoning: ${reasoning}`); + logger.log({ + message: `Evaluation reasoning: ${reasoning}`, + level: 1, + }); if (!success) { return { _success: false, diff --git a/packages/evals/tasks/amazon_add_to_cart.ts b/packages/evals/tasks/amazon_add_to_cart.ts index 07c1b0b5a..91cb7d7c5 100644 --- a/packages/evals/tasks/amazon_add_to_cart.ts +++ b/packages/evals/tasks/amazon_add_to_cart.ts @@ -20,8 +20,14 @@ export const amazon_add_to_cart: EvalFunction = async ({ const expectedUrl = "https://browserbase.github.io/stagehand-eval-sites/sites/amazon/sign-in.html"; - console.log("currentUrl", currentUrl); - console.log("expectedUrl", expectedUrl); + logger.log({ + message: "URL comparison", + level: 1, + auxiliary: { + currentUrl: { value: currentUrl, type: "string" }, + expectedUrl: { value: expectedUrl, type: "string" }, + }, + }); return { _success: currentUrl === expectedUrl, currentUrl, From 693bc95c1132247886de3eea3683b429ef2b3096 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Wed, 12 Nov 2025 21:50:37 -0500 Subject: [PATCH 3/7] more detailed google cua agent logger --- packages/core/lib/v3/agent/GoogleCUAClient.ts | 2 +- .../v3/agent/utils/googleCustomToolHandler.ts | 310 ++++++++++++++++-- 2 files changed, 289 insertions(+), 23 deletions(-) diff --git a/packages/core/lib/v3/agent/GoogleCUAClient.ts b/packages/core/lib/v3/agent/GoogleCUAClient.ts index fa3aa7d29..6bb213c16 100644 --- a/packages/core/lib/v3/agent/GoogleCUAClient.ts +++ b/packages/core/lib/v3/agent/GoogleCUAClient.ts @@ -466,7 +466,7 @@ export class GoogleCUAClient extends AgentClient { if (result.functionCalls.length > 0 || hasError) { // Filter out custom tool function calls as they've already been handled const computerUseFunctionCalls = result.functionCalls.filter( - (fc) => !isCustomTool(fc, this.tools), + (fc) => !isCustomTool(fc, this.tools, logger), ); if (computerUseFunctionCalls.length > 0) { diff --git a/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts b/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts index 8ce13a008..64f460eef 100644 --- a/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts +++ b/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts @@ -24,23 +24,68 @@ export async function executeGoogleCustomTool( functionCall: FunctionCall, logger: (message: LogLine) => void, ): Promise { + const startTime = Date.now(); + const toolCallId = `tool_${Date.now()}`; + try { logger({ category: "agent", - message: `Executing custom tool: ${toolName} with args: ${JSON.stringify(toolArgs)}`, + message: `Executing custom tool: ${toolName}`, level: 1, + auxiliary: { + toolName, + toolCallId, + arguments: toolArgs, + functionCallName: functionCall.name, + }, }); const tool = tools[toolName]; + if (!tool) { + const errorMessage = `Tool ${toolName} not found in toolset`; + logger({ + category: "agent", + message: errorMessage, + level: 0, + auxiliary: { + toolName, + availableTools: Object.keys(tools), + }, + }); + throw new Error(errorMessage); + } + + logger({ + category: "agent", + message: `Tool ${toolName} found, executing with ${Object.keys(toolArgs).length} argument(s)`, + level: 2, + auxiliary: { + toolName, + toolCallId, + argumentCount: Object.keys(toolArgs).length, + }, + }); + const toolResult = await tool.execute(toolArgs, { - toolCallId: `tool_${Date.now()}`, + toolCallId, messages: [], }); + const executionTime = Date.now() - startTime; + const resultString = JSON.stringify(toolResult); + const resultSize = resultString.length; + logger({ category: "agent", - message: `Tool ${toolName} completed successfully. Result: ${JSON.stringify(toolResult)}`, + message: `Tool ${toolName} completed successfully in ${executionTime}ms`, level: 1, + auxiliary: { + toolName, + toolCallId, + executionTime: { value: String(executionTime), unit: "ms" }, + resultSize: { value: String(resultSize), unit: "bytes" }, + result: toolResult, + }, }); // Create function response with the result @@ -48,7 +93,7 @@ export async function executeGoogleCustomTool( functionResponse: { name: toolName, response: { - result: JSON.stringify(toolResult), + result: resultString, }, }, }; @@ -58,13 +103,28 @@ export async function executeGoogleCustomTool( success: true, }; } catch (toolError) { + const executionTime = Date.now() - startTime; const errorMessage = toolError instanceof Error ? toolError.message : String(toolError); + const errorStack = toolError instanceof Error ? toolError.stack : undefined; + const errorType = + toolError instanceof Error + ? toolError.constructor.name + : typeof toolError; logger({ category: "agent", message: `Error executing custom tool ${toolName}: ${errorMessage}`, level: 0, + auxiliary: { + toolName, + toolCallId, + executionTime: { value: String(executionTime), unit: "ms" }, + errorType, + errorMessage, + ...(errorStack ? { errorStack } : {}), + arguments: toolArgs, + }, }); // Create error function response @@ -90,8 +150,24 @@ export async function executeGoogleCustomTool( export function isCustomTool( functionCall: FunctionCall, tools?: ToolSet, + logger?: (message: LogLine) => void, ): boolean { - return !!(tools && functionCall.name && functionCall.name in tools); + const isCustom = !!(tools && functionCall.name && functionCall.name in tools); + + if (logger) { + logger({ + category: "agent", + message: `Checking if function call "${functionCall.name}" is a custom tool: ${isCustom}`, + level: 2, + auxiliary: { + functionCallName: functionCall.name, + isCustomTool: isCustom, + availableCustomTools: tools ? Object.keys(tools) : [], + }, + }); + } + + return isCustom; } /** @@ -100,16 +176,56 @@ export function isCustomTool( */ export function convertToolSetToFunctionDeclarations( tools: ToolSet, + logger?: (message: LogLine) => void, ): FunctionDeclaration[] { + const toolCount = Object.keys(tools).length; + + if (logger) { + logger({ + category: "agent", + message: `Converting ${toolCount} tool(s) to Google FunctionDeclarations`, + level: 2, + auxiliary: { + toolCount, + toolNames: Object.keys(tools), + }, + }); + } + const functionDeclarations: FunctionDeclaration[] = []; + const failedConversions: string[] = []; for (const [name, tool] of Object.entries(tools)) { - const functionDeclaration = convertToolToFunctionDeclaration(name, tool); + const functionDeclaration = convertToolToFunctionDeclaration( + name, + tool, + logger, + ); if (functionDeclaration) { functionDeclarations.push(functionDeclaration); + } else { + failedConversions.push(name); } } + if (logger) { + logger({ + category: "agent", + message: `Converted ${functionDeclarations.length} of ${toolCount} tool(s) to FunctionDeclarations`, + level: functionDeclarations.length === toolCount ? 2 : 1, + auxiliary: { + successfulConversions: functionDeclarations.length, + totalTools: toolCount, + ...(failedConversions.length > 0 + ? { + failedConversions, + warning: "Some tools failed to convert and were excluded", + } + : {}), + }, + }); + } + return functionDeclarations; } @@ -119,8 +235,21 @@ export function convertToolSetToFunctionDeclarations( function convertToolToFunctionDeclaration( name: string, tool: { description?: string; inputSchema: unknown }, + logger?: (message: LogLine) => void, ): FunctionDeclaration | null { try { + if (logger) { + logger({ + category: "agent", + message: `Converting tool "${name}" to FunctionDeclaration`, + level: 2, + auxiliary: { + toolName: name, + hasDescription: !!tool.description, + }, + }); + } + // Convert Zod schema to JSON schema const jsonSchema = zodToJsonSchema(tool.inputSchema as z.ZodType) as { properties?: Record; @@ -128,16 +257,76 @@ function convertToolToFunctionDeclaration( type?: string; }; - const parameters = convertJsonSchemaToGoogleParameters(jsonSchema); + const propertyCount = jsonSchema.properties + ? Object.keys(jsonSchema.properties).length + : 0; + const requiredCount = jsonSchema.required?.length || 0; - return { + if (logger) { + logger({ + category: "agent", + message: `Tool "${name}" schema converted: ${propertyCount} property(ies), ${requiredCount} required`, + level: 2, + auxiliary: { + toolName: name, + propertyCount, + requiredCount, + properties: jsonSchema.properties + ? Object.keys(jsonSchema.properties) + : [], + }, + }); + } + + const parameters = convertJsonSchemaToGoogleParameters( + jsonSchema, + name, + logger, + ); + + const functionDeclaration = { name, description: tool.description || `Execute ${name}`, parameters, }; + + if (logger) { + logger({ + category: "agent", + message: `Successfully converted tool "${name}" to FunctionDeclaration`, + level: 2, + auxiliary: { + toolName: name, + parameterType: parameters.type, + parameterCount: Object.keys(parameters.properties || {}).length, + }, + }); + } + + return functionDeclaration; } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + const errorType = + error instanceof Error ? error.constructor.name : typeof error; + // Tool conversion failed - return null to filter this tool out // This typically indicates an invalid tool schema definition + if (logger) { + logger({ + category: "agent", + message: `Failed to convert tool "${name}" to FunctionDeclaration: ${errorMessage}`, + level: 0, + auxiliary: { + toolName: name, + errorType, + errorMessage, + ...(error instanceof Error && error.stack + ? { errorStack: error.stack } + : {}), + }, + }); + } + return null; } } @@ -145,16 +334,21 @@ function convertToolToFunctionDeclaration( /** * Convert JSON schema to Google's parameter format */ -function convertJsonSchemaToGoogleParameters(schema: { - properties?: Record; - required?: string[]; - type?: string; -}): { +function convertJsonSchemaToGoogleParameters( + schema: { + properties?: Record; + required?: string[]; + type?: string; + }, + toolName?: string, + logger?: (message: LogLine) => void, +): { type: Type; properties: Record; required?: string[]; } { const properties: Record = {}; + const typeMappings: Record = {}; if (schema.properties) { for (const [key, value] of Object.entries(schema.properties)) { @@ -163,8 +357,18 @@ function convertJsonSchemaToGoogleParameters(schema: { description?: string; items?: { type?: string }; }; + const jsonType = propSchema.type || "string"; + const googleType = mapJsonTypeToGoogleType( + jsonType, + key, + toolName, + logger, + ); + + typeMappings[key] = `${jsonType} -> ${Type[googleType]}`; + properties[key] = { - type: mapJsonTypeToGoogleType(propSchema.type || "string"), + type: googleType, ...(propSchema.description ? { description: propSchema.description } : {}), @@ -172,6 +376,19 @@ function convertJsonSchemaToGoogleParameters(schema: { } } + if (logger && Object.keys(typeMappings).length > 0) { + logger({ + category: "agent", + message: `Converted ${Object.keys(properties).length} property type(s) for ${toolName || "tool"}`, + level: 2, + auxiliary: { + ...(toolName ? { toolName } : {}), + typeMappings, + propertyCount: Object.keys(properties).length, + }, + }); + } + return { type: Type.OBJECT, properties, @@ -184,20 +401,69 @@ function convertJsonSchemaToGoogleParameters(schema: { /** * Map JSON schema types to Google's Type enum */ -function mapJsonTypeToGoogleType(jsonType: string): Type { - switch (jsonType.toLowerCase()) { +function mapJsonTypeToGoogleType( + jsonType: string, + propertyName?: string, + toolName?: string, + logger?: (message: LogLine) => void, +): Type { + const normalizedType = jsonType.toLowerCase(); + let mappedType: Type; + + switch (normalizedType) { case "string": - return Type.STRING; + mappedType = Type.STRING; + break; case "number": case "integer": - return Type.NUMBER; + mappedType = Type.NUMBER; + break; case "boolean": - return Type.BOOLEAN; + mappedType = Type.BOOLEAN; + break; case "array": - return Type.ARRAY; + mappedType = Type.ARRAY; + break; case "object": - return Type.OBJECT; + mappedType = Type.OBJECT; + break; default: - return Type.STRING; + mappedType = Type.STRING; + if (logger) { + logger({ + category: "agent", + message: `Unknown JSON schema type "${jsonType}", defaulting to STRING`, + level: 1, + auxiliary: { + ...(toolName ? { toolName } : {}), + ...(propertyName ? { propertyName } : {}), + originalType: jsonType, + mappedType: "STRING", + warning: "Type mapping fallback used", + }, + }); + } + break; + } + + if ( + logger && + normalizedType !== "string" && + normalizedType !== "number" && + normalizedType !== "integer" + ) { + logger({ + category: "agent", + message: `Mapped JSON type "${jsonType}" to Google type "${Type[mappedType]}"`, + level: 2, + auxiliary: { + ...(toolName ? { toolName } : {}), + ...(propertyName ? { propertyName } : {}), + originalType: jsonType, + mappedType: Type[mappedType], + }, + }); } + + return mappedType; } From 70d61d437f3994317a659f6cb48df92a47c10d8e Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Thu, 13 Nov 2025 09:43:17 -0500 Subject: [PATCH 4/7] fix lint and build --- .../core/lib/v3/agent/AnthropicCUAClient.ts | 182 +++++++++--------- packages/core/lib/v3/agent/GoogleCUAClient.ts | 8 +- packages/core/lib/v3/agent/OpenAICUAClient.ts | 112 ++++++----- .../v3/agent/utils/googleCustomToolHandler.ts | 179 +++++++++++------ 4 files changed, 265 insertions(+), 216 deletions(-) diff --git a/packages/core/lib/v3/agent/AnthropicCUAClient.ts b/packages/core/lib/v3/agent/AnthropicCUAClient.ts index 7f40c5947..c885707b1 100644 --- a/packages/core/lib/v3/agent/AnthropicCUAClient.ts +++ b/packages/core/lib/v3/agent/AnthropicCUAClient.ts @@ -402,107 +402,103 @@ export class AnthropicCUAClient extends AgentClient { id: string; usage: Record; }> { - try { - // For the API request, we use the inputItems directly - // These should already be properly formatted as a sequence of user/assistant messages - const messages: AnthropicMessage[] = []; - - for (const item of inputItems) { - if ("role" in item) { - // Skip system messages as Anthropic requires system as a top-level parameter - if (item.role !== "system") { - messages.push(item); - } + // For the API request, we use the inputItems directly + // These should already be properly formatted as a sequence of user/assistant messages + const messages: AnthropicMessage[] = []; + + for (const item of inputItems) { + if ("role" in item) { + // Skip system messages as Anthropic requires system as a top-level parameter + if (item.role !== "system") { + messages.push(item); } - // Note: We don't need special handling for tool_result items here anymore - // as they should already be properly wrapped in user messages } + // Note: We don't need special handling for tool_result items here anymore + // as they should already be properly wrapped in user messages + } - // Configure thinking capability if available - const thinking = this.thinkingBudget - ? { type: "enabled" as const, budget_tokens: this.thinkingBudget } - : undefined; - - // Create the request parameters - const requestParams: Record = { - model: this.modelName, - max_tokens: 4096, - messages: messages, - tools: [ - { - type: "computer_20250124", // Use the latest version for Claude 3.7 Sonnet - name: "computer", - display_width_px: this.currentViewport.width, - display_height_px: this.currentViewport.height, - display_number: 1, - }, - ], - betas: ["computer-use-2025-01-24"], - }; + // Configure thinking capability if available + const thinking = this.thinkingBudget + ? { type: "enabled" as const, budget_tokens: this.thinkingBudget } + : undefined; + + // Create the request parameters + const requestParams: Record = { + model: this.modelName, + max_tokens: 4096, + messages: messages, + tools: [ + { + type: "computer_20250124", // Use the latest version for Claude 3.7 Sonnet + name: "computer", + display_width_px: this.currentViewport.width, + display_height_px: this.currentViewport.height, + display_number: 1, + }, + ], + betas: ["computer-use-2025-01-24"], + }; - // Add custom tools if available - if (this.tools && Object.keys(this.tools).length > 0) { - const customTools = Object.entries(this.tools).map(([name, tool]) => { - // Convert Zod schema to proper JSON schema format for Anthropic - const jsonSchema = zodToJsonSchema(tool.inputSchema as z.ZodType) as { - properties?: Record; - required?: string[]; - }; + // Add custom tools if available + if (this.tools && Object.keys(this.tools).length > 0) { + const customTools = Object.entries(this.tools).map(([name, tool]) => { + // Convert Zod schema to proper JSON schema format for Anthropic + const jsonSchema = zodToJsonSchema(tool.inputSchema as z.ZodType) as { + properties?: Record; + required?: string[]; + }; - const inputSchema = { - type: "object", - properties: jsonSchema.properties || {}, - required: jsonSchema.required || [], - }; + const inputSchema = { + type: "object", + properties: jsonSchema.properties || {}, + required: jsonSchema.required || [], + }; - return { - name, - description: tool.description, - input_schema: inputSchema, - }; - }); + return { + name, + description: tool.description, + input_schema: inputSchema, + }; + }); - requestParams.tools = [ - ...(requestParams.tools as Record[]), - ...customTools, - ]; - } + requestParams.tools = [ + ...(requestParams.tools as Record[]), + ...customTools, + ]; + } - // Add system parameter if provided - if (this.userProvidedInstructions) { - requestParams.system = this.userProvidedInstructions; - } + // Add system parameter if provided + if (this.userProvidedInstructions) { + requestParams.system = this.userProvidedInstructions; + } - // Add thinking parameter if available - if (thinking) { - requestParams.thinking = thinking; - } + // Add thinking parameter if available + if (thinking) { + requestParams.thinking = thinking; + } - const startTime = Date.now(); - // Create the message using the Anthropic Messages API - // @ts-expect-error - The Anthropic SDK types are stricter than what we need - const response = await this.client.beta.messages.create(requestParams); - const endTime = Date.now(); - const elapsedMs = endTime - startTime; - const usage = { - input_tokens: response.usage.input_tokens, - output_tokens: response.usage.output_tokens, - inference_time_ms: elapsedMs, - }; + const startTime = Date.now(); + // Create the message using the Anthropic Messages API + // @ts-expect-error - The Anthropic SDK types are stricter than what we need + const response = await this.client.beta.messages.create(requestParams); + const endTime = Date.now(); + const elapsedMs = endTime - startTime; + const usage = { + input_tokens: response.usage.input_tokens, + output_tokens: response.usage.output_tokens, + inference_time_ms: elapsedMs, + }; - // Store the message ID for future use - this.lastMessageId = response.id; + // Store the message ID for future use + this.lastMessageId = response.id; - // Return the content and message ID - return { - // Cast the response content to our internal type - content: response.content as unknown as AnthropicContentBlock[], - id: response.id, - usage, - }; - } catch (error) { - throw error; - } + // Return the content and message ID + return { + // Cast the response content to our internal type + content: response.content as unknown as AnthropicContentBlock[], + id: response.id, + usage, + }; } async takeAction( @@ -943,12 +939,8 @@ export class AnthropicCUAClient extends AgentClient { // Use the screenshot provider if available if (this.screenshotProvider) { - try { - const base64Image = await this.screenshotProvider(); - return `data:image/png;base64,${base64Image}`; - } catch (error) { - throw error; - } + const base64Image = await this.screenshotProvider(); + return `data:image/png;base64,${base64Image}`; } throw new AgentScreenshotProviderError( diff --git a/packages/core/lib/v3/agent/GoogleCUAClient.ts b/packages/core/lib/v3/agent/GoogleCUAClient.ts index 6bb213c16..514b68eec 100644 --- a/packages/core/lib/v3/agent/GoogleCUAClient.ts +++ b/packages/core/lib/v3/agent/GoogleCUAClient.ts @@ -890,12 +890,8 @@ export class GoogleCUAClient extends AgentClient { // Use the screenshot provider if available if (this.screenshotProvider) { - try { - const base64Image = await this.screenshotProvider(); - return `data:image/png;base64,${base64Image}`; - } catch (error) { - throw error; - } + const base64Image = await this.screenshotProvider(); + return `data:image/png;base64,${base64Image}`; } throw new AgentScreenshotProviderError( diff --git a/packages/core/lib/v3/agent/OpenAICUAClient.ts b/packages/core/lib/v3/agent/OpenAICUAClient.ts index 3fcda32a0..65eb162a4 100644 --- a/packages/core/lib/v3/agent/OpenAICUAClient.ts +++ b/packages/core/lib/v3/agent/OpenAICUAClient.ts @@ -369,71 +369,67 @@ export class OpenAICUAClient extends AgentClient { responseId: string; usage: Record; }> { - try { - // Create the request parameters - const requestParams: Record = { - model: this.modelName, - tools: [ - { - type: "computer_use_preview", - display_width: this.currentViewport.width, - display_height: this.currentViewport.height, - environment: this.environment, - }, - ], - input: inputItems, - truncation: "auto", - }; + // Create the request parameters + const requestParams: Record = { + model: this.modelName, + tools: [ + { + type: "computer_use_preview", + display_width: this.currentViewport.width, + display_height: this.currentViewport.height, + environment: this.environment, + }, + ], + input: inputItems, + truncation: "auto", + }; - // Add custom tools if available - if (this.tools && Object.keys(this.tools).length > 0) { - const customTools = Object.entries(this.tools).map(([name, tool]) => ({ - type: "function" as const, + // Add custom tools if available + if (this.tools && Object.keys(this.tools).length > 0) { + const customTools = Object.entries(this.tools).map(([name, tool]) => ({ + type: "function" as const, + name, + function: { name, - function: { - name, - description: tool.description, - parameters: tool.inputSchema, - }, - })); - - requestParams.tools = [ - ...(requestParams.tools as Record[]), - ...customTools, - ]; - } + description: tool.description, + parameters: tool.inputSchema, + }, + })); - // Add previous_response_id if available - if (previousResponseId) { - requestParams.previous_response_id = previousResponseId; - } + requestParams.tools = [ + ...(requestParams.tools as Record[]), + ...customTools, + ]; + } - const startTime = Date.now(); - // Create the response using the OpenAI Responses API - // @ts-expect-error - Force type to match what the OpenAI SDK expects - const response = await this.client.responses.create(requestParams); - const endTime = Date.now(); - const elapsedMs = endTime - startTime; + // Add previous_response_id if available + if (previousResponseId) { + requestParams.previous_response_id = previousResponseId; + } - // Extract only the input_tokens and output_tokens - const usage = { - input_tokens: response.usage.input_tokens, - output_tokens: response.usage.output_tokens, - inference_time_ms: elapsedMs, - }; + const startTime = Date.now(); + // Create the response using the OpenAI Responses API + // @ts-expect-error - Force type to match what the OpenAI SDK expects + const response = await this.client.responses.create(requestParams); + const endTime = Date.now(); + const elapsedMs = endTime - startTime; + + // Extract only the input_tokens and output_tokens + const usage = { + input_tokens: response.usage.input_tokens, + output_tokens: response.usage.output_tokens, + inference_time_ms: elapsedMs, + }; - // Store the response ID for future use - this.lastResponseId = response.id; + // Store the response ID for future use + this.lastResponseId = response.id; - // Return the output and response ID - return { - output: response.output as unknown as ResponseItem[], - responseId: response.id, - usage, - }; - } catch (error) { - throw error; - } + // Return the output and response ID + return { + output: response.output as unknown as ResponseItem[], + responseId: response.id, + usage, + }; } async takeAction( diff --git a/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts b/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts index 64f460eef..978232c21 100644 --- a/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts +++ b/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts @@ -33,10 +33,10 @@ export async function executeGoogleCustomTool( message: `Executing custom tool: ${toolName}`, level: 1, auxiliary: { - toolName, - toolCallId, - arguments: toolArgs, - functionCallName: functionCall.name, + toolName: { value: toolName, type: "string" as const }, + toolCallId: { value: toolCallId, type: "string" as const }, + arguments: { value: JSON.stringify(toolArgs), type: "object" as const }, + functionCallName: { value: functionCall.name, type: "string" as const }, }, }); @@ -48,8 +48,11 @@ export async function executeGoogleCustomTool( message: errorMessage, level: 0, auxiliary: { - toolName, - availableTools: Object.keys(tools), + toolName: { value: toolName, type: "string" as const }, + availableTools: { + value: JSON.stringify(Object.keys(tools)), + type: "object" as const, + }, }, }); throw new Error(errorMessage); @@ -60,9 +63,12 @@ export async function executeGoogleCustomTool( message: `Tool ${toolName} found, executing with ${Object.keys(toolArgs).length} argument(s)`, level: 2, auxiliary: { - toolName, - toolCallId, - argumentCount: Object.keys(toolArgs).length, + toolName: { value: toolName, type: "string" as const }, + toolCallId: { value: toolCallId, type: "string" as const }, + argumentCount: { + value: String(Object.keys(toolArgs).length), + type: "integer" as const, + }, }, }); @@ -80,11 +86,11 @@ export async function executeGoogleCustomTool( message: `Tool ${toolName} completed successfully in ${executionTime}ms`, level: 1, auxiliary: { - toolName, - toolCallId, - executionTime: { value: String(executionTime), unit: "ms" }, - resultSize: { value: String(resultSize), unit: "bytes" }, - result: toolResult, + toolName: { value: toolName, type: "string" as const }, + toolCallId: { value: toolCallId, type: "string" as const }, + executionTime: { value: `${executionTime}ms`, type: "string" as const }, + resultSize: { value: `${resultSize} bytes`, type: "string" as const }, + result: { value: JSON.stringify(toolResult), type: "object" as const }, }, }); @@ -117,13 +123,15 @@ export async function executeGoogleCustomTool( message: `Error executing custom tool ${toolName}: ${errorMessage}`, level: 0, auxiliary: { - toolName, - toolCallId, - executionTime: { value: String(executionTime), unit: "ms" }, - errorType, - errorMessage, - ...(errorStack ? { errorStack } : {}), - arguments: toolArgs, + toolName: { value: toolName, type: "string" as const }, + toolCallId: { value: toolCallId, type: "string" as const }, + executionTime: { value: `${executionTime}ms`, type: "string" as const }, + errorType: { value: errorType, type: "string" as const }, + errorMessage: { value: errorMessage, type: "string" as const }, + ...(errorStack + ? { errorStack: { value: errorStack, type: "string" as const } } + : {}), + arguments: { value: JSON.stringify(toolArgs), type: "object" as const }, }, }); @@ -160,9 +168,12 @@ export function isCustomTool( message: `Checking if function call "${functionCall.name}" is a custom tool: ${isCustom}`, level: 2, auxiliary: { - functionCallName: functionCall.name, - isCustomTool: isCustom, - availableCustomTools: tools ? Object.keys(tools) : [], + functionCallName: { value: functionCall.name, type: "string" as const }, + isCustomTool: { value: String(isCustom), type: "boolean" as const }, + availableCustomTools: { + value: JSON.stringify(tools ? Object.keys(tools) : []), + type: "object" as const, + }, }, }); } @@ -186,8 +197,11 @@ export function convertToolSetToFunctionDeclarations( message: `Converting ${toolCount} tool(s) to Google FunctionDeclarations`, level: 2, auxiliary: { - toolCount, - toolNames: Object.keys(tools), + toolCount: { value: String(toolCount), type: "integer" as const }, + toolNames: { + value: JSON.stringify(Object.keys(tools)), + type: "object" as const, + }, }, }); } @@ -214,12 +228,21 @@ export function convertToolSetToFunctionDeclarations( message: `Converted ${functionDeclarations.length} of ${toolCount} tool(s) to FunctionDeclarations`, level: functionDeclarations.length === toolCount ? 2 : 1, auxiliary: { - successfulConversions: functionDeclarations.length, - totalTools: toolCount, + successfulConversions: { + value: String(functionDeclarations.length), + type: "integer" as const, + }, + totalTools: { value: String(toolCount), type: "integer" as const }, ...(failedConversions.length > 0 ? { - failedConversions, - warning: "Some tools failed to convert and were excluded", + failedConversions: { + value: JSON.stringify(failedConversions), + type: "object" as const, + }, + warning: { + value: "Some tools failed to convert and were excluded", + type: "string" as const, + }, } : {}), }, @@ -244,8 +267,11 @@ function convertToolToFunctionDeclaration( message: `Converting tool "${name}" to FunctionDeclaration`, level: 2, auxiliary: { - toolName: name, - hasDescription: !!tool.description, + toolName: { value: name, type: "string" as const }, + hasDescription: { + value: String(!!tool.description), + type: "boolean" as const, + }, }, }); } @@ -268,12 +294,21 @@ function convertToolToFunctionDeclaration( message: `Tool "${name}" schema converted: ${propertyCount} property(ies), ${requiredCount} required`, level: 2, auxiliary: { - toolName: name, - propertyCount, - requiredCount, - properties: jsonSchema.properties - ? Object.keys(jsonSchema.properties) - : [], + toolName: { value: name, type: "string" as const }, + propertyCount: { + value: String(propertyCount), + type: "integer" as const, + }, + requiredCount: { + value: String(requiredCount), + type: "integer" as const, + }, + properties: { + value: JSON.stringify( + jsonSchema.properties ? Object.keys(jsonSchema.properties) : [], + ), + type: "object" as const, + }, }, }); } @@ -296,9 +331,15 @@ function convertToolToFunctionDeclaration( message: `Successfully converted tool "${name}" to FunctionDeclaration`, level: 2, auxiliary: { - toolName: name, - parameterType: parameters.type, - parameterCount: Object.keys(parameters.properties || {}).length, + toolName: { value: name, type: "string" as const }, + parameterType: { + value: String(parameters.type), + type: "string" as const, + }, + parameterCount: { + value: String(Object.keys(parameters.properties || {}).length), + type: "integer" as const, + }, }, }); } @@ -317,11 +358,11 @@ function convertToolToFunctionDeclaration( message: `Failed to convert tool "${name}" to FunctionDeclaration: ${errorMessage}`, level: 0, auxiliary: { - toolName: name, - errorType, - errorMessage, + toolName: { value: name, type: "string" as const }, + errorType: { value: errorType, type: "string" as const }, + errorMessage: { value: errorMessage, type: "string" as const }, ...(error instanceof Error && error.stack - ? { errorStack: error.stack } + ? { errorStack: { value: error.stack, type: "string" as const } } : {}), }, }); @@ -382,9 +423,17 @@ function convertJsonSchemaToGoogleParameters( message: `Converted ${Object.keys(properties).length} property type(s) for ${toolName || "tool"}`, level: 2, auxiliary: { - ...(toolName ? { toolName } : {}), - typeMappings, - propertyCount: Object.keys(properties).length, + ...(toolName + ? { toolName: { value: toolName, type: "string" as const } } + : {}), + typeMappings: { + value: JSON.stringify(typeMappings), + type: "object" as const, + }, + propertyCount: { + value: String(Object.keys(properties).length), + type: "integer" as const, + }, }, }); } @@ -435,11 +484,23 @@ function mapJsonTypeToGoogleType( message: `Unknown JSON schema type "${jsonType}", defaulting to STRING`, level: 1, auxiliary: { - ...(toolName ? { toolName } : {}), - ...(propertyName ? { propertyName } : {}), - originalType: jsonType, - mappedType: "STRING", - warning: "Type mapping fallback used", + ...(toolName + ? { toolName: { value: toolName, type: "string" as const } } + : {}), + ...(propertyName + ? { + propertyName: { + value: propertyName, + type: "string" as const, + }, + } + : {}), + originalType: { value: jsonType, type: "string" as const }, + mappedType: { value: "STRING", type: "string" as const }, + warning: { + value: "Type mapping fallback used", + type: "string" as const, + }, }, }); } @@ -457,10 +518,14 @@ function mapJsonTypeToGoogleType( message: `Mapped JSON type "${jsonType}" to Google type "${Type[mappedType]}"`, level: 2, auxiliary: { - ...(toolName ? { toolName } : {}), - ...(propertyName ? { propertyName } : {}), - originalType: jsonType, - mappedType: Type[mappedType], + ...(toolName + ? { toolName: { value: toolName, type: "string" as const } } + : {}), + ...(propertyName + ? { propertyName: { value: propertyName, type: "string" as const } } + : {}), + originalType: { value: jsonType, type: "string" as const }, + mappedType: { value: Type[mappedType], type: "string" as const }, }, }); } From e18d2294b348a3d020b65e64a7ff3f7790e3cd25 Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Thu, 13 Nov 2025 13:15:23 -0500 Subject: [PATCH 5/7] revert agent logging --- .../core/lib/v3/agent/AnthropicCUAClient.ts | 218 +++++----- packages/core/lib/v3/agent/GoogleCUAClient.ts | 11 +- packages/core/lib/v3/agent/OpenAICUAClient.ts | 113 +++--- .../v3/agent/utils/googleCustomToolHandler.ts | 381 ++---------------- 4 files changed, 195 insertions(+), 528 deletions(-) diff --git a/packages/core/lib/v3/agent/AnthropicCUAClient.ts b/packages/core/lib/v3/agent/AnthropicCUAClient.ts index c885707b1..d30905e03 100644 --- a/packages/core/lib/v3/agent/AnthropicCUAClient.ts +++ b/packages/core/lib/v3/agent/AnthropicCUAClient.ts @@ -269,7 +269,7 @@ export class AnthropicCUAClient extends AgentClient { }); // Convert tool use to action and add to actions list - const action = this.convertToolUseToAction(toolUseItem, logger); + const action = this.convertToolUseToAction(toolUseItem); if (action) { logger({ category: "agent", @@ -402,103 +402,108 @@ export class AnthropicCUAClient extends AgentClient { id: string; usage: Record; }> { - // For the API request, we use the inputItems directly - // These should already be properly formatted as a sequence of user/assistant messages - const messages: AnthropicMessage[] = []; - - for (const item of inputItems) { - if ("role" in item) { - // Skip system messages as Anthropic requires system as a top-level parameter - if (item.role !== "system") { - messages.push(item); + try { + // For the API request, we use the inputItems directly + // These should already be properly formatted as a sequence of user/assistant messages + const messages: AnthropicMessage[] = []; + + for (const item of inputItems) { + if ("role" in item) { + // Skip system messages as Anthropic requires system as a top-level parameter + if (item.role !== "system") { + messages.push(item); + } } + // Note: We don't need special handling for tool_result items here anymore + // as they should already be properly wrapped in user messages } - // Note: We don't need special handling for tool_result items here anymore - // as they should already be properly wrapped in user messages - } - // Configure thinking capability if available - const thinking = this.thinkingBudget - ? { type: "enabled" as const, budget_tokens: this.thinkingBudget } - : undefined; - - // Create the request parameters - const requestParams: Record = { - model: this.modelName, - max_tokens: 4096, - messages: messages, - tools: [ - { - type: "computer_20250124", // Use the latest version for Claude 3.7 Sonnet - name: "computer", - display_width_px: this.currentViewport.width, - display_height_px: this.currentViewport.height, - display_number: 1, - }, - ], - betas: ["computer-use-2025-01-24"], - }; + // Configure thinking capability if available + const thinking = this.thinkingBudget + ? { type: "enabled" as const, budget_tokens: this.thinkingBudget } + : undefined; + + // Create the request parameters + const requestParams: Record = { + model: this.modelName, + max_tokens: 4096, + messages: messages, + tools: [ + { + type: "computer_20250124", // Use the latest version for Claude 3.7 Sonnet + name: "computer", + display_width_px: this.currentViewport.width, + display_height_px: this.currentViewport.height, + display_number: 1, + }, + ], + betas: ["computer-use-2025-01-24"], + }; - // Add custom tools if available - if (this.tools && Object.keys(this.tools).length > 0) { - const customTools = Object.entries(this.tools).map(([name, tool]) => { - // Convert Zod schema to proper JSON schema format for Anthropic - const jsonSchema = zodToJsonSchema(tool.inputSchema as z.ZodType) as { - properties?: Record; - required?: string[]; - }; + // Add custom tools if available + if (this.tools && Object.keys(this.tools).length > 0) { + const customTools = Object.entries(this.tools).map(([name, tool]) => { + // Convert Zod schema to proper JSON schema format for Anthropic + const jsonSchema = zodToJsonSchema(tool.inputSchema as z.ZodType) as { + properties?: Record; + required?: string[]; + }; - const inputSchema = { - type: "object", - properties: jsonSchema.properties || {}, - required: jsonSchema.required || [], - }; + const inputSchema = { + type: "object", + properties: jsonSchema.properties || {}, + required: jsonSchema.required || [], + }; - return { - name, - description: tool.description, - input_schema: inputSchema, - }; - }); + return { + name, + description: tool.description, + input_schema: inputSchema, + }; + }); - requestParams.tools = [ - ...(requestParams.tools as Record[]), - ...customTools, - ]; - } + requestParams.tools = [ + ...(requestParams.tools as Record[]), + ...customTools, + ]; + } - // Add system parameter if provided - if (this.userProvidedInstructions) { - requestParams.system = this.userProvidedInstructions; - } + // Add system parameter if provided + if (this.userProvidedInstructions) { + requestParams.system = this.userProvidedInstructions; + } - // Add thinking parameter if available - if (thinking) { - requestParams.thinking = thinking; - } + // Add thinking parameter if available + if (thinking) { + requestParams.thinking = thinking; + } - const startTime = Date.now(); - // Create the message using the Anthropic Messages API - // @ts-expect-error - The Anthropic SDK types are stricter than what we need - const response = await this.client.beta.messages.create(requestParams); - const endTime = Date.now(); - const elapsedMs = endTime - startTime; - const usage = { - input_tokens: response.usage.input_tokens, - output_tokens: response.usage.output_tokens, - inference_time_ms: elapsedMs, - }; + const startTime = Date.now(); + // Create the message using the Anthropic Messages API + // @ts-expect-error - The Anthropic SDK types are stricter than what we need + const response = await this.client.beta.messages.create(requestParams); + const endTime = Date.now(); + const elapsedMs = endTime - startTime; + const usage = { + input_tokens: response.usage.input_tokens, + output_tokens: response.usage.output_tokens, + inference_time_ms: elapsedMs, + }; - // Store the message ID for future use - this.lastMessageId = response.id; + // Store the message ID for future use + this.lastMessageId = response.id; - // Return the content and message ID - return { - // Cast the response content to our internal type - content: response.content as unknown as AnthropicContentBlock[], - id: response.id, - usage, - }; + // Return the content and message ID + return { + // Cast the response content to our internal type + content: response.content as unknown as AnthropicContentBlock[], + id: response.id, + usage, + }; + } catch (error) { + console.error("Error getting action from Anthropic:", error); + throw error; + } } async takeAction( @@ -731,10 +736,7 @@ export class AnthropicCUAClient extends AgentClient { return toolResults; } - private convertToolUseToAction( - item: ToolUseItem, - logger: (message: LogLine) => void, - ): AgentAction | null { + private convertToolUseToAction(item: ToolUseItem): AgentAction | null { try { const { name, input } = item; @@ -743,14 +745,7 @@ export class AnthropicCUAClient extends AgentClient { const action = input.action as string; if (!action) { - logger({ - category: "agent", - message: "Missing action in tool use item", - level: 0, - auxiliary: { - item: { value: JSON.stringify(item), type: "object" }, - }, - }); + console.warn("Missing action in tool use item:", item); return null; } @@ -906,24 +901,10 @@ export class AnthropicCUAClient extends AgentClient { return null; } - logger({ - category: "agent", - message: `Unknown tool name: ${name}`, - level: 0, - }); + console.warn(`Unknown tool name: ${name}`); return null; } catch (error) { - logger({ - category: "agent", - message: "Error converting tool use to action", - level: 0, - auxiliary: { - error: { - value: error instanceof Error ? error.message : String(error), - type: "string", - }, - }, - }); + console.error("Error converting tool use to action:", error); return null; } } @@ -939,8 +920,13 @@ export class AnthropicCUAClient extends AgentClient { // Use the screenshot provider if available if (this.screenshotProvider) { - const base64Image = await this.screenshotProvider(); - return `data:image/png;base64,${base64Image}`; + try { + const base64Image = await this.screenshotProvider(); + return `data:image/png;base64,${base64Image}`; + } catch (error) { + console.error("Error capturing screenshot:", error); + throw error; + } } throw new AgentScreenshotProviderError( diff --git a/packages/core/lib/v3/agent/GoogleCUAClient.ts b/packages/core/lib/v3/agent/GoogleCUAClient.ts index 514b68eec..fb7462960 100644 --- a/packages/core/lib/v3/agent/GoogleCUAClient.ts +++ b/packages/core/lib/v3/agent/GoogleCUAClient.ts @@ -466,7 +466,7 @@ export class GoogleCUAClient extends AgentClient { if (result.functionCalls.length > 0 || hasError) { // Filter out custom tool function calls as they've already been handled const computerUseFunctionCalls = result.functionCalls.filter( - (fc) => !isCustomTool(fc, this.tools, logger), + (fc) => !isCustomTool(fc, this.tools), ); if (computerUseFunctionCalls.length > 0) { @@ -890,8 +890,13 @@ export class GoogleCUAClient extends AgentClient { // Use the screenshot provider if available if (this.screenshotProvider) { - const base64Image = await this.screenshotProvider(); - return `data:image/png;base64,${base64Image}`; + try { + const base64Image = await this.screenshotProvider(); + return `data:image/png;base64,${base64Image}`; + } catch (error) { + console.error("Error capturing screenshot:", error); + throw error; + } } throw new AgentScreenshotProviderError( diff --git a/packages/core/lib/v3/agent/OpenAICUAClient.ts b/packages/core/lib/v3/agent/OpenAICUAClient.ts index 65eb162a4..573c18a0e 100644 --- a/packages/core/lib/v3/agent/OpenAICUAClient.ts +++ b/packages/core/lib/v3/agent/OpenAICUAClient.ts @@ -369,67 +369,72 @@ export class OpenAICUAClient extends AgentClient { responseId: string; usage: Record; }> { - // Create the request parameters - const requestParams: Record = { - model: this.modelName, - tools: [ - { - type: "computer_use_preview", - display_width: this.currentViewport.width, - display_height: this.currentViewport.height, - environment: this.environment, - }, - ], - input: inputItems, - truncation: "auto", - }; + try { + // Create the request parameters + const requestParams: Record = { + model: this.modelName, + tools: [ + { + type: "computer_use_preview", + display_width: this.currentViewport.width, + display_height: this.currentViewport.height, + environment: this.environment, + }, + ], + input: inputItems, + truncation: "auto", + }; - // Add custom tools if available - if (this.tools && Object.keys(this.tools).length > 0) { - const customTools = Object.entries(this.tools).map(([name, tool]) => ({ - type: "function" as const, - name, - function: { + // Add custom tools if available + if (this.tools && Object.keys(this.tools).length > 0) { + const customTools = Object.entries(this.tools).map(([name, tool]) => ({ + type: "function" as const, name, - description: tool.description, - parameters: tool.inputSchema, - }, - })); + function: { + name, + description: tool.description, + parameters: tool.inputSchema, + }, + })); + + requestParams.tools = [ + ...(requestParams.tools as Record[]), + ...customTools, + ]; + } - requestParams.tools = [ - ...(requestParams.tools as Record[]), - ...customTools, - ]; - } + // Add previous_response_id if available + if (previousResponseId) { + requestParams.previous_response_id = previousResponseId; + } - // Add previous_response_id if available - if (previousResponseId) { - requestParams.previous_response_id = previousResponseId; - } + const startTime = Date.now(); + // Create the response using the OpenAI Responses API + // @ts-expect-error - Force type to match what the OpenAI SDK expects + const response = await this.client.responses.create(requestParams); + const endTime = Date.now(); + const elapsedMs = endTime - startTime; - const startTime = Date.now(); - // Create the response using the OpenAI Responses API - // @ts-expect-error - Force type to match what the OpenAI SDK expects - const response = await this.client.responses.create(requestParams); - const endTime = Date.now(); - const elapsedMs = endTime - startTime; - - // Extract only the input_tokens and output_tokens - const usage = { - input_tokens: response.usage.input_tokens, - output_tokens: response.usage.output_tokens, - inference_time_ms: elapsedMs, - }; + // Extract only the input_tokens and output_tokens + const usage = { + input_tokens: response.usage.input_tokens, + output_tokens: response.usage.output_tokens, + inference_time_ms: elapsedMs, + }; - // Store the response ID for future use - this.lastResponseId = response.id; + // Store the response ID for future use + this.lastResponseId = response.id; - // Return the output and response ID - return { - output: response.output as unknown as ResponseItem[], - responseId: response.id, - usage, - }; + // Return the output and response ID + return { + output: response.output as unknown as ResponseItem[], + responseId: response.id, + usage, + }; + } catch (error) { + console.error("Error getting action from OpenAI:", error); + throw error; + } } async takeAction( diff --git a/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts b/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts index 978232c21..3343c775b 100644 --- a/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts +++ b/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts @@ -24,74 +24,23 @@ export async function executeGoogleCustomTool( functionCall: FunctionCall, logger: (message: LogLine) => void, ): Promise { - const startTime = Date.now(); - const toolCallId = `tool_${Date.now()}`; - try { logger({ category: "agent", - message: `Executing custom tool: ${toolName}`, + message: `Executing custom tool: ${toolName} with args: ${JSON.stringify(toolArgs)}`, level: 1, - auxiliary: { - toolName: { value: toolName, type: "string" as const }, - toolCallId: { value: toolCallId, type: "string" as const }, - arguments: { value: JSON.stringify(toolArgs), type: "object" as const }, - functionCallName: { value: functionCall.name, type: "string" as const }, - }, }); const tool = tools[toolName]; - if (!tool) { - const errorMessage = `Tool ${toolName} not found in toolset`; - logger({ - category: "agent", - message: errorMessage, - level: 0, - auxiliary: { - toolName: { value: toolName, type: "string" as const }, - availableTools: { - value: JSON.stringify(Object.keys(tools)), - type: "object" as const, - }, - }, - }); - throw new Error(errorMessage); - } - - logger({ - category: "agent", - message: `Tool ${toolName} found, executing with ${Object.keys(toolArgs).length} argument(s)`, - level: 2, - auxiliary: { - toolName: { value: toolName, type: "string" as const }, - toolCallId: { value: toolCallId, type: "string" as const }, - argumentCount: { - value: String(Object.keys(toolArgs).length), - type: "integer" as const, - }, - }, - }); - const toolResult = await tool.execute(toolArgs, { - toolCallId, + toolCallId: `tool_${Date.now()}`, messages: [], }); - const executionTime = Date.now() - startTime; - const resultString = JSON.stringify(toolResult); - const resultSize = resultString.length; - logger({ category: "agent", - message: `Tool ${toolName} completed successfully in ${executionTime}ms`, + message: `Tool ${toolName} completed successfully. Result: ${JSON.stringify(toolResult)}`, level: 1, - auxiliary: { - toolName: { value: toolName, type: "string" as const }, - toolCallId: { value: toolCallId, type: "string" as const }, - executionTime: { value: `${executionTime}ms`, type: "string" as const }, - resultSize: { value: `${resultSize} bytes`, type: "string" as const }, - result: { value: JSON.stringify(toolResult), type: "object" as const }, - }, }); // Create function response with the result @@ -99,7 +48,7 @@ export async function executeGoogleCustomTool( functionResponse: { name: toolName, response: { - result: resultString, + result: JSON.stringify(toolResult), }, }, }; @@ -109,30 +58,13 @@ export async function executeGoogleCustomTool( success: true, }; } catch (toolError) { - const executionTime = Date.now() - startTime; const errorMessage = toolError instanceof Error ? toolError.message : String(toolError); - const errorStack = toolError instanceof Error ? toolError.stack : undefined; - const errorType = - toolError instanceof Error - ? toolError.constructor.name - : typeof toolError; logger({ category: "agent", message: `Error executing custom tool ${toolName}: ${errorMessage}`, level: 0, - auxiliary: { - toolName: { value: toolName, type: "string" as const }, - toolCallId: { value: toolCallId, type: "string" as const }, - executionTime: { value: `${executionTime}ms`, type: "string" as const }, - errorType: { value: errorType, type: "string" as const }, - errorMessage: { value: errorMessage, type: "string" as const }, - ...(errorStack - ? { errorStack: { value: errorStack, type: "string" as const } } - : {}), - arguments: { value: JSON.stringify(toolArgs), type: "object" as const }, - }, }); // Create error function response @@ -158,27 +90,8 @@ export async function executeGoogleCustomTool( export function isCustomTool( functionCall: FunctionCall, tools?: ToolSet, - logger?: (message: LogLine) => void, ): boolean { - const isCustom = !!(tools && functionCall.name && functionCall.name in tools); - - if (logger) { - logger({ - category: "agent", - message: `Checking if function call "${functionCall.name}" is a custom tool: ${isCustom}`, - level: 2, - auxiliary: { - functionCallName: { value: functionCall.name, type: "string" as const }, - isCustomTool: { value: String(isCustom), type: "boolean" as const }, - availableCustomTools: { - value: JSON.stringify(tools ? Object.keys(tools) : []), - type: "object" as const, - }, - }, - }); - } - - return isCustom; + return !!(tools && functionCall.name && functionCall.name in tools); } /** @@ -187,68 +100,16 @@ export function isCustomTool( */ export function convertToolSetToFunctionDeclarations( tools: ToolSet, - logger?: (message: LogLine) => void, ): FunctionDeclaration[] { - const toolCount = Object.keys(tools).length; - - if (logger) { - logger({ - category: "agent", - message: `Converting ${toolCount} tool(s) to Google FunctionDeclarations`, - level: 2, - auxiliary: { - toolCount: { value: String(toolCount), type: "integer" as const }, - toolNames: { - value: JSON.stringify(Object.keys(tools)), - type: "object" as const, - }, - }, - }); - } - const functionDeclarations: FunctionDeclaration[] = []; - const failedConversions: string[] = []; for (const [name, tool] of Object.entries(tools)) { - const functionDeclaration = convertToolToFunctionDeclaration( - name, - tool, - logger, - ); + const functionDeclaration = convertToolToFunctionDeclaration(name, tool); if (functionDeclaration) { functionDeclarations.push(functionDeclaration); - } else { - failedConversions.push(name); } } - if (logger) { - logger({ - category: "agent", - message: `Converted ${functionDeclarations.length} of ${toolCount} tool(s) to FunctionDeclarations`, - level: functionDeclarations.length === toolCount ? 2 : 1, - auxiliary: { - successfulConversions: { - value: String(functionDeclarations.length), - type: "integer" as const, - }, - totalTools: { value: String(toolCount), type: "integer" as const }, - ...(failedConversions.length > 0 - ? { - failedConversions: { - value: JSON.stringify(failedConversions), - type: "object" as const, - }, - warning: { - value: "Some tools failed to convert and were excluded", - type: "string" as const, - }, - } - : {}), - }, - }); - } - return functionDeclarations; } @@ -258,24 +119,8 @@ export function convertToolSetToFunctionDeclarations( function convertToolToFunctionDeclaration( name: string, tool: { description?: string; inputSchema: unknown }, - logger?: (message: LogLine) => void, ): FunctionDeclaration | null { try { - if (logger) { - logger({ - category: "agent", - message: `Converting tool "${name}" to FunctionDeclaration`, - level: 2, - auxiliary: { - toolName: { value: name, type: "string" as const }, - hasDescription: { - value: String(!!tool.description), - type: "boolean" as const, - }, - }, - }); - } - // Convert Zod schema to JSON schema const jsonSchema = zodToJsonSchema(tool.inputSchema as z.ZodType) as { properties?: Record; @@ -283,91 +128,18 @@ function convertToolToFunctionDeclaration( type?: string; }; - const propertyCount = jsonSchema.properties - ? Object.keys(jsonSchema.properties).length - : 0; - const requiredCount = jsonSchema.required?.length || 0; - - if (logger) { - logger({ - category: "agent", - message: `Tool "${name}" schema converted: ${propertyCount} property(ies), ${requiredCount} required`, - level: 2, - auxiliary: { - toolName: { value: name, type: "string" as const }, - propertyCount: { - value: String(propertyCount), - type: "integer" as const, - }, - requiredCount: { - value: String(requiredCount), - type: "integer" as const, - }, - properties: { - value: JSON.stringify( - jsonSchema.properties ? Object.keys(jsonSchema.properties) : [], - ), - type: "object" as const, - }, - }, - }); - } + const parameters = convertJsonSchemaToGoogleParameters(jsonSchema); - const parameters = convertJsonSchemaToGoogleParameters( - jsonSchema, - name, - logger, - ); - - const functionDeclaration = { + return { name, description: tool.description || `Execute ${name}`, parameters, }; - - if (logger) { - logger({ - category: "agent", - message: `Successfully converted tool "${name}" to FunctionDeclaration`, - level: 2, - auxiliary: { - toolName: { value: name, type: "string" as const }, - parameterType: { - value: String(parameters.type), - type: "string" as const, - }, - parameterCount: { - value: String(Object.keys(parameters.properties || {}).length), - type: "integer" as const, - }, - }, - }); - } - - return functionDeclaration; } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - const errorType = - error instanceof Error ? error.constructor.name : typeof error; - - // Tool conversion failed - return null to filter this tool out - // This typically indicates an invalid tool schema definition - if (logger) { - logger({ - category: "agent", - message: `Failed to convert tool "${name}" to FunctionDeclaration: ${errorMessage}`, - level: 0, - auxiliary: { - toolName: { value: name, type: "string" as const }, - errorType: { value: errorType, type: "string" as const }, - errorMessage: { value: errorMessage, type: "string" as const }, - ...(error instanceof Error && error.stack - ? { errorStack: { value: error.stack, type: "string" as const } } - : {}), - }, - }); - } - + console.error( + `Error converting tool ${name} to function declaration:`, + error, + ); return null; } } @@ -375,21 +147,16 @@ function convertToolToFunctionDeclaration( /** * Convert JSON schema to Google's parameter format */ -function convertJsonSchemaToGoogleParameters( - schema: { - properties?: Record; - required?: string[]; - type?: string; - }, - toolName?: string, - logger?: (message: LogLine) => void, -): { +function convertJsonSchemaToGoogleParameters(schema: { + properties?: Record; + required?: string[]; + type?: string; +}): { type: Type; properties: Record; required?: string[]; } { const properties: Record = {}; - const typeMappings: Record = {}; if (schema.properties) { for (const [key, value] of Object.entries(schema.properties)) { @@ -398,18 +165,8 @@ function convertJsonSchemaToGoogleParameters( description?: string; items?: { type?: string }; }; - const jsonType = propSchema.type || "string"; - const googleType = mapJsonTypeToGoogleType( - jsonType, - key, - toolName, - logger, - ); - - typeMappings[key] = `${jsonType} -> ${Type[googleType]}`; - properties[key] = { - type: googleType, + type: mapJsonTypeToGoogleType(propSchema.type || "string"), ...(propSchema.description ? { description: propSchema.description } : {}), @@ -417,27 +174,6 @@ function convertJsonSchemaToGoogleParameters( } } - if (logger && Object.keys(typeMappings).length > 0) { - logger({ - category: "agent", - message: `Converted ${Object.keys(properties).length} property type(s) for ${toolName || "tool"}`, - level: 2, - auxiliary: { - ...(toolName - ? { toolName: { value: toolName, type: "string" as const } } - : {}), - typeMappings: { - value: JSON.stringify(typeMappings), - type: "object" as const, - }, - propertyCount: { - value: String(Object.keys(properties).length), - type: "integer" as const, - }, - }, - }); - } - return { type: Type.OBJECT, properties, @@ -450,85 +186,20 @@ function convertJsonSchemaToGoogleParameters( /** * Map JSON schema types to Google's Type enum */ -function mapJsonTypeToGoogleType( - jsonType: string, - propertyName?: string, - toolName?: string, - logger?: (message: LogLine) => void, -): Type { - const normalizedType = jsonType.toLowerCase(); - let mappedType: Type; - - switch (normalizedType) { +function mapJsonTypeToGoogleType(jsonType: string): Type { + switch (jsonType.toLowerCase()) { case "string": - mappedType = Type.STRING; - break; + return Type.STRING; case "number": case "integer": - mappedType = Type.NUMBER; - break; + return Type.NUMBER; case "boolean": - mappedType = Type.BOOLEAN; - break; + return Type.BOOLEAN; case "array": - mappedType = Type.ARRAY; - break; + return Type.ARRAY; case "object": - mappedType = Type.OBJECT; - break; + return Type.OBJECT; default: - mappedType = Type.STRING; - if (logger) { - logger({ - category: "agent", - message: `Unknown JSON schema type "${jsonType}", defaulting to STRING`, - level: 1, - auxiliary: { - ...(toolName - ? { toolName: { value: toolName, type: "string" as const } } - : {}), - ...(propertyName - ? { - propertyName: { - value: propertyName, - type: "string" as const, - }, - } - : {}), - originalType: { value: jsonType, type: "string" as const }, - mappedType: { value: "STRING", type: "string" as const }, - warning: { - value: "Type mapping fallback used", - type: "string" as const, - }, - }, - }); - } - break; - } - - if ( - logger && - normalizedType !== "string" && - normalizedType !== "number" && - normalizedType !== "integer" - ) { - logger({ - category: "agent", - message: `Mapped JSON type "${jsonType}" to Google type "${Type[mappedType]}"`, - level: 2, - auxiliary: { - ...(toolName - ? { toolName: { value: toolName, type: "string" as const } } - : {}), - ...(propertyName - ? { propertyName: { value: propertyName, type: "string" as const } } - : {}), - originalType: { value: jsonType, type: "string" as const }, - mappedType: { value: Type[mappedType], type: "string" as const }, - }, - }); + return Type.STRING; } - - return mappedType; } From d4e85248622bd5548b8d5fe1a56d30fa66942f9e Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Thu, 13 Nov 2025 14:37:20 -0500 Subject: [PATCH 6/7] update CUA logging --- packages/core/lib/v3/agent/AgentClient.ts | 2 + .../core/lib/v3/agent/AnthropicCUAClient.ts | 89 +++++++++++++++---- packages/core/lib/v3/agent/GoogleCUAClient.ts | 44 +++++++-- packages/core/lib/v3/agent/OpenAICUAClient.ts | 72 ++++++++++++--- .../v3/agent/utils/googleCustomToolHandler.ts | 6 +- .../core/lib/v3/handlers/v3CuaAgentHandler.ts | 2 +- 6 files changed, 172 insertions(+), 43 deletions(-) diff --git a/packages/core/lib/v3/agent/AgentClient.ts b/packages/core/lib/v3/agent/AgentClient.ts index cc530ac23..d5e35a8ad 100644 --- a/packages/core/lib/v3/agent/AgentClient.ts +++ b/packages/core/lib/v3/agent/AgentClient.ts @@ -4,6 +4,7 @@ import { AgentType, AgentExecutionOptions, } from "../types/public/agent"; +import { LogLine } from "../types/public/logs"; /** * Abstract base class for agent clients @@ -29,6 +30,7 @@ export abstract class AgentClient { abstract execute(options: AgentExecutionOptions): Promise; abstract captureScreenshot( + logger: (message: LogLine) => void, options?: Record, ): Promise; diff --git a/packages/core/lib/v3/agent/AnthropicCUAClient.ts b/packages/core/lib/v3/agent/AnthropicCUAClient.ts index d30905e03..eac93fa96 100644 --- a/packages/core/lib/v3/agent/AnthropicCUAClient.ts +++ b/packages/core/lib/v3/agent/AnthropicCUAClient.ts @@ -223,7 +223,7 @@ export class AnthropicCUAClient extends AgentClient { }> { try { // Get response from the model - const result = await this.getAction(inputItems); + const result = await this.getAction(inputItems, logger); const content = result.content; const usage = { input_tokens: result.usage.input_tokens, @@ -269,7 +269,7 @@ export class AnthropicCUAClient extends AgentClient { }); // Convert tool use to action and add to actions list - const action = this.convertToolUseToAction(toolUseItem); + const action = this.convertToolUseToAction(toolUseItem, logger); if (action) { logger({ category: "agent", @@ -397,7 +397,10 @@ export class AnthropicCUAClient extends AgentClient { ]; } - async getAction(inputItems: ResponseInputItem[]): Promise<{ + async getAction( + inputItems: ResponseInputItem[], + logger: (message: LogLine) => void, + ): Promise<{ content: AnthropicContentBlock[]; id: string; usage: Record; @@ -501,7 +504,20 @@ export class AnthropicCUAClient extends AgentClient { usage, }; } catch (error) { - console.error("Error getting action from Anthropic:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error getting action from Anthropic: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + }, + }); throw error; } } @@ -540,7 +556,7 @@ export class AnthropicCUAClient extends AgentClient { }); // Capture a screenshot for the response - const screenshot = await this.captureScreenshot(); + const screenshot = await this.captureScreenshot(logger); logger({ category: "agent", message: `Screenshot captured, length: ${screenshot.length}`, @@ -654,7 +670,7 @@ export class AnthropicCUAClient extends AgentClient { try { // For computer tool, try to capture a screenshot even on error if (item.name === "computer") { - const screenshot = await this.captureScreenshot(); + const screenshot = await this.captureScreenshot(logger); toolResults.push({ type: "tool_result", @@ -736,7 +752,10 @@ export class AnthropicCUAClient extends AgentClient { return toolResults; } - private convertToolUseToAction(item: ToolUseItem): AgentAction | null { + private convertToolUseToAction( + item: ToolUseItem, + logger: (message: LogLine) => void, + ): AgentAction | null { try { const { name, input } = item; @@ -745,7 +764,14 @@ export class AnthropicCUAClient extends AgentClient { const action = input.action as string; if (!action) { - console.warn("Missing action in tool use item:", item); + logger({ + category: "agent", + message: "Missing action in tool use item", + level: 1, + auxiliary: { + item: { value: JSON.stringify(item), type: "object" }, + }, + }); return null; } @@ -901,18 +927,38 @@ export class AnthropicCUAClient extends AgentClient { return null; } - console.warn(`Unknown tool name: ${name}`); + logger({ + category: "agent", + message: `Unknown tool name: ${name}`, + level: 1, + }); return null; } catch (error) { - console.error("Error converting tool use to action:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error converting tool use to action: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + }, + }); return null; } } - async captureScreenshot(options?: { - base64Image?: string; - currentUrl?: string; - }): Promise { + async captureScreenshot( + logger: (message: LogLine) => void, + options?: { + base64Image?: string; + currentUrl?: string; + }, + ): Promise { // Use provided options if available if (options?.base64Image) { return `data:image/png;base64,${options.base64Image}`; @@ -924,7 +970,20 @@ export class AnthropicCUAClient extends AgentClient { const base64Image = await this.screenshotProvider(); return `data:image/png;base64,${base64Image}`; } catch (error) { - console.error("Error capturing screenshot:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error capturing screenshot: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + }, + }); throw error; } } diff --git a/packages/core/lib/v3/agent/GoogleCUAClient.ts b/packages/core/lib/v3/agent/GoogleCUAClient.ts index fb7462960..88c169323 100644 --- a/packages/core/lib/v3/agent/GoogleCUAClient.ts +++ b/packages/core/lib/v3/agent/GoogleCUAClient.ts @@ -477,7 +477,7 @@ export class GoogleCUAClient extends AgentClient { level: 2, }); - const screenshot = await this.captureScreenshot(); + const screenshot = await this.captureScreenshot(logger); const base64Data = screenshot.replace( /^data:image\/png;base64,/, "", @@ -511,10 +511,15 @@ export class GoogleCUAClient extends AgentClient { functionResponses.push(functionResponsePart); } } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); logger({ category: "agent", - message: `Error capturing screenshot: ${error}`, + message: `Error capturing screenshot: ${errorMessage}`, level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + }, }); } } @@ -609,7 +614,7 @@ export class GoogleCUAClient extends AgentClient { }); // Convert function call to action(s) - const action = this.convertFunctionCallToAction(part.functionCall); + const action = this.convertFunctionCallToAction(part.functionCall, logger); if (action) { // Special handling for type_text_at - we need to click first if ( @@ -688,6 +693,7 @@ export class GoogleCUAClient extends AgentClient { */ private convertFunctionCallToAction( functionCall: FunctionCall, + logger: (message: LogLine) => void, ): AgentAction | null { const { name, args } = functionCall; @@ -857,7 +863,11 @@ export class GoogleCUAClient extends AgentClient { pageUrl: this.currentUrl, }; } - console.warn(`Unsupported Google CUA function: ${name}`); + logger({ + category: "agent", + message: `Unsupported Google CUA function: ${name}`, + level: 1, + }); return null; } } @@ -874,10 +884,13 @@ export class GoogleCUAClient extends AgentClient { }; } - async captureScreenshot(options?: { - base64Image?: string; - currentUrl?: string; - }): Promise { + async captureScreenshot( + logger: (message: LogLine) => void, + options?: { + base64Image?: string; + currentUrl?: string; + }, + ): Promise { // Update current URL if provided if (options?.currentUrl) { this.currentUrl = options.currentUrl; @@ -894,7 +907,20 @@ export class GoogleCUAClient extends AgentClient { const base64Image = await this.screenshotProvider(); return `data:image/png;base64,${base64Image}`; } catch (error) { - console.error("Error capturing screenshot:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error capturing screenshot: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + }, + }); throw error; } } diff --git a/packages/core/lib/v3/agent/OpenAICUAClient.ts b/packages/core/lib/v3/agent/OpenAICUAClient.ts index 573c18a0e..030058f43 100644 --- a/packages/core/lib/v3/agent/OpenAICUAClient.ts +++ b/packages/core/lib/v3/agent/OpenAICUAClient.ts @@ -214,7 +214,7 @@ export class OpenAICUAClient extends AgentClient { }> { try { // Get response from the model - const result = await this.getAction(inputItems, previousResponseId); + const result = await this.getAction(inputItems, previousResponseId, logger); const output = result.output; const responseId = result.responseId; const usage = { @@ -262,7 +262,7 @@ export class OpenAICUAClient extends AgentClient { message: `Found function_call: ${item.name}, call_id: ${item.call_id}`, level: 2, }); - const action = this.convertFunctionCallToAction(item); + const action = this.convertFunctionCallToAction(item, logger); if (action) { stepActions.push(action); logger({ @@ -363,7 +363,8 @@ export class OpenAICUAClient extends AgentClient { async getAction( inputItems: ResponseInputItem[], - previousResponseId?: string, + previousResponseId: string | undefined, + logger: (message: LogLine) => void, ): Promise<{ output: ResponseItem[]; responseId: string; @@ -432,7 +433,20 @@ export class OpenAICUAClient extends AgentClient { usage, }; } catch (error) { - console.error("Error getting action from OpenAI:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error getting action from OpenAI: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + }, + }); throw error; } } @@ -460,7 +474,7 @@ export class OpenAICUAClient extends AgentClient { } // Capture a screenshot - const screenshot = await this.captureScreenshot(); + const screenshot = await this.captureScreenshot(logger); // Create a computer_call_output for the next request const outputItem = { @@ -532,7 +546,7 @@ export class OpenAICUAClient extends AgentClient { try { // Capture a screenshot even on error - const screenshot = await this.captureScreenshot(); + const screenshot = await this.captureScreenshot(logger); const errorOutputItem = { type: "computer_call_output" as const, @@ -608,7 +622,7 @@ export class OpenAICUAClient extends AgentClient { ) { // Handle function calls (tool calls) try { - const action = this.convertFunctionCallToAction(item); + const action = this.convertFunctionCallToAction(item, logger); if (action && this.actionHandler) { await this.actionHandler(action); @@ -701,6 +715,7 @@ export class OpenAICUAClient extends AgentClient { private convertFunctionCallToAction( call: FunctionCallItem, + logger: (message: LogLine) => void, ): AgentAction | null { try { const args = JSON.parse(call.arguments); @@ -710,15 +725,33 @@ export class OpenAICUAClient extends AgentClient { params: args, }; } catch (error) { - console.error("Error parsing function call arguments:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error parsing function call arguments: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + callId: { value: call.call_id, type: "string" }, + callName: { value: call.name, type: "string" }, + }, + }); return null; } } - async captureScreenshot(options?: { - base64Image?: string; - currentUrl?: string; - }): Promise { + async captureScreenshot( + logger: (message: LogLine) => void, + options?: { + base64Image?: string; + currentUrl?: string; + }, + ): Promise { // Use provided options if available if (options?.base64Image) { return `data:image/png;base64,${options.base64Image}`; @@ -730,7 +763,20 @@ export class OpenAICUAClient extends AgentClient { const base64Image = await this.screenshotProvider(); return `data:image/png;base64,${base64Image}`; } catch (error) { - console.error("Error capturing screenshot:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error capturing screenshot: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + }, + }); throw error; } } diff --git a/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts b/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts index 3343c775b..e9f1397bf 100644 --- a/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts +++ b/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts @@ -135,11 +135,7 @@ function convertToolToFunctionDeclaration( description: tool.description || `Execute ${name}`, parameters, }; - } catch (error) { - console.error( - `Error converting tool ${name} to function declaration:`, - error, - ); + } catch { return null; } } diff --git a/packages/core/lib/v3/handlers/v3CuaAgentHandler.ts b/packages/core/lib/v3/handlers/v3CuaAgentHandler.ts index aab4f0039..ff575fca4 100644 --- a/packages/core/lib/v3/handlers/v3CuaAgentHandler.ts +++ b/packages/core/lib/v3/handlers/v3CuaAgentHandler.ts @@ -514,7 +514,7 @@ export class V3CuaAgentHandler { const page = await this.v3.context.awaitActivePage(); const base64Image = await page.screenshot({ fullPage: false }); const currentUrl = page.url(); - return await this.agentClient.captureScreenshot({ + return await this.agentClient.captureScreenshot(this.logger, { base64Image, currentUrl, }); From 6b536ff6fd7f292cc902acd85042bd840ccbb5fe Mon Sep 17 00:00:00 2001 From: Filip Michalsky Date: Thu, 13 Nov 2025 14:46:18 -0500 Subject: [PATCH 7/7] fix lint --- packages/core/lib/v3/agent/GoogleCUAClient.ts | 5 ++++- packages/core/lib/v3/agent/OpenAICUAClient.ts | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/core/lib/v3/agent/GoogleCUAClient.ts b/packages/core/lib/v3/agent/GoogleCUAClient.ts index 88c169323..767db62ce 100644 --- a/packages/core/lib/v3/agent/GoogleCUAClient.ts +++ b/packages/core/lib/v3/agent/GoogleCUAClient.ts @@ -614,7 +614,10 @@ export class GoogleCUAClient extends AgentClient { }); // Convert function call to action(s) - const action = this.convertFunctionCallToAction(part.functionCall, logger); + const action = this.convertFunctionCallToAction( + part.functionCall, + logger, + ); if (action) { // Special handling for type_text_at - we need to click first if ( diff --git a/packages/core/lib/v3/agent/OpenAICUAClient.ts b/packages/core/lib/v3/agent/OpenAICUAClient.ts index 030058f43..a8a12a8dd 100644 --- a/packages/core/lib/v3/agent/OpenAICUAClient.ts +++ b/packages/core/lib/v3/agent/OpenAICUAClient.ts @@ -214,7 +214,11 @@ export class OpenAICUAClient extends AgentClient { }> { try { // Get response from the model - const result = await this.getAction(inputItems, previousResponseId, logger); + const result = await this.getAction( + inputItems, + previousResponseId, + logger, + ); const output = result.output; const responseId = result.responseId; const usage = {