diff --git a/packages/core/lib/inferenceLogUtils.ts b/packages/core/lib/inferenceLogUtils.ts index a6a60ca61..cdae1ef7d 100644 --- a/packages/core/lib/inferenceLogUtils.ts +++ b/packages/core/lib/inferenceLogUtils.ts @@ -107,8 +107,13 @@ function readSummaryFile(inferenceType: string): Record { ) { return parsed; } - } catch { + } catch (error) { // If we fail to parse for any reason, fall back to empty array + // This is a debug utility file - using console.warn for visibility + console.warn( + `Failed to parse inference summary file ${summaryPath}:`, + error instanceof Error ? error.message : String(error), + ); } return { [arrayKey]: [] }; } diff --git a/packages/core/lib/v3/agent/AgentClient.ts b/packages/core/lib/v3/agent/AgentClient.ts index cc530ac23..d5e35a8ad 100644 --- a/packages/core/lib/v3/agent/AgentClient.ts +++ b/packages/core/lib/v3/agent/AgentClient.ts @@ -4,6 +4,7 @@ import { AgentType, AgentExecutionOptions, } from "../types/public/agent"; +import { LogLine } from "../types/public/logs"; /** * Abstract base class for agent clients @@ -29,6 +30,7 @@ export abstract class AgentClient { abstract execute(options: AgentExecutionOptions): Promise; abstract captureScreenshot( + logger: (message: LogLine) => void, options?: Record, ): Promise; diff --git a/packages/core/lib/v3/agent/AnthropicCUAClient.ts b/packages/core/lib/v3/agent/AnthropicCUAClient.ts index d30905e03..eac93fa96 100644 --- a/packages/core/lib/v3/agent/AnthropicCUAClient.ts +++ b/packages/core/lib/v3/agent/AnthropicCUAClient.ts @@ -223,7 +223,7 @@ export class AnthropicCUAClient extends AgentClient { }> { try { // Get response from the model - const result = await this.getAction(inputItems); + const result = await this.getAction(inputItems, logger); const content = result.content; const usage = { input_tokens: result.usage.input_tokens, @@ -269,7 +269,7 @@ export class AnthropicCUAClient extends AgentClient { }); // Convert tool use to action and add to actions list - const action = this.convertToolUseToAction(toolUseItem); + const action = this.convertToolUseToAction(toolUseItem, logger); if (action) { logger({ category: "agent", @@ -397,7 +397,10 @@ export class AnthropicCUAClient extends AgentClient { ]; } - async getAction(inputItems: ResponseInputItem[]): Promise<{ + async getAction( + inputItems: ResponseInputItem[], + logger: (message: LogLine) => void, + ): Promise<{ content: AnthropicContentBlock[]; id: string; usage: Record; @@ -501,7 +504,20 @@ export class AnthropicCUAClient extends AgentClient { usage, }; } catch (error) { - console.error("Error getting action from Anthropic:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error getting action from Anthropic: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + }, + }); throw error; } } @@ -540,7 +556,7 @@ export class AnthropicCUAClient extends AgentClient { }); // Capture a screenshot for the response - const screenshot = await this.captureScreenshot(); + const screenshot = await this.captureScreenshot(logger); logger({ category: "agent", message: `Screenshot captured, length: ${screenshot.length}`, @@ -654,7 +670,7 @@ export class AnthropicCUAClient extends AgentClient { try { // For computer tool, try to capture a screenshot even on error if (item.name === "computer") { - const screenshot = await this.captureScreenshot(); + const screenshot = await this.captureScreenshot(logger); toolResults.push({ type: "tool_result", @@ -736,7 +752,10 @@ export class AnthropicCUAClient extends AgentClient { return toolResults; } - private convertToolUseToAction(item: ToolUseItem): AgentAction | null { + private convertToolUseToAction( + item: ToolUseItem, + logger: (message: LogLine) => void, + ): AgentAction | null { try { const { name, input } = item; @@ -745,7 +764,14 @@ export class AnthropicCUAClient extends AgentClient { const action = input.action as string; if (!action) { - console.warn("Missing action in tool use item:", item); + logger({ + category: "agent", + message: "Missing action in tool use item", + level: 1, + auxiliary: { + item: { value: JSON.stringify(item), type: "object" }, + }, + }); return null; } @@ -901,18 +927,38 @@ export class AnthropicCUAClient extends AgentClient { return null; } - console.warn(`Unknown tool name: ${name}`); + logger({ + category: "agent", + message: `Unknown tool name: ${name}`, + level: 1, + }); return null; } catch (error) { - console.error("Error converting tool use to action:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error converting tool use to action: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + }, + }); return null; } } - async captureScreenshot(options?: { - base64Image?: string; - currentUrl?: string; - }): Promise { + async captureScreenshot( + logger: (message: LogLine) => void, + options?: { + base64Image?: string; + currentUrl?: string; + }, + ): Promise { // Use provided options if available if (options?.base64Image) { return `data:image/png;base64,${options.base64Image}`; @@ -924,7 +970,20 @@ export class AnthropicCUAClient extends AgentClient { const base64Image = await this.screenshotProvider(); return `data:image/png;base64,${base64Image}`; } catch (error) { - console.error("Error capturing screenshot:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error capturing screenshot: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + }, + }); throw error; } } diff --git a/packages/core/lib/v3/agent/GoogleCUAClient.ts b/packages/core/lib/v3/agent/GoogleCUAClient.ts index fb7462960..767db62ce 100644 --- a/packages/core/lib/v3/agent/GoogleCUAClient.ts +++ b/packages/core/lib/v3/agent/GoogleCUAClient.ts @@ -477,7 +477,7 @@ export class GoogleCUAClient extends AgentClient { level: 2, }); - const screenshot = await this.captureScreenshot(); + const screenshot = await this.captureScreenshot(logger); const base64Data = screenshot.replace( /^data:image\/png;base64,/, "", @@ -511,10 +511,15 @@ export class GoogleCUAClient extends AgentClient { functionResponses.push(functionResponsePart); } } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); logger({ category: "agent", - message: `Error capturing screenshot: ${error}`, + message: `Error capturing screenshot: ${errorMessage}`, level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + }, }); } } @@ -609,7 +614,10 @@ export class GoogleCUAClient extends AgentClient { }); // Convert function call to action(s) - const action = this.convertFunctionCallToAction(part.functionCall); + const action = this.convertFunctionCallToAction( + part.functionCall, + logger, + ); if (action) { // Special handling for type_text_at - we need to click first if ( @@ -688,6 +696,7 @@ export class GoogleCUAClient extends AgentClient { */ private convertFunctionCallToAction( functionCall: FunctionCall, + logger: (message: LogLine) => void, ): AgentAction | null { const { name, args } = functionCall; @@ -857,7 +866,11 @@ export class GoogleCUAClient extends AgentClient { pageUrl: this.currentUrl, }; } - console.warn(`Unsupported Google CUA function: ${name}`); + logger({ + category: "agent", + message: `Unsupported Google CUA function: ${name}`, + level: 1, + }); return null; } } @@ -874,10 +887,13 @@ export class GoogleCUAClient extends AgentClient { }; } - async captureScreenshot(options?: { - base64Image?: string; - currentUrl?: string; - }): Promise { + async captureScreenshot( + logger: (message: LogLine) => void, + options?: { + base64Image?: string; + currentUrl?: string; + }, + ): Promise { // Update current URL if provided if (options?.currentUrl) { this.currentUrl = options.currentUrl; @@ -894,7 +910,20 @@ export class GoogleCUAClient extends AgentClient { const base64Image = await this.screenshotProvider(); return `data:image/png;base64,${base64Image}`; } catch (error) { - console.error("Error capturing screenshot:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error capturing screenshot: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + }, + }); throw error; } } diff --git a/packages/core/lib/v3/agent/OpenAICUAClient.ts b/packages/core/lib/v3/agent/OpenAICUAClient.ts index 573c18a0e..a8a12a8dd 100644 --- a/packages/core/lib/v3/agent/OpenAICUAClient.ts +++ b/packages/core/lib/v3/agent/OpenAICUAClient.ts @@ -214,7 +214,11 @@ export class OpenAICUAClient extends AgentClient { }> { try { // Get response from the model - const result = await this.getAction(inputItems, previousResponseId); + const result = await this.getAction( + inputItems, + previousResponseId, + logger, + ); const output = result.output; const responseId = result.responseId; const usage = { @@ -262,7 +266,7 @@ export class OpenAICUAClient extends AgentClient { message: `Found function_call: ${item.name}, call_id: ${item.call_id}`, level: 2, }); - const action = this.convertFunctionCallToAction(item); + const action = this.convertFunctionCallToAction(item, logger); if (action) { stepActions.push(action); logger({ @@ -363,7 +367,8 @@ export class OpenAICUAClient extends AgentClient { async getAction( inputItems: ResponseInputItem[], - previousResponseId?: string, + previousResponseId: string | undefined, + logger: (message: LogLine) => void, ): Promise<{ output: ResponseItem[]; responseId: string; @@ -432,7 +437,20 @@ export class OpenAICUAClient extends AgentClient { usage, }; } catch (error) { - console.error("Error getting action from OpenAI:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error getting action from OpenAI: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + }, + }); throw error; } } @@ -460,7 +478,7 @@ export class OpenAICUAClient extends AgentClient { } // Capture a screenshot - const screenshot = await this.captureScreenshot(); + const screenshot = await this.captureScreenshot(logger); // Create a computer_call_output for the next request const outputItem = { @@ -532,7 +550,7 @@ export class OpenAICUAClient extends AgentClient { try { // Capture a screenshot even on error - const screenshot = await this.captureScreenshot(); + const screenshot = await this.captureScreenshot(logger); const errorOutputItem = { type: "computer_call_output" as const, @@ -608,7 +626,7 @@ export class OpenAICUAClient extends AgentClient { ) { // Handle function calls (tool calls) try { - const action = this.convertFunctionCallToAction(item); + const action = this.convertFunctionCallToAction(item, logger); if (action && this.actionHandler) { await this.actionHandler(action); @@ -701,6 +719,7 @@ export class OpenAICUAClient extends AgentClient { private convertFunctionCallToAction( call: FunctionCallItem, + logger: (message: LogLine) => void, ): AgentAction | null { try { const args = JSON.parse(call.arguments); @@ -710,15 +729,33 @@ export class OpenAICUAClient extends AgentClient { params: args, }; } catch (error) { - console.error("Error parsing function call arguments:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error parsing function call arguments: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + callId: { value: call.call_id, type: "string" }, + callName: { value: call.name, type: "string" }, + }, + }); return null; } } - async captureScreenshot(options?: { - base64Image?: string; - currentUrl?: string; - }): Promise { + async captureScreenshot( + logger: (message: LogLine) => void, + options?: { + base64Image?: string; + currentUrl?: string; + }, + ): Promise { // Use provided options if available if (options?.base64Image) { return `data:image/png;base64,${options.base64Image}`; @@ -730,7 +767,20 @@ export class OpenAICUAClient extends AgentClient { const base64Image = await this.screenshotProvider(); return `data:image/png;base64,${base64Image}`; } catch (error) { - console.error("Error capturing screenshot:", error); + const errorMessage = + error instanceof Error ? error.message : String(error); + logger({ + category: "agent", + message: `Error capturing screenshot: ${errorMessage}`, + level: 0, + auxiliary: { + error: { value: errorMessage, type: "string" }, + stack: { + value: error instanceof Error ? error.stack || "" : "", + type: "string", + }, + }, + }); throw error; } } diff --git a/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts b/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts index 3343c775b..e9f1397bf 100644 --- a/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts +++ b/packages/core/lib/v3/agent/utils/googleCustomToolHandler.ts @@ -135,11 +135,7 @@ function convertToolToFunctionDeclaration( description: tool.description || `Execute ${name}`, parameters, }; - } catch (error) { - console.error( - `Error converting tool ${name} to function declaration:`, - error, - ); + } catch { return null; } } diff --git a/packages/core/lib/v3/dom/piercer.runtime.ts b/packages/core/lib/v3/dom/piercer.runtime.ts index e84d09cb2..5802201db 100644 --- a/packages/core/lib/v3/dom/piercer.runtime.ts +++ b/packages/core/lib/v3/dom/piercer.runtime.ts @@ -33,8 +33,7 @@ declare global { } export function installV3ShadowPiercer(opts: V3ShadowPatchOptions = {}): void { - // hardcoded debug (remove later if desired) - const DEBUG = true; + const DEBUG = opts.debug ?? false; type PatchedFn = Element["attachShadow"] & { __v3Patched?: boolean; diff --git a/packages/core/lib/v3/handlers/v3CuaAgentHandler.ts b/packages/core/lib/v3/handlers/v3CuaAgentHandler.ts index aab4f0039..ff575fca4 100644 --- a/packages/core/lib/v3/handlers/v3CuaAgentHandler.ts +++ b/packages/core/lib/v3/handlers/v3CuaAgentHandler.ts @@ -514,7 +514,7 @@ export class V3CuaAgentHandler { const page = await this.v3.context.awaitActivePage(); const base64Image = await page.screenshot({ fullPage: false }); const currentUrl = page.url(); - return await this.agentClient.captureScreenshot({ + return await this.agentClient.captureScreenshot(this.logger, { base64Image, currentUrl, }); diff --git a/packages/core/lib/v3/llm/aisdk.ts b/packages/core/lib/v3/llm/aisdk.ts index ecf0d716c..dd317bf77 100644 --- a/packages/core/lib/v3/llm/aisdk.ts +++ b/packages/core/lib/v3/llm/aisdk.ts @@ -40,8 +40,10 @@ export class AISdkClient extends LLMClient { async createChatCompletion({ options, + logger, }: CreateChatCompletionOptions): Promise { - this.logger?.({ + const effectiveLogger = logger ?? this.logger; + effectiveLogger?.({ category: "aisdk", message: "creating chat completion", level: 2, @@ -145,7 +147,7 @@ export class AISdkClient extends LLMClient { }); } catch (err) { if (NoObjectGeneratedError.isInstance(err)) { - this.logger?.({ + effectiveLogger?.({ category: "AISDK error", message: err.message, level: 0, @@ -191,7 +193,7 @@ export class AISdkClient extends LLMClient { }, } as T; - this.logger?.({ + effectiveLogger?.({ category: "aisdk", message: "response", level: 1, @@ -277,7 +279,7 @@ export class AISdkClient extends LLMClient { }, } as T; - this.logger?.({ + effectiveLogger?.({ category: "aisdk", message: "response", level: 2, diff --git a/packages/core/lib/v3/understudy/consoleMessage.ts b/packages/core/lib/v3/understudy/consoleMessage.ts index 2a8e22074..a53f3cc35 100644 --- a/packages/core/lib/v3/understudy/consoleMessage.ts +++ b/packages/core/lib/v3/understudy/consoleMessage.ts @@ -1,5 +1,6 @@ import type { Protocol } from "devtools-protocol"; import type { Page } from "./page"; +import { v3Logger } from "../logger"; type RemoteObject = Protocol.Runtime.RemoteObject; @@ -14,7 +15,19 @@ function formatRemoteObject(obj: RemoteObject | undefined): string { if (typeof value === "string") return value; try { return JSON.stringify(value); - } catch { + } catch (error) { + v3Logger({ + category: "ctx", + message: "Failed to stringify console message value", + level: 2, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + valueType: { value: typeof value, type: "string" }, + }, + }); return String(value); } } diff --git a/packages/core/lib/v3/understudy/selectorResolver.ts b/packages/core/lib/v3/understudy/selectorResolver.ts index 06b68301d..3b6d47865 100644 --- a/packages/core/lib/v3/understudy/selectorResolver.ts +++ b/packages/core/lib/v3/understudy/selectorResolver.ts @@ -331,7 +331,19 @@ export class FrameSelectorResolver { typeof data.count === "number" ? data.count : Number(data.count); if (!Number.isFinite(num)) return 0; return Math.max(0, Math.floor(num)); - } catch { + } catch (error) { + v3Logger({ + category: "locator", + message: "Failed to count CSS selector matches", + level: 2, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + selector: { value, type: "string" }, + }, + }); return 0; } } @@ -370,7 +382,19 @@ export class FrameSelectorResolver { : Number(evalRes.result.value); if (!Number.isFinite(num)) return 0; return Math.max(0, Math.floor(num)); - } catch { + } catch (error) { + v3Logger({ + category: "locator", + message: "Failed to count XPath matches", + level: 2, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + xpath: { value, type: "string" }, + }, + }); return 0; } } @@ -386,7 +410,19 @@ export class FrameSelectorResolver { { objectId }, ); nodeId = rn.nodeId ?? null; - } catch { + } catch (error) { + v3Logger({ + category: "locator", + message: "Failed to resolve node from object ID", + level: 2, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + objectId: { value: objectId, type: "string" }, + }, + }); nodeId = null; } @@ -418,7 +454,18 @@ export class FrameSelectorResolver { const num = typeof value === "number" ? value : Number(value); if (!Number.isFinite(num)) return 0; return Math.max(0, Math.floor(num)); - } catch { + } catch (error) { + v3Logger({ + category: "locator", + message: "Failed to evaluate count expression", + level: 2, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + }, + }); return 0; } } @@ -445,7 +492,18 @@ export class FrameSelectorResolver { } return this.resolveFromObjectId(evalRes.result.objectId); - } catch { + } catch (error) { + v3Logger({ + category: "locator", + message: "Failed to evaluate element expression", + level: 2, + auxiliary: { + error: { + value: error instanceof Error ? error.message : String(error), + type: "string", + }, + }, + }); return null; } } diff --git a/packages/evals/index.eval.ts b/packages/evals/index.eval.ts index 709fd2274..e34c7d38b 100644 --- a/packages/evals/index.eval.ts +++ b/packages/evals/index.eval.ts @@ -384,9 +384,8 @@ const generateFilteredTestcases = (): Testcase[] => { return result; } catch (error) { // Log any errors that occur during task execution - console.error(`❌ ${input.name}: Error - ${error}`); logger.error({ - message: `Error in task ${input.name}`, + message: `❌ ${input.name}: Error - ${error}`, level: 0, auxiliary: { error: { diff --git a/packages/evals/tasks/agent/github_react_version.ts b/packages/evals/tasks/agent/github_react_version.ts index ddf732723..a5f980dff 100644 --- a/packages/evals/tasks/agent/github_react_version.ts +++ b/packages/evals/tasks/agent/github_react_version.ts @@ -21,8 +21,14 @@ export const github_react_version: EvalFunction = async ({ question: "Does the page show the latest version of react and the date it was published", }); - console.log(`evaluation: ${evaluation}`); - console.log(`reasoning: ${reasoning}`); + logger.log({ + message: "Evaluation results", + level: 1, + auxiliary: { + evaluation: { value: evaluation, type: "string" }, + reasoning: { value: reasoning, type: "string" }, + }, + }); // only use url check for now, as using extract on the version is prone to breaking in future const success = evaluation === "YES"; if (!success) { diff --git a/packages/evals/tasks/agent/hugging_face.ts b/packages/evals/tasks/agent/hugging_face.ts index e70259de8..12b6063f0 100644 --- a/packages/evals/tasks/agent/hugging_face.ts +++ b/packages/evals/tasks/agent/hugging_face.ts @@ -17,7 +17,10 @@ export const hugging_face: EvalFunction = async ({ "Search for a model on Hugging Face with an Apache-2.0 license that has received the highest number of likes.", maxSteps: Number(process.env.AGENT_EVAL_MAX_STEPS) || 20, }); - console.log(`agentResult: ${agentResult.message}`); + logger.log({ + message: `Agent result: ${agentResult.message}`, + level: 1, + }); const { evaluation, reasoning } = await evaluator.ask({ question: "Does the message mention 'kokoro-82m' or 'hexgrad/Kokoro-82M'?", @@ -27,7 +30,10 @@ export const hugging_face: EvalFunction = async ({ const success = evaluation === "YES"; - console.log(`reasoning: ${reasoning}`); + logger.log({ + message: `Evaluation reasoning: ${reasoning}`, + level: 1, + }); if (!success) { return { _success: false, diff --git a/packages/evals/tasks/amazon_add_to_cart.ts b/packages/evals/tasks/amazon_add_to_cart.ts index 07c1b0b5a..91cb7d7c5 100644 --- a/packages/evals/tasks/amazon_add_to_cart.ts +++ b/packages/evals/tasks/amazon_add_to_cart.ts @@ -20,8 +20,14 @@ export const amazon_add_to_cart: EvalFunction = async ({ const expectedUrl = "https://browserbase.github.io/stagehand-eval-sites/sites/amazon/sign-in.html"; - console.log("currentUrl", currentUrl); - console.log("expectedUrl", expectedUrl); + logger.log({ + message: "URL comparison", + level: 1, + auxiliary: { + currentUrl: { value: currentUrl, type: "string" }, + expectedUrl: { value: expectedUrl, type: "string" }, + }, + }); return { _success: currentUrl === expectedUrl, currentUrl,