diff --git a/.changeset/agent-execute-variables.md b/.changeset/agent-execute-variables.md new file mode 100644 index 000000000..ab8e4faf0 --- /dev/null +++ b/.changeset/agent-execute-variables.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Add variables support to v3 agentExecute API schema and remove experimental requirement diff --git a/packages/core/lib/v3/agent/tools/fillFormVision.ts b/packages/core/lib/v3/agent/tools/fillFormVision.ts index 677638425..34acf0f65 100644 --- a/packages/core/lib/v3/agent/tools/fillFormVision.ts +++ b/packages/core/lib/v3/agent/tools/fillFormVision.ts @@ -64,9 +64,12 @@ MANDATORY USE CASES (always use fillFormVision for these): try { const page = await v3.context.awaitActivePage(); - // Process coordinates and substitute variables for each field - // Keep original values (with %tokens%) for logging/caching, substituted values for typing - const processedFields = fields.map((field) => { + // Process coordinates per field. Keep the original `value` (with any + // `%variableName%` tokens) so substituted secrets never leak through + // the tool result, replay cache, returned AgentResult.actions, or + // anything that logs the action. Variables are substituted only at + // typing time below. + const safeFields = fields.map((field) => { const processed = processCoordinates( field.coordinates.x, field.coordinates.y, @@ -74,9 +77,8 @@ MANDATORY USE CASES (always use fillFormVision for these): v3, ); return { - ...field, - originalValue: field.value, // Keep original with %tokens% for cache - value: substituteVariables(field.value, variables), + action: field.action, + value: field.value, coordinates: { x: processed.x, y: processed.y }, }; }); @@ -97,7 +99,7 @@ MANDATORY USE CASES (always use fillFormVision for these): const shouldCollectXpath = v3.isAgentReplayActive(); const actions: Action[] = []; - for (const field of processedFields) { + for (const field of safeFields) { // Click the field, only requesting XPath when caching is enabled const xpath = await page.click( field.coordinates.x, @@ -106,10 +108,12 @@ MANDATORY USE CASES (always use fillFormVision for these): returnXpath: shouldCollectXpath, }, ); - await page.type(field.value); + // Substitute variables only at the moment of typing so the resolved + // value never leaves this scope. + await page.type(substituteVariables(field.value, variables)); - // Build Action with XPath for deterministic replay (only when caching) - // Use originalValue (with %tokens%) so cache stores references, not sensitive values + // Build Action with XPath for deterministic replay (only when caching). + // Store the placeholder value so cache entries don't contain secrets. if (shouldCollectXpath) { const normalizedXpath = ensureXPath(xpath); if (normalizedXpath) { @@ -117,7 +121,7 @@ MANDATORY USE CASES (always use fillFormVision for these): selector: normalizedXpath, description: field.action, method: "type", - arguments: [field.originalValue], + arguments: [field.value], }); } } @@ -140,7 +144,7 @@ MANDATORY USE CASES (always use fillFormVision for these): return { success: true, - playwrightArguments: processedFields, + playwrightArguments: safeFields, screenshotBase64, }; } catch (error) { diff --git a/packages/core/lib/v3/agent/utils/validateExperimentalFeatures.ts b/packages/core/lib/v3/agent/utils/validateExperimentalFeatures.ts index cfcad7140..f20f6a2c0 100644 --- a/packages/core/lib/v3/agent/utils/validateExperimentalFeatures.ts +++ b/packages/core/lib/v3/agent/utils/validateExperimentalFeatures.ts @@ -115,12 +115,6 @@ export function validateExperimentalFeatures( if (executeOptions.output) { features.push("output schema"); } - if ( - executeOptions.variables && - Object.keys(executeOptions.variables).length > 0 - ) { - features.push("variables"); - } } if (features.length > 0) { diff --git a/packages/core/lib/v3/types/public/agent.ts b/packages/core/lib/v3/types/public/agent.ts index 7b2ed8e07..846ebe70e 100644 --- a/packages/core/lib/v3/types/public/agent.ts +++ b/packages/core/lib/v3/types/public/agent.ts @@ -377,9 +377,8 @@ export interface AgentExecuteOptionsBase { * * Accepts both simple values and rich objects with descriptions (same type as `act`). * - * **Note:** Not supported in CUA mode (`mode: "cua"`). Requires `experimental: true`. + * **Note:** Not supported in CUA mode (`mode: "cua"`). * - * @experimental * @example * ```typescript * // Simple values diff --git a/packages/core/lib/v3/types/public/api.ts b/packages/core/lib/v3/types/public/api.ts index 4332053f5..daf5dea34 100644 --- a/packages/core/lib/v3/types/public/api.ts +++ b/packages/core/lib/v3/types/public/api.ts @@ -740,6 +740,10 @@ export const AgentExecuteOptionsSchema = z description: "Timeout in milliseconds for each agent tool call", example: 30000, }), + variables: VariablesSchema.optional().meta({ + description: + "Variables available to the agent via %variableName% syntax in supported tools", + }), }) .meta({ id: "AgentExecuteOptions" }); diff --git a/packages/core/tests/unit/agent-variables-validation.test.ts b/packages/core/tests/unit/agent-variables-validation.test.ts new file mode 100644 index 000000000..cd6f4b162 --- /dev/null +++ b/packages/core/tests/unit/agent-variables-validation.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { validateExperimentalFeatures } from "../../lib/v3/agent/utils/validateExperimentalFeatures.js"; +import { StagehandInvalidArgumentError } from "../../lib/v3/types/public/sdkErrors.js"; + +describe("agent variable experimental validation", () => { + it("allows variables without experimental mode", () => { + expect(() => + validateExperimentalFeatures({ + isExperimental: false, + agentConfig: { mode: "dom" }, + executeOptions: { + instruction: "fill %username%", + variables: { username: "john@example.com" }, + }, + }), + ).not.toThrow(); + }); + + it("allows rich variables without experimental mode", () => { + expect(() => + validateExperimentalFeatures({ + isExperimental: false, + agentConfig: { mode: "dom" }, + executeOptions: { + instruction: "fill %username%", + variables: { + username: { + value: "john@example.com", + description: "The login email", + }, + }, + }, + }), + ).not.toThrow(); + }); + + it("continues to reject variables in CUA mode", () => { + expect(() => + validateExperimentalFeatures({ + isExperimental: true, + agentConfig: { mode: "cua" }, + executeOptions: { + instruction: "fill %username%", + variables: { username: "john@example.com" }, + }, + }), + ).toThrow(StagehandInvalidArgumentError); + }); +}); diff --git a/packages/core/tests/unit/api-client-observe-variables.test.ts b/packages/core/tests/unit/api-client-observe-variables.test.ts index ee306642a..98856cf7d 100644 --- a/packages/core/tests/unit/api-client-observe-variables.test.ts +++ b/packages/core/tests/unit/api-client-observe-variables.test.ts @@ -96,4 +96,62 @@ describe("StagehandAPIClient variable serialization", () => { serverCache: undefined, }); }); + + it("preserves rich variables when sending the agentExecute request", async () => { + const client = new StagehandAPIClient({ + apiKey: "bb-test", + logger: vi.fn(), + }); + const executeMock = vi.fn().mockResolvedValue({ + success: true, + message: "ok", + actions: [], + completed: true, + }); + + ( + client as unknown as { + execute: typeof executeMock; + } + ).execute = executeMock; + + await client.agentExecute( + { mode: "dom" }, + { + instruction: "fill the form with %username% and %password%", + variables: { + username: "john@example.com", + password: { + value: "secret", + description: "The login password", + }, + }, + }, + ); + + expect(executeMock).toHaveBeenCalledWith({ + method: "agentExecute", + args: { + agentConfig: { + systemPrompt: undefined, + mode: "dom", + cua: undefined, + model: undefined, + executionModel: undefined, + }, + executeOptions: { + instruction: "fill the form with %username% and %password%", + variables: { + username: "john@example.com", + password: { + value: "secret", + description: "The login password", + }, + }, + }, + frameId: undefined, + shouldCache: undefined, + }, + }); + }); }); diff --git a/packages/core/tests/unit/api-variables-schema.test.ts b/packages/core/tests/unit/api-variables-schema.test.ts index cda10c19c..ecd33723f 100644 --- a/packages/core/tests/unit/api-variables-schema.test.ts +++ b/packages/core/tests/unit/api-variables-schema.test.ts @@ -35,4 +35,30 @@ describe("API variable schemas", () => { expect(result.success).toBe(true); }); + + it("preserves variables for agent execute requests", () => { + const result = Api.AgentExecuteRequestSchema.safeParse({ + agentConfig: { mode: "dom" }, + executeOptions: { + instruction: "fill the form with %username% and %password%", + variables: { + username: "john@example.com", + password: { + value: "secret-password", + description: "The login password", + }, + }, + }, + }); + + expect(result.success).toBe(true); + if (!result.success) throw result.error; + expect(result.data.executeOptions.variables).toEqual({ + username: "john@example.com", + password: { + value: "secret-password", + description: "The login password", + }, + }); + }); }); diff --git a/packages/docs/v3/basics/agent.mdx b/packages/docs/v3/basics/agent.mdx index edb6aa5ca..d22c6ccfc 100644 --- a/packages/docs/v3/basics/agent.mdx +++ b/packages/docs/v3/basics/agent.mdx @@ -486,14 +486,13 @@ const result = await agent.execute({ Use variables to pass sensitive data (like passwords, API keys, or personal information) to the agent without exposing the actual values to the LLM. The agent sees only variable names and descriptions, while the actual values are substituted at runtime. -**Non-CUA agents only.** Variables require `experimental: true` and are not available with Computer Use Agents. +**Non-CUA agents only.** Variables are not available with Computer Use Agents. ### Basic Usage ```typescript const stagehand = new Stagehand({ env: "LOCAL", - experimental: true, // Required for variables }); await stagehand.init(); diff --git a/packages/server-v3/openapi.v3.yaml b/packages/server-v3/openapi.v3.yaml index 2b3b554dc..ef26bf9a3 100644 --- a/packages/server-v3/openapi.v3.yaml +++ b/packages/server-v3/openapi.v3.yaml @@ -847,6 +847,9 @@ components: description: Timeout in milliseconds for each agent tool call example: 30000 type: number + variables: + description: Variables available to the agent via %variableName% syntax in supported tools + $ref: "#/components/schemas/Variables" required: - instruction AgentExecuteRequest: @@ -1767,6 +1770,9 @@ components: description: Timeout in milliseconds for each agent tool call example: 30000 type: number + variables: + description: Variables available to the agent via %variableName% syntax in supported tools + $ref: "#/components/schemas/Variables" required: - instruction additionalProperties: false diff --git a/packages/server-v3/tests/integration/v3/act.test.ts b/packages/server-v3/tests/integration/v3/act.test.ts index 3deb76e74..3d4e7d94a 100644 --- a/packages/server-v3/tests/integration/v3/act.test.ts +++ b/packages/server-v3/tests/integration/v3/act.test.ts @@ -347,6 +347,143 @@ describe("POST /v1/sessions/:id/act (V3)", () => { }); }); +// ============================================================================= +// Variables Tests (V3) - Substitute %variableName% placeholders into act input +// ============================================================================= + +describe("POST /v1/sessions/:id/act (V3) - variables", () => { + const LOGIN_URL = + "https://browserbase.github.io/stagehand-eval-sites/sites/login/"; + const EMAIL_XPATH = "/html/body/main/form/div[1]/input"; + const PASSWORD_XPATH = "/html/body/main/form/div[2]/input"; + + let varsSessionId: string; + let varsCdpUrl: string; + + before(async () => { + ({ sessionId: varsSessionId, cdpUrl: varsCdpUrl } = + await createSessionWithCdp(getHeaders("3.0.0"))); + }); + + beforeEach(async () => { + const navResponse = await navigateSession( + varsSessionId, + LOGIN_URL, + getHeaders("3.0.0"), + ); + assert.equal( + navResponse.status, + HTTP_OK, + "Navigate to login should succeed", + ); + }); + + after(async () => { + await endSession(varsSessionId, getHeaders("3.0.0")); + }); + + async function readInputValue(xpath: string): Promise { + const browser = await chromium.connectOverCDP(varsCdpUrl); + try { + const pages = browser.contexts()[0]!.pages(); + return await pages[0]!.locator(`xpath=${xpath}`).inputValue(); + } finally { + await browser.close(); + } + } + + it("should substitute simple variable into typed value", async () => { + const url = getBaseUrl(); + const openaiApiKey = requireEnv("OPENAI_API_KEY", OPENAI_API_KEY); + + const ctx = await fetchWithContext( + `${url}/v1/sessions/${varsSessionId}/act`, + { + method: "POST", + headers: { + ...getHeaders("3.0.0"), + "x-model-api-key": "", + }, + body: JSON.stringify({ + input: "type %username% into the email field", + options: { + model: { + modelName: "openai/gpt-4.1-mini", + apiKey: openaiApiKey, + }, + variables: { + username: "john@example.com", + }, + }, + }), + }, + ); + + assertFetchStatus(ctx, HTTP_OK, "act with simple variable should succeed"); + assertFetchOk(ctx.body !== null, "Response should have body", ctx); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + assertFetchOk( + !!ctx.body.data?.result?.success, + `act result should be success, got: ${ctx.body.data?.result?.message ?? ""}`, + ctx, + ); + + const emailValue = await readInputValue(EMAIL_XPATH); + assert.equal( + emailValue, + "john@example.com", + "Email field should contain substituted variable value", + ); + }); + + it("should substitute rich variable (with description) into typed value", async () => { + const url = getBaseUrl(); + const openaiApiKey = requireEnv("OPENAI_API_KEY", OPENAI_API_KEY); + + const ctx = await fetchWithContext( + `${url}/v1/sessions/${varsSessionId}/act`, + { + method: "POST", + headers: { + ...getHeaders("3.0.0"), + "x-model-api-key": "", + }, + body: JSON.stringify({ + input: "type %password% into the password field", + options: { + model: { + modelName: "openai/gpt-4.1-mini", + apiKey: openaiApiKey, + }, + variables: { + password: { + value: "secret123", + description: "The user's password", + }, + }, + }, + }), + }, + ); + + assertFetchStatus(ctx, HTTP_OK, "act with rich variable should succeed"); + assertFetchOk(ctx.body !== null, "Response should have body", ctx); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + assertFetchOk( + !!ctx.body.data?.result?.success, + `act result should be success, got: ${ctx.body.data?.result?.message ?? ""}`, + ctx, + ); + + const passwordValue = await readInputValue(PASSWORD_XPATH); + assert.equal( + passwordValue, + "secret123", + "Password field should contain substituted variable value", + ); + }); +}); + // ============================================================================= // SSE Streaming Tests (V3) // ============================================================================= diff --git a/packages/server-v3/tests/integration/v3/agentExecute.test.ts b/packages/server-v3/tests/integration/v3/agentExecute.test.ts index 0a0a14337..cee5d3f66 100644 --- a/packages/server-v3/tests/integration/v3/agentExecute.test.ts +++ b/packages/server-v3/tests/integration/v3/agentExecute.test.ts @@ -1,6 +1,8 @@ import assert from "node:assert/strict"; import { after, before, beforeEach, describe, it } from "node:test"; +import { chromium } from "playwright"; + import { assertFetchOk, assertFetchStatus, @@ -942,6 +944,165 @@ describe("POST /v1/sessions/:id/agentExecute - agentConfig.mode (V3)", () => { }); }); +// ============================================================================= +// V3 Variables Tests - executeOptions.variables substitution into agent tool calls +// ============================================================================= + +describe("POST /v1/sessions/:id/agentExecute (V3) - variables", () => { + const LOGIN_URL = + "https://browserbase.github.io/stagehand-eval-sites/sites/login/"; + const EMAIL_XPATH = "/html/body/main/form/div[1]/input"; + const PASSWORD_XPATH = "/html/body/main/form/div[2]/input"; + + let sessionId: string; + let cdpUrl: string; + const headers = getHeaders("3.0.0"); + + before(async () => { + ({ sessionId, cdpUrl } = await createSessionWithCdp(headers)); + }); + + beforeEach(async () => { + const navResponse = await navigateSession(sessionId, LOGIN_URL, headers); + assert.equal( + navResponse.status, + HTTP_OK, + "Navigate to login should succeed", + ); + }); + + after(async () => { + if (sessionId) { + await endSession(sessionId, headers); + sessionId = ""; + } + }); + + async function readInputValue(xpath: string): Promise { + const browser = await chromium.connectOverCDP(cdpUrl); + try { + const pages = browser.contexts()[0]!.pages(); + return await pages[0]!.locator(`xpath=${xpath}`).inputValue(); + } finally { + await browser.close(); + } + } + + it("should substitute simple variables into agent tool calls", async () => { + const url = getBaseUrl(); + const openaiApiKey = requireEnv("OPENAI_API_KEY", OPENAI_API_KEY); + + const ctx = await fetchWithContext<{ + success: boolean; + data?: { result: unknown; actionId?: string }; + }>(`${url}/v1/sessions/${sessionId}/agentExecute`, { + method: "POST", + headers: { + ...headers, + "x-model-api-key": "", + }, + body: JSON.stringify({ + agentConfig: { + model: { + modelName: "openai/gpt-4.1-nano", + apiKey: openaiApiKey, + }, + }, + executeOptions: { + instruction: + "Type %username% into the email field and %password% into the password field. Do not submit the form.", + maxSteps: 5, + variables: { + username: "john@example.com", + password: "secret123", + }, + }, + }), + }); + + assertFetchStatus( + ctx, + HTTP_OK, + "Agent execute with simple variables should succeed", + ); + assertFetchOk(ctx.body !== null, "Response body should be parseable", ctx); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + + const emailValue = await readInputValue(EMAIL_XPATH); + const passwordValue = await readInputValue(PASSWORD_XPATH); + assert.equal( + emailValue, + "john@example.com", + "Email field should contain substituted username variable value", + ); + assert.equal( + passwordValue, + "secret123", + "Password field should contain substituted password variable value", + ); + }); + + it("should substitute rich variables (with descriptions) into agent tool calls", async () => { + const url = getBaseUrl(); + const openaiApiKey = requireEnv("OPENAI_API_KEY", OPENAI_API_KEY); + + const ctx = await fetchWithContext<{ + success: boolean; + data?: { result: unknown; actionId?: string }; + }>(`${url}/v1/sessions/${sessionId}/agentExecute`, { + method: "POST", + headers: { + ...headers, + "x-model-api-key": "", + }, + body: JSON.stringify({ + agentConfig: { + model: { + modelName: "openai/gpt-4.1-nano", + apiKey: openaiApiKey, + }, + }, + executeOptions: { + instruction: + "Type %username% into the email field and %password% into the password field. Do not submit the form.", + maxSteps: 5, + variables: { + username: { + value: "alice@example.com", + description: "The login email", + }, + password: { + value: "rich-pw-456", + description: "The login password", + }, + }, + }, + }), + }); + + assertFetchStatus( + ctx, + HTTP_OK, + "Agent execute with rich variables should succeed", + ); + assertFetchOk(ctx.body !== null, "Response body should be parseable", ctx); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + + const emailValue = await readInputValue(EMAIL_XPATH); + const passwordValue = await readInputValue(PASSWORD_XPATH); + assert.equal( + emailValue, + "alice@example.com", + "Email field should contain substituted username variable value", + ); + assert.equal( + passwordValue, + "rich-pw-456", + "Password field should contain substituted password variable value", + ); + }); +}); + // ============================================================================= // SSE Streaming Tests (V3) // ============================================================================= diff --git a/packages/server-v3/tests/integration/v3/observe.test.ts b/packages/server-v3/tests/integration/v3/observe.test.ts index efc993059..2f6b71549 100644 --- a/packages/server-v3/tests/integration/v3/observe.test.ts +++ b/packages/server-v3/tests/integration/v3/observe.test.ts @@ -164,35 +164,59 @@ describe("POST /v1/sessions/:id/observe (V3)", () => { data?: { result: unknown[]; actionId?: string }; } - const ctx = await fetchWithContext( - `${url}/v1/sessions/${sessionId}/observe`, - { - method: "POST", - headers: getHeaders("3.0.0"), - body: JSON.stringify({ - instruction: "Find any link on the page", - options: { - variables: { - username: "john@example.com", - }, - }, - }), - }, + const navResponse = await navigateSession( + sessionId, + "https://browserbase.github.io/stagehand-eval-sites/sites/login/", + getHeaders("3.0.0"), ); + assert.equal( + navResponse.status, + HTTP_OK, + "Navigate to login should succeed", + ); + + try { + const ctx = await fetchWithContext( + `${url}/v1/sessions/${sessionId}/observe`, + { + method: "POST", + headers: getHeaders("3.0.0"), + body: JSON.stringify({ + instruction: "Find the field for entering %username%", + options: { + variables: { + username: "john@example.com", + password: "secret123", + }, + }, + }), + }, + ); - assertFetchStatus(ctx, HTTP_OK, "Observe with variables should succeed"); - assertFetchOk(ctx.body !== null, "Response body should be parseable", ctx); - assertFetchOk(ctx.body.success, "Response should indicate success", ctx); - assertFetchOk( - ctx.body.data !== undefined, - "Response should have data", - ctx, - ); - assertFetchOk( - Array.isArray(ctx.body.data.result), - "Result should be an array of observed elements", - ctx, - ); + assertFetchStatus(ctx, HTTP_OK, "Observe with variables should succeed"); + assertFetchOk( + ctx.body !== null, + "Response body should be parseable", + ctx, + ); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + assertFetchOk( + ctx.body.data !== undefined, + "Response should have data", + ctx, + ); + assertFetchOk( + Array.isArray(ctx.body.data.result), + "Result should be an array of observed elements", + ctx, + ); + } finally { + await navigateSession( + sessionId, + "https://example.com", + getHeaders("3.0.0"), + ); + } }); it("should observe with rich variables option", async () => { @@ -203,42 +227,69 @@ describe("POST /v1/sessions/:id/observe (V3)", () => { data?: { result: unknown[]; actionId?: string }; } - const ctx = await fetchWithContext( - `${url}/v1/sessions/${sessionId}/observe`, - { - method: "POST", - headers: getHeaders("3.0.0"), - body: JSON.stringify({ - instruction: "Find any link on the page", - options: { - variables: { - username: { - value: "john@example.com", - description: "The login email", + const navResponse = await navigateSession( + sessionId, + "https://browserbase.github.io/stagehand-eval-sites/sites/login/", + getHeaders("3.0.0"), + ); + assert.equal( + navResponse.status, + HTTP_OK, + "Navigate to login should succeed", + ); + + try { + const ctx = await fetchWithContext( + `${url}/v1/sessions/${sessionId}/observe`, + { + method: "POST", + headers: getHeaders("3.0.0"), + body: JSON.stringify({ + instruction: "Find the field for entering %username%", + options: { + variables: { + username: { + value: "john@example.com", + description: "The login email", + }, + password: { + value: "secret123", + description: "The login password", + }, }, }, - }, - }), - }, - ); + }), + }, + ); - assertFetchStatus( - ctx, - HTTP_OK, - "Observe with rich variables should succeed", - ); - assertFetchOk(ctx.body !== null, "Response body should be parseable", ctx); - assertFetchOk(ctx.body.success, "Response should indicate success", ctx); - assertFetchOk( - ctx.body.data !== undefined, - "Response should have data", - ctx, - ); - assertFetchOk( - Array.isArray(ctx.body.data.result), - "Result should be an array of observed elements", - ctx, - ); + assertFetchStatus( + ctx, + HTTP_OK, + "Observe with rich variables should succeed", + ); + assertFetchOk( + ctx.body !== null, + "Response body should be parseable", + ctx, + ); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + assertFetchOk( + ctx.body.data !== undefined, + "Response should have data", + ctx, + ); + assertFetchOk( + Array.isArray(ctx.body.data.result), + "Result should be an array of observed elements", + ctx, + ); + } finally { + await navigateSession( + sessionId, + "https://example.com", + getHeaders("3.0.0"), + ); + } }); it("should observe without instruction (observe all)", async () => {