From 284ca245bf101c3e5d438480efc968beba246124 Mon Sep 17 00:00:00 2001 From: Chromie Date: Mon, 4 May 2026 16:40:01 -0700 Subject: [PATCH 1/5] remove experimental requirement on agent variables (#2079) # why # what changed # test plan --- ## Summary by cubic Enable variables in v3 `agentExecute` without requiring experimental mode, and update API schemas to preserve both simple and rich variable shapes. CUA mode remains unsupported for variables. - **New Features** - Added `variables` to `AgentExecuteOptionsSchema` and server OpenAPI so values pass through unchanged. - Removed the experimental-feature check for variables in `validateExperimentalFeatures`; CUA-mode rejection stays. - Updated JSDoc on `AgentExecuteOptionsBase.variables` and added unit tests for validation, schema parsing, and client serialization. Written for commit 65b3c75eb541f78a897ce91cb9688fa959f05c0d. Summary will update on new commits. --------- Co-authored-by: Claude --- .changeset/agent-execute-variables.md | 5 ++ .../utils/validateExperimentalFeatures.ts | 6 -- packages/core/lib/v3/types/public/agent.ts | 3 +- packages/core/lib/v3/types/public/api.ts | 4 ++ .../unit/agent-variables-validation.test.ts | 49 ++++++++++++++++ .../unit/api-client-observe-variables.test.ts | 58 +++++++++++++++++++ .../tests/unit/api-variables-schema.test.ts | 26 +++++++++ packages/server-v3/openapi.v3.yaml | 6 ++ 8 files changed, 149 insertions(+), 8 deletions(-) create mode 100644 .changeset/agent-execute-variables.md create mode 100644 packages/core/tests/unit/agent-variables-validation.test.ts diff --git a/.changeset/agent-execute-variables.md b/.changeset/agent-execute-variables.md new file mode 100644 index 0000000000..ab8e4faf0b --- /dev/null +++ b/.changeset/agent-execute-variables.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Add variables support to v3 agentExecute API schema and remove experimental requirement diff --git a/packages/core/lib/v3/agent/utils/validateExperimentalFeatures.ts b/packages/core/lib/v3/agent/utils/validateExperimentalFeatures.ts index cfcad71403..f20f6a2c09 100644 --- a/packages/core/lib/v3/agent/utils/validateExperimentalFeatures.ts +++ b/packages/core/lib/v3/agent/utils/validateExperimentalFeatures.ts @@ -115,12 +115,6 @@ export function validateExperimentalFeatures( if (executeOptions.output) { features.push("output schema"); } - if ( - executeOptions.variables && - Object.keys(executeOptions.variables).length > 0 - ) { - features.push("variables"); - } } if (features.length > 0) { diff --git a/packages/core/lib/v3/types/public/agent.ts b/packages/core/lib/v3/types/public/agent.ts index 7b2ed8e070..846ebe70e8 100644 --- a/packages/core/lib/v3/types/public/agent.ts +++ b/packages/core/lib/v3/types/public/agent.ts @@ -377,9 +377,8 @@ export interface AgentExecuteOptionsBase { * * Accepts both simple values and rich objects with descriptions (same type as `act`). * - * **Note:** Not supported in CUA mode (`mode: "cua"`). Requires `experimental: true`. + * **Note:** Not supported in CUA mode (`mode: "cua"`). * - * @experimental * @example * ```typescript * // Simple values diff --git a/packages/core/lib/v3/types/public/api.ts b/packages/core/lib/v3/types/public/api.ts index 4332053f51..daf5dea344 100644 --- a/packages/core/lib/v3/types/public/api.ts +++ b/packages/core/lib/v3/types/public/api.ts @@ -740,6 +740,10 @@ export const AgentExecuteOptionsSchema = z description: "Timeout in milliseconds for each agent tool call", example: 30000, }), + variables: VariablesSchema.optional().meta({ + description: + "Variables available to the agent via %variableName% syntax in supported tools", + }), }) .meta({ id: "AgentExecuteOptions" }); diff --git a/packages/core/tests/unit/agent-variables-validation.test.ts b/packages/core/tests/unit/agent-variables-validation.test.ts new file mode 100644 index 0000000000..cd6f4b1624 --- /dev/null +++ b/packages/core/tests/unit/agent-variables-validation.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { validateExperimentalFeatures } from "../../lib/v3/agent/utils/validateExperimentalFeatures.js"; +import { StagehandInvalidArgumentError } from "../../lib/v3/types/public/sdkErrors.js"; + +describe("agent variable experimental validation", () => { + it("allows variables without experimental mode", () => { + expect(() => + validateExperimentalFeatures({ + isExperimental: false, + agentConfig: { mode: "dom" }, + executeOptions: { + instruction: "fill %username%", + variables: { username: "john@example.com" }, + }, + }), + ).not.toThrow(); + }); + + it("allows rich variables without experimental mode", () => { + expect(() => + validateExperimentalFeatures({ + isExperimental: false, + agentConfig: { mode: "dom" }, + executeOptions: { + instruction: "fill %username%", + variables: { + username: { + value: "john@example.com", + description: "The login email", + }, + }, + }, + }), + ).not.toThrow(); + }); + + it("continues to reject variables in CUA mode", () => { + expect(() => + validateExperimentalFeatures({ + isExperimental: true, + agentConfig: { mode: "cua" }, + executeOptions: { + instruction: "fill %username%", + variables: { username: "john@example.com" }, + }, + }), + ).toThrow(StagehandInvalidArgumentError); + }); +}); diff --git a/packages/core/tests/unit/api-client-observe-variables.test.ts b/packages/core/tests/unit/api-client-observe-variables.test.ts index ee306642a4..98856cf7d8 100644 --- a/packages/core/tests/unit/api-client-observe-variables.test.ts +++ b/packages/core/tests/unit/api-client-observe-variables.test.ts @@ -96,4 +96,62 @@ describe("StagehandAPIClient variable serialization", () => { serverCache: undefined, }); }); + + it("preserves rich variables when sending the agentExecute request", async () => { + const client = new StagehandAPIClient({ + apiKey: "bb-test", + logger: vi.fn(), + }); + const executeMock = vi.fn().mockResolvedValue({ + success: true, + message: "ok", + actions: [], + completed: true, + }); + + ( + client as unknown as { + execute: typeof executeMock; + } + ).execute = executeMock; + + await client.agentExecute( + { mode: "dom" }, + { + instruction: "fill the form with %username% and %password%", + variables: { + username: "john@example.com", + password: { + value: "secret", + description: "The login password", + }, + }, + }, + ); + + expect(executeMock).toHaveBeenCalledWith({ + method: "agentExecute", + args: { + agentConfig: { + systemPrompt: undefined, + mode: "dom", + cua: undefined, + model: undefined, + executionModel: undefined, + }, + executeOptions: { + instruction: "fill the form with %username% and %password%", + variables: { + username: "john@example.com", + password: { + value: "secret", + description: "The login password", + }, + }, + }, + frameId: undefined, + shouldCache: undefined, + }, + }); + }); }); diff --git a/packages/core/tests/unit/api-variables-schema.test.ts b/packages/core/tests/unit/api-variables-schema.test.ts index cda10c19c4..ecd33723f5 100644 --- a/packages/core/tests/unit/api-variables-schema.test.ts +++ b/packages/core/tests/unit/api-variables-schema.test.ts @@ -35,4 +35,30 @@ describe("API variable schemas", () => { expect(result.success).toBe(true); }); + + it("preserves variables for agent execute requests", () => { + const result = Api.AgentExecuteRequestSchema.safeParse({ + agentConfig: { mode: "dom" }, + executeOptions: { + instruction: "fill the form with %username% and %password%", + variables: { + username: "john@example.com", + password: { + value: "secret-password", + description: "The login password", + }, + }, + }, + }); + + expect(result.success).toBe(true); + if (!result.success) throw result.error; + expect(result.data.executeOptions.variables).toEqual({ + username: "john@example.com", + password: { + value: "secret-password", + description: "The login password", + }, + }); + }); }); diff --git a/packages/server-v3/openapi.v3.yaml b/packages/server-v3/openapi.v3.yaml index 2b3b554dcc..ef26bf9a34 100644 --- a/packages/server-v3/openapi.v3.yaml +++ b/packages/server-v3/openapi.v3.yaml @@ -847,6 +847,9 @@ components: description: Timeout in milliseconds for each agent tool call example: 30000 type: number + variables: + description: Variables available to the agent via %variableName% syntax in supported tools + $ref: "#/components/schemas/Variables" required: - instruction AgentExecuteRequest: @@ -1767,6 +1770,9 @@ components: description: Timeout in milliseconds for each agent tool call example: 30000 type: number + variables: + description: Variables available to the agent via %variableName% syntax in supported tools + $ref: "#/components/schemas/Variables" required: - instruction additionalProperties: false From d17da4f60acfed562306388510fd08e69a31604a Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 4 May 2026 16:43:30 -0700 Subject: [PATCH 2/5] update docs --- packages/docs/v3/basics/agent.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/docs/v3/basics/agent.mdx b/packages/docs/v3/basics/agent.mdx index edb6aa5ca0..d22c6ccfcd 100644 --- a/packages/docs/v3/basics/agent.mdx +++ b/packages/docs/v3/basics/agent.mdx @@ -486,14 +486,13 @@ const result = await agent.execute({ Use variables to pass sensitive data (like passwords, API keys, or personal information) to the agent without exposing the actual values to the LLM. The agent sees only variable names and descriptions, while the actual values are substituted at runtime. -**Non-CUA agents only.** Variables require `experimental: true` and are not available with Computer Use Agents. +**Non-CUA agents only.** Variables are not available with Computer Use Agents. ### Basic Usage ```typescript const stagehand = new Stagehand({ env: "LOCAL", - experimental: true, // Required for variables }); await stagehand.init(); From 4467fe178fc142c38147e0b718b7f60b73f8604a Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 4 May 2026 17:00:12 -0700 Subject: [PATCH 3/5] update tests --- .../tests/integration/v3/act.test.ts | 111 ++++++++++++ .../tests/integration/v3/agentExecute.test.ts | 161 +++++++++++++++++ .../tests/integration/v3/observe.test.ts | 171 ++++++++++++------ 3 files changed, 383 insertions(+), 60 deletions(-) diff --git a/packages/server-v3/tests/integration/v3/act.test.ts b/packages/server-v3/tests/integration/v3/act.test.ts index 3deb76e74e..4e4f19ef75 100644 --- a/packages/server-v3/tests/integration/v3/act.test.ts +++ b/packages/server-v3/tests/integration/v3/act.test.ts @@ -347,6 +347,117 @@ describe("POST /v1/sessions/:id/act (V3)", () => { }); }); +// ============================================================================= +// Variables Tests (V3) - Substitute %variableName% placeholders into act input +// ============================================================================= + +describe("POST /v1/sessions/:id/act (V3) - variables", () => { + const LOGIN_URL = + "https://browserbase.github.io/stagehand-eval-sites/sites/login/"; + const EMAIL_XPATH = "/html/body/main/form/div[1]/input"; + const PASSWORD_XPATH = "/html/body/main/form/div[2]/input"; + + let varsSessionId: string; + let varsCdpUrl: string; + + before(async () => { + ({ sessionId: varsSessionId, cdpUrl: varsCdpUrl } = + await createSessionWithCdp(getHeaders("3.0.0"))); + }); + + beforeEach(async () => { + const navResponse = await navigateSession( + varsSessionId, + LOGIN_URL, + getHeaders("3.0.0"), + ); + assert.equal( + navResponse.status, + HTTP_OK, + "Navigate to login should succeed", + ); + }); + + after(async () => { + await endSession(varsSessionId, getHeaders("3.0.0")); + }); + + async function readInputValue(xpath: string): Promise { + const browser = await chromium.connectOverCDP(varsCdpUrl); + try { + const pages = browser.contexts()[0]!.pages(); + return await pages[0]!.locator(`xpath=${xpath}`).inputValue(); + } finally { + await browser.close(); + } + } + + it("should substitute simple variable into typed value", async () => { + const url = getBaseUrl(); + + const ctx = await fetchWithContext( + `${url}/v1/sessions/${varsSessionId}/act`, + { + method: "POST", + headers: getHeaders("3.0.0"), + body: JSON.stringify({ + input: "type %username% into the email field", + options: { + variables: { + username: "john@example.com", + }, + }, + }), + }, + ); + + assertFetchStatus(ctx, HTTP_OK, "act with simple variable should succeed"); + assertFetchOk(ctx.body !== null, "Response should have body", ctx); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + + const emailValue = await readInputValue(EMAIL_XPATH); + assert.equal( + emailValue, + "john@example.com", + "Email field should contain substituted variable value", + ); + }); + + it("should substitute rich variable (with description) into typed value", async () => { + const url = getBaseUrl(); + + const ctx = await fetchWithContext( + `${url}/v1/sessions/${varsSessionId}/act`, + { + method: "POST", + headers: getHeaders("3.0.0"), + body: JSON.stringify({ + input: "type %password% into the password field", + options: { + variables: { + password: { + value: "secret123", + description: "The user's password", + }, + }, + }, + }), + }, + ); + + assertFetchStatus(ctx, HTTP_OK, "act with rich variable should succeed"); + assertFetchOk(ctx.body !== null, "Response should have body", ctx); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + + const passwordValue = await readInputValue(PASSWORD_XPATH); + assert.equal( + passwordValue, + "secret123", + "Password field should contain substituted variable value", + ); + }); +}); + // ============================================================================= // SSE Streaming Tests (V3) // ============================================================================= diff --git a/packages/server-v3/tests/integration/v3/agentExecute.test.ts b/packages/server-v3/tests/integration/v3/agentExecute.test.ts index 0a0a143378..cee5d3f667 100644 --- a/packages/server-v3/tests/integration/v3/agentExecute.test.ts +++ b/packages/server-v3/tests/integration/v3/agentExecute.test.ts @@ -1,6 +1,8 @@ import assert from "node:assert/strict"; import { after, before, beforeEach, describe, it } from "node:test"; +import { chromium } from "playwright"; + import { assertFetchOk, assertFetchStatus, @@ -942,6 +944,165 @@ describe("POST /v1/sessions/:id/agentExecute - agentConfig.mode (V3)", () => { }); }); +// ============================================================================= +// V3 Variables Tests - executeOptions.variables substitution into agent tool calls +// ============================================================================= + +describe("POST /v1/sessions/:id/agentExecute (V3) - variables", () => { + const LOGIN_URL = + "https://browserbase.github.io/stagehand-eval-sites/sites/login/"; + const EMAIL_XPATH = "/html/body/main/form/div[1]/input"; + const PASSWORD_XPATH = "/html/body/main/form/div[2]/input"; + + let sessionId: string; + let cdpUrl: string; + const headers = getHeaders("3.0.0"); + + before(async () => { + ({ sessionId, cdpUrl } = await createSessionWithCdp(headers)); + }); + + beforeEach(async () => { + const navResponse = await navigateSession(sessionId, LOGIN_URL, headers); + assert.equal( + navResponse.status, + HTTP_OK, + "Navigate to login should succeed", + ); + }); + + after(async () => { + if (sessionId) { + await endSession(sessionId, headers); + sessionId = ""; + } + }); + + async function readInputValue(xpath: string): Promise { + const browser = await chromium.connectOverCDP(cdpUrl); + try { + const pages = browser.contexts()[0]!.pages(); + return await pages[0]!.locator(`xpath=${xpath}`).inputValue(); + } finally { + await browser.close(); + } + } + + it("should substitute simple variables into agent tool calls", async () => { + const url = getBaseUrl(); + const openaiApiKey = requireEnv("OPENAI_API_KEY", OPENAI_API_KEY); + + const ctx = await fetchWithContext<{ + success: boolean; + data?: { result: unknown; actionId?: string }; + }>(`${url}/v1/sessions/${sessionId}/agentExecute`, { + method: "POST", + headers: { + ...headers, + "x-model-api-key": "", + }, + body: JSON.stringify({ + agentConfig: { + model: { + modelName: "openai/gpt-4.1-nano", + apiKey: openaiApiKey, + }, + }, + executeOptions: { + instruction: + "Type %username% into the email field and %password% into the password field. Do not submit the form.", + maxSteps: 5, + variables: { + username: "john@example.com", + password: "secret123", + }, + }, + }), + }); + + assertFetchStatus( + ctx, + HTTP_OK, + "Agent execute with simple variables should succeed", + ); + assertFetchOk(ctx.body !== null, "Response body should be parseable", ctx); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + + const emailValue = await readInputValue(EMAIL_XPATH); + const passwordValue = await readInputValue(PASSWORD_XPATH); + assert.equal( + emailValue, + "john@example.com", + "Email field should contain substituted username variable value", + ); + assert.equal( + passwordValue, + "secret123", + "Password field should contain substituted password variable value", + ); + }); + + it("should substitute rich variables (with descriptions) into agent tool calls", async () => { + const url = getBaseUrl(); + const openaiApiKey = requireEnv("OPENAI_API_KEY", OPENAI_API_KEY); + + const ctx = await fetchWithContext<{ + success: boolean; + data?: { result: unknown; actionId?: string }; + }>(`${url}/v1/sessions/${sessionId}/agentExecute`, { + method: "POST", + headers: { + ...headers, + "x-model-api-key": "", + }, + body: JSON.stringify({ + agentConfig: { + model: { + modelName: "openai/gpt-4.1-nano", + apiKey: openaiApiKey, + }, + }, + executeOptions: { + instruction: + "Type %username% into the email field and %password% into the password field. Do not submit the form.", + maxSteps: 5, + variables: { + username: { + value: "alice@example.com", + description: "The login email", + }, + password: { + value: "rich-pw-456", + description: "The login password", + }, + }, + }, + }), + }); + + assertFetchStatus( + ctx, + HTTP_OK, + "Agent execute with rich variables should succeed", + ); + assertFetchOk(ctx.body !== null, "Response body should be parseable", ctx); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + + const emailValue = await readInputValue(EMAIL_XPATH); + const passwordValue = await readInputValue(PASSWORD_XPATH); + assert.equal( + emailValue, + "alice@example.com", + "Email field should contain substituted username variable value", + ); + assert.equal( + passwordValue, + "rich-pw-456", + "Password field should contain substituted password variable value", + ); + }); +}); + // ============================================================================= // SSE Streaming Tests (V3) // ============================================================================= diff --git a/packages/server-v3/tests/integration/v3/observe.test.ts b/packages/server-v3/tests/integration/v3/observe.test.ts index efc9930595..2f6b71549c 100644 --- a/packages/server-v3/tests/integration/v3/observe.test.ts +++ b/packages/server-v3/tests/integration/v3/observe.test.ts @@ -164,35 +164,59 @@ describe("POST /v1/sessions/:id/observe (V3)", () => { data?: { result: unknown[]; actionId?: string }; } - const ctx = await fetchWithContext( - `${url}/v1/sessions/${sessionId}/observe`, - { - method: "POST", - headers: getHeaders("3.0.0"), - body: JSON.stringify({ - instruction: "Find any link on the page", - options: { - variables: { - username: "john@example.com", - }, - }, - }), - }, + const navResponse = await navigateSession( + sessionId, + "https://browserbase.github.io/stagehand-eval-sites/sites/login/", + getHeaders("3.0.0"), ); + assert.equal( + navResponse.status, + HTTP_OK, + "Navigate to login should succeed", + ); + + try { + const ctx = await fetchWithContext( + `${url}/v1/sessions/${sessionId}/observe`, + { + method: "POST", + headers: getHeaders("3.0.0"), + body: JSON.stringify({ + instruction: "Find the field for entering %username%", + options: { + variables: { + username: "john@example.com", + password: "secret123", + }, + }, + }), + }, + ); - assertFetchStatus(ctx, HTTP_OK, "Observe with variables should succeed"); - assertFetchOk(ctx.body !== null, "Response body should be parseable", ctx); - assertFetchOk(ctx.body.success, "Response should indicate success", ctx); - assertFetchOk( - ctx.body.data !== undefined, - "Response should have data", - ctx, - ); - assertFetchOk( - Array.isArray(ctx.body.data.result), - "Result should be an array of observed elements", - ctx, - ); + assertFetchStatus(ctx, HTTP_OK, "Observe with variables should succeed"); + assertFetchOk( + ctx.body !== null, + "Response body should be parseable", + ctx, + ); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + assertFetchOk( + ctx.body.data !== undefined, + "Response should have data", + ctx, + ); + assertFetchOk( + Array.isArray(ctx.body.data.result), + "Result should be an array of observed elements", + ctx, + ); + } finally { + await navigateSession( + sessionId, + "https://example.com", + getHeaders("3.0.0"), + ); + } }); it("should observe with rich variables option", async () => { @@ -203,42 +227,69 @@ describe("POST /v1/sessions/:id/observe (V3)", () => { data?: { result: unknown[]; actionId?: string }; } - const ctx = await fetchWithContext( - `${url}/v1/sessions/${sessionId}/observe`, - { - method: "POST", - headers: getHeaders("3.0.0"), - body: JSON.stringify({ - instruction: "Find any link on the page", - options: { - variables: { - username: { - value: "john@example.com", - description: "The login email", + const navResponse = await navigateSession( + sessionId, + "https://browserbase.github.io/stagehand-eval-sites/sites/login/", + getHeaders("3.0.0"), + ); + assert.equal( + navResponse.status, + HTTP_OK, + "Navigate to login should succeed", + ); + + try { + const ctx = await fetchWithContext( + `${url}/v1/sessions/${sessionId}/observe`, + { + method: "POST", + headers: getHeaders("3.0.0"), + body: JSON.stringify({ + instruction: "Find the field for entering %username%", + options: { + variables: { + username: { + value: "john@example.com", + description: "The login email", + }, + password: { + value: "secret123", + description: "The login password", + }, }, }, - }, - }), - }, - ); + }), + }, + ); - assertFetchStatus( - ctx, - HTTP_OK, - "Observe with rich variables should succeed", - ); - assertFetchOk(ctx.body !== null, "Response body should be parseable", ctx); - assertFetchOk(ctx.body.success, "Response should indicate success", ctx); - assertFetchOk( - ctx.body.data !== undefined, - "Response should have data", - ctx, - ); - assertFetchOk( - Array.isArray(ctx.body.data.result), - "Result should be an array of observed elements", - ctx, - ); + assertFetchStatus( + ctx, + HTTP_OK, + "Observe with rich variables should succeed", + ); + assertFetchOk( + ctx.body !== null, + "Response body should be parseable", + ctx, + ); + assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + assertFetchOk( + ctx.body.data !== undefined, + "Response should have data", + ctx, + ); + assertFetchOk( + Array.isArray(ctx.body.data.result), + "Result should be an array of observed elements", + ctx, + ); + } finally { + await navigateSession( + sessionId, + "https://example.com", + getHeaders("3.0.0"), + ); + } }); it("should observe without instruction (observe all)", async () => { From 3d61e0d6fadf2669d71e800f53d8136edd8c3b47 Mon Sep 17 00:00:00 2001 From: tkattkat Date: Tue, 5 May 2026 10:25:03 -0700 Subject: [PATCH 4/5] update fillform tool --- .../core/lib/v3/agent/tools/fillFormVision.ts | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/core/lib/v3/agent/tools/fillFormVision.ts b/packages/core/lib/v3/agent/tools/fillFormVision.ts index 6776384253..34acf0f651 100644 --- a/packages/core/lib/v3/agent/tools/fillFormVision.ts +++ b/packages/core/lib/v3/agent/tools/fillFormVision.ts @@ -64,9 +64,12 @@ MANDATORY USE CASES (always use fillFormVision for these): try { const page = await v3.context.awaitActivePage(); - // Process coordinates and substitute variables for each field - // Keep original values (with %tokens%) for logging/caching, substituted values for typing - const processedFields = fields.map((field) => { + // Process coordinates per field. Keep the original `value` (with any + // `%variableName%` tokens) so substituted secrets never leak through + // the tool result, replay cache, returned AgentResult.actions, or + // anything that logs the action. Variables are substituted only at + // typing time below. + const safeFields = fields.map((field) => { const processed = processCoordinates( field.coordinates.x, field.coordinates.y, @@ -74,9 +77,8 @@ MANDATORY USE CASES (always use fillFormVision for these): v3, ); return { - ...field, - originalValue: field.value, // Keep original with %tokens% for cache - value: substituteVariables(field.value, variables), + action: field.action, + value: field.value, coordinates: { x: processed.x, y: processed.y }, }; }); @@ -97,7 +99,7 @@ MANDATORY USE CASES (always use fillFormVision for these): const shouldCollectXpath = v3.isAgentReplayActive(); const actions: Action[] = []; - for (const field of processedFields) { + for (const field of safeFields) { // Click the field, only requesting XPath when caching is enabled const xpath = await page.click( field.coordinates.x, @@ -106,10 +108,12 @@ MANDATORY USE CASES (always use fillFormVision for these): returnXpath: shouldCollectXpath, }, ); - await page.type(field.value); + // Substitute variables only at the moment of typing so the resolved + // value never leaves this scope. + await page.type(substituteVariables(field.value, variables)); - // Build Action with XPath for deterministic replay (only when caching) - // Use originalValue (with %tokens%) so cache stores references, not sensitive values + // Build Action with XPath for deterministic replay (only when caching). + // Store the placeholder value so cache entries don't contain secrets. if (shouldCollectXpath) { const normalizedXpath = ensureXPath(xpath); if (normalizedXpath) { @@ -117,7 +121,7 @@ MANDATORY USE CASES (always use fillFormVision for these): selector: normalizedXpath, description: field.action, method: "type", - arguments: [field.originalValue], + arguments: [field.value], }); } } @@ -140,7 +144,7 @@ MANDATORY USE CASES (always use fillFormVision for these): return { success: true, - playwrightArguments: processedFields, + playwrightArguments: safeFields, screenshotBase64, }; } catch (error) { From 6dcc82101118e51e6ff2ef0f9d423dccb02cfaac Mon Sep 17 00:00:00 2001 From: miguel Date: Tue, 5 May 2026 10:30:51 -0700 Subject: [PATCH 5/5] update model in act integration tests --- .../tests/integration/v3/act.test.ts | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/packages/server-v3/tests/integration/v3/act.test.ts b/packages/server-v3/tests/integration/v3/act.test.ts index 4e4f19ef75..3d4e7d94a0 100644 --- a/packages/server-v3/tests/integration/v3/act.test.ts +++ b/packages/server-v3/tests/integration/v3/act.test.ts @@ -394,15 +394,23 @@ describe("POST /v1/sessions/:id/act (V3) - variables", () => { it("should substitute simple variable into typed value", async () => { const url = getBaseUrl(); + const openaiApiKey = requireEnv("OPENAI_API_KEY", OPENAI_API_KEY); const ctx = await fetchWithContext( `${url}/v1/sessions/${varsSessionId}/act`, { method: "POST", - headers: getHeaders("3.0.0"), + headers: { + ...getHeaders("3.0.0"), + "x-model-api-key": "", + }, body: JSON.stringify({ input: "type %username% into the email field", options: { + model: { + modelName: "openai/gpt-4.1-mini", + apiKey: openaiApiKey, + }, variables: { username: "john@example.com", }, @@ -414,6 +422,11 @@ describe("POST /v1/sessions/:id/act (V3) - variables", () => { assertFetchStatus(ctx, HTTP_OK, "act with simple variable should succeed"); assertFetchOk(ctx.body !== null, "Response should have body", ctx); assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + assertFetchOk( + !!ctx.body.data?.result?.success, + `act result should be success, got: ${ctx.body.data?.result?.message ?? ""}`, + ctx, + ); const emailValue = await readInputValue(EMAIL_XPATH); assert.equal( @@ -425,15 +438,23 @@ describe("POST /v1/sessions/:id/act (V3) - variables", () => { it("should substitute rich variable (with description) into typed value", async () => { const url = getBaseUrl(); + const openaiApiKey = requireEnv("OPENAI_API_KEY", OPENAI_API_KEY); const ctx = await fetchWithContext( `${url}/v1/sessions/${varsSessionId}/act`, { method: "POST", - headers: getHeaders("3.0.0"), + headers: { + ...getHeaders("3.0.0"), + "x-model-api-key": "", + }, body: JSON.stringify({ input: "type %password% into the password field", options: { + model: { + modelName: "openai/gpt-4.1-mini", + apiKey: openaiApiKey, + }, variables: { password: { value: "secret123", @@ -448,6 +469,11 @@ describe("POST /v1/sessions/:id/act (V3) - variables", () => { assertFetchStatus(ctx, HTTP_OK, "act with rich variable should succeed"); assertFetchOk(ctx.body !== null, "Response should have body", ctx); assertFetchOk(ctx.body.success, "Response should indicate success", ctx); + assertFetchOk( + !!ctx.body.data?.result?.success, + `act result should be success, got: ${ctx.body.data?.result?.message ?? ""}`, + ctx, + ); const passwordValue = await readInputValue(PASSWORD_XPATH); assert.equal(