From f848b6f5c04cafabfdc96fe356540892346d97df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 4 Sep 2026 23:03:03 +0200 Subject: [PATCH] fix(web): preserve the backend ref so snapshot refs match actionable refs The web/agent-browser backend mints refs in tree order and skips non-interactive nodes, so its `@eN` refs are not dense. agent-device was dropping that ref in `normalizeAgentBrowserSnapshot` and then re-minting a dense positional `e${index+1}` in `attachRefs`. The ref an agent reads off the snapshot (dense, positional) therefore did not equal the ref the backend resolves on the next action (tree-ordered). On the ShopDemo login screen the username textbox displayed as one ref while the backend's ref for the same position pointed at the passcode field, so `fill @e3` landed in the wrong input. Preserve the backend ref on each web node and make `attachRefs` keep a node's existing `ref` when present, falling back to dense numbering for backends that do not mint refs (iOS/Android/maestro are unaffected). --- .../kernel/src/snapshot-attach-refs.test.ts | 25 ++++++++++++++++ packages/kernel/src/snapshot.ts | 18 ++++++++++- .../__tests__/agent-browser-snapshot.test.ts | 30 +++++++++++++++++++ .../src/agent-browser-provider.test.ts | 3 ++ .../src/agent-browser-snapshot.ts | 8 ++++- 5 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 packages/kernel/src/snapshot-attach-refs.test.ts diff --git a/packages/kernel/src/snapshot-attach-refs.test.ts b/packages/kernel/src/snapshot-attach-refs.test.ts new file mode 100644 index 0000000000..b92bb8e54a --- /dev/null +++ b/packages/kernel/src/snapshot-attach-refs.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, test } from 'vitest'; +import { attachRefs, type RawSnapshotNode } from './snapshot.ts'; + +const node = (index: number, extra: Partial = {}): RawSnapshotNode => ({ + index, + ...extra, +}); + +describe('attachRefs', () => { + test('preserves a backend-minted ref instead of re-minting a dense positional one', () => { + // The web/agent-browser backend mints refs in tree order (non-dense — non-interactive + // nodes are skipped). The first node's backend ref is e2 because e1 went to a node + // that did not appear in the projected tree. Preserving it keeps the ref the agent + // reads off the snapshot identical to the ref the backend resolves on the next action. + const attached = attachRefs([node(0, { ref: 'e2' }), node(1, { ref: 'e3' })]); + expect(attached.map((n) => n.ref)).toEqual(['e2', 'e3']); + // index is preserved untouched — the positional identity is orthogonal to the ref. + expect(attached.map((n) => n.index)).toEqual([0, 1]); + }); + + test('re-mints dense positional refs for nodes without a backend ref', () => { + const attached = attachRefs([node(0), node(1, { ref: 'e7' }), node(2)]); + expect(attached.map((n) => n.ref)).toEqual(['e1', 'e7', 'e3']); + }); +}); diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 4fc96bcfdb..06e0e5a719 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -120,6 +120,15 @@ export type RawSnapshotNode = { hiddenContentBelow?: boolean; interactionBlocked?: 'covered'; presentationHints?: string[]; + /** + * Backend-minted ref for this node, when the capture backend already assigns a + * stable, actionable ref (e.g. the web/agent-browser backend resolves actions + * against its own `@eN` refs). `attachRefs` preserves this instead of re-minting + * a dense positional ref, so the ref an agent sees in the snapshot is the same + * ref the backend can resolve on the next action. Absent for backends that do + * not mint refs — those fall back to dense `e${index}` numbering. + */ + ref?: string; /** * Accessibility custom actions the element exposes (iOS * `UIAccessibilityCustomAction`, React Native `accessibilityActions`). Merged @@ -273,8 +282,15 @@ export type ScreenshotOverlayRef = { center: Point; }; +/** + * Assign a display ref to every node. A node that already carries a backend-minted + * `ref` keeps it (see `RawSnapshotNode.ref`) — the web/agent-browser backend resolves + * actions against its own refs, so re-minting a dense positional ref here would make + * the snapshot show one ref while actions act on a different element. Backends that do + * not mint refs get dense `e${index}` numbering, matching the historical behavior. + */ export function attachRefs(nodes: RawSnapshotNode[]): SnapshotNode[] { - return nodes.map((node, idx) => ({ ...node, ref: `e${idx + 1}` })); + return nodes.map((node, idx) => ({ ...node, ref: node.ref ?? `e${idx + 1}` })); } /** diff --git a/packages/platform-web/src/__tests__/agent-browser-snapshot.test.ts b/packages/platform-web/src/__tests__/agent-browser-snapshot.test.ts index 8664e487dc..c946616629 100644 --- a/packages/platform-web/src/__tests__/agent-browser-snapshot.test.ts +++ b/packages/platform-web/src/__tests__/agent-browser-snapshot.test.ts @@ -59,4 +59,34 @@ describe('normalizeAgentBrowserSnapshot', () => { expect(result.nodes[0]?.enabled).toBe(true); }); + + test('preserves the backend ref so the displayed ref is the actionable one', async () => { + // agent-browser mints refs in tree order and skips non-interactive nodes + // (the leading `generic` container never gets a ref), so the refs are NOT dense: + // e1 = generic (skipped here, no ref on it) + // e2 = username textbox + // e3 = passcode textbox <-- agent presses @e3 expecting username + // e4 = sign-in button + // The username textbox is the 2nd NODE in the snapshot but its backend ref is e2. + // If agent-device re-mints dense positional refs, the snapshot would show the + // username as @e2 while the passcode (node 3) shows @e3 — and an agent that + // reads "@e3 = passcode" would land in the passcode field when it meant the + // username, because the backend's own e3 resolves to a different element. + // Preserving the backend ref keeps display ref == actionable ref. + const result = await normalizeAgentBrowserSnapshot({ + snapshot: [ + '- textbox "Username" [ref=e2]', + '- textbox "Passcode" [ref=e3]', + '- button "Sign in" [ref=e4]', + ].join('\n'), + refs: { + e2: { role: 'textbox', name: 'Username' }, + e3: { role: 'textbox', name: 'Passcode' }, + e4: { role: 'button', name: 'Sign in' }, + }, + }); + + expect(result.nodes.map((node) => node.label)).toEqual(['Username', 'Passcode', 'Sign in']); + expect(result.nodes.map((node) => node.ref)).toEqual(['e2', 'e3', 'e4']); + }); }); diff --git a/packages/platform-web/src/agent-browser-provider.test.ts b/packages/platform-web/src/agent-browser-provider.test.ts index 43635685f5..d2629b2a14 100644 --- a/packages/platform-web/src/agent-browser-provider.test.ts +++ b/packages/platform-web/src/agent-browser-provider.test.ts @@ -666,6 +666,9 @@ function expectedNode( ) { return { index, + // The web normalizer now preserves the backend ref on each node. This fixture's + // refs happen to be dense in tree order, so the ref matches the positional `e{index}`. + ref: `e${index + 1}`, type, role: type, label, diff --git a/packages/platform-web/src/agent-browser-snapshot.ts b/packages/platform-web/src/agent-browser-snapshot.ts index 4f0211eab4..9994570ba6 100644 --- a/packages/platform-web/src/agent-browser-snapshot.ts +++ b/packages/platform-web/src/agent-browser-snapshot.ts @@ -31,7 +31,13 @@ export async function normalizeAgentBrowserSnapshot( if (fetchBox) await attachDraftRects(drafts, fetchBox); return { - nodes: drafts.map((draft, index) => ({ ...draft.node, index })), + // Preserve each draft's backend ref as the node's `ref`. agent-browser resolves + // actions (click/fill/hover) against its own `@eN` refs, which are minted in tree + // order and are NOT dense — so the ref an agent reads off the snapshot must be the + // backend ref, not a re-minted dense positional ref. `attachRefs` (downstream) keeps + // it. Without this the snapshot could show `@e3` = username while the action on + // `@e3` lands on the backend's `@e3` (a different element, e.g. the passcode field). + nodes: drafts.map((draft, index) => ({ ...draft.node, index, ref: draft.ref })), truncated: readBooleanProperty(data, 'truncated'), }; }