fix(web): preserve the backend ref so snapshot refs match actionable refs

packages/kernel/src/snapshot-attach-refs.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, test } from 'vitest';
+import { attachRefs, type RawSnapshotNode } from './snapshot.ts';
+
+const node = (index: number, extra: Partial<RawSnapshotNode> = {}): RawSnapshotNode => ({
+  index,
+  ...extra,
+});
+
+describe('attachRefs', () => {
+  test('preserves a backend-minted ref instead of re-minting a dense positional one', () => {
+    // The web/agent-browser backend mints refs in tree order (non-dense — non-interactive
+    // nodes are skipped). The first node's backend ref is e2 because e1 went to a node
+    // that did not appear in the projected tree. Preserving it keeps the ref the agent
+    // reads off the snapshot identical to the ref the backend resolves on the next action.
+    const attached = attachRefs([node(0, { ref: 'e2' }), node(1, { ref: 'e3' })]);
+    expect(attached.map((n) => n.ref)).toEqual(['e2', 'e3']);
+    // index is preserved untouched — the positional identity is orthogonal to the ref.
+    expect(attached.map((n) => n.index)).toEqual([0, 1]);
+  });
+
+  test('re-mints dense positional refs for nodes without a backend ref', () => {
+    const attached = attachRefs([node(0), node(1, { ref: 'e7' }), node(2)]);
+    expect(attached.map((n) => n.ref)).toEqual(['e1', 'e7', 'e3']);
+  });
+});

packages/kernel/src/snapshot.ts

@@ -120,6 +120,15 @@ export type RawSnapshotNode = {
   hiddenContentBelow?: boolean;
   interactionBlocked?: 'covered';
   presentationHints?: string[];
+  /**
+   * Backend-minted ref for this node, when the capture backend already assigns a
+   * stable, actionable ref (e.g. the web/agent-browser backend resolves actions
+   * against its own `@eN` refs). `attachRefs` preserves this instead of re-minting
+   * a dense positional ref, so the ref an agent sees in the snapshot is the same
+   * ref the backend can resolve on the next action. Absent for backends that do
+   * not mint refs — those fall back to dense `e${index}` numbering.
+   */
+  ref?: string;
   /**
    * Accessibility custom actions the element exposes (iOS
    * `UIAccessibilityCustomAction`, React Native `accessibilityActions`). Merged
@@ -273,8 +282,15 @@ export type ScreenshotOverlayRef = {
   center: Point;
 };
 
+/**
+ * Assign a display ref to every node. A node that already carries a backend-minted
+ * `ref` keeps it (see `RawSnapshotNode.ref`) — the web/agent-browser backend resolves
+ * actions against its own refs, so re-minting a dense positional ref here would make
+ * the snapshot show one ref while actions act on a different element. Backends that do
+ * not mint refs get dense `e${index}` numbering, matching the historical behavior.
+ */
 export function attachRefs(nodes: RawSnapshotNode[]): SnapshotNode[] {
-  return nodes.map((node, idx) => ({ ...node, ref: `e${idx + 1}` }));
+  return nodes.map((node, idx) => ({ ...node, ref: node.ref ?? `e${idx + 1}` }));
 }
 
 /**

packages/platform-web/src/__tests__/agent-browser-snapshot.test.ts

@@ -59,4 +59,34 @@ describe('normalizeAgentBrowserSnapshot', () => {
 
     expect(result.nodes[0]?.enabled).toBe(true);
   });
+
+  test('preserves the backend ref so the displayed ref is the actionable one', async () => {
+    // agent-browser mints refs in tree order and skips non-interactive nodes
+    // (the leading `generic` container never gets a ref), so the refs are NOT dense:
+    //   e1 = generic  (skipped here, no ref on it)
+    //   e2 = username textbox
+    //   e3 = passcode textbox   <-- agent presses @e3 expecting username
+    //   e4 = sign-in button
+    // The username textbox is the 2nd NODE in the snapshot but its backend ref is e2.
+    // If agent-device re-mints dense positional refs, the snapshot would show the
+    // username as @e2 while the passcode (node 3) shows @e3 — and an agent that
+    // reads "@e3 = passcode" would land in the passcode field when it meant the
+    // username, because the backend's own e3 resolves to a different element.
+    // Preserving the backend ref keeps display ref == actionable ref.
+    const result = await normalizeAgentBrowserSnapshot({
+      snapshot: [
+        '- textbox "Username" [ref=e2]',
+        '- textbox "Passcode" [ref=e3]',
+        '- button "Sign in" [ref=e4]',
+      ].join('\n'),
+      refs: {
+        e2: { role: 'textbox', name: 'Username' },
+        e3: { role: 'textbox', name: 'Passcode' },
+        e4: { role: 'button', name: 'Sign in' },
+      },
+    });
+
+    expect(result.nodes.map((node) => node.label)).toEqual(['Username', 'Passcode', 'Sign in']);
+    expect(result.nodes.map((node) => node.ref)).toEqual(['e2', 'e3', 'e4']);
+  });
 });

packages/platform-web/src/agent-browser-provider.test.ts

@@ -666,6 +666,9 @@ function expectedNode(
 ) {
   return {
     index,
+    // The web normalizer now preserves the backend ref on each node. This fixture's
+    // refs happen to be dense in tree order, so the ref matches the positional `e{index}`.
+    ref: `e${index + 1}`,
     type,
     role: type,
     label,

packages/platform-web/src/agent-browser-snapshot.ts

@@ -31,7 +31,13 @@ export async function normalizeAgentBrowserSnapshot(
   if (fetchBox) await attachDraftRects(drafts, fetchBox);
 
   return {
-    nodes: drafts.map((draft, index) => ({ ...draft.node, index })),
+    // Preserve each draft's backend ref as the node's `ref`. agent-browser resolves
+    // actions (click/fill/hover) against its own `@eN` refs, which are minted in tree
+    // order and are NOT dense — so the ref an agent reads off the snapshot must be the
+    // backend ref, not a re-minted dense positional ref. `attachRefs` (downstream) keeps
+    // it. Without this the snapshot could show `@e3` = username while the action on
+    // `@e3` lands on the backend's `@e3` (a different element, e.g. the passcode field).
+    nodes: drafts.map((draft, index) => ({ ...draft.node, index, ref: draft.ref })),
     truncated: readBooleanProperty(data, 'truncated'),
   };
 }