diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java index 871a91f6c..7305e5ab9 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java @@ -34,14 +34,26 @@ static void appendNode( if (windowMetadata != null) { appendWindowMetadata(xml, windowMetadata); } - appendNonEmptyAttribute(xml, "text", node.getText()); + CharSequence text = node.getText(); + if (text != null) { + appendAttribute(xml, "text", text); + } // getText() returns the HINT for an empty field on modern Android, so `text` alone cannot // distinguish a cleared field from one whose value equals its hint; only this flag can // (#2063 empty-fill verification). - appendTrueAttribute( - xml, - "hint-showing", - Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && node.isShowingHintText()); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + appendAttribute(xml, "hint-showing", Boolean.toString(node.isShowingHintText())); + } + appendAttribute(xml, "editable", Boolean.toString(node.isEditable())); + if (node.isEditable()) { + // These are accessibility offsets, not a measurement of the entered value's length. + int selectionStart = node.getTextSelectionStart(); + int selectionEnd = node.getTextSelectionEnd(); + if (selectionStart >= 0 && selectionEnd >= 0) { + appendAttribute(xml, "selection-start", Integer.toString(selectionStart)); + appendAttribute(xml, "selection-end", Integer.toString(selectionEnd)); + } + } appendNonEmptyAttribute(xml, "resource-id", node.getViewIdResourceName()); appendAttribute(xml, "class", node.getClassName()); appendNonEmptyAttribute(xml, "package", node.getPackageName()); @@ -66,7 +78,7 @@ static void appendNode( Boolean.toString( hasAccessibilityAction(node, AccessibilityAction.ACTION_SCROLL_BACKWARD))); } - appendTrueAttribute(xml, "password", node.isPassword()); + appendAttribute(xml, "password", Boolean.toString(node.isPassword())); appendAttribute( xml, "bounds", diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 4fc96bcfd..1a8a406e9 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -107,6 +107,13 @@ export type RawSnapshotNode = { enabled?: boolean; selected?: boolean; focused?: boolean; + /** Native accessibility facts; absent means unavailable, not false. */ + editable?: boolean; + password?: boolean; + hintShowing?: boolean; + /** Accessibility selection offsets, never a character count or proof of value equality. */ + selectionStart?: number; + selectionEnd?: number; visibleToUser?: boolean; hittable?: boolean; depth?: number; diff --git a/packages/platform-android/src/__tests__/ui-hierarchy-field-metadata.test.ts b/packages/platform-android/src/__tests__/ui-hierarchy-field-metadata.test.ts new file mode 100644 index 000000000..c2bb835c4 --- /dev/null +++ b/packages/platform-android/src/__tests__/ui-hierarchy-field-metadata.test.ts @@ -0,0 +1,35 @@ +import { expect, test } from 'vitest'; +import { buildUiHierarchySnapshot, parseUiHierarchyTree } from '../ui-hierarchy.ts'; + +test.each([false, true])( + 'editable field metadata survives snapshot presentation (raw=%s)', + (raw) => { + const tree = parseUiHierarchyTree(``); + const { nodes } = buildUiHierarchySnapshot(tree, undefined, { raw }); + expect(nodes.find((node) => node.identifier === 'field')).toMatchObject({ + value: '', + editable: true, + password: true, + hintShowing: false, + selectionStart: 0, + selectionEnd: 0, + }); + }, +); + +test('missing native field metadata remains unknown instead of becoming false or zero', () => { + const tree = parseUiHierarchyTree(``); + const { nodes } = buildUiHierarchySnapshot(tree, undefined, { raw: true }); + const field = nodes.find((node) => node.identifier === 'field'); + expect(field).toBeDefined(); + expect(field?.value).toBeUndefined(); + expect(field?.editable).toBeUndefined(); + expect(field?.password).toBeUndefined(); + expect(field?.hintShowing).toBeUndefined(); + expect(field?.selectionStart).toBeUndefined(); + expect(field?.selectionEnd).toBeUndefined(); +}); diff --git a/packages/platform-android/src/ui-hierarchy-builder.ts b/packages/platform-android/src/ui-hierarchy-builder.ts index e1851c4de..3c0b95639 100644 --- a/packages/platform-android/src/ui-hierarchy-builder.ts +++ b/packages/platform-android/src/ui-hierarchy-builder.ts @@ -352,6 +352,11 @@ function createAndroidRawSnapshotNode( rect: node.rect, enabled: node.enabled, focused: node.focused, + editable: node.editable, + password: node.password, + hintShowing: node.hintShowing, + selectionStart: node.selectionStart, + selectionEnd: node.selectionEnd, visibleToUser: node.visibleToUser, hittable: isAgentTarget(node) || undefined, depth: compactedAndroidNodeDepth(state.nodes, parentIndex), diff --git a/packages/platform-android/src/ui-hierarchy-node.ts b/packages/platform-android/src/ui-hierarchy-node.ts index c63b3adb2..db4d438af 100644 --- a/packages/platform-android/src/ui-hierarchy-node.ts +++ b/packages/platform-android/src/ui-hierarchy-node.ts @@ -14,6 +14,11 @@ export type AndroidUiHierarchy = { enabled?: boolean; visibleToUser?: boolean; focused?: boolean; + editable?: boolean; + password?: boolean; + hintShowing?: boolean; + selectionStart?: number; + selectionEnd?: number; // Two independent facts, never collapsed, and never undefined: the helper omits false attributes // while stock UiAutomator writes them out, so reading an absent attribute as a value gave two // encodings of one control opposite answers. diff --git a/packages/platform-android/src/ui-hierarchy.ts b/packages/platform-android/src/ui-hierarchy.ts index 123247c8e..e339b3c35 100644 --- a/packages/platform-android/src/ui-hierarchy.ts +++ b/packages/platform-android/src/ui-hierarchy.ts @@ -35,6 +35,9 @@ export type AndroidUiNodeMetadata = { focusable?: boolean; focused?: boolean; password?: boolean; + editable?: boolean; + selectionStart?: number; + selectionEnd?: number; /** * Helper-only: the `text` attribute is the field's HINT, not its value (an empty input's * `getText()` returns the hint on modern Android). Absent in raw uiautomator dumps. @@ -147,6 +150,9 @@ function readNodeAttributes(node: string): Omit { focusable: boolAttr('focusable'), focused: boolAttr('focused'), password: boolAttr('password'), + ...optionalBoolAttr('editable', 'editable'), + ...optionalNumberAttr('selectionStart', 'selection-start'), + ...optionalNumberAttr('selectionEnd', 'selection-end'), ...optionalBoolAttr('hintShowing', 'hint-showing'), ...optionalBoolAttr('visibleToUser', 'visible-to-user'), ...optionalNumberAttr('drawingOrder', 'drawing-order'), @@ -300,6 +306,11 @@ function normalizeAndroidUiHierarchyNode( rect: attrs.rect, enabled: attrs.enabled, focused: attrs.focused, + editable: attrs.editable, + password: attrs.password, + hintShowing: attrs.hintShowing, + selectionStart: attrs.selectionStart, + selectionEnd: attrs.selectionEnd, visibleToUser: attrs.visibleToUser, clickable: attrs.clickable === true, focusable: attrs.focusable === true, diff --git a/src/daemon/session-lifecycle/internal/__tests__/session-open-runtime.test.ts b/src/daemon/session-lifecycle/internal/__tests__/session-open-runtime.test.ts index 10fb6e5d6..0b23ba31d 100644 --- a/src/daemon/session-lifecycle/internal/__tests__/session-open-runtime.test.ts +++ b/src/daemon/session-lifecycle/internal/__tests__/session-open-runtime.test.ts @@ -15,6 +15,19 @@ vi.mock('../../../../core/dispatch-resolve.ts', async (importOriginal) => { }; }); vi.mock('../../../device-ready.ts', () => ({ ensureDeviceReady: vi.fn(async () => {}) })); +vi.mock('../../../../platform-runtime-apple-application-tools.ts', async (importOriginal) => { + const actual = + await importOriginal< + typeof import('../../../../platform-runtime-apple-application-tools.ts') + >(); + return { + ...actual, + createAppleApplicationTools: () => ({ + ...actual.createAppleApplicationTools(), + prewarmRunnerSession: vi.fn(async () => {}), + }), + }; +}); vi.mock('../../../../platform-runtime-runtime-hints.ts', async (importOriginal) => { const actual = await importOriginal(); diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 79973f152..c6b01627e 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -46,6 +46,14 @@ agent-device snapshot --diff # Alias for the same diff operation ## Efficient snapshot usage +Android structured snapshot nodes and `get attrs` expose native `editable`, `password`, +`hintShowing`, `selectionStart`, and `selectionEnd` facts when available. Missing fields mean +unknown; `hintShowing` requires Android API 26 or later. Selection values are accessibility +offsets, not character counts, and cannot verify a secure value or its equality to expected text. +An explicitly empty accessibility text remains `value: ""`; missing text remains unavailable. +These facts describe the accessibility observation, which may contain a hint or masked text, +rather than privileged access to an application's backing value. + - iOS and Android share the same mobile snapshot contract: visible-first output, actionable-now refs, and hidden list content communicated via discovery hints. - Default to `snapshot -i` for agent loops. - Default snapshot text is an agent-facing, token-efficient view for planning and targeting actions. It is visible-first and may collapse helper/accessibility noise; use `--raw` or `--json` when you need the full provider tree.