Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,26 @@ static void appendNode(
if (windowMetadata != null) {
appendWindowMetadata(xml, windowMetadata);
}
appendNonEmptyAttribute(xml, "text", node.getText());
CharSequence text = node.getText();
if (text != null) {
appendAttribute(xml, "text", text);
}
// getText() returns the HINT for an empty field on modern Android, so `text` alone cannot
// distinguish a cleared field from one whose value equals its hint; only this flag can
// (#2063 empty-fill verification).
appendTrueAttribute(
xml,
"hint-showing",
Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && node.isShowingHintText());
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
appendAttribute(xml, "hint-showing", Boolean.toString(node.isShowingHintText()));
}
appendAttribute(xml, "editable", Boolean.toString(node.isEditable()));
if (node.isEditable()) {
// These are accessibility offsets, not a measurement of the entered value's length.
int selectionStart = node.getTextSelectionStart();
int selectionEnd = node.getTextSelectionEnd();
if (selectionStart >= 0 && selectionEnd >= 0) {
appendAttribute(xml, "selection-start", Integer.toString(selectionStart));
appendAttribute(xml, "selection-end", Integer.toString(selectionEnd));
}
}
appendNonEmptyAttribute(xml, "resource-id", node.getViewIdResourceName());
appendAttribute(xml, "class", node.getClassName());
appendNonEmptyAttribute(xml, "package", node.getPackageName());
Expand All @@ -66,7 +78,7 @@ static void appendNode(
Boolean.toString(
hasAccessibilityAction(node, AccessibilityAction.ACTION_SCROLL_BACKWARD)));
}
appendTrueAttribute(xml, "password", node.isPassword());
appendAttribute(xml, "password", Boolean.toString(node.isPassword()));
appendAttribute(
xml,
"bounds",
Expand Down
7 changes: 7 additions & 0 deletions packages/kernel/src/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,13 @@ export type RawSnapshotNode = {
enabled?: boolean;
selected?: boolean;
focused?: boolean;
/** Native accessibility facts; absent means unavailable, not false. */
editable?: boolean;
password?: boolean;
hintShowing?: boolean;
/** Accessibility selection offsets, never a character count or proof of value equality. */
selectionStart?: number;
selectionEnd?: number;
visibleToUser?: boolean;
hittable?: boolean;
depth?: number;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import { expect, test } from 'vitest';
import { buildUiHierarchySnapshot, parseUiHierarchyTree } from '../ui-hierarchy.ts';

test.each([false, true])(
'editable field metadata survives snapshot presentation (raw=%s)',
(raw) => {
const tree = parseUiHierarchyTree(`<hierarchy><node class="android.widget.EditText"
resource-id="field" text="" bounds="[0,0][200,100]" visible-to-user="true"
editable="true" password="true" hint-showing="false" selection-start="0" selection-end="0"
focusable="true" focused="true" /></hierarchy>`);
const { nodes } = buildUiHierarchySnapshot(tree, undefined, { raw });
expect(nodes.find((node) => node.identifier === 'field')).toMatchObject({
value: '',
editable: true,
password: true,
hintShowing: false,
selectionStart: 0,
selectionEnd: 0,
});
},
);

test('missing native field metadata remains unknown instead of becoming false or zero', () => {
const tree = parseUiHierarchyTree(`<hierarchy><node class="android.widget.EditText"
resource-id="field" bounds="[0,0][200,100]" focusable="true" /></hierarchy>`);
const { nodes } = buildUiHierarchySnapshot(tree, undefined, { raw: true });
const field = nodes.find((node) => node.identifier === 'field');
expect(field).toBeDefined();
expect(field?.value).toBeUndefined();
expect(field?.editable).toBeUndefined();
expect(field?.password).toBeUndefined();
expect(field?.hintShowing).toBeUndefined();
expect(field?.selectionStart).toBeUndefined();
expect(field?.selectionEnd).toBeUndefined();
});
5 changes: 5 additions & 0 deletions packages/platform-android/src/ui-hierarchy-builder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,11 @@ function createAndroidRawSnapshotNode(
rect: node.rect,
enabled: node.enabled,
focused: node.focused,
editable: node.editable,
password: node.password,
hintShowing: node.hintShowing,
selectionStart: node.selectionStart,
selectionEnd: node.selectionEnd,
visibleToUser: node.visibleToUser,
hittable: isAgentTarget(node) || undefined,
depth: compactedAndroidNodeDepth(state.nodes, parentIndex),
Expand Down
5 changes: 5 additions & 0 deletions packages/platform-android/src/ui-hierarchy-node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ export type AndroidUiHierarchy = {
enabled?: boolean;
visibleToUser?: boolean;
focused?: boolean;
editable?: boolean;
password?: boolean;
hintShowing?: boolean;
selectionStart?: number;
selectionEnd?: number;
// Two independent facts, never collapsed, and never undefined: the helper omits false attributes
// while stock UiAutomator writes them out, so reading an absent attribute as a value gave two
// encodings of one control opposite answers.
Expand Down
11 changes: 11 additions & 0 deletions packages/platform-android/src/ui-hierarchy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ export type AndroidUiNodeMetadata = {
focusable?: boolean;
focused?: boolean;
password?: boolean;
editable?: boolean;
selectionStart?: number;
selectionEnd?: number;
/**
* Helper-only: the `text` attribute is the field's HINT, not its value (an empty input's
* `getText()` returns the hint on modern Android). Absent in raw uiautomator dumps.
Expand Down Expand Up @@ -147,6 +150,9 @@ function readNodeAttributes(node: string): Omit<AndroidUiNodeMetadata, 'rect'> {
focusable: boolAttr('focusable'),
focused: boolAttr('focused'),
password: boolAttr('password'),
...optionalBoolAttr('editable', 'editable'),
...optionalNumberAttr('selectionStart', 'selection-start'),
...optionalNumberAttr('selectionEnd', 'selection-end'),
...optionalBoolAttr('hintShowing', 'hint-showing'),
...optionalBoolAttr('visibleToUser', 'visible-to-user'),
...optionalNumberAttr('drawingOrder', 'drawing-order'),
Expand Down Expand Up @@ -300,6 +306,11 @@ function normalizeAndroidUiHierarchyNode(
rect: attrs.rect,
enabled: attrs.enabled,
focused: attrs.focused,
editable: attrs.editable,
password: attrs.password,
hintShowing: attrs.hintShowing,
selectionStart: attrs.selectionStart,
selectionEnd: attrs.selectionEnd,
visibleToUser: attrs.visibleToUser,
clickable: attrs.clickable === true,
focusable: attrs.focusable === true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,19 @@ vi.mock('../../../../core/dispatch-resolve.ts', async (importOriginal) => {
};
});
vi.mock('../../../device-ready.ts', () => ({ ensureDeviceReady: vi.fn(async () => {}) }));
vi.mock('../../../../platform-runtime-apple-application-tools.ts', async (importOriginal) => {
const actual =
await importOriginal<
typeof import('../../../../platform-runtime-apple-application-tools.ts')
>();
return {
...actual,
createAppleApplicationTools: () => ({
...actual.createAppleApplicationTools(),
prewarmRunnerSession: vi.fn(async () => {}),
}),
};
});
vi.mock('../../../../platform-runtime-runtime-hints.ts', async (importOriginal) => {
const actual =
await importOriginal<typeof import('../../../../platform-runtime-runtime-hints.ts')>();
Expand Down
8 changes: 8 additions & 0 deletions website/docs/docs/snapshots.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,14 @@ agent-device snapshot --diff # Alias for the same diff operation

## Efficient snapshot usage

Android structured snapshot nodes and `get attrs` expose native `editable`, `password`,
`hintShowing`, `selectionStart`, and `selectionEnd` facts when available. Missing fields mean
unknown; `hintShowing` requires Android API 26 or later. Selection values are accessibility
offsets, not character counts, and cannot verify a secure value or its equality to expected text.
An explicitly empty accessibility text remains `value: ""`; missing text remains unavailable.
These facts describe the accessibility observation, which may contain a hint or masked text,
rather than privileged access to an application's backing value.

- iOS and Android share the same mobile snapshot contract: visible-first output, actionable-now refs, and hidden list content communicated via discovery hints.
- Default to `snapshot -i` for agent loops.
- Default snapshot text is an agent-facing, token-efficient view for planning and targeting actions. It is visible-first and may collapse helper/accessibility noise; use `--raw` or `--json` when you need the full provider tree.
Expand Down