From 8da9a0ebced65ca3cf2c0fd160c4b3bafeab3f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 10:42:10 +0200 Subject: [PATCH 1/9] perf(android): reuse automation helper for input and screenshots --- android/snapshot-helper/README.md | 6 + .../AccessibilityNodeReads.java | 5 + .../AccessibilityTreeCapture.java | 2 +- .../AutomationCommandHandler.java | 203 ++++++++++++++++++ .../SnapshotInstrumentation.java | 11 +- ...002-persistent-platform-helper-sessions.md | 11 +- ...utomation-performance-2026-10-19-base.json | 68 ++++++ ...omation-performance-2026-10-19-change.json | 67 ++++++ ...droid-automation-performance-2026-10-19.md | 65 ++++++ .../src/element-text-runtime.test.ts | 25 +++ .../contracts/src/element-text-runtime.ts | 17 +- packages/contracts/src/interactor-types.ts | 8 +- packages/contracts/src/screenshot.ts | 15 +- .../__tests__/selector-text-runtime.test.ts | 21 ++ .../execution/android-foreground-surface.ts | 1 + ...android-escape-dispatch-disclosure.test.ts | 1 + .../src/execution/selector-text-runtime.ts | 10 +- .../src/__tests__/fill-verification.test.ts | 38 ++++ .../src/__tests__/screenshot.test.ts | 18 ++ .../snapshot-helper-session-protocol.test.ts | 30 +++ .../snapshot-helper-session.fixtures.ts | 12 ++ .../src/__tests__/text-input-fill.test.ts | 24 +++ .../__tests__/touch-helper-session.test.ts | 146 +++++++++++++ .../src/automation-helper-artifact.ts | 102 +++++++++ .../src/automation-helper.test.ts | 22 ++ .../platform-android/src/automation-helper.ts | 48 +++++ .../platform-android/src/fill-verification.ts | 81 +++++-- .../platform-android/src/input-actions.ts | 6 + packages/platform-android/src/observation.ts | 7 + packages/platform-android/src/screenshot.ts | 22 +- .../src/snapshot-helper-capture.ts | 16 +- .../src/snapshot-helper-session-lifecycle.ts | 4 +- .../src/snapshot-helper-session-protocol.ts | 13 +- .../src/snapshot-helper-session.ts | 12 +- .../src/snapshot-helper-types.ts | 1 + packages/platform-android/src/snapshot.ts | 4 + packages/platform-android/src/text-input.ts | 11 +- packages/platform-android/src/touch-helper.ts | 110 +--------- scripts/bench-android-automation.mjs | 94 ++++++++ .../capture/screenshot-options.test.ts | 14 ++ src/commands/capture/screenshot.ts | 2 +- src/core/interactors/android.ts | 2 +- website/docs/docs/commands.md | 3 +- 43 files changed, 1208 insertions(+), 170 deletions(-) create mode 100644 android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java create mode 100644 docs/evidence/android-automation-performance-2026-10-19-base.json create mode 100644 docs/evidence/android-automation-performance-2026-10-19-change.json create mode 100644 docs/evidence/android-automation-performance-2026-10-19.md create mode 100644 packages/platform-android/src/automation-helper-artifact.ts create mode 100644 packages/platform-android/src/automation-helper.test.ts create mode 100644 packages/platform-android/src/automation-helper.ts create mode 100644 scripts/bench-android-automation.mjs diff --git a/android/snapshot-helper/README.md b/android/snapshot-helper/README.md index 05f561166d..bc5855f273 100644 --- a/android/snapshot-helper/README.md +++ b/android/snapshot-helper/README.md @@ -95,11 +95,17 @@ connection: the client connects, sends a single command line, reads the response closes that connection; the process stays alive for the next connection: - `snapshot ` — capture and return an XML snapshot, same semantics as the default mode +- `snapshot-immediate ` — capture fresh XML without an idle wait when the caller owns settling +- `input ` — inject a tap, key, or point-targeted text replacement +- `focus ` — read the focused window type and root package after clearing the accessibility cache +- `screenshot ` — return a PNG in the `payloadBase64` header - `viewport ` — return interactive-window viewport bounds - `gesture ` — inject a planned touch gesture (same payload as the one-shot `gesture` mode) - `quit ` — acknowledge and stop the session +The ready record advertises optional operations in `sessionCommands`. A host must check support before sending them. Text replacement retains a focus tap; the host owns post-write verification. Input failures must not be resent because a mutation may already have reached the app. + `viewport` and `gesture` responses are headers-only (no body): `kind`, `injectedEvents`, `elapsedMs` for `gesture`, and `x`, `y`, `width`, `height` for `viewport`. `snapshot` responses carry the XML body after the header block, as described below. The response protocol literal is always diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityNodeReads.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityNodeReads.java index 432651cfae..bfbd928f49 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityNodeReads.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityNodeReads.java @@ -18,6 +18,11 @@ final class AccessibilityNodeReads { private AccessibilityNodeReads() {} + static AccessibilityNodeInfo rootWithoutPrefetch(AccessibilityWindowInfo window) { + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU + ? window.getRoot(0) : window.getRoot(); + } + static AccessibilityNodeInfo root(AccessibilityWindowInfo window) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { return window.getRoot(PREFETCH_STRATEGY); diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java index f56e47767f..d9830dfb8b 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java @@ -90,7 +90,7 @@ private static Result captureOnce(UiAutomation automation, int maxDepth, int max missingRootWindowTypes); } - private static void clearAccessibilityCache(UiAutomation automation) { + static void clearAccessibilityCache(UiAutomation automation) { // Navigation 3 and any in-place content swap that keeps the same window/Activity render every // destination inside one AndroidComposeView. A persistent helper session reuses a single // UiAutomation connection across captures, and its per-connection accessibility node cache is diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java new file mode 100644 index 0000000000..63ec7c71e0 --- /dev/null +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java @@ -0,0 +1,203 @@ +package com.callstack.agentdevice.snapshothelper; + +import android.app.UiAutomation; +import android.graphics.Bitmap; +import android.graphics.Rect; +import android.os.Bundle; +import android.os.SystemClock; +import android.util.Base64; +import android.view.InputDevice; +import android.view.KeyCharacterMap; +import android.view.KeyEvent; +import android.view.MotionEvent; +import android.view.accessibility.AccessibilityNodeInfo; +import android.view.accessibility.AccessibilityWindowInfo; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import org.json.JSONObject; + +/** Executes session input and image capture inside the instrumentation's automation context. */ +final class AutomationCommandHandler { + private AutomationCommandHandler() {} + + static void writeSessionResponse( + OutputStream output, String requestId, String action, String payload, UiAutomation automation) + throws IOException { + Bundle result = SessionResponseWriter.sessionResponseBundle(requestId); + try { + if ("screenshot".equals(action)) { + screenshot(result, automation); + } else if ("focus".equals(action)) { + focus(result, automation); + } else { + input(automation, new JSONObject(new String(Base64.decode(payload, Base64.DEFAULT), StandardCharsets.UTF_8))); + } + result.putString("ok", "true"); + SessionResponseWriter.writeSessionResponse(output, result, ""); + } catch (Throwable error) { + SessionResponseWriter.writeSessionError(output, requestId, error.getClass().getName(), + error.getMessage() == null ? error.getClass().getName() : error.getMessage()); + } + } + + private static void input(UiAutomation automation, JSONObject input) throws Exception { + switch (input.getString("kind")) { + case "tap": + tap(automation, input.getDouble("x"), input.getDouble("y")); + return; + case "key": + key(automation, input.getInt("keyCode")); + return; + case "fill": + fill(automation, input.getInt("x"), input.getInt("y"), input.getString("text")); + return; + default: + throw new IllegalArgumentException("Unknown input kind"); + } + } + + private static void tap(UiAutomation automation, double x, double y) { + if (Double.isNaN(x) || Double.isInfinite(x) || Double.isNaN(y) || Double.isInfinite(y) + || x < 0 || y < 0) { + throw new IllegalArgumentException("Invalid tap coordinates"); + } + long down = SystemClock.uptimeMillis(); + touch(automation, down, MotionEvent.ACTION_DOWN, (float) x, (float) y); + touch(automation, down, MotionEvent.ACTION_UP, (float) x, (float) y); + } + + private static void touch(UiAutomation automation, long down, int action, float x, float y) { + MotionEvent event = MotionEvent.obtain(down, SystemClock.uptimeMillis(), action, x, y, 0); + event.setSource(InputDevice.SOURCE_TOUCHSCREEN); + try { + if (!automation.injectInputEvent(event, true)) { + throw new IllegalStateException("Touch injection was refused"); + } + } finally { + event.recycle(); + } + } + + private static void key(UiAutomation automation, int code) { + if (code < 0 || code > KeyEvent.getMaxKeyCode()) throw new IllegalArgumentException("Invalid key code"); + long down = SystemClock.uptimeMillis(); + for (int action : new int[] {KeyEvent.ACTION_DOWN, KeyEvent.ACTION_UP}) { + KeyEvent event = new KeyEvent(down, SystemClock.uptimeMillis(), action, code, 0, 0, + KeyCharacterMap.VIRTUAL_KEYBOARD, 0, 0, InputDevice.SOURCE_KEYBOARD); + if (!automation.injectInputEvent(event, true)) { + throw new IllegalStateException("Key injection was refused"); + } + } + } + + private static void fill(UiAutomation automation, int x, int y, String text) { + AccessibilityNodeInfo target = null; + try { + for (AccessibilityWindowInfo window : automation.getWindows()) { + try { + if (window.getType() == AccessibilityWindowInfo.TYPE_INPUT_METHOD) continue; + AccessibilityNodeInfo root = AccessibilityNodeReads.root(window); + if (root == null) continue; + AccessibilityNodeInfo candidate; + try { + candidate = editableAtPoint(root, x, y, 0); + } finally { + root.recycle(); + } + if (candidate == null) continue; + if (target != null) { + candidate.recycle(); + throw new IllegalStateException("Multiple editable fields at the fill point"); + } + target = candidate; + } finally { + window.recycle(); + } + } + if (target == null) throw new IllegalStateException("No editable field at the fill point"); + tap(automation, x, y); + if (!target.refresh()) throw new IllegalStateException("Fill target disappeared after focus"); + if (!target.isFocused() && !target.performAction(AccessibilityNodeInfo.ACTION_FOCUS)) { + throw new IllegalStateException("Field focus was refused"); + } + Bundle arguments = new Bundle(); + arguments.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text); + if (!target.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)) { + throw new IllegalStateException("Field text replacement was refused"); + } + } finally { + if (target != null) target.recycle(); + } + } + + private static AccessibilityNodeInfo editableAtPoint(AccessibilityNodeInfo node, int x, int y, int depth) { + if (depth > 128) throw new IllegalStateException("Fill target traversal exceeded its depth limit"); + Rect bounds = new Rect(); + node.getBoundsInScreen(bounds); + AccessibilityNodeInfo target = null; + try { + for (int i = 0; i < node.getChildCount(); i++) { + AccessibilityNodeInfo child = AccessibilityNodeReads.child(node, i); + if (child == null) continue; + AccessibilityNodeInfo candidate; + try { + candidate = editableAtPoint(child, x, y, depth + 1); + } finally { + child.recycle(); + } + if (candidate == null) continue; + if (target != null) { + candidate.recycle(); + throw new IllegalStateException("Multiple editable fields at the fill point"); + } + target = candidate; + } + if (target != null) { + AccessibilityNodeInfo found = target; + target = null; + return found; + } + return node.isEditable() && node.isEnabled() && node.isVisibleToUser() && bounds.contains(x, y) + ? AccessibilityNodeInfo.obtain(node) : null; + } finally { + if (target != null) target.recycle(); + } + } + + private static void screenshot(Bundle result, UiAutomation automation) { + Bitmap image = automation.takeScreenshot(); + if (image == null) throw new IllegalStateException("Screen capture was unavailable"); + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + if (!image.compress(Bitmap.CompressFormat.PNG, 100, bytes)) { + throw new IllegalStateException("PNG encoding failed"); + } + result.putString("payloadBase64", Base64.encodeToString(bytes.toByteArray(), Base64.NO_WRAP)); + } finally { + image.recycle(); + } + } + + private static void focus(Bundle result, UiAutomation automation) throws Exception { + AccessibilityTreeCapture.prepare(automation, 0, 0); + AccessibilityTreeCapture.clearAccessibilityCache(automation); + for (AccessibilityWindowInfo window : automation.getWindows()) { + try { + if (!window.isFocused()) continue; + result.putString("windowType", Integer.toString(window.getType())); + AccessibilityNodeInfo root = AccessibilityNodeReads.rootWithoutPrefetch(window); + if (root == null) continue; + try { + CharSequence packageName = root.getPackageName(); + if (packageName != null) result.putString("focusedPackage", packageName.toString()); + } finally { + root.recycle(); + } + } finally { + window.recycle(); + } + } + } +} diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index ed2dc03f11..4721a51fe5 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -180,6 +180,7 @@ private void runSnapshotSession( putBaseMetadata( ready, waitForIdleTimeoutMs, waitForIdleQuietMs, timeoutMs, maxDepth, maxNodes); ready.putString("sessionReady", "true"); + ready.putString("sessionCommands", "input,screenshot,focus,snapshot-immediate"); ready.putString("sessionPort", Integer.toString(sessionPort)); sendStatus(2, ready); @@ -201,6 +202,12 @@ private void runSnapshotSession( SessionResponseWriter.writeSessionOk(socket.getOutputStream(), requestId); return; } + if ("input".equals(action) || "screenshot".equals(action) || "focus".equals(action)) { + AutomationCommandHandler.writeSessionResponse( + socket.getOutputStream(), requestId, action, + parts.length > 2 ? parts[2] : "", getConnectedUiAutomationUnchecked()); + continue; + } if ("viewport".equals(action)) { TouchCommandHandler.writeSessionViewport( socket.getOutputStream(), requestId, getConnectedUiAutomationUnchecked()); @@ -214,7 +221,7 @@ private void runSnapshotSession( getConnectedUiAutomationUnchecked()); continue; } - if (!"snapshot".equals(action)) { + if (!"snapshot".equals(action) && !"snapshot-immediate".equals(action)) { SessionResponseWriter.writeSessionError( socket.getOutputStream(), requestId, @@ -226,7 +233,7 @@ private void runSnapshotSession( socket.getOutputStream(), requestId, waitForIdleQuietMs, - waitForIdleTimeoutMs, + "snapshot-immediate".equals(action) ? 0 : waitForIdleTimeoutMs, timeoutMs, maxDepth, maxNodes); diff --git a/docs/adr/0002-persistent-platform-helper-sessions.md b/docs/adr/0002-persistent-platform-helper-sessions.md index 2db4cba683..9978f6effe 100644 --- a/docs/adr/0002-persistent-platform-helper-sessions.md +++ b/docs/adr/0002-persistent-platform-helper-sessions.md @@ -14,7 +14,7 @@ Accepted - When startup costs are high and an automation context is reusable, use persistent platform helper sessions. - The daemon owns helper lifetime policy. Platform modules own helper-manager mechanics. - Unless a separate cache contract has explicit invalidation, a helper must execute each command against fresh platform state. -- On Android, snapshot capture, gesture viewport resolution, and instrumentation-backed planned-touch injection share one bundled automation helper. +- On Android, snapshot capture, pointer and key input, text replacement, screenshots, window-focus reads, gesture viewport resolution, and instrumentation-backed planned-touch injection share one bundled automation helper. - On Android, after a failed session command, retry one-shot only for idempotent reads, and only after the failed session has stopped. - On Android, return a session gesture failure directly. Do not retry it one-shot. - For Android snapshots, if the packaged helper is unavailable or invalid, agent-device must fail closed. It must not substitute the legacy `adb uiautomator dump` snapshot engine. @@ -94,8 +94,13 @@ Before that amendment, touch synthesis shipped as a separate instrumentation hel Thus, the daemon had to stop the persistent snapshot session before every gesture, and the next snapshot restarted it lazily. After a failed session command, one-shot retry applies only to idempotent reads, such as viewport resolution. The retry runs only after the failed session has stopped. -Gesture injection is not idempotent because events may already be partially injected. -Thus, session gesture failures return directly without a one-shot retry. +Pointer, key, text, and gesture injection are not idempotent because a mutation may have reached the app before the response fails. +Thus, session mutation failures return directly without a one-shot retry. +The helper advertises optional session commands in its ready record. +The host sends an optional command only when the helper advertises it. +Text replacement retains the focus tap and host verification. +Fresh application-window focus can answer the common dialog-readiness and in-app guards. +System and unclassified windows retain the window-dump diagnostics. Helper reuse must never let process ownership cause interference between commands. For iOS, keep the XCTest runner session as the reference implementation for lifecycle and invalidation behavior. diff --git a/docs/evidence/android-automation-performance-2026-10-19-base.json b/docs/evidence/android-automation-performance-2026-10-19-base.json new file mode 100644 index 0000000000..4490abfbe2 --- /dev/null +++ b/docs/evidence/android-automation-performance-2026-10-19-base.json @@ -0,0 +1,68 @@ +{ + "observedAt": "2026-10-19T08:29:05.470Z", + "version": "0.21.29-dev", + "serial": "emulator-5554", + "rounds": 20, + "summary": { + "open": { + "count": 1, + "p50": 1974.88125, + "p95": 1974.88125 + }, + "restart": { + "count": 3, + "p50": 1221.4566250000003, + "p95": 1272.7720409999997 + }, + "observe": { + "count": 20, + "p50": 27.026958000000377, + "p95": 29.024333000000297 + }, + "tap": { + "count": 20, + "p50": 105.79833400000098, + "p95": 131.45187499999975 + }, + "fill": { + "count": 20, + "p50": 558.3288339999999, + "p95": 3296.912625000001 + }, + "screenshot": { + "count": 10, + "p50": 1237.9075420000008, + "p95": 1325.500250000001 + } + }, + "samples": { + "open": [1974.88125], + "restart": [1221.4566250000003, 1272.7720409999997, 1094.0003749999996], + "observe": [ + 45.778957999999875, 27.57654100000036, 27.712750000000597, 27.990334000000075, + 26.00025000000005, 26.740875000000415, 28.017417000000023, 27.026958000000377, + 28.392417000000023, 26.539334000000053, 26.187415999999757, 26.233374999999796, + 27.746541999999863, 26.64958299999944, 25.823707999999897, 26.324791999999434, + 29.024333000000297, 27.611957999999504, 27.144374999999854, 25.519750000000386 + ], + "tap": [ + 156.7671249999994, 111.00983299999916, 99.53199999999924, 131.45187499999975, + 101.18212500000027, 108.20045800000025, 104.59895899999992, 99.9613330000002, + 100.58424999999988, 107.36154100000022, 114.32608299999993, 112.42979200000082, + 125.20504200000141, 103.49154100000123, 105.79833400000098, 110.19045799999913, + 104.1442500000012, 113.18316699999923, 100.42487500000061, 102.52449999999953 + ], + "fill": [ + 509.73954199999935, 522.5573750000003, 526.1622920000009, 558.3288339999999, + 531.6915420000005, 508.30204200000117, 534.4593750000004, 524.6943329999995, + 561.4105410000011, 557.9260830000003, 565.5302920000013, 529.8140419999982, 560.0029170000016, + 3296.912625000001, 3621.0452920000025, 2794.0020839999997, 613.8256250000013, + 591.187291000002, 606.5397499999999, 754.4855409999982 + ], + "screenshot": [ + 1325.500250000001, 1245.3615000000027, 1243.3869999999952, 1226.1485419999954, + 1237.9075420000008, 1225.732624999997, 1258.799165999997, 1230.1937089999992, + 1241.7689580000006, 1218.0068329999995 + ] + } +} diff --git a/docs/evidence/android-automation-performance-2026-10-19-change.json b/docs/evidence/android-automation-performance-2026-10-19-change.json new file mode 100644 index 0000000000..55991c2755 --- /dev/null +++ b/docs/evidence/android-automation-performance-2026-10-19-change.json @@ -0,0 +1,67 @@ +{ + "observedAt": "2026-10-19T08:29:40.174Z", + "version": "0.21.29-dev", + "serial": "emulator-5554", + "rounds": 20, + "summary": { + "open": { + "count": 1, + "p50": 1822.68275, + "p95": 1822.68275 + }, + "restart": { + "count": 3, + "p50": 1438.9537920000002, + "p95": 1747.7173329999996 + }, + "observe": { + "count": 20, + "p50": 28.247334000000592, + "p95": 43.49158400000033 + }, + "tap": { + "count": 20, + "p50": 52.498374999999214, + "p95": 71.78887500000019 + }, + "fill": { + "count": 20, + "p50": 274.9039160000011, + "p95": 320.104292 + }, + "screenshot": { + "count": 10, + "p50": 69.09745800000019, + "p95": 90.72204200000124 + } + }, + "samples": { + "open": [1822.68275], + "restart": [1352.413083, 1438.9537920000002, 1747.7173329999996], + "observe": [ + 43.49158400000033, 68.1205829999999, 42.22912499999984, 41.98558300000059, 32.813750000000255, + 31.26354199999969, 27.980375000000095, 28.247334000000592, 28.18345799999952, + 27.63783400000011, 28.944290999999794, 27.74183399999947, 28.38483400000041, + 27.308415999999852, 27.91345799999999, 27.493083000000297, 28.324792000000343, + 30.343583000000763, 27.84783299999981, 27.909208000000035 + ], + "tap": [ + 91.6572500000002, 48.43120799999997, 55.57158299999992, 51.092915999999605, 53.25258399999984, + 52.498374999999214, 52.05000000000018, 64.53916600000048, 56.65462500000103, + 55.79291700000067, 71.78887500000019, 49.233249999999316, 53.193208000000595, + 49.52549999999974, 56.17620800000077, 48.1856659999994, 50.54033300000083, 48.412874999999985, + 56.652500000000146, 51.12199999999939 + ], + "fill": [ + 285.4419170000001, 272.2380000000012, 267.15691700000025, 279.33129200000076, + 276.94137500000033, 320.104292, 283.2405839999992, 483.2623330000006, 287.7722080000003, + 274.9039160000011, 268.5384589999994, 280.07829200000015, 263.04104200000074, + 272.88083299999926, 268.9792500000003, 265.0731660000001, 268.2274589999997, + 271.8718340000014, 275.08550000000105, 281.9104589999988 + ], + "screenshot": [ + 90.72204200000124, 70.9776669999992, 66.74666699999943, 69.09745800000019, 77.99150000000009, + 69.82012500000201, 68.6362500000032, 67.60212500000125, 69.22020799999882, 67.85250000000087 + ] + } +} diff --git a/docs/evidence/android-automation-performance-2026-10-19.md b/docs/evidence/android-automation-performance-2026-10-19.md new file mode 100644 index 0000000000..10b6bbcdea --- /dev/null +++ b/docs/evidence/android-automation-performance-2026-10-19.md @@ -0,0 +1,65 @@ +# Android automation performance, 2026-10-19 + +The comparison starts from `eef1a3f76d7216da006ff59c975620c8b9dd7182`. +The reference experiment is [tester-army/e2e#1234](https://github.com/tester-army/e2e/pull/1234), inspected at `2593e45025edc52b2c381e809b41cb63d5878e2b`. +Its measurements are author-reported. Its direct engine sets text through `ACTION_SET_TEXT`, injects input through `UiAutomation`, and captures screenshots inside its instrumentation process. + +## Reproduce + +Use a dedicated Pixel 7 API 36 emulator and the current `examples/test-app` Release APK. +The local run used `Pixel_7_CI`, serial `emulator-5554`, Android 16, Node 26.8.1. +Animations remained enabled. The Device panel stream remained attached for both variants. +The APK came from the main-branch artifact `fingerprint.22184ce7fc5743fac60a0b5e673374310cea6154.android` (artifact `10611725290`, producer run `35529367803`) and was repacked with this checkout's JavaScript. + +Build each checkout and its Android helpers separately. Run the variants in sequence on the same device and APK: + +```sh +ANDROID_SERIAL=emulator-5554 BENCH_ROUNDS=20 \ + node scripts/bench-android-automation.mjs /absolute/checkout /private/tmp/android-benchmark.json +``` + +The script uses the public SDK and the Checkout form's `field-email`. +It excludes locator snapshots from tap and fill timings. +Every fill is independently read back and checked. +The JSON contains raw samples, nearest-rank p50/p95, the checkout, version, serial, and observation time. +Open is one cold sample and is not evidence of a startup speedup. + +## Causes and boundaries + +- Ordinary taps and key events used `adb shell input` despite a live automation helper. +- Tap readiness and escape checks issued repeated window/activity dumps. Fresh app-window focus can answer the common case in process. System windows retain the diagnostic path. +- Fill cleared and committed text through IME broadcasts or shell commands. The helper can replace the selected editable node's text directly. It must still focus by tapping, raise the system keyboard when appropriate, preserve provider-native injection precedence, and verify the observed value. +- Fill's ownership probes repeated `dumpsys input_method` even when captured window provenance already identified the IME. Plain XML without window metadata retains the probe. +- Default screenshots enabled demo mode and waited one second. Immediate capture is the new default. `--normalize-status-bar` and SDK `stabilize: true` request the previous normalized, delayed capture. +- The helper retains fresh acquisition; none of these changes caches snapshot results. +- Mutation failures are not resent. Optional commands require the helper's advertised support. Timeout diagnostics omit encoded input payloads. + +The direct experiment's 4.4 ms fill is not a verified-fill comparison. Agent-device retains two matching samples separated by 150 ms, plus identity, masking, formatting, and IME checks. This sets a latency floor above the reference text action alone. + +## Test-app measurements + +Twenty samples per steady-state operation; screenshots have ten and warm restarts have three. Values are milliseconds. + +| Operation | Base p50 | Change p50 | Base p95 | Change p95 | +| --- | ---: | ---: | ---: | ---: | +| observe | 27.0 | 28.2 | 29.0 | 43.5 | +| tap | 105.8 | 52.5 | 131.5 | 71.8 | +| fill | 558.3 | 274.9 | 3296.9 | 320.1 | +| screenshot | 1237.9 | 69.1 | 1325.5 | 90.7 | +| restart | 1221.5 | 1439.0 | 1272.8 | 1747.7 | + +Observe did not improve. Restarts did not improve in this three-sample run. The cold open had one sample per variant and must not be treated as a measured improvement. + +Fill-owned captures skip the helper idle wait because the host already samples the live field until two observations agree across 150 ms. This request does not change session identity or the next public snapshot's idle policy. + +## Validation + +The complete test-app Android suite passed with no skipped scenarios: `152951 ms` on the final implementation. The run includes inventory, automation and rotation, Maestro selection, Unicode forms, system keyboard, capture and close, lifecycle and permission recovery, recording and performance artifacts, catalog traversal, checkout, gesture, and drag replays. Artifact: `test/artifacts/android-emulator/1792398951596-10411/coverage-report.json`. + +The unchanged base passed the smoke selection in `61434 ms` and the three full-tier scenarios in `108610 ms`. These were separate invocations, so their sum is not a paired full-suite benchmark. The result does not establish the reference experiment's 2.4x total-suite gain. + +Full-suite investigation exposed a pre-existing Android live-text bug on both base and the change: `get text id="automation-window"` resolved a node labeled `portrait` but read neighbouring `active` text at its former point. A minimal regression failed with `active` before the fix. The selected resource id, package, and class now reach the live read. A moved target returns its own text; a missing, cross-package, or duplicate identity cannot select a neighbour. + +An early native-fill implementation omitted the keyboard-opening tap; the lifecycle scenario detected it and the focus tap was restored. One checkout submission miss occurred before the final synchronization and freshness hardening; six isolated checkout replays passed, so its cause was not established. The final combined suite passed, including every replay without retries. + +Input-socket and screenshot-default regressions failed on unchanged production code (`3 failed, 17 passed`) and passed with the implementation. The focus-path regression also failed on unchanged base. The text-identity regression was observed red and green at its owning hierarchy seam, and its contract and daemon forwarding tests passed. diff --git a/packages/contracts/src/element-text-runtime.test.ts b/packages/contracts/src/element-text-runtime.test.ts index f9804d61f6..197cbc1d93 100644 --- a/packages/contracts/src/element-text-runtime.test.ts +++ b/packages/contracts/src/element-text-runtime.test.ts @@ -95,3 +95,28 @@ test('an advertised read with no interactor implementation fails as a contract b /advertised readTextAtPoint/.test(error.message), ); }); + +test('a live text binding forwards the resolved target identity to its owner', async () => { + const target = { + identifier: 'window-state', + bundleId: 'com.example', + type: 'android.widget.TextView', + }; + let received: unknown; + const runtime = bindElementTextRuntime({ + device: { platform: 'android' } as DeviceInfo, + signal: new AbortController().signal, + resolveInteractor: async () => + ({ + readTextAtPoint: async (_point, options) => { + received = options?.target; + return 'portrait'; + }, + }) as Interactor, + }); + assert.deepEqual( + await runtime.readTextAtPoint({ point: { x: 50, y: 50 }, options: { target } }), + { status: 'read', text: 'portrait' }, + ); + assert.equal(received, target); +}); diff --git a/packages/contracts/src/element-text-runtime.ts b/packages/contracts/src/element-text-runtime.ts index abfda48730..a96a4e26d9 100644 --- a/packages/contracts/src/element-text-runtime.ts +++ b/packages/contracts/src/element-text-runtime.ts @@ -1,18 +1,22 @@ import type { DeviceInfo } from '@agent-device/kernel/device'; -import type { Point } from '@agent-device/kernel/snapshot'; +import type { Point, SnapshotNode } from '@agent-device/kernel/snapshot'; import type { Interactor, RunnerContext } from './interactor-types.ts'; import { invalidRuntimeContract } from './runtime-contract-error.ts'; import type { RuntimeOperationFact } from './platform-runtime.ts'; import type { SessionSurface } from './session-surface.ts'; import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; -/** - * Neutral intent for one point-addressed element read. The point is already resolved from the - * node the caller matched, so the operation names no command, request, session, or CLI flag. - */ +/** Identity carried from a resolved node into its live text read. */ +export type ElementTextTarget = Pick; + +/** A live text read at a resolved point, with optional identity to preserve the selected control. */ export type ReadTextAtPointInput = Readonly<{ point: Point; - options?: Readonly<{ appBundleId?: string; surface?: SessionSurface }>; + options?: Readonly<{ + appBundleId?: string; + surface?: SessionSurface; + target?: ElementTextTarget; + }>; /** Same runner metadata a capture needs; reuses that type rather than restating it. */ execution?: SnapshotRuntimeExecution; }>; @@ -113,6 +117,7 @@ export function bindElementTextRuntime( appBundleId: input.options?.appBundleId, surface: input.options?.surface, signal, + ...(input.options?.target ? { target: input.options.target } : {}), }), ); }, diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index d5cf0a7343..e0660d0223 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -1,4 +1,5 @@ import type { AppStateRuntimeResult } from './app-state-runtime.ts'; +import type { ElementTextTarget } from './element-text-runtime.ts'; import type { BackMode } from './back-mode.ts'; import type { IosSystemSurfaceProvenance } from './ios-system-surface.ts'; import type { DeviceRotation } from './device-rotation.ts'; @@ -375,7 +376,12 @@ export type Interactor = { */ readTextAtPoint?( point: Point, - options?: { appBundleId?: string; surface?: SessionSurface; signal?: AbortSignal }, + options?: { + appBundleId?: string; + surface?: SessionSurface; + signal?: AbortSignal; + target?: ElementTextTarget; + }, ): Promise; /** * Native text-presence reading, when the backend has one that does not require a tree capture. diff --git a/packages/contracts/src/screenshot.ts b/packages/contracts/src/screenshot.ts index e3610f8a5d..813fd3924a 100644 --- a/packages/contracts/src/screenshot.ts +++ b/packages/contracts/src/screenshot.ts @@ -164,7 +164,7 @@ export const SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS: readonly ScreenshotSpecificFl type: 'boolean', usageLabel: '--no-stabilize', usageDescription: - 'Screenshot: skip Android demo-mode/status-bar stabilization and settle delay for low-latency capture loops', + 'Screenshot: capture immediately without Android status-bar stabilization (the default)', projectConfig: true, recorded: true, }, @@ -174,7 +174,7 @@ export const SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS: readonly ScreenshotSpecificFl type: 'boolean', usageLabel: '--normalize-status-bar', usageDescription: - 'Screenshot: on iOS simulators temporarily normalize status-bar chrome for deterministic screenshot diffs', + 'Screenshot: temporarily normalize iOS simulator or Android status-bar chrome for deterministic screenshot diffs; Android waits for transient UI to fade', projectConfig: true, recorded: true, }, @@ -270,7 +270,8 @@ export function screenshotOptionsFromFlags( pixelDensity: flags?.screenshotPixelDensity, fullscreen: flags?.screenshotFullscreen, scale: flags?.screenshotScale, - stabilize: flags?.screenshotNoStabilize ? false : undefined, + stabilize: + flags?.screenshotNoStabilize === undefined ? undefined : !flags.screenshotNoStabilize, normalizeStatusBar: flags?.screenshotNormalizeStatusBar, }); } @@ -285,7 +286,8 @@ export function screenshotFlagsFromOptions( screenshotFullscreen: options.screenshotFullscreen ?? options.fullscreen, screenshotScale: options.screenshotScale, screenshotNoStabilize: - options.screenshotNoStabilize ?? (options.stabilize === false ? true : undefined), + options.screenshotNoStabilize ?? + (options.stabilize === undefined ? undefined : !options.stabilize), screenshotNormalizeStatusBar: options.screenshotNormalizeStatusBar ?? options.normalizeStatusBar, }); @@ -327,6 +329,7 @@ export function appendScreenshotScriptFlags( parts.push('--scale', String(flags.screenshotScale)); } if (flags?.screenshotNoStabilize) parts.push('--no-stabilize'); + if (flags?.screenshotNoStabilize === false) parts.push('--no-stabilize=false'); if (flags?.screenshotNormalizeStatusBar) parts.push('--normalize-status-bar'); } @@ -357,6 +360,10 @@ function readScreenshotBooleanScriptFlag( flags: Partial, index: number, ): { handled: true; nextIndex: number } | undefined { + if (token === '--no-stabilize=false') { + flags.screenshotNoStabilize = false; + return { handled: true, nextIndex: index }; + } const definition = SCREENSHOT_SCRIPT_BOOLEAN_FLAGS.find((entry) => entry.tokens.some((candidate) => candidate === token), ); diff --git a/packages/daemon/src/execution/__tests__/selector-text-runtime.test.ts b/packages/daemon/src/execution/__tests__/selector-text-runtime.test.ts index 6d15c93d2f..492ad36854 100644 --- a/packages/daemon/src/execution/__tests__/selector-text-runtime.test.ts +++ b/packages/daemon/src/execution/__tests__/selector-text-runtime.test.ts @@ -129,3 +129,24 @@ describe('readTextForNode', () => { ).rejects.toThrow(/runner transport closed/); }); }); + +describe('Android text identity', () => { + it('carries the selected resource identity into the live read', async () => { + readTextAtPoint.mockClear(); + await readTextForNode({ + ...baseParams, + device: { platform: 'android' } as never, + node: node({ + type: 'android.widget.TextView', + identifier: 'window-state', + bundleId: 'com.example', + label: 'portrait', + }), + }); + expect(readTextAtPoint.mock.calls[0]?.[0].options?.target).toEqual({ + identifier: 'window-state', + bundleId: 'com.example', + type: 'android.widget.TextView', + }); + }); +}); diff --git a/packages/daemon/src/execution/android-foreground-surface.ts b/packages/daemon/src/execution/android-foreground-surface.ts index c9d52e5f73..343c2fe18d 100644 --- a/packages/daemon/src/execution/android-foreground-surface.ts +++ b/packages/daemon/src/execution/android-foreground-surface.ts @@ -31,6 +31,7 @@ export async function detectAndroidEscapeSurface( ); } + if (await observation.readAppFocus(session.device, session.appBundleId)) return null; const foreground = await observation.readAppState(session.device); const foregroundPackage = foreground.package?.trim(); if (!foregroundPackage || foregroundPackage === session.appBundleId) return null; diff --git a/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts index da6bd3bafc..bd5dfd7adc 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts @@ -21,6 +21,7 @@ async function pressThatLeftTheApp(): Promise { appBundleId: 'com.example.app', }); const observation = { + readAppFocus: async () => false, readAppState: async () => ({ package: 'com.android.settings' }), isPermissionPackage: async () => false, } as unknown as AndroidObservationAdapter; diff --git a/packages/daemon/src/execution/selector-text-runtime.ts b/packages/daemon/src/execution/selector-text-runtime.ts index db6ece8456..8bdafcfc80 100644 --- a/packages/daemon/src/execution/selector-text-runtime.ts +++ b/packages/daemon/src/execution/selector-text-runtime.ts @@ -44,7 +44,15 @@ export async function readTextForNode(params: { const context = contextFromFlags(flags, appBundleId, traceOutPath); const outcome = await readTextAtPoint({ point: center, - options: { appBundleId, surface }, + options: { + appBundleId, + surface, + ...(device.platform === 'android' && node.identifier + ? { + target: { identifier: node.identifier, bundleId: node.bundleId, type: node.type }, + } + : {}), + }, execution: runtimeExecutionFromContext(context), }); if (outcome.status === 'read') return outcome.text; diff --git a/packages/platform-android/src/__tests__/fill-verification.test.ts b/packages/platform-android/src/__tests__/fill-verification.test.ts index b699f01f09..25502c4321 100644 --- a/packages/platform-android/src/__tests__/fill-verification.test.ts +++ b/packages/platform-android/src/__tests__/fill-verification.test.ts @@ -162,10 +162,15 @@ test('the pre-action target read shares the daemon-session helper with the sampl assert.equal(target?.resourceId, 'com.example:id/field'); assert.equal(session.captureCount(), 3); assert.equal(session.spawnArgs.length, 1); + assert.equal( + session.requests.filter((command) => command.startsWith('snapshot-immediate ')).length, + session.captureCount(), + ); assert.equal(session.forwardRemovals(), 0); }); type FillHelperSession = { + requests: string[]; provider: AndroidAdbProvider; spawnArgs: (readonly string[])[]; processes: FakeAndroidProcess[]; @@ -179,11 +184,13 @@ function createFillHelperSession( const calls: (readonly string[])[] = []; const spawnArgs: (readonly string[])[] = []; const processes: FakeAndroidProcess[] = []; + const requests: string[] = []; let captureCount = 0; const provider = createPersistentSnapshotHelperProvider({ calls, spawnArgs, processes, + sessionRequests: requests, sessionXml: () => { captureCount += 1; return filledFieldXml(options.textForCapture?.(captureCount) ?? 'chips'); @@ -194,6 +201,7 @@ function createFillHelperSession( spawnArgs, processes, captureCount: () => captureCount, + requests, forwardRemovals: () => calls.filter(isAndroidHelperForwardRemoval).length, }; } @@ -260,3 +268,33 @@ test('a commit counts as dropped when the field shows its hint or keeps its old assert.equal(isAndroidFillCommitDropped(observed('Jan'), before), false); assert.equal(isAndroidFillCommitDropped(observed('Jan'), null), false); }); + +test('helper window provenance identifies a custom IME without an adb keyboard probe', async () => { + const calls: (readonly string[])[] = []; + const spawnArgs: (readonly string[])[] = []; + const processes: FakeAndroidProcess[] = []; + const provider = createPersistentSnapshotHelperProvider({ + calls, + spawnArgs, + processes, + sessionXml: () => ` + ${filledFieldXml('chips').replace('focused="true"', 'focused="false"').replace('', '').replace('', '').replace('', '')} + + `, + }); + let probes = 0; + const verification = await withFillHelperProvider( + { + ...provider, + exec: async (args, options) => { + if (args.includes('input_method')) probes++; + return await provider.exec(args, options); + }, + }, + () => + verifyAndroidFilledText(device, 10, 10, 'chips', { helperSessionScope: 'daemon-session' }), + ); + assert.equal(probes, 0); + assert.equal(verification.reason, 'ime_capture'); + assert.equal(verification.actualInput?.packageName, 'dev.custom.keyboard'); +}); diff --git a/packages/platform-android/src/__tests__/screenshot.test.ts b/packages/platform-android/src/__tests__/screenshot.test.ts index 445a701520..057a19e2bc 100644 --- a/packages/platform-android/src/__tests__/screenshot.test.ts +++ b/packages/platform-android/src/__tests__/screenshot.test.ts @@ -128,3 +128,21 @@ async function withTempScreenshot( await fs.rm(tmpDir, { recursive: true, force: true }); } } + +test('Android screenshots capture immediately by default and stabilize only when requested', async () => { + await withTempScreenshot('screenshot-default-latency-', async (outPath) => { + await screenshotAndroid(device, outPath); + assert.equal(mockSleep.mock.calls.length, 0); + assert.equal( + mockRunCmd.mock.calls.some(([, args]) => args.includes('broadcast')), + false, + ); + assert.deepEqual(await fs.readFile(outPath), VALID_PNG); + await screenshotAndroid(device, outPath, { normalizeStatusBar: true }); + assert.deepEqual(mockSleep.mock.calls, [[1000]]); + assert.equal( + mockRunCmd.mock.calls.some(([, args]) => args.includes('broadcast')), + true, + ); + }); +}); diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts index 6358b5f154..21dd1bbfc1 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; import { assertAndroidSnapshotHelperTouchSessionHeaders, isAndroidSnapshotHelperSessionCommandAcknowledged, @@ -139,3 +140,32 @@ function sessionResponse(params: { .map(([key, value]) => `${key}=${value}`) .join('\n')}\n\n${params.xml}`; } + +test('helper timeout diagnostics omit encoded text input', async () => { + const net = await import('node:net'); + const { sendAndroidSnapshotHelperSessionCommand } = + await import('../snapshot-helper-session-protocol.ts'); + const sockets = new Set(); + const server = net.createServer((socket) => { + sockets.add(socket); + socket.on('close', () => sockets.delete(socket)); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + assert(address && typeof address !== 'string'); + const encoded = Buffer.from(JSON.stringify({ text: 'private fill value' })).toString('base64'); + try { + await assert.rejects( + sendAndroidSnapshotHelperSessionCommand(address.port, `input request-1 ${encoded}`, 20), + (error) => { + assert(error instanceof AppError); + assert(!JSON.stringify(error).includes(encoded)); + assert.equal(error.details?.command, 'input'); + return true; + }, + ); + } finally { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(resolve)); + } +}); diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts b/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts index 181f5846e2..e82803e1cd 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts @@ -57,6 +57,8 @@ export type PersistentSnapshotHelperProviderOptions = { processes: FakeAndroidProcess[]; sessionResponseMode?: 'ok' | 'malformed'; sessionXml?: (sessionIndex: number, snapshotCount: number) => string; + sessionRequests?: string[]; + sessionInput?: (input: { kind: string; text?: string }) => void; stalledSessionCleanup?: boolean; oneShotAttempts?: (readonly string[])[]; oneShotXml?: string; @@ -84,6 +86,7 @@ export function createPersistentSnapshotHelperProvider( const server = net.createServer((socket) => { socket.once('data', (chunk) => { const command = chunk.toString('utf8').trim(); + options.sessionRequests?.push(command); const [, requestId = ''] = command.split(/\s+/, 2); if (command.startsWith('quit')) { socket.end(sessionResponse({ requestId, body: '' })); @@ -94,6 +97,13 @@ export function createPersistentSnapshotHelperProvider( socket.end('malformed session response'); return; } + if (command.startsWith('input')) { + options.sessionInput?.( + JSON.parse(Buffer.from(command.split(' ')[2]!, 'base64').toString('utf8')), + ); + socket.end(sessionResponse({ requestId, body: '' })); + return; + } if (command.startsWith('viewport')) { socket.end( sessionResponse({ @@ -136,6 +146,7 @@ export function createPersistentSnapshotHelperProvider( process.stdout.write( [ 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', + 'INSTRUMENTATION_STATUS: sessionCommands=input,screenshot,focus,snapshot-immediate', 'INSTRUMENTATION_STATUS: sessionReady=true', 'INSTRUMENTATION_STATUS_CODE: 2', '', @@ -269,6 +280,7 @@ export function createSessionProvider(options: SessionProviderOptions): AndroidA process.stdout.write( [ 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', + 'INSTRUMENTATION_STATUS: sessionCommands=input,screenshot,focus,snapshot-immediate', 'INSTRUMENTATION_STATUS: sessionReady=true', 'INSTRUMENTATION_STATUS_CODE: 2', '', diff --git a/packages/platform-android/src/__tests__/text-input-fill.test.ts b/packages/platform-android/src/__tests__/text-input-fill.test.ts index 74bda6e04b..c00963c970 100644 --- a/packages/platform-android/src/__tests__/text-input-fill.test.ts +++ b/packages/platform-android/src/__tests__/text-input-fill.test.ts @@ -643,6 +643,9 @@ test('fillAndroid runs the whole attempt on one warm daemon-session helper', asy calls, spawnArgs, processes, + sessionInput: (input) => { + if (input.kind === 'fill') typed = input.text ?? ''; + }, sessionXml: () => { captureCount += 1; return androidInputXml({ text: typed }); @@ -837,3 +840,24 @@ test('fillAndroid with an empty text fails, not unconfirmed, when a provider fil }, ); }); + +test('a live text read keeps the resolved identity when its old point covers a sibling', () => { + const xml = ` + + + `; + const target = { + identifier: 'window-state', + bundleId: 'com.example', + type: 'android.widget.TextView', + }; + assert.equal(readAndroidTextAtPointInHierarchy(xml, 50, 50, target), 'portrait'); + assert.equal( + readAndroidTextAtPointInHierarchy(xml, 50, 50, { ...target, bundleId: 'other.app' }), + null, + ); + assert.equal( + readAndroidTextAtPointInHierarchy(xml.replace('app-state', 'window-state'), 50, 50, target), + null, + ); +}); diff --git a/packages/platform-android/src/__tests__/touch-helper-session.test.ts b/packages/platform-android/src/__tests__/touch-helper-session.test.ts index 02643178bf..ee62c00ad5 100644 --- a/packages/platform-android/src/__tests__/touch-helper-session.test.ts +++ b/packages/platform-android/src/__tests__/touch-helper-session.test.ts @@ -177,6 +177,7 @@ function createFakeTouchHelperSessionProvider( process.stdout.write( [ 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', + 'INSTRUMENTATION_STATUS: sessionCommands=input,screenshot,focus,snapshot-immediate', 'INSTRUMENTATION_STATUS: sessionReady=true', 'INSTRUMENTATION_STATUS_CODE: 2', '', @@ -811,3 +812,148 @@ for (const row of SESSION_DISPATCH_ROWS) { }); }); } + +test('input actions reuse the live automation socket without adb input calls', async () => { + const { pressAndroid, backAndroid, pressAndroidEnter } = await import('../input-actions.ts'); + const device = makeIsolatedDevice(); + const inputs: unknown[] = []; + const provider = createFakeTouchHelperSessionProvider((command, requestId) => { + if (command.startsWith('snapshot')) return snapshotSessionResponse(requestId); + if (command.startsWith('input')) { + inputs.push(JSON.parse(Buffer.from(command.split(' ')[2]!, 'base64').toString('utf8'))); + } + return sessionHeaderResponse({ + agentDeviceProtocol: 'android-snapshot-helper-v1', + requestId, + ok: 'true', + }); + }); + const adbCalls: (readonly string[])[] = []; + await withAndroidAdbProvider( + { + ...provider, + exec: currentVersionAdb(async (args) => { + adbCalls.push(args); + return { exitCode: 0, stdout: '', stderr: '' }; + }), + }, + { serial: device.id }, + async () => { + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: getAndroidSnapshotHelperSessionDeviceKey(device), + packageName: manifest.packageName, + instrumentationRunner: manifest.instrumentationRunner, + }); + await pressAndroid(device, 12, 34); + await backAndroid(device); + await pressAndroidEnter(device); + }, + ); + assert.deepEqual(inputs, [ + { kind: 'tap', x: 12, y: 34 }, + { kind: 'key', keyCode: 4 }, + { kind: 'key', keyCode: 66 }, + ]); + assert.equal( + adbCalls.some((args) => args.includes('input')), + false, + ); +}); + +test('a failed helper input is not resent through adb', async () => { + const { pressAndroid } = await import('../input-actions.ts'); + const device = makeIsolatedDevice(); + let inputCalls = 0; + const provider = createFakeTouchHelperSessionProvider((command, requestId) => { + if (command.startsWith('snapshot')) return snapshotSessionResponse(requestId); + inputCalls++; + return sessionHeaderResponse({ + agentDeviceProtocol: 'android-snapshot-helper-v1', + requestId, + ok: 'false', + errorType: 'java.lang.IllegalStateException', + message: 'Input dispatch failed', + }); + }); + const adbCalls: (readonly string[])[] = []; + await withAndroidAdbProvider( + { + ...provider, + exec: currentVersionAdb(async (args) => { + adbCalls.push(args); + return { exitCode: 0, stdout: '', stderr: '' }; + }), + }, + { serial: device.id }, + async () => { + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: getAndroidSnapshotHelperSessionDeviceKey(device), + packageName: manifest.packageName, + instrumentationRunner: manifest.instrumentationRunner, + }); + await assert.rejects( + pressAndroid(device, 12, 34), + (error: AppError) => error.details?.dispatched === 'unknown', + ); + }, + ); + assert.equal(inputCalls, 1); + assert.equal( + adbCalls.some((args) => args.includes('input')), + false, + ); +}); + +test('fresh helper focus answers app guards while system windows retain ANR diagnostics', async () => { + const { createAndroidObservationAdapter } = await import('../observation.ts'); + const device = makeIsolatedDevice(); + let systemWindow = false; + const provider = createFakeTouchHelperSessionProvider((command, requestId) => { + if (command.startsWith('snapshot')) return snapshotSessionResponse(requestId); + return sessionHeaderResponse({ + agentDeviceProtocol: 'android-snapshot-helper-v1', + requestId, + ok: 'true', + focusedPackage: systemWindow ? 'android' : 'com.example.app', + windowType: systemWindow ? '3' : '1', + }); + }); + const runAdb = vi.fn(async () => ({ + exitCode: 0, + stderr: '', + stdout: 'mCurrentFocus=Window{1 u0 Application Not Responding: com.example.app}', + })); + const observer = createAndroidObservationAdapter({ + runAdb, + readSnapshotNodes: async () => [], + openApp: async () => {}, + }); + await withAndroidAdbProvider( + { ...provider, exec: currentVersionAdb(async () => ({ exitCode: 0, stdout: '', stderr: '' })) }, + { serial: device.id }, + async () => { + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: getAndroidSnapshotHelperSessionDeviceKey(device), + packageName: manifest.packageName, + instrumentationRunner: manifest.instrumentationRunner, + }); + assert.deepEqual(await observer.readBlockingDialog(device), { status: 'clear' }); + assert.equal( + await observer.readAppFocus(device, 'com.example.app', { requireNoBlockingDialog: true }), + true, + ); + assert.equal(runAdb.mock.calls.length, 0); + systemWindow = true; + const blocked = await observer.readBlockingDialog(device); + assert.equal(blocked.status, 'dialog'); + if (blocked.status === 'dialog') assert.equal(blocked.focus.package, 'com.example.app'); + assert.equal(runAdb.mock.calls.length, 1); + }, + ); +}); diff --git a/packages/platform-android/src/automation-helper-artifact.ts b/packages/platform-android/src/automation-helper-artifact.ts new file mode 100644 index 0000000000..215462dffb --- /dev/null +++ b/packages/platform-android/src/automation-helper-artifact.ts @@ -0,0 +1,102 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { + resolveAndroidAdbProvider, + type AndroidAdbExecutor, + type AndroidAdbProvider, +} from './adb-executor.ts'; +import { emitDiagnostic, withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; +import { resolveAndroidHelperArtifact } from './helper-package-install.ts'; +import { parseAndroidSnapshotHelperManifest } from './snapshot-helper-artifact.ts'; +import { ensureAndroidSnapshotHelper } from './snapshot-helper-install.ts'; +import { stopAndroidSnapshotHelperSession } from './snapshot-helper-session-lifecycle.ts'; +import { + getAndroidSnapshotHelperSessionDeviceKey, + recoverAndroidSnapshotHelperRetirement, +} from './snapshot-helper-retirement.ts'; +import type { + AndroidSnapshotHelperArtifact, + AndroidSnapshotHelperInstallResult, +} from './snapshot-helper-types.ts'; +const HELPER_INSTALL_TIMEOUT_MS = 30_000; +type PreparedAndroidAutomationHelper = { + adb: AndroidAdbExecutor; + adbProvider: AndroidAdbProvider; + artifact: AndroidSnapshotHelperArtifact; + install: AndroidSnapshotHelperInstallResult; + deviceKey: string; +}; + +// Sessions are keyed by device, so reuse must also prove the live session runs the helper binary +// this command selected; a mismatch stops the session and the command falls back to one-shot. +export function androidAutomationHelperIdentity(artifact: AndroidSnapshotHelperArtifact): { + packageName: string; + runner: string; + helperVersion: string; + helperVersionCode: number; + sha256: string; +} { + return { + packageName: artifact.manifest.packageName, + runner: artifact.manifest.instrumentationRunner, + helperVersion: artifact.manifest.version, + helperVersionCode: artifact.manifest.versionCode, + sha256: artifact.manifest.sha256, + }; +} + +export async function prepareAndroidAutomationHelper( + device: DeviceInfo, +): Promise { + const adbProvider = resolveAndroidAdbProvider(device); + // Same artifact precedence as snapshot capture: a provider-supplied helper artifact overrides + // the bundled one so both transports install and run the single shared helper (issue #1275). + const artifact = + adbProvider.snapshotHelperArtifact ?? (await resolveAndroidAutomationHelperArtifact()); + const deviceKey = getAndroidSnapshotHelperSessionDeviceKey(device); + await recoverAndroidSnapshotHelperRetirement({ + deviceKey, + adb: adbProvider.exec, + }); + const install = await withDiagnosticTimer( + 'android_touch_helper_install', + async () => + await ensureAndroidSnapshotHelper({ + adb: adbProvider.exec, + adbProvider, + artifact, + deviceKey, + timeoutMs: HELPER_INSTALL_TIMEOUT_MS, + }), + { + packageName: artifact.manifest.packageName, + versionCode: artifact.manifest.versionCode, + }, + ); + emitDiagnostic({ + phase: 'android_touch_helper_install_decision', + data: install, + }); + if (install.installed) { + // An APK replacement kills the running instrumentation, so a persistent session started + // against the previous binary must not serve this command; the next snapshot restarts it. + await stopAndroidSnapshotHelperSession(deviceKey); + } + return { adb: adbProvider.exec, adbProvider, artifact, install, deviceKey }; +} + +async function resolveAndroidAutomationHelperArtifact(): Promise { + return await resolveAndroidHelperArtifact({ + helperDirName: 'snapshot-helper', + manifestFileName: (version) => `agent-device-android-snapshot-helper-${version}.manifest.json`, + parseManifest: (value) => { + const manifest = parseAndroidSnapshotHelperManifest(value); + return { + ...manifest, + assetName: + manifest.assetName ?? `agent-device-android-snapshot-helper-${manifest.version}.apk`, + }; + }, + unavailableMessage: + 'Android automation requires the bundled Android automation helper artifact, but it was not found or could not be read', + }); +} diff --git a/packages/platform-android/src/automation-helper.test.ts b/packages/platform-android/src/automation-helper.test.ts new file mode 100644 index 0000000000..c45bd66668 --- /dev/null +++ b/packages/platform-android/src/automation-helper.test.ts @@ -0,0 +1,22 @@ +import { expect, test, vi } from 'vitest'; + +vi.mock('./snapshot-helper-session-lifecycle.ts', () => ({ + getLiveAndroidSnapshotHelperSession: () => ({ commands: new Set(['snapshot']) }), + stopAndroidSnapshotHelperSession: vi.fn(), +})); +vi.mock('./automation-helper-artifact.ts', () => ({ + prepareAndroidAutomationHelper: vi.fn(() => { + throw new Error('An unsupported command must not prepare or replace the helper'); + }), + androidAutomationHelperIdentity: vi.fn(), +})); +import { runAndroidAutomationHelper } from './automation-helper.ts'; +import { prepareAndroidAutomationHelper } from './automation-helper-artifact.ts'; +import { ANDROID_EMULATOR } from './__tests__/test-utils/device-fixtures.ts'; + +test('an older helper is not sent unadvertised input, screenshot, or focus commands', async () => { + for (const action of ['input', 'screenshot', 'focus'] as const) { + await expect(runAndroidAutomationHelper(ANDROID_EMULATOR, action)).resolves.toBeUndefined(); + } + expect(prepareAndroidAutomationHelper).not.toHaveBeenCalled(); +}); diff --git a/packages/platform-android/src/automation-helper.ts b/packages/platform-android/src/automation-helper.ts new file mode 100644 index 0000000000..e6416963a3 --- /dev/null +++ b/packages/platform-android/src/automation-helper.ts @@ -0,0 +1,48 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { + getLiveAndroidSnapshotHelperSession, + stopAndroidSnapshotHelperSession, +} from './snapshot-helper-session-lifecycle.ts'; +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import { getAndroidSnapshotHelperSessionDeviceKey } from './snapshot-helper-retirement.ts'; +import { runAndroidSnapshotHelperSessionTouchCommand } from './snapshot-helper-session.ts'; +import { + prepareAndroidAutomationHelper, + androidAutomationHelperIdentity, +} from './automation-helper-artifact.ts'; + +export type AndroidHelperInput = + | { kind: 'tap'; x: number; y: number } + | { kind: 'key'; keyCode: number } + | { kind: 'fill'; x: number; y: number; text: string }; + +/** Reuses the session's automation context without starting instrumentation for sessionless input. */ +export async function runAndroidAutomationHelper( + device: DeviceInfo, + action: 'input' | 'screenshot' | 'focus', + input?: AndroidHelperInput, +): Promise | undefined> { + const deviceKey = getAndroidSnapshotHelperSessionDeviceKey(device); + if (!getLiveAndroidSnapshotHelperSession(deviceKey)?.commands.has(action)) return undefined; + const prepared = await prepareAndroidAutomationHelper(device); + try { + return await runAndroidSnapshotHelperSessionTouchCommand({ + deviceKey, + action, + helper: androidAutomationHelperIdentity(prepared.artifact), + payloadBase64: input ? Buffer.from(JSON.stringify(input)).toString('base64') : undefined, + timeoutMs: action === 'focus' ? 2_000 : 30_000, + }); + } catch (error) { + if (action !== 'focus') throw error; + await stopAndroidSnapshotHelperSession(deviceKey); + emitDiagnostic({ + level: 'warn', + phase: 'android_helper_focus_fallback', + data: { + error: error instanceof Error ? error.message : String(error), + }, + }); + return undefined; + } +} diff --git a/packages/platform-android/src/fill-verification.ts b/packages/platform-android/src/fill-verification.ts index ca5a2ae7da..01289f6874 100644 --- a/packages/platform-android/src/fill-verification.ts +++ b/packages/platform-android/src/fill-verification.ts @@ -15,6 +15,8 @@ import { isAndroidInputMethodOwnedNode } from '@agent-device/contracts/android-i import { captureAndroidUiHierarchyXml } from './snapshot.ts'; import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; +import { ANDROID_WINDOW_TYPE_INPUT_METHOD } from './ui-hierarchy-node.ts'; +import type { ElementTextTarget } from '@agent-device/contracts/element-text-runtime'; import type { FillUnconfirmedVerification, FillVerificationTarget, @@ -39,6 +41,7 @@ type AndroidTextAtPointScan = { type AndroidFillVerificationContext = { activeInputMethodPackage?: string | null; + inputMethodPackages?: ReadonlySet; }; /** @@ -57,12 +60,24 @@ export async function verifyAndroidFilledText( expected: string, helper: AndroidHelperSessionOptions = {}, ): Promise { - const context = await readAndroidFillVerificationContext(device); + const deadline = Deadline.fromTimeoutMs(FILL_VERIFICATION_DEADLINE_MS); + let initialXml: string | undefined = await captureAndroidUiHierarchyXml(device, { + ...helper, + skipIdleWait: true, + }); + const context = await readAndroidFillVerificationContext(device, initialXml); const { value } = await pollUntilSettled({ - probe: () => inspectAndroidFilledText(device, x, y, expected, context, helper), + probe: async () => { + if (initialXml !== undefined) { + const xml = initialXml; + initialXml = undefined; + return verifyAndroidFilledTextInHierarchy(xml, x, y, expected, context); + } + return await inspectAndroidFilledText(device, x, y, expected, context, helper); + }, settled: (verification, previous) => verification.reason === 'ime_capture' || (verification.ok && previous?.ok === true), - deadline: Deadline.fromTimeoutMs(FILL_VERIFICATION_DEADLINE_MS), + deadline, intervalMs: FILL_VERIFICATION_SAMPLE_INTERVAL_MS, }); return value; @@ -72,12 +87,13 @@ export async function readAndroidTextAtPoint( device: DeviceInfo, x: number, y: number, - helper: AndroidHelperSessionOptions = {}, + helper: AndroidHelperSessionOptions & { target?: ElementTextTarget } = {}, ): Promise { return readAndroidTextAtPointInHierarchy( await captureAndroidUiHierarchyXml(device, helper), x, y, + helper.target, ); } @@ -87,13 +103,9 @@ async function readAndroidFillTargetAtPoint( y: number, helper: AndroidHelperSessionOptions, ): Promise { - const context = await readAndroidFillVerificationContext(device); - return inspectAndroidTextAtPointInHierarchy( - await captureAndroidUiHierarchyXml(device, helper), - x, - y, - context, - ).targetInput; + const xml = await captureAndroidUiHierarchyXml(device, { ...helper, skipIdleWait: true }); + const context = await readAndroidFillVerificationContext(device, xml); + return inspectAndroidTextAtPointInHierarchy(xml, x, y, context).targetInput; } export async function readAndroidFillTargetBeforeMutation( @@ -213,7 +225,18 @@ export function readAndroidTextAtPointInHierarchy( xml: string, x: number, y: number, + target?: ElementTextTarget, ): string | null { + if (target?.identifier) { + const matches = [...androidUiNodes(xml)].filter( + (node) => + node.resourceId === target.identifier && + (!target.bundleId || node.packageName === target.bundleId) && + (!target.type || node.className === target.type), + ); + if (matches.length !== 1) return null; + return androidFillCandidateFromNode(matches[0]!, {})?.text ?? null; + } // Reads are point-targeted: a focused sibling may be a different app field or an IME // composing surface, so it must not override the node that contains the requested point. return inspectAndroidTextAtPointInHierarchy(xml, x, y).targetInput?.text ?? null; @@ -252,7 +275,7 @@ async function inspectAndroidFilledText( // Each delay samples the live hierarchy again — settling is what the samples observe, so they // share the helper session but never a capture. return verifyAndroidFilledTextInHierarchy( - await captureAndroidUiHierarchyXml(device, helper), + await captureAndroidUiHierarchyXml(device, { ...helper, skipIdleWait: true }), x, y, expected, @@ -272,8 +295,10 @@ function inspectAndroidTextAtPointInHierarchy( anyAtPoint: null, }; - for (const node of androidUiNodes(xml)) { - const candidate = androidFillCandidateFromNode(node, context); + const nodes = [...androidUiNodes(xml)]; + const observedContext = androidFillWindowContext(nodes) ?? context; + for (const node of nodes) { + const candidate = androidFillCandidateFromNode(node, observedContext); if (candidate) updateAndroidTextAtPointScan(scan, candidate, x, y); } @@ -397,11 +422,13 @@ function androidFillCandidateFromNode( rect: node.rect, focused: node.focused ?? false, password: node.password === true, - inputMethodOwned: isAndroidInputMethodOwnedNode({ - packageName: node.packageName, - resourceId: node.resourceId, - activeInputMethodPackage: context.activeInputMethodPackage, - }), + inputMethodOwned: + context.inputMethodPackages?.has(node.packageName ?? '') === true || + isAndroidInputMethodOwnedNode({ + packageName: node.packageName, + resourceId: node.resourceId, + activeInputMethodPackage: context.activeInputMethodPackage, + }), area, editText: isEditTextClass(node.className ?? ''), hintShowing: node.hintShowing === true, @@ -411,7 +438,10 @@ function androidFillCandidateFromNode( async function readAndroidFillVerificationContext( device: DeviceInfo, + xml: string, ): Promise { + const context = androidFillWindowContext([...androidUiNodes(xml)]); + if (context) return context; try { const state = await getAndroidKeyboardState(device); return { activeInputMethodPackage: state.inputMethodPackage }; @@ -427,6 +457,19 @@ async function readAndroidFillVerificationContext( } } +function androidFillWindowContext( + nodes: readonly AndroidUiNodeMetadata[], +): AndroidFillVerificationContext | undefined { + if (!nodes.some((node) => node.windowType !== undefined)) return undefined; + return { + inputMethodPackages: new Set( + nodes + .filter((node) => node.windowType === ANDROID_WINDOW_TYPE_INPUT_METHOD) + .flatMap((node) => (node.packageName ? [node.packageName] : [])), + ), + }; +} + function updateAndroidTextAtPointScan( scan: AndroidTextAtPointScan, candidate: AndroidFillVerificationCandidate, diff --git a/packages/platform-android/src/input-actions.ts b/packages/platform-android/src/input-actions.ts index aa8c3b2800..84795014ff 100644 --- a/packages/platform-android/src/input-actions.ts +++ b/packages/platform-android/src/input-actions.ts @@ -29,6 +29,7 @@ import { runAndroidShell } from './adb.ts'; import { androidAdbResultError, discloseAdbInputDispatch } from './adb-failure.ts'; import { executeAndroidTouchPlan, readAndroidGestureViewportReading } from './touch-executor.ts'; import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; +import { runAndroidAutomationHelper } from './automation-helper.ts'; /** * What a caller may steer on a dispatched input command: cancellation only. The wider executor @@ -44,6 +45,7 @@ export async function pressAndroid( options?: AndroidInputDispatchOptions, ): Promise { try { + if (!options?.signal && (await runAndroidAutomationHelper(device, 'input', { kind: 'tap', x, y }))) return; // Picked field by field so no runtime caller can smuggle `allowFailure: true` past the type. await runAndroidShell(device, ['input', 'tap', x, y], { allowFailure: false, @@ -83,6 +85,7 @@ export async function backAndroid( device: DeviceInfo, options?: AndroidInputDispatchOptions, ): Promise { + if (!options?.signal && (await runAndroidAutomationHelper(device, 'input', { kind: 'key', keyCode: 4 }))) return; await runAndroidShell( device, ['input', 'keyevent', '4'], @@ -91,10 +94,12 @@ export async function backAndroid( } export async function homeAndroid(device: DeviceInfo): Promise { + if (await runAndroidAutomationHelper(device, 'input', { kind: 'key', keyCode: 3 })) return; await runAndroidShell(device, ['input', 'keyevent', '3']); } export async function pressAndroidEnter(device: DeviceInfo): Promise { + if (await runAndroidAutomationHelper(device, 'input', { kind: 'key', keyCode: 66 })) return; await runAndroidShell(device, ['input', 'keyevent', 'ENTER']); } @@ -211,6 +216,7 @@ export async function probeAndroidDisplayRotation( } export async function appSwitcherAndroid(device: DeviceInfo): Promise { + if (await runAndroidAutomationHelper(device, 'input', { kind: 'key', keyCode: 187 })) return; await runAndroidShell(device, ['input', 'keyevent', '187']); } diff --git a/packages/platform-android/src/observation.ts b/packages/platform-android/src/observation.ts index b93bb6bf17..0cb20d8643 100644 --- a/packages/platform-android/src/observation.ts +++ b/packages/platform-android/src/observation.ts @@ -7,6 +7,7 @@ import type { AppStateRuntimeResult } from '@agent-device/contracts/app-state-ru import type { DeviceInfo } from '@agent-device/kernel/device'; import { deviceShellArgv } from '@agent-device/kernel/device-shell'; import { AppError } from '@agent-device/kernel/errors'; +import { runAndroidAutomationHelper } from './automation-helper.ts'; import { ANDROID_FOCUS_MARKERS, ANDROID_FOCUSED_WINDOW_MARKER, @@ -43,6 +44,8 @@ export function createAndroidObservationAdapter( readAppState: async (device) => await readAppState(host, device), readBlockingDialog: async (device) => await readBlockingDialog(host, device), async readAppFocus(device, appBundleId, options = {}) { + const focus = await runAndroidAutomationHelper(device, 'focus'); + if (focus?.focusedPackage === appBundleId && focus.windowType === '1') return true; const read = createDumpReader(host, device); if (options.requireNoBlockingDialog) { const observation = await readBlockingDialog(host, device, read); @@ -110,6 +113,10 @@ async function readBlockingDialog( device: DeviceInfo, read: DumpReader = createDumpReader(host, device), ): Promise { + const focus = await runAndroidAutomationHelper(device, 'focus'); + if (focus?.windowType === '1' && focus.focusedPackage && focus.focusedPackage !== 'android') { + return { status: 'clear' }; + } for (const args of orderedDumps(device, 'blockingDialog')) { const parsed = readAndroidBlockingDialogFocus(await read(args)); if (parsed.focus) return { status: 'dialog', focus: parsed.focus }; diff --git a/packages/platform-android/src/screenshot.ts b/packages/platform-android/src/screenshot.ts index af4803e772..1c70816d4e 100644 --- a/packages/platform-android/src/screenshot.ts +++ b/packages/platform-android/src/screenshot.ts @@ -5,6 +5,7 @@ import type { ShellWord } from '@agent-device/kernel/device-shell'; import { runAndroidExecOut, runAndroidShell, sleep } from './adb.ts'; import { requireAndroidAdbHost } from './adb-host.ts'; import { probeAndroidDisplayRotation } from './input-actions.ts'; +import { runAndroidAutomationHelper } from './automation-helper.ts'; // PNG file signature: 0x89 P N G \r \n 0x1A \n const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); @@ -17,6 +18,7 @@ const ANDROID_SCREENSHOT_ROTATION_GRACE_MS = 250; export type AndroidScreenshotOptions = { stabilize?: boolean; + normalizeStatusBar?: boolean; }; export async function screenshotAndroid( @@ -24,7 +26,7 @@ export async function screenshotAndroid( outPath: string, options: AndroidScreenshotOptions = {}, ): Promise { - if (options.stabilize === false) { + if (!(options.stabilize ?? options.normalizeStatusBar ?? false)) { return await captureAndroidScreenshotWithRotation(device, outPath); } @@ -98,30 +100,28 @@ async function settledWithin(pending: Promise, graceMs: number): Promise { - const result = await runAndroidExecOut(device, ['screencap', '-p'], { - binaryStdout: true, - }); - if (!result.stdoutBuffer) { + const helper = await runAndroidAutomationHelper(device, 'screenshot'); + const bytes = helper + ? Buffer.from(helper.payloadBase64 ?? '', 'base64') + : (await runAndroidExecOut(device, ['screencap', '-p'], { binaryStdout: true })).stdoutBuffer; + if (!bytes) { throw new AppError('COMMAND_FAILED', 'Failed to capture screenshot'); } // On multi-display devices (e.g. Galaxy Z Fold), adb screencap may write a // warning to stdout before the PNG data. Strip any leading garbage by // locating the PNG signature and discarding everything before it. - const pngOffset = result.stdoutBuffer.indexOf(PNG_SIGNATURE); + const pngOffset = bytes.indexOf(PNG_SIGNATURE); if (pngOffset < 0) { throw new AppError('COMMAND_FAILED', 'Screenshot data does not contain a valid PNG header'); } - const pngEndOffset = findPngEndOffset(result.stdoutBuffer, pngOffset); + const pngEndOffset = findPngEndOffset(bytes, pngOffset); if (!pngEndOffset) { throw new AppError('COMMAND_FAILED', 'Screenshot data does not contain a complete PNG payload'); } - await requireAndroidAdbHost().files.writeBytes( - outPath, - result.stdoutBuffer.subarray(pngOffset, pngEndOffset), - ); + await requireAndroidAdbHost().files.writeBytes(outPath, bytes.subarray(pngOffset, pngEndOffset)); } function findPngEndOffset(buffer: Buffer, pngStartOffset: number): number | null { diff --git a/packages/platform-android/src/snapshot-helper-capture.ts b/packages/platform-android/src/snapshot-helper-capture.ts index 6b69039b15..7443fbc083 100644 --- a/packages/platform-android/src/snapshot-helper-capture.ts +++ b/packages/platform-android/src/snapshot-helper-capture.ts @@ -65,11 +65,17 @@ export async function captureAndroidSnapshotWithHelper( }); let result: Awaited>; try { - result = await options.adb(buildAndroidSnapshotHelperArgs(resolved), { - allowFailure: true, - timeoutMs: resolved.commandTimeoutMs, - signal: options.signal, - }); + result = await options.adb( + buildAndroidSnapshotHelperArgs({ + ...resolved, + ...(options.skipIdleWait ? { waitForIdleTimeoutMs: 0 } : {}), + }), + { + allowFailure: true, + timeoutMs: resolved.commandTimeoutMs, + signal: options.signal, + }, + ); } catch (error) { if (!options.signal?.aborted) throw error; await retireCanceledAndroidSnapshotHelperCapture({ diff --git a/packages/platform-android/src/snapshot-helper-session-lifecycle.ts b/packages/platform-android/src/snapshot-helper-session-lifecycle.ts index fddfa69945..853e3c0640 100644 --- a/packages/platform-android/src/snapshot-helper-session-lifecycle.ts +++ b/packages/platform-android/src/snapshot-helper-session-lifecycle.ts @@ -86,6 +86,7 @@ export type AndroidSnapshotHelperSession = { process: AndroidAdbProcess; startedAtMs: number; capturedCount: number; + commands: ReadonlySet; }; /** A session this caller may run commands on, with the budgets it was started under. */ @@ -312,13 +313,14 @@ async function startAndroidSnapshotHelperSession(params: { process: childProcess, startedAtMs: Date.now(), capturedCount: 0, + commands: new Set(), }; try { // A helper that announces itself late is a slow `am instrument`, which the one-shot transport it // falls back to pays too, so the wait gets a share of the helper-command budget rather than a // smaller guess. The caller's own deadline reaches it as an abort on `options.signal`, which is // what bounds this below the budget when the command itself is short. - await waitForAndroidSnapshotHelperSessionReady( + session.commands = await waitForAndroidSnapshotHelperSessionReady( childProcess, params.startBudgetMs, params.options.signal, diff --git a/packages/platform-android/src/snapshot-helper-session-protocol.ts b/packages/platform-android/src/snapshot-helper-session-protocol.ts index 23a06863b7..fe3d436714 100644 --- a/packages/platform-android/src/snapshot-helper-session-protocol.ts +++ b/packages/platform-android/src/snapshot-helper-session-protocol.ts @@ -48,7 +48,7 @@ export function sendAndroidSnapshotHelperSessionCommand( socket.destroy(); reject( new AppError('COMMAND_FAILED', 'Android snapshot helper session request timed out', { - command, + command: command.split(' ', 1)[0], timeoutMs, port, }), @@ -85,6 +85,7 @@ export async function requestAndroidSnapshotHelperSessionSnapshot(params: { port: number; commandTimeoutMs: number; signal?: AbortSignal; + skipIdleWait?: boolean; }): Promise { const requestId = `snapshot-${Date.now()}-${Math.random().toString(16).slice(2)}`; // The socket waits out the whole command budget the session was given. The device-side @@ -92,7 +93,7 @@ export async function requestAndroidSnapshotHelperSessionSnapshot(params: { // built from that wait plus a fixed overhead cut off slow-but-healthy captures (tester-army/e2e#841). const response = await sendAndroidSnapshotHelperSessionCommand( params.port, - `snapshot ${requestId}`, + `${params.skipIdleWait ? 'snapshot-immediate' : 'snapshot'} ${requestId}`, params.commandTimeoutMs, params.signal, ); @@ -106,7 +107,7 @@ export function waitForAndroidSnapshotHelperSessionReady( childProcess: AndroidAdbProcess, timeoutMs: number, signal?: AbortSignal, -): Promise { +): Promise> { return new Promise((resolve, reject) => { let output = ''; const timer = setTimeout(() => { @@ -123,11 +124,13 @@ export function waitForAndroidSnapshotHelperSessionReady( output += chunk.toString(); if ( output.includes(`agentDeviceProtocol=${ANDROID_SNAPSHOT_HELPER_PROTOCOL}`) && - output.includes('sessionReady=true') + output.includes('sessionReady=true') && + output.includes('INSTRUMENTATION_STATUS_CODE: 2') ) { clearTimeout(timer); signal?.removeEventListener('abort', onAbort); - resolve(); + const commands = output.match(/INSTRUMENTATION_STATUS: sessionCommands=([^\r\n]*)\r?\n/); + resolve(new Set(commands?.[1]?.split(',').filter(Boolean))); } }; const onAbort = () => { diff --git a/packages/platform-android/src/snapshot-helper-session.ts b/packages/platform-android/src/snapshot-helper-session.ts index 8829c8889e..cd157302e5 100644 --- a/packages/platform-android/src/snapshot-helper-session.ts +++ b/packages/platform-android/src/snapshot-helper-session.ts @@ -1,6 +1,5 @@ /** - * Commands that run OVER a live automation-helper session: snapshot capture and the touch commands - * that piggyback on it. Session ownership itself — starting, reusing, retiring — belongs to + * Commands that run over a live automation-helper session: acquisition, input, and focus reads. Session ownership itself — starting, reusing, retiring — belongs to * `snapshot-helper-session-lifecycle.ts`, which this module acquires through. */ import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; @@ -57,6 +56,7 @@ async function captureFromAndroidSnapshotHelperSession(params: { port: session.port, commandTimeoutMs: resolved.commandTimeoutMs, signal: options.signal, + skipIdleWait: options.skipIdleWait && session.commands.has('snapshot-immediate'), }); session.capturedCount += 1; return { @@ -97,11 +97,11 @@ async function captureFromAndroidSnapshotHelperSession(params: { // (a second instrumentation for the helper package would force-stop the session that has it). They // never start a session: without one, callers use the same helper APK through a one-shot // `am instrument` run instead. -export type AndroidTouchHelperAction = 'gesture' | 'viewport'; +export type AndroidTouchHelperAction = 'gesture' | 'viewport' | 'input' | 'screenshot' | 'focus'; /** - * Only a gesture injects input, so only its failures say whether input reached the device. Its - * failure is `unknown` unless the host adb refused the launch: an `ok=false` result carries the + * Gestures and input commands mutate the app, so their failures disclose whether input reached it. A + * mutation failure is `unknown` unless the host adb refused the launch: an `ok=false` result carries the * thrown `errorType` but no count of events injected before the throw, so a parse refusal and a * mid-injection failure read alike. */ @@ -109,7 +109,7 @@ export function discloseHelperTouchDispatch( action: AndroidTouchHelperAction, error: unknown, ): unknown { - if (action !== 'gesture' || !(error instanceof AppError)) return error; + if ((action !== 'gesture' && action !== 'input') || !(error instanceof AppError)) return error; return discloseDispatch(error, isAdbHostRefusal(error) ? 'no' : 'unknown'); } diff --git a/packages/platform-android/src/snapshot-helper-types.ts b/packages/platform-android/src/snapshot-helper-types.ts index 739b692e79..70346dbd57 100644 --- a/packages/platform-android/src/snapshot-helper-types.ts +++ b/packages/platform-android/src/snapshot-helper-types.ts @@ -78,6 +78,7 @@ export type AndroidSnapshotHelperCaptureOptions = { maxDepth?: number; maxNodes?: number; emitChunks?: boolean; + skipIdleWait?: boolean; }; export type AndroidSnapshotHelperMetadata = { diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index e7f559b0b4..b4fdce54ea 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -102,6 +102,7 @@ const HELPER_CONTENT_RECAPTURE_DELAY_MS = 250; */ const HELPER_CONTENT_RECAPTURE_WINDOW_MS = 10_000; export type AndroidSnapshotOptions = SnapshotOptions & { + skipIdleWait?: boolean; appBundleId?: string; signal?: AbortSignal; helperArtifact?: AndroidSnapshotHelperArtifact; @@ -442,6 +443,7 @@ async function installAndroidSnapshotHelper( } async function captureAndroidUiHierarchyFromHelper(params: { + skipIdleWait?: boolean; signal?: AbortSignal; adb: AndroidAdbExecutor; adbProvider: AndroidAdbProvider; @@ -456,6 +458,7 @@ async function captureAndroidUiHierarchyFromHelper(params: { deviceKey: helperDeviceKey, signal, }); + captureOptions.skipIdleWait = params.skipIdleWait; try { const sessionCapture = await withDiagnosticTimer( 'android_snapshot_helper_session_capture', @@ -550,6 +553,7 @@ async function captureAndroidHelperContentAttempt(params: { await stopAndroidSnapshotHelperSession(helperDeviceKey); } const capture = await captureAndroidUiHierarchyFromHelper({ + skipIdleWait: options.skipIdleWait, signal: options.signal, adb, adbProvider, diff --git a/packages/platform-android/src/text-input.ts b/packages/platform-android/src/text-input.ts index 5bf56ae143..2cf2b17e05 100644 --- a/packages/platform-android/src/text-input.ts +++ b/packages/platform-android/src/text-input.ts @@ -36,6 +36,7 @@ import { getAndroidTestImeOwnership } from './ime-state.ts'; import { discloseAdbInputDispatch } from './adb-failure.ts'; import { focusAndroid } from './input-actions.ts'; import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; +import { runAndroidAutomationHelper } from './automation-helper.ts'; /** * `input text` truncates long strings in some app/IME states (#531), so ASCII is written in short @@ -84,6 +85,14 @@ export async function fillAndroid( const verification = await verifyAndroidFilledText(device, x, y, text, helper); return completeAndroidFillVerification(text, beforeTarget, verification); } + if ( + delayMs === 0 && + (await runAndroidAutomationHelper(device, 'input', { kind: 'fill', x, y, text })) + ) { + emitAndroidTextDiagnostic('fill', 'android-helper', text); + const verification = await verifyAndroidFilledText(device, x, y, text, helper); + return completeAndroidFillVerification(text, beforeTarget, verification); + } let lastVerification: AndroidFillVerification | null = null; let dispatchedSteps = 0; try { @@ -530,7 +539,7 @@ function chunkAndroidInputText(text: string, chunkSize: number): string[] { function emitAndroidTextDiagnostic( action: AndroidTextInputAction, - backend: 'provider-native' | 'adb-shell' | 'test-ime', + backend: 'provider-native' | 'adb-shell' | 'test-ime' | 'android-helper', text: string, ): void { emitDiagnostic({ diff --git a/packages/platform-android/src/touch-helper.ts b/packages/platform-android/src/touch-helper.ts index 4a62c3b195..ef7a69ba14 100644 --- a/packages/platform-android/src/touch-helper.ts +++ b/packages/platform-android/src/touch-helper.ts @@ -1,3 +1,7 @@ +import { + prepareAndroidAutomationHelper, + androidAutomationHelperIdentity, +} from './automation-helper-artifact.ts'; import type { PointerTrajectory } from '@agent-device/contracts/gesture-plan-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { Rect } from '@agent-device/kernel/snapshot'; @@ -6,12 +10,7 @@ import { attachAdbFailureHint } from './adb-failure.ts'; import { execFailureDetails } from '@agent-device/host-kit/command'; import { emitDiagnostic, withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; -import { - resolveAndroidAdbProvider, - runAdbShell, - type AndroidAdbExecutor, - type AndroidAdbProvider, -} from './adb-executor.ts'; +import { runAdbShell, type AndroidAdbExecutor } from './adb-executor.ts'; import { parseInstrumentationRecords, readInstrumentationResultNumber, @@ -21,9 +20,6 @@ import { type AndroidGestureViewportReading, } from './gesture-viewport.ts'; import type { AndroidLoweredTouchPlan } from './touch-plan-lowering.ts'; -import { resolveAndroidHelperArtifact } from './helper-package-install.ts'; -import { parseAndroidSnapshotHelperManifest } from './snapshot-helper-artifact.ts'; -import { ensureAndroidSnapshotHelper } from './snapshot-helper-install.ts'; import { discloseHelperTouchDispatch, runAndroidSnapshotHelperSessionTouchCommand, @@ -33,23 +29,16 @@ import { ensureAndroidSnapshotHelperSession, stopAndroidSnapshotHelperSession, } from './snapshot-helper-session-lifecycle.ts'; -import { - getAndroidSnapshotHelperSessionDeviceKey, - recoverAndroidSnapshotHelperRetirement, -} from './snapshot-helper-retirement.ts'; import { buildAndroidSnapshotHelperCaptureOptions } from './snapshot-helper-capture.ts'; import { ANDROID_SNAPSHOT_HELPER_PROTOCOL, type AndroidHelperSessionOptions, - type AndroidSnapshotHelperArtifact, - type AndroidSnapshotHelperInstallResult, } from './snapshot-helper-types.ts'; export const ANDROID_TOUCH_PLAN_PROTOCOL = 'android-touch-plan-v1'; const HELPER_GESTURE_TIMEOUT_MS = 45_000; const HELPER_GESTURE_TIMEOUT_OVERHEAD_MS = 15_000; const HELPER_VIEWPORT_TIMEOUT_MS = 45_000; -const HELPER_INSTALL_TIMEOUT_MS = 30_000; const HELPER_NO_FINAL_RESULT = 'ANDROID_TOUCH_HELPER_NO_FINAL_RESULT'; const HELPER_REPORTED_FAILURE = 'ANDROID_TOUCH_HELPER_REPORTED_FAILURE'; @@ -64,19 +53,11 @@ type AndroidTouchHelperGestureRequest = { pointers: AndroidPlannedPointerTrajectory[]; }; -type PreparedAndroidTouchHelper = { - adb: AndroidAdbExecutor; - adbProvider: AndroidAdbProvider; - artifact: AndroidSnapshotHelperArtifact; - install: AndroidSnapshotHelperInstallResult; - deviceKey: string; -}; - export async function executeAndroidTouchHelperPlan( device: DeviceInfo, plan: AndroidLoweredTouchPlan, ): Promise> { - const prepared = await prepareAndroidTouchHelper(device); + const prepared = await prepareAndroidAutomationHelper(device); const request = normalizeAndroidTouchHelperGestureRequest(plan); const timeoutMs = Math.max( HELPER_GESTURE_TIMEOUT_MS, @@ -94,7 +75,7 @@ export async function executeAndroidTouchHelperPlan( const sessionHeaders = await runAndroidSnapshotHelperSessionTouchCommand({ deviceKey: prepared.deviceKey, action: 'gesture', - helper: touchSessionHelperIdentity(prepared.artifact), + helper: androidAutomationHelperIdentity(prepared.artifact), payloadBase64, timeoutMs, }); @@ -130,7 +111,7 @@ export async function readAndroidTouchHelperViewportReading( device: DeviceInfo, helper: AndroidHelperSessionOptions = {}, ): Promise { - const prepared = await prepareAndroidTouchHelper(device); + const prepared = await prepareAndroidAutomationHelper(device); if (helper.helperSessionScope === 'daemon-session') { // Without a live session both this read and the gesture that follows would each start their // own `am instrument`. The session outlives the command under this scope, so warming it here @@ -148,7 +129,7 @@ export async function readAndroidTouchHelperViewportReading( const sessionHeaders = await runAndroidSnapshotHelperSessionTouchCommand({ deviceKey: prepared.deviceKey, action: 'viewport', - helper: touchSessionHelperIdentity(prepared.artifact), + helper: androidAutomationHelperIdentity(prepared.artifact), timeoutMs: HELPER_VIEWPORT_TIMEOUT_MS, }); if (sessionHeaders) return readViewportResult(sessionHeaders); @@ -173,79 +154,6 @@ export async function readAndroidTouchHelperViewportReading( }); } -// Sessions are keyed by device, so reuse must also prove the live session runs the helper binary -// this command selected; a mismatch stops the session and the command falls back to one-shot. -function touchSessionHelperIdentity(artifact: AndroidSnapshotHelperArtifact): { - packageName: string; - runner: string; - helperVersion: string; - helperVersionCode: number; - sha256: string; -} { - return { - packageName: artifact.manifest.packageName, - runner: artifact.manifest.instrumentationRunner, - helperVersion: artifact.manifest.version, - helperVersionCode: artifact.manifest.versionCode, - sha256: artifact.manifest.sha256, - }; -} - -async function prepareAndroidTouchHelper(device: DeviceInfo): Promise { - const adbProvider = resolveAndroidAdbProvider(device); - // Same artifact precedence as snapshot capture: a provider-supplied helper artifact overrides - // the bundled one so both transports install and run the single shared helper (issue #1275). - const artifact = - adbProvider.snapshotHelperArtifact ?? (await resolveAndroidTouchHelperArtifact()); - const deviceKey = getAndroidSnapshotHelperSessionDeviceKey(device); - await recoverAndroidSnapshotHelperRetirement({ - deviceKey, - adb: adbProvider.exec, - }); - const install = await withDiagnosticTimer( - 'android_touch_helper_install', - async () => - await ensureAndroidSnapshotHelper({ - adb: adbProvider.exec, - adbProvider, - artifact, - deviceKey, - timeoutMs: HELPER_INSTALL_TIMEOUT_MS, - }), - { - packageName: artifact.manifest.packageName, - versionCode: artifact.manifest.versionCode, - }, - ); - emitDiagnostic({ - phase: 'android_touch_helper_install_decision', - data: install, - }); - if (install.installed) { - // An APK replacement kills the running instrumentation, so a persistent session started - // against the previous binary must not serve this command; the next snapshot restarts it. - await stopAndroidSnapshotHelperSession(deviceKey); - } - return { adb: adbProvider.exec, adbProvider, artifact, install, deviceKey }; -} - -async function resolveAndroidTouchHelperArtifact(): Promise { - return await resolveAndroidHelperArtifact({ - helperDirName: 'snapshot-helper', - manifestFileName: (version) => `agent-device-android-snapshot-helper-${version}.manifest.json`, - parseManifest: (value) => { - const manifest = parseAndroidSnapshotHelperManifest(value); - return { - ...manifest, - assetName: - manifest.assetName ?? `agent-device-android-snapshot-helper-${manifest.version}.apk`, - }; - }, - unavailableMessage: - 'Android touch gestures require the bundled Android automation helper artifact, but it was not found or could not be read', - }); -} - export function normalizeAndroidTouchHelperGestureRequest( plan: AndroidLoweredTouchPlan, ): AndroidTouchHelperGestureRequest { diff --git a/scripts/bench-android-automation.mjs b/scripts/bench-android-automation.mjs new file mode 100644 index 0000000000..7eae21bbfa --- /dev/null +++ b/scripts/bench-android-automation.mjs @@ -0,0 +1,94 @@ +import assert from 'node:assert/strict'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const checkout = path.resolve(process.argv[2] ?? '.'); +const output = path.resolve(process.argv[3] ?? '.tmp/android-automation-benchmark.json'); +await mkdir(path.dirname(output), { recursive: true }); +const rounds = Number(process.env.BENCH_ROUNDS ?? 20); +assert(Number.isInteger(rounds) && rounds > 0, 'BENCH_ROUNDS must be a positive integer'); +const serial = process.env.ANDROID_SERIAL; +assert(serial, 'Set ANDROID_SERIAL to an exclusively owned emulator'); +const { createAgentDeviceClient } = await import( + pathToFileURL(path.join(checkout, 'dist/src/index.js')) +); +const client = createAgentDeviceClient({ + session: 'android-automation-benchmark', + stateDir: `${output}.state`, + platform: 'android', + serial, +}); +const samples = {}; +async function measure(name, run) { + const start = performance.now(); + await run(); + (samples[name] ??= []).push(performance.now() - start); +} +async function email() { + const snapshot = await client.capture.snapshot({ interactiveOnly: true }); + const node = snapshot.nodes.find( + (n) => n.identifier === 'field-email' || n.identifier?.endsWith(':id/field-email'), + ); + assert(node?.ref, 'The current test-app Checkout form must expose field-email'); + return { ref: `@${node.ref}~s${snapshot.refsGeneration}` }; +} +let opened = false; +try { + await measure('open', async () => { + await client.apps.open({ + app: 'com.callstack.agentdevicelab', + url: 'agent-device-test-app:///form', + foreground: true, + }); + opened = true; + }); + await email(); + for (let i = 0; i < 3; i++) { + await measure('restart', () => + client.apps.open({ + app: 'com.callstack.agentdevicelab', + url: 'agent-device-test-app:///form', + relaunch: true, + }), + ); + } + await email(); + for (let i = 0; i < rounds; i++) + await measure('observe', () => client.capture.snapshot({ interactiveOnly: true })); + for (let i = 0; i < rounds; i++) { + const target = await email(); + await measure('tap', () => client.interactions.press(target)); + } + for (let i = 0; i < rounds; i++) { + const target = await email(); + const text = `user${String(i).padStart(2, '0')}@example.test`; + await measure('fill', () => client.interactions.fill({ ...target, text })); + const result = await client.interactions.get({ format: 'text', selector: 'id="field-email"' }); + assert.equal(result.text, text, 'A fast fill must actually reach the selected field'); + } + for (let i = 0; i < Math.min(rounds, 10); i++) { + await measure('screenshot', () => client.capture.screenshot({ path: `${output}.${i}.png` })); + } +} finally { + if (opened) await client.apps.close(); +} +const percentile = (values, p) => + values.toSorted((a, b) => a - b)[Math.ceil(values.length * p) - 1]; +const summary = Object.fromEntries( + Object.entries(samples).map(([name, values]) => [ + name, + { + count: values.length, + p50: percentile(values, 0.5), + p95: percentile(values, 0.95), + }, + ]), +); +await mkdir(path.dirname(output), { recursive: true }); +const version = JSON.parse(await readFile(path.join(checkout, 'package.json'), 'utf8')).version; +await writeFile( + output, + `${JSON.stringify({ observedAt: new Date().toISOString(), checkout, version, serial, rounds, summary, samples }, null, 2)}\n`, +); +console.log(JSON.stringify(summary, null, 2)); diff --git a/src/commands/capture/screenshot-options.test.ts b/src/commands/capture/screenshot-options.test.ts index fb365809bd..254e65ebcc 100644 --- a/src/commands/capture/screenshot-options.test.ts +++ b/src/commands/capture/screenshot-options.test.ts @@ -151,3 +151,17 @@ test('screenshot script flags use the shared recorded flag contract', () => { 'screenshotNormalizeStatusBar', ]); }); + +test('explicit screenshot stabilization survives SDK projection and script recording', () => { + const projected = screenshotFlagsFromOptions({ stabilize: true }); + assert.deepEqual(screenshotOptionsFromFlags(projected), { stabilize: true }); + const parts: string[] = []; + appendScreenshotScriptFlags(parts, projected); + assert.deepEqual(parts, ['--no-stabilize=false']); + const recorded: Partial = {}; + assert.deepEqual(readScreenshotScriptFlag({ args: parts, index: 0, flags: recorded }), { + handled: true, + nextIndex: 0, + }); + assert.deepEqual(screenshotOptionsFromFlags(recorded), { stabilize: true }); +}); diff --git a/src/commands/capture/screenshot.ts b/src/commands/capture/screenshot.ts index 1843d4b50c..a8149b92e9 100644 --- a/src/commands/capture/screenshot.ts +++ b/src/commands/capture/screenshot.ts @@ -74,7 +74,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.", + "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.", }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index b9bb6da9d1..0f3cc2f431 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -90,11 +90,11 @@ export function createAndroidInteractor( performGesture: (plan) => executeAndroidTouchPlan(device, plan), gestureViewport: () => readAndroidGestureViewport(device, { helperSessionScope }), screenshot: (outPath, options) => screenshotAndroid(device, outPath, options), - // uiautomator reads the node covering a point; `undefined` means nothing covers it. readTextAtPoint: async (point, options) => { const { readAndroidTextAtPoint } = await import('@agent-device/platform-android/mechanics'); const read = await readAndroidTextAtPoint(device, point.x, point.y, { helperSessionScope: androidHelperSessionScope(options?.appBundleId), + target: options?.target, }); return read ?? undefined; }, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 85e97f3b35..d34e6c3b85 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -962,7 +962,7 @@ agent-device screenshot page.png # Explicit screenshot path agent-device screenshot page.png --scale 0.3 # Resize both dimensions to 30% for agent context agent-device screenshot page.png --overlay-refs # Draw current @eN refs and target rectangles onto the PNG agent-device screenshot page.png --crop-on 'label="Save"' # Crop the capture to the frame the selector resolves on the same screen -agent-device screenshot baseline.png --normalize-status-bar # Normalize iOS simulator chrome for reusable diff baselines +agent-device screenshot baseline.png --normalize-status-bar # Normalize iOS simulator or Android chrome for reusable diff baselines agent-device screenshot page.png --platform web --fullscreen # On web, --fullscreen/--full/-f captures the entire document agent-device viewport 1280 900 --platform web # Resize the active web viewport for fixed-layout or 100vh apps agent-device screenshot textedit.png # App-session window capture on macOS @@ -983,6 +983,7 @@ agent-device record stop # Stop active recording **Screenshots** - `--scale ` resizes both dimensions, from `0.01` to `1` (full resolution). Set `AGENT_DEVICE_SCREENSHOT_SCALE=0.3` or `screenshotScale` in config as a token-conscious default; an explicit `--scale` overrides it. Leave it unset or pass `--scale 1` for pixel-diff baselines. `--max-size` is refused everywhere (CLI, `.ad` scripts, Node options, config, `AGENT_DEVICE_SCREENSHOT_MAX_SIZE`) with guidance to use `--scale`. +- Android screenshots capture immediately by default. Use `--normalize-status-bar` to set deterministic status-bar chrome and wait for transient UI to fade. In the Node client, `stabilize: true` also enables this wait. - `--overlay-refs` takes a fresh snapshot and burns visible `@eN` refs and their rectangles into the PNG. With `--scale`, refs are drawn at the final size, so avoid scales where labels become unreadable. In `--json` mode each ref also has a screenshot-space `center` for `press ` fallback. - `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators and Android emulators, needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. - `--fullscreen` on macOS applies only to app sessions. Other surfaces (`desktop`, `menubar`, `frontmost-app`) always capture the main display and refuse `--fullscreen` with `INVALID_ARGS`. From 77044ad5ee187aec163c055e9d00e174c0420a40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 10:44:04 +0200 Subject: [PATCH 2/9] refactor(contracts): isolate live text target identity --- packages/contracts/src/element-text-runtime.ts | 7 +++---- packages/contracts/src/element-text-target.ts | 4 ++++ packages/contracts/src/interactor-types.ts | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 packages/contracts/src/element-text-target.ts diff --git a/packages/contracts/src/element-text-runtime.ts b/packages/contracts/src/element-text-runtime.ts index a96a4e26d9..dce162cb80 100644 --- a/packages/contracts/src/element-text-runtime.ts +++ b/packages/contracts/src/element-text-runtime.ts @@ -1,14 +1,13 @@ +import type { ElementTextTarget } from './element-text-target.ts'; +export type { ElementTextTarget } from './element-text-target.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import type { Point, SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { Point } from '@agent-device/kernel/snapshot'; import type { Interactor, RunnerContext } from './interactor-types.ts'; import { invalidRuntimeContract } from './runtime-contract-error.ts'; import type { RuntimeOperationFact } from './platform-runtime.ts'; import type { SessionSurface } from './session-surface.ts'; import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; -/** Identity carried from a resolved node into its live text read. */ -export type ElementTextTarget = Pick; - /** A live text read at a resolved point, with optional identity to preserve the selected control. */ export type ReadTextAtPointInput = Readonly<{ point: Point; diff --git a/packages/contracts/src/element-text-target.ts b/packages/contracts/src/element-text-target.ts new file mode 100644 index 0000000000..6c66a6a214 --- /dev/null +++ b/packages/contracts/src/element-text-target.ts @@ -0,0 +1,4 @@ +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; + +/** Identity carried from a resolved node into its live text read. */ +export type ElementTextTarget = Pick; diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index e0660d0223..a17438a9aa 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -1,5 +1,5 @@ import type { AppStateRuntimeResult } from './app-state-runtime.ts'; -import type { ElementTextTarget } from './element-text-runtime.ts'; +import type { ElementTextTarget } from './element-text-target.ts'; import type { BackMode } from './back-mode.ts'; import type { IosSystemSurfaceProvenance } from './ios-system-surface.ts'; import type { DeviceRotation } from './device-rotation.ts'; From 5f06b79b7567991b727cf5453f35c862f13ae518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 10:48:05 +0200 Subject: [PATCH 3/9] chore(bench): expose Android automation benchmark command --- docs/evidence/android-automation-performance-2026-10-19.md | 2 +- package.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/evidence/android-automation-performance-2026-10-19.md b/docs/evidence/android-automation-performance-2026-10-19.md index 10b6bbcdea..e58659d225 100644 --- a/docs/evidence/android-automation-performance-2026-10-19.md +++ b/docs/evidence/android-automation-performance-2026-10-19.md @@ -15,7 +15,7 @@ Build each checkout and its Android helpers separately. Run the variants in sequ ```sh ANDROID_SERIAL=emulator-5554 BENCH_ROUNDS=20 \ - node scripts/bench-android-automation.mjs /absolute/checkout /private/tmp/android-benchmark.json + pnpm bench:android:automation /absolute/checkout /private/tmp/android-benchmark.json ``` The script uses the public SDK and the Checkout form's `field-email`. diff --git a/package.json b/package.json index a248112e84..41f384ed90 100644 --- a/package.json +++ b/package.json @@ -123,6 +123,7 @@ "package:npm": "pnpm build:package && pnpm check:package", "release:prepare": "rm -rf .tmp/release && pnpm check:mcp-metadata && pnpm build:package && pnpm check:package -- --pack-destination .tmp/release && node scripts/release-workspace.mjs pack", "ad": "node bin/agent-device.mjs", + "bench:android:automation": "node scripts/bench-android-automation.mjs", "bench:help-conformance": "node scripts/help-conformance-bench.mjs", "maestro:conformance": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/maestro-conformance/format-generated-json.test.mjs packages/maestro/test/conformance/verify.test.ts packages/maestro/test/conformance/differential/engine-process.test.ts packages/maestro/test/conformance/differential/report-output.test.ts packages/maestro/test/conformance/differential/run.test.ts packages/maestro/test/conformance/differential/invariants.test.ts", "maestro:conformance:regenerate": "node --experimental-strip-types scripts/maestro-conformance/regenerate.mjs", From e01211046ab91af3f87265a033cbc08641bf3fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 10:58:40 +0200 Subject: [PATCH 4/9] fix(android): preserve lazy helper loading and align regression fixtures --- .../evidence/android-automation-performance-2026-10-19.md | 2 ++ .../internal/interaction-dispatch-disclosure.test.ts | 1 + .../src/__tests__/snapshot-helper-session.fixtures.ts | 4 ++-- packages/platform-android/src/__tests__/snapshot.test.ts | 4 ++-- packages/platform-android/src/input-actions.ts | 6 +++++- packages/platform-android/src/observation.ts | 3 ++- packages/platform-android/src/screenshot.ts | 2 +- packages/platform-android/src/text-input.ts | 2 +- packages/platform-android/src/touch-helper.ts | 8 ++++---- src/commands/schema/cli-help-command-usage.test.ts | 4 ++-- .../provider-scenarios/android-lifecycle.test.ts | 2 +- .../provider-scenarios/android-tv-remote.test.ts | 7 +++++++ 12 files changed, 30 insertions(+), 15 deletions(-) diff --git a/docs/evidence/android-automation-performance-2026-10-19.md b/docs/evidence/android-automation-performance-2026-10-19.md index e58659d225..7728b95e56 100644 --- a/docs/evidence/android-automation-performance-2026-10-19.md +++ b/docs/evidence/android-automation-performance-2026-10-19.md @@ -63,3 +63,5 @@ Full-suite investigation exposed a pre-existing Android live-text bug on both ba An early native-fill implementation omitted the keyboard-opening tap; the lifecycle scenario detected it and the focus tap was restored. One checkout submission miss occurred before the final synchronization and freshness hardening; six isolated checkout replays passed, so its cause was not established. The final combined suite passed, including every replay without retries. Input-socket and screenshot-default regressions failed on unchanged production code (`3 failed, 17 passed`) and passed with the implementation. The focus-path regression also failed on unchanged base. The text-identity regression was observed red and green at its owning hierarchy seam, and its contract and daemon forwarding tests passed. + +The broad unit selection initially exposed stale screenshot expectations, an inconsistent foreground fixture, and added eager imports. Their fixes passed 1,016 focused tests. The Android TV provider timeout also reproduced on unchanged base: its accepted IME installation was followed by an empty registration list. The fixture now reports the installed service; both Android provider scenario files passed seven tests without a timeout increase. diff --git a/packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts index 90eefb5662..5e6df54486 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts @@ -336,6 +336,7 @@ async function pressThenForegroundReadRefused(): Promise { const readRefusal = new AppError('COMMAND_FAILED', 'adb device offline', { dispatched: 'no' }); const androidObservation: AndroidObservationAdapter = { ...clearAndroidObservationFixture, + readAppFocus: async () => false, readAppState: async () => { if (mockTapPoint.mock.calls.length > 0) throw readRefusal; return { package: 'com.example.app' }; diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts b/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts index e82803e1cd..f97ac254b8 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts @@ -146,7 +146,7 @@ export function createPersistentSnapshotHelperProvider( process.stdout.write( [ 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', - 'INSTRUMENTATION_STATUS: sessionCommands=input,screenshot,focus,snapshot-immediate', + `INSTRUMENTATION_STATUS: sessionCommands=${options.sessionInput ? 'input,' : ''}snapshot-immediate`, 'INSTRUMENTATION_STATUS: sessionReady=true', 'INSTRUMENTATION_STATUS_CODE: 2', '', @@ -280,7 +280,7 @@ export function createSessionProvider(options: SessionProviderOptions): AndroidA process.stdout.write( [ 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', - 'INSTRUMENTATION_STATUS: sessionCommands=input,screenshot,focus,snapshot-immediate', + 'INSTRUMENTATION_STATUS: sessionCommands=snapshot-immediate', 'INSTRUMENTATION_STATUS: sessionReady=true', 'INSTRUMENTATION_STATUS_CODE: 2', '', diff --git a/packages/platform-android/src/__tests__/snapshot.test.ts b/packages/platform-android/src/__tests__/snapshot.test.ts index 3d40f55f2b..15d063a768 100644 --- a/packages/platform-android/src/__tests__/snapshot.test.ts +++ b/packages/platform-android/src/__tests__/snapshot.test.ts @@ -112,11 +112,11 @@ afterEach(async () => { await resetAndroidSnapshotHelperSessions(); }); -test('screenshotAndroid waits for transient UI to settle before capture', async () => { +test('screenshotAndroid waits for transient UI when stabilization is requested', async () => { const events: string[] = []; await withTempScreenshot('screenshot-settle-', async (outPath) => { mockScreenshotEvents(events); - await screenshotAndroid(device, outPath); + await screenshotAndroid(device, outPath, { stabilize: true }); // Demo mode is one setting and two broadcasts, all before the settle. assert.deepEqual(events, ['enable', 'enable', 'enable', 'settle:1000', 'capture', 'disable']); diff --git a/packages/platform-android/src/input-actions.ts b/packages/platform-android/src/input-actions.ts index 84795014ff..e0d24806ca 100644 --- a/packages/platform-android/src/input-actions.ts +++ b/packages/platform-android/src/input-actions.ts @@ -29,7 +29,6 @@ import { runAndroidShell } from './adb.ts'; import { androidAdbResultError, discloseAdbInputDispatch } from './adb-failure.ts'; import { executeAndroidTouchPlan, readAndroidGestureViewportReading } from './touch-executor.ts'; import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; -import { runAndroidAutomationHelper } from './automation-helper.ts'; /** * What a caller may steer on a dispatched input command: cancellation only. The wider executor @@ -44,6 +43,7 @@ export async function pressAndroid( y: number, options?: AndroidInputDispatchOptions, ): Promise { + const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); try { if (!options?.signal && (await runAndroidAutomationHelper(device, 'input', { kind: 'tap', x, y }))) return; // Picked field by field so no runtime caller can smuggle `allowFailure: true` past the type. @@ -85,6 +85,7 @@ export async function backAndroid( device: DeviceInfo, options?: AndroidInputDispatchOptions, ): Promise { + const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); if (!options?.signal && (await runAndroidAutomationHelper(device, 'input', { kind: 'key', keyCode: 4 }))) return; await runAndroidShell( device, @@ -94,11 +95,13 @@ export async function backAndroid( } export async function homeAndroid(device: DeviceInfo): Promise { + const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); if (await runAndroidAutomationHelper(device, 'input', { kind: 'key', keyCode: 3 })) return; await runAndroidShell(device, ['input', 'keyevent', '3']); } export async function pressAndroidEnter(device: DeviceInfo): Promise { + const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); if (await runAndroidAutomationHelper(device, 'input', { kind: 'key', keyCode: 66 })) return; await runAndroidShell(device, ['input', 'keyevent', 'ENTER']); } @@ -216,6 +219,7 @@ export async function probeAndroidDisplayRotation( } export async function appSwitcherAndroid(device: DeviceInfo): Promise { + const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); if (await runAndroidAutomationHelper(device, 'input', { kind: 'key', keyCode: 187 })) return; await runAndroidShell(device, ['input', 'keyevent', '187']); } diff --git a/packages/platform-android/src/observation.ts b/packages/platform-android/src/observation.ts index 0cb20d8643..74d805891a 100644 --- a/packages/platform-android/src/observation.ts +++ b/packages/platform-android/src/observation.ts @@ -7,7 +7,6 @@ import type { AppStateRuntimeResult } from '@agent-device/contracts/app-state-ru import type { DeviceInfo } from '@agent-device/kernel/device'; import { deviceShellArgv } from '@agent-device/kernel/device-shell'; import { AppError } from '@agent-device/kernel/errors'; -import { runAndroidAutomationHelper } from './automation-helper.ts'; import { ANDROID_FOCUS_MARKERS, ANDROID_FOCUSED_WINDOW_MARKER, @@ -44,6 +43,7 @@ export function createAndroidObservationAdapter( readAppState: async (device) => await readAppState(host, device), readBlockingDialog: async (device) => await readBlockingDialog(host, device), async readAppFocus(device, appBundleId, options = {}) { + const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); const focus = await runAndroidAutomationHelper(device, 'focus'); if (focus?.focusedPackage === appBundleId && focus.windowType === '1') return true; const read = createDumpReader(host, device); @@ -113,6 +113,7 @@ async function readBlockingDialog( device: DeviceInfo, read: DumpReader = createDumpReader(host, device), ): Promise { + const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); const focus = await runAndroidAutomationHelper(device, 'focus'); if (focus?.windowType === '1' && focus.focusedPackage && focus.focusedPackage !== 'android') { return { status: 'clear' }; diff --git a/packages/platform-android/src/screenshot.ts b/packages/platform-android/src/screenshot.ts index 1c70816d4e..ce8ddc0312 100644 --- a/packages/platform-android/src/screenshot.ts +++ b/packages/platform-android/src/screenshot.ts @@ -5,7 +5,6 @@ import type { ShellWord } from '@agent-device/kernel/device-shell'; import { runAndroidExecOut, runAndroidShell, sleep } from './adb.ts'; import { requireAndroidAdbHost } from './adb-host.ts'; import { probeAndroidDisplayRotation } from './input-actions.ts'; -import { runAndroidAutomationHelper } from './automation-helper.ts'; // PNG file signature: 0x89 P N G \r \n 0x1A \n const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); @@ -100,6 +99,7 @@ async function settledWithin(pending: Promise, graceMs: number): Promise { + const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); const helper = await runAndroidAutomationHelper(device, 'screenshot'); const bytes = helper ? Buffer.from(helper.payloadBase64 ?? '', 'base64') diff --git a/packages/platform-android/src/text-input.ts b/packages/platform-android/src/text-input.ts index 2cf2b17e05..5386ae6268 100644 --- a/packages/platform-android/src/text-input.ts +++ b/packages/platform-android/src/text-input.ts @@ -36,7 +36,6 @@ import { getAndroidTestImeOwnership } from './ime-state.ts'; import { discloseAdbInputDispatch } from './adb-failure.ts'; import { focusAndroid } from './input-actions.ts'; import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; -import { runAndroidAutomationHelper } from './automation-helper.ts'; /** * `input text` truncates long strings in some app/IME states (#531), so ASCII is written in short @@ -85,6 +84,7 @@ export async function fillAndroid( const verification = await verifyAndroidFilledText(device, x, y, text, helper); return completeAndroidFillVerification(text, beforeTarget, verification); } + const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); if ( delayMs === 0 && (await runAndroidAutomationHelper(device, 'input', { kind: 'fill', x, y, text })) diff --git a/packages/platform-android/src/touch-helper.ts b/packages/platform-android/src/touch-helper.ts index ef7a69ba14..d70919f5f9 100644 --- a/packages/platform-android/src/touch-helper.ts +++ b/packages/platform-android/src/touch-helper.ts @@ -1,7 +1,3 @@ -import { - prepareAndroidAutomationHelper, - androidAutomationHelperIdentity, -} from './automation-helper-artifact.ts'; import type { PointerTrajectory } from '@agent-device/contracts/gesture-plan-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { Rect } from '@agent-device/kernel/snapshot'; @@ -57,6 +53,8 @@ export async function executeAndroidTouchHelperPlan( device: DeviceInfo, plan: AndroidLoweredTouchPlan, ): Promise> { + const { prepareAndroidAutomationHelper, androidAutomationHelperIdentity } = + await import('./automation-helper-artifact.ts'); const prepared = await prepareAndroidAutomationHelper(device); const request = normalizeAndroidTouchHelperGestureRequest(plan); const timeoutMs = Math.max( @@ -111,6 +109,8 @@ export async function readAndroidTouchHelperViewportReading( device: DeviceInfo, helper: AndroidHelperSessionOptions = {}, ): Promise { + const { prepareAndroidAutomationHelper, androidAutomationHelperIdentity } = + await import('./automation-helper-artifact.ts'); const prepared = await prepareAndroidAutomationHelper(device); if (helper.helperSessionScope === 'daemon-session') { // Without a live session both this read and the gesture that follows would each start their diff --git a/src/commands/schema/cli-help-command-usage.test.ts b/src/commands/schema/cli-help-command-usage.test.ts index 7590c8dd78..ff4d7aaecd 100644 --- a/src/commands/schema/cli-help-command-usage.test.ts +++ b/src/commands/schema/cli-help-command-usage.test.ts @@ -25,9 +25,9 @@ test('usageForCommand documents screenshot web aliases and stabilization flags', assert.match(help, /--fullscreen, --full, -f/); assert.match(help, /entire page/i); assert.match(help, /--no-stabilize/); - assert.match(help, /low-latency Android capture loops/); + assert.match(help, /Android captures immediately by default/); assert.match(help, /--normalize-status-bar/); - assert.match(help, /deterministic iOS simulator chrome/); + assert.match(help, /deterministic iOS simulator or Android chrome/); assert.match(help, /--crop-on /); assert.match(help, /crop the capture to the frame of the selector/); }); diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index 7de74923ed..c14c18c30f 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -1216,7 +1216,7 @@ function assertAndroidInteractionContract(world: AndroidSettingsWorld): void { assert.equal( adbCalls.filter((call) => call.join(' ') === 'shell settings put global sysui_demo_allowed 1') .length, - 2, + 0, ); } diff --git a/test/integration/provider-scenarios/android-tv-remote.test.ts b/test/integration/provider-scenarios/android-tv-remote.test.ts index 4f3fe41030..9fe76fe8d4 100644 --- a/test/integration/provider-scenarios/android-tv-remote.test.ts +++ b/test/integration/provider-scenarios/android-tv-remote.test.ts @@ -19,6 +19,13 @@ test('Provider-backed integration Android TV remote flow sends D-pad keyevents', const adbProvider: AndroidAdbProvider = { exec: async (args) => { adbCalls.push([...args]); + if (args.join(' ') === 'shell ime list -a -s') { + return { + stdout: 'com.callstack.agentdevice.imehelper/.TestInputMethodService\n', + stderr: '', + exitCode: 0, + }; + } return { stdout: '', stderr: '', exitCode: 0 }; }, }; From 3f8800c990a5dc124343f24eb8e5fc959d161683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 12:32:45 +0200 Subject: [PATCH 5/9] docs(perf): verify original Android benchmark cases and costs --- ...droid-automation-performance-2026-10-19.md | 2 + .../android-e2e-parity-2026-10-19-cases.json | 604 ++++++++++++++++++ ...roid-e2e-parity-2026-10-19-operations.json | 217 +++++++ .../evidence/android-e2e-parity-2026-10-19.md | 84 +++ 4 files changed, 907 insertions(+) create mode 100644 docs/evidence/android-e2e-parity-2026-10-19-cases.json create mode 100644 docs/evidence/android-e2e-parity-2026-10-19-operations.json create mode 100644 docs/evidence/android-e2e-parity-2026-10-19.md diff --git a/docs/evidence/android-automation-performance-2026-10-19.md b/docs/evidence/android-automation-performance-2026-10-19.md index 7728b95e56..2ffe554e9c 100644 --- a/docs/evidence/android-automation-performance-2026-10-19.md +++ b/docs/evidence/android-automation-performance-2026-10-19.md @@ -4,6 +4,8 @@ The comparison starts from `eef1a3f76d7216da006ff59c975620c8b9dd7182`. The reference experiment is [tester-army/e2e#1234](https://github.com/tester-army/e2e/pull/1234), inspected at `2593e45025edc52b2c381e809b41cb63d5878e2b`. Its measurements are author-reported. Its direct engine sets text through `ACTION_SET_TEXT`, injects input through `UiAutomation`, and captures screenshots inside its instrumentation process. +The follow-up [original-app comparison](android-e2e-parity-2026-10-19.md) runs the exact 42 Android cases and operation benchmark. Both agent-device variants passed all 42 cases; the direct engine passed 40 and failed two. The base/head suite times were 327.95/321.25 seconds. The fixture primitive gains below do not establish the reference experiment's total-suite gain. + ## Reproduce Use a dedicated Pixel 7 API 36 emulator and the current `examples/test-app` Release APK. diff --git a/docs/evidence/android-e2e-parity-2026-10-19-cases.json b/docs/evidence/android-e2e-parity-2026-10-19-cases.json new file mode 100644 index 0000000000..315a2c9935 --- /dev/null +++ b/docs/evidence/android-e2e-parity-2026-10-19-cases.json @@ -0,0 +1,604 @@ +{ + "referenceCommit": "2593e45025edc52b2c381e809b41cb63d5878e2b", + "agentDeviceBaseCommit": "eef1a3f76d7216da006ff59c975620c8b9dd7182", + "agentDeviceHeadCommit": "ddba75f5b474cb4cdac11b15f75292f2870396c7", + "apkSha256": "b163e985ebd8340bf10447cdf6826aea37e429ee8c4382f9179c863c8c0619b2", + "apkSource": "apps/mobile-benchmark at referenceCommit; local Release build, arm64-v8a", + "device": "Pixel_7_CI / emulator-5554 / Android 16 API 36", + "runner": "e2e 0.21.2 / Node 26.8.1 / darwin arm64", + "config": "apps/mobile-benchmark/e2e.android.config.ts", + "workers": 1, + "runnerRetries": 0, + "sameSelectedCases": true, + "sameRunnableCases": true, + "runnableTestIdsSha256": "fc5487b7d87d94e48a1315303d7aa3da1446adf152a54a2181dfbacf9596b2c7", + "runs": { + "base": { + "runId": "01a153a6-415b-7d76-9327-abc70b772569", + "startedAt": "2026-10-19T10:12:48.456Z", + "finishedAt": "2026-10-19T10:18:16.408Z", + "durationMs": 327952, + "status": "passed", + "summary": { + "discovered": 45, + "selected": 43, + "executed": 42, + "passed": 42, + "failed": 0, + "interrupted": 0, + "flaky": 0, + "skipped": 1 + }, + "reportSha256": "fb5811851cec21c69fa25e5f6bf776636c2be507d65e8167ef2d2c42344c6a3a" + }, + "head": { + "runId": "01a153a0-9100-7c37-8b9b-2d982bded990", + "startedAt": "2026-10-19T10:06:36.383Z", + "finishedAt": "2026-10-19T10:11:57.633Z", + "durationMs": 321250, + "status": "passed", + "summary": { + "discovered": 45, + "selected": 43, + "executed": 42, + "passed": 42, + "failed": 0, + "interrupted": 0, + "flaky": 0, + "skipped": 1 + }, + "reportSha256": "0a8d8175ba8145079aebb9236b18407fc8cfe38185037972329007a318e9e68d" + }, + "direct": { + "runId": "01a153ac-2384-78f5-a765-32b352482a2e", + "startedAt": "2026-10-19T10:19:12.763Z", + "finishedAt": "2026-10-19T10:22:35.366Z", + "durationMs": 202603, + "status": "failed", + "summary": { + "discovered": 45, + "selected": 43, + "executed": 42, + "passed": 40, + "failed": 2, + "interrupted": 0, + "flaky": 0, + "skipped": 1 + }, + "reportSha256": "a4f5adfa00d0d0a844e3fe1a47d44d58e02eb9f13b685b102fcae23874651d49" + } + }, + "cases": [ + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20label%20query%20resolves%20a%20labeled%20text%20field", + "sourceLine": 32, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 3401, + "headStatus": "passed", + "headDurationMs": 3444, + "directStatus": "passed", + "directDurationMs": 1687 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20labeled%20field%20answers%20name%2C%20display%20value%2C%20value%20reads%2C%20fill%2C%20and%20clear", + "sourceLine": 37, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 8183, + "headStatus": "passed", + "headDurationMs": 7941, + "directStatus": "passed", + "directDurationMs": 1894 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20placeholder%20query%20and%20attribute%20reads%20find%20the%20field", + "sourceLine": 55, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 3131, + "headStatus": "passed", + "headDurationMs": 3040, + "directStatus": "passed", + "directDurationMs": 1534 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20secure%20field%20takes%20a%20fill%20and%20denies%20value%20reads", + "sourceLine": 64, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 4627, + "headStatus": "passed", + "headDurationMs": 4370, + "directStatus": "passed", + "directDurationMs": 1553 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::press%20sends%20a%20character%20and%20Enter%20through%20the%20soft%20keyboard", + "sourceLine": 72, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 5958, + "headStatus": "passed", + "headDurationMs": 5825, + "directStatus": "passed", + "directDurationMs": 1730 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20field%20autofocuses%20on%20mount%20and%20focus%28%29%20moves%20the%20focus", + "sourceLine": 81, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 9066, + "headStatus": "passed", + "headDurationMs": 9064, + "directStatus": "passed", + "directDurationMs": 1872 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::toBeFocused%20follows%20the%20focus", + "sourceLine": 97, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 8269, + "headStatus": "passed", + "headDurationMs": 8215, + "directStatus": "passed", + "directDurationMs": 2741 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20switch%20checks%2C%20unchecks%2C%20and%20reports%20its%20state", + "sourceLine": 107, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 9219, + "headStatus": "passed", + "headDurationMs": 9006, + "directStatus": "passed", + "directDurationMs": 4075 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::checkbox%20and%20radio%20roles%20answer%20getByRole%2C%20and%20a%20checkbox%20toggles%20by%20tap", + "sourceLine": 130, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 6393, + "headStatus": "passed", + "headDurationMs": 6137, + "directStatus": "passed", + "directDurationMs": 1886 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::checkbox%20and%20radio%20roles%20check%2C%20uncheck%2C%20and%20report%20their%20state", + "sourceLine": 147, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 11014, + "headStatus": "passed", + "headDurationMs": 10877, + "directStatus": "passed", + "directDurationMs": 6367 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20tab%20role%20answers%20getByRole", + "sourceLine": 180, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 2857, + "headStatus": "passed", + "headDurationMs": 2821, + "directStatus": "passed", + "directDurationMs": 1325 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20radio%20group%20reports%20the%20checked%20option", + "sourceLine": 188, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 8487, + "headStatus": "passed", + "headDurationMs": 8240, + "directStatus": "passed", + "directDurationMs": 3923 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::buttons%20report%20disabled%2C%20enabled%2C%20and%20a%20state%20that%20arrives%20later", + "sourceLine": 201, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 7600, + "headStatus": "passed", + "headDurationMs": 7459, + "directStatus": "passed", + "directDurationMs": 3194 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20display%3A%20none%20element%20is%20absent%20rather%20than%20hidden%2C%20its%20visible%20twin%20attached", + "sourceLine": 219, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 5643, + "headStatus": "passed", + "headDurationMs": 5611, + "directStatus": "passed", + "directDurationMs": 2368 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20header%20answers%20its%20accessible%20name", + "sourceLine": 235, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 2932, + "headStatus": "passed", + "headDurationMs": 2873, + "directStatus": "passed", + "directDurationMs": 1298 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20tab%20row%20answers%20count%2C%20all%2C%20texts%2C%20filters%2C%20positions%2C%20and%20switching", + "sourceLine": 244, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 4790, + "headStatus": "passed", + "headDurationMs": 4715, + "directStatus": "passed", + "directDurationMs": 1727 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20list%20answers%20counts%2C%20list-form%20text%2C%20reads%2C%20and%20boxes%20in%20order", + "sourceLine": 271, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 4614, + "headStatus": "passed", + "headDurationMs": 4636, + "directStatus": "failed", + "directDurationMs": 647, + "directError": { + "category": "test", + "code": "ACTION_FAILED", + "message": "node 346.63 is off screen: getByLabel(\"Control Inventory. \")", + "retryable": false, + "phase": "beforeEach", + "source": { + "file": "tests/fixtures.ts", + "line": 81, + "column": 3 + } + } + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20row%20contains%20its%20texts", + "sourceLine": 300, + "selected": true, + "skip": { + "cause": "explicit", + "reason": "React Native flattens a plain View out of the accessibility tree on both platforms: XCTest and the Android helper report a View with a testID as a leaf beside its children, so filter({ has }) and a query scoped to it match nothing; only ScrollView, TextInput, and Text hosts nest" + }, + "baseStatus": "skipped", + "baseDurationMs": null, + "headStatus": "skipped", + "headDurationMs": null, + "directStatus": "skipped", + "directDurationMs": null + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20long%20press%20lands%20as%20one", + "sourceLine": 312, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 6628, + "headStatus": "passed", + "headDurationMs": 6488, + "directStatus": "passed", + "directDurationMs": 2343 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20double%20tap%20lands%20inside%20the%20300%20ms%20window", + "sourceLine": 327, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 7382, + "headStatus": "passed", + "headDurationMs": 7411, + "directStatus": "passed", + "directDurationMs": 1730 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::pointer%20verbs%20land%20at%20the%20point%20asked%20for", + "sourceLine": 338, + "selected": false, + "skip": { + "cause": "platform-unavailable", + "reason": "test declares platforms [ios]" + }, + "baseStatus": "skipped", + "baseDurationMs": null, + "headStatus": "skipped", + "headDurationMs": null, + "directStatus": "skipped", + "directDurationMs": null + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::an%20ambiguous%20locator%20fails%20at%20once", + "sourceLine": 384, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 4335, + "headStatus": "passed", + "headDurationMs": 4273, + "directStatus": "passed", + "directDurationMs": 1482 + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::screenshot%2C%20restart%2C%20and%20clearState%20reopen%20the%20app%20on%20its%20home%20list", + "sourceLine": 390, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 7884, + "headStatus": "passed", + "headDurationMs": 6570, + "directStatus": "passed", + "directDurationMs": 2483 + }, + { + "testId": "tests/controls.e2e.ts::refusals::verbs%20a%20device%20never%20declares%20are%20refused%20before%20the%20engine", + "sourceLine": 404, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 1, + "headStatus": "passed", + "headDurationMs": 1, + "directStatus": "passed", + "directDurationMs": 3 + }, + { + "testId": "tests/controls.e2e.ts::refusals::keys%20the%20soft%20keyboard%20cannot%20send%20are%20refused%20by%20the%20engine", + "sourceLine": 418, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 157, + "headStatus": "passed", + "headDurationMs": 165, + "directStatus": "passed", + "directDurationMs": 165 + }, + { + "testId": "tests/controls.e2e.ts::refusals::a%20missing%20locator%20fails%20as%20not%20found%20once%20its%20timeout%20passes", + "sourceLine": 429, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 1107, + "headStatus": "passed", + "headDurationMs": 1113, + "directStatus": "passed", + "directDurationMs": 1072 + }, + { + "testId": "tests/flows-lists.e2e.ts::infinite%20scroll%20reaches%20the%20target%20item", + "sourceLine": 9, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 14782, + "headStatus": "passed", + "headDurationMs": 14316, + "directStatus": "passed", + "directDurationMs": 22370 + }, + { + "testId": "tests/flows-lists.e2e.ts::product%20catalog%20adds%20exactly%20two%20of%20the%20right%20variant", + "sourceLine": 17, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 9200, + "headStatus": "passed", + "headDurationMs": 8986, + "directStatus": "passed", + "directDurationMs": 1990 + }, + { + "testId": "tests/flows-lists.e2e.ts::gestures%3A%20long-press%2C%20then%20swipe%20left", + "sourceLine": 27, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 7467, + "headStatus": "passed", + "headDurationMs": 7339, + "directStatus": "passed", + "directDurationMs": 13198 + }, + { + "testId": "tests/flows-lists.e2e.ts::gestures%3A%20double-tap%20completes%20the%20flow", + "sourceLine": 40, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 8981, + "headStatus": "passed", + "headDurationMs": 8745, + "directStatus": "passed", + "directDurationMs": 3109 + }, + { + "testId": "tests/flows-lists.e2e.ts::sticky%20chrome%3A%20accept%20below%20the%20fold%2C%20then%20continue", + "sourceLine": 50, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 10534, + "headStatus": "passed", + "headDurationMs": 10532, + "directStatus": "passed", + "directDurationMs": 13379 + }, + { + "testId": "tests/flows-lists.e2e.ts::async%20states%3A%20load%2C%20pull%20to%20refresh%2C%20claim%20through%20the%20toast", + "sourceLine": 62, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 10734, + "headStatus": "passed", + "headDurationMs": 10900, + "directStatus": "passed", + "directDurationMs": 7623 + }, + { + "testId": "tests/flows-lists.e2e.ts::huge%20virtualized%20list%20reaches%20row%20512", + "sourceLine": 102, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 50998, + "headStatus": "passed", + "headDurationMs": 48378, + "directStatus": "passed", + "directDurationMs": 50821 + }, + { + "testId": "tests/flows-lists.e2e.ts::web%20view%20coupon%20form%20applies%20the%20code%20shown%20on%20the%20page", + "sourceLine": 120, + "selected": false, + "skip": { + "cause": "platform-unavailable", + "reason": "test declares platforms [ios]" + }, + "baseStatus": "skipped", + "baseDurationMs": null, + "headStatus": "skipped", + "headDurationMs": null, + "directStatus": "skipped", + "directDurationMs": null + }, + { + "testId": "tests/flows.e2e.ts::text%20inputs%20unlock%20the%20submit%20button", + "sourceLine": 11, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 5832, + "headStatus": "passed", + "headDurationMs": 7074, + "directStatus": "passed", + "directDurationMs": 1987 + }, + { + "testId": "tests/flows.e2e.ts::filled%20text%20inputs%20report%20their%20values", + "sourceLine": 25, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 5159, + "headStatus": "passed", + "headDurationMs": 5929, + "directStatus": "passed", + "directDurationMs": 1569 + }, + { + "testId": "tests/flows.e2e.ts::modal%20flow%20confirms%20through%20the%20native%20alert", + "sourceLine": 36, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 7689, + "headStatus": "passed", + "headDurationMs": 7922, + "directStatus": "failed", + "directDurationMs": 1767, + "directError": { + "category": "test", + "code": "ACTION_FAILED", + "message": "node 800.31 is off screen: getByTestId(\"continue-button\")", + "retryable": false, + "phase": "body", + "source": { + "file": "tests/flows.e2e.ts", + "line": 39, + "column": 3 + } + } + }, + { + "testId": "tests/flows.e2e.ts::bottom%20tabs%20act%20inside%20the%20last%20tab", + "sourceLine": 49, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 6006, + "headStatus": "passed", + "headDurationMs": 5767, + "directStatus": "passed", + "directDurationMs": 12224 + }, + { + "testId": "tests/flows.e2e.ts::error%20recovery%20retries%2C%20then%20confirms%20the%20delete", + "sourceLine": 63, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 7745, + "headStatus": "passed", + "headDurationMs": 7811, + "directStatus": "passed", + "directDurationMs": 4528 + }, + { + "testId": "tests/flows.e2e.ts::choice%20controls%20place%20the%20exact%20order", + "sourceLine": 72, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 7851, + "headStatus": "passed", + "headDurationMs": 7833, + "directStatus": "passed", + "directDurationMs": 1760 + }, + { + "testId": "tests/flows.e2e.ts::debounced%20search%20selects%20the%20target%20once%20results%20arrive", + "sourceLine": 93, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 7950, + "headStatus": "passed", + "headDurationMs": 7670, + "directStatus": "passed", + "directDurationMs": 3032 + }, + { + "testId": "tests/login-form.e2e.ts::login%20form::rejects%20a%20malformed%20email", + "sourceLine": 18, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 8165, + "headStatus": "passed", + "headDurationMs": 7710, + "directStatus": "passed", + "directDurationMs": 2012 + }, + { + "testId": "tests/login-form.e2e.ts::login%20form::rejects%20an%20unknown%20account", + "sourceLine": 26, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 8146, + "headStatus": "passed", + "headDurationMs": 7800, + "directStatus": "passed", + "directDurationMs": 2082 + }, + { + "testId": "tests/login-form.e2e.ts::login%20form::signs%20in%20and%20logs%20out", + "sourceLine": 34, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 12211, + "headStatus": "passed", + "headDurationMs": 11504, + "directStatus": "passed", + "directDurationMs": 6956 + }, + { + "testId": "tests/smoke.e2e.ts::home%20lists%20the%20scenarios%20and%20opens%20one", + "sourceLine": 5, + "selected": true, + "baseStatus": "passed", + "baseDurationMs": 4330, + "headStatus": "passed", + "headDurationMs": 4246, + "directStatus": "passed", + "directDurationMs": 2336 + } + ] +} diff --git a/docs/evidence/android-e2e-parity-2026-10-19-operations.json b/docs/evidence/android-e2e-parity-2026-10-19-operations.json new file mode 100644 index 0000000000..eb47663258 --- /dev/null +++ b/docs/evidence/android-e2e-parity-2026-10-19-operations.json @@ -0,0 +1,217 @@ +{ + "referenceCommit": "2593e45025edc52b2c381e809b41cb63d5878e2b", + "apkSha256": "b163e985ebd8340bf10447cdf6826aea37e429ee8c4382f9179c863c8c0619b2", + "benchmarkScript": "packages/android/bench/bench.ts --rounds 20", + "baseCommit": "eef1a3f76d7216da006ff59c975620c8b9dd7182", + "headCommit": "ddba75f5b474cb4cdac11b15f75292f2870396c7", + "notes": "Original unmodified benchmark; direct samples come from the head pairing. Cold initialization has one sample. Native verified-fill and screenshot guarantees differ from direct operation contracts.", + "samples": { + "base": { + "prepare + init (cold)": [1529.9775], + "restart app": [1199.5876250000001, 1168.563209, 1077.8197080000004], + "observe (home list)": [ + 26.446333999999297, 26.305582999999388, 25.86200000000008, 23.896874999999454, + 23.85649999999987, 23.235124999999243, 23.265666999999667, 23.722541000000092, + 22.593000000000757, 21.676916999999776, 23.012499999999818, 23.323999999999614, + 21.665624999999636, 21.801916000000347, 23.034292000000278, 25.581833000000188, + 22.37291699999969, 22.46849999999995, 21.51366699999926, 21.443707999999788 + ], + "swipe (viewport)": [ + 694.0999590000001, 412.26991700000053, 405.2490420000004, 444.01458300000013, + 424.91829100000086 + ], + "tap row -> next screen visible": [ + 258.6743750000005, 426.4684579999994, 477.47758299999987, 615.1892499999994, + 411.19045800000094 + ], + "back -> list visible": [ + 405.9442080000008, 306.1894160000011, 259.55645800000093, 417.8524579999994, + 421.08304099999987 + ], + "observe (form)": [ + 146.4332919999997, 101.61391699999876, 150.5727910000005, 72.62433299999975, + 22.387833000000683, 22.362457999999606, 22.9619160000002, 18.966207999998005, + 17.77141699999993, 17.67883300000176, 17.986291999997775, 18.33483299999716, + 17.625625000000582, 17.8089579999978, 17.9587500000016, 20.79175000000032, + 19.11120799999844, 18.23712499999965, 20.343166999999085, 18.21049999999741 + ], + "locate getByTestId": [ + 18.814040999997815, 17.533917000000656, 18.350374999998166, 17.38616700000057, + 16.836542000000918, 17.53204199999891, 17.897291000001132, 18.095124999999825, + 18.28629099999671, 17.24620800000048, 17.564416000001074, 18.51854199999798, + 17.97745799999757, 17.889541000000463, 17.826916999998502, 17.761540999999852, + 18.806459000003088, 17.987166999999317, 18.471332999997685, 19.85612499999843 + ], + "tap": [ + 116.91799999999785, 165.7128339999981, 119.96308299999873, 122.34612500000003, + 155.60037499999817, 121.5961669999997, 123.35312499999782, 122.07795799999803, + 114.87654200000179, 123.81437499999811, 112.41774999999689, 122.42745899999863, + 118.41458399999829, 115.50658400000248, 121.30354200000147, 110.2821249999979, + 114.06295800000225, 112.78974999999627, 111.28112499999406, 126.77858299999934 + ], + "fill (19 chars)": [ + 468.88387499999953, 975.8170000000027, 988.150416000004, 2794.582791999994, + 1054.988916000002, 2798.2055420000033, 954.4806250000038, 971.6698329999999, + 962.2762920000023, 979.554167000002, 968.8219160000008, 977.8647500000006, + 969.3966660000006, 1026.529290999999, 963.0484169999982, 2808.975790999997, + 958.3973750000005, 949.1790420000034, 972.1281670000026, 954.5080000000016 + ], + "keyboard.type (10 chars)": [ + 88.67316700000083, 101.84741699999722, 121.07875000000058, 84.60054200000013, + 79.92433300000266, 91.77129100001184, 76.86787500001083, 81.64600000000792, + 85.20266700000502, 95.80220800000825 + ], + "press Enter on node": [ + 487.36108299999614, 30.62262500000361, 27.76379199999792, 30.901332999987062, + 28.83062500000233, 29.332457999989856, 30.981124999991152, 33.61529200000223, + 29.038249999997788, 28.06708300000173 + ], + "screenshot": [ + 1453.572250000012, 1425.6530000000057, 1448.2772919999989, 1448.195333999989, + 1463.3300829999935 + ], + "observe with pixels": [ + 1444.6934159999946, 1453.2294170000096, 1540.2688749999943, 1454.4534580000036, + 1441.7382089999883 + ] + }, + "head": { + "prepare + init (cold)": [1494.6224169999998], + "restart app": [868.1754170000002, 958.7219579999996, 849.0516249999996], + "observe (home list)": [ + 26.666959000000134, 26.30712500000027, 26.94604199999958, 24.62433299999975, + 30.270375000000058, 25.386875000000146, 22.448209000000134, 20.812041000000136, + 24.415291000000252, 23.770958999999493, 22.928375000000415, 24.54200000000037, + 23.528124999999818, 23.04729199999929, 23.352583000000777, 24.330041999999594, + 23.762500000000728, 23.487834000000475, 24.143833000000086, 26.94562500000029 + ], + "swipe (viewport)": [ + 621.8104579999999, 382.58150000000023, 404.3797079999995, 367.32208399999945, + 400.8157089999995 + ], + "tap row -> next screen visible": [ + 250.65650000000096, 419.6733750000003, 485.2935000000016, 399.36941600000137, + 502.76320900000064 + ], + "back -> list visible": [ + 376.9512080000004, 231.51266700000087, 235.21287500000108, 232.84254200000032, + 246.9188749999994 + ], + "observe (form)": [ + 117.77474999999868, 70.91749999999956, 61.093417000000045, 50.164042000000336, + 66.85812500000065, 64.56770799999867, 18.23004100000071, 16.44745800000055, + 18.339999999998327, 18.147249999999985, 17.638999999999214, 18.465125000000626, + 17.950125000001208, 18.104874999999083, 18.22149999999965, 17.577499999999418, + 17.53820900000028, 17.225915999999415, 17.305500000000393, 16.851666999998997 + ], + "locate getByTestId": [ + 18.34925000000112, 16.549791999999798, 17.469583000000057, 16.82837500000096, + 16.558999999999287, 16.389541999998983, 17.557208000000173, 17.885041999999885, + 17.225416999999652, 16.98687500000051, 17.488499999999476, 17.283165999999255, + 17.09020899999996, 17.126541999999972, 17.052708999999595, 17.10450000000128, + 16.37054200000057, 16.665374999998676, 16.86358299999847, 18.414040999999997 + ], + "tap": [ + 76.74612499999967, 61.947416999999405, 72.87854099999822, 61.670249999999214, + 68.7677909999984, 64.78904200000034, 103.77758299999914, 66.92695899999671, + 57.11250000000291, 58.18516699999964, 70.34441599999991, 68.8212919999969, 64.1785000000018, + 77.51312499999767, 71.12879200000316, 74.24387500000012, 71.14233400000376, + 65.60533399999986, 67.72016699999949, 59.7488749999975 + ], + "fill (19 chars)": [ + 256.84620800000266, 743.0553750000036, 759.9048750000002, 756.7259580000027, + 765.773417000004, 765.357041999996, 744.8195829999968, 748.7268750000003, 764.4585829999996, + 748.0647909999971, 762.0945829999982, 743.7299999999959, 751.7755409999954, + 745.5300829999978, 948.5913749999963, 861.3603339999972, 946.9823749999996, + 915.7157909999951, 877.7937079999974, 876.8785839999982 + ], + "keyboard.type (10 chars)": [ + 99.75366699999722, 111.97358399999939, 118.77095900000131, 120.95545799999672, + 92.96062499999971, 111.39287499999773, 115.41312499999913, 114.8380829999951, + 96.2410419999942, 89.93470900000102 + ], + "press Enter on node": [ + 539.5690410000025, 65.39737500000047, 96.466207999998, 82.24408399999811, 79.87220800000068, + 68.41308299999946, 110.43729199999507, 102.9151669999992, 98.02158299999428, + 65.20712500000081 + ], + "screenshot": [ + 421.59779100000014, 310.58570899999904, 343.3423330000005, 196.09537499999715, + 183.1460839999927 + ], + "observe with pixels": [ + 209.23770899999363, 182.3966250000085, 182.53804200000013, 182.4375410000066, + 179.74883400001272 + ] + }, + "direct": { + "prepare + init (cold)": [230.29099999999744], + "restart app": [244.20333299999766, 468.70987500000047, 647.5611669999926], + "observe (home list)": [ + 12.987957999997889, 159.95495899999514, 13.967499999998836, 6.867832999996608, + 15.024250000002212, 25.34329200000502, 16.11662499999511, 6.226875000007567, + 19.576874999998836, 12.332667000009678, 32.46141699999862, 32.105667000010726, + 11.972791000007419, 18.015958000003593, 7.458166999989771, 10.882291999994777, + 6.243916999999783, 13.57499999999709, 7.635999999998603, 12.01279200000863 + ], + "swipe (viewport)": [ + 542.6985829999903, 465.18512499998906, 493.2251249999972, 560.8984580000106, + 1215.3907080000063 + ], + "tap row -> next screen visible": [ + 870.9782919999998, 177.63633299998764, 155.90083299999242, 389.2433330000058, + 140.6924999999901 + ], + "back -> list visible": [ + 565.359209000002, 416.27791699999943, 185.8007079999952, 629.7403750000085, + 399.52987500000745 + ], + "observe (form)": [ + 18.570667000007234, 52.294624999994994, 66.8796250000014, 31.19708399999945, + 39.168541999999434, 42.12783300000592, 40.815833000000566, 31.619957999995677, + 34.31174999999348, 31.268708999996306, 46.3260420000006, 2.890417000002344, + 5.573041999989073, 7.242832999996608, 2.63916700000118, 2.056625000011991, + 1.6024580000084825, 2.1255420000088634, 8.237374999996973, 2.523583999995026 + ], + "locate getByTestId": [ + 2.5434170000080485, 1.9828750000015134, 1.7421249999897555, 1.9113340000039898, + 1.6794999999983702, 1.6973330000037095, 1.6288329999952111, 1.5403749999968568, + 1.545958000002429, 1.5285000000003492, 1.641749999995227, 1.5176250000076834, + 1.5429589999985183, 1.7096670000028098, 1.6344169999938458, 1.9348339999996824, + 1.7111250000016298, 1.690667000002577, 1.5134580000012647, 1.6767909999907715 + ], + "tap": [ + 43.26729200000409, 48.21479199999885, 62.46349999999802, 52.26366699999198, + 40.35487500000454, 42.88920900000085, 42.97912499999802, 34.65595899999607, + 45.02187500000582, 49.78675000001385, 38.34791699999187, 43.99754199999734, + 31.466791000013473, 35.91729199999827, 36.59937500000524, 27.592709000004106, + 34.09466699999757, 27.239374999990105, 27.320292000003974, 28.847500000003492 + ], + "fill (19 chars)": [ + 3.837707999991835, 7.213707999995677, 16.47637499999837, 9.818624999999884, + 6.697417000003043, 14.107041999988724, 35.852166999990004, 7.837625000000116, + 6.041540999998688, 15.40020899999945, 22.695584000000963, 3.987833999999566, + 30.342958000008366, 14.949374999996508, 21.307749999992666, 21.347292000005837, + 8.041083999996772, 48.86291599999822, 25.10654199999408, 8.139292000007117 + ], + "keyboard.type (10 chars)": [ + 553.8614169999928, 373.6178749999963, 457.3865839999926, 440.8177080000023, + 571.1499999999942, 373.5888330000016, 352.2668340000091, 380.9855000000098, + 347.8806659999973, 371.33116699999664 + ], + "press Enter on node": [ + 63.167583000002196, 43.389958999992814, 49.77445899999293, 45.81241699999373, + 72.059957999998, 55.794250000006286, 28.79816700000083, 29.502082999999402, + 26.162083999995957, 39.57462500000838 + ], + "screenshot": [ + 91.22400000000198, 54.53900000000431, 86.97687499999302, 65.86887500000012, + 53.98862499999814 + ], + "observe with pixels": [ + 54.37358299999323, 54.296541000003344, 56.32408400000713, 55.67783300000883, + 53.17912499999511 + ] + } + } +} diff --git a/docs/evidence/android-e2e-parity-2026-10-19.md b/docs/evidence/android-e2e-parity-2026-10-19.md new file mode 100644 index 0000000000..c0da233e95 --- /dev/null +++ b/docs/evidence/android-e2e-parity-2026-10-19.md @@ -0,0 +1,84 @@ +# Original Android e2e benchmark, 2026-10-19 + +The original Android selection now ran against the original benchmark app. +Both agent-device base and PR head passed all 42 runnable cases without runner retries. +The earlier `examples/test-app` run did not establish this case parity. + +The SDK code under test is `ddba75f5b474cb4cdac11b15f75292f2870396c7`. +The unchanged base is `eef1a3f76d7216da006ff59c975620c8b9dd7182`. +The app, runner, direct engine, tests, and configuration come from [tester-army/e2e#1234](https://github.com/tester-army/e2e/pull/1234), commit `2593e45025edc52b2c381e809b41cb63d5878e2b`. + +## App and method + +No downloadable APK was present in the releases or retained Android artifacts. Both retained artifact archives contained test evidence, not an APK. The workflow caches the native build with Actions cache. The downloaded source was built locally with its documented CI procedure: frozen dependencies, `pnpm build`, Expo prebuild, and `:app:assembleRelease` with JDK 17. The APK contains the pinned source's Release JavaScript and arm64-v8a native libraries. No app or test behavior was edited. + +- App: `dev.e2e.benchmark`, version `1.0.0`, version code `1`. +- APK: `47,530,660` bytes; SHA-256 `b163e985ebd8340bf10447cdf6826aea37e429ee8c4382f9179c863c8c0619b2`. +- Device: `Pixel_7_CI`, `emulator-5554`, Android 16 / API 36. +- Host: darwin arm64, Node `26.8.1`; runner: e2e `0.21.2`. +- One worker; zero runner retries; animations enabled; Device panel stream attached. +- Same installed APK for all variants. Run order: PR head, base, direct. +- Each agent-device checkout had its own dependencies, build, Android helper artifacts, and daemon state directory. The temporary e2e checkout's `packages/mobile/node_modules/agent-device` symlink selected the checkout under test. + +The original runner's `list --config e2e.android.config.ts` discovered 45 declarations. It selected 43 for Android and executed 42. Row containment is explicitly skipped in the source. The pointer-coordinate case and WebView case are iOS-only. Exact selected and runnable test identities match across all three reports; this is not a comparison of counts alone. + +```sh +BENCH_ENGINE=agent-device AGENT_DEVICE_STATE_DIR=/absolute/isolated-state \ + ANDROID_SERIAL=emulator-5554 E2E_TELEMETRY_DISABLED=1 \ + node node_modules/e2e/dist/cli/bin.js run \ + --config e2e.android.config.ts --workers 1 --retries 0 --output .e2e/run-name +``` + +Run that command from `apps/mobile-benchmark`. To select the direct engine, omit `BENCH_ENGINE`. The direct server was built with `pnpm --filter @e2e-dev/android build:server`. The APK was installed before the runs; `E2E_MOBILE_BENCHMARK_ANDROID_APP` was unset. + +## Exact suite results + +| Variant | Passed | Failed | Source skip | Elapsed | +| --- | ---: | ---: | ---: | ---: | +| Agent-device base | 42 | 0 | 1 | 327.95 s | +| Agent-device PR head | 42 | 0 | 1 | 321.25 s | +| Direct engine | 40 | 2 | 1 | 202.60 s | + +This is one run per variant. The 6.70-second base/head difference is a diagnostic observation, not evidence of a substantial suite speedup. One Device-panel observation screenshot was taken during the head run. The direct run failed two assertions, so its duration does not establish an all-passing speedup. + +The direct failures were `ACTION_FAILED` with an off-screen node: + +- `controls.e2e.ts:271`, list counts/texts/reads/boxes: the scenario-row tap failed in `tests/fixtures.ts:81`, before the test body. +- `flows.e2e.ts:36`, modal and native alert: the `continue-button` tap failed at line 39. + +Both cases passed on both agent-device variants. Every attempted case reported complete cleanup. The [case results](android-e2e-parity-2026-10-19-cases.json) include every declaration, selection flag, status, duration, report digest, and both failure details. + +## Original operation benchmark + +The original, unmodified `packages/android/bench/bench.ts --rounds 20` completed on base and head. Each invocation also ran the direct engine. The following table uses the direct samples paired with head. The [raw samples](android-e2e-parity-2026-10-19-operations.json) preserve each operation's count. Values are nearest-rank p50 milliseconds. Initialization has one sample; restart has three; other operations have 5–20. These small counts do not establish a startup improvement. + +| Operation | Base | Head | Direct | +| --- | ---: | ---: | ---: | +| Prepare + init | 1530 | 1495 | 230 | +| Restart | 1169 | 868 | 469 | +| Observe home | 23.0 | 24.1 | 13.0 | +| Viewport swipe | 425 | 401 | 543 | +| Tap row until next screen visible | 426 | 420 | 178 | +| Back until list visible | 406 | 235 | 416 | +| Observe form | 19.0 | 18.1 | 18.6 | +| Locate test id | 17.9 | 17.1 | 1.7 | +| Tap | 120 | 67.7 | 38.3 | +| Fill, 19 characters | 972 | 760 | 14.1 | +| Keyboard type, 10 characters | 85.2 | 111 | 374 | +| Press Enter on node | 29.3 | 82.2 | 43.4 | +| Screenshot | 1448 | 311 | 65.9 | +| Observe with pixels | 1453 | 182 | 54.4 | + +The complete operation table includes the typing and Enter rows that did not improve. Agent-device fill verifies its value across two observations separated by 150 ms; direct text replacement is a different completion contract. + +## Remaining costs + +The head suite's session recorded 762 snapshots totaling 89.16 seconds, 43 opens totaling 39.75 seconds, and 97 presses totaling 6.54 seconds. Its 88 e2e `locator.tap` steps totaled 106.77 seconds, including reads and readiness waits. These aggregates overlap and must not be added together. + +The mobile engine's selected-suite configuration sets `transition: 1500`. Its `settled()` method waits for a new or moved target, then observes it again. The standalone operation benchmark uses the engine's 500 ms default. Pairing each of its 20 fill samples with the corresponding daemon request gives 516.8 ms p50 outside the head daemon, versus 521.7 ms on base. Verified daemon fill itself is 243 ms p50 on head, versus 452 ms on base. This consumer wait remains after native input becomes faster. + +For the five screenshot-artifact calls, the corresponding daemon screenshot requests are 108 ms p50 on head, versus 1218 ms on base. The e2e screenshot API additionally observes the screen and decodes, masks, and re-encodes secure-field regions through `maskedScreenshot()` and `redactSecure()`. Its 311 ms head p50 therefore measures more than raw capture. Across all ten daemon screenshot requests, including the five pixel observations, head p50 is 58 ms. + +The experiment's 2.4x all-passing suite gain has not been reproduced by this change. The result proves exact Android case coverage and faster input/capture primitives. Snapshot acquisition, consumer transition policy, app lifecycle, and screenshot redaction remain material costs. + +Full reports, both benchmark pairings, and the APK are retained locally under `.tmp/e2e-1234-benchmark/`. The native source-helper probe reported `android-helper` and `helperVersion: 0.21.29-dev`. Each isolated daemon stopped with known cleanup and no orphaned claims. The disposable base checkout was removed after measurement. From 34d7844566304b132c2114765775c0ff93a91a02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 17:34:20 +0200 Subject: [PATCH 6/9] fix(android): recover only proven safe helper refusals and reads --- .../AutomationCommandHandler.java | 38 +++-- ...002-persistent-platform-helper-sessions.md | 5 +- packages/contracts/src/screenshot.ts | 2 +- .../snapshot-helper-session.fixtures.ts | 9 +- .../src/automation-helper-fallback.test.ts | 158 ++++++++++++++++++ .../platform-android/src/automation-helper.ts | 23 ++- .../src/fill-verification-text-read.test.ts | 27 +++ .../platform-android/src/fill-verification.ts | 9 +- .../src/snapshot-helper-session-lifecycle.ts | 11 +- .../src/snapshot-helper-session.ts | 16 +- website/docs/docs/commands.md | 4 +- 11 files changed, 265 insertions(+), 37 deletions(-) create mode 100644 packages/platform-android/src/automation-helper-fallback.test.ts create mode 100644 packages/platform-android/src/fill-verification-text-read.test.ts diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java index 63ec7c71e0..2e4a92b0a1 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java @@ -26,52 +26,61 @@ static void writeSessionResponse( OutputStream output, String requestId, String action, String payload, UiAutomation automation) throws IOException { Bundle result = SessionResponseWriter.sessionResponseBundle(requestId); + InputDispatch dispatch = new InputDispatch(); try { if ("screenshot".equals(action)) { screenshot(result, automation); } else if ("focus".equals(action)) { focus(result, automation); } else { - input(automation, new JSONObject(new String(Base64.decode(payload, Base64.DEFAULT), StandardCharsets.UTF_8))); + input(automation, new JSONObject(new String(Base64.decode(payload, Base64.DEFAULT), StandardCharsets.UTF_8)), dispatch); } result.putString("ok", "true"); SessionResponseWriter.writeSessionResponse(output, result, ""); } catch (Throwable error) { - SessionResponseWriter.writeSessionError(output, requestId, error.getClass().getName(), - error.getMessage() == null ? error.getClass().getName() : error.getMessage()); + result.putString("ok", "false"); + result.putString("errorType", error.getClass().getName()); + result.putString("message", error.getMessage() == null ? error.getClass().getName() : error.getMessage()); + if ("input".equals(action) && !dispatch.attempted) result.putString("injected", "false"); + SessionResponseWriter.writeSessionResponse(output, result, ""); } } - private static void input(UiAutomation automation, JSONObject input) throws Exception { + private static final class InputDispatch { + boolean attempted; + } + + private static void input(UiAutomation automation, JSONObject input, InputDispatch dispatch) throws Exception { switch (input.getString("kind")) { case "tap": - tap(automation, input.getDouble("x"), input.getDouble("y")); + tap(automation, input.getDouble("x"), input.getDouble("y"), dispatch); return; case "key": - key(automation, input.getInt("keyCode")); + key(automation, input.getInt("keyCode"), dispatch); return; case "fill": - fill(automation, input.getInt("x"), input.getInt("y"), input.getString("text")); + fill(automation, input.getInt("x"), input.getInt("y"), input.getString("text"), dispatch); return; default: throw new IllegalArgumentException("Unknown input kind"); } } - private static void tap(UiAutomation automation, double x, double y) { + private static void tap(UiAutomation automation, double x, double y, InputDispatch dispatch) { if (Double.isNaN(x) || Double.isInfinite(x) || Double.isNaN(y) || Double.isInfinite(y) || x < 0 || y < 0) { throw new IllegalArgumentException("Invalid tap coordinates"); } long down = SystemClock.uptimeMillis(); - touch(automation, down, MotionEvent.ACTION_DOWN, (float) x, (float) y); - touch(automation, down, MotionEvent.ACTION_UP, (float) x, (float) y); + touch(automation, down, MotionEvent.ACTION_DOWN, (float) x, (float) y, dispatch); + touch(automation, down, MotionEvent.ACTION_UP, (float) x, (float) y, dispatch); } - private static void touch(UiAutomation automation, long down, int action, float x, float y) { + private static void touch(UiAutomation automation, long down, int action, float x, float y, InputDispatch dispatch) { MotionEvent event = MotionEvent.obtain(down, SystemClock.uptimeMillis(), action, x, y, 0); event.setSource(InputDevice.SOURCE_TOUCHSCREEN); try { + dispatch.attempted = true; if (!automation.injectInputEvent(event, true)) { throw new IllegalStateException("Touch injection was refused"); } @@ -80,19 +89,20 @@ private static void touch(UiAutomation automation, long down, int action, float } } - private static void key(UiAutomation automation, int code) { + private static void key(UiAutomation automation, int code, InputDispatch dispatch) { if (code < 0 || code > KeyEvent.getMaxKeyCode()) throw new IllegalArgumentException("Invalid key code"); long down = SystemClock.uptimeMillis(); for (int action : new int[] {KeyEvent.ACTION_DOWN, KeyEvent.ACTION_UP}) { KeyEvent event = new KeyEvent(down, SystemClock.uptimeMillis(), action, code, 0, 0, KeyCharacterMap.VIRTUAL_KEYBOARD, 0, 0, InputDevice.SOURCE_KEYBOARD); + dispatch.attempted = true; if (!automation.injectInputEvent(event, true)) { throw new IllegalStateException("Key injection was refused"); } } } - private static void fill(UiAutomation automation, int x, int y, String text) { + private static void fill(UiAutomation automation, int x, int y, String text, InputDispatch dispatch) { AccessibilityNodeInfo target = null; try { for (AccessibilityWindowInfo window : automation.getWindows()) { @@ -117,7 +127,7 @@ private static void fill(UiAutomation automation, int x, int y, String text) { } } if (target == null) throw new IllegalStateException("No editable field at the fill point"); - tap(automation, x, y); + tap(automation, x, y, dispatch); if (!target.refresh()) throw new IllegalStateException("Fill target disappeared after focus"); if (!target.isFocused() && !target.performAction(AccessibilityNodeInfo.ACTION_FOCUS)) { throw new IllegalStateException("Field focus was refused"); diff --git a/docs/adr/0002-persistent-platform-helper-sessions.md b/docs/adr/0002-persistent-platform-helper-sessions.md index 9978f6effe..7d784b5b3c 100644 --- a/docs/adr/0002-persistent-platform-helper-sessions.md +++ b/docs/adr/0002-persistent-platform-helper-sessions.md @@ -95,7 +95,10 @@ Thus, the daemon had to stop the persistent snapshot session before every gestur After a failed session command, one-shot retry applies only to idempotent reads, such as viewport resolution. The retry runs only after the failed session has stopped. Pointer, key, text, and gesture injection are not idempotent because a mutation may have reached the app before the response fails. -Thus, session mutation failures return directly without a one-shot retry. +Thus, session mutation failures return directly without a one-shot retry unless a validated helper response proves that no input injection was attempted. +This exception permits the existing host input path only after the session has stopped. +Idempotent screenshot and focus failures also stop the session before the host read. +Piggyback commands retire a cached session whose process has ended before using a host path. The helper advertises optional session commands in its ready record. The host sends an optional command only when the helper advertises it. Text replacement retains the focus tap and host verification. diff --git a/packages/contracts/src/screenshot.ts b/packages/contracts/src/screenshot.ts index 813fd3924a..48e16a4373 100644 --- a/packages/contracts/src/screenshot.ts +++ b/packages/contracts/src/screenshot.ts @@ -174,7 +174,7 @@ export const SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS: readonly ScreenshotSpecificFl type: 'boolean', usageLabel: '--normalize-status-bar', usageDescription: - 'Screenshot: temporarily normalize iOS simulator or Android status-bar chrome for deterministic screenshot diffs; Android waits for transient UI to fade', + 'Screenshot: temporarily normalize iOS simulator or ADB-backed Android status-bar chrome for deterministic screenshot diffs; ADB-backed Android waits for transient UI to fade', projectConfig: true, recorded: true, }, diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts b/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts index f97ac254b8..d39450342e 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts @@ -59,6 +59,8 @@ export type PersistentSnapshotHelperProviderOptions = { sessionXml?: (sessionIndex: number, snapshotCount: number) => string; sessionRequests?: string[]; sessionInput?: (input: { kind: string; text?: string }) => void; + sessionCommands?: string[]; + sessionCommand?: (command: string) => Record | undefined; stalledSessionCleanup?: boolean; oneShotAttempts?: (readonly string[])[]; oneShotXml?: string; @@ -97,6 +99,11 @@ export function createPersistentSnapshotHelperProvider( socket.end('malformed session response'); return; } + const response = options.sessionCommand?.(command); + if (response) { + socket.end(sessionResponse({ requestId, body: '', metadata: response })); + return; + } if (command.startsWith('input')) { options.sessionInput?.( JSON.parse(Buffer.from(command.split(' ')[2]!, 'base64').toString('utf8')), @@ -146,7 +153,7 @@ export function createPersistentSnapshotHelperProvider( process.stdout.write( [ 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', - `INSTRUMENTATION_STATUS: sessionCommands=${options.sessionInput ? 'input,' : ''}snapshot-immediate`, + `INSTRUMENTATION_STATUS: sessionCommands=${options.sessionCommands?.join(',') ?? `${options.sessionInput ? 'input,' : ''}snapshot-immediate`}`, 'INSTRUMENTATION_STATUS: sessionReady=true', 'INSTRUMENTATION_STATUS_CODE: 2', '', diff --git a/packages/platform-android/src/automation-helper-fallback.test.ts b/packages/platform-android/src/automation-helper-fallback.test.ts new file mode 100644 index 0000000000..51b44a3d99 --- /dev/null +++ b/packages/platform-android/src/automation-helper-fallback.test.ts @@ -0,0 +1,158 @@ +import { afterEach, expect, test } from 'vitest'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { once } from 'node:events'; +import { withAndroidAdbProvider } from './adb-executor.ts'; +import { fillAndroid } from './text-input.ts'; +import { pressAndroid, backAndroid } from './input-actions.ts'; +import { screenshotAndroid } from './screenshot.ts'; +import { readAndroidFillTargetBeforeMutation } from './fill-verification.ts'; +import { resetAndroidSnapshotHelperSessions } from './snapshot-helper-session-lifecycle.ts'; +import { + createPersistentSnapshotHelperProvider, + type FakeAndroidProcess, +} from './__tests__/snapshot-helper-session.fixtures.ts'; +import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from './__tests__/test-utils/android-snapshot-helper.ts'; +import { ANDROID_EMULATOR as device } from './__tests__/test-utils/device-fixtures.ts'; +import { mkdtempForTest } from './__tests__/test-utils/android-host-test-setup.ts'; + +const png = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+b9xkAAAAASUVORK5CYII=', + 'base64', +); +const helper = { helperSessionScope: 'daemon-session' } as const; +afterEach(resetAndroidSnapshotHelperSessions); + +function fixture(response: Record) { + const calls: (readonly string[])[] = []; + const requests: string[] = []; + const processes: FakeAndroidProcess[] = []; + let value = ''; + const provider = createPersistentSnapshotHelperProvider({ + calls, + processes, + spawnArgs: [], + sessionRequests: requests, + sessionCommands: ['input', 'screenshot', 'focus', 'snapshot-immediate'], + sessionCommand: (command) => + /^(input|screenshot|focus) /.test(command) ? response : undefined, + sessionXml: () => + ``, + }); + const exec = provider.exec; + provider.exec = async (args, options) => { + if (args[0] === 'shell' && args[1] === 'input') { + calls.push(args); + if (args[2] === 'text') value += args[3]; + if (args.includes('KEYCODE_DEL')) value = ''; + return { exitCode: 0, stdout: '', stderr: '' }; + } + if (args[0] === 'exec-out') { + calls.push(args); + return { exitCode: 0, stdout: '', stderr: '', stdoutBuffer: png }; + } + return await exec(args, options); + }; + return { + calls, + requests, + processes, + provider: { ...provider, snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT }, + }; +} + +const refused = { + ok: 'false', + errorType: 'java.lang.IllegalStateException', + message: 'No editable field at the fill point', +}; + +test('a validated no-injection refusal completes fill through adb without a helper resend', async () => { + const f = fixture({ ...refused, injected: 'false' }); + await withAndroidAdbProvider(f.provider, { serial: device.id }, async () => { + await fillAndroid(device, 10, 10, 'chips', 0, helper); + expect((await readAndroidFillTargetBeforeMutation(device, 10, 10, helper))?.text).toBe('chips'); + }); + expect(f.calls.filter((args) => args[0] === 'shell' && args[2] === 'text')).toHaveLength(1); + expect(f.requests.filter((request) => request.startsWith('input '))).toHaveLength(1); + expect(f.requests.some((request) => request.startsWith('quit '))).toBe(true); +}); + +test.each([undefined, 'true', 'unknown'])( + 'the same refusal with injected=%s never resends input', + async (injected) => { + const f = fixture(injected ? { ...refused, injected } : refused); + await expect( + withAndroidAdbProvider(f.provider, { serial: device.id }, () => + fillAndroid(device, 10, 10, 'chips', 0, helper), + ), + ).rejects.toMatchObject({ details: { dispatched: 'unknown' } }); + expect(f.calls.some((args) => args[0] === 'shell' && args[1] === 'input')).toBe(false); + }, +); + +test.each([ + { + ok: 'false', + errorType: 'java.lang.IllegalStateException', + message: 'Screen capture was unavailable', + }, + { ok: 'true', requestId: 'wrong-request' }, +])( + 'a failed helper screenshot %j stops its session and captures a PNG through screencap', + async (response) => { + const f = fixture(response); + const dir = await mkdtempForTest('android-helper-screenshot-fallback-'); + const output = path.join(dir, 'fallback.png'); + await withAndroidAdbProvider(f.provider, { serial: device.id }, async () => { + await readAndroidFillTargetBeforeMutation(device, 10, 10, helper); + await screenshotAndroid(device, output); + }); + expect(await fs.readFile(output)).toEqual(png); + expect(f.requests.filter((request) => request.startsWith('screenshot '))).toHaveLength(1); + expect(f.requests.some((request) => request.startsWith('quit '))).toBe(true); + expect( + f.calls.filter((args) => args[0] === 'exec-out' && args.includes('screencap')), + ).toHaveLength(1); + }, +); + +test('a cached ended helper is retired before a pointer command takes the host path', async () => { + const f = fixture({ ok: 'true' }); + await withAndroidAdbProvider(f.provider, { serial: device.id }, async () => { + await readAndroidFillTargetBeforeMutation(device, 10, 10, helper); + const exited = once(f.processes[0]!, 'close'); + f.processes[0]!.kill(); + await exited; + await pressAndroid(device, 10, 10); + }); + expect(f.requests.some((request) => request.startsWith('input '))).toBe(false); + expect(f.calls.filter((args) => args[0] === 'shell' && args[2] === 'tap')).toHaveLength(1); + expect(f.calls.some((args) => args[0] === 'forward' && args[1] === '--remove')).toBe(true); +}); + +test.each([ + ['tap', () => pressAndroid(device, 10, 10)], + ['key', () => backAndroid(device)], +] as const)( + 'a typed no-injection %s refusal uses the host input path once', + async (_kind, action) => { + const f = fixture({ ...refused, injected: 'false' }); + await withAndroidAdbProvider(f.provider, { serial: device.id }, async () => { + await readAndroidFillTargetBeforeMutation(device, 10, 10, helper); + await action(); + }); + expect(f.requests.filter((request) => request.startsWith('input '))).toHaveLength(1); + expect(f.calls.filter((args) => args[0] === 'shell' && args[1] === 'input')).toHaveLength(1); + }, +); + +test('an uncorrelated no-injection reply cannot authorize a mutation resend', async () => { + const f = fixture({ ...refused, injected: 'false', requestId: 'wrong-request' }); + await expect( + withAndroidAdbProvider(f.provider, { serial: device.id }, () => + fillAndroid(device, 10, 10, 'chips', 0, helper), + ), + ).rejects.toMatchObject({ details: { dispatched: 'unknown' } }); + expect(f.calls.some((args) => args[0] === 'shell' && args[1] === 'input')).toBe(false); +}); diff --git a/packages/platform-android/src/automation-helper.ts b/packages/platform-android/src/automation-helper.ts index e6416963a3..bf6aa93f4f 100644 --- a/packages/platform-android/src/automation-helper.ts +++ b/packages/platform-android/src/automation-helper.ts @@ -1,3 +1,4 @@ +import { AppError } from '@agent-device/kernel/errors'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { getLiveAndroidSnapshotHelperSession, @@ -16,14 +17,21 @@ export type AndroidHelperInput = | { kind: 'key'; keyCode: number } | { kind: 'fill'; x: number; y: number; text: string }; +const ACTION_POLICY = { + input: { mutation: true, timeoutMs: 30_000 }, + screenshot: { mutation: false, timeoutMs: 30_000 }, + focus: { mutation: false, timeoutMs: 2_000 }, +} as const; + /** Reuses the session's automation context without starting instrumentation for sessionless input. */ export async function runAndroidAutomationHelper( device: DeviceInfo, - action: 'input' | 'screenshot' | 'focus', + action: keyof typeof ACTION_POLICY, input?: AndroidHelperInput, ): Promise | undefined> { const deviceKey = getAndroidSnapshotHelperSessionDeviceKey(device); - if (!getLiveAndroidSnapshotHelperSession(deviceKey)?.commands.has(action)) return undefined; + if (!(await getLiveAndroidSnapshotHelperSession(deviceKey))?.commands.has(action)) + return undefined; const prepared = await prepareAndroidAutomationHelper(device); try { return await runAndroidSnapshotHelperSessionTouchCommand({ @@ -31,16 +39,21 @@ export async function runAndroidAutomationHelper( action, helper: androidAutomationHelperIdentity(prepared.artifact), payloadBase64: input ? Buffer.from(JSON.stringify(input)).toString('base64') : undefined, - timeoutMs: action === 'focus' ? 2_000 : 30_000, + timeoutMs: ACTION_POLICY[action].timeoutMs, }); } catch (error) { - if (action !== 'focus') throw error; + if ( + ACTION_POLICY[action].mutation && + !(error instanceof AppError && error.details?.dispatched === 'no') + ) + throw error; await stopAndroidSnapshotHelperSession(deviceKey); emitDiagnostic({ level: 'warn', - phase: 'android_helper_focus_fallback', + phase: `android_helper_${action}_fallback`, data: { error: error instanceof Error ? error.message : String(error), + dispatched: error instanceof AppError ? error.details?.dispatched : undefined, }, }); return undefined; diff --git a/packages/platform-android/src/fill-verification-text-read.test.ts b/packages/platform-android/src/fill-verification-text-read.test.ts new file mode 100644 index 0000000000..96ae6ecdcb --- /dev/null +++ b/packages/platform-android/src/fill-verification-text-read.test.ts @@ -0,0 +1,27 @@ +import { expect, test } from 'vitest'; +import { readAndroidTextAtPointInHierarchy } from './fill-verification.ts'; +const row = (id: string, text: string, bounds: string) => + ``; +const identity = { + identifier: 'com.example:id/title', + bundleId: 'com.example', + type: 'android.widget.TextView', +}; + +test('a shared row resource id uses the live text at the requested point', () => { + const xml = `${row(identity.identifier, 'first live', '[0,0][200,100]')}${row(identity.identifier, 'second live', '[0,100][200,200]')}`; + expect(readAndroidTextAtPointInHierarchy(xml, 10, 150, identity)).toBe('second live'); +}); + +test('a missing or ambiguous identity preserves a readable point', () => { + const xml = `${row(identity.identifier, 'first', '[0,0][200,100]')}${row(identity.identifier, 'second', '[0,100][200,200]')}${row('other', 'point value', '[0,200][200,300]')}`; + expect(readAndroidTextAtPointInHierarchy(xml, 10, 250, identity)).toBe('point value'); + expect( + readAndroidTextAtPointInHierarchy(xml, 10, 250, { ...identity, identifier: 'missing' }), + ).toBe('point value'); +}); + +test('a unique identity can still read a field that moved away from the recorded point', () => { + const xml = `${row(identity.identifier, 'moved live', '[0,100][200,200]')}`; + expect(readAndroidTextAtPointInHierarchy(xml, 10, 10, identity)).toBe('moved live'); +}); diff --git a/packages/platform-android/src/fill-verification.ts b/packages/platform-android/src/fill-verification.ts index 01289f6874..4d35174716 100644 --- a/packages/platform-android/src/fill-verification.ts +++ b/packages/platform-android/src/fill-verification.ts @@ -234,8 +234,13 @@ export function readAndroidTextAtPointInHierarchy( (!target.bundleId || node.packageName === target.bundleId) && (!target.type || node.className === target.type), ); - if (matches.length !== 1) return null; - return androidFillCandidateFromNode(matches[0]!, {})?.text ?? null; + const candidates = matches.flatMap((node) => { + const candidate = androidFillCandidateFromNode(node, {}); + return candidate ? [candidate] : []; + }); + if (matches.length === 1 && candidates.length === 1) return candidates[0]!.text; + const atPoint = candidates.filter((candidate) => containsPoint(candidate.rect, x, y)); + if (atPoint.length === 1) return atPoint[0]!.text; } // Reads are point-targeted: a focused sibling may be a different app field or an IME // composing surface, so it must not override the node that contains the requested point. diff --git a/packages/platform-android/src/snapshot-helper-session-lifecycle.ts b/packages/platform-android/src/snapshot-helper-session-lifecycle.ts index 853e3c0640..72be1819e7 100644 --- a/packages/platform-android/src/snapshot-helper-session-lifecycle.ts +++ b/packages/platform-android/src/snapshot-helper-session-lifecycle.ts @@ -148,10 +148,15 @@ export async function acquireAndroidSnapshotHelperSession( * The live session for a device, or `undefined` when none is running. Commands that may only * piggyback on an existing session — never start one — read ownership through this. */ -export function getLiveAndroidSnapshotHelperSession( +export async function getLiveAndroidSnapshotHelperSession( deviceKey: string, -): AndroidSnapshotHelperSession | undefined { - return sessions().get(deviceKey); +): Promise { + const session = sessions().get(deviceKey); + if (session && hasAndroidSnapshotHelperProcessEnded(session.process)) { + await stopAndroidSnapshotHelperSession(deviceKey, { force: true }); + return undefined; + } + return session; } async function resolveAndroidSnapshotHelperSession(params: { diff --git a/packages/platform-android/src/snapshot-helper-session.ts b/packages/platform-android/src/snapshot-helper-session.ts index cd157302e5..ce64b0825a 100644 --- a/packages/platform-android/src/snapshot-helper-session.ts +++ b/packages/platform-android/src/snapshot-helper-session.ts @@ -120,7 +120,7 @@ export async function runAndroidSnapshotHelperSessionTouchCommand(params: { payloadBase64?: string; timeoutMs: number; }): Promise | undefined> { - const session = getLiveAndroidSnapshotHelperSession(params.deviceKey); + const session = await getLiveAndroidSnapshotHelperSession(params.deviceKey); if (!session) return undefined; if (!matchesSessionHelperIdentity(session.helper, params.helper)) { // A different helper binary was selected for this device (e.g. a provider-supplied artifact). @@ -159,14 +159,14 @@ export async function runAndroidSnapshotHelperSessionTouchCommand(params: { } if (headers.ok !== 'true') { // The helper ran and reported a structured failure; the session itself stays healthy. - throw discloseHelperTouchDispatch( - params.action, - new AppError( - 'COMMAND_FAILED', - headers.message || headers.errorType || `Android automation helper ${params.action} failed`, - { errorType: headers.errorType, helper: headers }, - ), + const failure = new AppError( + 'COMMAND_FAILED', + headers.message || headers.errorType || `Android automation helper ${params.action} failed`, + { errorType: headers.errorType, helper: headers }, ); + throw params.action === 'input' && headers.injected === 'false' + ? discloseDispatch(failure, 'no') + : discloseHelperTouchDispatch(params.action, failure); } return headers; } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index d34e6c3b85..a7b17254db 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -962,7 +962,7 @@ agent-device screenshot page.png # Explicit screenshot path agent-device screenshot page.png --scale 0.3 # Resize both dimensions to 30% for agent context agent-device screenshot page.png --overlay-refs # Draw current @eN refs and target rectangles onto the PNG agent-device screenshot page.png --crop-on 'label="Save"' # Crop the capture to the frame the selector resolves on the same screen -agent-device screenshot baseline.png --normalize-status-bar # Normalize iOS simulator or Android chrome for reusable diff baselines +agent-device screenshot baseline.png --normalize-status-bar # Normalize iOS simulator or ADB-backed Android chrome for reusable diff baselines agent-device screenshot page.png --platform web --fullscreen # On web, --fullscreen/--full/-f captures the entire document agent-device viewport 1280 900 --platform web # Resize the active web viewport for fixed-layout or 100vh apps agent-device screenshot textedit.png # App-session window capture on macOS @@ -983,7 +983,7 @@ agent-device record stop # Stop active recording **Screenshots** - `--scale ` resizes both dimensions, from `0.01` to `1` (full resolution). Set `AGENT_DEVICE_SCREENSHOT_SCALE=0.3` or `screenshotScale` in config as a token-conscious default; an explicit `--scale` overrides it. Leave it unset or pass `--scale 1` for pixel-diff baselines. `--max-size` is refused everywhere (CLI, `.ad` scripts, Node options, config, `AGENT_DEVICE_SCREENSHOT_MAX_SIZE`) with guidance to use `--scale`. -- Android screenshots capture immediately by default. Use `--normalize-status-bar` to set deterministic status-bar chrome and wait for transient UI to fade. In the Node client, `stabilize: true` also enables this wait. +- ADB-backed Android screenshots capture immediately by default. Use `--normalize-status-bar` to set deterministic status-bar chrome and wait for transient UI to fade. In the Node client, `stabilize: true` also enables this wait. - `--overlay-refs` takes a fresh snapshot and burns visible `@eN` refs and their rectangles into the PNG. With `--scale`, refs are drawn at the final size, so avoid scales where labels become unreadable. In `--json` mode each ref also has a screenshot-space `center` for `press ` fallback. - `--crop-on ` crops the capture to the frame the selector resolves on the same screen. It works only on iOS simulators and Android emulators, needs exactly one framed node, cannot be combined with `--overlay-refs` or `--fullscreen`, and warns when the frame is clipped. The crop is re-encoded, so do not byte-compare it with an older crop. - `--fullscreen` on macOS applies only to app sessions. Other surfaces (`desktop`, `menubar`, `frontmost-app`) always capture the main display and refuse `--fullscreen` with `INVALID_ARGS`. From 157a4a7720f3974ffef1afb8992ee97976a747bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 17:36:05 +0200 Subject: [PATCH 7/9] fix(android): await helper admission for transient capture --- .../src/__tests__/text-input-fill.test.ts | 2 +- .../src/automation-helper-fallback.test.ts | 2 +- packages/platform-android/src/fill-verification.ts | 9 ++++++++- packages/platform-android/src/snapshot.ts | 9 +++++---- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/packages/platform-android/src/__tests__/text-input-fill.test.ts b/packages/platform-android/src/__tests__/text-input-fill.test.ts index c00963c970..2e285d8f9e 100644 --- a/packages/platform-android/src/__tests__/text-input-fill.test.ts +++ b/packages/platform-android/src/__tests__/text-input-fill.test.ts @@ -858,6 +858,6 @@ test('a live text read keeps the resolved identity when its old point covers a s ); assert.equal( readAndroidTextAtPointInHierarchy(xml.replace('app-state', 'window-state'), 50, 50, target), - null, + 'active', ); }); diff --git a/packages/platform-android/src/automation-helper-fallback.test.ts b/packages/platform-android/src/automation-helper-fallback.test.ts index 51b44a3d99..d45642d0aa 100644 --- a/packages/platform-android/src/automation-helper-fallback.test.ts +++ b/packages/platform-android/src/automation-helper-fallback.test.ts @@ -91,7 +91,7 @@ test.each([undefined, 'true', 'unknown'])( }, ); -test.each([ +test.each>([ { ok: 'false', errorType: 'java.lang.IllegalStateException', diff --git a/packages/platform-android/src/fill-verification.ts b/packages/platform-android/src/fill-verification.ts index 4d35174716..3f7cdb0f3a 100644 --- a/packages/platform-android/src/fill-verification.ts +++ b/packages/platform-android/src/fill-verification.ts @@ -244,7 +244,14 @@ export function readAndroidTextAtPointInHierarchy( } // Reads are point-targeted: a focused sibling may be a different app field or an IME // composing surface, so it must not override the node that contains the requested point. - return inspectAndroidTextAtPointInHierarchy(xml, x, y).targetInput?.text ?? null; + const pointTarget = inspectAndroidTextAtPointInHierarchy(xml, x, y).targetInput; + if ( + !pointTarget || + (target?.bundleId && pointTarget.packageName !== target.bundleId) || + (target?.type && pointTarget.className !== target.type) + ) + return null; + return pointTarget.text; } export function androidFillFailureMessage(verification: AndroidFillVerification | null): string { diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index b4fdce54ea..c107161876 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -313,7 +313,7 @@ async function captureAndroidUiHierarchyWithHelper( artifact: AndroidSnapshotHelperArtifact, ): Promise { const helperDeviceKey = getAndroidSnapshotHelperSessionDeviceKey(device); - const releaseHelperSession = releasesHelperSessionAfterCapture(options, helperDeviceKey); + const releaseHelperSession = await releasesHelperSessionAfterCapture(options, helperDeviceKey); try { return await captureAndroidHelperContentWithinWindow({ options, @@ -385,11 +385,12 @@ function resolveContentRecaptureDeadlineMs(options: AndroidSnapshotOptions): num } /** A transient read keeps a session it found running and releases one it had to start. */ -function releasesHelperSessionAfterCapture( +async function releasesHelperSessionAfterCapture( options: AndroidSnapshotOptions, helperDeviceKey: string, -): boolean { - if (options.transient) return getLiveAndroidSnapshotHelperSession(helperDeviceKey) === undefined; +): Promise { + if (options.transient) + return (await getLiveAndroidSnapshotHelperSession(helperDeviceKey)) === undefined; return options.helperSessionScope !== 'daemon-session'; } From ecbba260accb89d801f02bb1b27d9150758383cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 17:42:01 +0200 Subject: [PATCH 8/9] docs(evidence): retain case results with shared tuple labels --- .../android-e2e-parity-2026-10-19-cases.json | 412 ++++++------------ 1 file changed, 139 insertions(+), 273 deletions(-) diff --git a/docs/evidence/android-e2e-parity-2026-10-19-cases.json b/docs/evidence/android-e2e-parity-2026-10-19-cases.json index 315a2c9935..cfe8cef12f 100644 --- a/docs/evidence/android-e2e-parity-2026-10-19-cases.json +++ b/docs/evidence/android-e2e-parity-2026-10-19-cases.json @@ -73,188 +73,134 @@ "testId": "tests/controls.e2e.ts::control%20inventory::a%20label%20query%20resolves%20a%20labeled%20text%20field", "sourceLine": 32, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 3401, - "headStatus": "passed", - "headDurationMs": 3444, - "directStatus": "passed", - "directDurationMs": 1687 + "base": ["passed", 3401], + "head": ["passed", 3444], + "direct": ["passed", 1687] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20labeled%20field%20answers%20name%2C%20display%20value%2C%20value%20reads%2C%20fill%2C%20and%20clear", "sourceLine": 37, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 8183, - "headStatus": "passed", - "headDurationMs": 7941, - "directStatus": "passed", - "directDurationMs": 1894 + "base": ["passed", 8183], + "head": ["passed", 7941], + "direct": ["passed", 1894] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20placeholder%20query%20and%20attribute%20reads%20find%20the%20field", "sourceLine": 55, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 3131, - "headStatus": "passed", - "headDurationMs": 3040, - "directStatus": "passed", - "directDurationMs": 1534 + "base": ["passed", 3131], + "head": ["passed", 3040], + "direct": ["passed", 1534] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20secure%20field%20takes%20a%20fill%20and%20denies%20value%20reads", "sourceLine": 64, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 4627, - "headStatus": "passed", - "headDurationMs": 4370, - "directStatus": "passed", - "directDurationMs": 1553 + "base": ["passed", 4627], + "head": ["passed", 4370], + "direct": ["passed", 1553] }, { "testId": "tests/controls.e2e.ts::control%20inventory::press%20sends%20a%20character%20and%20Enter%20through%20the%20soft%20keyboard", "sourceLine": 72, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 5958, - "headStatus": "passed", - "headDurationMs": 5825, - "directStatus": "passed", - "directDurationMs": 1730 + "base": ["passed", 5958], + "head": ["passed", 5825], + "direct": ["passed", 1730] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20field%20autofocuses%20on%20mount%20and%20focus%28%29%20moves%20the%20focus", "sourceLine": 81, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 9066, - "headStatus": "passed", - "headDurationMs": 9064, - "directStatus": "passed", - "directDurationMs": 1872 + "base": ["passed", 9066], + "head": ["passed", 9064], + "direct": ["passed", 1872] }, { "testId": "tests/controls.e2e.ts::control%20inventory::toBeFocused%20follows%20the%20focus", "sourceLine": 97, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 8269, - "headStatus": "passed", - "headDurationMs": 8215, - "directStatus": "passed", - "directDurationMs": 2741 + "base": ["passed", 8269], + "head": ["passed", 8215], + "direct": ["passed", 2741] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20switch%20checks%2C%20unchecks%2C%20and%20reports%20its%20state", "sourceLine": 107, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 9219, - "headStatus": "passed", - "headDurationMs": 9006, - "directStatus": "passed", - "directDurationMs": 4075 + "base": ["passed", 9219], + "head": ["passed", 9006], + "direct": ["passed", 4075] }, { "testId": "tests/controls.e2e.ts::control%20inventory::checkbox%20and%20radio%20roles%20answer%20getByRole%2C%20and%20a%20checkbox%20toggles%20by%20tap", "sourceLine": 130, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 6393, - "headStatus": "passed", - "headDurationMs": 6137, - "directStatus": "passed", - "directDurationMs": 1886 + "base": ["passed", 6393], + "head": ["passed", 6137], + "direct": ["passed", 1886] }, { "testId": "tests/controls.e2e.ts::control%20inventory::checkbox%20and%20radio%20roles%20check%2C%20uncheck%2C%20and%20report%20their%20state", "sourceLine": 147, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 11014, - "headStatus": "passed", - "headDurationMs": 10877, - "directStatus": "passed", - "directDurationMs": 6367 + "base": ["passed", 11014], + "head": ["passed", 10877], + "direct": ["passed", 6367] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20tab%20role%20answers%20getByRole", "sourceLine": 180, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 2857, - "headStatus": "passed", - "headDurationMs": 2821, - "directStatus": "passed", - "directDurationMs": 1325 + "base": ["passed", 2857], + "head": ["passed", 2821], + "direct": ["passed", 1325] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20radio%20group%20reports%20the%20checked%20option", "sourceLine": 188, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 8487, - "headStatus": "passed", - "headDurationMs": 8240, - "directStatus": "passed", - "directDurationMs": 3923 + "base": ["passed", 8487], + "head": ["passed", 8240], + "direct": ["passed", 3923] }, { "testId": "tests/controls.e2e.ts::control%20inventory::buttons%20report%20disabled%2C%20enabled%2C%20and%20a%20state%20that%20arrives%20later", "sourceLine": 201, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 7600, - "headStatus": "passed", - "headDurationMs": 7459, - "directStatus": "passed", - "directDurationMs": 3194 + "base": ["passed", 7600], + "head": ["passed", 7459], + "direct": ["passed", 3194] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20display%3A%20none%20element%20is%20absent%20rather%20than%20hidden%2C%20its%20visible%20twin%20attached", "sourceLine": 219, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 5643, - "headStatus": "passed", - "headDurationMs": 5611, - "directStatus": "passed", - "directDurationMs": 2368 + "base": ["passed", 5643], + "head": ["passed", 5611], + "direct": ["passed", 2368] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20header%20answers%20its%20accessible%20name", "sourceLine": 235, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 2932, - "headStatus": "passed", - "headDurationMs": 2873, - "directStatus": "passed", - "directDurationMs": 1298 + "base": ["passed", 2932], + "head": ["passed", 2873], + "direct": ["passed", 1298] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20tab%20row%20answers%20count%2C%20all%2C%20texts%2C%20filters%2C%20positions%2C%20and%20switching", "sourceLine": 244, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 4790, - "headStatus": "passed", - "headDurationMs": 4715, - "directStatus": "passed", - "directDurationMs": 1727 + "base": ["passed", 4790], + "head": ["passed", 4715], + "direct": ["passed", 1727] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20list%20answers%20counts%2C%20list-form%20text%2C%20reads%2C%20and%20boxes%20in%20order", "sourceLine": 271, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 4614, - "headStatus": "passed", - "headDurationMs": 4636, - "directStatus": "failed", - "directDurationMs": 647, "directError": { "category": "test", "code": "ACTION_FAILED", @@ -266,7 +212,10 @@ "line": 81, "column": 3 } - } + }, + "base": ["passed", 4614], + "head": ["passed", 4636], + "direct": ["failed", 647] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20row%20contains%20its%20texts", @@ -276,34 +225,25 @@ "cause": "explicit", "reason": "React Native flattens a plain View out of the accessibility tree on both platforms: XCTest and the Android helper report a View with a testID as a leaf beside its children, so filter({ has }) and a query scoped to it match nothing; only ScrollView, TextInput, and Text hosts nest" }, - "baseStatus": "skipped", - "baseDurationMs": null, - "headStatus": "skipped", - "headDurationMs": null, - "directStatus": "skipped", - "directDurationMs": null + "base": ["skipped", null], + "head": ["skipped", null], + "direct": ["skipped", null] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20long%20press%20lands%20as%20one", "sourceLine": 312, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 6628, - "headStatus": "passed", - "headDurationMs": 6488, - "directStatus": "passed", - "directDurationMs": 2343 + "base": ["passed", 6628], + "head": ["passed", 6488], + "direct": ["passed", 2343] }, { "testId": "tests/controls.e2e.ts::control%20inventory::a%20double%20tap%20lands%20inside%20the%20300%20ms%20window", "sourceLine": 327, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 7382, - "headStatus": "passed", - "headDurationMs": 7411, - "directStatus": "passed", - "directDurationMs": 1730 + "base": ["passed", 7382], + "head": ["passed", 7411], + "direct": ["passed", 1730] }, { "testId": "tests/controls.e2e.ts::control%20inventory::pointer%20verbs%20land%20at%20the%20point%20asked%20for", @@ -313,144 +253,105 @@ "cause": "platform-unavailable", "reason": "test declares platforms [ios]" }, - "baseStatus": "skipped", - "baseDurationMs": null, - "headStatus": "skipped", - "headDurationMs": null, - "directStatus": "skipped", - "directDurationMs": null + "base": ["skipped", null], + "head": ["skipped", null], + "direct": ["skipped", null] }, { "testId": "tests/controls.e2e.ts::control%20inventory::an%20ambiguous%20locator%20fails%20at%20once", "sourceLine": 384, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 4335, - "headStatus": "passed", - "headDurationMs": 4273, - "directStatus": "passed", - "directDurationMs": 1482 + "base": ["passed", 4335], + "head": ["passed", 4273], + "direct": ["passed", 1482] }, { "testId": "tests/controls.e2e.ts::control%20inventory::screenshot%2C%20restart%2C%20and%20clearState%20reopen%20the%20app%20on%20its%20home%20list", "sourceLine": 390, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 7884, - "headStatus": "passed", - "headDurationMs": 6570, - "directStatus": "passed", - "directDurationMs": 2483 + "base": ["passed", 7884], + "head": ["passed", 6570], + "direct": ["passed", 2483] }, { "testId": "tests/controls.e2e.ts::refusals::verbs%20a%20device%20never%20declares%20are%20refused%20before%20the%20engine", "sourceLine": 404, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 1, - "headStatus": "passed", - "headDurationMs": 1, - "directStatus": "passed", - "directDurationMs": 3 + "base": ["passed", 1], + "head": ["passed", 1], + "direct": ["passed", 3] }, { "testId": "tests/controls.e2e.ts::refusals::keys%20the%20soft%20keyboard%20cannot%20send%20are%20refused%20by%20the%20engine", "sourceLine": 418, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 157, - "headStatus": "passed", - "headDurationMs": 165, - "directStatus": "passed", - "directDurationMs": 165 + "base": ["passed", 157], + "head": ["passed", 165], + "direct": ["passed", 165] }, { "testId": "tests/controls.e2e.ts::refusals::a%20missing%20locator%20fails%20as%20not%20found%20once%20its%20timeout%20passes", "sourceLine": 429, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 1107, - "headStatus": "passed", - "headDurationMs": 1113, - "directStatus": "passed", - "directDurationMs": 1072 + "base": ["passed", 1107], + "head": ["passed", 1113], + "direct": ["passed", 1072] }, { "testId": "tests/flows-lists.e2e.ts::infinite%20scroll%20reaches%20the%20target%20item", "sourceLine": 9, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 14782, - "headStatus": "passed", - "headDurationMs": 14316, - "directStatus": "passed", - "directDurationMs": 22370 + "base": ["passed", 14782], + "head": ["passed", 14316], + "direct": ["passed", 22370] }, { "testId": "tests/flows-lists.e2e.ts::product%20catalog%20adds%20exactly%20two%20of%20the%20right%20variant", "sourceLine": 17, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 9200, - "headStatus": "passed", - "headDurationMs": 8986, - "directStatus": "passed", - "directDurationMs": 1990 + "base": ["passed", 9200], + "head": ["passed", 8986], + "direct": ["passed", 1990] }, { "testId": "tests/flows-lists.e2e.ts::gestures%3A%20long-press%2C%20then%20swipe%20left", "sourceLine": 27, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 7467, - "headStatus": "passed", - "headDurationMs": 7339, - "directStatus": "passed", - "directDurationMs": 13198 + "base": ["passed", 7467], + "head": ["passed", 7339], + "direct": ["passed", 13198] }, { "testId": "tests/flows-lists.e2e.ts::gestures%3A%20double-tap%20completes%20the%20flow", "sourceLine": 40, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 8981, - "headStatus": "passed", - "headDurationMs": 8745, - "directStatus": "passed", - "directDurationMs": 3109 + "base": ["passed", 8981], + "head": ["passed", 8745], + "direct": ["passed", 3109] }, { "testId": "tests/flows-lists.e2e.ts::sticky%20chrome%3A%20accept%20below%20the%20fold%2C%20then%20continue", "sourceLine": 50, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 10534, - "headStatus": "passed", - "headDurationMs": 10532, - "directStatus": "passed", - "directDurationMs": 13379 + "base": ["passed", 10534], + "head": ["passed", 10532], + "direct": ["passed", 13379] }, { "testId": "tests/flows-lists.e2e.ts::async%20states%3A%20load%2C%20pull%20to%20refresh%2C%20claim%20through%20the%20toast", "sourceLine": 62, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 10734, - "headStatus": "passed", - "headDurationMs": 10900, - "directStatus": "passed", - "directDurationMs": 7623 + "base": ["passed", 10734], + "head": ["passed", 10900], + "direct": ["passed", 7623] }, { "testId": "tests/flows-lists.e2e.ts::huge%20virtualized%20list%20reaches%20row%20512", "sourceLine": 102, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 50998, - "headStatus": "passed", - "headDurationMs": 48378, - "directStatus": "passed", - "directDurationMs": 50821 + "base": ["passed", 50998], + "head": ["passed", 48378], + "direct": ["passed", 50821] }, { "testId": "tests/flows-lists.e2e.ts::web%20view%20coupon%20form%20applies%20the%20code%20shown%20on%20the%20page", @@ -460,45 +361,30 @@ "cause": "platform-unavailable", "reason": "test declares platforms [ios]" }, - "baseStatus": "skipped", - "baseDurationMs": null, - "headStatus": "skipped", - "headDurationMs": null, - "directStatus": "skipped", - "directDurationMs": null + "base": ["skipped", null], + "head": ["skipped", null], + "direct": ["skipped", null] }, { "testId": "tests/flows.e2e.ts::text%20inputs%20unlock%20the%20submit%20button", "sourceLine": 11, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 5832, - "headStatus": "passed", - "headDurationMs": 7074, - "directStatus": "passed", - "directDurationMs": 1987 + "base": ["passed", 5832], + "head": ["passed", 7074], + "direct": ["passed", 1987] }, { "testId": "tests/flows.e2e.ts::filled%20text%20inputs%20report%20their%20values", "sourceLine": 25, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 5159, - "headStatus": "passed", - "headDurationMs": 5929, - "directStatus": "passed", - "directDurationMs": 1569 + "base": ["passed", 5159], + "head": ["passed", 5929], + "direct": ["passed", 1569] }, { "testId": "tests/flows.e2e.ts::modal%20flow%20confirms%20through%20the%20native%20alert", "sourceLine": 36, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 7689, - "headStatus": "passed", - "headDurationMs": 7922, - "directStatus": "failed", - "directDurationMs": 1767, "directError": { "category": "test", "code": "ACTION_FAILED", @@ -510,95 +396,75 @@ "line": 39, "column": 3 } - } + }, + "base": ["passed", 7689], + "head": ["passed", 7922], + "direct": ["failed", 1767] }, { "testId": "tests/flows.e2e.ts::bottom%20tabs%20act%20inside%20the%20last%20tab", "sourceLine": 49, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 6006, - "headStatus": "passed", - "headDurationMs": 5767, - "directStatus": "passed", - "directDurationMs": 12224 + "base": ["passed", 6006], + "head": ["passed", 5767], + "direct": ["passed", 12224] }, { "testId": "tests/flows.e2e.ts::error%20recovery%20retries%2C%20then%20confirms%20the%20delete", "sourceLine": 63, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 7745, - "headStatus": "passed", - "headDurationMs": 7811, - "directStatus": "passed", - "directDurationMs": 4528 + "base": ["passed", 7745], + "head": ["passed", 7811], + "direct": ["passed", 4528] }, { "testId": "tests/flows.e2e.ts::choice%20controls%20place%20the%20exact%20order", "sourceLine": 72, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 7851, - "headStatus": "passed", - "headDurationMs": 7833, - "directStatus": "passed", - "directDurationMs": 1760 + "base": ["passed", 7851], + "head": ["passed", 7833], + "direct": ["passed", 1760] }, { "testId": "tests/flows.e2e.ts::debounced%20search%20selects%20the%20target%20once%20results%20arrive", "sourceLine": 93, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 7950, - "headStatus": "passed", - "headDurationMs": 7670, - "directStatus": "passed", - "directDurationMs": 3032 + "base": ["passed", 7950], + "head": ["passed", 7670], + "direct": ["passed", 3032] }, { "testId": "tests/login-form.e2e.ts::login%20form::rejects%20a%20malformed%20email", "sourceLine": 18, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 8165, - "headStatus": "passed", - "headDurationMs": 7710, - "directStatus": "passed", - "directDurationMs": 2012 + "base": ["passed", 8165], + "head": ["passed", 7710], + "direct": ["passed", 2012] }, { "testId": "tests/login-form.e2e.ts::login%20form::rejects%20an%20unknown%20account", "sourceLine": 26, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 8146, - "headStatus": "passed", - "headDurationMs": 7800, - "directStatus": "passed", - "directDurationMs": 2082 + "base": ["passed", 8146], + "head": ["passed", 7800], + "direct": ["passed", 2082] }, { "testId": "tests/login-form.e2e.ts::login%20form::signs%20in%20and%20logs%20out", "sourceLine": 34, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 12211, - "headStatus": "passed", - "headDurationMs": 11504, - "directStatus": "passed", - "directDurationMs": 6956 + "base": ["passed", 12211], + "head": ["passed", 11504], + "direct": ["passed", 6956] }, { "testId": "tests/smoke.e2e.ts::home%20lists%20the%20scenarios%20and%20opens%20one", "sourceLine": 5, "selected": true, - "baseStatus": "passed", - "baseDurationMs": 4330, - "headStatus": "passed", - "headDurationMs": 4246, - "directStatus": "passed", - "directDurationMs": 2336 + "base": ["passed", 4330], + "head": ["passed", 4246], + "direct": ["passed", 2336] } - ] + ], + "caseResultTuple": ["status", "durationMs"] } From 1977e685b23b20b5d66bed85b62fad5b169d3af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 18:01:03 +0200 Subject: [PATCH 9/9] test(android): preserve cancellable input across helper rebase --- .../src/automation-helper-fallback.test.ts | 20 +++++++++++++++++++ .../platform-android/src/input-actions.ts | 12 +++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/platform-android/src/automation-helper-fallback.test.ts b/packages/platform-android/src/automation-helper-fallback.test.ts index d45642d0aa..7bbc5ff057 100644 --- a/packages/platform-android/src/automation-helper-fallback.test.ts +++ b/packages/platform-android/src/automation-helper-fallback.test.ts @@ -156,3 +156,23 @@ test('an uncorrelated no-injection reply cannot authorize a mutation resend', as ).rejects.toMatchObject({ details: { dispatched: 'unknown' } }); expect(f.calls.some((args) => args[0] === 'shell' && args[1] === 'input')).toBe(false); }); + +test.each(['tap', 'back'])( + 'cancellable %s preserves the host signal with a live helper', + async (kind) => { + const f = fixture({ ok: 'true' }); + const signal = new AbortController().signal; + const exec = f.provider.exec; + f.provider.exec = async (args, options) => { + if (args[0] === 'shell' && args[1] === 'input') expect(options?.signal).toBe(signal); + return await exec(args, options); + }; + await withAndroidAdbProvider(f.provider, { serial: device.id }, async () => { + await readAndroidFillTargetBeforeMutation(device, 10, 10, helper); + if (kind === 'tap') await pressAndroid(device, 10, 10, { signal }); + else await backAndroid(device, { signal }); + }); + expect(f.requests.some((request) => request.startsWith('input '))).toBe(false); + expect(f.calls.filter((args) => args[0] === 'shell' && args[1] === 'input')).toHaveLength(1); + }, +); diff --git a/packages/platform-android/src/input-actions.ts b/packages/platform-android/src/input-actions.ts index e0d24806ca..a36d5fbfeb 100644 --- a/packages/platform-android/src/input-actions.ts +++ b/packages/platform-android/src/input-actions.ts @@ -45,7 +45,11 @@ export async function pressAndroid( ): Promise { const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); try { - if (!options?.signal && (await runAndroidAutomationHelper(device, 'input', { kind: 'tap', x, y }))) return; + if ( + !options?.signal && + (await runAndroidAutomationHelper(device, 'input', { kind: 'tap', x, y })) + ) + return; // Picked field by field so no runtime caller can smuggle `allowFailure: true` past the type. await runAndroidShell(device, ['input', 'tap', x, y], { allowFailure: false, @@ -86,7 +90,11 @@ export async function backAndroid( options?: AndroidInputDispatchOptions, ): Promise { const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); - if (!options?.signal && (await runAndroidAutomationHelper(device, 'input', { kind: 'key', keyCode: 4 }))) return; + if ( + !options?.signal && + (await runAndroidAutomationHelper(device, 'input', { kind: 'key', keyCode: 4 })) + ) + return; await runAndroidShell( device, ['input', 'keyevent', '4'],