perf(android): reuse live helper for input and capture

android/snapshot-helper/README.md

@@ -95,11 +95,17 @@ connection: the client connects, sends a single command line, reads the response
 closes that connection; the process stays alive for the next connection:
 
 - `snapshot <requestId>` — capture and return an XML snapshot, same semantics as the default mode
+- `snapshot-immediate <requestId>` — capture fresh XML without an idle wait when the caller owns settling
+- `input <requestId> <payloadBase64>` — inject a tap, key, or point-targeted text replacement
+- `focus <requestId>` — read the focused window type and root package after clearing the accessibility cache
+- `screenshot <requestId>` — return a PNG in the `payloadBase64` header
 - `viewport <requestId>` — return interactive-window viewport bounds
 - `gesture <requestId> <payloadBase64>` — inject a planned touch gesture (same payload as the
   one-shot `gesture` mode)
 - `quit <requestId>` — acknowledge and stop the session
 
+The ready record advertises optional operations in `sessionCommands`. A host must check support before sending them. Text replacement retains a focus tap; the host owns post-write verification. Input failures must not be resent because a mutation may already have reached the app.
+
 `viewport` and `gesture` responses are headers-only (no body): `kind`, `injectedEvents`,
 `elapsedMs` for `gesture`, and `x`, `y`, `width`, `height` for `viewport`. `snapshot` responses carry
 the XML body after the header block, as described below. The response protocol literal is always

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityNodeReads.java

@@ -18,6 +18,11 @@ final class AccessibilityNodeReads {
 
   private AccessibilityNodeReads() {}
 
+  static AccessibilityNodeInfo rootWithoutPrefetch(AccessibilityWindowInfo window) {
+    return Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU
+        ? window.getRoot(0) : window.getRoot();
+  }
+
   static AccessibilityNodeInfo root(AccessibilityWindowInfo window) {
     if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
       return window.getRoot(PREFETCH_STRATEGY);

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java

@@ -90,7 +90,7 @@ private static Result captureOnce(UiAutomation automation, int maxDepth, int max
         missingRootWindowTypes);
   }
 
-  private static void clearAccessibilityCache(UiAutomation automation) {
+  static void clearAccessibilityCache(UiAutomation automation) {
     // Navigation 3 and any in-place content swap that keeps the same window/Activity render every
     // destination inside one AndroidComposeView. A persistent helper session reuses a single
     // UiAutomation connection across captures, and its per-connection accessibility node cache is

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java

@@ -0,0 +1,213 @@
+package com.callstack.agentdevice.snapshothelper;
+
+import android.app.UiAutomation;
+import android.graphics.Bitmap;
+import android.graphics.Rect;
+import android.os.Bundle;
+import android.os.SystemClock;
+import android.util.Base64;
+import android.view.InputDevice;
+import android.view.KeyCharacterMap;
+import android.view.KeyEvent;
+import android.view.MotionEvent;
+import android.view.accessibility.AccessibilityNodeInfo;
+import android.view.accessibility.AccessibilityWindowInfo;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import org.json.JSONObject;
+
+/** Executes session input and image capture inside the instrumentation's automation context. */
+final class AutomationCommandHandler {
+  private AutomationCommandHandler() {}
+
+  static void writeSessionResponse(
+      OutputStream output, String requestId, String action, String payload, UiAutomation automation)
+      throws IOException {
+    Bundle result = SessionResponseWriter.sessionResponseBundle(requestId);
+    InputDispatch dispatch = new InputDispatch();
+    try {
+      if ("screenshot".equals(action)) {
+        screenshot(result, automation);
+      } else if ("focus".equals(action)) {
+        focus(result, automation);
+      } else {
+        input(automation, new JSONObject(new String(Base64.decode(payload, Base64.DEFAULT), StandardCharsets.UTF_8)), dispatch);
+      }
+      result.putString("ok", "true");
+      SessionResponseWriter.writeSessionResponse(output, result, "");
+    } catch (Throwable error) {
+      result.putString("ok", "false");
+      result.putString("errorType", error.getClass().getName());
+      result.putString("message", error.getMessage() == null ? error.getClass().getName() : error.getMessage());
+      if ("input".equals(action) && !dispatch.attempted) result.putString("injected", "false");
+      SessionResponseWriter.writeSessionResponse(output, result, "");
+    }
+  }
+
+  private static final class InputDispatch {
+    boolean attempted;
+  }
+
+  private static void input(UiAutomation automation, JSONObject input, InputDispatch dispatch) throws Exception {
+    switch (input.getString("kind")) {
+      case "tap":
+        tap(automation, input.getDouble("x"), input.getDouble("y"), dispatch);
+        return;
+      case "key":
+        key(automation, input.getInt("keyCode"), dispatch);
+        return;
+      case "fill":
+        fill(automation, input.getInt("x"), input.getInt("y"), input.getString("text"), dispatch);
+        return;
+      default:
+        throw new IllegalArgumentException("Unknown input kind");
+    }
+  }
+
+  private static void tap(UiAutomation automation, double x, double y, InputDispatch dispatch) {
+    if (Double.isNaN(x) || Double.isInfinite(x) || Double.isNaN(y) || Double.isInfinite(y)
+        || x < 0 || y < 0) {
+      throw new IllegalArgumentException("Invalid tap coordinates");
+    }
+    long down = SystemClock.uptimeMillis();
+    touch(automation, down, MotionEvent.ACTION_DOWN, (float) x, (float) y, dispatch);
+    touch(automation, down, MotionEvent.ACTION_UP, (float) x, (float) y, dispatch);
+  }
+
+  private static void touch(UiAutomation automation, long down, int action, float x, float y, InputDispatch dispatch) {
+    MotionEvent event = MotionEvent.obtain(down, SystemClock.uptimeMillis(), action, x, y, 0);
+    event.setSource(InputDevice.SOURCE_TOUCHSCREEN);
+    try {
+      dispatch.attempted = true;
+      if (!automation.injectInputEvent(event, true)) {
+        throw new IllegalStateException("Touch injection was refused");
+      }
+    } finally {
+      event.recycle();
+    }
+  }
+
+  private static void key(UiAutomation automation, int code, InputDispatch dispatch) {
+    if (code < 0 || code > KeyEvent.getMaxKeyCode()) throw new IllegalArgumentException("Invalid key code");
+    long down = SystemClock.uptimeMillis();
+    for (int action : new int[] {KeyEvent.ACTION_DOWN, KeyEvent.ACTION_UP}) {
+      KeyEvent event = new KeyEvent(down, SystemClock.uptimeMillis(), action, code, 0, 0,
+          KeyCharacterMap.VIRTUAL_KEYBOARD, 0, 0, InputDevice.SOURCE_KEYBOARD);
+      dispatch.attempted = true;
+      if (!automation.injectInputEvent(event, true)) {
+        throw new IllegalStateException("Key injection was refused");
+      }
+    }
+  }
+
+  private static void fill(UiAutomation automation, int x, int y, String text, InputDispatch dispatch) {
+    AccessibilityNodeInfo target = null;
+    try {
+      for (AccessibilityWindowInfo window : automation.getWindows()) {
+        try {
+          if (window.getType() == AccessibilityWindowInfo.TYPE_INPUT_METHOD) continue;
+          AccessibilityNodeInfo root = AccessibilityNodeReads.root(window);
+          if (root == null) continue;
+          AccessibilityNodeInfo candidate;
+          try {
+            candidate = editableAtPoint(root, x, y, 0);
+          } finally {
+            root.recycle();
+          }
+          if (candidate == null) continue;
+          if (target != null) {
+            candidate.recycle();
+            throw new IllegalStateException("Multiple editable fields at the fill point");
+          }
+          target = candidate;
+        } finally {
+          window.recycle();
+        }
+      }
+      if (target == null) throw new IllegalStateException("No editable field at the fill point");
+      tap(automation, x, y, dispatch);
+      if (!target.refresh()) throw new IllegalStateException("Fill target disappeared after focus");
+      if (!target.isFocused() && !target.performAction(AccessibilityNodeInfo.ACTION_FOCUS)) {
+        throw new IllegalStateException("Field focus was refused");
+      }
+      Bundle arguments = new Bundle();
+      arguments.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text);
+      if (!target.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)) {
+        throw new IllegalStateException("Field text replacement was refused");
+      }
+    } finally {
+      if (target != null) target.recycle();
+    }
+  }
+
+  private static AccessibilityNodeInfo editableAtPoint(AccessibilityNodeInfo node, int x, int y, int depth) {
+    if (depth > 128) throw new IllegalStateException("Fill target traversal exceeded its depth limit");
+    Rect bounds = new Rect();
+    node.getBoundsInScreen(bounds);
+    AccessibilityNodeInfo target = null;
+    try {
+      for (int i = 0; i < node.getChildCount(); i++) {
+        AccessibilityNodeInfo child = AccessibilityNodeReads.child(node, i);
+        if (child == null) continue;
+        AccessibilityNodeInfo candidate;
+        try {
+          candidate = editableAtPoint(child, x, y, depth + 1);
+        } finally {
+          child.recycle();
+        }
+        if (candidate == null) continue;
+        if (target != null) {
+          candidate.recycle();
+          throw new IllegalStateException("Multiple editable fields at the fill point");
+        }
+        target = candidate;
+      }
+      if (target != null) {
+        AccessibilityNodeInfo found = target;
+        target = null;
+        return found;
+      }
+      return node.isEditable() && node.isEnabled() && node.isVisibleToUser() && bounds.contains(x, y)
+          ? AccessibilityNodeInfo.obtain(node) : null;
+    } finally {
+      if (target != null) target.recycle();
+    }
+  }
+
+  private static void screenshot(Bundle result, UiAutomation automation) {
+    Bitmap image = automation.takeScreenshot();
+    if (image == null) throw new IllegalStateException("Screen capture was unavailable");
+    try {
+      ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+      if (!image.compress(Bitmap.CompressFormat.PNG, 100, bytes)) {
+        throw new IllegalStateException("PNG encoding failed");
+      }
+      result.putString("payloadBase64", Base64.encodeToString(bytes.toByteArray(), Base64.NO_WRAP));
+    } finally {
+      image.recycle();
+    }
+  }
+
+  private static void focus(Bundle result, UiAutomation automation) throws Exception {
+    AccessibilityTreeCapture.prepare(automation, 0, 0);
+    AccessibilityTreeCapture.clearAccessibilityCache(automation);
+    for (AccessibilityWindowInfo window : automation.getWindows()) {
+      try {
+        if (!window.isFocused()) continue;
+        result.putString("windowType", Integer.toString(window.getType()));
+        AccessibilityNodeInfo root = AccessibilityNodeReads.rootWithoutPrefetch(window);
+        if (root == null) continue;
+        try {
+          CharSequence packageName = root.getPackageName();
+          if (packageName != null) result.putString("focusedPackage", packageName.toString());
+        } finally {
+          root.recycle();
+        }
+      } finally {
+        window.recycle();
+      }
+    }
+  }
+}

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java

@@ -180,6 +180,7 @@ private void runSnapshotSession(
       putBaseMetadata(
           ready, waitForIdleTimeoutMs, waitForIdleQuietMs, timeoutMs, maxDepth, maxNodes);
       ready.putString("sessionReady", "true");
+      ready.putString("sessionCommands", "input,screenshot,focus,snapshot-immediate");
       ready.putString("sessionPort", Integer.toString(sessionPort));
       sendStatus(2, ready);
 
@@ -201,6 +202,12 @@ private void runSnapshotSession(
             SessionResponseWriter.writeSessionOk(socket.getOutputStream(), requestId);
             return;
           }
+          if ("input".equals(action) || "screenshot".equals(action) || "focus".equals(action)) {
+            AutomationCommandHandler.writeSessionResponse(
+                socket.getOutputStream(), requestId, action,
+                parts.length > 2 ? parts[2] : "", getConnectedUiAutomationUnchecked());
+            continue;
+          }
           if ("viewport".equals(action)) {
             TouchCommandHandler.writeSessionViewport(
                 socket.getOutputStream(), requestId, getConnectedUiAutomationUnchecked());
@@ -214,7 +221,7 @@ private void runSnapshotSession(
                 getConnectedUiAutomationUnchecked());
             continue;
           }
-          if (!"snapshot".equals(action)) {
+          if (!"snapshot".equals(action) && !"snapshot-immediate".equals(action)) {
             SessionResponseWriter.writeSessionError(
                 socket.getOutputStream(),
                 requestId,
@@ -226,7 +233,7 @@ private void runSnapshotSession(
               socket.getOutputStream(),
               requestId,
               waitForIdleQuietMs,
-              waitForIdleTimeoutMs,
+              "snapshot-immediate".equals(action) ? 0 : waitForIdleTimeoutMs,
               timeoutMs,
               maxDepth,
               maxNodes);

docs/adr/0002-persistent-platform-helper-sessions.md

@@ -14,7 +14,7 @@ Accepted
 - When startup costs are high and an automation context is reusable, use persistent platform helper sessions.
 - The daemon owns helper lifetime policy. Platform modules own helper-manager mechanics.
 - Unless a separate cache contract has explicit invalidation, a helper must execute each command against fresh platform state.
-- On Android, snapshot capture, gesture viewport resolution, and instrumentation-backed planned-touch injection share one bundled automation helper.
+- On Android, snapshot capture, pointer and key input, text replacement, screenshots, window-focus reads, gesture viewport resolution, and instrumentation-backed planned-touch injection share one bundled automation helper.
 - On Android, after a failed session command, retry one-shot only for idempotent reads, and only after the failed session has stopped.
 - On Android, return a session gesture failure directly. Do not retry it one-shot.
 - For Android snapshots, if the packaged helper is unavailable or invalid, agent-device must fail closed. It must not substitute the legacy `adb uiautomator dump` snapshot engine.
@@ -94,8 +94,16 @@ Before that amendment, touch synthesis shipped as a separate instrumentation hel
 Thus, the daemon had to stop the persistent snapshot session before every gesture, and the next snapshot restarted it lazily.
 After a failed session command, one-shot retry applies only to idempotent reads, such as viewport resolution.
 The retry runs only after the failed session has stopped.
-Gesture injection is not idempotent because events may already be partially injected.
-Thus, session gesture failures return directly without a one-shot retry.
+Pointer, key, text, and gesture injection are not idempotent because a mutation may have reached the app before the response fails.
+Thus, session mutation failures return directly without a one-shot retry unless a validated helper response proves that no input injection was attempted.
+This exception permits the existing host input path only after the session has stopped.
+Idempotent screenshot and focus failures also stop the session before the host read.
+Piggyback commands retire a cached session whose process has ended before using a host path.
+The helper advertises optional session commands in its ready record.
+The host sends an optional command only when the helper advertises it.
+Text replacement retains the focus tap and host verification.
+Fresh application-window focus can answer the common dialog-readiness and in-app guards.
+System and unclassified windows retain the window-dump diagnostics.
 Helper reuse must never let process ownership cause interference between commands.
 
 For iOS, keep the XCTest runner session as the reference implementation for lifecycle and invalidation behavior.

docs/evidence/android-automation-performance-2026-10-19-base.json

@@ -0,0 +1,68 @@
+{
+  "observedAt": "2026-10-19T08:29:05.470Z",
+  "version": "0.21.29-dev",
+  "serial": "emulator-5554",
+  "rounds": 20,
+  "summary": {
+    "open": {
+      "count": 1,
+      "p50": 1974.88125,
+      "p95": 1974.88125
+    },
+    "restart": {
+      "count": 3,
+      "p50": 1221.4566250000003,
+      "p95": 1272.7720409999997
+    },
+    "observe": {
+      "count": 20,
+      "p50": 27.026958000000377,
+      "p95": 29.024333000000297
+    },
+    "tap": {
+      "count": 20,
+      "p50": 105.79833400000098,
+      "p95": 131.45187499999975
+    },
+    "fill": {
+      "count": 20,
+      "p50": 558.3288339999999,
+      "p95": 3296.912625000001
+    },
+    "screenshot": {
+      "count": 10,
+      "p50": 1237.9075420000008,
+      "p95": 1325.500250000001
+    }
+  },
+  "samples": {
+    "open": [1974.88125],
+    "restart": [1221.4566250000003, 1272.7720409999997, 1094.0003749999996],
+    "observe": [
+      45.778957999999875, 27.57654100000036, 27.712750000000597, 27.990334000000075,
+      26.00025000000005, 26.740875000000415, 28.017417000000023, 27.026958000000377,
+      28.392417000000023, 26.539334000000053, 26.187415999999757, 26.233374999999796,
+      27.746541999999863, 26.64958299999944, 25.823707999999897, 26.324791999999434,
+      29.024333000000297, 27.611957999999504, 27.144374999999854, 25.519750000000386
+    ],
+    "tap": [
+      156.7671249999994, 111.00983299999916, 99.53199999999924, 131.45187499999975,
+      101.18212500000027, 108.20045800000025, 104.59895899999992, 99.9613330000002,
+      100.58424999999988, 107.36154100000022, 114.32608299999993, 112.42979200000082,
+      125.20504200000141, 103.49154100000123, 105.79833400000098, 110.19045799999913,
+      104.1442500000012, 113.18316699999923, 100.42487500000061, 102.52449999999953
+    ],
+    "fill": [
+      509.73954199999935, 522.5573750000003, 526.1622920000009, 558.3288339999999,
+      531.6915420000005, 508.30204200000117, 534.4593750000004, 524.6943329999995,
+      561.4105410000011, 557.9260830000003, 565.5302920000013, 529.8140419999982, 560.0029170000016,
+      3296.912625000001, 3621.0452920000025, 2794.0020839999997, 613.8256250000013,
+      591.187291000002, 606.5397499999999, 754.4855409999982
+    ],
+    "screenshot": [
+      1325.500250000001, 1245.3615000000027, 1243.3869999999952, 1226.1485419999954,
+      1237.9075420000008, 1225.732624999997, 1258.799165999997, 1230.1937089999992,
+      1241.7689580000006, 1218.0068329999995
+    ]
+  }
+}