diff --git a/android/snapshot-helper/README.md b/android/snapshot-helper/README.md index bc5855f273..a8b7afd67e 100644 --- a/android/snapshot-helper/README.md +++ b/android/snapshot-helper/README.md @@ -162,3 +162,7 @@ The release manifest is a stable provider contract for the current helper protoc resolve the APK from `apkUrl`, verify `sha256`, install it with `adb install -r `, and run `instrumentationRunner`. Manifests up to 0.21.3 also carried an `installArgs` array; it only ever spelled `install -r` (plus `-t` while the helper was `testOnly`), so consumers can ignore it. + +Capture metadata reports `cacheInvalidated` when the capture cleared the platform node cache through `UiAutomation.clearCache()`. A complete capture with that evidence need not change routes after a control press. Incomplete captures and captures without that evidence retain freshness recovery. + +Warm application open commands use the same automation session. Close retains the existing foreground and process-absence checks. A failed application command is never resent. diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java index d9830dfb8b..e640ef1b99 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java @@ -36,13 +36,13 @@ static void prepare(UiAutomation automation, long quietMs, long timeoutMs) static Result capture( UiAutomation automation, int maxDepth, int maxNodes, long stabilizationTimeoutMs) throws InterruptedException, AccessibilityCaptureStabilizer.IncompleteCaptureException { - clearAccessibilityCache(automation); return AccessibilityCaptureStabilizer.capture( () -> captureOnce(automation, maxDepth, maxNodes), stabilizationTimeoutMs); } @SuppressWarnings("deprecation") private static Result captureOnce(UiAutomation automation, int maxDepth, int maxNodes) { + boolean cacheInvalidated = clearAccessibilityCache(automation); AccessibilityTreeXml.Stats stats = new AccessibilityTreeXml.Stats(); StringBuilder xml = new StringBuilder(); xml.append(""); @@ -87,10 +87,11 @@ private static Result captureOnce(UiAutomation automation, int maxDepth, int max windowCount, stats.nodeCount, stats.truncated, - missingRootWindowTypes); + missingRootWindowTypes, + cacheInvalidated); } - static void clearAccessibilityCache(UiAutomation automation) { + static boolean clearAccessibilityCache(UiAutomation automation) { // Navigation 3 and any in-place content swap that keeps the same window/Activity render every // destination inside one AndroidComposeView. A persistent helper session reuses a single // UiAutomation connection across captures, and its per-connection accessibility node cache is @@ -98,16 +99,16 @@ static void clearAccessibilityCache(UiAutomation automation) { // // Declared residue (agent-device #1832): the reset mechanism differs by API level — the public // clearCache() on 34+, a setServiceInfo() re-apply below — and the two are not known to be - // equivalent. This is an acquisition-freshness difference the payload does not disclose. + // equivalent. Only the public cache reset establishes cacheInvalidated evidence. if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { try { - automation.clearCache(); - return; + return automation.clearCache(); } catch (RuntimeException ignored) { // Fall back to the setServiceInfo() flush below if the platform rejects the public API. } } clearAccessibilityCacheViaServiceInfo(automation); + return false; } private static void clearAccessibilityCacheViaServiceInfo(UiAutomation automation) { @@ -269,6 +270,7 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture { final int windowCount; final int nodeCount; final boolean truncated; + final boolean cacheInvalidated; // AccessibilityWindowInfo types of listed windows whose root read null or threw, so the host // can tell a window it could not read (an input method, say) from one that was not on screen. final List missingRootWindowTypes; @@ -281,7 +283,8 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture { int windowCount, int nodeCount, boolean truncated, - List missingRootWindowTypes) { + List missingRootWindowTypes, + boolean cacheInvalidated) { this.xml = xml; this.rootPresent = rootPresent; this.foregroundWindowRootsPresent = foregroundWindowRootsPresent; @@ -289,6 +292,7 @@ static final class Result implements AccessibilityCaptureStabilizer.Capture { this.windowCount = windowCount; this.nodeCount = nodeCount; this.truncated = truncated; + this.cacheInvalidated = cacheInvalidated; this.missingRootWindowTypes = missingRootWindowTypes; } diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/ApplicationCommandHandler.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/ApplicationCommandHandler.java new file mode 100644 index 0000000000..d3424984ce --- /dev/null +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/ApplicationCommandHandler.java @@ -0,0 +1,59 @@ +package com.callstack.agentdevice.snapshothelper; + +import android.app.UiAutomation; +import android.os.Bundle; +import android.util.Base64; +import android.os.ParcelFileDescriptor; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import org.json.JSONObject; + +final class ApplicationCommandHandler { + private ApplicationCommandHandler() {} + + static void execute(UiAutomation automation, JSONObject input, Bundle result) throws Exception { + String packageName = input.getString("packageName"); + if (!packageName.matches("[A-Za-z][A-Za-z0-9_]*(\\.[A-Za-z][A-Za-z0-9_]*)+")) { + throw new IllegalArgumentException("Invalid application package"); + } + switch (input.getString("kind")) { + case "open": + open(automation, packageName, result); + return; + default: + throw new IllegalArgumentException("Unknown application operation"); + } + } + + private static void open(UiAutomation automation, String packageName, Bundle result) throws Exception { + String resolved = shell(automation, "cmd package resolve-activity --brief -a android.intent.action.MAIN -c android.intent.category.LAUNCHER " + packageName).trim(); + String[] lines = resolved.split("\\n"); + String component = lines[lines.length - 1].trim(); + if (!component.matches(java.util.regex.Pattern.quote(packageName) + "/[A-Za-z_.$][A-Za-z0-9_.$]*")) { + result.putString("componentUnavailable", "true"); + return; + } + String output = shell(automation, "am start -W -a android.intent.action.MAIN -c android.intent.category.DEFAULT -c android.intent.category.LAUNCHER -n '" + component + "'"); + result.putString("launchOutputBase64", Base64.encodeToString(output.getBytes(StandardCharsets.UTF_8), Base64.NO_WRAP)); + } + + private static String shell(UiAutomation automation, String command) throws Exception { + ParcelFileDescriptor[] pipes = automation.executeShellCommandRw("sh"); + if (pipes == null || pipes.length != 2) throw new IllegalStateException("Application shell command was unavailable"); + try (InputStream stream = new ParcelFileDescriptor.AutoCloseInputStream(pipes[0]); + OutputStream input = new ParcelFileDescriptor.AutoCloseOutputStream(pipes[1]); + ByteArrayOutputStream bytes = new ByteArrayOutputStream()) { + input.write(("exec 2>&1\n" + command + "\n").getBytes(StandardCharsets.UTF_8)); + input.close(); + byte[] buffer = new byte[4096]; + int count; + while ((count = stream.read(buffer)) != -1) { + if (bytes.size() + count > 65536) throw new IllegalStateException("Application command output exceeded its limit"); + bytes.write(buffer, 0, count); + } + return bytes.toString(StandardCharsets.UTF_8.name()); + } + } +} diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java index 2e4a92b0a1..a6940da647 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java @@ -28,7 +28,9 @@ static void writeSessionResponse( Bundle result = SessionResponseWriter.sessionResponseBundle(requestId); InputDispatch dispatch = new InputDispatch(); try { - if ("screenshot".equals(action)) { + if ("app".equals(action)) { + ApplicationCommandHandler.execute(automation, new JSONObject(new String(Base64.decode(payload, Base64.DEFAULT), StandardCharsets.UTF_8)), result); + } else if ("screenshot".equals(action)) { screenshot(result, automation); } else if ("focus".equals(action)) { focus(result, automation); diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index 4721a51fe5..789db4f26e 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -122,6 +122,7 @@ private static void putCaptureMetadata( DisplayExtent.Extent display, long elapsedMs) { result.putString("rootPresent", Boolean.toString(capture.rootPresent)); + result.putString("cacheInvalidated", Boolean.toString(capture.cacheInvalidated)); result.putString("captureMode", capture.captureMode); result.putString("windowCount", Integer.toString(capture.windowCount)); result.putString( @@ -180,7 +181,8 @@ private void runSnapshotSession( putBaseMetadata( ready, waitForIdleTimeoutMs, waitForIdleQuietMs, timeoutMs, maxDepth, maxNodes); ready.putString("sessionReady", "true"); - ready.putString("sessionCommands", "input,screenshot,focus,snapshot-immediate"); + ready.putString("sessionCommands", "input,screenshot,focus,snapshot-immediate" + + (android.os.Build.VERSION.SDK_INT >= 31 ? ",app" : "")); ready.putString("sessionPort", Integer.toString(sessionPort)); sendStatus(2, ready); @@ -202,7 +204,7 @@ private void runSnapshotSession( SessionResponseWriter.writeSessionOk(socket.getOutputStream(), requestId); return; } - if ("input".equals(action) || "screenshot".equals(action) || "focus".equals(action)) { + if ("input".equals(action) || "screenshot".equals(action) || "focus".equals(action) || "app".equals(action)) { AutomationCommandHandler.writeSessionResponse( socket.getOutputStream(), requestId, action, parts.length > 2 ? parts[2] : "", getConnectedUiAutomationUnchecked()); diff --git a/docs/adr/0011-interaction-guarantee-contract.md b/docs/adr/0011-interaction-guarantee-contract.md index a669d69414..521a320a0c 100644 --- a/docs/adr/0011-interaction-guarantee-contract.md +++ b/docs/adr/0011-interaction-guarantee-contract.md @@ -17,6 +17,7 @@ Completed process history is in git history. - The Apple runner does not resend a mutating command whose first send may have run. - Multiple mutating selector matches collapse only if they form one ancestor–descendant chain and every member resolves to the same actionable node. Otherwise, the mutation fails with `AMBIGUOUS_MATCH`. Maestro's explicit expected-point/non-hittable compatibility path stays separate. - Keep synthesized iOS gesture policy outside the element dispatch-path matrix. +- On Android, the press/click moving-target guard samples fresh geometry until the target remains on one surface at one position for 100 ms, within a 1,500 ms transition window. A target that cannot establish that evidence is refused. Other platforms retain their existing guard schedule. ## Context diff --git a/docs/adr/0014-session-ref-frame-lifetime.md b/docs/adr/0014-session-ref-frame-lifetime.md index c8c2aa0f7f..01a35056ef 100644 --- a/docs/adr/0014-session-ref-frame-lifetime.md +++ b/docs/adr/0014-session-ref-frame-lifetime.md @@ -194,6 +194,8 @@ A ref-targeting mutation is admitted only when all of these conditions hold: The target is resolved against the authorized frame, never against a newer operational observation by positional coincidence. Android freshness captures may verify current identity or provide comparison evidence. +On Android, an identifier that names exactly one same-local-identity node in both the authorized frame and the observation can carry fresh geometry across a positional reindex when both nodes name the same application package. +This does not change the frame namespace or authorize a different identity. But they must not redefine which node an already-admitted `@eN` authorizes. Read-only ref consumers remain fail-open with a structured staleness warning only when the retained frame contains the evidence necessary to resolve that ref. diff --git a/docs/evidence/android-edited-field-experiment-2026-10-19-results.json b/docs/evidence/android-edited-field-experiment-2026-10-19-results.json new file mode 100644 index 0000000000..a9ece8c615 --- /dev/null +++ b/docs/evidence/android-edited-field-experiment-2026-10-19-results.json @@ -0,0 +1,221 @@ +{ + "date": "2026-10-19", + "baseCommit": "8b29f466fc2941eb24f9b3df8b60b3aef862d8cd", + "runtimeCommit": "86d42d43ea2a06cc4290fb1bef56b780765b7d1c", + "adapterCommit": "5e1d8952f4b0fd0d6a12b084ed8151e71e7cd889", + "device": "Pixel_7_CI / emulator-5554 / Android 16 API 36", + "workers": 1, + "runnerRetries": 0, + "benchmarkApkSha256": "b163e985ebd8340bf10447cdf6826aea37e429ee8c4382f9179c863c8c0619b2", + "testAppApkSha256": "4f332d39e76899b86e7006cda5d43ade6211733fe85ffe4552735e3e56c867b4", + "rejectedHostShortcut": { + "disposition": "Rejected. Covering button can receive click while field has correct empty value.", + "caseAbbaSeconds": [8.94, 6.194, 6.087, 8.866], + "clearAbbaMs": [1733, 451, 436, 1731], + "enterAbbaMs": [1617, 286, 285, 1603], + "focusedMeasuredOutcomes": { + "correct": 15, + "wrong-target": 3, + "mutation-after-error": 3 + }, + "focusedTimerOutcomes": { + "correct": 12, + "refusal": 9 + }, + "testAppTimerSeconds": 37.714, + "testAppMeasuredSeconds": 32.634, + "excludedRuns": [ + "Initial overlapping-field setup failure", + "Preparation cohort with only one eligible shortcut action" + ] + }, + "acceptedVerifierFix": { + "base": { + "trials": 6, + "outcomes": { + "correct": 2, + "mutation-after-error": 4 + }, + "movingClearMs": [1646, 1638, 1620, 1635], + "movingClearMedianMs": 1636.5 + }, + "final": { + "trials": 6, + "outcomes": { + "correct": 6 + }, + "movingClearMs": [311, 299, 334, 413], + "movingClearMedianMs": 322.5 + }, + "movingClearMedianSavedMs": 1314.0, + "negativeControl": { + "expectedFailure": true, + "value": "_", + "decoy": "D", + "clears": 1, + "edits": 0, + "clicks": 1, + "outcome": "wrong-target" + }, + "originalRegression": { + "baseFailed": 1, + "basePassed": 4 + }, + "reviewRegression": { + "previousCandidateFailed": 4, + "previousCandidatePassed": 1, + "finalPassed": 5 + }, + "focusedTestsPassed": 40 + }, + "validation": { + "benchmarkHead": { + "runId": "01a1549f-a646-773d-bca2-44daffb097b7", + "durationSeconds": 309.967, + "summary": { + "discovered": 45, + "selected": 43, + "executed": 42, + "passed": 42, + "failed": 0, + "interrupted": 0, + "flaky": 0, + "skipped": 1 + }, + "fixtureRecoveries": 0, + "extraAttempts": 0 + }, + "benchmarkContemporaryBase": { + "runId": "01a154a5-caae-797d-8b39-578194786bfd", + "durationSeconds": 304.726, + "summary": { + "discovered": 45, + "selected": 43, + "executed": 42, + "passed": 42, + "failed": 0, + "interrupted": 0, + "flaky": 0, + "skipped": 1 + }, + "fixtureRecoveries": 0, + "extraAttempts": 0 + }, + "sameRunnableIds": true, + "testAppWorkflows": { + "runId": "01a154ab-32ef-710a-a5e8-0f5cc3d49eb4", + "durationSeconds": 46.556, + "summary": { + "discovered": 4, + "selected": 4, + "executed": 4, + "passed": 4, + "failed": 0, + "interrupted": 0, + "flaky": 0, + "skipped": 0 + }, + "fixtureRecoveries": 0, + "extraAttempts": 0 + }, + "testAppCoverage": { + "completedScenarios": [ + "smoke:fixture-bootstrap", + "smoke:inventory", + "smoke:automation-system", + "smoke:maestro-clickable-first", + "smoke:form-input", + "smoke:keyboard-ime", + "smoke:capture-close", + "full:lifecycle-system", + "full:observability-artifacts", + "full:fixture-replays" + ], + "commandCount": 41, + "behaviorCount": 15, + "infrastructureReissues": 0, + "sessionClosed": true, + "durationMs": 149338, + "initialTouchMode": true, + "negativeVisibilityProbes": 8, + "unexpectedStepFailures": 0 + }, + "androidSnapshot": { + "backend": "android-helper", + "pixelDensity": 2.625, + "helperVersion": "0.21.29-dev", + "helperApiVersion": "2", + "helperTransport": "persistent-session", + "helperSessionReused": true, + "installReason": "current", + "waitForIdleTimeoutMs": 500, + "waitForIdleQuietMs": 100, + "timeoutMs": 2000, + "maxDepth": 128, + "maxNodes": 5000, + "rootPresent": true, + "cacheInvalidated": true, + "captureMode": "interactive-windows", + "windowCount": 2, + "missingRootWindowTypes": [], + "nodeCount": 138, + "helperTruncated": false, + "elapsedMs": 15 + }, + "independentReview": "No findings at runtimeCommit. Pre-action uniqueness finding fixed and red/green proven.", + "sameTestAppCoverageIdentitiesAsPrior": true, + "testAppSetup": { + "initialFailure": "Non-touch mode focused a lower control and scrolled the startup landmark offscreen before any fill.", + "interruptedReruns": 1, + "repair": "One real tap on a snapshot-confirmed fixture button; observed inTouchMode=true.", + "sourceOrTestEdits": false + }, + "benchmarkOperationTotalsMs": { + "base": { + "fill": 14119, + "clear": 1731, + "tap": 83809, + "scrollUntilVisible": 66961 + }, + "head": { + "fill": 13911, + "clear": 1754, + "tap": 83802, + "scrollUntilVisible": 73162 + } + } + }, + "limits": [ + "API 36 is the only live platform.", + "One head/base suite pair is not statistical evidence.", + "No whole-suite speedup or experimental-engine parity claim.", + "The retained point-input race is not fixed by the verifier." + ], + "archive": { + "filename": "android-edited-field-experiment-2026-10-19.tar.gz", + "sha256": "a8ac1266326861aaaf821344433e7b6c4c507caa0eb56d30d864e97498f68626", + "localOnly": true, + "removedOwnedWorktrees": 2, + "reviewedShaUnchanged": true, + "reviewedStatusUnchanged": true, + "ownedDaemonStatesStopped": 27, + "daemonStopFailures": 0 + }, + "postReview": { + "runtimeCommit": "14a2d7a28dfa0e509e02247323aa56b72c98e1fc", + "change": "Behavior-equivalent separation of identity corroboration from point scanning to satisfy complexity gate.", + "focusedTestsPassed": 40, + "fallowPassed": true, + "nativePublicSdkTrialsPassed": 6, + "nativeTrialCounters": "Each target clear=1, decoy unchanged, decoy edits=0, cover clicks=0.", + "independentReview": "No findings; predicates and ownership equivalent.", + "firstHarnessRun": "Six correct cases; duplicate final close returned SESSION_NOT_FOUND. Corrected harness rerun exited 0.", + "additionalOwnedDaemonStatesStopped": 1, + "fullBenchmarkAndTestAppEvidenceCommit": "86d42d43ea2a06cc4290fb1bef56b780765b7d1c", + "archive": { + "filename": "android-edited-field-post-review-2026-10-19.tar.gz", + "sha256": "ba0df22db185ea4d5134082866687a1a9317e26e94b4bcb815fbfe03f8dce28e", + "localOnly": true + } + } +} diff --git a/docs/evidence/android-edited-field-experiment-2026-10-19.md b/docs/evidence/android-edited-field-experiment-2026-10-19.md new file mode 100644 index 0000000000..6aed75bdc0 --- /dev/null +++ b/docs/evidence/android-edited-field-experiment-2026-10-19.md @@ -0,0 +1,60 @@ +# Android edited-field experiments, 2026-10-19 + +The safe result is a clear-verification fix: a field that moves after being cleared now confirms its empty value by a corroborated identity. Four moving-field trials previously changed the correct field but returned an error after a median 1636.5 ms. At the measured runtime, all four succeeded, with a median 322.5 ms: 1314 ms saved per affected clear. + +The larger adapter timer shortcut was rejected. It saved seconds in normal tests but clicked a covering button in three oracle trials, even though the requested field was correctly cleared. Exact value assertions alone miss that side effect. + +[Machine-readable results](android-edited-field-experiment-2026-10-19-results.json) retain inputs, outcomes, and validation counts. [The preceding startup experiment](android-startup-experiment-2026-10-19.md) remains a separate rejected policy. + +## Inputs and isolation + +The runtime base was `8b29f466fc2941eb24f9b3df8b60b3aef862d8cd`. The measured runtime was `86d42d43ea2a06cc4290fb1bef56b780765b7d1c`. The adapter was `5e1d8952f4b0fd0d6a12b084ed8151e71e7cd889`. Separate disposable checkouts received independent dependency installations and builds. Runs used Pixel_7_CI / emulator-5554, Android 16 API 36, sequentially, one worker, and zero runner retries. No heavy tests or builds ran beside device timings. + +The benchmark APK SHA-256 remained `b163e985ebd8340bf10447cdf6826aea37e429ee8c4382f9179c863c8c0619b2`. The test-app APK remained `4f332d39e76899b86e7006cda5d43ade6211733fe85ffe4552735e3e56c867b4`. + +## Large wait found, unsafe shortcut rejected + +The adapter refinds fields using role, test ID, name, and text. Android textbox text can echo the edited value. A successful edit can therefore make the same stationary field appear newly arrived, triggering the remainder of the configured 1500 ms transition timer. + +The prototype admitted only a uniquely identified, previously edited, focused Android textbox. It required three complete same-app observations spanning at least 100 ms, unchanged geometry, expected value, focus, keyboard, and surface facts. Other mutations or failed actions cleared eligibility. Password fields were excluded. Native dispatch guards remained enabled. + +| Same original benchmark case, ABBA | Case time | Clear | Enter | +| --- | ---: | ---: | ---: | +| Timer | 8.940 s | 1733 ms | 1617 ms | +| Measured shortcut | 6.194 s | 451 ms | 286 ms | +| Measured shortcut | 6.087 s | 436 ms | 285 ms | +| Timer | 8.866 s | 1731 ms | 1603 ms | + +All four case runs passed with no fixture recoveries. Four test-app workflows also passed: 37.714 s with the timer and 32.634 s with the shortcut. These measurements belong to the rejected prototype. It did not run the full original suite. + +The focused native oracle repeated seven cells three times: static, overlays at 200/250/300 ms, movement, a slide, and duplicate identity. The shortcut produced 15 correct actions, three covering-button clicks, and three errors after the correct mutation. The timer produced 12 correct actions and nine refusals with no activation. These selected timing cells demonstrate counterexamples; they do not estimate production failure rates. + +Each trial checked a unique trial token, exactly one target clear, unchanged decoy text, zero decoy edits, and zero covering-button clicks after 800 ms. The failed overlay trials had both an empty target and a positive click counter. A deliberate covering-button click after a correct clear also failed the oracle as intended. Setup-failed and shortcut-ineligible preparation runs are excluded from these counts. + +## Narrow verifier fix retained + +The old verifier dropped a focused input when its raw text became empty. If that field moved, it instead inspected the old coordinates, saw the decoy, and exhausted its 1500 ms deadline despite a successful clear. + +The fix carries the captured target through native-helper verification. Moved-field corroboration requires complete, freshly invalidated captures before and after the mutation, no missing window roots, and exactly one raw package/resource-ID match in both trees. The observed field must retain its class, be the sole focused edit, belong to the app, be nonsensitive, and have an exactly empty value. Missing or ambiguous evidence retains the existing point-verification behavior. Provider-native and shell/IME paths do not gain this corroboration. + +The native fixture places a decoy at the target's old point after moving the target. With adapter transition zero in both arms, the unchanged base passed two static controls and returned four errors after correctly clearing the moving target. The measured runtime passed all six trials. Moving clears measured 299–413 ms, median 322.5 ms, versus 1620–1646 ms, median 1636.5 ms. Every trial recorded one target clear, unchanged decoy text, and no decoy edits or clicks. This is a verifier improvement, not proof that point dispatch is atomic. + +## Validation + +The original regression test failed on the unchanged base: one failed, four passed. Focused final coverage passed all 40 tests. Independent review found missing pre-action uniqueness proof in an earlier candidate; that finding was fixed. The matched helper-transport regression then changed from four failures and one pass at `2aaf2c6c59` to five passes at the measured runtime. The final independent source review found no remaining issues. + +Both original benchmark runs had zero fixture recoveries and zero extra attempts. The historical 273.122 s run is not treated as a contemporary base. In the new pair, fill totals were 14.119/13.911 s, clear 1.731/1.754 s, and taps 83.809/83.802 s (base/head). Scrolling totals were 66.961/73.162 s. One pair cannot establish a statistical improvement or regression. The test-app scenario, command, and behavior identities match the previous full verification. Eight expected negative visibility probes preceded scroll-to-visible actions; they were not infrastructure reissues or failed mutations. + +The first full CLI attempt stopped before any fill: Android was in non-touch mode after the Enter-key workflow, and focus scrolled the Automation lab heading offscreen. A visible fixture tap changed the observed input event to `inTouchMode=true`. The unchanged suite then ran with that explicit precondition. The failed setup, an interrupted premature rerun, and the repair commands remain in the archive; they are excluded from passing-run counts. + +The benchmark and test-app runs used the original adapter after restoring and rebuilding it. The benchmark selected the same 42 runnable test IDs as the earlier accepted run, with one authored skip. Aggregate runs are regression evidence; this experiment makes no whole-suite speedup or experimental-engine parity claim. API 36 is the only live platform tested. Unknown capture metadata and older APIs retain the existing verification path. + +Two Claude Opus public-source review rounds challenged the shortcut. The follow-up corrected an initial mistaken 500 ms timer bound to the configured 1500 ms, and identified native fill's unconditional point tap. These reviews covered public code and design, not the complete unpublished patch. The app-side counter oracle established the actual wrong-click result. + +## Disposition and next target + +Promote only the verifier fix. Retain the adapter timer and native input guards. The next larger experiment is identity-bound native fill/clear that can resolve and validate the target without injecting an unconditional point tap. It needs counter-based tests for overlays, duplicate IDs, remounts, keyboard movement, and touch-handler fidelity before any host wait is removed. A native accessibility action alone does not establish fidelity to custom touch handlers. + +The local archive retains runtime patches/source, rejected adapter patches, fixture source/APK, harnesses, reports, logs, and per-file checksums: `android-edited-field-experiment-2026-10-19.tar.gz`, SHA-256 `a8ac1266326861aaaf821344433e7b6c4c507caa0eb56d30d864e97498f68626`. It is a local handoff artifact, not a repository-hosted download. The initial 27 owned daemon states were stopped. Both disposable checkouts were removed. The reviewed SHA and clean status were verified unchanged before promotion. Only the final guarded verifier fix was promoted. + +The final code at `14a2d7a28dfa0e509e02247323aa56b72c98e1fc` separates identity corroboration from point scanning to satisfy the complexity gate. Independent review confirmed equivalent checks. All 40 focused tests and six native counter trials through the public SDK passed on this refactor. Whole-suite benchmark and test-app evidence above remains attributed to `86d42d43ea`. The first new harness run completed six correct cases but failed a duplicate final close; the corrected harness exited zero. One additional owned daemon was stopped. The post-review archive is `android-edited-field-post-review-2026-10-19.tar.gz`, SHA-256 `ba0df22db185ea4d5134082866687a1a9317e26e94b4bcb815fbfe03f8dce28e`. diff --git a/docs/evidence/android-readiness-2026-10-19-results.json b/docs/evidence/android-readiness-2026-10-19-results.json new file mode 100644 index 0000000000..4c63357766 --- /dev/null +++ b/docs/evidence/android-readiness-2026-10-19-results.json @@ -0,0 +1,677 @@ +{ + "date": "2026-10-19", + "runtimeCommit": "fc612424d16cbaa6418107cb2552157f33a6e410", + "adapterCommit": "5e1d8952f4b0fd0d6a12b084ed8151e71e7cd889", + "adapterRuntimeCommit": "59307b157079b0b1a665b82afd0aa6b3845c2e2c", + "parentCommit": "0d4b58c341bda7372d05e69a6bf58db6248e4f47", + "parentRuntimeCommit": "ddba75f5b474cb4cdac11b15f75292f2870396c7", + "referenceCommit": "2593e45025edc52b2c381e809b41cb63d5878e2b", + "apkSha256": "b163e985ebd8340bf10447cdf6826aea37e429ee8c4382f9179c863c8c0619b2", + "device": "Pixel_7_CI / emulator-5554 / Android 16 API 36", + "workers": 1, + "runnerRetries": 0, + "sameSelectedIds": true, + "sameRunnableIds": true, + "runnableTestIdsSha256": "6c06cc1d2590e3b6c451d26b0912f973cd6703a5c58a932ba3ee544d1b55e19d", + "runs": { + "parent": { + "id": "01a153a0-9100-7c37-8b9b-2d982bded990", + "startedAt": "2026-10-19T10:06:36.383Z", + "finishedAt": "2026-10-19T10:11:57.633Z", + "durationMs": 321250, + "summary": { + "discovered": 45, + "selected": 43, + "executed": 42, + "passed": 42, + "failed": 0, + "interrupted": 0, + "flaky": 0, + "skipped": 1 + }, + "fixtureRecoveryTestIds": [] + }, + "direct": { + "id": "01a153ac-2384-78f5-a765-32b352482a2e", + "startedAt": "2026-10-19T10:19:12.763Z", + "finishedAt": "2026-10-19T10:22:35.366Z", + "durationMs": 202603, + "summary": { + "discovered": 45, + "selected": 43, + "executed": 42, + "passed": 40, + "failed": 2, + "interrupted": 0, + "flaky": 0, + "skipped": 1 + }, + "fixtureRecoveryTestIds": [ + "tests/flows-lists.e2e.ts::infinite%20scroll%20reaches%20the%20target%20item", + "tests/flows-lists.e2e.ts::gestures%3A%20long-press%2C%20then%20swipe%20left", + "tests/flows-lists.e2e.ts::sticky%20chrome%3A%20accept%20below%20the%20fold%2C%20then%20continue", + "tests/flows.e2e.ts::bottom%20tabs%20act%20inside%20the%20last%20tab" + ] + }, + "candidate": { + "id": "01a153ff-a255-7377-b15c-175c0b6eac39", + "startedAt": "2026-10-19T11:50:26.329Z", + "finishedAt": "2026-10-19T11:54:59.451Z", + "durationMs": 273122, + "summary": { + "discovered": 45, + "selected": 43, + "executed": 42, + "passed": 42, + "failed": 0, + "interrupted": 0, + "flaky": 0, + "skipped": 1 + }, + "fixtureRecoveryTestIds": [] + } + }, + "sharedDirectPassingCases": { + "count": 40, + "totalsMs": { + "parent": 308199, + "direct": 199428, + "candidate": 264813 + } + }, + "cases": [ + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20label%20query%20resolves%20a%20labeled%20text%20field", + "title": "a label query resolves a labeled text field", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 3444, + "directMs": 1687, + "candidateMs": 3266, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20labeled%20field%20answers%20name%2C%20display%20value%2C%20value%20reads%2C%20fill%2C%20and%20clear", + "title": "a labeled field answers name, display value, value reads, fill, and clear", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 7941, + "directMs": 1894, + "candidateMs": 7927, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20placeholder%20query%20and%20attribute%20reads%20find%20the%20field", + "title": "a placeholder query and attribute reads find the field", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 3040, + "directMs": 1534, + "candidateMs": 2953, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20secure%20field%20takes%20a%20fill%20and%20denies%20value%20reads", + "title": "a secure field takes a fill and denies value reads", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 4370, + "directMs": 1553, + "candidateMs": 4569, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::press%20sends%20a%20character%20and%20Enter%20through%20the%20soft%20keyboard", + "title": "press sends a character and Enter through the soft keyboard", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 5825, + "directMs": 1730, + "candidateMs": 6178, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20field%20autofocuses%20on%20mount%20and%20focus%28%29%20moves%20the%20focus", + "title": "a field autofocuses on mount and focus() moves the focus", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 9064, + "directMs": 1872, + "candidateMs": 4222, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::toBeFocused%20follows%20the%20focus", + "title": "toBeFocused follows the focus", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 8215, + "directMs": 2741, + "candidateMs": 4767, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20switch%20checks%2C%20unchecks%2C%20and%20reports%20its%20state", + "title": "a switch checks, unchecks, and reports its state", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 9006, + "directMs": 4075, + "candidateMs": 6292, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::checkbox%20and%20radio%20roles%20answer%20getByRole%2C%20and%20a%20checkbox%20toggles%20by%20tap", + "title": "checkbox and radio roles answer getByRole, and a checkbox toggles by tap", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 6137, + "directMs": 1886, + "candidateMs": 4143, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::checkbox%20and%20radio%20roles%20check%2C%20uncheck%2C%20and%20report%20their%20state", + "title": "checkbox and radio roles check, uncheck, and report their state", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 10877, + "directMs": 6367, + "candidateMs": 8400, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20tab%20role%20answers%20getByRole", + "title": "a tab role answers getByRole", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 2821, + "directMs": 1325, + "candidateMs": 2864, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20radio%20group%20reports%20the%20checked%20option", + "title": "a radio group reports the checked option", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 8240, + "directMs": 3923, + "candidateMs": 5934, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::buttons%20report%20disabled%2C%20enabled%2C%20and%20a%20state%20that%20arrives%20later", + "title": "buttons report disabled, enabled, and a state that arrives later", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 7459, + "directMs": 3194, + "candidateMs": 5084, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20display%3A%20none%20element%20is%20absent%20rather%20than%20hidden%2C%20its%20visible%20twin%20attached", + "title": "a display: none element is absent rather than hidden, its visible twin attached", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 5611, + "directMs": 2368, + "candidateMs": 4478, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20header%20answers%20its%20accessible%20name", + "title": "a header answers its accessible name", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 2873, + "directMs": 1298, + "candidateMs": 2814, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20tab%20row%20answers%20count%2C%20all%2C%20texts%2C%20filters%2C%20positions%2C%20and%20switching", + "title": "a tab row answers count, all, texts, filters, positions, and switching", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 4715, + "directMs": 1727, + "candidateMs": 4694, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20list%20answers%20counts%2C%20list-form%20text%2C%20reads%2C%20and%20boxes%20in%20order", + "title": "a list answers counts, list-form text, reads, and boxes in order", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 4636, + "directMs": 647, + "candidateMs": 3537, + "directStatus": "failed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20row%20contains%20its%20texts", + "title": "a row contains its texts", + "file": "tests/controls.e2e.ts", + "status": "skipped", + "parentMs": null, + "directMs": null, + "candidateMs": null, + "directStatus": "skipped" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20long%20press%20lands%20as%20one", + "title": "a long press lands as one", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 6488, + "directMs": 2343, + "candidateMs": 5508, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::a%20double%20tap%20lands%20inside%20the%20300%20ms%20window", + "title": "a double tap lands inside the 300 ms window", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 7411, + "directMs": 1730, + "candidateMs": 4796, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::pointer%20verbs%20land%20at%20the%20point%20asked%20for", + "title": "pointer verbs land at the point asked for", + "file": "tests/controls.e2e.ts", + "status": "skipped", + "parentMs": null, + "directMs": null, + "candidateMs": null, + "directStatus": "skipped" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::an%20ambiguous%20locator%20fails%20at%20once", + "title": "an ambiguous locator fails at once", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 4273, + "directMs": 1482, + "candidateMs": 4913, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::control%20inventory::screenshot%2C%20restart%2C%20and%20clearState%20reopen%20the%20app%20on%20its%20home%20list", + "title": "screenshot, restart, and clearState reopen the app on its home list", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 6570, + "directMs": 2483, + "candidateMs": 7129, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::refusals::verbs%20a%20device%20never%20declares%20are%20refused%20before%20the%20engine", + "title": "verbs a device never declares are refused before the engine", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 1, + "directMs": 3, + "candidateMs": 3, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::refusals::keys%20the%20soft%20keyboard%20cannot%20send%20are%20refused%20by%20the%20engine", + "title": "keys the soft keyboard cannot send are refused by the engine", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 165, + "directMs": 165, + "candidateMs": 1490, + "directStatus": "passed" + }, + { + "testId": "tests/controls.e2e.ts::refusals::a%20missing%20locator%20fails%20as%20not%20found%20once%20its%20timeout%20passes", + "title": "a missing locator fails as not found once its timeout passes", + "file": "tests/controls.e2e.ts", + "status": "passed", + "parentMs": 1113, + "directMs": 1072, + "candidateMs": 1116, + "directStatus": "passed" + }, + { + "testId": "tests/flows-lists.e2e.ts::infinite%20scroll%20reaches%20the%20target%20item", + "title": "infinite scroll reaches the target item", + "file": "tests/flows-lists.e2e.ts", + "status": "passed", + "parentMs": 14316, + "directMs": 22370, + "candidateMs": 14463, + "directStatus": "passed" + }, + { + "testId": "tests/flows-lists.e2e.ts::product%20catalog%20adds%20exactly%20two%20of%20the%20right%20variant", + "title": "product catalog adds exactly two of the right variant", + "file": "tests/flows-lists.e2e.ts", + "status": "passed", + "parentMs": 8986, + "directMs": 1990, + "candidateMs": 4315, + "directStatus": "passed" + }, + { + "testId": "tests/flows-lists.e2e.ts::gestures%3A%20long-press%2C%20then%20swipe%20left", + "title": "gestures: long-press, then swipe left", + "file": "tests/flows-lists.e2e.ts", + "status": "passed", + "parentMs": 7339, + "directMs": 13198, + "candidateMs": 7231, + "directStatus": "passed" + }, + { + "testId": "tests/flows-lists.e2e.ts::gestures%3A%20double-tap%20completes%20the%20flow", + "title": "gestures: double-tap completes the flow", + "file": "tests/flows-lists.e2e.ts", + "status": "passed", + "parentMs": 8745, + "directMs": 3109, + "candidateMs": 8890, + "directStatus": "passed" + }, + { + "testId": "tests/flows-lists.e2e.ts::sticky%20chrome%3A%20accept%20below%20the%20fold%2C%20then%20continue", + "title": "sticky chrome: accept below the fold, then continue", + "file": "tests/flows-lists.e2e.ts", + "status": "passed", + "parentMs": 10532, + "directMs": 13379, + "candidateMs": 6412, + "directStatus": "passed" + }, + { + "testId": "tests/flows-lists.e2e.ts::async%20states%3A%20load%2C%20pull%20to%20refresh%2C%20claim%20through%20the%20toast", + "title": "async states: load, pull to refresh, claim through the toast", + "file": "tests/flows-lists.e2e.ts", + "status": "passed", + "parentMs": 10900, + "directMs": 7623, + "candidateMs": 10450, + "directStatus": "passed" + }, + { + "testId": "tests/flows-lists.e2e.ts::huge%20virtualized%20list%20reaches%20row%20512", + "title": "huge virtualized list reaches row 512", + "file": "tests/flows-lists.e2e.ts", + "status": "passed", + "parentMs": 48378, + "directMs": 50821, + "candidateMs": 48668, + "directStatus": "passed" + }, + { + "testId": "tests/flows-lists.e2e.ts::web%20view%20coupon%20form%20applies%20the%20code%20shown%20on%20the%20page", + "title": "web view coupon form applies the code shown on the page", + "file": "tests/flows-lists.e2e.ts", + "status": "skipped", + "parentMs": null, + "directMs": null, + "candidateMs": null, + "directStatus": "skipped" + }, + { + "testId": "tests/flows.e2e.ts::text%20inputs%20unlock%20the%20submit%20button", + "title": "text inputs unlock the submit button", + "file": "tests/flows.e2e.ts", + "status": "passed", + "parentMs": 7074, + "directMs": 1987, + "candidateMs": 5285, + "directStatus": "passed" + }, + { + "testId": "tests/flows.e2e.ts::filled%20text%20inputs%20report%20their%20values", + "title": "filled text inputs report their values", + "file": "tests/flows.e2e.ts", + "status": "passed", + "parentMs": 5929, + "directMs": 1569, + "candidateMs": 4636, + "directStatus": "passed" + }, + { + "testId": "tests/flows.e2e.ts::modal%20flow%20confirms%20through%20the%20native%20alert", + "title": "modal flow confirms through the native alert", + "file": "tests/flows.e2e.ts", + "status": "passed", + "parentMs": 7922, + "directMs": 1767, + "candidateMs": 4293, + "directStatus": "failed" + }, + { + "testId": "tests/flows.e2e.ts::bottom%20tabs%20act%20inside%20the%20last%20tab", + "title": "bottom tabs act inside the last tab", + "file": "tests/flows.e2e.ts", + "status": "passed", + "parentMs": 5767, + "directMs": 12224, + "candidateMs": 4164, + "directStatus": "passed" + }, + { + "testId": "tests/flows.e2e.ts::error%20recovery%20retries%2C%20then%20confirms%20the%20delete", + "title": "error recovery retries, then confirms the delete", + "file": "tests/flows.e2e.ts", + "status": "passed", + "parentMs": 7811, + "directMs": 4528, + "candidateMs": 6420, + "directStatus": "passed" + }, + { + "testId": "tests/flows.e2e.ts::choice%20controls%20place%20the%20exact%20order", + "title": "choice controls place the exact order", + "file": "tests/flows.e2e.ts", + "status": "passed", + "parentMs": 7833, + "directMs": 1760, + "candidateMs": 4145, + "directStatus": "passed" + }, + { + "testId": "tests/flows.e2e.ts::debounced%20search%20selects%20the%20target%20once%20results%20arrive", + "title": "debounced search selects the target once results arrive", + "file": "tests/flows.e2e.ts", + "status": "passed", + "parentMs": 7670, + "directMs": 3032, + "candidateMs": 7599, + "directStatus": "passed" + }, + { + "testId": "tests/login-form.e2e.ts::login%20form::rejects%20a%20malformed%20email", + "title": "rejects a malformed email", + "file": "tests/login-form.e2e.ts", + "status": "passed", + "parentMs": 7710, + "directMs": 2012, + "candidateMs": 6454, + "directStatus": "passed" + }, + { + "testId": "tests/login-form.e2e.ts::login%20form::rejects%20an%20unknown%20account", + "title": "rejects an unknown account", + "file": "tests/login-form.e2e.ts", + "status": "passed", + "parentMs": 7800, + "directMs": 2082, + "candidateMs": 6378, + "directStatus": "passed" + }, + { + "testId": "tests/login-form.e2e.ts::login%20form::signs%20in%20and%20logs%20out", + "title": "signs in and logs out", + "file": "tests/login-form.e2e.ts", + "status": "passed", + "parentMs": 11504, + "directMs": 6956, + "candidateMs": 11668, + "directStatus": "passed" + }, + { + "testId": "tests/smoke.e2e.ts::home%20lists%20the%20scenarios%20and%20opens%20one", + "title": "home lists the scenarios and opens one", + "file": "tests/smoke.e2e.ts", + "status": "passed", + "parentMs": 4246, + "directMs": 2336, + "candidateMs": 4095, + "directStatus": "passed" + } + ], + "testApp": { + "fixtureCommit": "498f939840858eb545cd8f30cc7c542c27c24c6c", + "checkoutViewportCorrection": { + "originalFailure": "Order summary was above the viewport after fixed scroll up0.6", + "verification": "one additional scroll revealed form-success and exact submitted summary without resubmission", + "correction": "scroll up --until id=\"form-success\"; both original summary assertions retained" + }, + "runtimeCommit": "fc612424d16cbaa6418107cb2552157f33a6e410", + "apkSha256": "4f332d39e76899b86e7006cda5d43ade6211733fe85ffe4552735e3e56c867b4", + "completedScenarios": [ + "smoke:fixture-bootstrap", + "smoke:inventory", + "smoke:automation-system", + "smoke:maestro-clickable-first", + "smoke:form-input", + "smoke:keyboard-ime", + "smoke:capture-close", + "full:lifecycle-system", + "full:observability-artifacts", + "full:fixture-replays" + ], + "behaviorIds": [ + "android-resource-id-selectors", + "cold-start-deep-link-navigation", + "helper-backed-gesture-recovery", + "home-recents-restoration", + "ime-owned-input-recovery", + "long-list-scroll-recovery", + "maestro-clickable-first-selection", + "orientation-fixture-state", + "push-broadcast-delivery", + "runtime-permission-recovery", + "safe-back-navigation", + "safe-keyboard-dismissal", + "system-ime-keyboard", + "test-ime-restoration", + "test-ime-unicode-input" + ], + "commands": [ + "alert", + "app-switcher", + "apps", + "appstate", + "artifacts", + "back", + "batch", + "capabilities", + "click", + "close", + "devices", + "diff", + "doctor", + "events", + "fill", + "find", + "focus", + "gesture", + "get", + "home", + "install", + "is", + "keyboard", + "logs", + "longpress", + "open", + "orientation", + "perf", + "press", + "push", + "record", + "replay", + "screenshot", + "scroll", + "settings", + "snapshot", + "swipe", + "test", + "trace", + "type", + "wait" + ], + "timings": { + "runDurationMs": 145716, + "scenarios": [ + { + "durationMs": 1143, + "id": "smoke:fixture-bootstrap" + }, + { + "durationMs": 1166, + "id": "smoke:inventory" + }, + { + "durationMs": 27145, + "id": "smoke:automation-system" + }, + { + "durationMs": 3423, + "id": "smoke:maestro-clickable-first" + }, + { + "durationMs": 4436, + "id": "smoke:form-input" + }, + { + "durationMs": 3702, + "id": "smoke:keyboard-ime" + }, + { + "durationMs": 2250, + "id": "smoke:capture-close" + }, + { + "durationMs": 32341, + "id": "full:lifecycle-system" + }, + { + "durationMs": 20964, + "id": "full:observability-artifacts" + }, + { + "durationMs": 47603, + "id": "full:fixture-replays" + } + ] + } + }, + "validation": { + "commit": "fc612424d16cbaa6418107cb2552157f33a6e410", + "exactCommit": true, + "checkoutChanged": false, + "relatedTestsPassed": 3910, + "regressionBase": { + "failed": 12, + "passed": 6, + "skipped": 36 + }, + "regressionHead": { + "passed": 18, + "skipped": 36 + }, + "adapterUnitTestsPassed": 352 + } +} diff --git a/docs/evidence/android-readiness-2026-10-19.md b/docs/evidence/android-readiness-2026-10-19.md new file mode 100644 index 0000000000..05e20da20d --- /dev/null +++ b/docs/evidence/android-readiness-2026-10-19.md @@ -0,0 +1,66 @@ +# Android readiness and lifecycle, 2026-10-19 + +The final candidate passed all 42 original Android cases without runner retries or the fixture's authored second-tap recovery. Runtime fell from 321.25 to 273.12 seconds, a 48.13-second reduction (15.0%). This measures the combined agent-device and e2e adapter changes. It does not establish parity with the experimental engine. + +[Follow-up startup experiments](android-startup-experiment-2026-10-19.md) reached 229.672 seconds but failed timed-overlay safety tests. Those policies were rejected. The follow-up also reproduces a capture-to-input race near the retained timer's boundary. + +The runtime is `fc612424d16cbaa6418107cb2552157f33a6e410`. The adapter runtime is `59307b157079b0b1a665b82afd0aa6b3845c2e2c`; final adapter commit `5e1d8952f4b0fd0d6a12b084ed8151e71e7cd889` changes only test assertions. The parent is `0d4b58c341bda7372d05e69a6bf58db6248e4f47`, whose runtime code is `ddba75f5b474cb4cdac11b15f75292f2870396c7`. The original app, tests, runner, and direct engine are [tester-army/e2e#1234](https://github.com/tester-army/e2e/pull/1234), commit `2593e45025edc52b2c381e809b41cb63d5878e2b`. + +[Machine-readable results](android-readiness-2026-10-19-results.json) retain every selected test identity, status, duration, comparison, and validation boundary. + +## Method and results + +All variants used the same locally built Release APK (`dev.e2e.benchmark`), Pixel_7_CI / emulator-5554, Android 16 API 36, one worker, zero runner retries, enabled animations, and the attached Device panel stream. APK SHA-256: `b163e985ebd8340bf10447cdf6826aea37e429ee8c4382f9179c863c8c0619b2`. Each SDK checkout used its own dependencies, built native helpers, and isolated daemon state. App and test source were unchanged. The adapter benchmark config adds an opt-in environment switch. + +| Variant | Passed | Failed | Source skip | Fixture recoveries | Elapsed | +| --- | ---: | ---: | ---: | ---: | ---: | +| Parent agent-device | 42 | 0 | 1 | 0 | 321.25 s | +| Final candidate plus adapter | 42 | 0 | 1 | 0 | 273.12 s | +| Experimental direct engine | 40 | 2 | 1 | 4 | 202.60 s | + +Exact selected and runnable identities match. Of 45 declarations, Android selects 43: 42 run, and row containment is explicitly skipped. Two declarations are iOS-only. Counts alone were not used to establish coverage. The direct engine's failures and recovery calls prevent treating its whole-run duration as an all-passing reliability comparison. + +On the 40 identities that passed on every engine, test durations total 308.199 seconds on the parent, 264.813 seconds on the candidate, and 199.428 seconds on direct. The remaining gap is 65.385 seconds. Startup transition waits and lifecycle proof remain costs; scrolling was not the source of the original comparative gap. + +| Case | Parent | Candidate | Reduction | +| --- | ---: | ---: | ---: | +| Autofocus and moving focus | 9.064 s | 4.222 s | 53.4% | +| Product catalog | 8.986 s | 4.315 s | 52.0% | +| Choice controls and order | 7.833 s | 4.145 s | 47.1% | +| Modal and native alert | 7.922 s | 4.293 s | 45.8% | + +These are local observations, not statistical estimates or hosted-CI measurements. The earlier broad-delegation candidates passed 42 cases in 233.26 and 229.75 seconds, but needed one and two authored fixture recoveries. Those timings are excluded from the final speedup. A passing runner summary hid the missed initial taps; the final comparison checks `tests/fixtures.ts:92` explicitly and requires zero calls. + +From `apps/mobile-benchmark`, after selecting the built SDK in the adapter's dependencies: + +```sh +E2E_MOBILE_DEVICE_READINESS=1 BENCH_ENGINE=agent-device \ + AGENT_DEVICE_STATE_DIR=/absolute/isolated-state \ + ANDROID_SERIAL=emulator-5554 E2E_TELEMETRY_DISABLED=1 \ + node node_modules/e2e/dist/cli/bin.js run \ + --config e2e.android.config.ts --workers 1 --retries 0 --output .e2e/readiness +``` + +For the parent, omit `E2E_MOBILE_DEVICE_READINESS` and select the parent SDK. For direct, omit `BENCH_ENGINE`. The app APK was installed before timing. No device boot, APK compilation, or installation time is included. + +## Safety boundaries + +- Complete captures that prove native cache invalidation no longer require a route change after a successful control press. Empty, sharply dropped, truncated, missing-root, and unproved captures retain their recovery checks. +- Android target motion uses a measured quiet position with a bounded transition recheck. Ref adoption requires a unique local identity with an identifier and the same known application package. Ambiguous or changed identities fail before dispatch. Ref recovery disclosure remains intact. +- Adapter delegation requires the daemon's advertised capability and an actual completed prior target press. Snapshot reads preserve this eligibility. Other command attempts and failed or cancelled mutations retire it. No-op toggles retire it. Startup, anonymous targets, inner-control rows, fill, keys, long press, and swipes retain the adapter's existing checks. +- Warm package open uses the helper only for the supported package-launch form. It preserves MAIN/DEFAULT/LAUNCHER intent semantics, quotes nested activity names, distinguishes installed-but-unlaunchable packages, and preserves multiline launch status and identity through Base64. Failed mutations are never resent. +- Existing foreground/process termination fences, IME ownership, and the documented frame-stat reset remain in place. Native stop consolidation was not shipped. + +## Validation + +The exact runtime commit passed `pnpm check:affected --run --base 0d4b58c341bda7372d05e69a6bf58db6248e4f47 --head fc612424d16cbaa6418107cb2552157f33a6e410`: 3,910 related tests, no failures, `exactCommit=true`, `checkoutChanged=false`. The native helper built locally. Independent reviews found no unresolved code findings. + +The same focused regression command on unchanged parent production code yielded 12 failures and 6 passes; the fixed implementation yielded 18 passes. Thirty-six unrelated cases were filtered in each run. The failures cover false stale-route recovery, lost SDK capability projection, helper launch behavior, and ref reindex/ambiguity handling. Additional focused lifecycle and motion checks passed 52 cases. Adapter unit checks passed 352 cases, including startup, back, failed mutations, no-op toggles, anonymous controls, and nested controls. + +The full `examples/test-app` Android suite also ran on the final runtime and rebuilt helper. All 10 scenarios passed. Its exact 41 commands and 15 behavior identities match the parent report. This includes permissions, system/Test IME restoration, keyboard dismissal, lifecycle, screenshot/recording, and retry-free replay. The Release fixture APK SHA-256 is `4f332d39e76899b86e7006cda5d43ade6211733fe85ffe4552735e3e56c867b4`. The fixture contains its bundled JavaScript and does not require Metro. + +That test-app run uses fixture commit `498f939840858eb545cd8f30cc7c542c27c24c6c`. An earlier run exposed a fixed-scroll assumption in its checkout replay: the submit succeeded, but the summary remained above the viewport. One additional scroll revealed both the success card and exact expected summary without resubmitting. The replay now uses `scroll up --until 'id="form-success"'`; it retains both original summary assertions and does not retry submit. The full suite passed in 145.716 seconds. Three additional independent checkout runs also passed without retries. This fixture correction does not affect the original e2e benchmark measurements. + +The final adapter passed its root `pnpm check` and all 352 mobile unit tests against its actual pinned published SDK, `agent-device@0.21.28`. Those checks validate compatibility of the default path; they do not prove the unreleased opt-in capability works with that SDK. + +Coverage, native CI builds, device/provider lanes, and required checks at the pushed head remain GitHub obligations. The companion adapter PR is a draft until a released agent-device SDK preserves the capability and its pinned dependency/lockfile are updated; published `agent-device@0.21.28` drops the field. The measured runs explicitly linked the built SDK. diff --git a/docs/evidence/android-startup-experiment-2026-10-19-results.json b/docs/evidence/android-startup-experiment-2026-10-19-results.json new file mode 100644 index 0000000000..64139d98cb --- /dev/null +++ b/docs/evidence/android-startup-experiment-2026-10-19-results.json @@ -0,0 +1,361 @@ +{ + "reviewedRuntimeSha": "d01199dcbe3e7e62a216d3ce4d071ff8b4a60ca1", + "adapterSha": "5e1d8952f4b0fd0d6a12b084ed8151e71e7cd889", + "referenceSha": "2593e45025edc52b2c381e809b41cb63d5878e2b", + "device": "Pixel_7_CI / emulator-5554 / API 36", + "workers": 1, + "runnerRetries": 0, + "candidateStatus": "rejected: late overlays can activate a decoy", + "fullBenchmark": { + "startedAt": "2026-10-19T12:36:55.564Z", + "finishedAt": "2026-10-19T12:40:45.236Z", + "elapsedSeconds": 229.672, + "summary": { + "discovered": 45, + "selected": 43, + "executed": 42, + "passed": 42, + "failed": 0, + "interrupted": 0, + "flaky": 0, + "skipped": 1 + }, + "fixtureRecoveryCalls": 0, + "sameSelectedIdentitiesAsPublishedBaseline": true + }, + "ourApp": { + "timer": { + "startedAt": "2026-10-19T12:52:05.563Z", + "finishedAt": "2026-10-19T12:52:27.012Z", + "elapsedSeconds": 21.449, + "summary": { + "discovered": 3, + "selected": 3, + "executed": 3, + "passed": 3, + "failed": 0, + "interrupted": 0, + "flaky": 0, + "skipped": 0 + }, + "fixtureRecoveryCalls": 0 + }, + "measured": { + "startedAt": "2026-10-19T12:52:28.776Z", + "finishedAt": "2026-10-19T12:52:46.985Z", + "elapsedSeconds": 18.209, + "summary": { + "discovered": 3, + "selected": 3, + "executed": 3, + "passed": 3, + "failed": 0, + "interrupted": 0, + "flaky": 0, + "skipped": 0 + }, + "fixtureRecoveryCalls": 0 + } + }, + "cohorts": [ + { + "name": "readiness-oracle-timer", + "trials": 39, + "outcomes": { + "activation": 30, + "refusal-unconfirmed": 9 + }, + "activationMedianMs": 1619.5 + }, + { + "name": "readiness-oracle-measured", + "trials": 39, + "outcomes": { + "activation": 27, + "refusal-unconfirmed": 12 + }, + "activationMedianMs": 626 + }, + { + "name": "readiness-oracle-zero", + "trials": 39, + "outcomes": { + "activation": 13, + "decoy-hit": 17, + "silent-miss": 6, + "refusal-unconfirmed": 3 + }, + "activationMedianMs": 394 + }, + { + "name": "readiness-oracle-direct", + "trials": 39, + "outcomes": { + "activation": 20, + "silent-miss": 7, + "decoy-hit": 9, + "refusal-unconfirmed": 3 + }, + "activationMedianMs": 660.0 + }, + { + "name": "readiness-overlay-sweep", + "trials": 50, + "outcomes": { + "refusal-unconfirmed": 27, + "decoy-hit": 4, + "activation": 19 + }, + "activationMedianMs": 512 + }, + { + "name": "readiness-input-observe", + "trials": 15, + "outcomes": { + "activation": 8, + "refusal-unconfirmed": 6, + "decoy-hit": 1 + }, + "activationMedianMs": 523.5 + }, + { + "name": "readiness-input-observe-spinner", + "trials": 9, + "outcomes": { + "refusal-unconfirmed": 4, + "activation": 5 + }, + "activationMedianMs": 609 + }, + { + "name": "readiness-timer-late-control", + "trials": 30, + "outcomes": { + "refusal-unconfirmed": 12, + "decoy-hit": 3, + "activation": 15 + }, + "activationMedianMs": 1613 + }, + { + "name": "readiness-negative-controls", + "trials": 4, + "outcomes": { + "silent-miss": 1, + "decoy-hit": 1, + "duplicate-activation": 1, + "stale-activity": 1 + } + }, + { + "name": "readiness-input-app-trace", + "trials": 15, + "outcomes": { + "refusal-unconfirmed": 4, + "activation": 9, + "decoy-hit": 2 + }, + "activationMedianMs": 540 + }, + { + "name": "readiness-input-bypass-down", + "trials": 15, + "outcomes": { + "activation": 10, + "refusal-unconfirmed": 3, + "decoy-hit": 2 + }, + "activationMedianMs": 538.5 + } + ], + "wrongTargetTimelines": { + "readiness-input-observe": [ + { + "trial": "measured-overlay-500-4-1792414705780", + "delayMs": 500, + "firstTargetReadUptimeMs": 1558263408, + "lastTargetReadBeforeClickUptimeMs": 1558263903, + "decoyClickUptimeMs": 1558263966, + "nativeInput": [ + "begin action=0 at=1558263929 x=219.0 y=1034.0", + "end action=0 at=1558263946 waitedMs=17", + "begin action=1 at=1558263946 x=219.0 y=1034.0", + "end action=1 at=1558263961 waitedMs=15" + ], + "appEvents": [] + } + ], + "readiness-timer-late-control": [ + { + "trial": "timer-overlay-1550-0-1792414946579", + "delayMs": 1550, + "firstTargetReadUptimeMs": 1558504252, + "lastTargetReadBeforeClickUptimeMs": 1558505783, + "decoyClickUptimeMs": 1558505836, + "nativeInput": [ + "begin action=0 at=1558505805 x=219.0 y=1034.0", + "end action=0 at=1558505820 waitedMs=15", + "begin action=1 at=1558505820 x=219.0 y=1034.0", + "end action=1 at=1558505830 waitedMs=10" + ], + "appEvents": [] + }, + { + "trial": "timer-overlay-1550-1-1792414980639", + "delayMs": 1550, + "firstTargetReadUptimeMs": 1558538400, + "lastTargetReadBeforeClickUptimeMs": 1558539921, + "decoyClickUptimeMs": 1558539990, + "nativeInput": [ + "begin action=0 at=1558539942 x=219.0 y=1034.0", + "end action=0 at=1558539970 waitedMs=28", + "begin action=1 at=1558539970 x=219.0 y=1034.0", + "end action=1 at=1558539982 waitedMs=12" + ], + "appEvents": [] + }, + { + "trial": "timer-overlay-1550-2-1792415014076", + "delayMs": 1550, + "firstTargetReadUptimeMs": 1558571670, + "lastTargetReadBeforeClickUptimeMs": 1558573198, + "decoyClickUptimeMs": 1558573247, + "nativeInput": [ + "begin action=0 at=1558573217 x=219.0 y=1034.0", + "end action=0 at=1558573231 waitedMs=14", + "begin action=1 at=1558573231 x=219.0 y=1034.0", + "end action=1 at=1558573242 waitedMs=11" + ], + "appEvents": [] + } + ], + "readiness-input-app-trace": [ + { + "trial": "measured-overlay-550-1-1792415108112", + "delayMs": 550, + "firstTargetReadUptimeMs": 1558665754, + "lastTargetReadBeforeClickUptimeMs": 1558666275, + "decoyClickUptimeMs": 1558666334, + "nativeInput": [ + "begin action=0 at=1558666291 x=219.0 y=1034.0", + "end action=0 at=1558666313 waitedMs=22", + "begin action=1 at=1558666313 x=219.0 y=1034.0", + "end action=1 at=1558666325 waitedMs=12" + ], + "appEvents": [ + "disturb trial=measured-overlay-550-1-1792415108112 mode=overlay at=1558666304", + "receipt kind=Blocking_sheet trial=measured-overlay-550-1-1792415108112 action=0 at=1558666312 eventAt=1558666291 downAt=1558666291 rect=Rect(74, 950 - 363, 1118)", + "receipt kind=Blocking_sheet trial=measured-overlay-550-1-1792415108112 action=1 at=1558666314 eventAt=1558666313 downAt=1558666291 rect=Rect(74, 950 - 363, 1118)" + ] + }, + { + "trial": "measured-overlay-500-3-1792415119698", + "delayMs": 500, + "firstTargetReadUptimeMs": 1558677630, + "lastTargetReadBeforeClickUptimeMs": 1558678091, + "decoyClickUptimeMs": 1558678165, + "nativeInput": [ + "begin action=0 at=1558678123 x=219.0 y=1034.0", + "end action=0 at=1558678146 waitedMs=23", + "begin action=1 at=1558678146 x=219.0 y=1034.0", + "end action=1 at=1558678162 waitedMs=16" + ], + "appEvents": [ + "disturb trial=measured-overlay-500-3-1792415119698 mode=overlay at=1558678131", + "receipt kind=Blocking_sheet trial=measured-overlay-500-3-1792415119698 action=0 at=1558678146 eventAt=1558678123 downAt=1558678123 rect=Rect(74, 950 - 363, 1118)", + "receipt kind=Blocking_sheet trial=measured-overlay-500-3-1792415119698 action=1 at=1558678147 eventAt=1558678146 downAt=1558678123 rect=Rect(74, 950 - 363, 1118)" + ] + } + ], + "readiness-input-bypass-down": [ + { + "trial": "measured-overlay-550-3-1792415207912", + "delayMs": 550, + "firstTargetReadUptimeMs": 1558765530, + "lastTargetReadBeforeClickUptimeMs": 1558766067, + "decoyClickUptimeMs": 1558766151, + "nativeInput": [ + "begin action=0 at=1558766107 x=219.0 y=1034.0", + "end action=0 at=1558766132 waitedMs=25", + "begin action=1 at=1558766132 x=219.0 y=1034.0", + "end action=1 at=1558766142 waitedMs=10" + ], + "appEvents": [ + "disturb trial=measured-overlay-550-3-1792415207912 mode=overlay at=1558766081", + "receipt kind=Blocking_sheet trial=measured-overlay-550-3-1792415207912 action=0 at=1558766130 eventAt=1558766107 downAt=1558766107 rect=Rect(74, 950 - 363, 1118)", + "receipt kind=Blocking_sheet trial=measured-overlay-550-3-1792415207912 action=1 at=1558766133 eventAt=1558766132 downAt=1558766107 rect=Rect(74, 950 - 363, 1118)" + ] + }, + { + "trial": "measured-overlay-450-4-1792415210386", + "delayMs": 450, + "firstTargetReadUptimeMs": 1558768149, + "lastTargetReadBeforeClickUptimeMs": 1558768584, + "decoyClickUptimeMs": 1558768634, + "nativeInput": [ + "begin action=0 at=1558768601 x=219.0 y=1034.0", + "end action=0 at=1558768614 waitedMs=13", + "begin action=1 at=1558768614 x=219.0 y=1034.0", + "end action=1 at=1558768634 waitedMs=20" + ], + "appEvents": [ + "disturb trial=measured-overlay-450-4-1792415210386 mode=overlay at=1558768599", + "receipt kind=Blocking_sheet trial=measured-overlay-450-4-1792415210386 action=0 at=1558768612 eventAt=1558768601 downAt=1558768601 rect=Rect(74, 950 - 363, 1118)", + "receipt kind=Blocking_sheet trial=measured-overlay-450-4-1792415210386 action=1 at=1558768615 eventAt=1558768614 downAt=1558768601 rect=Rect(74, 950 - 363, 1118)" + ] + } + ] + }, + "observedInjectionWaitMs": { + "DOWN": { + "n": 9, + "min": 7, + "median": 12, + "max": 21 + }, + "UP": { + "n": 9, + "min": 8, + "median": 14, + "max": 16 + } + }, + "publishedBaselineElapsedSeconds": 273.122, + "candidateReductionSeconds": 43.45, + "candidateReductionPercent": 15.9, + "bypassDownPath": { + "threeArgumentEvents": 12, + "fallbackEvents": 0, + "DOWNWaitMs": { + "n": 12, + "min": 6, + "median": 15.5, + "max": 25 + }, + "UPWaitMs": { + "n": 12, + "min": 10, + "median": 13, + "max": 20 + }, + "promoted": false + }, + "oracleInterpretation": "Refusals with zero handler activations are separate from successful activations. Cohorts use deliberately selected timing offsets and are not estimates of production failure rates.", + "limitations": [ + "One local API 36 emulator; no statistical or CI claim.", + "No older API or window-transition certification for the bypass, which already failed its falsification fixture.", + "Benchmark and three test-app workflows passed the measured candidate; timed overlays falsified safety.", + "Public-only Opus reviews were design/source challenges, not approval of the complete unpublished patch." + ], + "artifact": { + "name": "android-startup-experiment-2026-10-19.tar.gz", + "sha256": "9d7a7cda4f2c465069ad640efb90182127ae60d5c86f492355bb5eefecc7decd", + "checksummedFiles": 259 + }, + "cleanup": { + "ownedDaemonStatesStopped": 22, + "disposableWorktreesRemoved": 2, + "reviewedShaBeforeEvidenceEdits": "d01199dcbe3e7e62a216d3ce4d071ff8b4a60ca1", + "reviewedStatusBeforeEvidenceEdits": "clean" + } +} diff --git a/docs/evidence/android-startup-experiment-2026-10-19.md b/docs/evidence/android-startup-experiment-2026-10-19.md new file mode 100644 index 0000000000..213ec74dcd --- /dev/null +++ b/docs/evidence/android-startup-experiment-2026-10-19.md @@ -0,0 +1,92 @@ +# Android startup readiness experiments, 2026-10-19 + +The startup early-exit experiment saved 43.450 seconds on the original 42-case suite, but it activated a covering control in timed-overlay tests. Do not promote it. Removing the framework animation wait from DOWN also failed that oracle. This follow-up changes evidence only; the published runtime and adapter policies remain unchanged. + +The retained 1500 ms startup timer also failed when overlays arrived near its boundary. The timer mitigates a race; it does not establish an atomic relationship between the observed target and delivered input. A longer timer alone cannot provide that guarantee. + +[Machine-readable results](android-startup-experiment-2026-10-19-results.json) retain cohort counts, native timestamps, rejected policies, and validation limits. [The previous accepted measurements](android-readiness-2026-10-19.md) remain attributed to their original commits. + +## Inputs and isolation + +The reviewed runtime was `d01199dcbe3e7e62a216d3ce4d071ff8b4a60ca1`. The adapter was `5e1d8952f4b0fd0d6a12b084ed8151e71e7cd889`. The direct-engine reference was `2593e45025edc52b2c381e809b41cb63d5878e2b`. Separate disposable checkouts received their own dependency installations and builds. The experiments ran sequentially on Pixel_7_CI / emulator-5554, Android 16 API 36, with one worker and zero runner retries. + +The benchmark APK and its 42 runnable Android tests were unchanged. Its SHA-256 remains `b163e985ebd8340bf10447cdf6826aea37e429ee8c4382f9179c863c8c0619b2`. The test-app APK remains `4f332d39e76899b86e7006cda5d43ade6211733fe85ffe4552735e3e56c867b4`. + +The adapter experiment replaced only the startup path with no prior action tree. It required a complete same-app observation, one uniquely matching target, unchanged surface and keyboard facts, and at least three equal target rectangles spanning 100 ms. It polled every 50 ms within the remaining transition budget. Missing, ambiguous, or unsettled targets refused. The existing device press guards still ran afterward. The zero-wait control skipped this startup observation policy; it retained the remaining runtime guards. + +## Performance result, rejected for safety + +| Run | Passed | Fixture recovery taps | Elapsed | +| --- | ---: | ---: | ---: | +| Published runtime plus adapter, retained timer | 42 | 0 | 273.122 s | +| Startup measured-readiness experiment | 42 | 0 | 229.672 s | +| Test-app affected workflows, timer | 3 | 0 | 21.449 s | +| Test-app affected workflows, measured | 3 | 0 | 18.209 s | + +The full benchmark selected the same identities: 43 selected, 42 executed and passed, and one authored skip. The startup experiment reduced elapsed time by 15.9%. This is one local measurement, not a statistical or CI claim. A timer/measured/measured/timer pilot preceded the full run. + +The three test-app workflows checked startup form navigation and exact controlled values, checkout submission and the exact submitted summary, and startup settings navigation. These runs exercise the changed adapter route. They do not replace the previously recorded full test-app verification at the unchanged runtime. + +Passing app suites did not prove the startup policy safe. The additional fixture below falsified it without relying on runner status or an authored recovery tap. + +## Activation oracle and counterexamples + +A separate native fixture, `dev.agentdevice.readinessprobe`, exposes target and decoy handler counters plus a per-trial identity. Disturbances begin after the first accessibility read of the target. Each trial makes one requested action, reads the counters after 800 ms, and checks the trial identity. There are no action retries. A valid activation must produce exactly one target hit and zero decoy hits. Refusals with zero activations remain separate outcomes. + +The first cohort contains 13 cells repeated three times: static, immediate and delayed slides, overlays, remounts, an instant shift, keyboard movement, duplicate identities, and continuous target motion. + +| Policy | Correct activations | Refused, no activation | Wrong activations | Silent misses | +| --- | ---: | ---: | ---: | ---: | +| Retained timer | 30 | 9 | 0 | 0 | +| Measured startup | 27 | 12 | 0 | 0 | +| Zero startup wait | 13 | 3 | 17 | 6 | +| Original direct engine | 20 | 3 | 9 | 7 | + +The timer and measured policies initially appeared safe. A wider sweep changed that conclusion: + +| Falsification cohort | Trials | Correct | Refused | Wrong target | +| --- | ---: | ---: | ---: | ---: | +| Measured startup, overlay delays 200–650 ms | 50 | 19 | 27 | 4 | +| Retained timer, overlay delays 1300–1900 ms | 30 | 15 | 12 | 3 | +| Measured startup, native input instrumentation | 15 | 8 | 6 | 1 | +| Measured startup, app touch instrumentation | 15 | 9 | 4 | 2 | +| Measured startup, DOWN-only animation-wait bypass | 15 | 10 | 3 | 2 | + +These deliberately selected timing distributions are different. They demonstrate counterexamples; they do not estimate production failure rates or establish a relative reliability rate. A tap delivered successfully before a future overlay appears is valid. The wrong-target rows instead contain actual decoy handler activations. + +An unrelated background ticker produced two correct activations and one readiness refusal in three trials. Static controls activated in all three companion trials. Continuous target motion refused in all three. Target-scoped readiness therefore still needs a budget and capture-cost check; global visual quiet would also penalize unrelated updates. + +Four negative controls each failed the oracle for the intended reason: empty-point input produced `target=0, decoy=0`; explicit decoy input produced `0,1`; duplicate input produced `2,0`; and a replaced activity returned a different trial identity. Setup errors and the earlier harness classification correction are excluded from the cohort totals. + +## Native delivery trace + +One instrumented wrong-target trial provides this sequence on the shared device uptime clock: + +| Event | Uptime | Since final target read | +| --- | ---: | ---: | +| Final target accessibility read | 1558666275 ms | 0 ms | +| Native DOWN injection begins | 1558666291 ms | 16 ms | +| Overlay disturbance begins | 1558666304 ms | 29 ms | +| Covering control receives DOWN | 1558666312 ms | 37 ms | +| Covering control receives UP | 1558666314 ms | 39 ms | +| Decoy click handler increments | 1558666334 ms | 59 ms | + +The preceding instrumentation cohort measured DOWN injection at 7–21 ms, median 12 ms. Its wrong-target trial waited 17 ms. This does not support a large global animation wait as the source of this failure or of the remaining suite-wide gap. + +The DOWN-only bypass reused the existing three-argument dispatcher. Logs confirm 12 three-argument calls and zero fallback calls; three trials refused before injection. DOWN took 6–25 ms, median 15.5 ms. Two decoy activations falsified the bypass as a solution. No speed or compatibility claim follows from this small diagnostic cohort. + +Android's connection code synchronizes input transactions before DOWN and after UP, passing through the animation-wait policy. Disabling that policy still leaves transaction synchronization and synchronous event delivery. See [AOSP Android 16 UiAutomationConnection](https://github.com/aosp-mirror/platform_frameworks_base/blob/android16-release/core/java/android/app/UiAutomationConnection.java#L126-L163). This is consistent with the measured short waits and surviving race. + +## Disposition and next experiment + +Three public-source review rounds with Claude Opus challenged delayed motion, identity, window transforms, input ordering, and negative-control quality. The review corrected an overbroad claim about future animations: success is judged at input delivery, not against a later final rectangle. It also distinguished native target handles from unique selector resolution. These reviews were design and source challenges, not approval of the complete unpublished patch. + +Reject both experimental policies. Keep their measured performance attributed to rejected candidates. Preserve complete same-app observations, unique target identity, offscreen and cover checks, keyboard checks, typed refusals, and dispatch provenance. + +The next useful experiment is one native operation that refreshes the resolved target, validates its current input surface, and dispatches with the smallest possible gap. It must measure actual app-side delivery, not just RPC completion. This is a hypothesis to test, not a demonstrated fix. Use the timed overlays, incoming windows, IME transitions, and the oldest supported Android API as falsification cases. Avoid reintroducing nearest-match selection or a global quiet requirement. + +The other large opportunity remains repeated full-tree capture and readiness polling. Moving those observations closer to the native target could remove round trips without weakening freshness. The earlier cost audit measured 50.111 seconds of first-tap idle, 36.222 seconds of other locator idle, and 58.802 seconds across 736 daemon snapshot requests on the shared passing cases. These categories can overlap and must not be added together. + +The local archive retains patches, fixture source and APKs, harnesses, reports, logs, and per-file checksums: `android-startup-experiment-2026-10-19.tar.gz`, SHA-256 `9d7a7cda4f2c465069ad640efb90182127ae60d5c86f492355bb5eefecc7decd`. It is a local handoff artifact, not a repository-hosted download. Twenty-two owned daemon states were stopped. Both disposable checkouts were removed. The reviewed SHA and clean tracked status were verified before these evidence-only edits. No runtime candidate was promoted. + +[The edited-field follow-up](android-edited-field-experiment-2026-10-19.md) rejects a same-field timer shortcut and retains a guarded clear-verification fix. diff --git a/examples/test-app/replays/checkout-form-android.ad b/examples/test-app/replays/checkout-form-android.ad index efbe0402fe..6a75c93b09 100644 --- a/examples/test-app/replays/checkout-form-android.ad +++ b/examples/test-app/replays/checkout-form-android.ad @@ -18,7 +18,7 @@ wait "Delivery" 5000 scroll down 0.7 click id="checkbox-agree" click id="submit-order" -scroll up 0.6 +scroll up --until 'id="form-success"' wait "Order summary" 5000 wait "Ada Lovelace chose pickup with cash payment." 5000 close diff --git a/packages/capture-kit/src/snapshot/snapshot-freshness/android.test.ts b/packages/capture-kit/src/snapshot/snapshot-freshness/android.test.ts index d4cd87bbc7..9b543a9b8d 100644 --- a/packages/capture-kit/src/snapshot/snapshot-freshness/android.test.ts +++ b/packages/capture-kit/src/snapshot/snapshot-freshness/android.test.ts @@ -153,3 +153,38 @@ test('isLikelySnapshotStuckOnPreviousRoute ignores missing baselines and tiny tr const tiny = makeSnapshotState(labeledNodes(5)).nodes; assert.equal(isLikelySnapshotStuckOnPreviousRoute(buildSnapshotSignatures(tiny), tiny), false); }); + +test.each(['unchanged', 'selected', 'checked', 'focused', 'enabled'] as const)( + 'a cache-invalidated capture trusts a complete same-screen %s state after a press', + (state) => { + const baseline = makeSnapshotState(labeledNodes(40)); + const current = makeSnapshotState(labeledNodes(40)); + if (state !== 'unchanged') current.nodes[1]![state] = true; + assert.equal( + androidFreshnessReason( + { snapshot: current, rawNodeCount: 40, cacheInvalidated: true }, + freshnessWindow({ + action: 'press', + baselineCount: 40, + baselineSignatures: buildSnapshotSignatures(baseline.nodes), + routeComparable: true, + }), + { interactiveOnly: false }, + ), + null, + ); + }, +); + +test('cache invalidation does not excuse an empty or content-poor capture', () => { + for (const interactiveOnly of [true, false]) { + assert.equal( + androidFreshnessReason( + { snapshot: makeSnapshotState([]), rawNodeCount: 40, cacheInvalidated: true }, + freshnessWindow({ baselineCount: 40 }), + { interactiveOnly }, + ), + interactiveOnly ? 'empty-interactive' : 'sharp-drop', + ); + } +}); diff --git a/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts b/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts index 26955e8ab5..9d5773a56c 100644 --- a/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts +++ b/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts @@ -90,6 +90,8 @@ export function androidFreshnessReason( return !hasMeaningfulSnapshotContent(attempt.snapshot) ? 'sharp-drop' : null; } + if (attempt.cacheInvalidated === true) return null; + return window.routeComparable && isNavigationSensitiveAction(window.action) && isLikelySnapshotStuckOnPreviousRoute(window.baselineSignatures, attempt.snapshot.nodes) diff --git a/packages/capture-kit/src/snapshot/snapshot-freshness/types.ts b/packages/capture-kit/src/snapshot/snapshot-freshness/types.ts index 3b7fb6609d..633cba4126 100644 --- a/packages/capture-kit/src/snapshot/snapshot-freshness/types.ts +++ b/packages/capture-kit/src/snapshot/snapshot-freshness/types.ts @@ -34,4 +34,6 @@ export type SnapshotFreshnessMode = 'default' | 'ref-refresh'; export type SnapshotFreshnessAttempt = { snapshot: SnapshotState; rawNodeCount: number | undefined; + /** The producer proved a complete traversal after invalidating the platform node cache. */ + cacheInvalidated?: boolean; }; diff --git a/packages/contracts/src/client-device-view.ts b/packages/contracts/src/client-device-view.ts index 549f88529b..2017a9fb42 100644 --- a/packages/contracts/src/client-device-view.ts +++ b/packages/contracts/src/client-device-view.ts @@ -51,6 +51,7 @@ export type AgentDeviceDevice = { export type AgentDeviceCapabilitiesResult = { device: AgentDeviceDevice; availableCommands: string[]; + targetReadiness?: { press: 'bounded-geometry' }; }; export type AgentDeviceSessionDevice = { diff --git a/packages/contracts/src/snapshot-types.ts b/packages/contracts/src/snapshot-types.ts index 3c0c56e093..a8291fba63 100644 --- a/packages/contracts/src/snapshot-types.ts +++ b/packages/contracts/src/snapshot-types.ts @@ -72,6 +72,8 @@ export type AndroidSnapshotBackendMetadata = { maxDepth?: number; maxNodes?: number; rootPresent?: boolean; + /** This capture cleared the platform accessibility node cache through UiAutomation.clearCache(). */ + cacheInvalidated?: boolean; captureMode?: string; systemSurfaceOnly?: boolean; windowCount?: number; diff --git a/packages/daemon/src/core/__tests__/session-snapshot-freshness.test.ts b/packages/daemon/src/core/__tests__/session-snapshot-freshness.test.ts index 98463fe445..cea42e97a0 100644 --- a/packages/daemon/src/core/__tests__/session-snapshot-freshness.test.ts +++ b/packages/daemon/src/core/__tests__/session-snapshot-freshness.test.ts @@ -1,11 +1,13 @@ import assert from 'node:assert/strict'; import { afterEach, test, vi } from 'vitest'; +import type { SnapshotFreshnessAttemptShape } from '@agent-device/capture-kit/snapshot-freshness'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { clearAndroidSnapshotFreshness, getActiveAndroidSnapshotFreshness, markAndroidSnapshotFreshness, + captureAndroidFreshnessRecoveredAttempt, } from '../session-snapshot-freshness.ts'; import { makeSession } from './post-gesture-stabilization-fixtures.ts'; @@ -65,3 +67,43 @@ test('clear removes an active freshness window', () => { clearAndroidSnapshotFreshness(session); assert.equal(session.androidSnapshotFreshness, undefined); }); + +test.each(['complete', 'truncated', 'missing-root', 'unknown-cache'] as const)( + 'freshness trusts only complete cache-invalidated acquisitions: %s', + async (mode) => { + vi.useFakeTimers(); + const session = makeSession('android'); + session.snapshot = makeSnapshotState(labeledNodes(40), { comparisonSafe: true }); + markAndroidSnapshotFreshness(session, 'press'); + let captures = 0; + const result = captureAndroidFreshnessRecoveredAttempt( + { + session, + interactiveOnly: false, + capture: async () => { + captures++; + return { + snapshot: session.snapshot!, + annotations: { + androidSnapshot: { + backend: 'android-helper' as const, + rootPresent: true, + cacheInvalidated: mode !== 'unknown-cache', + helperTruncated: mode === 'truncated', + missingRootWindowTypes: mode === 'missing-root' ? [1] : [], + }, + }, + }; + }, + }, + session.androidSnapshotFreshness!, + ); + await vi.runAllTimersAsync(); + const capture: SnapshotFreshnessAttemptShape = await result; + assert.equal(captures, mode === 'complete' ? 1 : 4); + assert.equal( + capture.annotations.freshness?.staleAfterRetries, + mode === 'complete' ? undefined : true, + ); + }, +); diff --git a/packages/daemon/src/core/session-snapshot-freshness.ts b/packages/daemon/src/core/session-snapshot-freshness.ts index a61eadc56d..3cf8888d98 100644 --- a/packages/daemon/src/core/session-snapshot-freshness.ts +++ b/packages/daemon/src/core/session-snapshot-freshness.ts @@ -93,7 +93,15 @@ export async function captureAndroidFreshnessRecoveredAttempt< capture: params.capture, classify: (attempt) => androidFreshnessReason( - { snapshot: attempt.snapshot, rawNodeCount: attempt.annotations.analysis?.rawNodeCount }, + { + snapshot: attempt.snapshot, + rawNodeCount: attempt.annotations.analysis?.rawNodeCount, + cacheInvalidated: + attempt.annotations.androidSnapshot?.cacheInvalidated === true && + attempt.annotations.androidSnapshot.rootPresent === true && + attempt.annotations.androidSnapshot.helperTruncated === false && + attempt.annotations.androidSnapshot.missingRootWindowTypes?.length === 0, + }, freshness, { interactiveOnly: params.interactiveOnly, mode: params.androidFreshnessMode }, ), diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/__tests__/session-capabilities.test.ts b/packages/daemon/src/sessions/session-lifecycle/internal/__tests__/session-capabilities.test.ts index 37ee0e5680..5a10f8a755 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/__tests__/session-capabilities.test.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/__tests__/session-capabilities.test.ts @@ -82,7 +82,12 @@ async function projectAndroidCapabilities(sessionName: string) { }); expect(response?.ok).toBe(true); const data = response?.ok ? response.data : undefined; - return { runtime, device: data?.device, availableCommands: data?.availableCommands }; + return { + runtime, + device: data?.device, + availableCommands: data?.availableCommands, + targetReadiness: data?.targetReadiness, + }; } test('capabilities reports supported commands for the selected session device', async () => { @@ -772,3 +777,8 @@ const dumpEmptyAndroidNetwork: PlatformRuntimeOperations['networkDump'] = async }, notes: [], }); + +test('Android advertises the bounded press geometry guard to consuming harnesses', async () => { + const { targetReadiness } = await projectAndroidCapabilities('android-target-readiness'); + expect(targetReadiness).toEqual({ press: 'bounded-geometry' }); +}); diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/inventory.ts b/packages/daemon/src/sessions/session-lifecycle/internal/inventory.ts index ad01c3bd5b..3303eafaaa 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/inventory.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/inventory.ts @@ -285,6 +285,9 @@ async function capabilitiesInventoryResponse(params: { ok: true, data: { device: publicDeviceInfo(device), + ...(device.platform === 'android' && factOwnedCapabilityAvailable('press', facts) + ? { targetReadiness: { press: 'bounded-geometry' } } + : {}), availableCommands: listRuntimeFactCommands().filter((command) => { // A session that already owns an app identity answers appstate itself, so it stays // available even when the sessionless runtime probe cannot see a foreground app. diff --git a/packages/platform-android/src/__tests__/fill-verification.test.ts b/packages/platform-android/src/__tests__/fill-verification.test.ts index 25502c4321..81201caefd 100644 --- a/packages/platform-android/src/__tests__/fill-verification.test.ts +++ b/packages/platform-android/src/__tests__/fill-verification.test.ts @@ -159,7 +159,7 @@ test('the pre-action target read shares the daemon-session helper with the sampl return before; }); - assert.equal(target?.resourceId, 'com.example:id/field'); + assert.equal(target?.targetInput?.resourceId, 'com.example:id/field'); assert.equal(session.captureCount(), 3); assert.equal(session.spawnArgs.length, 1); assert.equal( @@ -179,7 +179,11 @@ type FillHelperSession = { }; function createFillHelperSession( - options: { textForCapture?: (captureIndex: number) => string } = {}, + options: { + textForCapture?: (captureIndex: number) => string; + xmlForCapture?: (captureIndex: number) => string; + metadataForCapture?: (captureIndex: number) => Record; + } = {}, ): FillHelperSession { const calls: (readonly string[])[] = []; const spawnArgs: (readonly string[])[] = []; @@ -191,9 +195,13 @@ function createFillHelperSession( spawnArgs, processes, sessionRequests: requests, + sessionSnapshotMetadata: (_, captureIndex) => options.metadataForCapture?.(captureIndex) ?? {}, sessionXml: () => { captureCount += 1; - return filledFieldXml(options.textForCapture?.(captureCount) ?? 'chips'); + return ( + options.xmlForCapture?.(captureCount) ?? + filledFieldXml(options.textForCapture?.(captureCount) ?? 'chips') + ); }, }); return { @@ -298,3 +306,40 @@ test('helper window provenance identifies a custom IME without an adb keyboard p assert.equal(verification.reason, 'ime_capture'); assert.equal(verification.actualInput?.packageName, 'dev.custom.keyboard'); }); + +for (const mode of ['unique', 'duplicate', 'truncated', 'cache-unknown', 'missing-root'] as const) { + test(`moved clear requires unique fresh complete pre-action capture: ${mode}`, async () => { + const beforeXml = filledFieldXml('old'); + const movedXml = + '' + + '' + + '' + + ''; + const session = createFillHelperSession({ + xmlForCapture: (i) => + i === 1 + ? mode === 'duplicate' + ? beforeXml.replace( + '', + '', + ) + : beforeXml + : movedXml, + metadataForCapture: (i) => ({ + cacheInvalidated: i === 1 && mode === 'cache-unknown' ? '' : 'true', + missingRootWindowTypes: i === 1 && mode === 'missing-root' ? '2' : '', + ...(i === 1 && mode === 'truncated' ? { truncated: 'true' } : {}), + }), + }); + const verification = await underFakeTimers(() => + withFillHelperProvider(session.provider, async () => { + const helper = { helperSessionScope: 'daemon-session' as const }; + const before = await readAndroidFillTargetBeforeMutation(device, 10, 10, helper); + return verifyAndroidFilledText(device, 10, 10, '', helper, before); + }), + ); + assert.equal(verification.ok, mode === 'unique'); + if (mode === 'unique') + assert.equal(verification.actualInput?.resourceId, 'com.example:id/field'); + }); +} diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts index 21dd1bbfc1..e1c7a9bb3c 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-session-protocol.test.ts @@ -38,6 +38,7 @@ test('parses the session envelope and snapshot metadata', () => { maxDepth: undefined, maxNodes: undefined, rootPresent: undefined, + cacheInvalidated: undefined, truncated: undefined, elapsedMs: undefined, displayWidth: undefined, diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts b/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts index d39450342e..6086c3ef7a 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-session.fixtures.ts @@ -58,6 +58,7 @@ export type PersistentSnapshotHelperProviderOptions = { sessionResponseMode?: 'ok' | 'malformed'; sessionXml?: (sessionIndex: number, snapshotCount: number) => string; sessionRequests?: string[]; + sessionSnapshotMetadata?: (sessionIndex: number, snapshotCount: number) => Record; sessionInput?: (input: { kind: string; text?: string }) => void; sessionCommands?: string[]; sessionCommand?: (command: string) => Record | undefined; @@ -142,6 +143,7 @@ export function createPersistentSnapshotHelperProvider( nodeCount: '1', truncated: 'false', elapsedMs: '8', + ...options.sessionSnapshotMetadata?.(sessionIndex, snapshotCount), }, }), ); diff --git a/packages/platform-android/src/__tests__/snapshot-helper.test.ts b/packages/platform-android/src/__tests__/snapshot-helper.test.ts index 14b642e78f..cc6e8ff820 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper.test.ts @@ -56,6 +56,7 @@ test('parseAndroidSnapshotHelperOutput reconstructs XML chunks and metadata', () maxDepth: '128', maxNodes: '5000', rootPresent: 'true', + cacheInvalidated: 'true', captureMode: 'interactive-windows', windowCount: '2', nodeCount: '1', @@ -76,6 +77,7 @@ test('parseAndroidSnapshotHelperOutput reconstructs XML chunks and metadata', () maxDepth: 128, maxNodes: 5000, rootPresent: true, + cacheInvalidated: true, captureMode: 'interactive-windows', windowCount: 2, missingRootWindowTypes: undefined, diff --git a/packages/platform-android/src/__tests__/text-input-fill.test.ts b/packages/platform-android/src/__tests__/text-input-fill.test.ts index 2e285d8f9e..bbfffd23c2 100644 --- a/packages/platform-android/src/__tests__/text-input-fill.test.ts +++ b/packages/platform-android/src/__tests__/text-input-fill.test.ts @@ -633,11 +633,11 @@ test('readAndroidTextAtPointInHierarchy reads the EditText under the requested p assert.equal(readAndroidTextAtPointInHierarchy(hierarchy, 100, 250), 'fallback@example.com'); }); -test('fillAndroid runs the whole attempt on one warm daemon-session helper', async () => { +test('fillAndroid confirms a moved clear through one warm helper', async () => { const calls: (readonly string[])[] = []; const spawnArgs: (readonly string[])[] = []; const processes: FakeAndroidProcess[] = []; - let typed = ''; + let typed = 'before'; let captureCount = 0; const provider = createPersistentSnapshotHelperProvider({ calls, @@ -646,9 +646,18 @@ test('fillAndroid runs the whole attempt on one warm daemon-session helper', asy sessionInput: (input) => { if (input.kind === 'fill') typed = input.text ?? ''; }, + sessionSnapshotMetadata: () => ({ cacheInvalidated: 'true', missingRootWindowTypes: '' }), sessionXml: () => { captureCount += 1; - return androidInputXml({ text: typed }); + const xml = phoneInputXml(typed); + return captureCount === 1 + ? xml + : xml + .replace('[0,0][200,100]', '[400,0][600,100]') + .replace( + '', + '', + ); }, }); @@ -659,6 +668,7 @@ test('fillAndroid runs the whole attempt on one warm daemon-session helper', asy exec: async (args, options) => { if (isDeleteKey(args)) typed = ''; if (isTextInput(args)) typed = args[3] ?? ''; + if (args.includes('--show-versioncode')) return provider.exec(args, options); if (args[0] === 'shell' && args[1] !== 'am') return adbResult(''); return await provider.exec(args, options); }, @@ -666,13 +676,14 @@ test('fillAndroid runs the whole attempt on one warm daemon-session helper', asy }, { serial: ANDROID_EMULATOR.id }, async () => { - await fillAndroid(ANDROID_EMULATOR, 10, 10, 'chips', 0, { + await fillAndroid(ANDROID_EMULATOR, 10, 10, '', 0, { helperSessionScope: 'daemon-session', }); }, ); // One pre-action target read plus the two settling samples the text held for. + assert.equal(typed, ''); assert.equal(captureCount, 3); assert.equal(spawnArgs.length, 1); assert.equal(calls.filter(isAndroidHelperForwardRemoval).length, 0); diff --git a/packages/platform-android/src/app-lifecycle.ts b/packages/platform-android/src/app-lifecycle.ts index 7000caa0a4..901a9e6c65 100644 --- a/packages/platform-android/src/app-lifecycle.ts +++ b/packages/platform-android/src/app-lifecycle.ts @@ -222,6 +222,8 @@ async function startAndroidApp( if (options.url !== undefined) { return await openAndroidAppBoundDeepLink(device, app, options); } + const helperLaunch = await tryAndroidHelperLaunch(device, app, options); + if (helperLaunch) return helperLaunch; const resolved = await resolveAndroidApp(device, app); const launchCategory = resolveAndroidLauncherCategory(device); if (resolved.type === 'intent') { @@ -240,6 +242,41 @@ async function startAndroidApp( return await openAndroidPackage(device, resolved.value, launchCategory, options); } +async function tryAndroidHelperLaunch( + device: DeviceInfo, + app: string, + options: OpenAndroidAppOptions, +): Promise { + const packageName = app.trim(); + if ( + device.target === 'tv' || + options.activity !== undefined || + androidLaunchArgs(options).length > 0 || + options.terminateRunningApp || + !/^[A-Za-z][A-Za-z0-9_]*(\.[A-Za-z][A-Za-z0-9_]*)+$/.test(packageName) + ) + return undefined; + const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); + const result = await runAndroidAutomationHelper(device, 'app', { + kind: 'open', + packageName, + }); + if (!result) return undefined; + if ( + result.componentUnavailable === 'true' && + !(await isAndroidPackageInstalled(device, packageName)) + ) + throw buildAndroidPackageNotInstalledError(packageName); + const stdout = Buffer.from(result.launchOutputBase64 ?? '', 'base64').toString('utf8'); + if (!/^Status: ok\r?$/m.test(stdout)) { + throw new AppError('COMMAND_FAILED', 'Android helper could not launch the application', { + packageName, + stdout, + }); + } + return { stdout, stderr: '', exitCode: 0 }; +} + /** `am start -W` names the activity this launch resumed on its `Activity: /` line. */ function launchedAndroidPackage(result: AndroidAdbExecutorResult): string | undefined { const line = result.stdout.split('\n').find((entry) => entry.trim().startsWith('Activity:')); diff --git a/packages/platform-android/src/automation-helper-app-session.test.ts b/packages/platform-android/src/automation-helper-app-session.test.ts new file mode 100644 index 0000000000..d6c0505f72 --- /dev/null +++ b/packages/platform-android/src/automation-helper-app-session.test.ts @@ -0,0 +1,81 @@ +import { afterEach, expect, test } from 'vitest'; +import { once } from 'node:events'; +import { openAndroidApp } from './app-lifecycle.ts'; +import { withAndroidAdbProvider } from './adb-executor.ts'; +import { readAndroidFillTargetBeforeMutation } from './fill-verification.ts'; +import { resetAndroidSnapshotHelperSessions } from './snapshot-helper-session-lifecycle.ts'; +import { + createPersistentSnapshotHelperProvider, + type FakeAndroidProcess, +} from './__tests__/snapshot-helper-session.fixtures.ts'; +import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT as artifact } from './__tests__/test-utils/android-snapshot-helper.ts'; +import { ANDROID_EMULATOR as device } from './__tests__/test-utils/device-fixtures.ts'; +afterEach(resetAndroidSnapshotHelperSessions); + +test.each(['advertised', 'unsupported', 'ended', 'non-native-name'] as const)( + 'app open through an %s session uses exactly one launch transport', + async (mode) => { + const calls: (readonly string[])[] = [], + requests: string[] = [], + processes: FakeAndroidProcess[] = []; + const pkg = mode === 'non-native-name' ? '_com.example.app' : 'com.example.app'; + const provider = createPersistentSnapshotHelperProvider({ + calls, + processes, + spawnArgs: [], + sessionRequests: requests, + sessionCommands: + mode === 'unsupported' ? ['snapshot-immediate'] : ['snapshot-immediate', 'app'], + sessionCommand: (command) => + command.startsWith('app ') + ? { + launchOutputBase64: Buffer.from( + `Status: ok\nActivity: ${pkg}/.MainActivity\n`, + ).toString('base64'), + } + : undefined, + }); + const exec = provider.exec; + await withAndroidAdbProvider( + { + ...provider, + snapshotHelperArtifact: artifact, + exec: async (args, options) => { + if (args[0] === 'shell' && args[1] === 'am' && args[2] === 'start') { + calls.push(args); + return { + stdout: `Status: ok\nActivity: ${pkg}/.MainActivity\n`, + stderr: '', + exitCode: 0, + }; + } + return await exec(args, options); + }, + }, + { serial: device.id }, + async () => { + await readAndroidFillTargetBeforeMutation(device, 10, 10, { + helperSessionScope: 'daemon-session', + }); + if (mode === 'ended') { + const exited = once(processes[0]!, 'close'); + processes[0]!.kill(); + await exited; + } + await expect(openAndroidApp(device, ` ${pkg} `)).resolves.toBe(pkg); + }, + ); + const appRequests = requests.filter((request) => request.startsWith('app ')); + expect(appRequests).toHaveLength(mode === 'advertised' ? 1 : 0); + expect( + calls.filter((args) => args[0] === 'shell' && args[1] === 'am' && args[2] === 'start'), + ).toHaveLength(mode === 'advertised' ? 0 : 1); + if (mode === 'advertised') + expect(JSON.parse(Buffer.from(appRequests[0]!.split(' ')[2]!, 'base64').toString())).toEqual({ + kind: 'open', + packageName: pkg, + }); + if (mode === 'ended') + expect(calls.some((args) => args[0] === 'forward' && args[1] === '--remove')).toBe(true); + }, +); diff --git a/packages/platform-android/src/automation-helper-app.test.ts b/packages/platform-android/src/automation-helper-app.test.ts new file mode 100644 index 0000000000..283a4f9d6b --- /dev/null +++ b/packages/platform-android/src/automation-helper-app.test.ts @@ -0,0 +1,68 @@ +import { beforeEach, expect, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { withAndroidAdbProvider } from './adb-executor.ts'; +import { openAndroidApp } from './app-lifecycle.ts'; +import { runAndroidAutomationHelper } from './automation-helper.ts'; +import { ANDROID_EMULATOR } from './__tests__/test-utils/device-fixtures.ts'; +import './__tests__/test-utils/android-host-test-setup.ts'; + +vi.mock('./automation-helper.ts', () => ({ runAndroidAutomationHelper: vi.fn() })); +const helper = vi.mocked(runAndroidAutomationHelper); +const hostShell = vi.fn(async (): Promise<{ stdout: string; stderr: string; exitCode: number }> => { + throw new Error('Unexpected host shell after helper admission'); +}); +const pkg = 'com.example.app'; + +beforeEach(() => { + helper.mockReset(); + hostShell.mockReset(); + hostShell.mockRejectedValue(new Error('Unexpected host shell after helper admission')); +}); + +async function isolated(run: () => Promise): Promise { + return await withAndroidAdbProvider({ exec: hostShell }, { serial: ANDROID_EMULATOR.id }, run); +} + +test('a warm app open uses the helper launch result and preserves launched package identity', async () => { + helper.mockResolvedValue({ + launchOutputBase64: Buffer.from( + 'Status: ok\nActivity: com.example.app/.MainActivity\n', + ).toString('base64'), + }); + await expect(isolated(() => openAndroidApp(ANDROID_EMULATOR, pkg))).resolves.toBe(pkg); + expect(helper).toHaveBeenCalledWith(ANDROID_EMULATOR, 'app', { kind: 'open', packageName: pkg }); + expect(hostShell).not.toHaveBeenCalled(); +}); + +test('a helper error after possible application mutation is returned without a shell resend', async () => { + const failure = new AppError('COMMAND_FAILED', 'Helper transport lost after dispatch'); + helper.mockRejectedValue(failure); + await expect(isolated(() => openAndroidApp(ANDROID_EMULATOR, pkg))).rejects.toBe(failure); + expect(hostShell).not.toHaveBeenCalled(); +}); + +test('an unsuccessful helper launch is a failure and is never retried as a host launch', async () => { + helper.mockResolvedValue({ + launchOutputBase64: Buffer.from('Status: timeout\n').toString('base64'), + }); + await expect(isolated(() => openAndroidApp(ANDROID_EMULATOR, pkg))).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + }); + expect(hostShell).not.toHaveBeenCalled(); +}); + +test.each([true, false])( + 'an unavailable helper component distinguishes installed=%s without resending a launch', + async (installed) => { + helper.mockResolvedValue({ componentUnavailable: 'true' }); + hostShell.mockResolvedValue({ + stdout: installed ? 'package:/data/app/base.apk\n' : '', + stderr: '', + exitCode: installed ? 0 : 1, + }); + await expect(isolated(() => openAndroidApp(ANDROID_EMULATOR, pkg))).rejects.toMatchObject({ + code: installed ? 'COMMAND_FAILED' : 'APP_NOT_INSTALLED', + }); + expect(hostShell).toHaveBeenCalledTimes(1); + }, +); diff --git a/packages/platform-android/src/automation-helper-fallback.test.ts b/packages/platform-android/src/automation-helper-fallback.test.ts index 7bbc5ff057..73ea363f5a 100644 --- a/packages/platform-android/src/automation-helper-fallback.test.ts +++ b/packages/platform-android/src/automation-helper-fallback.test.ts @@ -71,7 +71,9 @@ test('a validated no-injection refusal completes fill through adb without a help const f = fixture({ ...refused, injected: 'false' }); await withAndroidAdbProvider(f.provider, { serial: device.id }, async () => { await fillAndroid(device, 10, 10, 'chips', 0, helper); - expect((await readAndroidFillTargetBeforeMutation(device, 10, 10, helper))?.text).toBe('chips'); + expect( + (await readAndroidFillTargetBeforeMutation(device, 10, 10, helper))?.targetInput?.text, + ).toBe('chips'); }); expect(f.calls.filter((args) => args[0] === 'shell' && args[2] === 'text')).toHaveLength(1); expect(f.requests.filter((request) => request.startsWith('input '))).toHaveLength(1); diff --git a/packages/platform-android/src/automation-helper.ts b/packages/platform-android/src/automation-helper.ts index bf6aa93f4f..c8d2fcc0de 100644 --- a/packages/platform-android/src/automation-helper.ts +++ b/packages/platform-android/src/automation-helper.ts @@ -15,10 +15,12 @@ import { export type AndroidHelperInput = | { kind: 'tap'; x: number; y: number } | { kind: 'key'; keyCode: number } - | { kind: 'fill'; x: number; y: number; text: string }; + | { kind: 'fill'; x: number; y: number; text: string } + | { kind: 'open'; packageName: string }; const ACTION_POLICY = { input: { mutation: true, timeoutMs: 30_000 }, + app: { mutation: true, timeoutMs: 30_000 }, screenshot: { mutation: false, timeoutMs: 30_000 }, focus: { mutation: false, timeoutMs: 2_000 }, } as const; diff --git a/packages/platform-android/src/fill-verification-empty-target.test.ts b/packages/platform-android/src/fill-verification-empty-target.test.ts new file mode 100644 index 0000000000..96fd04b7b0 --- /dev/null +++ b/packages/platform-android/src/fill-verification-empty-target.test.ts @@ -0,0 +1,147 @@ +import { expect, test } from 'vitest'; +import { verifyAndroidFilledTextInHierarchy } from './fill-verification.ts'; +import type { AndroidFillVerificationNode } from './fill-diagnostics.ts'; +import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; +const metadata: AndroidSnapshotBackendMetadata = { + backend: 'android-helper', + rootPresent: true, + cacheInvalidated: true, + helperTruncated: false, + systemSurfaceOnly: false, + missingRootWindowTypes: [], +}; +const target: AndroidFillVerificationNode = { + text: 'G', + className: 'android.widget.EditText', + resourceId: 'app:id/target', + packageName: 'app', + rect: { x: 0, y: 0, width: 100, height: 40 }, + focused: true, + password: false, + inputMethodOwned: false, + area: 4000, + hintShowing: false, + placeholder: null, +}; +function field(id: string, text: string, x: number, focused: boolean, pkg = 'app') { + return ``; +} +function verify( + nodes: string, + before: AndroidFillVerificationNode = target, + captureMetadata: AndroidSnapshotBackendMetadata | undefined = metadata, +) { + return verifyAndroidFilledTextInHierarchy( + `${nodes}`, + 50, + 20, + '', + { captureMetadata }, + { targetInput: before, uniqueIdentity: true }, + ); +} +test('does not accept a different empty focused field instead of the requested target', () => { + expect( + verify(field('app:id/target', 'G', 0, false) + field('app:id/other', '', 200, true)).ok, + ).toBe(false); +}); +test('does not recover an empty target with a duplicated identity', () => { + expect( + verify( + field('app:id/decoy', 'D', 0, false) + + field('app:id/target', '', 200, true) + + field('app:id/target', '', 350, false), + ).ok, + ).toBe(false); +}); +test('does not recover an empty target from another package', () => { + expect( + verify(field('app:id/decoy', 'D', 0, false) + field('app:id/target', '', 200, true, 'other')) + .ok, + ).toBe(false); +}); +test('does not recover an empty field that no longer has focus', () => { + expect( + verify(field('app:id/decoy', 'D', 0, false) + field('app:id/target', '', 200, false)).ok, + ).toBe(false); +}); + +test('does not recover an empty field with the same ID but a different class', () => { + expect( + verify( + field('app:id/decoy', 'D', 0, false) + + field('app:id/target', '', 200, true).replace('android.widget.EditText', 'custom.EditText'), + ).ok, + ).toBe(false); +}); +test('does not accept whitespace as a cleared value', () => { + expect( + verify(field('app:id/decoy', 'D', 0, false) + field('app:id/target', ' ', 200, true)).ok, + ).toBe(false); +}); +test.each(['composing', ''])('does not override a competing focused IME input (%s)', (imeText) => { + expect( + verify( + field('app:id/decoy', 'D', 0, false) + + field('app:id/target', '', 200, true) + + field('keyboard:id/input', imeText, 350, true, 'com.google.android.inputmethod.latin'), + ).ok, + ).toBe(false); +}); +test('requires the captured target identity before confirming a moved empty field', () => { + const xml = + '' + + field('app:id/decoy', 'D', 0, false) + + field('app:id/target', '', 200, true) + + ''; + expect(verifyAndroidFilledTextInHierarchy(xml, 50, 20, '').ok).toBe(false); +}); + +test('does not confirm a moved empty field without an identified pre-action target', () => { + const nodes = field('app:id/decoy', 'D', 0, false) + field('app:id/target', '', 200, true); + expect(verify(nodes, { ...target, resourceId: null }).ok).toBe(false); +}); +test('retains masked-field verification instead of corroborating an empty password field by identity', () => { + const nodes = field('app:id/decoy', 'D', 0, false) + field('app:id/target', '', 200, true); + expect(verify(nodes, { ...target, password: true }).ok).toBe(false); +}); + +test.each([ + ['unknown capture', {}], + ['truncated tree', { helperTruncated: true }], + ['unknown truncation', { helperTruncated: undefined }], + ['uninvalidated cache', { cacheInvalidated: false }], + ['unknown cache state', { cacheInvalidated: undefined }], + ['missing root', { rootPresent: false }], + ['system-only capture', { systemSurfaceOnly: true }], + ['missing window root', { missingRootWindowTypes: [2] }], + ['unknown window completeness', { missingRootWindowTypes: undefined }], +] satisfies [string, Partial][])( + 'does not confirm a moved empty field from %s', + (name, patch) => { + const nodes = field('app:id/decoy', 'D', 0, false) + field('app:id/target', '', 200, true); + const captureMetadata = + name === 'unknown capture' + ? { backend: 'android-helper' as const } + : { ...metadata, ...patch }; + expect(verify(nodes, target, captureMetadata).ok).toBe(false); + }, +); + +test('does not corroborate a moved empty field without a unique pre-action identity', () => { + const xml = + '' + + field('app:id/decoy', 'D', 0, false) + + field('app:id/target', '', 200, true) + + ''; + expect( + verifyAndroidFilledTextInHierarchy( + xml, + 50, + 20, + '', + { captureMetadata: metadata }, + { targetInput: target, uniqueIdentity: false }, + ).ok, + ).toBe(false); +}); diff --git a/packages/platform-android/src/fill-verification.ts b/packages/platform-android/src/fill-verification.ts index 3f7cdb0f3a..07ee722e95 100644 --- a/packages/platform-android/src/fill-verification.ts +++ b/packages/platform-android/src/fill-verification.ts @@ -12,7 +12,8 @@ import { } from './fill-diagnostics.ts'; import { getAndroidKeyboardState } from './device-input-state.ts'; import { isAndroidInputMethodOwnedNode } from '@agent-device/contracts/android-input-ownership'; -import { captureAndroidUiHierarchyXml } from './snapshot.ts'; +import { captureAndroidUiHierarchy, captureAndroidUiHierarchyXml } from './snapshot.ts'; +import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; import { ANDROID_WINDOW_TYPE_INPUT_METHOD } from './ui-hierarchy-node.ts'; @@ -39,7 +40,13 @@ type AndroidTextAtPointScan = { anyAtPoint: AndroidFillVerificationCandidate | null; }; +type AndroidFillTargetCapture = { + targetInput: AndroidFillVerificationNode | null; + uniqueIdentity: boolean; +}; + type AndroidFillVerificationContext = { + captureMetadata?: AndroidSnapshotBackendMetadata; activeInputMethodPackage?: string | null; inputMethodPackages?: ReadonlySet; }; @@ -59,21 +66,33 @@ export async function verifyAndroidFilledText( y: number, expected: string, helper: AndroidHelperSessionOptions = {}, + beforeCapture?: AndroidFillTargetCapture | null, ): Promise { const deadline = Deadline.fromTimeoutMs(FILL_VERIFICATION_DEADLINE_MS); - let initialXml: string | undefined = await captureAndroidUiHierarchyXml(device, { - ...helper, - skipIdleWait: true, - }); - const context = await readAndroidFillVerificationContext(device, initialXml); + let initialCapture: Awaited> | undefined = + await captureAndroidUiHierarchy(device, { + ...helper, + skipIdleWait: true, + }); + const context = await readAndroidFillVerificationContext(device, initialCapture.xml); const { value } = await pollUntilSettled({ probe: async () => { - if (initialXml !== undefined) { - const xml = initialXml; - initialXml = undefined; - return verifyAndroidFilledTextInHierarchy(xml, x, y, expected, context); + if (initialCapture) { + const capture = initialCapture; + initialCapture = undefined; + return verifyAndroidFilledTextInHierarchy( + capture.xml, + x, + y, + expected, + { + ...context, + captureMetadata: capture.metadata, + }, + beforeCapture, + ); } - return await inspectAndroidFilledText(device, x, y, expected, context, helper); + return await inspectAndroidFilledText(device, x, y, expected, context, helper, beforeCapture); }, settled: (verification, previous) => verification.reason === 'ime_capture' || (verification.ok && previous?.ok === true), @@ -102,10 +121,16 @@ async function readAndroidFillTargetAtPoint( x: number, y: number, helper: AndroidHelperSessionOptions, -): Promise { - const xml = await captureAndroidUiHierarchyXml(device, { ...helper, skipIdleWait: true }); - const context = await readAndroidFillVerificationContext(device, xml); - return inspectAndroidTextAtPointInHierarchy(xml, x, y, context).targetInput; +): Promise { + const capture = await captureAndroidUiHierarchy(device, { ...helper, skipIdleWait: true }); + const context = await readAndroidFillVerificationContext(device, capture.xml); + const targetInput = inspectAndroidTextAtPointInHierarchy(capture.xml, x, y, context).targetInput; + return { + targetInput, + uniqueIdentity: + completeFreshAndroidFillCapture(capture.metadata) && + uniqueAndroidFillTarget([...androidUiNodes(capture.xml)], targetInput), + }; } export async function readAndroidFillTargetBeforeMutation( @@ -113,7 +138,7 @@ export async function readAndroidFillTargetBeforeMutation( x: number, y: number, helper: AndroidHelperSessionOptions = {}, -): Promise { +): Promise { try { return await readAndroidFillTargetAtPoint(device, x, y, helper); } catch (error) { @@ -203,8 +228,19 @@ export function verifyAndroidFilledTextInHierarchy( y: number, expected: string, context: AndroidFillVerificationContext = {}, + beforeCapture?: AndroidFillTargetCapture | null, ): AndroidFillVerification { - const inspection = inspectAndroidTextAtPointInHierarchy(xml, x, y, context); + const inspection = inspectAndroidTextAtPointInHierarchy( + xml, + x, + y, + context, + expected === '' && + beforeCapture?.uniqueIdentity && + completeFreshAndroidFillCapture(context.captureMetadata) + ? beforeCapture.targetInput + : undefined, + ); if (isAndroidImeCapture(inspection)) { return { ok: false, @@ -283,15 +319,41 @@ async function inspectAndroidFilledText( expected: string, context: AndroidFillVerificationContext, helper: AndroidHelperSessionOptions, + beforeCapture?: AndroidFillTargetCapture | null, ): Promise { // Each delay samples the live hierarchy again — settling is what the samples observe, so they // share the helper session but never a capture. + const capture = await captureAndroidUiHierarchy(device, { ...helper, skipIdleWait: true }); return verifyAndroidFilledTextInHierarchy( - await captureAndroidUiHierarchyXml(device, { ...helper, skipIdleWait: true }), + capture.xml, x, y, expected, - context, + { ...context, captureMetadata: capture.metadata }, + beforeCapture, + ); +} + +function completeFreshAndroidFillCapture(metadata?: AndroidSnapshotBackendMetadata): boolean { + return ( + metadata?.rootPresent === true && + metadata.cacheInvalidated === true && + metadata.helperTruncated === false && + metadata.systemSurfaceOnly !== true && + metadata.missingRootWindowTypes?.length === 0 + ); +} + +function uniqueAndroidFillTarget( + nodes: readonly AndroidUiNodeMetadata[], + target: AndroidFillVerificationNode | null, +): boolean { + return ( + !!target?.resourceId && + !!target.packageName && + nodes.filter( + (node) => node.resourceId === target.resourceId && node.packageName === target.packageName, + ).length === 1 ); } @@ -300,6 +362,7 @@ function inspectAndroidTextAtPointInHierarchy( x: number, y: number, context: AndroidFillVerificationContext = {}, + beforeTarget?: AndroidFillVerificationNode | null, ): AndroidTextAtPointInspection { const scan: AndroidTextAtPointScan = { focusedEdit: null, @@ -309,12 +372,44 @@ function inspectAndroidTextAtPointInHierarchy( const nodes = [...androidUiNodes(xml)]; const observedContext = androidFillWindowContext(nodes) ?? context; - for (const node of nodes) { - const candidate = androidFillCandidateFromNode(node, observedContext); - if (candidate) updateAndroidTextAtPointScan(scan, candidate, x, y); + const candidates = nodes + .map((node) => androidFillCandidateFromNode(node, observedContext)) + .filter((candidate) => candidate !== null); + for (const candidate of candidates) { + updateAndroidTextAtPointScan(scan, candidate, x, y); } + const field = identifiedEmptyAndroidFillTarget(nodes, candidates, beforeTarget); + return field ? { targetInput: field, actualInput: field } : androidTextAtPointInspection(scan); +} - return androidTextAtPointInspection(scan); +function identifiedEmptyAndroidFillTarget( + nodes: readonly AndroidUiNodeMetadata[], + candidates: readonly AndroidFillVerificationCandidate[], + beforeTarget?: AndroidFillVerificationNode | null, +): AndroidFillVerificationCandidate | null { + if ( + !beforeTarget || + isSensitiveFillDiagnosticNode(beforeTarget) || + beforeTarget.inputMethodOwned || + !uniqueAndroidFillTarget(nodes, beforeTarget) + ) { + return null; + } + const field = candidates.find( + (candidate) => candidate.editText && sameAndroidFillTarget(beforeTarget, candidate), + ); + const focusedEdits = candidates.filter((candidate) => candidate.editText && candidate.focused); + if ( + field && + focusedEdits.length === 1 && + field.focused && + field.className === beforeTarget.className && + !field.inputMethodOwned && + !isSensitiveFillDiagnosticNode(field) && + observedAndroidValue(field) === '' + ) + return field; + return null; } function isAndroidImeCapture(inspection: AndroidTextAtPointInspection): boolean { diff --git a/packages/platform-android/src/lifecycle.test.ts b/packages/platform-android/src/lifecycle.test.ts index 0abd087eb9..a7602680ab 100644 --- a/packages/platform-android/src/lifecycle.test.ts +++ b/packages/platform-android/src/lifecycle.test.ts @@ -19,6 +19,10 @@ vi.mock('./app-lifecycle.ts', async (importOriginal) => ({ killAndroidApp: vi.fn(async () => {}), })); +vi.mock('./automation-helper.ts', () => ({ + runAndroidAutomationHelper: vi.fn(async () => undefined), +})); + const mockKillAndroidApp = vi.mocked(killAndroidApp); const device: DeviceInfo = { diff --git a/packages/platform-android/src/snapshot-helper-capture.ts b/packages/platform-android/src/snapshot-helper-capture.ts index 7443fbc083..97ad5558df 100644 --- a/packages/platform-android/src/snapshot-helper-capture.ts +++ b/packages/platform-android/src/snapshot-helper-capture.ts @@ -376,6 +376,7 @@ function readHelperMetadata(finalResult: Record): AndroidSnapsho maxDepth: readOptionalNumber(finalResult.maxDepth), maxNodes: readOptionalNumber(finalResult.maxNodes), rootPresent: readOptionalBoolean(finalResult.rootPresent), + cacheInvalidated: readOptionalBoolean(finalResult.cacheInvalidated), captureMode: readOptionalCaptureMode(finalResult.captureMode), windowCount: readOptionalNumber(finalResult.windowCount), missingRootWindowTypes: readInstrumentationResultIntegerList( diff --git a/packages/platform-android/src/snapshot-helper-session-protocol.ts b/packages/platform-android/src/snapshot-helper-session-protocol.ts index fe3d436714..c40ee6595f 100644 --- a/packages/platform-android/src/snapshot-helper-session-protocol.ts +++ b/packages/platform-android/src/snapshot-helper-session-protocol.ts @@ -289,6 +289,7 @@ function readSessionMetadata(headers: Record): AndroidSnapshotHe maxDepth: readInstrumentationResultNumber(headers.maxDepth), maxNodes: readInstrumentationResultNumber(headers.maxNodes), rootPresent: readInstrumentationResultBoolean(headers.rootPresent), + cacheInvalidated: readInstrumentationResultBoolean(headers.cacheInvalidated), captureMode: headers.captureMode === 'interactive-windows' || headers.captureMode === 'active-window' ? headers.captureMode diff --git a/packages/platform-android/src/snapshot-helper-session.ts b/packages/platform-android/src/snapshot-helper-session.ts index ce64b0825a..99fabb8866 100644 --- a/packages/platform-android/src/snapshot-helper-session.ts +++ b/packages/platform-android/src/snapshot-helper-session.ts @@ -97,7 +97,13 @@ async function captureFromAndroidSnapshotHelperSession(params: { // (a second instrumentation for the helper package would force-stop the session that has it). They // never start a session: without one, callers use the same helper APK through a one-shot // `am instrument` run instead. -export type AndroidTouchHelperAction = 'gesture' | 'viewport' | 'input' | 'screenshot' | 'focus'; +export type AndroidTouchHelperAction = + | 'gesture' + | 'viewport' + | 'input' + | 'screenshot' + | 'focus' + | 'app'; /** * Gestures and input commands mutate the app, so their failures disclose whether input reached it. A diff --git a/packages/platform-android/src/snapshot-helper-types.ts b/packages/platform-android/src/snapshot-helper-types.ts index 70346dbd57..945fd87869 100644 --- a/packages/platform-android/src/snapshot-helper-types.ts +++ b/packages/platform-android/src/snapshot-helper-types.ts @@ -90,6 +90,8 @@ export type AndroidSnapshotHelperMetadata = { maxDepth?: number; maxNodes?: number; rootPresent?: boolean; + /** This capture cleared the platform accessibility node cache through UiAutomation.clearCache(). */ + cacheInvalidated?: boolean; captureMode?: AndroidSnapshotCaptureMode; windowCount?: number; /** diff --git a/packages/platform-android/src/snapshot-types.ts b/packages/platform-android/src/snapshot-types.ts index 33dada3395..5ca8274e74 100644 --- a/packages/platform-android/src/snapshot-types.ts +++ b/packages/platform-android/src/snapshot-types.ts @@ -38,6 +38,7 @@ export type AndroidSnapshotBackendMetadata = { maxDepth?: number; maxNodes?: number; rootPresent?: boolean; + cacheInvalidated?: boolean; captureMode?: AndroidSnapshotCaptureMode; systemSurfaceOnly?: boolean; windowCount?: number; diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index c107161876..95645e6a7a 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -119,16 +119,14 @@ export async function captureAndroidUiHierarchyXml( device: DeviceInfo, options: AndroidSnapshotOptions = {}, ): Promise { - const adb = resolveAndroidAdbProvider(device, options.helperAdb).exec; - return (await captureAndroidUiHierarchy(device, options, adb)).xml; + return (await captureAndroidUiHierarchy(device, options)).xml; } export async function snapshotAndroid( device: DeviceInfo, options: AndroidSnapshotOptions = {}, ): Promise { - const adb = resolveAndroidAdbProvider(device, options.helperAdb).exec; - const capture = await captureAndroidUiHierarchy(device, options, adb); + const capture = await captureAndroidUiHierarchy(device, options); // The one place Android answers the viewport question (#3182): the helper's own display read, // through the shared guard, so both the healthy and the presentation-failed capture below publish // the same box the tree was measured against, and a display the helper could not address stays @@ -281,10 +279,9 @@ function applyHiddenContentHintsToInteractiveSnapshot(params: { ); } -async function captureAndroidUiHierarchy( +export async function captureAndroidUiHierarchy( device: DeviceInfo, - options: AndroidSnapshotOptions, - adb: AndroidAdbExecutor, + options: AndroidSnapshotOptions = {}, ): Promise { const adbProvider = resolveAndroidAdbProvider(device, options.helperAdb); const helper = await withDiagnosticTimer( @@ -295,7 +292,12 @@ async function captureAndroidUiHierarchy( ), ); if (helper.artifact) { - return await captureAndroidUiHierarchyWithHelper(device, options, adb, helper.artifact); + return await captureAndroidUiHierarchyWithHelper( + device, + options, + adbProvider.exec, + helper.artifact, + ); } emitDiagnostic({ @@ -517,6 +519,7 @@ function formatAndroidHelperCaptureResult( maxDepth: capture.metadata.maxDepth, maxNodes: capture.metadata.maxNodes, rootPresent: capture.metadata.rootPresent, + cacheInvalidated: capture.metadata.cacheInvalidated, captureMode: capture.metadata.captureMode, windowCount: capture.metadata.windowCount, missingRootWindowTypes: capture.metadata.missingRootWindowTypes, diff --git a/packages/platform-android/src/text-input.ts b/packages/platform-android/src/text-input.ts index 5386ae6268..b88a2fd909 100644 --- a/packages/platform-android/src/text-input.ts +++ b/packages/platform-android/src/text-input.ts @@ -76,7 +76,8 @@ export async function fillAndroid( delayMs = 0, helper: AndroidHelperSessionOptions = {}, ): Promise { - const beforeTarget = await readAndroidFillTargetBeforeMutation(device, x, y, helper); + const beforeCapture = await readAndroidFillTargetBeforeMutation(device, x, y, helper); + const beforeTarget = beforeCapture?.targetInput ?? null; const providerText = resolveAndroidTextInjector(device); if (providerText) { await providerText({ action: 'fill', target: { x, y }, text, delayMs }); @@ -90,7 +91,7 @@ export async function fillAndroid( (await runAndroidAutomationHelper(device, 'input', { kind: 'fill', x, y, text })) ) { emitAndroidTextDiagnostic('fill', 'android-helper', text); - const verification = await verifyAndroidFilledText(device, x, y, text, helper); + const verification = await verifyAndroidFilledText(device, x, y, text, helper, beforeCapture); return completeAndroidFillVerification(text, beforeTarget, verification); } let lastVerification: AndroidFillVerification | null = null; diff --git a/src/__tests__/agent-device-client-target-readiness.test.ts b/src/__tests__/agent-device-client-target-readiness.test.ts new file mode 100644 index 0000000000..a34082adff --- /dev/null +++ b/src/__tests__/agent-device-client-target-readiness.test.ts @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createAgentDeviceClient } from '../agent-device-client.ts'; +import { createTransport } from './client-transport-fixture.ts'; + +test.each([undefined, { press: 'bounded-geometry' }, { press: 'unknown' }])( + 'the SDK preserves only recognized daemon target-readiness claims: %j', + async (targetReadiness) => { + const setup = createTransport(() => ({ + ok: true, + data: { + device: { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + target: 'mobile', + kind: 'emulator', + booted: true, + }, + availableCommands: ['press'], + targetReadiness, + }, + })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + const result = await client.devices.capabilities(); + assert.deepEqual( + result.targetReadiness, + targetReadiness?.press === 'bounded-geometry' ? { press: 'bounded-geometry' } : undefined, + ); + }, +); diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index 8c7df8b941..37c4e4a849 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -222,6 +222,9 @@ export function createAgentDeviceClient( return { device: normalizeDevice(data.device), availableCommands, + ...(asOptionalRecord(data.targetReadiness)?.press === 'bounded-geometry' + ? { targetReadiness: { press: 'bounded-geometry' as const } } + : {}), }; }, boot: async (options = {}) => await executeCommand>('boot', options), diff --git a/src/commands/interaction/runtime/moving-target-gate.test.ts b/src/commands/interaction/runtime/moving-target-gate.test.ts index 7375487d34..6bf2d01d8d 100644 --- a/src/commands/interaction/runtime/moving-target-gate.test.ts +++ b/src/commands/interaction/runtime/moving-target-gate.test.ts @@ -1,4 +1,5 @@ import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; import { test } from 'vitest'; import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; import type { PreresolvedInteractionTarget } from '@agent-device/contracts/interaction'; @@ -45,6 +46,7 @@ function slideTree(x: number, over?: { label?: string; surfaceBundleId?: string depth: 1, parentIndex: 0, type: 'Button', + bundleId: 'com.example.app', label: over?.label ?? 'Slide target', rect: { x, y: BUTTON_Y, width: 120, height: 50 }, hittable: true, @@ -200,7 +202,7 @@ test('the @ref path takes its own observations, because its resolution spends no }); assert.deepEqual(await device.press('ref'), [centerOf(RESTING_X)]); - assert.equal(device.captures(), 2, 'both re-observations, none for the first resolution'); + assert.equal(device.captures(), 3, 'fresh observations span the target quiet window'); }); test('a raw-coordinate press has no resolved element to re-observe', async () => { @@ -370,11 +372,7 @@ test('a target newly arrived on a different surface is re-observed, not certifie }); assert.deepEqual(await device.press('selector'), [centerOf(RESTING_X)]); - assert.equal( - device.captures(), - 3, - 'resolution plus both re-observations, not the screening alone', - ); + assert.equal(device.captures(), 4, 'resolution plus observations across the quiet window'); }); test('a replacement that took the ref at the frame coordinates is refused, not agreed', async () => { @@ -434,11 +432,7 @@ test('the refused selector press publishes a wait that survives the shell', asyn const device = createGuardedDevice({ stored: slideTree(MID_FLIGHT_X, { label }), // Still moving at the end of the window: the two re-observations disagree. - trees: [ - slideTree(MID_FLIGHT_X, { label }), - slideTree(MID_FLIGHT_X + 40, { label }), - slideTree(MID_FLIGHT_X + 80, { label }), - ], + trees: Array.from({ length: 40 }, (_, index) => slideTree(120 + (index % 2) * 40, { label })), baselineAgeMs: 100, baselineTree: slideTree(-400, { label }), }); @@ -560,3 +554,82 @@ test('a clock skew ahead of the mutation does not produce a negative wait', () = assert.ok(decision.kind === 'recheck'); assert.ok(decision.waitMs >= MOVING_TARGET_GATE_SAMPLE_GAP_MS); }); + +test('Android admits the same uniquely identified target after another node reindexes its observation', async () => { + const frame = slideTree(MID_FLIGHT_X); + frame.nodes[1]!.identifier = 'stable-target'; + const fresh = slideTree(RESTING_X); + fresh.nodes[1]!.identifier = 'stable-target'; + fresh.nodes[1]!.index = 2; + fresh.nodes[1]!.ref = 'e3'; + fresh.nodes.splice(1, 0, { + ref: 'e2', + index: 1, + type: 'Text', + label: 'New status', + parentIndex: 0, + depth: 1, + }); + const device = createGuardedDevice({ + stored: frame, + refFrame: frame, + trees: [fresh], + ...DISTURBING_HISTORY, + }); + assert.deepEqual(await device.press('ref'), [centerOf(RESTING_X)]); +}); + +test('Android refuses to certify an identity duplicated in the new observation', async () => { + const frame = slideTree(MID_FLIGHT_X); + frame.nodes[1]!.identifier = 'stable-target'; + const fresh = slideTree(RESTING_X); + fresh.nodes[1]!.identifier = 'stable-target'; + fresh.nodes.push({ + ...fresh.nodes[1]!, + index: 2, + ref: 'e3', + rect: { x: 160, y: BUTTON_Y, width: 120, height: 50 }, + }); + const device = createGuardedDevice({ + stored: frame, + refFrame: frame, + trees: [fresh], + ...DISTURBING_HISTORY, + }); + await assert.rejects( + () => device.press('ref'), + (error: unknown) => { + assert.equal( + (error as AppError).details?.reason, + INTERACTION_ERROR_REASONS.targetStillMoving, + ); + return true; + }, + ); + assert.deepEqual(device.taps(), []); +}); + +test('Android refuses to adopt matching identity from another application', async () => { + const frame = slideTree(MID_FLIGHT_X); + frame.nodes[1]!.identifier = 'stable-target'; + const fresh = slideTree(RESTING_X); + fresh.nodes[1]!.identifier = 'stable-target'; + fresh.nodes[1]!.bundleId = 'com.example.other'; + const device = createGuardedDevice({ + stored: frame, + refFrame: frame, + trees: [fresh], + ...DISTURBING_HISTORY, + }); + await assert.rejects( + () => device.press('ref'), + (error: unknown) => { + assert.equal( + (error as AppError).details?.reason, + INTERACTION_ERROR_REASONS.targetStillMoving, + ); + return true; + }, + ); + assert.deepEqual(device.taps(), []); +}); diff --git a/src/commands/interaction/runtime/moving-target-gate.ts b/src/commands/interaction/runtime/moving-target-gate.ts index 48ce52b637..865a2a2039 100644 --- a/src/commands/interaction/runtime/moving-target-gate.ts +++ b/src/commands/interaction/runtime/moving-target-gate.ts @@ -28,7 +28,7 @@ import type { ResolveInteractionTargetParams } from './interaction-resolution-re * photograph of a surface that may still be rearranging itself, and the ADB-backed Android helper is * measured reporting frames in flight — so an interaction issued right after an action that moved * the UI can resolve a point the target has already left and report success on the vacancy. The exposure is the runtime path's own capture-to-dispatch - * distance, so the gate is backend-neutral. + * distance. Android requires a bounded quiet interval; other platforms retain the full observation window. * * Two judgements decide when it is real: {@link movingTargetGateDecision} asks whether this position * is still owed a second observation, and kernel's `areCapturedRectsAtSamePosition` asks whether the @@ -38,7 +38,9 @@ import type { ResolveInteractionTargetParams } from './interaction-resolution-re * anchored on the mutation's END, so the gate waits out only what is LEFT of it — which is also why * the refusal's hint names `--settle` as the caller-opted superset. * - * This is the seam that spends the window, in front of the dispatch and behind every pre-dispatch + * Android uses the window as a deadline and returns after a measured 100 ms quiet position. + * Other backends retain the end-of-window recheck. + * This seam runs in front of the dispatch and behind every pre-dispatch * guard that already judged the target (occlusion, off-screen, keyboard), so it re-decides none of * them: it takes the second and third observations, and dispatches the requested interaction at the * position two of them agree on. @@ -69,11 +71,15 @@ export async function guardResolvedTargetAgainstMotion( if (params.expectedResolvedTarget || params.preresolvedTarget) return resolved; const gate = await readMovingTargetGate(runtime, options, resolved.node, resolved.preAction); if (!gate) return resolved; - return await resolveAtRestTarget(runtime, options, params, resolved, gate); + return runtime.backend.platform === 'android' + ? await resolveAndroidAtRestTarget(runtime, options, params, resolved, gate) + : await resolveAtRestTarget(runtime, options, params, resolved, gate); } /** How long after a mutation its target positions stay owed a second observation. */ export const MOVING_TARGET_GATE_WINDOW_MS = 500; +const ANDROID_MOVING_TARGET_WINDOW_MS = 1_500; +const ANDROID_TARGET_QUIET_MS = 100; /** * How far apart the two observations must be for agreeing to mean "at rest". Below one capture @@ -104,14 +110,16 @@ export function movingTargetGateDecision(params: { /** Whether the action's own pre-action tree already shows the resolved node at this rect. */ targetShownAtRest: boolean; nowMs: number; + windowMs?: number; }): MovingTargetGateDecision { if (params.baselineFinishedAtMs === undefined) return { kind: 'idle' }; const ageMs = params.nowMs - params.baselineFinishedAtMs; - if (ageMs >= MOVING_TARGET_GATE_WINDOW_MS) return { kind: 'not-recent' }; + const windowMs = params.windowMs ?? MOVING_TARGET_GATE_WINDOW_MS; + if (ageMs >= windowMs) return { kind: 'not-recent' }; if (params.targetShownAtRest) return { kind: 'stationary' }; return { kind: 'recheck', - waitMs: Math.max(MOVING_TARGET_GATE_WINDOW_MS - ageMs, MOVING_TARGET_GATE_SAMPLE_GAP_MS), + waitMs: Math.max(windowMs - ageMs, MOVING_TARGET_GATE_SAMPLE_GAP_MS), }; } @@ -135,6 +143,9 @@ async function readMovingTargetGate( baselineFinishedAtMs: baseline.finishedAt, targetShownAtRest: baselineShowsTargetAtRest(baseline, resolvedNode, resolvedSurface), nowMs: now(runtime), + ...(runtime.backend.platform === 'android' + ? { windowMs: ANDROID_MOVING_TARGET_WINDOW_MS } + : {}), }); return decision.kind === 'recheck' ? decision : undefined; } @@ -212,6 +223,31 @@ async function resolveAtRestTarget( throw motionRefusal(resolved, second.rect, now(runtime) - waitedFrom); } +async function resolveAndroidAtRestTarget( + runtime: AgentDeviceRuntime, + options: CommandContext & { target: InteractionTarget }, + params: ResolveInteractionTargetParams, + resolved: Exclude, + gate: MovingTargetRecheck, +): Promise { + const startedAt = now(runtime); + const deadline = startedAt + Math.max(gate.waitMs, ANDROID_TARGET_QUIET_MS); + let previous = await reObserveTarget(runtime, options, params); + if (!previous.measured) throw motionRefusal(resolved, undefined, now(runtime) - startedAt); + let quietSince = now(runtime); + let observations = 0; + while (observations < 2 || now(runtime) < deadline) { + await sleep(runtime, MOVING_TARGET_GATE_SAMPLE_GAP_MS); + const next = onSameSurface(previous, await reObserveTarget(runtime, options, params)); + observations++; + if (!next.measured) throw motionRefusal(resolved, undefined, now(runtime) - startedAt); + if (!areCapturedRectsAtSamePosition(previous.rect, next.rect)) quietSince = now(runtime); + if (now(runtime) - quietSince >= ANDROID_TARGET_QUIET_MS) return next.resolved; + previous = next; + } + throw motionRefusal(resolved, previous.rect, now(runtime) - startedAt); +} + /** * Rest is one position on one surface, which is why the screening refuses to compare positions * across surfaces at all (#2438). A look taken on another presentation — a system sheet arriving diff --git a/src/commands/interaction/runtime/ref-target-resolution.ts b/src/commands/interaction/runtime/ref-target-resolution.ts index d8a846b4b5..d14e2cd8af 100644 --- a/src/commands/interaction/runtime/ref-target-resolution.ts +++ b/src/commands/interaction/runtime/ref-target-resolution.ts @@ -171,6 +171,7 @@ async function resolveSnapshotForRef( // element. The caller re-observes (snapshot) or uses a selector. if (outcome.kind !== 'resolved') throw refMissRefusal(outcome, target.ref); return reconcileFreshObservation({ + platform: runtime.backend.platform, session, frameTree, target, @@ -190,6 +191,7 @@ async function resolveSnapshotForRef( * authorized frame node so a positional coincidence cannot retarget the action. */ function reconcileFreshObservation(params: { + platform: string; session: CommandSessionRecord; frameTree: SnapshotState; target: Extract; @@ -204,6 +206,8 @@ function reconcileFreshObservation(params: { if (observation === frameTree) { return { snapshot: frameTree, resolved: authorized, adoptedFrom: 'observation' }; } + const identified = reconcileAndroidIdentifiedObservation({ ...params, observation }); + if (identified) return identified; const observed = tryResolveRefNode(observation.nodes, target.ref, { fallbackLabel }); if ( observed.kind === 'resolved' && @@ -217,6 +221,41 @@ function reconcileFreshObservation(params: { return { snapshot: frameTree, resolved: authorized, adoptedFrom: 'authorized-frame' }; } +function reconcileAndroidIdentifiedObservation(params: { + platform: string; + frameTree: SnapshotState; + observation: SnapshotState; + authorized: ResolvedRefNode; +}): RefObservationReconciliation | undefined { + const { platform, frameTree, observation, authorized } = params; + if (platform !== 'android' || !authorized.node.identifier || !authorized.node.bundleId) + return undefined; + if ( + surfaceScopedNodes(frameTree).surfaceBundleId !== + surfaceScopedNodes(observation).surfaceBundleId + ) + return undefined; + const identity = readNodeLocalIdentity(authorized.node); + const matches = (node: SnapshotNode) => + localIdentitiesEqual(identity, readNodeLocalIdentity(node)); + if (frameTree.nodes.filter(matches).length !== 1) return undefined; + const candidates = observation.nodes.filter( + (node) => hasSnapshotRef(node) && node.bundleId === authorized.node.bundleId && matches(node), + ); + const node = candidates[0]; + if (candidates.length !== 1 || !node || !hasSnapshotRef(node)) + return { snapshot: frameTree, resolved: authorized, adoptedFrom: 'authorized-frame' }; + return { + snapshot: observation, + resolved: buildRefResolution( + authorized.ref, + node, + authorized.resolution.kind === 'label-fallback' ? 'label-fallback' : 'exact', + ), + adoptedFrom: 'observation', + }; +} + /** The runtime-ref resolver: `exact` for a resolved `@ref`, `label-fallback` for trailing-label recovery. */ /** * What one tree makes of a ref: the node it authorizes (exact, or the trailing-label recovery), a diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index a7b17254db..c1c2b438e5 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -236,7 +236,7 @@ agent-device capabilities --session checkout --json - `devices` lists available targets after applying `--platform` and [isolation scopes](#device-isolation-scopes). - Each device carries `model` (such as `iPhone 17 Pro` or `Pixel 9`) and `osVersion` (such as `26.1` or `16`) when platform tooling reports them: `simctl` for iOS simulators, `devicectl` or `xctrace` for physical iOS devices, and `ro.product.model` and `ro.build.version.release` for running Android devices. Android emulators report the system image model, such as `sdk_gphone64_arm64`. Stopped AVDs and other platforms omit both fields. They describe a device and are not selectors. -- `capabilities` lists the commands the session device, or an explicit `--platform`/`--device`/`--udid`/`--serial` target, supports. JSON output is `{ device, availableCommands }`; use `availableCommands` instead of keeping your own platform support table. +- `capabilities` lists the commands the session device, or an explicit `--platform`/`--device`/`--udid`/`--serial` target, supports. JSON output includes `device` and `availableCommands`; use `availableCommands` instead of keeping your own platform support table. Android also reports `targetReadiness.press: "bounded-geometry"` when element presses check moving targets before dispatch. Test harnesses can use that capability to delegate tap readiness instead of adding a fixed transition delay. It does not wait for the result of the tap. ### HarmonyOS command boundary