diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index fe7fcb190d..9852a26361 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -211,7 +211,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ min: 0, usageLabel: '--settle-quiet ', usageDescription: - 'Settle: quiet window the UI must hold to count as settled (default 500ms). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the --timeout settle deadline', + 'Settle: quiet window the UI must hold to count as settled (default 500ms; 300ms when the settled tree comes from the iOS simulator snapshot bridge). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the --timeout settle deadline', projectConfig: true, recorded: false, }, diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index e636d621a6..cc21f0a6d1 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -25,7 +25,8 @@ export type RepeatedPressOptions = { * Opt-in (#1101): after the action, wait for the UI to go quiet and return the * settled diff vs the pre-action tree (`settle` on the result) in the same * response. Best-effort — never fails the action. `settleQuietMs` tunes the - * quiet window (default 500ms); `timeoutMs` bounds the settle wait (default + * quiet window (default 500ms, or 300ms when the settled tree comes from the iOS + * simulator snapshot bridge); `timeoutMs` bounds the settle wait (default * 10s) when `settle` is true. A bare `timeoutMs` without `settle` is ignored * for compatibility; `settleQuietMs` still requires `settle`. */ diff --git a/src/__tests__/daemon-composition/generic-settle.test.ts b/src/__tests__/daemon-composition/generic-settle.test.ts index d7a3111ab3..3b948d3a6d 100644 --- a/src/__tests__/daemon-composition/generic-settle.test.ts +++ b/src/__tests__/daemon-composition/generic-settle.test.ts @@ -1,6 +1,7 @@ import { composedPlatformServicesFixture } from './platform-services-fixture.ts'; import { beforeEach, expect, test, vi } from 'vitest'; import type { CommandFlags } from '@agent-device/contracts/command'; +import type { CommandClock } from '@agent-device/contracts/runtime-contract'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { APP_SURFACE, @@ -113,18 +114,36 @@ async function emulateCaptureSnapshotForSession( return snapshot; } -function mockCommandDispatch(snapshots: Array) { +function mockCommandDispatch( + snapshots: Array, + producer: NonNullable = 'apple-runner', +) { let snapshotCalls = 0; mockDispatch.mockImplementation(async (command) => { if (command === 'snapshot') { const nodes = snapshots[Math.min(snapshotCalls, snapshots.length - 1)]; snapshotCalls += 1; - return { nodes, backend: 'xctest', producer: 'apple-runner' }; + return { nodes, backend: 'xctest', producer }; } return {}; }); } +/** + * A clock the settle loop advances instead of waiting. The default-window cases below assert the + * window a run RESOLVED, which needs no real elapsed time: the loop's own budget arithmetic runs on + * this clock, so the suite neither slows nor goes unstable under load. + */ +function fakeClock(stepMs = 300): CommandClock { + let elapsed = 0; + return { + now: () => elapsed, + sleep: async (ms: number) => { + elapsed += ms > 0 ? ms : stepMs; + }, + }; +} + const contextFromFlags = () => ({}) as never; /** @@ -161,13 +180,14 @@ function seedSession( type GenericSettleTree = { nodes: RawSnapshotNode[]; systemSurface?: IosSystemSurfaceProvenance; + producer?: 'apple-runner' | 'simulator-ax-bridge'; }; function snapshotPayload(tree: GenericSettleTree) { return { nodes: tree.nodes, backend: 'xctest' as const, - producer: 'apple-runner' as const, + producer: tree.producer ?? ('apple-runner' as const), ...(tree.systemSurface ? { systemSurface: tree.systemSurface } : {}), }; } @@ -205,6 +225,8 @@ async function dispatchGeneric(params: { runtimeBindings?: Readonly< Pick[0], 'inspectFacts' | 'bindDevice'> >; + /** Injects a clock into every command surface the route builds, so no test waits real time. */ + clock?: CommandClock; }): Promise { const req: DaemonRequest = { token: 't', @@ -214,7 +236,9 @@ async function dispatchGeneric(params: { ...(params.flags ? { flags: params.flags } : {}), }; return await dispatchGenericCommand({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: params.clock + ? async (config) => await testCreateCommandSurface({ ...config, clock: params.clock }) + : testCreateCommandSurface, platformServices: composedPlatformServicesFixture(), req, ref: params.sessionStore.lookup(params.sessionName)!, @@ -801,3 +825,80 @@ for (const change of ['rebuild', 'retire'] as const) { } }); } + +// The default quiet window keys on the capture the settle loop PERFORMS (#3470 follow-up), not on +// the baseline it diffs: on this route the baseline is the session's stored tree, which can be +// several commands stale and served by a different producer. All three run on an injected clock, so +// they assert the resolved window without real waiting. +test('scroll --settle without --settle-quiet uses the shorter default when the loop reads the simulator bridge', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-default-quiet'; + seedSession(sessionName, sessionStore, { + nodes: BEFORE_NODES, + producer: 'simulator-ax-bridge', + }); + mockCommandDispatch([AFTER_NODES], 'simulator-ax-bridge'); + + const response = await dispatchGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + // No settleQuietMs: the route's own default window is what is under test. + flags: { settle: true, timeoutMs: 2_000 }, + clock: fakeClock(), + }); + + const settle = expectOkData(response).settle as SettlePayload; + expect(settle.settled).toBe(true); + expect(settle.quietMs).toBe(300); +}); + +test('scroll --settle keeps the 500ms default when the loop reads the Apple runner', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-default-quiet-runner'; + seedSession(sessionName, sessionStore, { + nodes: BEFORE_NODES, + producer: 'apple-runner', + }); + mockCommandDispatch([AFTER_NODES], 'apple-runner'); + + const response = await dispatchGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { settle: true, timeoutMs: 2_000 }, + clock: fakeClock(), + }); + + const settle = expectOkData(response).settle as SettlePayload; + expect(settle.settled).toBe(true); + expect(settle.quietMs).toBe(500); +}); + +test('a stale bridge-sourced baseline does not shorten a settle the Apple runner performs', async () => { + // The mismatch the stored baseline can carry: the tree on file came from the bridge, but the + // captures that pay this window come from the runner, whose read the short window was never + // measured on. Keying on the baseline would hold 300ms against an unmeasured read. + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-default-quiet-mismatch'; + seedSession(sessionName, sessionStore, { + nodes: BEFORE_NODES, + producer: 'simulator-ax-bridge', + }); + mockCommandDispatch([AFTER_NODES], 'apple-runner'); + + const response = await dispatchGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { settle: true, timeoutMs: 2_000 }, + clock: fakeClock(), + }); + + const settle = expectOkData(response).settle as SettlePayload; + expect(settle.settled).toBe(true); + expect(settle.quietMs).toBe(500); +}); diff --git a/src/commands/interaction/runtime/selector-wait.ts b/src/commands/interaction/runtime/selector-wait.ts index aeb3b9c3db..4b81e00798 100644 --- a/src/commands/interaction/runtime/selector-wait.ts +++ b/src/commands/interaction/runtime/selector-wait.ts @@ -97,6 +97,7 @@ type StableCaptureResult = { waitedMs: number; captures: number; nodeCount: number; + failure?: { error: unknown }; }; export type SelectorWaitOperations = { diff --git a/src/commands/interaction/runtime/settle-default-quiet.test.ts b/src/commands/interaction/runtime/settle-default-quiet.test.ts new file mode 100644 index 0000000000..48fdc1d4e8 --- /dev/null +++ b/src/commands/interaction/runtime/settle-default-quiet.test.ts @@ -0,0 +1,184 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { SnapshotProvenance, SnapshotState } from '@agent-device/kernel/snapshot'; +import { ref } from './selector-read-utils.ts'; +import { + buttonSnapshot, + createSettleDevice, + welcomeSnapshot, +} from './__tests__/settle-device-fixtures.ts'; + +// The `--settle` DEFAULT quiet window keys on the capture the settle loop PERFORMS (#3470 +// follow-up), because that read is the one the window prices: the loop holds its quiet window open +// across captures of its own. The pre-action baseline cannot answer this — on the ref route it is +// the stored ref frame, on the generic route the session's stored tree, and either can be several +// commands stale and served by a producer other than the one settle then reads through. A caller +// that asks for a window with `--settle-quiet` always outranks the default. +// +// Budgets are injected (fake clock) — no real waiting. + +function withProvenance

(state: SnapshotState, provenance: P) { + return { ...state, ...provenance }; +} + +const BRIDGE = { backend: 'xctest', producer: 'simulator-ax-bridge' } as const; +const RUNNER = { backend: 'xctest', producer: 'apple-runner' } as const; + +type LoopCapturePath = SnapshotProvenance | undefined | ((capture: number) => SnapshotProvenance); + +/** + * A press whose baseline and whose loop captures come from declared capture paths, so the two can + * be made to agree or to disagree. The pressed button stays on screen, so no capture is a broad + * replacement and the window under test is the one the loop priced, not the confirmation floor. + */ +function pressWithCapturePaths(params: { + baseline?: SnapshotProvenance; + loop?: LoopCapturePath; + settle?: { quietMs?: number; timeoutMs?: number }; +}) { + const baseline = params.baseline + ? withProvenance(buttonSnapshot(), params.baseline) + : buttonSnapshot(); + let captures = 0; + const device = createSettleDevice({ + stored: baseline, + captureSnapshot: () => { + // A ref press captures nothing before the action, so every call is the loop's own sample. + captures += 1; + const provenance = typeof params.loop === 'function' ? params.loop(captures) : params.loop; + return { + snapshot: provenance ? withProvenance(buttonSnapshot(), provenance) : buttonSnapshot(), + }; + }, + }); + return device.interactions.press(ref('@e1'), { session: 'default', settle: params.settle ?? {} }); +} + +test('a settle the simulator snapshot bridge performs uses the shorter default window', async () => { + const result = await pressWithCapturePaths({ loop: BRIDGE }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, true); + assert.equal(settle.quietMs, 300); + assert.ok(settle.waitedMs < 500, `bridge settle held ${settle.waitedMs}ms`); +}); + +test('an explicit --settle-quiet outranks the per-capture default window', async () => { + const result = await pressWithCapturePaths({ loop: BRIDGE, settle: { quietMs: 700 } }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.quietMs, 700); +}); + +// Each of these is a read the 300 ms default was NOT measured on, so shortening its window would be +// an unmeasured change of behaviour rather than the measured one. The Apple runner shares the +// `xctest` CHANNEL with the bridge, which is why the rule keys on the producer. +const UNMEASURED_CAPTURES: Array<[string, SnapshotProvenance?]> = [ + ['the Apple runner tree read (same channel, different producer)', RUNNER], + ['an Appium-sourced tree on the same channel', { backend: 'xctest', producer: 'appium-source' }], + ['the Android helper', { backend: 'android', producer: 'android-uiautomator' }], + ['a capture that declares no provenance at all', undefined], +]; + +for (const [label, provenance] of UNMEASURED_CAPTURES) { + test(`${label} keeps the 500ms default window`, async () => { + const result = await pressWithCapturePaths({ loop: provenance }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, true); + assert.equal(settle.quietMs, 500); + assert.ok(settle.waitedMs >= 500, `${label} settled after ${settle.waitedMs}ms`); + }); +} + +test('a bridge read during settle earns the shorter window even from a runner-issued ref frame', async () => { + // The ref frame the action was authorized against came from the runner, so keying the window on + // the baseline would hold the unmeasured 500 ms against captures the bridge answered. + const result = await pressWithCapturePaths({ baseline: RUNNER, loop: BRIDGE }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.quietMs, 300); + assert.ok(settle.waitedMs < 500, `bridge settle held ${settle.waitedMs}ms`); +}); + +test('a runner read during settle keeps the 500ms window despite a bridge-issued baseline', async () => { + // The unsafe direction of the same mismatch: holding 300 ms for a read priced at 500 ms would + // settle on a window the measurement never covered. + const result = await pressWithCapturePaths({ baseline: BRIDGE, loop: RUNNER }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.quietMs, 500); + assert.ok(settle.waitedMs >= 500, `runner settle held only ${settle.waitedMs}ms`); +}); + +test('the capture that decides the verdict prices the whole quiet span, even after a producer switch', async () => { + // The quiet span is elapsed time over which the tree did not change, whichever producer read it. + // A run that starts on the runner and continues on the bridge settles on the bridge's window. + const result = await pressWithCapturePaths({ + loop: (capture) => (capture === 1 ? RUNNER : BRIDGE), + }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, true); + assert.equal(settle.quietMs, 300); + assert.ok(settle.waitedMs < 500, `producer switch held ${settle.waitedMs}ms`); +}); + +test('a capture that breaks mid-settle reports the window the run had priced', async () => { + let captures = 0; + const device = createSettleDevice({ + stored: buttonSnapshot(), + captureSnapshot: () => { + captures += 1; + if (captures === 1) return { snapshot: withProvenance(buttonSnapshot(), BRIDGE) }; + throw new Error('AX bridge crashed'); + }, + }); + + const result = await device.interactions.press(ref('@e1'), { session: 'default', settle: {} }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, false); + assert.match(settle.hint ?? '', /Settle observation unavailable \(AX bridge crashed\)/); + assert.equal(settle.quietMs, 300); +}); + +test('the shorter default window never shortens a broad-transition confirmation', async () => { + // The trap this layer owes layer 1: a 300 ms default sits under the 1.5 s floor, so a rule that + // armed on `requestedQuietMs >= DEFAULT_STABLE_QUIET_MS` would silently switch the #1831 + // confirmation off for every default settle the bridge answers. Here the pre-action tree is the + // tiny-baseline shape, so the confirmation owns the wait and the 300 ms default still holds its + // 1.5 s floor (#3470). + const sparseBaseline = withProvenance(buttonSnapshot(), BRIDGE); + let captures = 0; + const device = createSettleDevice({ + stored: sparseBaseline, + captureSnapshot: () => { + captures += 1; + return { + snapshot: captures === 1 ? sparseBaseline : withProvenance(welcomeSnapshot(), BRIDGE), + }; + }, + }); + + const result = await device.interactions.press(ref('@e1'), { + session: 'default', + settle: { timeoutMs: 5_000 }, + }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, true); + assert.equal(settle.quietMs, 300); + assert.ok( + settle.waitedMs >= 1_500, + `confirmation did not outlast the shorter default: waitedMs=${settle.waitedMs}`, + ); +}); diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index 8ccff098d1..8beefccf19 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -27,6 +27,7 @@ import { DEFAULT_STABLE_QUIET_MS, DEFAULT_STABLE_TIMEOUT_MS, runStableCaptureLoop, + type StableCaptureLoopResult, TINY_STABLE_TREE_HINT, TINY_STABLE_TREE_NODE_COUNT, } from './stable-capture.ts'; @@ -126,29 +127,51 @@ async function settleAfterAction( options: CommandContext, params: SettleParams & { baseline: SurfaceScopedNodes; actionPoint?: Point }, ): Promise { - const quietMs = params.quietMs ?? DEFAULT_STABLE_QUIET_MS; + // The loop resolves an unrequested window from the capture that pays it, so the window is only + // known once one is taken. `base` carries the general default for the pre-capture shape the + // observation still has to report; once the loop returns, its own window replaces it, including on + // a failure while the outcome is read. const timeoutMs = params.timeoutMs ?? DEFAULT_STABLE_TIMEOUT_MS; - const base: SettleObservation = { settled: false, waitedMs: 0, captures: 0, quietMs, timeoutMs }; + const base: SettleObservation = { + settled: false, + waitedMs: 0, + captures: 0, + quietMs: params.quietMs ?? DEFAULT_STABLE_QUIET_MS, + timeoutMs, + }; + let outcome: StableCaptureLoopResult | undefined; try { - const outcome = await runStableCaptureLoop(runtime, options, { - quietMs, + outcome = await runStableCaptureLoop(runtime, options, { + ...(params.quietMs === undefined ? {} : { quietMs: params.quietMs }), timeoutMs, resetBudgetOnPrivateAxRecovery: true, broadTransitionBaselineNodes: params.baseline.nodes, }); + if (outcome.failure) return settleUnavailable(base, outcome.quietMs, outcome.failure.error); return await readSettledOutcome(runtime, options, params, base, outcome); } catch (error) { - // Never fail the action over the observation: report that settling itself - // broke and let the caller fall back to an explicit snapshot. - return { - observation: { - ...base, - hint: `Settle observation unavailable (${error instanceof Error ? error.message : String(error)}). The action itself succeeded; take a snapshot to observe the result.`, - }, - }; + return settleUnavailable(base, outcome?.quietMs ?? base.quietMs, error); } } +/** + * Never fail the action over the observation: report that settling itself broke, with the window the + * run held, and let the caller fall back to an explicit snapshot. + */ +function settleUnavailable( + base: SettleObservation, + quietMs: number, + error: unknown, +): SettleOutcome { + return { + observation: { + ...base, + quietMs, + hint: `Settle observation unavailable (${error instanceof Error ? error.message : String(error)}). The action itself succeeded; take a snapshot to observe the result.`, + }, + }; +} + /** Turns a finished stable-capture loop into the settled observation payload. */ async function readSettledOutcome( runtime: AgentDeviceRuntime, @@ -162,6 +185,9 @@ async function readSettledOutcome( settled: outcome.settled, waitedMs: outcome.waitedMs, captures: outcome.captures, + // The window the loop actually ran. For an unrequested settle that is the one its own captures + // priced, which `base` cannot know because it is built before the first read. + quietMs: outcome.quietMs, }; if (!outcome.lastCapture) { return { diff --git a/src/commands/interaction/runtime/stable-capture.test.ts b/src/commands/interaction/runtime/stable-capture.test.ts index 7e3dbc05e4..31d2fceac2 100644 --- a/src/commands/interaction/runtime/stable-capture.test.ts +++ b/src/commands/interaction/runtime/stable-capture.test.ts @@ -326,6 +326,76 @@ test('settle confirms a tiny modal replacement through a partial post-action pro ); }); +test('a dismissal confirmation is not dropped by a later empty projection', async () => { + // A confirmation is a fact about the transition, so once armed it holds the floor for the rest of + // the run: later captures that cannot support the comparison (here, empty projections that match + // each other) never return the run to the caller's shorter window. + const { runtime } = transitionRuntime({ + settledAtMs: 0, + emptyProjectionAtCaptures: [2, 3], + }); + + const outcome = await runStableCaptureLoop( + runtime, + { session: 'default' }, + { + quietMs: 150, + timeoutMs: 5_000, + broadTransitionBaselineNodes: elementThreadsNoticeSnapshot.nodes, + }, + ); + + assert.equal(outcome.settled, true); + assert.ok( + outcome.waitedMs >= 1_500, + `empty projection disarmed the confirmation: settled after ${outcome.waitedMs}ms`, + ); + assert.ok( + outcome.lastCapture!.snapshot.nodes.length > 0, + 'settled on an empty projection instead of the dismissed screen', + ); +}); + +// The two arming inputs that belong to the capture, not to the run: the viewport-root completeness +// and the declared capture backend. A dismissal of anything bigger than an alert or sheet (the +// `<= TINY_STABLE_TREE_NODE_COUNT` escape hatch) has to meet BOTH before the confirmation may arm, +// and a mid-animation first read that misses either one leaves the question open for the next +// capture rather than settling the run on its plain window. These captures declare no producer, so +// that plain window is the 500 ms default; the 1.5 s floor is what the assertions tell apart. +const MID_ANIMATION_FIRST_CAPTURES: Array< + [string, NonNullable[0]>] +> = [ + ['a rootless first capture', { rootlessUntilMs: 400 }], + ['an empty first capture', { emptyProjectionAtCaptures: [1] }], + ['a first capture read through another channel', { otherChannelAtCaptures: [1] }], +]; +for (const [label, midAnimation] of MID_ANIMATION_FIRST_CAPTURES) { + test(`settle arms the confirmation on the first rooted XCTest capture after ${label}`, async () => { + const { runtime } = transitionRuntime({ settledAtMs: 0, ...midAnimation }); + + const outcome = await runStableCaptureLoop( + runtime, + { session: 'default' }, + { + timeoutMs: 5_000, + // A full-screen settings table: the tiny-presented-baseline escape hatch does not apply, so + // arming depends entirely on the captures below. + broadTransitionBaselineNodes: elementSettingsSnapshot([1_138, 1_423, 2_144, 2_434]).nodes, + }, + ); + + assert.equal(outcome.settled, true); + assert.equal( + outcome.lastCapture?.snapshot.nodes.some((node) => node.label === 'action file'), + false, + ); + assert.ok( + outcome.waitedMs >= 1_500, + `${label} ruled the confirmation out for the run: settled after ${outcome.waitedMs}ms`, + ); + }); +} + test('settle keeps the default quiet window for a larger partial projection', async () => { const { runtime } = transitionRuntime({ omitViewportRoot: true, settledAtMs: 1_200 }); @@ -458,6 +528,12 @@ function transitionRuntime( omitViewportRoot?: boolean; settledAtMs?: number; sessionSnapshot?: SnapshotState; + /** 1-based capture indices that answer with an empty projection (mid-animation read). */ + emptyProjectionAtCaptures?: number[]; + /** Captures taken before this deadline omit the viewport root (mid-animation read). */ + rootlessUntilMs?: number; + /** 1-based capture indices that declare a non-XCTest capture channel. */ + otherChannelAtCaptures?: number[]; } = {}, ) { const captureBackend = options.captureBackend ?? 'tree'; @@ -476,17 +552,36 @@ function transitionRuntime( captureSnapshot: async () => { const keepsBaseline = options.firstCaptureKeepsBaseline === true && captures === 0; captures += 1; + if (options.emptyProjectionAtCaptures?.includes(captures)) { + return { + snapshot: withCaptureBackend( + { ...elementSettledRoomSnapshot, nodes: [] }, + captureBackend, + ), + }; + } const snapshot = keepsBaseline ? elementThreadsNoticeSnapshot : withCaptureBackend( elapsedMs < settledAtMs ? elementTransientRoomSnapshot : elementSettledRoomSnapshot, captureBackend, ); + const rootless = + options.omitViewportRoot === true || + (options.rootlessUntilMs !== undefined && elapsedMs < options.rootlessUntilMs); + const declaresOtherChannel = options.otherChannelAtCaptures?.includes(captures) === true; + const rootlessSnapshot = rootless + ? { ...snapshot, nodes: snapshot.nodes.slice(1) } + : snapshot; return { snapshot: withoutSnapshotBackend( - options.omitViewportRoot === true - ? { ...snapshot, nodes: snapshot.nodes.slice(1) } - : snapshot, + declaresOtherChannel + ? { + ...rootlessSnapshot, + backend: 'android' as const, + producer: 'android-uiautomator' as const, + } + : rootlessSnapshot, options.omitSnapshotBackend === true, ), }; diff --git a/src/commands/interaction/runtime/stable-capture.ts b/src/commands/interaction/runtime/stable-capture.ts index ce39bcf2a7..05a4bac9f1 100644 --- a/src/commands/interaction/runtime/stable-capture.ts +++ b/src/commands/interaction/runtime/stable-capture.ts @@ -1,7 +1,9 @@ import type { SnapshotNode, SnapshotPreferredBackend, + SnapshotProvenance, SnapshotQualityVerdict, + SnapshotState, } from '@agent-device/kernel/snapshot'; import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; @@ -34,8 +36,7 @@ import { withoutSettleChrome } from '@agent-device/capture-kit/snapshot-chrome'; const STABLE_POLL_INTERVAL_MS = 300; const STABLE_MIN_POLL_MS = 25; // Never slower than the default poll, never faster than the minimum: a tight window gets a cadence -// that can measure it, a window the confirmation extends gets one that does not sample the whole -// extended wait at the caller's original grid. +// that can measure it, and a window the confirmation extends is not sampled on the caller's grid. function stableCapturePollMs(quietMs: number): number { return Math.min(STABLE_POLL_INTERVAL_MS, Math.max(STABLE_MIN_POLL_MS, quietMs)); } @@ -47,14 +48,35 @@ const QUIET_DEADLINE_EPSILON_MS = 2; // Broad replacements (for example, dismissing a modal into a room) can expose a coherent but // transitional AX tree for the default quiet window. Confirm those transitions longer without // adding latency to local mutations whose pre/post trees substantially overlap. -// -// The confirmation is decided by the transition alone: a caller asking for a shorter quiet window -// asked for a tighter verdict, not for a different set of transitions to distrust. It used to arm -// only at `requestedQuietMs >= DEFAULT_STABLE_QUIET_MS`, which made `--settle-quiet 499` behave -// differently from 500 and left `settleQuietMs: 150` callers (tester-army/e2e) with no guard at all -// (#3470). A confirmed replacement is held for the caller's own window, floored at 1.5 s. const BROAD_TRANSITION_CONFIRMATION_QUIET_MS = 1_500; export const DEFAULT_STABLE_QUIET_MS = 500; +/** + * The default quiet window each producer's capture earns. The iOS simulator snapshot bridge answers + * well inside the general default, so holding the general window on that read spends a whole extra + * capture on a window its tree had already satisfied. Keyed on the producer, not the channel: the + * bridge and the Apple runner share the `xctest` channel and read at different costs, and only the + * bridge carries the measurement. Every producer states its price here, so a new one cannot inherit + * an unmeasured window by omission. (PR #3522 holds the interleaved before/after medians and the + * late-diff accuracy check.) + */ +const DEFAULT_QUIET_MS_BY_PRODUCER = { + 'apple-runner': DEFAULT_STABLE_QUIET_MS, + 'simulator-ax-bridge': 300, + 'appium-source': DEFAULT_STABLE_QUIET_MS, + 'limrun-ios-tree': DEFAULT_STABLE_QUIET_MS, + 'android-uiautomator': DEFAULT_STABLE_QUIET_MS, + 'harmonyos-uitest': DEFAULT_STABLE_QUIET_MS, + 'macos-helper': DEFAULT_STABLE_QUIET_MS, + 'linux-atspi': DEFAULT_STABLE_QUIET_MS, + 'agent-browser': DEFAULT_STABLE_QUIET_MS, +} as const satisfies Record; + +/** A capture that declares no producer has no measured price and keeps the general default. */ +function defaultQuietMsForCapture(snapshot: SnapshotState): number { + return snapshot.producer === undefined + ? DEFAULT_STABLE_QUIET_MS + : DEFAULT_QUIET_MS_BY_PRODUCER[snapshot.producer]; +} export const DEFAULT_STABLE_TIMEOUT_MS = 10_000; // Below this node count a settled tree is suspicious: real app surfaces have // more than a handful of accessibility nodes, splash/loading screens do not. @@ -72,178 +94,364 @@ export type StableCaptureLoopResult = { waitedMs: number; captures: number; nodeCount: number; + /** + * The quiet window the run was holding when it returned: the caller's, or the default resolved + * from the last capture that paid it. A run that completed no capture never priced a window, so + * it reports the general default it started from. + */ + quietMs: number; /** The most recent completed capture, settled or not. */ lastCapture?: CapturedSnapshot; + /** A capture threw; the run ended there, without a verdict. */ + failure?: { error: unknown }; }; export async function runStableCaptureLoop( runtime: AgentDeviceRuntime, options: CommandContext & SelectorSnapshotOptions, params: { - quietMs: number; + /** + * The quiet window the caller asked for. Absent means the loop resolves its own default from + * the capture it performs, because that capture is the read the window prices + * ({@link defaultQuietMsForCapture}); a caller with its own default passes it explicitly. + */ + quietMs?: number; timeoutMs: number; resetBudgetOnPrivateAxRecovery?: boolean; /** Immutable pre-action projection; the stored session may advance before settle begins. */ broadTransitionBaselineNodes?: SnapshotNode[]; }, ): Promise { - const { quietMs, timeoutMs } = params; + const { timeoutMs } = params; const start = now(runtime); let deadlineMs = start + timeoutMs; let privateAxRecoveryBudgetReset = false; const session = await runtime.sessions.get(options.session ?? 'default'); - let transitionBaseline: StableCaptureSignal | undefined; + // The quiet window the run holds. An unrequested window is priced by the latest capture the loop + // performs, never by the baseline's producer, and applies to the whole quiet span so far: the + // capture that decides the verdict is the read the window prices. A producer switch mid-wait can + // therefore shorten the window; a confirmed broad replacement never drops below its own floor. + // A window the caller asked for is fixed. + let quietMs = params.quietMs ?? DEFAULT_STABLE_QUIET_MS; + const confirmation = broadTransitionConfirmation({ + baselineNodes: params.broadTransitionBaselineNodes, + platform: runtime.backend.platform, + appBundleId: session?.appBundleId, + }); let preferredBackend = preferredSnapshotBackendForVerdict(session?.snapshot?.snapshotQuality); - // Cadence derives from the quiet window in force (never slower than the default - // poll): a caller asking for a 50ms quiet window should not be forced onto a - // 300ms grid, and a run the confirmation has extended to 1.5s should not spend - // that wait on the caller's tight grid — every sample is a capture round trip. - // Tests inject the budget instead of waiting real time. - let pollMs = stableCapturePollMs(quietMs); + const pacing = createStableCapturePacing(start); let captures = 0; - let lastSignal: StableCaptureSignal | undefined; - let lastNodeCount = 0; let lastCapture: CapturedSnapshot | undefined; - let quietSinceMs = start; - let requiredQuietMs = quietMs; + const result = ( + settled: boolean, + stalled: boolean, + waitedMs: number, + ): StableCaptureLoopResult => ({ + settled, + stalled, + waitedMs, + captures, + nodeCount: pacing.lastNodeCount, + quietMs, + lastCapture, + }); + const unfinished = (read: Exclude) => ({ + ...result(false, read.kind === 'stalled', now(runtime) - start), + ...(read.kind === 'failed' ? { failure: { error: read.error } } : {}), + }); while (now(runtime) < deadlineMs) { - const capture = await captureStableSignalWithinDeadline( + const read = await captureStableSignalWithinDeadline( runtime, options, deadlineMs - now(runtime), preferredBackend, ); - if (!capture) { - return { - settled: false, - stalled: true, - waitedMs: now(runtime) - start, - captures, - nodeCount: lastNodeCount, - lastCapture, - }; - } + if (read.kind !== 'captured') return unfinished(read); + const { capture } = read; captures += 1; lastCapture = capture; - transitionBaseline ??= stableCaptureTransitionBaseline( - params.broadTransitionBaselineNodes, - capture.snapshot, - runtime.backend.platform, - session?.appBundleId, - ); + const window = stableCaptureWindow(capture.snapshot, params.quietMs, confirmation); + quietMs = window.quietMs; + const { requiredQuietMs, pollMs } = window; const signal = stableCaptureSignal(capture.snapshot); - requiredQuietMs = stableCaptureNextQuietWindowMs({ - baseline: transitionBaseline, - snapshot: capture.snapshot, - appBundleId: session?.appBundleId, - requestedQuietMs: quietMs, - currentQuietMs: requiredQuietMs, - }); - // Recomputed from the window in force rather than only when it changed: the window is a - // monotonic max, so this is the same value, and the cadence cannot drift from it. - pollMs = stableCapturePollMs(requiredQuietMs); preferredBackend ??= preferredSnapshotBackendForVerdict(capture.snapshot.snapshotQuality); const nowMs = now(runtime); - const recoveredDeadlineMs = extendedDeadlineAfterPrivateAxRecovery({ - resetRequested: params.resetBudgetOnPrivateAxRecovery, + const nodeCount = capture.snapshot.nodes.length; + const verdict = capture.snapshot.snapshotQuality; + const recovered = recoverStableCaptureBudget({ + pacing, + resetBudgetOnPrivateAxRecovery: params.resetBudgetOnPrivateAxRecovery, alreadyReset: privateAxRecoveryBudgetReset, - verdict: capture.snapshot.snapshotQuality, - nowMs, - timeoutMs, - deadlineMs, - }); - if (recoveredDeadlineMs !== undefined) { - privateAxRecoveryBudgetReset = true; - deadlineMs = recoveredDeadlineMs; - quietSinceMs = nowMs; - lastSignal = signal; - lastNodeCount = capture.snapshot.nodes.length; - const recoveryDelayMs = stableCaptureDelayMs({ + state: { + deadlineMs, + timeoutMs, nowMs, - quietSinceMs, - quietMs: requiredQuietMs, + verdict, + signal, + nodeCount, + requiredQuietMs, pollMs, - deadlineMs, - }); - if (recoveryDelayMs <= 0) break; - await sleep(runtime, recoveryDelayMs); - continue; + }, + }); + if (recovered) { + privateAxRecoveryBudgetReset = true; + deadlineMs = recovered.deadlineMs; } - if (!stableCaptureSignalsEqual(lastSignal, signal)) { - lastSignal = signal; - lastNodeCount = capture.snapshot.nodes.length; - quietSinceMs = nowMs; - } else if (captures >= 2 && nowMs - quietSinceMs >= requiredQuietMs) { + const step = recovered + ? { settled: false, delayMs: recovered.delayMs } + : pacing.observe({ signal, nodeCount, nowMs, requiredQuietMs, pollMs, deadlineMs, captures }); + if (step.settled) return result(true, false, nowMs - start); + if (step.delayMs <= 0) break; + await sleep(runtime, step.delayMs); + } + return result(false, false, now(runtime) - start); +} + +/** + * The quiet window one capture prices, the window it must actually hold once the broad-transition + * confirmation has had its say, and the cadence that can measure that. Sampling an extended wait on + * the caller's tight grid would spend the whole extension in capture round trips. + */ +function stableCaptureWindow( + snapshot: SnapshotState, + requestedQuietMs: number | undefined, + confirmation: StableCaptureConfirmation, +): { quietMs: number; requiredQuietMs: number; pollMs: number } { + const quietMs = requestedQuietMs ?? defaultQuietMsForCapture(snapshot); + const requiredQuietMs = confirmation.requiredQuietMs(snapshot, quietMs); + return { quietMs, requiredQuietMs, pollMs: stableCapturePollMs(requiredQuietMs) }; +} + +/** + * The once-per-run budget extension a private-AX recovery buys. The recovery re-read the same + * surface through another backend, so the quiet window restarts on that capture rather than carrying + * suspicion across the switch. Undefined when the capture asked for no reset (or the run already + * spent its one). + */ +function recoverStableCaptureBudget(params: { + pacing: StableCapturePacing; + alreadyReset: boolean; + resetBudgetOnPrivateAxRecovery?: boolean; + state: { + deadlineMs: number; + timeoutMs: number; + nowMs: number; + verdict: SnapshotQualityVerdict | undefined; + signal: StableCaptureSignal; + nodeCount: number; + requiredQuietMs: number; + pollMs: number; + }; +}): { deadlineMs: number; delayMs: number } | undefined { + const { state } = params; + const extendedDeadlineMs = extendedDeadlineAfterPrivateAxRecovery({ + resetRequested: params.resetBudgetOnPrivateAxRecovery, + alreadyReset: params.alreadyReset, + verdict: state.verdict, + nowMs: state.nowMs, + timeoutMs: state.timeoutMs, + deadlineMs: state.deadlineMs, + }); + if (extendedDeadlineMs === undefined) return undefined; + params.pacing.restart({ + signal: state.signal, + nodeCount: state.nodeCount, + nowMs: state.nowMs, + }); + return { + deadlineMs: extendedDeadlineMs, + delayMs: params.pacing.delayMs({ + nowMs: state.nowMs, + requiredQuietMs: state.requiredQuietMs, + pollMs: state.pollMs, + deadlineMs: extendedDeadlineMs, + }), + }; +} + +type StableCapturePacing = ReturnType; + +/** + * The settle loop's per-capture bookkeeping: the last capture's shape, when the UI last changed, and + * the quiet-window arithmetic that decides whether a capture settles the run. Owns that state so the + * loop above stays the read of one capture at a time. + */ +function createStableCapturePacing(start: number) { + let lastSignal: StableCaptureSignal | undefined; + let lastNodeCount = 0; + let quietSinceMs = start; + return { + /** Node count of the most recent capture, reported on every result shape. */ + get lastNodeCount(): number { + return lastNodeCount; + }, + /** Folds one capture in: settled, or how long to wait before the next one (<= 0 ends the run). */ + observe(params: { + signal: StableCaptureSignal; + nodeCount: number; + nowMs: number; + requiredQuietMs: number; + pollMs: number; + deadlineMs: number; + captures: number; + }): { settled: boolean; delayMs: number } { + if (!stableCaptureSignalsEqual(lastSignal, params.signal)) { + // The UI is still moving: the quiet window restarts from this capture. + lastSignal = params.signal; + lastNodeCount = params.nodeCount; + quietSinceMs = params.nowMs; + } else if (params.captures >= 2 && params.nowMs - quietSinceMs >= params.requiredQuietMs) { + return { settled: true, delayMs: 0 }; + } return { - settled: true, - stalled: false, - waitedMs: nowMs - start, - captures, - nodeCount: lastNodeCount, - lastCapture, + settled: false, + delayMs: stableCaptureDelayMs({ + nowMs: params.nowMs, + quietSinceMs, + quietMs: params.requiredQuietMs, + pollMs: params.pollMs, + deadlineMs: params.deadlineMs, + }), }; - } - const delayMs = stableCaptureDelayMs({ - nowMs, - quietSinceMs, - quietMs: requiredQuietMs, - pollMs, - deadlineMs, - }); - if (delayMs <= 0) break; - await sleep(runtime, delayMs); - } + }, + /** Adopts a capture as the new start of the quiet window without deciding anything. */ + restart(params: { signal: StableCaptureSignal; nodeCount: number; nowMs: number }): void { + lastSignal = params.signal; + lastNodeCount = params.nodeCount; + quietSinceMs = params.nowMs; + }, + /** The wait before the next capture, given the window currently in force. */ + delayMs(params: { + nowMs: number; + requiredQuietMs: number; + pollMs: number; + deadlineMs: number; + }): number { + return stableCaptureDelayMs({ + nowMs: params.nowMs, + quietSinceMs, + quietMs: params.requiredQuietMs, + pollMs: params.pollMs, + deadlineMs: params.deadlineMs, + }); + }, + }; +} + +/** + * The broad-transition confirmation, as one decision the settle loop asks about each capture. It + * compares each capture against the pre-action tree until one proves the replacement, and holds that + * window for the rest of the run. + * + * Two gates stand before the comparison. The pre-action projection is constant for the run and the + * only input allowed to rule the confirmation out: a caller that passed none has nothing to confirm + * against. The capture gate — the capture proves an iOS XCTest read and carries a complete viewport + * projection — belongs to each capture, because the first read after a dismissal can still show the + * dismissed surface, arrive rootless or empty mid-animation, or come from another backend than the + * next poll will. A capture that fails it leaves the question open; the first capture that passes + * fixes the comparison baseline, and every later capture is compared against that baseline until one + * proves the replacement. + * + * Arming never consults the requested quiet window: `--settle-quiet` asks for a tighter verdict, not + * for a different set of transitions to distrust. A confirmed replacement is held for at least the + * caller's own window, floored at 1.5 s. + */ +function broadTransitionConfirmation(params: { + baselineNodes: SnapshotNode[] | undefined; + platform: AgentDeviceRuntime['backend']['platform']; + appBundleId: string | undefined; +}): StableCaptureConfirmation { + const { baselineNodes, platform, appBundleId } = params; + // Open question, not a ruling: `undefined` is the state a capture that cannot support the + // comparison leaves behind, so arming never depends on which capture happens to land first. + let transitionBaseline: StableCaptureSignal | undefined; + // The comparison refuses a node-less side outright, so a mid-animation empty read must not be able + // to cancel the floor: without the latch it hands the loop its short window back, and the next + // empty read matches it, settling a dismissal that owed the floor. + let confirmedQuietMs: number | undefined; return { - settled: false, - stalled: false, - waitedMs: now(runtime) - start, - captures, - nodeCount: lastNodeCount, - lastCapture, + // While unconfirmed, do not memoize the verdict on `stableCaptureSignalsEqual`: signal identity + // omits `role`, parent links, and >1px-free geometry, all of which the chrome classifier reads, + // and one pass costs tens of microseconds against the 300 ms poll cadence. + requiredQuietMs(snapshot, currentQuietMs) { + if (transitionBaseline === undefined && baselineNodes) { + transitionBaseline = broadTransitionBaseline({ + baselineNodes, + capture: snapshot, + platform, + appBundleId, + }); + } + if (!transitionBaseline) return currentQuietMs; + if (confirmedQuietMs === undefined) { + const candidate = stableCaptureComparisonSignal(snapshot, snapshot.nodes, appBundleId); + if (stableCaptureSignalsHaveBroadReplacement(transitionBaseline, candidate)) { + confirmedQuietMs = Math.max(currentQuietMs, BROAD_TRANSITION_CONFIRMATION_QUIET_MS); + } + } + // Still maxed against the current window: the loop can price a wider one after the latch, on a + // mid-run fallback from the bridge to the runner. + return confirmedQuietMs === undefined + ? currentQuietMs + : Math.max(currentQuietMs, confirmedQuietMs); + }, }; } -function stableCaptureTransitionBaseline( - baselineNodes: SnapshotNode[] | undefined, - snapshot: Parameters[0] | undefined, - platform: AgentDeviceRuntime['backend']['platform'], - appBundleId: string | undefined, -): StableCaptureSignal | undefined { - // SnapshotState.backend is optional at the runtime boundary. The bound iOS backend remains - // authoritative when a presented capture omits that provenance; a declared non-XCTest backend - // still wins so this confirmation cannot leak onto another capture implementation. - if (!baselineNodes || !snapshot || !isBoundIosXCTestCapture(snapshot.backend, platform)) { - return undefined; - } +type StableCaptureConfirmation = { + /** The quiet window this capture demands, at least the loop's current one. */ + requiredQuietMs: ( + snapshot: Parameters[0], + currentQuietMs: number, + ) => number; +}; + +/** + * The pre-action tree to confirm against, or `undefined` when the capture in hand cannot support the + * comparison: the capture must prove an iOS XCTest session (`SnapshotState.backend` is optional at + * the runtime boundary, so the bound platform stays authoritative when a capture omits that + * provenance; a declared non-XCTest backend still wins so this confirmation cannot leak onto another + * capture implementation), and it must be trustworthy evidence about the screen. `undefined` asks the + * next capture, rather than ruling the transition out for the run. + */ +function broadTransitionBaseline(params: { + baselineNodes: SnapshotNode[]; + capture: Parameters[0]; + platform: AgentDeviceRuntime['backend']['platform']; + appBundleId: string | undefined; +}): StableCaptureSignal | undefined { + const { baselineNodes, capture, platform, appBundleId } = params; + if (!isBoundIosXCTestCapture(capture.backend, platform)) return undefined; // A broad *screen* replacement needs a complete post-action viewport projection. The baseline // comes from settle's authoritative pre-action capture, but a presented iOS modal can omit its // Application root and still be the complete interaction surface. Requiring the root only from // the candidate prevents scoped/synthetic post-action fragments from extending settle latency. - const hasCompleteViewportProjection = snapshot.nodes.some(isViewportRootNode); + const hasCompleteViewportProjection = capture.nodes.some(isViewportRootNode); // Tiny pre-action surfaces are the other trustworthy completeness signal: iOS presents alerts // and sheets as a handful of nodes, and their first post-dismissal capture can temporarily omit // the viewport root. Treating that rootless replacement as an arbitrary scoped projection lets // a coherent transitional tree win the default 500ms quiet window. const hasTinyPresentedBaseline = baselineNodes.length <= TINY_STABLE_TREE_NODE_COUNT; if (!hasCompleteViewportProjection && !hasTinyPresentedBaseline) return undefined; - return stableCaptureReplacementSignal({ ...snapshot, nodes: baselineNodes }, appBundleId); + return stableCaptureComparisonSignal(capture, baselineNodes, appBundleId); } /** - * The comparison signal for the broad-replacement verdict: the capture minus the chrome the - * settle diff also collapses (`withoutSettleChrome`). Keyboard surface arriving or leaving — - * ~40 keys, the candidate bar, the dock buttons on a fill — is not a screen replacement, and - * counting it as one made every keyboard-raising action pay the 1.5 s confirmation window - * (#3469). The survivor rule lives in the shared classifier, so this signal and the diff agree - * on what keyboard chrome is; here it only decides the quiet window, never the visible diff. + * One side of the broad-replacement comparison: a tree minus the chrome the settle diff also + * collapses (`withoutSettleChrome`). Keyboard surface arriving or leaving — ~40 keys, the candidate + * bar, the dock buttons on a fill — is not a screen replacement, and counting it as one made every + * keyboard-raising action pay the 1.5 s confirmation window (#3469). The survivor rule lives in the + * shared classifier, so this signal and the diff agree on what keyboard chrome is; here it only + * decides the quiet window, never the visible diff. */ -function stableCaptureReplacementSignal( +function stableCaptureComparisonSignal( snapshot: Parameters[0], + nodes: SnapshotNode[], appBundleId: string | undefined, ): StableCaptureSignal { return stableCaptureSignal({ ...snapshot, - nodes: withoutSettleChrome(snapshot.nodes, appBundleId), + nodes: withoutSettleChrome(nodes, appBundleId), }); } @@ -255,30 +463,6 @@ function isBoundIosXCTestCapture( return backend === undefined && platform === 'ios'; } -/** - * The quiet window the candidate capture demands: the broad-replacement verdict runs on the - * chrome-free candidate signal (see `stableCaptureReplacementSignal`), so a keyboard pop never - * reaches the confirmation branch at all. Do not memoize this on `stableCaptureSignalsEqual`: - * the signal identity omits `role`, parent links, and >1px-free geometry, all of which the - * classifier reads, and one pass costs tens of microseconds against the 300 ms poll cadence. - */ -function stableCaptureNextQuietWindowMs(params: { - baseline: StableCaptureSignal | undefined; - snapshot: Parameters[0]; - appBundleId: string | undefined; - requestedQuietMs: number; - currentQuietMs: number; -}): number { - if (!params.baseline) { - return params.currentQuietMs; - } - const replacementSignal = stableCaptureReplacementSignal(params.snapshot, params.appBundleId); - if (!stableCaptureSignalsHaveBroadReplacement(params.baseline, replacementSignal)) { - return params.currentQuietMs; - } - return Math.max(params.requestedQuietMs, BROAD_TRANSITION_CONFIRMATION_QUIET_MS); -} - function isPrivateAxRecovery(verdict: SnapshotQualityVerdict | undefined): boolean { // 'deferred' = the penalty circuit breaker routed straight to private AX; the capture paid no // grind, so there is nothing to give the settle budget back for. @@ -351,7 +535,7 @@ function stableCaptureDelayMs(params: { // subsequent ref/get/find lookups against the same session. (`--settle` DOES // store its final capture, but explicitly, in one place, after the loop.) // -// Resolves undefined when the capture does not return within remainingMs. A +// Resolves `stalled` when the capture does not return within remainingMs. A // stalled backend capture (observed with macOS AX captures) must not push the // stable wait past the user-supplied timeout into the daemon request timeout. // The deadline cancels and joins the capture before returning so no late capture @@ -361,17 +545,26 @@ async function captureStableSignalWithinDeadline( options: CommandContext & SelectorSnapshotOptions, remainingMs: number, preferredBackend?: SnapshotPreferredBackend, -): Promise { - const result = await runWithinWaitDeadline(runtime, options, remainingMs, async (signal) => { - return await captureSelectorSnapshot( - runtime, - { ...options, signal }, - { - updateSession: false, - interactiveOnly: true, - preferredBackend, - }, - ); - }); - return result.timedOut ? undefined : result.value; +): Promise { + try { + const result = await runWithinWaitDeadline(runtime, options, remainingMs, async (signal) => { + return await captureSelectorSnapshot( + runtime, + { ...options, signal }, + { + updateSession: false, + interactiveOnly: true, + preferredBackend, + }, + ); + }); + return result.timedOut ? { kind: 'stalled' } : { kind: 'captured', capture: result.value }; + } catch (error) { + return { kind: 'failed', error }; + } } + +type StableCaptureRead = + | { kind: 'captured'; capture: CapturedSnapshot } + | { kind: 'stalled' } + | { kind: 'failed'; error: unknown }; diff --git a/src/commands/interaction/runtime/wait-stable.test.ts b/src/commands/interaction/runtime/wait-stable.test.ts index 9e55cae284..e59185691b 100644 --- a/src/commands/interaction/runtime/wait-stable.test.ts +++ b/src/commands/interaction/runtime/wait-stable.test.ts @@ -334,6 +334,38 @@ test('runtime wait stable times out with capture stats when never settling', asy assert.ok(captures > 1); }); +test('runtime wait stable surfaces a capture failure instead of reporting a timeout', async () => { + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => { + throw new Error('AX bridge crashed'); + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default' }]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); + + await assert.rejects( + () => + device.selectors.wait({ + session: 'default', + target: { kind: 'stable', quietMs: 500, timeoutMs: 1_000 }, + }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.notEqual( + (error as { details?: { reason?: string } }).details?.reason, + 'wait_stable_timeout', + ); + assert.match(error.message, /AX bridge crashed/); + return true; + }, + ); +}); + test('runtime wait stable times out when a backend capture stalls past the wait budget', async () => { const clock = createFakeClock(); let captureAborted = false; diff --git a/src/commands/interaction/runtime/wait-stable.ts b/src/commands/interaction/runtime/wait-stable.ts index 15240a7928..d16acc6ae5 100644 --- a/src/commands/interaction/runtime/wait-stable.ts +++ b/src/commands/interaction/runtime/wait-stable.ts @@ -21,6 +21,7 @@ export async function waitForStable( quietMs: quiet, timeoutMs: timeout, }); + if (outcome.failure) throw outcome.failure.error; if (!outcome.settled) { throw new AppError('COMMAND_FAILED', 'wait timed out waiting for a stable UI', { reason: WAIT_REASONS.stableTimeout, diff --git a/src/commands/post-action-observation-grammar.ts b/src/commands/post-action-observation-grammar.ts index dd1b250d35..b6f77aa0df 100644 --- a/src/commands/post-action-observation-grammar.ts +++ b/src/commands/post-action-observation-grammar.ts @@ -34,7 +34,7 @@ const settleFields = () => ({ 'After the action, wait for the UI to go quiet and return the settled diff vs the pre-action tree in the same response. Best-effort; never fails the action.', ), settleQuietMs: integerField( - 'Settle: quiet window in milliseconds (default 500). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500 whatever window is asked for, within the timeoutMs settle deadline.', + 'Settle: quiet window in milliseconds (default 500; 300 when the settled tree comes from the iOS simulator snapshot bridge). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the timeoutMs settle deadline.', { min: 0 }, ), timeoutMs: integerField('Settle: wait deadline in milliseconds (default 10000).', { min: 1 }),