From 57ce1e586cad8e94f7f844924eaafae784a5d7c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 13 Oct 2026 09:37:51 +0200 Subject: [PATCH 1/6] fix(settle): resolve the default quiet window from the capture that pays it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--settle` asked every capture the same 500 ms quiet window. The iOS simulator snapshot bridge answers well inside it, so the loop spends a whole extra capture holding a window the tree has already satisfied — and the window is paid twice per settled command: once as the settle's final window, then again as the next command's baseline capture. The loop now prices the window from the capture it performs rather than the one the action was performed against: an unrequested `--settle` resolves 300 ms when that read came from the simulator snapshot bridge, and 500 ms for the Apple runner's own tree read, the Android helper, and any capture that declares no provenance — only the bridge carries the measurement. It re-prices on every capture, so a mid-run fallback to a more expensive read widens the remaining wait, and it reports the window it ran on (`settle.quietMs`) so the response states which default it applied. Keying on the baseline's producer was the first cut and is wrong: the window governs the loop's own reads, and a stored ref frame or session tree can be several commands stale or served by another producer. Keying on the channel would be wrong too — the bridge and the Apple runner share `xctest` and read at different costs. `--settle-quiet` still outranks both, and `wait stable` keeps the general default of its own. Cadence follows the window in force, so the shorter default also samples on a grid that can measure it. The loop's window, quiet clock, and recovery budget moved to one owner each in the same pass, which is what makes the re-pricing a single line rather than a fourth piece of loop state. The trap this layer owes the layer under it is pinned by name: a 300 ms default sits under the 1.5 s confirmation floor, so the arming rule that layer removed must not come back. Mutation check: with `requestedQuietMs >= DEFAULT_STABLE_QUIET_MS` restored in the arming path, that case is the only one in the file that fails — i.e. the shorter default would have silently switched the #1831 guard off for every default settle the bridge answers. --- .../src/flag-definitions-action.ts | 2 +- packages/contracts/src/client-gesture.ts | 3 +- .../daemon-composition/generic-settle.test.ts | 109 ++++- .../runtime/settle-default-quiet.test.ts | 143 ++++++ src/commands/interaction/runtime/settle.ts | 17 +- .../runtime/stable-capture.test.ts | 41 ++ .../interaction/runtime/stable-capture.ts | 448 ++++++++++++------ .../post-action-observation-grammar.ts | 2 +- 8 files changed, 622 insertions(+), 143 deletions(-) create mode 100644 src/commands/interaction/runtime/settle-default-quiet.test.ts diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index fe7fcb190d..9852a26361 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -211,7 +211,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ min: 0, usageLabel: '--settle-quiet ', usageDescription: - 'Settle: quiet window the UI must hold to count as settled (default 500ms). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the --timeout settle deadline', + 'Settle: quiet window the UI must hold to count as settled (default 500ms; 300ms when the settled tree comes from the iOS simulator snapshot bridge). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the --timeout settle deadline', projectConfig: true, recorded: false, }, diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index e636d621a6..cc21f0a6d1 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -25,7 +25,8 @@ export type RepeatedPressOptions = { * Opt-in (#1101): after the action, wait for the UI to go quiet and return the * settled diff vs the pre-action tree (`settle` on the result) in the same * response. Best-effort — never fails the action. `settleQuietMs` tunes the - * quiet window (default 500ms); `timeoutMs` bounds the settle wait (default + * quiet window (default 500ms, or 300ms when the settled tree comes from the iOS + * simulator snapshot bridge); `timeoutMs` bounds the settle wait (default * 10s) when `settle` is true. A bare `timeoutMs` without `settle` is ignored * for compatibility; `settleQuietMs` still requires `settle`. */ diff --git a/src/__tests__/daemon-composition/generic-settle.test.ts b/src/__tests__/daemon-composition/generic-settle.test.ts index d7a3111ab3..3b948d3a6d 100644 --- a/src/__tests__/daemon-composition/generic-settle.test.ts +++ b/src/__tests__/daemon-composition/generic-settle.test.ts @@ -1,6 +1,7 @@ import { composedPlatformServicesFixture } from './platform-services-fixture.ts'; import { beforeEach, expect, test, vi } from 'vitest'; import type { CommandFlags } from '@agent-device/contracts/command'; +import type { CommandClock } from '@agent-device/contracts/runtime-contract'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { APP_SURFACE, @@ -113,18 +114,36 @@ async function emulateCaptureSnapshotForSession( return snapshot; } -function mockCommandDispatch(snapshots: Array) { +function mockCommandDispatch( + snapshots: Array, + producer: NonNullable = 'apple-runner', +) { let snapshotCalls = 0; mockDispatch.mockImplementation(async (command) => { if (command === 'snapshot') { const nodes = snapshots[Math.min(snapshotCalls, snapshots.length - 1)]; snapshotCalls += 1; - return { nodes, backend: 'xctest', producer: 'apple-runner' }; + return { nodes, backend: 'xctest', producer }; } return {}; }); } +/** + * A clock the settle loop advances instead of waiting. The default-window cases below assert the + * window a run RESOLVED, which needs no real elapsed time: the loop's own budget arithmetic runs on + * this clock, so the suite neither slows nor goes unstable under load. + */ +function fakeClock(stepMs = 300): CommandClock { + let elapsed = 0; + return { + now: () => elapsed, + sleep: async (ms: number) => { + elapsed += ms > 0 ? ms : stepMs; + }, + }; +} + const contextFromFlags = () => ({}) as never; /** @@ -161,13 +180,14 @@ function seedSession( type GenericSettleTree = { nodes: RawSnapshotNode[]; systemSurface?: IosSystemSurfaceProvenance; + producer?: 'apple-runner' | 'simulator-ax-bridge'; }; function snapshotPayload(tree: GenericSettleTree) { return { nodes: tree.nodes, backend: 'xctest' as const, - producer: 'apple-runner' as const, + producer: tree.producer ?? ('apple-runner' as const), ...(tree.systemSurface ? { systemSurface: tree.systemSurface } : {}), }; } @@ -205,6 +225,8 @@ async function dispatchGeneric(params: { runtimeBindings?: Readonly< Pick[0], 'inspectFacts' | 'bindDevice'> >; + /** Injects a clock into every command surface the route builds, so no test waits real time. */ + clock?: CommandClock; }): Promise { const req: DaemonRequest = { token: 't', @@ -214,7 +236,9 @@ async function dispatchGeneric(params: { ...(params.flags ? { flags: params.flags } : {}), }; return await dispatchGenericCommand({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: params.clock + ? async (config) => await testCreateCommandSurface({ ...config, clock: params.clock }) + : testCreateCommandSurface, platformServices: composedPlatformServicesFixture(), req, ref: params.sessionStore.lookup(params.sessionName)!, @@ -801,3 +825,80 @@ for (const change of ['rebuild', 'retire'] as const) { } }); } + +// The default quiet window keys on the capture the settle loop PERFORMS (#3470 follow-up), not on +// the baseline it diffs: on this route the baseline is the session's stored tree, which can be +// several commands stale and served by a different producer. All three run on an injected clock, so +// they assert the resolved window without real waiting. +test('scroll --settle without --settle-quiet uses the shorter default when the loop reads the simulator bridge', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-default-quiet'; + seedSession(sessionName, sessionStore, { + nodes: BEFORE_NODES, + producer: 'simulator-ax-bridge', + }); + mockCommandDispatch([AFTER_NODES], 'simulator-ax-bridge'); + + const response = await dispatchGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + // No settleQuietMs: the route's own default window is what is under test. + flags: { settle: true, timeoutMs: 2_000 }, + clock: fakeClock(), + }); + + const settle = expectOkData(response).settle as SettlePayload; + expect(settle.settled).toBe(true); + expect(settle.quietMs).toBe(300); +}); + +test('scroll --settle keeps the 500ms default when the loop reads the Apple runner', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-default-quiet-runner'; + seedSession(sessionName, sessionStore, { + nodes: BEFORE_NODES, + producer: 'apple-runner', + }); + mockCommandDispatch([AFTER_NODES], 'apple-runner'); + + const response = await dispatchGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { settle: true, timeoutMs: 2_000 }, + clock: fakeClock(), + }); + + const settle = expectOkData(response).settle as SettlePayload; + expect(settle.settled).toBe(true); + expect(settle.quietMs).toBe(500); +}); + +test('a stale bridge-sourced baseline does not shorten a settle the Apple runner performs', async () => { + // The mismatch the stored baseline can carry: the tree on file came from the bridge, but the + // captures that pay this window come from the runner, whose read the short window was never + // measured on. Keying on the baseline would hold 300ms against an unmeasured read. + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-default-quiet-mismatch'; + seedSession(sessionName, sessionStore, { + nodes: BEFORE_NODES, + producer: 'simulator-ax-bridge', + }); + mockCommandDispatch([AFTER_NODES], 'apple-runner'); + + const response = await dispatchGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { settle: true, timeoutMs: 2_000 }, + clock: fakeClock(), + }); + + const settle = expectOkData(response).settle as SettlePayload; + expect(settle.settled).toBe(true); + expect(settle.quietMs).toBe(500); +}); diff --git a/src/commands/interaction/runtime/settle-default-quiet.test.ts b/src/commands/interaction/runtime/settle-default-quiet.test.ts new file mode 100644 index 0000000000..ee60d3e3b3 --- /dev/null +++ b/src/commands/interaction/runtime/settle-default-quiet.test.ts @@ -0,0 +1,143 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { SnapshotProvenance, SnapshotState } from '@agent-device/kernel/snapshot'; +import { ref } from './selector-read-utils.ts'; +import { + buttonSnapshot, + createSettleDevice, + welcomeSnapshot, +} from './__tests__/settle-device-fixtures.ts'; + +// The `--settle` DEFAULT quiet window keys on the capture the settle loop PERFORMS (#3470 +// follow-up), because that read is the one the window prices: the loop holds its quiet window open +// across captures of its own. The pre-action baseline cannot answer this — on the ref route it is +// the stored ref frame, on the generic route the session's stored tree, and either can be several +// commands stale and served by a producer other than the one settle then reads through. A caller +// that asks for a window with `--settle-quiet` always outranks the default. +// +// Budgets are injected (fake clock) — no real waiting. + +function withProvenance

(state: SnapshotState, provenance: P) { + return { ...state, ...provenance }; +} + +const BRIDGE = { backend: 'xctest', producer: 'simulator-ax-bridge' } as const; +const RUNNER = { backend: 'xctest', producer: 'apple-runner' } as const; + +/** + * A press whose baseline and whose loop captures come from declared capture paths, so the two can + * be made to agree or to disagree. + */ +function pressWithCapturePaths(params: { + baseline?: SnapshotProvenance; + loop?: SnapshotProvenance; + settle?: { quietMs?: number; timeoutMs?: number }; +}) { + const baseline = params.baseline + ? withProvenance(buttonSnapshot(), params.baseline) + : buttonSnapshot(); + let captures = 0; + const device = createSettleDevice({ + stored: baseline, + captureSnapshot: () => { + captures += 1; + // A ref press captures nothing before the action, so every call is the loop's own sample. + const settled = welcomeSnapshot(); + return { + snapshot: + captures === 1 ? baseline : params.loop ? withProvenance(settled, params.loop) : settled, + }; + }, + }); + return device.interactions.press(ref('@e1'), { session: 'default', settle: params.settle ?? {} }); +} + +test('a settle the simulator snapshot bridge performs uses the shorter default window', async () => { + const result = await pressWithCapturePaths({ loop: BRIDGE }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, true); + assert.equal(settle.quietMs, 300); +}); + +test('an explicit --settle-quiet outranks the per-capture default window', async () => { + const result = await pressWithCapturePaths({ loop: BRIDGE, settle: { quietMs: 700 } }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.quietMs, 700); +}); + +// Each of these is a read the 300 ms default was NOT measured on, so shortening its window would be +// an unmeasured change of behaviour rather than the measured one. The Apple runner shares the +// `xctest` CHANNEL with the bridge, which is why the rule keys on the producer. +const UNMEASURED_CAPTURES: Array<[string, SnapshotProvenance?]> = [ + ['the Apple runner tree read (same channel, different producer)', RUNNER], + ['an Appium-sourced tree on the same channel', { backend: 'xctest', producer: 'appium-source' }], + ['the Android helper', { backend: 'android', producer: 'android-uiautomator' }], + ['a capture that declares no provenance at all', undefined], +]; + +for (const [label, provenance] of UNMEASURED_CAPTURES) { + test(`${label} keeps the 500ms default window`, async () => { + const result = await pressWithCapturePaths({ loop: provenance }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.quietMs, 500); + }); +} + +test('a bridge read during settle earns the shorter window even from a runner-issued ref frame', async () => { + // The ref frame the action was authorized against came from the runner, so keying the window on + // the baseline would hold the unmeasured 500 ms against captures the bridge answered. + const result = await pressWithCapturePaths({ baseline: RUNNER, loop: BRIDGE }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.quietMs, 300); +}); + +test('a runner read during settle keeps the 500ms window despite a bridge-issued baseline', async () => { + // The unsafe direction of the same mismatch: holding 300 ms for a read priced at 500 ms would + // settle on a window the measurement never covered. + const result = await pressWithCapturePaths({ baseline: BRIDGE, loop: RUNNER }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.quietMs, 500); +}); + +test('the shorter default window never shortens a broad-transition confirmation', async () => { + // The trap this layer owes layer 1: a 300 ms default sits under the 1.5 s floor, so a rule that + // armed on `requestedQuietMs >= DEFAULT_STABLE_QUIET_MS` would silently switch the #1831 + // confirmation off for every default settle the bridge answers. Here the pre-action tree is the + // tiny-baseline shape, so the confirmation owns the wait and the 300 ms default still holds its + // 1.5 s floor (#3470). + const sparseBaseline = withProvenance(buttonSnapshot(), BRIDGE); + let captures = 0; + const device = createSettleDevice({ + stored: sparseBaseline, + captureSnapshot: () => { + captures += 1; + return { + snapshot: captures === 1 ? sparseBaseline : withProvenance(welcomeSnapshot(), BRIDGE), + }; + }, + }); + + const result = await device.interactions.press(ref('@e1'), { + session: 'default', + settle: { timeoutMs: 5_000 }, + }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, true); + assert.equal(settle.quietMs, 300); + assert.ok( + settle.waitedMs >= 1_500, + `confirmation did not outlast the shorter default: waitedMs=${settle.waitedMs}`, + ); +}); diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index 8ccff098d1..aa3893515a 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -126,12 +126,20 @@ async function settleAfterAction( options: CommandContext, params: SettleParams & { baseline: SurfaceScopedNodes; actionPoint?: Point }, ): Promise { - const quietMs = params.quietMs ?? DEFAULT_STABLE_QUIET_MS; + // The loop resolves an unrequested window from the capture that pays it, so the window is only + // known once one is taken. `base` carries the general default for the pre-capture shape the + // observation still has to report; `readSettledOutcome` replaces it with the window the loop ran. const timeoutMs = params.timeoutMs ?? DEFAULT_STABLE_TIMEOUT_MS; - const base: SettleObservation = { settled: false, waitedMs: 0, captures: 0, quietMs, timeoutMs }; + const base: SettleObservation = { + settled: false, + waitedMs: 0, + captures: 0, + quietMs: params.quietMs ?? DEFAULT_STABLE_QUIET_MS, + timeoutMs, + }; try { const outcome = await runStableCaptureLoop(runtime, options, { - quietMs, + ...(params.quietMs === undefined ? {} : { quietMs: params.quietMs }), timeoutMs, resetBudgetOnPrivateAxRecovery: true, broadTransitionBaselineNodes: params.baseline.nodes, @@ -162,6 +170,9 @@ async function readSettledOutcome( settled: outcome.settled, waitedMs: outcome.waitedMs, captures: outcome.captures, + // The window the loop actually ran. For an unrequested settle that is the one its own captures + // priced, which `base` cannot know because it is built before the first read. + quietMs: outcome.quietMs, }; if (!outcome.lastCapture) { return { diff --git a/src/commands/interaction/runtime/stable-capture.test.ts b/src/commands/interaction/runtime/stable-capture.test.ts index 7e3dbc05e4..32ac691133 100644 --- a/src/commands/interaction/runtime/stable-capture.test.ts +++ b/src/commands/interaction/runtime/stable-capture.test.ts @@ -326,6 +326,37 @@ test('settle confirms a tiny modal replacement through a partial post-action pro ); }); +test('a dismissal confirmation is not dropped by a later empty projection', async () => { + // The confirmation is a fact about the transition, so it survives a capture that stops qualifying. + // The comparison refuses a node-less side, so re-deciding it per capture hands the loop its short + // window again mid-confirmation — and the next empty read matches it, settling on two empty + // projections 302ms after a dismissal that owed 1.5s (#3470). + const { runtime } = transitionRuntime({ + settledAtMs: 0, + emptyProjectionAtCaptures: [2, 3], + }); + + const outcome = await runStableCaptureLoop( + runtime, + { session: 'default' }, + { + quietMs: 150, + timeoutMs: 5_000, + broadTransitionBaselineNodes: elementThreadsNoticeSnapshot.nodes, + }, + ); + + assert.equal(outcome.settled, true); + assert.ok( + outcome.waitedMs >= 1_500, + `empty projection disarmed the confirmation: settled after ${outcome.waitedMs}ms`, + ); + assert.ok( + outcome.lastCapture!.snapshot.nodes.length > 0, + 'settled on an empty projection instead of the dismissed screen', + ); +}); + test('settle keeps the default quiet window for a larger partial projection', async () => { const { runtime } = transitionRuntime({ omitViewportRoot: true, settledAtMs: 1_200 }); @@ -458,6 +489,8 @@ function transitionRuntime( omitViewportRoot?: boolean; settledAtMs?: number; sessionSnapshot?: SnapshotState; + /** 1-based capture indices that answer with an empty projection (mid-animation read). */ + emptyProjectionAtCaptures?: number[]; } = {}, ) { const captureBackend = options.captureBackend ?? 'tree'; @@ -476,6 +509,14 @@ function transitionRuntime( captureSnapshot: async () => { const keepsBaseline = options.firstCaptureKeepsBaseline === true && captures === 0; captures += 1; + if (options.emptyProjectionAtCaptures?.includes(captures)) { + return { + snapshot: withCaptureBackend( + { ...elementSettledRoomSnapshot, nodes: [] }, + captureBackend, + ), + }; + } const snapshot = keepsBaseline ? elementThreadsNoticeSnapshot : withCaptureBackend( diff --git a/src/commands/interaction/runtime/stable-capture.ts b/src/commands/interaction/runtime/stable-capture.ts index ce39bcf2a7..cf30b8f398 100644 --- a/src/commands/interaction/runtime/stable-capture.ts +++ b/src/commands/interaction/runtime/stable-capture.ts @@ -2,6 +2,7 @@ import type { SnapshotNode, SnapshotPreferredBackend, SnapshotQualityVerdict, + SnapshotState, } from '@agent-device/kernel/snapshot'; import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; @@ -34,8 +35,7 @@ import { withoutSettleChrome } from '@agent-device/capture-kit/snapshot-chrome'; const STABLE_POLL_INTERVAL_MS = 300; const STABLE_MIN_POLL_MS = 25; // Never slower than the default poll, never faster than the minimum: a tight window gets a cadence -// that can measure it, a window the confirmation extends gets one that does not sample the whole -// extended wait at the caller's original grid. +// that can measure it, and a window the confirmation extends is not sampled on the caller's grid. function stableCapturePollMs(quietMs: number): number { return Math.min(STABLE_POLL_INTERVAL_MS, Math.max(STABLE_MIN_POLL_MS, quietMs)); } @@ -55,6 +55,21 @@ const QUIET_DEADLINE_EPSILON_MS = 2; // (#3470). A confirmed replacement is held for the caller's own window, floored at 1.5 s. const BROAD_TRANSITION_CONFIRMATION_QUIET_MS = 1_500; export const DEFAULT_STABLE_QUIET_MS = 500; +/** + * The default quiet window a capture earns. The iOS simulator snapshot bridge answers well inside + * the general default, so holding the general window on that read spends a whole extra capture on a + * window its tree had already satisfied. Keyed on the producer, not the channel: the bridge and the + * Apple runner share the `xctest` channel and read at different costs, and only the bridge carries + * the measurement. Every other read — the runner's own tree read, the Android helper, a capture that + * declares no provenance — keeps {@link DEFAULT_STABLE_QUIET_MS}. + * (PR #3522 holds the interleaved before/after medians and the late-diff accuracy check.) + */ +function defaultQuietMsForCapture(snapshot: SnapshotState): number { + return snapshot.producer === 'simulator-ax-bridge' + ? SIMULATOR_BRIDGE_SETTLE_QUIET_MS + : DEFAULT_STABLE_QUIET_MS; +} +const SIMULATOR_BRIDGE_SETTLE_QUIET_MS = 300; export const DEFAULT_STABLE_TIMEOUT_MS = 10_000; // Below this node count a settled tree is suspicious: real app surfaces have // more than a handful of accessibility nodes, splash/loading screens do not. @@ -72,6 +87,12 @@ export type StableCaptureLoopResult = { waitedMs: number; captures: number; nodeCount: number; + /** + * The quiet window the run was holding when it returned: the caller's, or the default resolved + * from the last capture that paid it. A run that completed no capture never priced a window, so + * it reports the general default it started from. + */ + quietMs: number; /** The most recent completed capture, settled or not. */ lastCapture?: CapturedSnapshot; }; @@ -80,32 +101,46 @@ export async function runStableCaptureLoop( runtime: AgentDeviceRuntime, options: CommandContext & SelectorSnapshotOptions, params: { - quietMs: number; + /** + * The quiet window the caller asked for. Absent means the loop resolves its own default from + * the capture it performs, because that capture is the read the window prices + * ({@link defaultQuietMsForCapture}); a caller with its own default passes it explicitly. + */ + quietMs?: number; timeoutMs: number; resetBudgetOnPrivateAxRecovery?: boolean; /** Immutable pre-action projection; the stored session may advance before settle begins. */ broadTransitionBaselineNodes?: SnapshotNode[]; }, ): Promise { - const { quietMs, timeoutMs } = params; + const { timeoutMs } = params; const start = now(runtime); let deadlineMs = start + timeoutMs; let privateAxRecoveryBudgetReset = false; const session = await runtime.sessions.get(options.session ?? 'default'); - let transitionBaseline: StableCaptureSignal | undefined; + const window = createStableCaptureWindow(params.quietMs); + const confirmation = broadTransitionConfirmation({ + baselineNodes: params.broadTransitionBaselineNodes, + platform: runtime.backend.platform, + appBundleId: session?.appBundleId, + }); let preferredBackend = preferredSnapshotBackendForVerdict(session?.snapshot?.snapshotQuality); - // Cadence derives from the quiet window in force (never slower than the default - // poll): a caller asking for a 50ms quiet window should not be forced onto a - // 300ms grid, and a run the confirmation has extended to 1.5s should not spend - // that wait on the caller's tight grid — every sample is a capture round trip. - // Tests inject the budget instead of waiting real time. - let pollMs = stableCapturePollMs(quietMs); + const pacing = createStableCapturePacing(start); let captures = 0; - let lastSignal: StableCaptureSignal | undefined; - let lastNodeCount = 0; let lastCapture: CapturedSnapshot | undefined; - let quietSinceMs = start; - let requiredQuietMs = quietMs; + const result = ( + settled: boolean, + stalled: boolean, + waitedMs: number, + ): StableCaptureLoopResult => ({ + settled, + stalled, + waitedMs, + captures, + nodeCount: pacing.lastNodeCount, + quietMs: window.baseQuietMs, + lastCapture, + }); while (now(runtime) < deadlineMs) { const capture = await captureStableSignalWithinDeadline( runtime, @@ -113,137 +148,308 @@ export async function runStableCaptureLoop( deadlineMs - now(runtime), preferredBackend, ); - if (!capture) { - return { - settled: false, - stalled: true, - waitedMs: now(runtime) - start, - captures, - nodeCount: lastNodeCount, - lastCapture, - }; - } + if (!capture) return result(false, true, now(runtime) - start); captures += 1; lastCapture = capture; - transitionBaseline ??= stableCaptureTransitionBaseline( - params.broadTransitionBaselineNodes, - capture.snapshot, - runtime.backend.platform, - session?.appBundleId, - ); + const defaultQuietMs = window.price(capture.snapshot); const signal = stableCaptureSignal(capture.snapshot); - requiredQuietMs = stableCaptureNextQuietWindowMs({ - baseline: transitionBaseline, - snapshot: capture.snapshot, - appBundleId: session?.appBundleId, - requestedQuietMs: quietMs, - currentQuietMs: requiredQuietMs, - }); - // Recomputed from the window in force rather than only when it changed: the window is a - // monotonic max, so this is the same value, and the cadence cannot drift from it. - pollMs = stableCapturePollMs(requiredQuietMs); + // One value carries both: how long this capture must stay quiet, and therefore the cadence that + // can measure it. Sampling an extended wait on the caller's tight grid would spend the whole + // extension in capture round trips. + const requiredQuietMs = confirmation.requiredQuietMs(capture.snapshot, defaultQuietMs); + const pollMs = stableCapturePollMs(requiredQuietMs); preferredBackend ??= preferredSnapshotBackendForVerdict(capture.snapshot.snapshotQuality); const nowMs = now(runtime); - const recoveredDeadlineMs = extendedDeadlineAfterPrivateAxRecovery({ - resetRequested: params.resetBudgetOnPrivateAxRecovery, + const nodeCount = capture.snapshot.nodes.length; + const verdict = capture.snapshot.snapshotQuality; + const recovered = recoverStableCaptureBudget({ + pacing, + resetBudgetOnPrivateAxRecovery: params.resetBudgetOnPrivateAxRecovery, alreadyReset: privateAxRecoveryBudgetReset, - verdict: capture.snapshot.snapshotQuality, - nowMs, - timeoutMs, - deadlineMs, - }); - if (recoveredDeadlineMs !== undefined) { - privateAxRecoveryBudgetReset = true; - deadlineMs = recoveredDeadlineMs; - quietSinceMs = nowMs; - lastSignal = signal; - lastNodeCount = capture.snapshot.nodes.length; - const recoveryDelayMs = stableCaptureDelayMs({ + state: { + deadlineMs, + timeoutMs, nowMs, - quietSinceMs, - quietMs: requiredQuietMs, + verdict, + signal, + nodeCount, + requiredQuietMs, pollMs, - deadlineMs, - }); - if (recoveryDelayMs <= 0) break; - await sleep(runtime, recoveryDelayMs); + }, + }); + if (recovered) { + privateAxRecoveryBudgetReset = true; + deadlineMs = recovered.deadlineMs; + if (recovered.delayMs <= 0) break; + await sleep(runtime, recovered.delayMs); continue; } - if (!stableCaptureSignalsEqual(lastSignal, signal)) { - lastSignal = signal; - lastNodeCount = capture.snapshot.nodes.length; - quietSinceMs = nowMs; - } else if (captures >= 2 && nowMs - quietSinceMs >= requiredQuietMs) { - return { - settled: true, - stalled: false, - waitedMs: nowMs - start, - captures, - nodeCount: lastNodeCount, - lastCapture, - }; - } - const delayMs = stableCaptureDelayMs({ + const observed = pacing.observe({ + signal, + nodeCount, nowMs, - quietSinceMs, - quietMs: requiredQuietMs, + requiredQuietMs, pollMs, deadlineMs, + captures, }); - if (delayMs <= 0) break; - await sleep(runtime, delayMs); + if (observed.settled) return result(true, false, nowMs - start); + if (observed.delayMs <= 0) break; + await sleep(runtime, observed.delayMs); } + return result(false, false, now(runtime) - start); +} + +/** + * The quiet window the run holds. An unrequested window is re-priced from each capture the loop + * performs: the read that pays the window is the loop's own, not the one the baseline came from, + * which can be several commands stale and served by another producer, and a mid-run fallback changes + * the price mid-wait. The confirmation applies its own floor on top of the returned price, so + * re-pricing is never a shorter wait. A window the caller asked for is fixed. + */ +function createStableCaptureWindow(requestedQuietMs: number | undefined) { + let baseQuietMs = requestedQuietMs ?? DEFAULT_STABLE_QUIET_MS; return { - settled: false, - stalled: false, - waitedMs: now(runtime) - start, - captures, - nodeCount: lastNodeCount, - lastCapture, + get baseQuietMs(): number { + return baseQuietMs; + }, + /** Re-prices the window from a capture the loop just completed, and reports that price. */ + price(snapshot: SnapshotState): number { + if (requestedQuietMs === undefined) baseQuietMs = defaultQuietMsForCapture(snapshot); + return baseQuietMs; + }, }; } -function stableCaptureTransitionBaseline( - baselineNodes: SnapshotNode[] | undefined, - snapshot: Parameters[0] | undefined, - platform: AgentDeviceRuntime['backend']['platform'], - appBundleId: string | undefined, -): StableCaptureSignal | undefined { - // SnapshotState.backend is optional at the runtime boundary. The bound iOS backend remains - // authoritative when a presented capture omits that provenance; a declared non-XCTest backend - // still wins so this confirmation cannot leak onto another capture implementation. - if (!baselineNodes || !snapshot || !isBoundIosXCTestCapture(snapshot.backend, platform)) { - return undefined; - } +/** + * The once-per-run budget extension a private-AX recovery buys. The recovery re-read the same + * surface through another backend, so the quiet window restarts on that capture rather than carrying + * suspicion across the switch. Undefined when the capture asked for no reset (or the run already + * spent its one). + */ +function recoverStableCaptureBudget(params: { + pacing: StableCapturePacing; + alreadyReset: boolean; + resetBudgetOnPrivateAxRecovery?: boolean; + state: { + deadlineMs: number; + timeoutMs: number; + nowMs: number; + verdict: SnapshotQualityVerdict | undefined; + signal: StableCaptureSignal; + nodeCount: number; + requiredQuietMs: number; + pollMs: number; + }; +}): { deadlineMs: number; delayMs: number } | undefined { + const { state } = params; + const extendedDeadlineMs = extendedDeadlineAfterPrivateAxRecovery({ + resetRequested: params.resetBudgetOnPrivateAxRecovery, + alreadyReset: params.alreadyReset, + verdict: state.verdict, + nowMs: state.nowMs, + timeoutMs: state.timeoutMs, + deadlineMs: state.deadlineMs, + }); + if (extendedDeadlineMs === undefined) return undefined; + params.pacing.restart({ + signal: state.signal, + nodeCount: state.nodeCount, + nowMs: state.nowMs, + }); + return { + deadlineMs: extendedDeadlineMs, + delayMs: params.pacing.delayMs({ + nowMs: state.nowMs, + requiredQuietMs: state.requiredQuietMs, + pollMs: state.pollMs, + deadlineMs: extendedDeadlineMs, + }), + }; +} + +type StableCapturePacing = ReturnType; + +/** + * The settle loop's per-capture bookkeeping: the last capture's shape, when the UI last changed, and + * the quiet-window arithmetic that decides whether a capture settles the run. Owns that state so the + * loop above stays the read of one capture at a time. + */ +function createStableCapturePacing(start: number) { + let lastSignal: StableCaptureSignal | undefined; + let lastNodeCount = 0; + let quietSinceMs = start; + return { + /** Node count of the most recent capture, reported on every result shape. */ + get lastNodeCount(): number { + return lastNodeCount; + }, + /** Folds one capture in: settled, or how long to wait before the next one (<= 0 ends the run). */ + observe(params: { + signal: StableCaptureSignal; + nodeCount: number; + nowMs: number; + requiredQuietMs: number; + pollMs: number; + deadlineMs: number; + captures: number; + }): { settled: boolean; delayMs: number } { + if (!stableCaptureSignalsEqual(lastSignal, params.signal)) { + // The UI is still moving: the quiet window restarts from this capture. + lastSignal = params.signal; + lastNodeCount = params.nodeCount; + quietSinceMs = params.nowMs; + } else if (params.captures >= 2 && params.nowMs - quietSinceMs >= params.requiredQuietMs) { + return { settled: true, delayMs: 0 }; + } + return { + settled: false, + delayMs: stableCaptureDelayMs({ + nowMs: params.nowMs, + quietSinceMs, + quietMs: params.requiredQuietMs, + pollMs: params.pollMs, + deadlineMs: params.deadlineMs, + }), + }; + }, + /** Adopts a capture as the new start of the quiet window without deciding anything. */ + restart(params: { signal: StableCaptureSignal; nodeCount: number; nowMs: number }): void { + lastSignal = params.signal; + lastNodeCount = params.nodeCount; + quietSinceMs = params.nowMs; + }, + /** The wait before the next capture, given the window currently in force. */ + delayMs(params: { + nowMs: number; + requiredQuietMs: number; + pollMs: number; + deadlineMs: number; + }): number { + return stableCaptureDelayMs({ + nowMs: params.nowMs, + quietSinceMs, + quietMs: params.requiredQuietMs, + pollMs: params.pollMs, + deadlineMs: params.deadlineMs, + }); + }, + }; +} + +/** + * The broad-transition confirmation, as one decision the settle loop asks about each capture. It + * answers the arming question once for the run and then compares each capture against the pre-action + * tree until one proves the replacement, holding that window for the rest of the run. + * + * Arming never consults the requested quiet window: the old `requestedQuietMs < + * DEFAULT_STABLE_QUIET_MS` bar is what made `--settle-quiet 499` behave differently from 500 and left + * `settleQuietMs: 150` callers without the guard at all. A confirmed replacement is held for at least + * the caller's own window, floored at 1.5 s: `--settle-quiet` asks for a tighter verdict, not for a + * different set of transitions to distrust (#3470). + */ +function broadTransitionConfirmation(params: { + baselineNodes: SnapshotNode[] | undefined; + platform: AgentDeviceRuntime['backend']['platform']; + appBundleId: string | undefined; +}): StableCaptureConfirmation { + // undefined until the first capture answers it; null once the transition is ruled out, so an + // ordinary local mutation keeps its chrome-free comparison off the cadence. + let transitionBaseline: StableCaptureSignal | null | undefined; + // The comparison refuses a node-less side outright, so a mid-animation empty read must not be able + // to cancel the floor: without the latch it hands the loop its short window back, and the next + // empty read matches it, settling a dismissal that owed the floor. + let confirmedQuietMs: number | undefined; + return { + // While unconfirmed, do not memoize the verdict on `stableCaptureSignalsEqual`: signal identity + // omits `role`, parent links, and >1px-free geometry, all of which the chrome classifier reads, + // and one pass costs tens of microseconds against the 300 ms poll cadence. + requiredQuietMs(snapshot, currentQuietMs) { + // Not `??=`: that operator re-runs the right side whenever it resolves to `null`, which is + // exactly the ruled-out answer a non-iOS or backend-less run gives, on every single poll. + if (transitionBaseline === undefined) { + transitionBaseline = broadTransitionBaseline({ + baselineNodes: params.baselineNodes, + capture: snapshot, + platform: params.platform, + appBundleId: params.appBundleId, + }); + } + if (transitionBaseline === null) return currentQuietMs; + if (confirmedQuietMs === undefined) { + const candidate = stableCaptureComparisonSignal( + snapshot, + snapshot.nodes, + params.appBundleId, + ); + if (stableCaptureSignalsHaveBroadReplacement(transitionBaseline, candidate)) { + confirmedQuietMs = Math.max(currentQuietMs, BROAD_TRANSITION_CONFIRMATION_QUIET_MS); + } + } + // Still maxed against the current window: the loop can price a wider one after the latch, on a + // mid-run fallback from the bridge to the runner. + return confirmedQuietMs === undefined + ? currentQuietMs + : Math.max(currentQuietMs, confirmedQuietMs); + }, + }; +} + +type StableCaptureConfirmation = { + /** The quiet window this capture demands, at least the loop's current one. */ + requiredQuietMs: ( + snapshot: Parameters[0], + currentQuietMs: number, + ) => number; +}; + +/** + * The pre-action tree to confirm against, or `null` when the transition in front of the loop is not + * one this confirmation exists for: the chrome-free signal for the pre-action tree, when the capture + * proves an iOS XCTest session (`SnapshotState.backend` is optional at the runtime boundary, so the + * bound platform stays authoritative when a capture omits that provenance; a declared non-XCTest + * backend still wins so this confirmation cannot leak onto another capture implementation) and that + * tree is trustworthy evidence about the screen. Whether the screen was actually replaced stays + * per-capture — the first capture after a dismissal can still show the dismissed surface. + */ +function broadTransitionBaseline(params: { + baselineNodes: SnapshotNode[] | undefined; + capture: Parameters[0]; + platform: AgentDeviceRuntime['backend']['platform']; + appBundleId: string | undefined; +}): StableCaptureSignal | null { + const { baselineNodes, capture, platform, appBundleId } = params; + if (!baselineNodes || !isBoundIosXCTestCapture(capture.backend, platform)) return null; // A broad *screen* replacement needs a complete post-action viewport projection. The baseline // comes from settle's authoritative pre-action capture, but a presented iOS modal can omit its // Application root and still be the complete interaction surface. Requiring the root only from // the candidate prevents scoped/synthetic post-action fragments from extending settle latency. - const hasCompleteViewportProjection = snapshot.nodes.some(isViewportRootNode); + const hasCompleteViewportProjection = capture.nodes.some(isViewportRootNode); // Tiny pre-action surfaces are the other trustworthy completeness signal: iOS presents alerts // and sheets as a handful of nodes, and their first post-dismissal capture can temporarily omit // the viewport root. Treating that rootless replacement as an arbitrary scoped projection lets // a coherent transitional tree win the default 500ms quiet window. const hasTinyPresentedBaseline = baselineNodes.length <= TINY_STABLE_TREE_NODE_COUNT; - if (!hasCompleteViewportProjection && !hasTinyPresentedBaseline) return undefined; - return stableCaptureReplacementSignal({ ...snapshot, nodes: baselineNodes }, appBundleId); + if (!hasCompleteViewportProjection && !hasTinyPresentedBaseline) return null; + return stableCaptureComparisonSignal(capture, baselineNodes, appBundleId); } /** - * The comparison signal for the broad-replacement verdict: the capture minus the chrome the - * settle diff also collapses (`withoutSettleChrome`). Keyboard surface arriving or leaving — - * ~40 keys, the candidate bar, the dock buttons on a fill — is not a screen replacement, and - * counting it as one made every keyboard-raising action pay the 1.5 s confirmation window - * (#3469). The survivor rule lives in the shared classifier, so this signal and the diff agree - * on what keyboard chrome is; here it only decides the quiet window, never the visible diff. + * One side of the broad-replacement comparison: a tree minus the chrome the settle diff also + * collapses (`withoutSettleChrome`). Keyboard surface arriving or leaving — ~40 keys, the candidate + * bar, the dock buttons on a fill — is not a screen replacement, and counting it as one made every + * keyboard-raising action pay the 1.5 s confirmation window (#3469). The survivor rule lives in the + * shared classifier, so this signal and the diff agree on what keyboard chrome is; here it only + * decides the quiet window, never the visible diff. */ -function stableCaptureReplacementSignal( +function stableCaptureComparisonSignal( snapshot: Parameters[0], + nodes: SnapshotNode[], appBundleId: string | undefined, ): StableCaptureSignal { return stableCaptureSignal({ ...snapshot, - nodes: withoutSettleChrome(snapshot.nodes, appBundleId), + nodes: withoutSettleChrome(nodes, appBundleId), }); } @@ -255,30 +461,6 @@ function isBoundIosXCTestCapture( return backend === undefined && platform === 'ios'; } -/** - * The quiet window the candidate capture demands: the broad-replacement verdict runs on the - * chrome-free candidate signal (see `stableCaptureReplacementSignal`), so a keyboard pop never - * reaches the confirmation branch at all. Do not memoize this on `stableCaptureSignalsEqual`: - * the signal identity omits `role`, parent links, and >1px-free geometry, all of which the - * classifier reads, and one pass costs tens of microseconds against the 300 ms poll cadence. - */ -function stableCaptureNextQuietWindowMs(params: { - baseline: StableCaptureSignal | undefined; - snapshot: Parameters[0]; - appBundleId: string | undefined; - requestedQuietMs: number; - currentQuietMs: number; -}): number { - if (!params.baseline) { - return params.currentQuietMs; - } - const replacementSignal = stableCaptureReplacementSignal(params.snapshot, params.appBundleId); - if (!stableCaptureSignalsHaveBroadReplacement(params.baseline, replacementSignal)) { - return params.currentQuietMs; - } - return Math.max(params.requestedQuietMs, BROAD_TRANSITION_CONFIRMATION_QUIET_MS); -} - function isPrivateAxRecovery(verdict: SnapshotQualityVerdict | undefined): boolean { // 'deferred' = the penalty circuit breaker routed straight to private AX; the capture paid no // grind, so there is nothing to give the settle budget back for. diff --git a/src/commands/post-action-observation-grammar.ts b/src/commands/post-action-observation-grammar.ts index dd1b250d35..b6f77aa0df 100644 --- a/src/commands/post-action-observation-grammar.ts +++ b/src/commands/post-action-observation-grammar.ts @@ -34,7 +34,7 @@ const settleFields = () => ({ 'After the action, wait for the UI to go quiet and return the settled diff vs the pre-action tree in the same response. Best-effort; never fails the action.', ), settleQuietMs: integerField( - 'Settle: quiet window in milliseconds (default 500). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500 whatever window is asked for, within the timeoutMs settle deadline.', + 'Settle: quiet window in milliseconds (default 500; 300 when the settled tree comes from the iOS simulator snapshot bridge). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the timeoutMs settle deadline.', { min: 0 }, ), timeoutMs: integerField('Settle: wait deadline in milliseconds (default 10000).', { min: 1 }), From dc196409371b0f3b288a60c88da509231abb6090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 14:10:55 +0200 Subject: [PATCH 2/6] fix(settle): re-check broad-transition arming on every capture until one arms The arming decision was made on the first capture and then stored as null for the rest of the run, but two of its inputs belong to the capture, not to the run: viewport-root completeness and the declared capture channel. A dismissal of anything larger than an alert or sheet needs both before the confirmation may arm, so a mid-animation rootless or empty first read -- or one served by another channel -- ruled the 1.5s floor out for a settle that owed it. Only the pre-action projection is constant for the run, so it is the only input allowed to rule the confirmation out; the per-capture half now re-asks until one capture arms. --- .../runtime/stable-capture.test.ts | 65 +++++++++++++++++-- .../interaction/runtime/stable-capture.ts | 59 +++++++++-------- 2 files changed, 91 insertions(+), 33 deletions(-) diff --git a/src/commands/interaction/runtime/stable-capture.test.ts b/src/commands/interaction/runtime/stable-capture.test.ts index 32ac691133..246d38e4ce 100644 --- a/src/commands/interaction/runtime/stable-capture.test.ts +++ b/src/commands/interaction/runtime/stable-capture.test.ts @@ -329,8 +329,8 @@ test('settle confirms a tiny modal replacement through a partial post-action pro test('a dismissal confirmation is not dropped by a later empty projection', async () => { // The confirmation is a fact about the transition, so it survives a capture that stops qualifying. // The comparison refuses a node-less side, so re-deciding it per capture hands the loop its short - // window again mid-confirmation — and the next empty read matches it, settling on two empty - // projections 302ms after a dismissal that owed 1.5s (#3470). + // window again mid-confirmation — and the next empty read matches it, so two empty projections + // hold a quiet window on a dismissal the floor still owed (#3470). const { runtime } = transitionRuntime({ settledAtMs: 0, emptyProjectionAtCaptures: [2, 3], @@ -357,6 +357,46 @@ test('a dismissal confirmation is not dropped by a later empty projection', asyn ); }); +// The two arming inputs that belong to the capture, not to the run: the viewport-root completeness +// and the declared capture backend. A dismissal of anything bigger than an alert or sheet (the +// `<= TINY_STABLE_TREE_NODE_COUNT` escape hatch) has to meet BOTH before the confirmation may arm, so +// a mid-animation first read decides nothing — it must leave the question open for the next capture. +// Ruling it out on that first read settled these on the plain window, which after this layer is +// 300 ms on the simulator bridge (#3470). +const MID_ANIMATION_FIRST_CAPTURES: Array< + [string, NonNullable[0]>] +> = [ + ['a rootless first capture', { rootlessUntilMs: 400 }], + ['an empty first capture', { emptyProjectionAtCaptures: [1] }], + ['a first capture read through another channel', { otherChannelAtCaptures: [1] }], +]; +for (const [label, midAnimation] of MID_ANIMATION_FIRST_CAPTURES) { + test(`settle arms the confirmation on the first rooted XCTest capture after ${label}`, async () => { + const { runtime } = transitionRuntime({ settledAtMs: 0, ...midAnimation }); + + const outcome = await runStableCaptureLoop( + runtime, + { session: 'default' }, + { + timeoutMs: 5_000, + // A full-screen settings table: the tiny-presented-baseline escape hatch does not apply, so + // arming depends entirely on the captures below. + broadTransitionBaselineNodes: elementSettingsSnapshot([1_138, 1_423, 2_144, 2_434]).nodes, + }, + ); + + assert.equal(outcome.settled, true); + assert.equal( + outcome.lastCapture?.snapshot.nodes.some((node) => node.label === 'action file'), + false, + ); + assert.ok( + outcome.waitedMs >= 1_500, + `${label} ruled the confirmation out for the run: settled after ${outcome.waitedMs}ms`, + ); + }); +} + test('settle keeps the default quiet window for a larger partial projection', async () => { const { runtime } = transitionRuntime({ omitViewportRoot: true, settledAtMs: 1_200 }); @@ -491,6 +531,10 @@ function transitionRuntime( sessionSnapshot?: SnapshotState; /** 1-based capture indices that answer with an empty projection (mid-animation read). */ emptyProjectionAtCaptures?: number[]; + /** Captures taken before this deadline omit the viewport root (mid-animation read). */ + rootlessUntilMs?: number; + /** 1-based capture indices that declare a non-XCTest capture channel. */ + otherChannelAtCaptures?: number[]; } = {}, ) { const captureBackend = options.captureBackend ?? 'tree'; @@ -523,11 +567,22 @@ function transitionRuntime( elapsedMs < settledAtMs ? elementTransientRoomSnapshot : elementSettledRoomSnapshot, captureBackend, ); + const rootless = + options.omitViewportRoot === true || + (options.rootlessUntilMs !== undefined && elapsedMs < options.rootlessUntilMs); + const declaresOtherChannel = options.otherChannelAtCaptures?.includes(captures) === true; + const rootlessSnapshot = rootless + ? { ...snapshot, nodes: snapshot.nodes.slice(1) } + : snapshot; return { snapshot: withoutSnapshotBackend( - options.omitViewportRoot === true - ? { ...snapshot, nodes: snapshot.nodes.slice(1) } - : snapshot, + declaresOtherChannel + ? { + ...rootlessSnapshot, + backend: 'android' as const, + producer: 'android-uiautomator' as const, + } + : rootlessSnapshot, options.omitSnapshotBackend === true, ), }; diff --git a/src/commands/interaction/runtime/stable-capture.ts b/src/commands/interaction/runtime/stable-capture.ts index cf30b8f398..7cc0c7034b 100644 --- a/src/commands/interaction/runtime/stable-capture.ts +++ b/src/commands/interaction/runtime/stable-capture.ts @@ -339,8 +339,17 @@ function createStableCapturePacing(start: number) { /** * The broad-transition confirmation, as one decision the settle loop asks about each capture. It - * answers the arming question once for the run and then compares each capture against the pre-action - * tree until one proves the replacement, holding that window for the rest of the run. + * compares each capture against the pre-action tree until one proves the replacement, and holds that + * window for the rest of the run. + * + * Arming is split by input lifetime. The pre-action projection is the one arming input that is + * constant for the run, and the only one allowed to rule the confirmation out: a caller that passed + * none has nothing to confirm against and no later capture can supply one. Whether the capture in + * hand proves an iOS XCTest read and carries a complete viewport projection is not constant — the + * first read after a dismissal can still show the dismissed surface, arrive rootless or empty + * mid-animation, or come from another backend than the next poll will. That half is re-asked on + * every capture until one arms; ruling it out on the first capture settled the #1831 + * transitional-tree case on the short window instead of the floor (#3470). * * Arming never consults the requested quiet window: the old `requestedQuietMs < * DEFAULT_STABLE_QUIET_MS` bar is what made `--settle-quiet 499` behave differently from 500 and left @@ -353,9 +362,10 @@ function broadTransitionConfirmation(params: { platform: AgentDeviceRuntime['backend']['platform']; appBundleId: string | undefined; }): StableCaptureConfirmation { - // undefined until the first capture answers it; null once the transition is ruled out, so an - // ordinary local mutation keeps its chrome-free comparison off the cadence. - let transitionBaseline: StableCaptureSignal | null | undefined; + const { baselineNodes, platform, appBundleId } = params; + // Open question, not a ruling: `undefined` is the state a capture that cannot support the + // comparison leaves behind, so arming never depends on which capture happens to land first. + let transitionBaseline: StableCaptureSignal | undefined; // The comparison refuses a node-less side outright, so a mid-animation empty read must not be able // to cancel the floor: without the latch it hands the loop its short window back, and the next // empty read matches it, settling a dismissal that owed the floor. @@ -365,23 +375,17 @@ function broadTransitionConfirmation(params: { // omits `role`, parent links, and >1px-free geometry, all of which the chrome classifier reads, // and one pass costs tens of microseconds against the 300 ms poll cadence. requiredQuietMs(snapshot, currentQuietMs) { - // Not `??=`: that operator re-runs the right side whenever it resolves to `null`, which is - // exactly the ruled-out answer a non-iOS or backend-less run gives, on every single poll. - if (transitionBaseline === undefined) { + if (transitionBaseline === undefined && baselineNodes) { transitionBaseline = broadTransitionBaseline({ - baselineNodes: params.baselineNodes, + baselineNodes, capture: snapshot, - platform: params.platform, - appBundleId: params.appBundleId, + platform, + appBundleId, }); } - if (transitionBaseline === null) return currentQuietMs; + if (!transitionBaseline) return currentQuietMs; if (confirmedQuietMs === undefined) { - const candidate = stableCaptureComparisonSignal( - snapshot, - snapshot.nodes, - params.appBundleId, - ); + const candidate = stableCaptureComparisonSignal(snapshot, snapshot.nodes, appBundleId); if (stableCaptureSignalsHaveBroadReplacement(transitionBaseline, candidate)) { confirmedQuietMs = Math.max(currentQuietMs, BROAD_TRANSITION_CONFIRMATION_QUIET_MS); } @@ -404,22 +408,21 @@ type StableCaptureConfirmation = { }; /** - * The pre-action tree to confirm against, or `null` when the transition in front of the loop is not - * one this confirmation exists for: the chrome-free signal for the pre-action tree, when the capture - * proves an iOS XCTest session (`SnapshotState.backend` is optional at the runtime boundary, so the - * bound platform stays authoritative when a capture omits that provenance; a declared non-XCTest - * backend still wins so this confirmation cannot leak onto another capture implementation) and that - * tree is trustworthy evidence about the screen. Whether the screen was actually replaced stays - * per-capture — the first capture after a dismissal can still show the dismissed surface. + * The pre-action tree to confirm against, or `undefined` when the capture in hand cannot support the + * comparison: the capture must prove an iOS XCTest session (`SnapshotState.backend` is optional at + * the runtime boundary, so the bound platform stays authoritative when a capture omits that + * provenance; a declared non-XCTest backend still wins so this confirmation cannot leak onto another + * capture implementation), and it must be trustworthy evidence about the screen. `undefined` asks the + * next capture, rather than ruling the transition out for the run. */ function broadTransitionBaseline(params: { - baselineNodes: SnapshotNode[] | undefined; + baselineNodes: SnapshotNode[]; capture: Parameters[0]; platform: AgentDeviceRuntime['backend']['platform']; appBundleId: string | undefined; -}): StableCaptureSignal | null { +}): StableCaptureSignal | undefined { const { baselineNodes, capture, platform, appBundleId } = params; - if (!baselineNodes || !isBoundIosXCTestCapture(capture.backend, platform)) return null; + if (!isBoundIosXCTestCapture(capture.backend, platform)) return undefined; // A broad *screen* replacement needs a complete post-action viewport projection. The baseline // comes from settle's authoritative pre-action capture, but a presented iOS modal can omit its // Application root and still be the complete interaction surface. Requiring the root only from @@ -430,7 +433,7 @@ function broadTransitionBaseline(params: { // the viewport root. Treating that rootless replacement as an arbitrary scoped projection lets // a coherent transitional tree win the default 500ms quiet window. const hasTinyPresentedBaseline = baselineNodes.length <= TINY_STABLE_TREE_NODE_COUNT; - if (!hasCompleteViewportProjection && !hasTinyPresentedBaseline) return null; + if (!hasCompleteViewportProjection && !hasTinyPresentedBaseline) return undefined; return stableCaptureComparisonSignal(capture, baselineNodes, appBundleId); } From 2f3c3144fe02a130078d33420252ebabe26b34d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 18:04:48 +0200 Subject: [PATCH 3/6] fix(settle): hold the quiet window in the loop that re-prices it The window helper had one call site and one reader per side of it, so the loop holds the window directly: an unrequested window is re-priced from each capture it performs, a requested one stays fixed. Behaviour is unchanged; the priced value the result reports is the same window. --- .../interaction/runtime/stable-capture.ts | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/src/commands/interaction/runtime/stable-capture.ts b/src/commands/interaction/runtime/stable-capture.ts index 7cc0c7034b..0b5d2489cb 100644 --- a/src/commands/interaction/runtime/stable-capture.ts +++ b/src/commands/interaction/runtime/stable-capture.ts @@ -118,7 +118,13 @@ export async function runStableCaptureLoop( let deadlineMs = start + timeoutMs; let privateAxRecoveryBudgetReset = false; const session = await runtime.sessions.get(options.session ?? 'default'); - const window = createStableCaptureWindow(params.quietMs); + // The quiet window the run holds. An unrequested window is re-priced from each capture the loop + // performs: the read that pays the window is the loop's own, not the one the baseline came from, + // which can be several commands stale and served by another producer, and a mid-run fallback + // changes the price mid-wait. The confirmation applies its own floor on top of the price, so + // re-pricing is never a shorter wait. A window the caller asked for is fixed. + const requestedQuietMs = params.quietMs; + let quietMs = requestedQuietMs ?? DEFAULT_STABLE_QUIET_MS; const confirmation = broadTransitionConfirmation({ baselineNodes: params.broadTransitionBaselineNodes, platform: runtime.backend.platform, @@ -138,7 +144,7 @@ export async function runStableCaptureLoop( waitedMs, captures, nodeCount: pacing.lastNodeCount, - quietMs: window.baseQuietMs, + quietMs, lastCapture, }); while (now(runtime) < deadlineMs) { @@ -151,12 +157,12 @@ export async function runStableCaptureLoop( if (!capture) return result(false, true, now(runtime) - start); captures += 1; lastCapture = capture; - const defaultQuietMs = window.price(capture.snapshot); + if (requestedQuietMs === undefined) quietMs = defaultQuietMsForCapture(capture.snapshot); const signal = stableCaptureSignal(capture.snapshot); // One value carries both: how long this capture must stay quiet, and therefore the cadence that // can measure it. Sampling an extended wait on the caller's tight grid would spend the whole // extension in capture round trips. - const requiredQuietMs = confirmation.requiredQuietMs(capture.snapshot, defaultQuietMs); + const requiredQuietMs = confirmation.requiredQuietMs(capture.snapshot, quietMs); const pollMs = stableCapturePollMs(requiredQuietMs); preferredBackend ??= preferredSnapshotBackendForVerdict(capture.snapshot.snapshotQuality); const nowMs = now(runtime); @@ -200,27 +206,6 @@ export async function runStableCaptureLoop( return result(false, false, now(runtime) - start); } -/** - * The quiet window the run holds. An unrequested window is re-priced from each capture the loop - * performs: the read that pays the window is the loop's own, not the one the baseline came from, - * which can be several commands stale and served by another producer, and a mid-run fallback changes - * the price mid-wait. The confirmation applies its own floor on top of the returned price, so - * re-pricing is never a shorter wait. A window the caller asked for is fixed. - */ -function createStableCaptureWindow(requestedQuietMs: number | undefined) { - let baseQuietMs = requestedQuietMs ?? DEFAULT_STABLE_QUIET_MS; - return { - get baseQuietMs(): number { - return baseQuietMs; - }, - /** Re-prices the window from a capture the loop just completed, and reports that price. */ - price(snapshot: SnapshotState): number { - if (requestedQuietMs === undefined) baseQuietMs = defaultQuietMsForCapture(snapshot); - return baseQuietMs; - }, - }; -} - /** * The once-per-run budget extension a private-AX recovery buys. The recovery re-read the same * surface through another backend, so the quiet window restarts on that capture rather than carrying From 3be3ad5adbd44a7e01679b5e63367e2d5c857d5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 18:50:33 +0200 Subject: [PATCH 4/6] fix(settle): price the quiet window per producer and report it on every path - Price the default quiet window from an exhaustive per-producer table, so a new producer must state its window. - Return capture failures from the stable-capture loop with the window the run priced, so a broken settle reports 300 ms after a bridge read instead of the general default. - Extract the per-capture window step out of runStableCaptureLoop. - Pin the shortened bridge wait on a non-broad fixture, and pin that the deciding capture prices the whole quiet span across a producer switch. - State the confirmation gate and test rationale as invariants. --- .../runtime/settle-default-quiet.test.ts | 53 +++++- src/commands/interaction/runtime/settle.ts | 35 +++- .../runtime/stable-capture.test.ts | 15 +- .../interaction/runtime/stable-capture.ts | 171 ++++++++++-------- 4 files changed, 176 insertions(+), 98 deletions(-) diff --git a/src/commands/interaction/runtime/settle-default-quiet.test.ts b/src/commands/interaction/runtime/settle-default-quiet.test.ts index ee60d3e3b3..48fdc1d4e8 100644 --- a/src/commands/interaction/runtime/settle-default-quiet.test.ts +++ b/src/commands/interaction/runtime/settle-default-quiet.test.ts @@ -24,13 +24,16 @@ function withProvenance

(state: SnapshotState, prov const BRIDGE = { backend: 'xctest', producer: 'simulator-ax-bridge' } as const; const RUNNER = { backend: 'xctest', producer: 'apple-runner' } as const; +type LoopCapturePath = SnapshotProvenance | undefined | ((capture: number) => SnapshotProvenance); + /** * A press whose baseline and whose loop captures come from declared capture paths, so the two can - * be made to agree or to disagree. + * be made to agree or to disagree. The pressed button stays on screen, so no capture is a broad + * replacement and the window under test is the one the loop priced, not the confirmation floor. */ function pressWithCapturePaths(params: { baseline?: SnapshotProvenance; - loop?: SnapshotProvenance; + loop?: LoopCapturePath; settle?: { quietMs?: number; timeoutMs?: number }; }) { const baseline = params.baseline @@ -40,12 +43,11 @@ function pressWithCapturePaths(params: { const device = createSettleDevice({ stored: baseline, captureSnapshot: () => { - captures += 1; // A ref press captures nothing before the action, so every call is the loop's own sample. - const settled = welcomeSnapshot(); + captures += 1; + const provenance = typeof params.loop === 'function' ? params.loop(captures) : params.loop; return { - snapshot: - captures === 1 ? baseline : params.loop ? withProvenance(settled, params.loop) : settled, + snapshot: provenance ? withProvenance(buttonSnapshot(), provenance) : buttonSnapshot(), }; }, }); @@ -59,6 +61,7 @@ test('a settle the simulator snapshot bridge performs uses the shorter default w assert.ok(settle); assert.equal(settle.settled, true); assert.equal(settle.quietMs, 300); + assert.ok(settle.waitedMs < 500, `bridge settle held ${settle.waitedMs}ms`); }); test('an explicit --settle-quiet outranks the per-capture default window', async () => { @@ -85,7 +88,9 @@ for (const [label, provenance] of UNMEASURED_CAPTURES) { const settle = result.settle; assert.ok(settle); + assert.equal(settle.settled, true); assert.equal(settle.quietMs, 500); + assert.ok(settle.waitedMs >= 500, `${label} settled after ${settle.waitedMs}ms`); }); } @@ -97,6 +102,7 @@ test('a bridge read during settle earns the shorter window even from a runner-is const settle = result.settle; assert.ok(settle); assert.equal(settle.quietMs, 300); + assert.ok(settle.waitedMs < 500, `bridge settle held ${settle.waitedMs}ms`); }); test('a runner read during settle keeps the 500ms window despite a bridge-issued baseline', async () => { @@ -107,6 +113,41 @@ test('a runner read during settle keeps the 500ms window despite a bridge-issued const settle = result.settle; assert.ok(settle); assert.equal(settle.quietMs, 500); + assert.ok(settle.waitedMs >= 500, `runner settle held only ${settle.waitedMs}ms`); +}); + +test('the capture that decides the verdict prices the whole quiet span, even after a producer switch', async () => { + // The quiet span is elapsed time over which the tree did not change, whichever producer read it. + // A run that starts on the runner and continues on the bridge settles on the bridge's window. + const result = await pressWithCapturePaths({ + loop: (capture) => (capture === 1 ? RUNNER : BRIDGE), + }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, true); + assert.equal(settle.quietMs, 300); + assert.ok(settle.waitedMs < 500, `producer switch held ${settle.waitedMs}ms`); +}); + +test('a capture that breaks mid-settle reports the window the run had priced', async () => { + let captures = 0; + const device = createSettleDevice({ + stored: buttonSnapshot(), + captureSnapshot: () => { + captures += 1; + if (captures === 1) return { snapshot: withProvenance(buttonSnapshot(), BRIDGE) }; + throw new Error('AX bridge crashed'); + }, + }); + + const result = await device.interactions.press(ref('@e1'), { session: 'default', settle: {} }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.settled, false); + assert.match(settle.hint ?? '', /Settle observation unavailable \(AX bridge crashed\)/); + assert.equal(settle.quietMs, 300); }); test('the shorter default window never shortens a broad-transition confirmation', async () => { diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index aa3893515a..8beefccf19 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -27,6 +27,7 @@ import { DEFAULT_STABLE_QUIET_MS, DEFAULT_STABLE_TIMEOUT_MS, runStableCaptureLoop, + type StableCaptureLoopResult, TINY_STABLE_TREE_HINT, TINY_STABLE_TREE_NODE_COUNT, } from './stable-capture.ts'; @@ -128,7 +129,8 @@ async function settleAfterAction( ): Promise { // The loop resolves an unrequested window from the capture that pays it, so the window is only // known once one is taken. `base` carries the general default for the pre-capture shape the - // observation still has to report; `readSettledOutcome` replaces it with the window the loop ran. + // observation still has to report; once the loop returns, its own window replaces it, including on + // a failure while the outcome is read. const timeoutMs = params.timeoutMs ?? DEFAULT_STABLE_TIMEOUT_MS; const base: SettleObservation = { settled: false, @@ -137,26 +139,39 @@ async function settleAfterAction( quietMs: params.quietMs ?? DEFAULT_STABLE_QUIET_MS, timeoutMs, }; + let outcome: StableCaptureLoopResult | undefined; try { - const outcome = await runStableCaptureLoop(runtime, options, { + outcome = await runStableCaptureLoop(runtime, options, { ...(params.quietMs === undefined ? {} : { quietMs: params.quietMs }), timeoutMs, resetBudgetOnPrivateAxRecovery: true, broadTransitionBaselineNodes: params.baseline.nodes, }); + if (outcome.failure) return settleUnavailable(base, outcome.quietMs, outcome.failure.error); return await readSettledOutcome(runtime, options, params, base, outcome); } catch (error) { - // Never fail the action over the observation: report that settling itself - // broke and let the caller fall back to an explicit snapshot. - return { - observation: { - ...base, - hint: `Settle observation unavailable (${error instanceof Error ? error.message : String(error)}). The action itself succeeded; take a snapshot to observe the result.`, - }, - }; + return settleUnavailable(base, outcome?.quietMs ?? base.quietMs, error); } } +/** + * Never fail the action over the observation: report that settling itself broke, with the window the + * run held, and let the caller fall back to an explicit snapshot. + */ +function settleUnavailable( + base: SettleObservation, + quietMs: number, + error: unknown, +): SettleOutcome { + return { + observation: { + ...base, + quietMs, + hint: `Settle observation unavailable (${error instanceof Error ? error.message : String(error)}). The action itself succeeded; take a snapshot to observe the result.`, + }, + }; +} + /** Turns a finished stable-capture loop into the settled observation payload. */ async function readSettledOutcome( runtime: AgentDeviceRuntime, diff --git a/src/commands/interaction/runtime/stable-capture.test.ts b/src/commands/interaction/runtime/stable-capture.test.ts index 246d38e4ce..e2fc4e15ff 100644 --- a/src/commands/interaction/runtime/stable-capture.test.ts +++ b/src/commands/interaction/runtime/stable-capture.test.ts @@ -327,10 +327,9 @@ test('settle confirms a tiny modal replacement through a partial post-action pro }); test('a dismissal confirmation is not dropped by a later empty projection', async () => { - // The confirmation is a fact about the transition, so it survives a capture that stops qualifying. - // The comparison refuses a node-less side, so re-deciding it per capture hands the loop its short - // window again mid-confirmation — and the next empty read matches it, so two empty projections - // hold a quiet window on a dismissal the floor still owed (#3470). + // A confirmation is a fact about the transition, so once armed it holds the floor for the rest of + // the run: later captures that cannot support the comparison (here, empty projections that match + // each other) never return the run to the caller's shorter window. const { runtime } = transitionRuntime({ settledAtMs: 0, emptyProjectionAtCaptures: [2, 3], @@ -359,10 +358,10 @@ test('a dismissal confirmation is not dropped by a later empty projection', asyn // The two arming inputs that belong to the capture, not to the run: the viewport-root completeness // and the declared capture backend. A dismissal of anything bigger than an alert or sheet (the -// `<= TINY_STABLE_TREE_NODE_COUNT` escape hatch) has to meet BOTH before the confirmation may arm, so -// a mid-animation first read decides nothing — it must leave the question open for the next capture. -// Ruling it out on that first read settled these on the plain window, which after this layer is -// 300 ms on the simulator bridge (#3470). +// `<= TINY_STABLE_TREE_NODE_COUNT` escape hatch) has to meet BOTH before the confirmation may arm, +// and a mid-animation first read that meets neither leaves the question open for the next capture +// rather than settling the run on its plain window. These captures declare no producer, so that plain +// window is the 500 ms default; the 1.5 s floor is what the assertions tell apart. const MID_ANIMATION_FIRST_CAPTURES: Array< [string, NonNullable[0]>] > = [ diff --git a/src/commands/interaction/runtime/stable-capture.ts b/src/commands/interaction/runtime/stable-capture.ts index 0b5d2489cb..05a4bac9f1 100644 --- a/src/commands/interaction/runtime/stable-capture.ts +++ b/src/commands/interaction/runtime/stable-capture.ts @@ -1,6 +1,7 @@ import type { SnapshotNode, SnapshotPreferredBackend, + SnapshotProvenance, SnapshotQualityVerdict, SnapshotState, } from '@agent-device/kernel/snapshot'; @@ -47,29 +48,35 @@ const QUIET_DEADLINE_EPSILON_MS = 2; // Broad replacements (for example, dismissing a modal into a room) can expose a coherent but // transitional AX tree for the default quiet window. Confirm those transitions longer without // adding latency to local mutations whose pre/post trees substantially overlap. -// -// The confirmation is decided by the transition alone: a caller asking for a shorter quiet window -// asked for a tighter verdict, not for a different set of transitions to distrust. It used to arm -// only at `requestedQuietMs >= DEFAULT_STABLE_QUIET_MS`, which made `--settle-quiet 499` behave -// differently from 500 and left `settleQuietMs: 150` callers (tester-army/e2e) with no guard at all -// (#3470). A confirmed replacement is held for the caller's own window, floored at 1.5 s. const BROAD_TRANSITION_CONFIRMATION_QUIET_MS = 1_500; export const DEFAULT_STABLE_QUIET_MS = 500; /** - * The default quiet window a capture earns. The iOS simulator snapshot bridge answers well inside - * the general default, so holding the general window on that read spends a whole extra capture on a - * window its tree had already satisfied. Keyed on the producer, not the channel: the bridge and the - * Apple runner share the `xctest` channel and read at different costs, and only the bridge carries - * the measurement. Every other read — the runner's own tree read, the Android helper, a capture that - * declares no provenance — keeps {@link DEFAULT_STABLE_QUIET_MS}. - * (PR #3522 holds the interleaved before/after medians and the late-diff accuracy check.) + * The default quiet window each producer's capture earns. The iOS simulator snapshot bridge answers + * well inside the general default, so holding the general window on that read spends a whole extra + * capture on a window its tree had already satisfied. Keyed on the producer, not the channel: the + * bridge and the Apple runner share the `xctest` channel and read at different costs, and only the + * bridge carries the measurement. Every producer states its price here, so a new one cannot inherit + * an unmeasured window by omission. (PR #3522 holds the interleaved before/after medians and the + * late-diff accuracy check.) */ +const DEFAULT_QUIET_MS_BY_PRODUCER = { + 'apple-runner': DEFAULT_STABLE_QUIET_MS, + 'simulator-ax-bridge': 300, + 'appium-source': DEFAULT_STABLE_QUIET_MS, + 'limrun-ios-tree': DEFAULT_STABLE_QUIET_MS, + 'android-uiautomator': DEFAULT_STABLE_QUIET_MS, + 'harmonyos-uitest': DEFAULT_STABLE_QUIET_MS, + 'macos-helper': DEFAULT_STABLE_QUIET_MS, + 'linux-atspi': DEFAULT_STABLE_QUIET_MS, + 'agent-browser': DEFAULT_STABLE_QUIET_MS, +} as const satisfies Record; + +/** A capture that declares no producer has no measured price and keeps the general default. */ function defaultQuietMsForCapture(snapshot: SnapshotState): number { - return snapshot.producer === 'simulator-ax-bridge' - ? SIMULATOR_BRIDGE_SETTLE_QUIET_MS - : DEFAULT_STABLE_QUIET_MS; + return snapshot.producer === undefined + ? DEFAULT_STABLE_QUIET_MS + : DEFAULT_QUIET_MS_BY_PRODUCER[snapshot.producer]; } -const SIMULATOR_BRIDGE_SETTLE_QUIET_MS = 300; export const DEFAULT_STABLE_TIMEOUT_MS = 10_000; // Below this node count a settled tree is suspicious: real app surfaces have // more than a handful of accessibility nodes, splash/loading screens do not. @@ -95,6 +102,8 @@ export type StableCaptureLoopResult = { quietMs: number; /** The most recent completed capture, settled or not. */ lastCapture?: CapturedSnapshot; + /** A capture threw; the run ended there, without a verdict. */ + failure?: { error: unknown }; }; export async function runStableCaptureLoop( @@ -118,13 +127,12 @@ export async function runStableCaptureLoop( let deadlineMs = start + timeoutMs; let privateAxRecoveryBudgetReset = false; const session = await runtime.sessions.get(options.session ?? 'default'); - // The quiet window the run holds. An unrequested window is re-priced from each capture the loop - // performs: the read that pays the window is the loop's own, not the one the baseline came from, - // which can be several commands stale and served by another producer, and a mid-run fallback - // changes the price mid-wait. The confirmation applies its own floor on top of the price, so - // re-pricing is never a shorter wait. A window the caller asked for is fixed. - const requestedQuietMs = params.quietMs; - let quietMs = requestedQuietMs ?? DEFAULT_STABLE_QUIET_MS; + // The quiet window the run holds. An unrequested window is priced by the latest capture the loop + // performs, never by the baseline's producer, and applies to the whole quiet span so far: the + // capture that decides the verdict is the read the window prices. A producer switch mid-wait can + // therefore shorten the window; a confirmed broad replacement never drops below its own floor. + // A window the caller asked for is fixed. + let quietMs = params.quietMs ?? DEFAULT_STABLE_QUIET_MS; const confirmation = broadTransitionConfirmation({ baselineNodes: params.broadTransitionBaselineNodes, platform: runtime.backend.platform, @@ -147,23 +155,25 @@ export async function runStableCaptureLoop( quietMs, lastCapture, }); + const unfinished = (read: Exclude) => ({ + ...result(false, read.kind === 'stalled', now(runtime) - start), + ...(read.kind === 'failed' ? { failure: { error: read.error } } : {}), + }); while (now(runtime) < deadlineMs) { - const capture = await captureStableSignalWithinDeadline( + const read = await captureStableSignalWithinDeadline( runtime, options, deadlineMs - now(runtime), preferredBackend, ); - if (!capture) return result(false, true, now(runtime) - start); + if (read.kind !== 'captured') return unfinished(read); + const { capture } = read; captures += 1; lastCapture = capture; - if (requestedQuietMs === undefined) quietMs = defaultQuietMsForCapture(capture.snapshot); + const window = stableCaptureWindow(capture.snapshot, params.quietMs, confirmation); + quietMs = window.quietMs; + const { requiredQuietMs, pollMs } = window; const signal = stableCaptureSignal(capture.snapshot); - // One value carries both: how long this capture must stay quiet, and therefore the cadence that - // can measure it. Sampling an extended wait on the caller's tight grid would spend the whole - // extension in capture round trips. - const requiredQuietMs = confirmation.requiredQuietMs(capture.snapshot, quietMs); - const pollMs = stableCapturePollMs(requiredQuietMs); preferredBackend ??= preferredSnapshotBackendForVerdict(capture.snapshot.snapshotQuality); const nowMs = now(runtime); const nodeCount = capture.snapshot.nodes.length; @@ -186,26 +196,32 @@ export async function runStableCaptureLoop( if (recovered) { privateAxRecoveryBudgetReset = true; deadlineMs = recovered.deadlineMs; - if (recovered.delayMs <= 0) break; - await sleep(runtime, recovered.delayMs); - continue; } - const observed = pacing.observe({ - signal, - nodeCount, - nowMs, - requiredQuietMs, - pollMs, - deadlineMs, - captures, - }); - if (observed.settled) return result(true, false, nowMs - start); - if (observed.delayMs <= 0) break; - await sleep(runtime, observed.delayMs); + const step = recovered + ? { settled: false, delayMs: recovered.delayMs } + : pacing.observe({ signal, nodeCount, nowMs, requiredQuietMs, pollMs, deadlineMs, captures }); + if (step.settled) return result(true, false, nowMs - start); + if (step.delayMs <= 0) break; + await sleep(runtime, step.delayMs); } return result(false, false, now(runtime) - start); } +/** + * The quiet window one capture prices, the window it must actually hold once the broad-transition + * confirmation has had its say, and the cadence that can measure that. Sampling an extended wait on + * the caller's tight grid would spend the whole extension in capture round trips. + */ +function stableCaptureWindow( + snapshot: SnapshotState, + requestedQuietMs: number | undefined, + confirmation: StableCaptureConfirmation, +): { quietMs: number; requiredQuietMs: number; pollMs: number } { + const quietMs = requestedQuietMs ?? defaultQuietMsForCapture(snapshot); + const requiredQuietMs = confirmation.requiredQuietMs(snapshot, quietMs); + return { quietMs, requiredQuietMs, pollMs: stableCapturePollMs(requiredQuietMs) }; +} + /** * The once-per-run budget extension a private-AX recovery buys. The recovery re-read the same * surface through another backend, so the quiet window restarts on that capture rather than carrying @@ -327,20 +343,18 @@ function createStableCapturePacing(start: number) { * compares each capture against the pre-action tree until one proves the replacement, and holds that * window for the rest of the run. * - * Arming is split by input lifetime. The pre-action projection is the one arming input that is - * constant for the run, and the only one allowed to rule the confirmation out: a caller that passed - * none has nothing to confirm against and no later capture can supply one. Whether the capture in - * hand proves an iOS XCTest read and carries a complete viewport projection is not constant — the - * first read after a dismissal can still show the dismissed surface, arrive rootless or empty - * mid-animation, or come from another backend than the next poll will. That half is re-asked on - * every capture until one arms; ruling it out on the first capture settled the #1831 - * transitional-tree case on the short window instead of the floor (#3470). + * Two gates stand before the comparison. The pre-action projection is constant for the run and the + * only input allowed to rule the confirmation out: a caller that passed none has nothing to confirm + * against. The capture gate — the capture proves an iOS XCTest read and carries a complete viewport + * projection — belongs to each capture, because the first read after a dismissal can still show the + * dismissed surface, arrive rootless or empty mid-animation, or come from another backend than the + * next poll will. A capture that fails it leaves the question open; the first capture that passes + * fixes the comparison baseline, and every later capture is compared against that baseline until one + * proves the replacement. * - * Arming never consults the requested quiet window: the old `requestedQuietMs < - * DEFAULT_STABLE_QUIET_MS` bar is what made `--settle-quiet 499` behave differently from 500 and left - * `settleQuietMs: 150` callers without the guard at all. A confirmed replacement is held for at least - * the caller's own window, floored at 1.5 s: `--settle-quiet` asks for a tighter verdict, not for a - * different set of transitions to distrust (#3470). + * Arming never consults the requested quiet window: `--settle-quiet` asks for a tighter verdict, not + * for a different set of transitions to distrust. A confirmed replacement is held for at least the + * caller's own window, floored at 1.5 s. */ function broadTransitionConfirmation(params: { baselineNodes: SnapshotNode[] | undefined; @@ -521,7 +535,7 @@ function stableCaptureDelayMs(params: { // subsequent ref/get/find lookups against the same session. (`--settle` DOES // store its final capture, but explicitly, in one place, after the loop.) // -// Resolves undefined when the capture does not return within remainingMs. A +// Resolves `stalled` when the capture does not return within remainingMs. A // stalled backend capture (observed with macOS AX captures) must not push the // stable wait past the user-supplied timeout into the daemon request timeout. // The deadline cancels and joins the capture before returning so no late capture @@ -531,17 +545,26 @@ async function captureStableSignalWithinDeadline( options: CommandContext & SelectorSnapshotOptions, remainingMs: number, preferredBackend?: SnapshotPreferredBackend, -): Promise { - const result = await runWithinWaitDeadline(runtime, options, remainingMs, async (signal) => { - return await captureSelectorSnapshot( - runtime, - { ...options, signal }, - { - updateSession: false, - interactiveOnly: true, - preferredBackend, - }, - ); - }); - return result.timedOut ? undefined : result.value; +): Promise { + try { + const result = await runWithinWaitDeadline(runtime, options, remainingMs, async (signal) => { + return await captureSelectorSnapshot( + runtime, + { ...options, signal }, + { + updateSession: false, + interactiveOnly: true, + preferredBackend, + }, + ); + }); + return result.timedOut ? { kind: 'stalled' } : { kind: 'captured', capture: result.value }; + } catch (error) { + return { kind: 'failed', error }; + } } + +type StableCaptureRead = + | { kind: 'captured'; capture: CapturedSnapshot } + | { kind: 'stalled' } + | { kind: 'failed'; error: unknown }; From c09f627d5eac8cae982f073573fa21294701d066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 18:59:22 +0200 Subject: [PATCH 5/6] fix(wait): surface a stable-capture failure from wait stable The stable-capture loop returns a capture failure instead of throwing it. `wait stable` now rethrows that failure rather than mapping it to a stable timeout. The arming-input test comment now says a mid-animation read that misses either input leaves arming open. --- .../interaction/runtime/selector-wait.ts | 1 + .../runtime/stable-capture.test.ts | 4 +-- .../interaction/runtime/wait-stable.test.ts | 32 +++++++++++++++++++ .../interaction/runtime/wait-stable.ts | 1 + 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/commands/interaction/runtime/selector-wait.ts b/src/commands/interaction/runtime/selector-wait.ts index aeb3b9c3db..4b81e00798 100644 --- a/src/commands/interaction/runtime/selector-wait.ts +++ b/src/commands/interaction/runtime/selector-wait.ts @@ -97,6 +97,7 @@ type StableCaptureResult = { waitedMs: number; captures: number; nodeCount: number; + failure?: { error: unknown }; }; export type SelectorWaitOperations = { diff --git a/src/commands/interaction/runtime/stable-capture.test.ts b/src/commands/interaction/runtime/stable-capture.test.ts index e2fc4e15ff..88594c397a 100644 --- a/src/commands/interaction/runtime/stable-capture.test.ts +++ b/src/commands/interaction/runtime/stable-capture.test.ts @@ -359,8 +359,8 @@ test('a dismissal confirmation is not dropped by a later empty projection', asyn // The two arming inputs that belong to the capture, not to the run: the viewport-root completeness // and the declared capture backend. A dismissal of anything bigger than an alert or sheet (the // `<= TINY_STABLE_TREE_NODE_COUNT` escape hatch) has to meet BOTH before the confirmation may arm, -// and a mid-animation first read that meets neither leaves the question open for the next capture -// rather than settling the run on its plain window. These captures declare no producer, so that plain +// and a mid-animation first read that misses either one leaves the question open for the next +// capture rather than settling the run on its plain window. These captures declare no producer, so that plain // window is the 500 ms default; the 1.5 s floor is what the assertions tell apart. const MID_ANIMATION_FIRST_CAPTURES: Array< [string, NonNullable[0]>] diff --git a/src/commands/interaction/runtime/wait-stable.test.ts b/src/commands/interaction/runtime/wait-stable.test.ts index 9e55cae284..e59185691b 100644 --- a/src/commands/interaction/runtime/wait-stable.test.ts +++ b/src/commands/interaction/runtime/wait-stable.test.ts @@ -334,6 +334,38 @@ test('runtime wait stable times out with capture stats when never settling', asy assert.ok(captures > 1); }); +test('runtime wait stable surfaces a capture failure instead of reporting a timeout', async () => { + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => { + throw new Error('AX bridge crashed'); + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default' }]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); + + await assert.rejects( + () => + device.selectors.wait({ + session: 'default', + target: { kind: 'stable', quietMs: 500, timeoutMs: 1_000 }, + }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.notEqual( + (error as { details?: { reason?: string } }).details?.reason, + 'wait_stable_timeout', + ); + assert.match(error.message, /AX bridge crashed/); + return true; + }, + ); +}); + test('runtime wait stable times out when a backend capture stalls past the wait budget', async () => { const clock = createFakeClock(); let captureAborted = false; diff --git a/src/commands/interaction/runtime/wait-stable.ts b/src/commands/interaction/runtime/wait-stable.ts index 15240a7928..d16acc6ae5 100644 --- a/src/commands/interaction/runtime/wait-stable.ts +++ b/src/commands/interaction/runtime/wait-stable.ts @@ -21,6 +21,7 @@ export async function waitForStable( quietMs: quiet, timeoutMs: timeout, }); + if (outcome.failure) throw outcome.failure.error; if (!outcome.settled) { throw new AppError('COMMAND_FAILED', 'wait timed out waiting for a stable UI', { reason: WAIT_REASONS.stableTimeout, From 40d86823361571925a20a6838b08dbfc50d97b20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 19:04:50 +0200 Subject: [PATCH 6/6] test(settle): rewrap the arming-input comment to the column width --- src/commands/interaction/runtime/stable-capture.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/interaction/runtime/stable-capture.test.ts b/src/commands/interaction/runtime/stable-capture.test.ts index 88594c397a..31d2fceac2 100644 --- a/src/commands/interaction/runtime/stable-capture.test.ts +++ b/src/commands/interaction/runtime/stable-capture.test.ts @@ -360,8 +360,8 @@ test('a dismissal confirmation is not dropped by a later empty projection', asyn // and the declared capture backend. A dismissal of anything bigger than an alert or sheet (the // `<= TINY_STABLE_TREE_NODE_COUNT` escape hatch) has to meet BOTH before the confirmation may arm, // and a mid-animation first read that misses either one leaves the question open for the next -// capture rather than settling the run on its plain window. These captures declare no producer, so that plain -// window is the 500 ms default; the 1.5 s floor is what the assertions tell apart. +// capture rather than settling the run on its plain window. These captures declare no producer, so +// that plain window is the 500 ms default; the 1.5 s floor is what the assertions tell apart. const MID_ANIMATION_FIRST_CAPTURES: Array< [string, NonNullable[0]>] > = [