diff --git a/.github/actions/smoke-ios-run/action.yml b/.github/actions/smoke-ios-run/action.yml index 34830a6a14..c11a117407 100644 --- a/.github/actions/smoke-ios-run/action.yml +++ b/.github/actions/smoke-ios-run/action.yml @@ -47,7 +47,7 @@ runs: with: runtime-version: ${{ env.IOS_RUNTIME_VERSION }} preferred-device-name: iPhone 17 Pro - - name: Run targeted iOS runner XCTest regressions + - name: Run iOS runner platform checks id: ios-xctest if: steps.xctest-impact.outputs.run != 'false' run: | @@ -66,13 +66,11 @@ runs: -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ -resultBundlePath "$GITHUB_WORKSPACE/test/artifacts/ios-runner-regressions.xcresult" \ "${IOS_PR_TEST_ARGS[@]}" 2>&1 | tee /tmp/agent-device-runner-regressions.log - node --input-type=module -e ' - import { readFileSync } from "node:fs"; - const log = readFileSync("/tmp/agent-device-runner-regressions.log", "utf8"); - if (!/\] AGENT_DEVICE_RUNNER_OPTIONAL_PROBE_WAIT_COMPLETED$/m.test(log)) { - throw new Error("Optional observation ended the runner test before its wait completed"); - } - ' + xcrun xcresulttool get test-results summary \ + --path "$GITHUB_WORKSPACE/test/artifacts/ios-runner-regressions.xcresult" \ + --compact > "$GITHUB_WORKSPACE/test/artifacts/ios-runner-regressions-summary.json" + RESULT_SUMMARY_PATH="$GITHUB_WORKSPACE/test/artifacts/ios-runner-regressions-summary.json" \ + XCTEST_LANE=pr node --experimental-strip-types scripts/xctest-run-summary.ts shell: bash - name: Preflight iOS runner through public CLI run: | diff --git a/.github/workflows/xctest-nightly.yml b/.github/workflows/xctest-nightly.yml index 3c20c78b86..74f1299f36 100644 --- a/.github/workflows/xctest-nightly.yml +++ b/.github/workflows/xctest-nightly.yml @@ -1,50 +1,9 @@ name: XCTest Nightly -# The full iOS runner XCTest suite (#1781 A7). The iOS PR phase in apple.yml selects a source-derived -# subset of the target's methods in an `-only-testing:` list; everything outside that list ran -# nowhere at all. This lane drops the filter and runs the test plan whole, skipping only -# `testCommand` — which is the runner's server entry point rather than a test (see the step -# below). -# -# No count is quoted here on purpose. `pnpm check:xctest-selection` prints the live split -# (declared / PR-selected / skipped / nightly-only) and is the only place those numbers are -# derived rather than remembered. A hand-copied figure in a comment is the same class of -# defect this lane exists to close: between this file being written and first review, the -# suite gained three methods and the PR list gained five entries, so every number originally -# written here was already wrong. -# -# Under the lane rule in #1781: -# -# - Catches: regressions in the simulator-only tests (guard `… && os(iOS)`: they launch -# the host app, route through SpringBoard, or assert an iOS-only branch) that apple.yml's -# PR list leaves out. Since the A7 classification the pure-decision tests also run on -# the macOS phase on every PR (apple.yml), so this lane's -# unique reach is the simulator-only remainder — `pnpm check:xctest-selection` prints -# both numbers. `testCommand` is skipped, and the tests guarded to macOS/tvOS only -# cannot execute on this lane's iOS Simulator destination (the host lane runs them). -# - Evidence: the first execution found #1812 in the dark set (2 of 155 red, both real); -# the other ~114 never-run tests all passed. -# - Cost: this is a public repository, so nothing here is billed. The real cost is one macOS -# concurrency slot per night. The Apple runner build shares apple.yml's cache (identical -# inputs and build variant), but scheduled runs only see DEFAULT-BRANCH caches, so the hit -# depends on apple.yml's `push: main` run having populated the cache at the same source -# hash — on a miss this lane also pays a full runner build. Adds nothing to any PR. -# - Kill criterion: what ONLY this lane reaches is 11 tests / ~42 s of simulator time -# (measured from the first night's xcresult) — the app-launching, SpringBoard-probing -# class. Once apple.yml runs the whole iOS-compiled bundle instead of its selected -# list — the follow-up stacked on the classification — every test reaches a PR lane and -# this lane goes. That trade is not free: those 11 are the flakiest tests in the suite, -# and they would land on a blocking 15-minute gate that is already cancelled ~19% of the -# time, so measure their flake rate here before moving them. -# -# Scheduled + manual only. It is deliberately NOT a PR gate: apple.yml is already over half of -# this repo's CI spend (#1781 A9), and moving the unrun set onto it would make that worse for -# tests whose flakiness on a shared runner is unknown. -# -# Note on `concurrency` below: it is keyed on `github.ref`, so a manual dispatch on the same -# ref cancels an in-flight scheduled run. That is the intended trade — an operator asking for -# a run wants the fresh one — but it means "dispatch while the nightly is running" loses the -# nightly's result rather than queueing behind it. +# Run all native iOS regressions nightly and on manual dispatch. PRs keep the host +# decision suite, shared platform-branch checks, and public CLI E2E smoke. +# Native fault injection and identity checks remain here; healthy E2E does not +# replace those contracts. The source-derived count guards against a zero-test run. on: schedule: @@ -63,13 +22,6 @@ jobs: full-suite: name: iOS Runner Full XCTest Suite runs-on: macos-26 - # Measured, not guessed. A full local run of this exact command on matching toolchain - # (Xcode 26.2, iOS 26.2 Simulator) executed 154 tests in 120s wall — the suite itself is - # ~2 minutes, not the bulk of the job. What dominates the budget is the runner build on a - # derived-data cache MISS, plus checkout and simulator boot. the iOS phase in apple.yml has an 80-minute limit - # while doing strictly more than this lane (same build and boot, plus replays, prepare, - # and the fixture E2E), so half of that is generous headroom for a cache-miss night and - # still caps a wedged run at 45 minutes instead of two hours of held macOS slot. timeout-minutes: 45 env: # Must match apple.yml. The runtime version rides in the derived-data cache key, and @@ -141,7 +93,14 @@ jobs: -xctestrun "$XCTESTRUN_PATH" \ -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ -skip-testing:AgentDeviceRunnerUITests/RunnerTests/testCommand \ - -resultBundlePath "$RESULT_BUNDLE_PATH" + -resultBundlePath "$RESULT_BUNDLE_PATH" 2>&1 | tee .tmp/xctest-nightly/runner.log + node --input-type=module -e ' + import { readFileSync } from "node:fs"; + const log = readFileSync(".tmp/xctest-nightly/runner.log", "utf8"); + if (!/\] AGENT_DEVICE_RUNNER_OPTIONAL_PROBE_WAIT_COMPLETED$/m.test(log)) { + throw new Error("Optional observation ended the runner test before its wait completed"); + } + ' # Best-effort and never the job's verdict on its own; the step below is what asserts. # `--compact` first because a red night's summary is the large one, and the job summary @@ -166,7 +125,7 @@ jobs: # guards (XCTEST_LANE above), so a guard that compiles a file of tests out of iOS reads # as red too. The failure list is capped so a badly red night cannot blow the 1 MiB # job-summary limit and lose the headline with it. - - name: Report the run and assert it executed the source-derived set + - name: Report the run and assert the source-derived count if: always() run: | set -euo pipefail diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index d2f63502d6..b334d7301d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -156,10 +156,11 @@ final class RunnerTests: XCTestCase { // `-D AGENT_DEVICE_RUNNER_UNIT_TESTS` and are classified by their `#if` guard (#1781 A7): // - `#if AGENT_DEVICE_RUNNER_UNIT_TESTS` alone: a pure runner decision (rule table, // geometry, parser, policy, journal, dispatch bookkeeping) that needs no launched app. - // Runs on the macOS host lane on every PR (ci.yml, no simulator) and on the iOS lanes. + // Runs on the macOS host lane on every PR and on the full iOS nightly lane. // - `… && os(iOS)` (or a nested `#if os(iOS)`): runner/XCTest semantics — launches the // host app, routes through SpringBoard, swizzles XCUIApplication, or asserts an - // iOS-only branch. Simulator lanes only (ios.yml PR list, xctest-nightly.yml). + // iOS-only branch. Full simulator coverage runs nightly (xctest-nightly.yml). + // - Shared methods with platform-dependent bodies also run on the PR simulator lane. // `pnpm check:xctest-selection` derives each lane's reachable set from these guards and // fails when a declared test is reachable by no lane, so a test gated to a platform nothing // runs (the old tvOS-only pair) cannot go dark silently. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/FixtureExistence.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/FixtureExistence.swift new file mode 100644 index 0000000000..eaa3b6f7b7 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/FixtureExistence.swift @@ -0,0 +1,9 @@ +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +import XCTest + +extension XCUIElement { + func waitForFixtureExistence(timeout: TimeInterval) -> Bool { + exists || waitForExistence(timeout: timeout) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift index f8862aa6d5..ad3dea5012 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift @@ -131,7 +131,7 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) // Resolution is a pure read; a contended hosted simulator can spend far more than the 10 s // command timeout on it, so it runs under the sibling fixtures' resolution allowance. let alert = try XCTUnwrap( @@ -160,7 +160,7 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) let alert = try XCTUnwrap( resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.alertResolutionAllowance)) ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift index eb2abe5914..87b7bdae2f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift @@ -5,6 +5,7 @@ import ObjectiveC.runtime private final class AlertTapDeadlineStub: NSObject { static var status: RunnerTapSynthesisStatus? + static var interruptionHandlingDisabled: Bool? @objc(synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:) class func synthesizeTap( @@ -15,7 +16,8 @@ private final class AlertTapDeadlineStub: NSObject { deadline: NSDate?, errorMessage: AutoreleasingUnsafeMutablePointer? ) -> RunnerTapSynthesisStatus { - status ?? (deadline == nil ? .succeeded : .deadlineExceeded) + interruptionHandlingDisabled = (application.value(forKey: "doesNotHandleUIInterruptions") as? NSNumber)?.boolValue + return status ?? (deadline == nil ? .succeeded : .deadlineExceeded) } } #endif @@ -49,7 +51,7 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) let alert = try resolveAlertBeforeTheCommand() let response = handleAlert(alert, action: "accept", deadline: .distantPast) XCTAssertFalse(response.ok) @@ -67,7 +69,7 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) let alert = try resolveAlertBeforeTheCommand() alertButtonHittabilityProbeOverrideForTesting = { probeDeadline in while Date() < probeDeadline { @@ -90,7 +92,7 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) let alert = try resolveAlertBeforeTheCommand() let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) let frame = button.frame @@ -110,7 +112,7 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) let alert = try resolveAlertBeforeTheCommand() let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) let frame = button.frame @@ -121,6 +123,7 @@ extension RunnerTests { method_setImplementation(method, method_getImplementation(stub)) defer { AlertTapDeadlineStub.status = nil + AlertTapDeadlineStub.interruptionHandlingDisabled = nil method_setImplementation(method, original) } @@ -150,7 +153,7 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) let alert = try resolveAlertBeforeTheCommand() let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) @@ -165,7 +168,7 @@ extension RunnerTests { } @MainActor - func testAlertActivationDoesNotConsultInterruptionHandlingWithANotificationBanner() throws { + func testAlertActivationConfirmsBeforeDeadlineWithoutHandlingTheNotificationBanner() throws { try withNotificationBannerAlert { alert in var consultedInterruptions: [String] = [] let monitor = addUIInterruptionMonitor(withDescription: "alert activation banner") { element in @@ -174,24 +177,41 @@ extension RunnerTests { } defer { removeUIInterruptionMonitor(monitor) } - _ = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) - - XCTAssertEqual(consultedInterruptions, [], "alert activation consulted XCTest's interruption handling") - } - } - - @MainActor - func testAlertActivationConfirmsBeforeDeadlineWithANotificationBanner() throws { - try withNotificationBannerAlert { alert in + try assertAlertActivationDisablesInterruptionHandling(alert) let deadline = Date().addingTimeInterval(RunnerTests.alertActivationDeadline) let response = handleAlert(alert, action: "accept", deadline: deadline) + XCTAssertEqual(consultedInterruptions, [], "alert activation consulted XCTest's interruption handling") XCTAssertTrue(response.ok, String(describing: response.error)) XCTAssertLessThan(Date(), deadline) XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") } } + @MainActor + private func assertAlertActivationDisablesInterruptionHandling(_ alert: RunnerAlert) throws { + let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) + let frame = button.frame + let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:") + let method = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector)) + let stub = try XCTUnwrap(class_getClassMethod(AlertTapDeadlineStub.self, selector)) + let original = method_getImplementation(method) + AlertTapDeadlineStub.status = .deadlineExceeded + AlertTapDeadlineStub.interruptionHandlingDisabled = nil + method_setImplementation(method, method_getImplementation(stub)) + defer { + AlertTapDeadlineStub.status = nil + AlertTapDeadlineStub.interruptionHandlingDisabled = nil + method_setImplementation(method, original) + } + + let outcome = activateAlertButton( + alert, button: button, action: "accept", frame: frame, deadline: .distantFuture + ) + XCTAssertNil(outcome) + XCTAssertEqual(AlertTapDeadlineStub.interruptionHandlingDisabled, true) + } + @MainActor private func withNotificationBannerAlert(_ assertions: (RunnerAlert) throws -> Void) throws { app.launchArguments = ["--agent-device-alert-replacement-regression", "--agent-device-alert-banner"] @@ -204,7 +224,7 @@ extension RunnerTests { _ = banner.waitForNonExistence(timeout: 15) } try acceptNotificationAuthorizationUntilAlertAppears() - XCTAssertTrue(banner.waitForExistence(timeout: appExistenceTimeout), "the fixture keeps a banner up") + XCTAssertTrue(banner.waitForFixtureExistence(timeout: appExistenceTimeout), "the fixture keeps a banner up") let alert = try resolveAlertBeforeTheCommand() try assertions(alert) @@ -255,7 +275,7 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) let original = try resolveAlertBeforeTheCommand() alertResolutionOverrideForTesting = { _ in original } let timeoutMs = Int(RunnerTests.alertActivationDeadline * 1000) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift index 6f4b102237..168f510b98 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift @@ -87,7 +87,7 @@ extension RunnerTests { private func launchCrowdedScreen(extraArguments: [String]) { app.launchArguments = ["--agent-device-crowded-screen"] + extraArguments app.launch() - XCTAssertTrue(app.staticTexts["agent-device-crowded-row-499"].waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.staticTexts["agent-device-crowded-row-499"].waitForFixtureExistence(timeout: appExistenceTimeout)) } @MainActor diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift index e98cb87986..bf1a5121eb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift @@ -304,50 +304,60 @@ extension RunnerTests { } @MainActor - func testSkipAppActivationPreflightIncludesForegroundCachedCoordinateOnlyTaps() throws { - app.launch() - mainOwned.app = app - mainOwned.bundleId = nil - defer { - mainOwned.app = nil - mainOwned.bundleId = nil - app.terminate() - } - let tap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# - ) - - XCTAssertTrue(shouldSkipAppActivationPreflight(tap)) - } - - @MainActor - func testSkipAppActivationPreflightRejectsMissingChangedAndBackgroundTargets() throws { + func testSkipAppActivationPreflightRequiresCachedForegroundCoordinateOnlyTap() throws { let coordinateTap = try runnerCommandFixture( #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# ) + let scroll = try runnerCommandFixture( + #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# + ) mainOwned.app = nil mainOwned.bundleId = nil - XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) + XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap), "a tap needs a cached target") + XCTAssertFalse(shouldSkipAppActivationPreflight(scroll), "a scroll needs a cached target") app.launch() mainOwned.app = app - mainOwned.bundleId = "com.example.current" defer { mainOwned.app = nil mainOwned.bundleId = nil app.terminate() } + XCTAssertEqual(app.state, .runningForeground) + XCTAssertTrue(shouldSkipAppActivationPreflight(coordinateTap)) + + let guardedCommands: [(String, String)] = [ + ("selector tap", #"{"command":"tap","commandId":"selector-tap","selectorKey":"label","selectorValue":"Search","synthesized":true}"#), + ("mixed sequence", """ + {"command":"sequence","commandId":"mixed-sequence","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"doubleTap","x":30,"y":40} + ]} + """), + ("drag", #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"#), + ("scroll", #"{"command":"scroll","commandId":"scroll-2","direction":"down","pixels":400}"#), + ("sequence", """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"longPress","x":10,"y":200,"durationMs":300} + ]} + """) + ] + for (name, json) in guardedCommands { + let command = try runnerCommandFixture(json) + XCTAssertFalse(shouldSkipAppActivationPreflight(command), name) + } + + mainOwned.bundleId = "com.example.current" let changedBundleTap = try runnerCommandFixture( #"{"command":"tap","commandId":"tap-2","appBundleId":"com.example.other","x":10,"y":20}"# ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(changedBundleTap)) + XCTAssertFalse(shouldSkipAppActivationPreflight(changedBundleTap), "a different bundle needs activation") app.terminate() - mainOwned.app = app mainOwned.bundleId = nil - - XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) + XCTAssertEqual(app.state, .notRunning) + XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap), "a terminated cached target needs activation") } @MainActor @@ -536,80 +546,6 @@ extension RunnerTests { XCTAssertFalse(hasAbandonedMainThreadWork()) } - @MainActor - func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws { - app.launch() - mainOwned.app = app - mainOwned.bundleId = nil - defer { - mainOwned.app = nil - mainOwned.bundleId = nil - app.terminate() - } - let selectorTap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","selectorKey":"label","selectorValue":"Search","synthesized":true}"# - ) - let standardDrag = try runnerCommandFixture( - #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# - ) - let mixedSequence = try runnerCommandFixture( - """ - {"command":"sequence","commandId":"seq-1","steps":[ - {"kind":"tap","x":10,"y":20,"synthesized":true}, - {"kind":"doubleTap","x":30,"y":40} - ]} - """ - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(selectorTap)) - XCTAssertFalse(shouldSkipAppActivationPreflight(standardDrag)) - XCTAssertFalse(shouldSkipAppActivationPreflight(mixedSequence)) - } - - // Launches nothing, but still simulator-only: `shouldSkipAppActivationPreflight` is - // `#if os(iOS) …guards… #else return false #endif`, so on macOS this asserts a compile-time - // literal and no edit to the iOS body could make it red. Its five siblings above and below - // are gated for the same reason. - @MainActor - func testSkipAppActivationPreflightRequiresCachedForegroundTarget() throws { - mainOwned.app = nil - mainOwned.bundleId = nil - let scroll = try runnerCommandFixture( - #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) - } - - @MainActor - func testSkipAppActivationPreflightKeepsDragScrollAndSequenceOnForegroundGuard() throws { - app.launch() - mainOwned.app = app - mainOwned.bundleId = nil - defer { - mainOwned.app = nil - mainOwned.bundleId = nil - app.terminate() - } - let drag = try runnerCommandFixture( - #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# - ) - let scroll = try runnerCommandFixture( - #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# - ) - let sequence = try runnerCommandFixture( - """ - {"command":"sequence","commandId":"seq-1","steps":[ - {"kind":"tap","x":10,"y":20,"synthesized":true}, - {"kind":"longPress","x":10,"y":200,"durationMs":300} - ]} - """ - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(drag)) - XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) - XCTAssertFalse(shouldSkipAppActivationPreflight(sequence)) - } #endif func testDispatchReturnsBusyBeforeQueueingMainThreadWork() throws { @@ -706,7 +642,7 @@ extension RunnerTests { private func startMacTargetAndLoseTheForeground() throws -> XCUIApplication { let target = XCUIApplication(bundleIdentifier: Self.macBackgroundTargetBundleId) target.launch() - XCTAssertTrue(target.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(target.waitForFixtureExistence(timeout: appExistenceTimeout)) _ = XCUIApplication(bundleIdentifier: Self.macForegroundStealBundleId).activate() XCTAssertTrue( target.wait(for: .runningBackground, timeout: 10), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift index b06f589e04..7786296fa5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift @@ -4,6 +4,10 @@ import AgentDeviceSnapshotPresentation #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) import ObjectiveC.runtime +private final class RunnerCoordinateTapObserver { + var count = 0 +} + private final class RunnerSynthesizedTapFailureStub: NSObject { @objc(synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:) class func synthesizeTap( @@ -20,6 +24,27 @@ private final class RunnerSynthesizedTapFailureStub: NSObject { } extension RunnerTests { + private func observeCoordinateTaps(droppingFirst: Bool = false) throws -> ( + observer: RunnerCoordinateTapObserver, restore: () -> Void + ) { + let selector = NSSelectorFromString("tap") + let method = try XCTUnwrap(class_getInstanceMethod(XCUICoordinate.self, selector)) + let original = method_getImplementation(method) + let observer = RunnerCoordinateTapObserver() + let block: @convention(block) (AnyObject) -> Void = { coordinate in + observer.count += 1 + if droppingFirst && observer.count == 1 { return } + typealias Tap = @convention(c) (AnyObject, Selector) -> Void + unsafeBitCast(original, to: Tap.self)(coordinate, selector) + } + let replacement = imp_implementationWithBlock(block) + method_setImplementation(method, replacement) + return (observer, { + method_setImplementation(method, original) + imp_removeBlock(replacement) + }) + } + /// Makes private tap synthesis fail until the returned closure restores it. func forceSynthesizedTapFailure() throws -> () -> Void { let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:") @@ -81,12 +106,12 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(textField.waitForFixtureExistence(timeout: appExistenceTimeout)) let otherElement = app.staticTexts["Agent Device Runner"] - XCTAssertTrue(otherElement.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(otherElement.waitForFixtureExistence(timeout: appExistenceTimeout)) XCTAssertFalse( keyboardFocusConfirmed(app: app, element: textField), "an untapped field must not confirm focus, or the fallback would fire immediately" @@ -95,8 +120,11 @@ extension RunnerTests { let tapCommand = try runnerCommandFixture( #"{"command":"tap","commandId":"tap-focus-confirmation","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# ) + let taps = try observeCoordinateTaps() + defer { taps.restore() } let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) + XCTAssertEqual(taps.observer.count, 1, "confirmed focus must not dispatch a second tap") try XCTSkipIf( isKeyboardVisible(app: app), "software keyboard is up: this simulator cannot exercise the hidden-keyboard responder path" @@ -115,6 +143,30 @@ extension RunnerTests { ) } + @MainActor + func testTextInputTapRecoversWhenItsFirstTapDoesNotTakeFocus() throws { + app.launchArguments = ["--agent-device-text-entry-regression"] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let field = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) + XCTAssertFalse(keyboardFocusConfirmed(app: app, element: field)) + let taps = try observeCoordinateTaps(droppingFirst: true) + defer { taps.restore() } + let failures = currentXCTestFailureCount() + let command = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-recovery-focus","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + + XCTAssertTrue(try executeOnMainPrepared(command: command, activeApp: app).ok) + XCTAssertEqual(taps.observer.count, 2) + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: field)) + XCTAssertFalse(didRecordXCTestFailure(since: failures)) + } + // #3060: a Flutter password field reports `TextField` through the legacy accessibility attributes and // `Other` through the modern ones once it takes focus, so it stops answering the type-bound query that // resolved it. The runner classified its resolved element AFTER dispatching, which re-ran that query, @@ -144,9 +196,9 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let field = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame mainOwned.app = app mainOwned.bundleId = "com.callstack.agentdevice.runner" @@ -191,9 +243,9 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) XCTAssertTrue( - app.textFields["agent-device-hardware-keyboard-input"].waitForExistence(timeout: appExistenceTimeout) + app.textFields["agent-device-hardware-keyboard-input"].waitForFixtureExistence(timeout: appExistenceTimeout) ) mainOwned.app = app mainOwned.bundleId = "com.callstack.agentdevice.runner" @@ -234,9 +286,9 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let field = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) // The fixture keeps its keyboard down with an empty inputView, but a simulator with the software // keyboard forced on takes the fill's keyboard-visible branch, which has different resolution. // That is an environment fact, so the test that pins the hidden-keyboard branch skips on it. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index eb8c68072d..a4a83c3df1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -31,12 +31,12 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(textField.waitForFixtureExistence(timeout: appExistenceTimeout)) let nonTextTarget = app.staticTexts["Agent Device Runner"] - XCTAssertTrue(nonTextTarget.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(nonTextTarget.waitForFixtureExistence(timeout: appExistenceTimeout)) let nonTextFrame = nonTextTarget.frame XCTAssertFalse(nonTextFrame.isEmpty) mainOwned.app = app @@ -89,11 +89,11 @@ extension RunnerTests { XCTAssertTrue(seeded.ok, String(describing: seeded.error)) let nonTextTarget = app.buttons["agent-device-tap-witness"] - XCTAssertTrue(nonTextTarget.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(nonTextTarget.waitForFixtureExistence(timeout: appExistenceTimeout)) let nonTextFrame = nonTextTarget.frame XCTAssertFalse(nonTextFrame.isEmpty) let pressCount = app.staticTexts["agent-device-tap-witness-presses"] - XCTAssertTrue(pressCount.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(pressCount.waitForFixtureExistence(timeout: appExistenceTimeout)) let fillCommand = try runnerCommandFixture( #"{"command":"type","commandId":"fill-penalized-non-text-point","text":"must-not-replace","textEntryMode":"replace","x":\#(nonTextFrame.midX),"y":\#(nonTextFrame.midY)}"# diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift index 96d3380248..b73dde9bf2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -73,67 +73,15 @@ extension RunnerTests { } extension RunnerTests { - /// #2995: resolving a window must read the window it books and nothing behind it. Reads are - /// counted rather than matched by identity because which window `XCUIApplication.windows` reports - /// first is XCTest's to choose; the property is that the first answer is the last read. @MainActor - func testRunnerWindowResolutionStopsReadingAtTheWindowItChose() throws { - let outcome = try resolveRunnerWindowAgainstTwoWindows( - answering: [RunnerWindowFrameReads.usableFrame] - ) - XCTAssertNotNil(outcome.resolved.window, "an app with two usable windows resolves one of them") - XCTAssertEqual(outcome.reads, 1, "a window behind the chosen one was read") - } - - /// The closest negative, so the single read above cannot pass because the resolver never looked at - /// a second window at all: when the first candidate reports no frame, the second one must be asked. - @MainActor - func testRunnerWindowResolutionKeepsLookingPastAWindowWithNoFrame() throws { - let outcome = try resolveRunnerWindowAgainstTwoWindows( - answering: [.zero, RunnerWindowFrameReads.usableFrame] - ) - XCTAssertNotNil( - outcome.resolved.window, - "the second window qualifies, so the resolver owes a window rather than app.frame" - ) - XCTAssertEqual(outcome.reads, 2, "a refused first window costs the second window its read") - } - - /// The other half of stopping early: when no candidate qualifies, the walk must still have asked - /// every one of them, and it must refuse a window rather than book a zero rectangle to normalize - /// against. - @MainActor - func testRunnerWindowResolutionRefusesAWindowWhenEveryCandidateMeasuresZero() throws { - let outcome = try resolveRunnerWindowAgainstTwoWindows(answering: [.zero]) - XCTAssertNil(outcome.resolved.window, "no candidate qualifies, so no window is booked against") - XCTAssertEqual( - outcome.reads, - 3, - "the walk gives up only after asking both windows, and the fallback's own app.frame read is the third" - ) - // `XCUIApplication` declares no `frame` of its own, so the third read is the same swizzled - // `XCUIElement` getter and the fallback frame is the probe's answer. A real app frame here would - // mean the fallback reads geometry the probe never saw. - XCTAssertEqual( - outcome.resolved.frame, - .zero, - "the app.frame fallback goes through the same read as a window's" - ) - } - - /// Launches the two-window fixture, installs the frame probe answering `frames` in read order - /// (repeating its last answer), and resolves once. - @MainActor - private func resolveRunnerWindowAgainstTwoWindows( - answering frames: [CGRect] - ) throws -> (resolved: (window: XCUIElement?, frame: CGRect), reads: Int) { + func testRunnerWindowResolutionReadsOnlyThroughTheFirstUsableWindow() throws { app.launchArguments = ["--agent-device-second-window"] app.launch() defer { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) XCTAssertEqual( app.windows.allElementsBoundByIndex.filter(\.exists).count, 2, @@ -148,14 +96,28 @@ extension RunnerTests { ) else { XCTFail("unable to install the window frame probe") - return ((nil, .zero), 0) + return } - RunnerWindowFrameReads.begin(answering: frames) let originalImplementation = method_getImplementation(frameMethod) method_setImplementation(frameMethod, method_getImplementation(probeMethod)) defer { method_setImplementation(frameMethod, originalImplementation) } - return (resolveRunnerWindow(app: app), RunnerWindowFrameReads.recorded()) + let cases: [(name: String, frames: [CGRect], hasWindow: Bool, reads: Int)] = [ + ("first usable window", [RunnerWindowFrameReads.usableFrame], true, 1), + ("empty first window", [.zero, RunnerWindowFrameReads.usableFrame], true, 2), + ("all windows empty", [.zero], false, 3) + ] + for testCase in cases { + RunnerWindowFrameReads.begin(answering: testCase.frames) + let resolved = resolveRunnerWindow(app: app) + XCTAssertEqual(resolved.window != nil, testCase.hasWindow, testCase.name) + XCTAssertEqual(RunnerWindowFrameReads.recorded(), testCase.reads, testCase.name) + XCTAssertEqual( + resolved.frame, + testCase.hasWindow ? RunnerWindowFrameReads.usableFrame : .zero, + testCase.name + ) + } } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SelectorMatchPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SelectorMatchPolicyTests.swift index 2d3249f90f..243c01ef1b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SelectorMatchPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SelectorMatchPolicyTests.swift @@ -93,7 +93,7 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let matches = app.descendants(matching: .any) .matching(identifier: duplicateIdentifier) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift index ab069702e5..5e103b00c4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift @@ -188,7 +188,7 @@ extension RunnerTests { app.terminate() } let label = app.staticTexts["Agent Device Runner"] - XCTAssertTrue(label.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(label.waitForFixtureExistence(timeout: appExistenceTimeout)) let point = CGPoint(x: label.frame.midX, y: label.frame.midY) mainOwned.accessibilityHealth = .unavailable let command = try runnerCommandFixture( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift index 5e3323cb29..e71ab44527 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift @@ -381,7 +381,7 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.staticTexts["agent-device-crowded-row-499"].waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.staticTexts["agent-device-crowded-row-499"].waitForFixtureExistence(timeout: appExistenceTimeout)) let hint = CaptureHint( projection: .regular, depth: nil, @@ -419,7 +419,7 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.staticTexts["agent-device-crowded-row-499"].waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.staticTexts["agent-device-crowded-row-499"].waitForFixtureExistence(timeout: appExistenceTimeout)) let hint = CaptureHint( projection: .regular, depth: nil, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index f9ebaf62b0..5e78b2ad30 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -41,9 +41,9 @@ extension RunnerTests { func focusSynthesizedReplacementField(extraLaunchArguments: [String] = []) throws -> XCUIElement { app.launchArguments = ["--agent-device-text-entry-regression"] + extraLaunchArguments app.launch() - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(textField.waitForFixtureExistence(timeout: appExistenceTimeout)) mainOwned.app = app mainOwned.bundleId = "com.callstack.agentdevice.runner" mainOwned.processIdentifier = try XCTUnwrap(Self.processIdentifier(of: app)) @@ -150,7 +150,7 @@ extension RunnerTests { ] + (prefilled ? ["--agent-device-text-entry-prefilled"] : [])) defer { tearDownSynthesizedReplacementField() } let togglePresses = app.staticTexts["agent-device-masked-toggle-presses"] - XCTAssertTrue(togglePresses.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(togglePresses.waitForFixtureExistence(timeout: appExistenceTimeout)) XCTAssertEqual( textField.value as? String, prefilled ? String(repeating: "\u{2022}", count: 6) : "", @@ -259,11 +259,11 @@ extension RunnerTests { func launchFieldThatMovesOnFocus() throws -> (field: XCUIElement, neighbour: XCUIElement) { app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-moves-on-focus"] app.launch() - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let field = app.textFields["agent-device-hardware-keyboard-input"] let neighbour = app.textFields["agent-device-text-entry-neighbour"] - XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout)) - XCTAssertTrue(neighbour.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(neighbour.waitForFixtureExistence(timeout: appExistenceTimeout)) mainOwned.app = app mainOwned.bundleId = "com.callstack.agentdevice.runner" mainOwned.processIdentifier = try XCTUnwrap(Self.processIdentifier(of: app)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift index c20ae6e5a3..92d4d13479 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemModalTests.swift @@ -8,7 +8,7 @@ extension RunnerTests { app.launchArguments = ["--agent-device-native-states"] app.launch() defer { app.terminate() } - XCTAssertTrue(app.switches["native-off"].waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.switches["native-off"].waitForFixtureExistence(timeout: appExistenceTimeout)) let resolved = ResolvedBlockingSystemModal(root: app, ownerApp: app, actions: actionableElements(in: app)) let payload = try XCTUnwrap(snapshotForBlockingSystemModal(resolved)) let off = try XCTUnwrap(payload.nodes?.first { $0.identifier == "native-off" }) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift index 57dbbac5e2..2f455a3e36 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift @@ -252,7 +252,7 @@ extension RunnerTests { app.terminate() } let field = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame let text = "ada@example" let command = try JSONDecoder().decode(Command.self, from: JSONSerialization.data(withJSONObject: [ @@ -292,7 +292,7 @@ extension RunnerTests { app.terminate() } let field = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame let command = try JSONDecoder().decode(Command.self, from: JSONSerialization.data(withJSONObject: [ "command": "type", "textEntryMode": "replace", "text": "hello\n", @@ -380,7 +380,7 @@ extension RunnerTests { app.terminate() } let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(textField.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = textField.frame let command = try JSONDecoder().decode( Command.self, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift index 9d342ee0e6..32a06ce44f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift @@ -7,9 +7,9 @@ extension RunnerTests { app.launchArguments = ["--agent-device-text-entry-regression"] app.launch() defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let field = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) field.tap() XCTAssertTrue(keyboardFocusConfirmed(app: app, element: field)) let failures = currentXCTestFailureCount() diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift index aca9d03c6a..405ed409aa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift @@ -10,7 +10,7 @@ extension RunnerTests { app.terminate() } let field = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let point = CGPoint(x: field.frame.midX, y: field.frame.midY) let completed = expectation(description: "optional probe completed inside runner wait") DispatchQueue.main.async { @@ -61,7 +61,7 @@ extension RunnerTests { app.terminate() } let field = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame mainOwned.app = app mainOwned.bundleId = "com.callstack.agentdevice.runner" @@ -93,7 +93,7 @@ extension RunnerTests { app.terminate() } let field = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let point = CGPoint(x: field.frame.midX, y: field.frame.midY) let expected = XCTIssue(type: .assertionFailure, compactDescription: "Required query failure must escape optional containment") let options = XCTExpectedFailure.Options() @@ -120,7 +120,7 @@ extension RunnerTests { app.terminate() } let field = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Failed to get matching snapshot: kAXErrorIllegalArgument") let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY)) @@ -140,7 +140,7 @@ extension RunnerTests { app.terminate() } let target = app.staticTexts["Agent Device Runner"] - XCTAssertTrue(target.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(target.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = target.frame mainOwned.app = app mainOwned.bundleId = "com.callstack.agentdevice.runner" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index b3d61de721..c0ff279316 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -131,10 +131,10 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(textField.waitForFixtureExistence(timeout: appExistenceTimeout)) let tapCommand = try runnerCommandFixture( #"{"command":"tap","commandId":"tap-disappearing-input","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# ) @@ -392,10 +392,10 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(textField.waitForFixtureExistence(timeout: appExistenceTimeout)) let tapCommand = try runnerCommandFixture( #"{"command":"tap","commandId":"tap-soft-keyboard-input","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# ) @@ -464,11 +464,11 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(textField.waitForFixtureExistence(timeout: appExistenceTimeout)) let togglePresses = app.staticTexts["agent-device-masked-toggle-presses"] - XCTAssertTrue(togglePresses.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(togglePresses.waitForFixtureExistence(timeout: appExistenceTimeout)) XCTAssertEqual(togglePresses.label, "presses=0") try tapHardwareKeyboardInput(commandId: "tap-masked-input") @@ -506,9 +506,9 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(textField.waitForFixtureExistence(timeout: appExistenceTimeout)) XCTAssertFalse(textField.frame.isEmpty) return textField } @@ -525,11 +525,11 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let textField = arguments.contains("--agent-device-text-entry-unnamed-input") ? app.textFields.element(boundBy: 0) : app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(textField.waitForFixtureExistence(timeout: appExistenceTimeout)) return textField } diff --git a/docs/evidence/ios-xctest-cost-profile-2026-10-19.csv b/docs/evidence/ios-xctest-cost-profile-2026-10-19.csv new file mode 100644 index 0000000000..b6ead4b793 --- /dev/null +++ b/docs/evidence/ios-xctest-cost-profile-2026-10-19.csv @@ -0,0 +1,119 @@ +identifier,baseline_status,baseline_seconds,candidate_status,candidate_seconds +AgentDeviceRunnerUITests/RunnerTests/testAbandonedTreeCaptureSkipsQuerySweepAndHonorsWarmupExemption,passed,12.418,passed,11.604 +AgentDeviceRunnerUITests/RunnerTests/testActivateTargetSkipsForegroundAndActivatesNonForegroundApplication,passed,0.025,passed,0.036 +AgentDeviceRunnerUITests/RunnerTests/testAlertAcceptDoesNotActivateAReplacementWithASharedButton,passed,7.158,passed,4.653 +AgentDeviceRunnerUITests/RunnerTests/testAlertActivationAfterDeadlineDoesNotTapTheOriginal,passed,4.62,passed,3.439 +AgentDeviceRunnerUITests/RunnerTests/testAlertActivationConfirmsBeforeDeadlineWithoutHandlingTheNotificationBanner,passed,18.39,passed,5.356 +AgentDeviceRunnerUITests/RunnerTests/testAlertActivationIgnoresAnAppThatNeverSettlesBeforeTheDeadline,passed,5.791,passed,4.382 +AgentDeviceRunnerUITests/RunnerTests/testAlertCannotProveAnIdenticalReplacementAndDoesNotActivateIt,passed,35.291,passed,33.865 +AgentDeviceRunnerUITests/RunnerTests/testAlertDeadlineBeforeActivationLeavesTheOriginalUntouched,passed,4.654,passed,3.442 +AgentDeviceRunnerUITests/RunnerTests/testAlertDeadlineDuringSynthesisPreparationLeavesTheOriginalUntouched,passed,4.515,passed,3.431 +AgentDeviceRunnerUITests/RunnerTests/testAlertDismissAnswersTheTrackingPromptDeclineAndNeverAuthorizes,passed,5.262,passed,4.399 +AgentDeviceRunnerUITests/RunnerTests/testAlertDismissDoesNotActivateAReplacementWithTheSameTitle,passed,5.995,passed,4.989 +AgentDeviceRunnerUITests/RunnerTests/testAlertDismissOnADismissPopupWindowChoosesThePopoverCloseButton,passed,6.589,passed,5.247 +AgentDeviceRunnerUITests/RunnerTests/testAlertDismissRefusesAnAffirmativeOnlyPromptWithoutActivatingIt,passed,4.567,passed,3.44 +AgentDeviceRunnerUITests/RunnerTests/testAlertDispatchResolvesItsOwnModalWithoutCoordinateTapRoutingProbe,passed,0.025,passed,0.016 +AgentDeviceRunnerUITests/RunnerTests/testAlertHittableProbeCompletingAfterDeadlineLeavesTheOriginalUntouched,passed,5.729,passed,4.341 +AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionCannotBypassRequestedDeadline,passed,1.025,passed,1.021 +AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionFindsADismissPopupMarkerOnACrowdedScreen,passed,6.245,passed,5.382 +AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionFindsAWindowThatIsItselfTheDismissPopupMarker,passed,6.207,passed,5.982 +AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionWithoutAnAlertDoesNotReadEveryElementOfTheScreen,passed,4.311,passed,3.634 +AgentDeviceRunnerUITests/RunnerTests/testAlertTitleIsTheAlertsOwnTextNotLabelledChromeInsideTheRoot,passed,3.231,passed,3.595 +AgentDeviceRunnerUITests/RunnerTests/testAnnouncedRelaunchWindowDoesNotSurviveALaterInvalidation,passed,0.266,passed,0.27 +AgentDeviceRunnerUITests/RunnerTests/testAnnouncedRelaunchWithNoBoundTargetCoversWhicheverBundleActivatesNext,passed,0.01,passed,0.013 +AgentDeviceRunnerUITests/RunnerTests/testApplicationStateNamesMatchTheAppStateContract,passed,0.008,passed,0.01 +AgentDeviceRunnerUITests/RunnerTests/testApplicationStateRawValuesMatchTheActivationDecoder,passed,0.008,passed,0.009 +AgentDeviceRunnerUITests/RunnerTests/testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand,passed,10.736,passed,4.461 +AgentDeviceRunnerUITests/RunnerTests/testBareSubmitKeyRefusesWhenPrivateSynthesisIsUnavailable,passed,5.168,passed,3.118 +AgentDeviceRunnerUITests/RunnerTests/testBareSubmitKeyUsesSynthesizedFirstResponderAfterHiddenKeyboardTap,passed,10.288,passed,3.789 +AgentDeviceRunnerUITests/RunnerTests/testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden,passed,20.142,passed,9.195 +AgentDeviceRunnerUITests/RunnerTests/testBlockingModalSnapshotLeavesWarmupExemptionForTheFirstCapturePlan,passed,0.012,passed,0.015 +AgentDeviceRunnerUITests/RunnerTests/testBlockingModalSnapshotPreservesNativeSwitchValues,passed,4.497,passed,3.449 +AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain,passed,5.095,passed,5.112 +AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw,passed,5.071,passed,5.071 +AgentDeviceRunnerUITests/RunnerTests/testCollapsedTabQueryReportsWhenItsProcessingSliceCannotStartIt,passed,5.067,passed,4.106 +AgentDeviceRunnerUITests/RunnerTests/testCommitCadenceLogLineEmitsLengthsOnlyNeverContents,passed,0.014,passed,0.017 +AgentDeviceRunnerUITests/RunnerTests/testCommonPrefixLengthWalksTheExpectedPrefixOnly,passed,0.01,passed,0.01 +AgentDeviceRunnerUITests/RunnerTests/testCoordinateFillOnInputThatLeavesTheTreeOnFocusFailsTypedAndRecordsNothing,passed,8.119,passed,6.0 +AgentDeviceRunnerUITests/RunnerTests/testCoordinateTapOnInputThatLeavesTheTreeOnFocusReportsTapped,passed,5.62,passed,3.548 +AgentDeviceRunnerUITests/RunnerTests/testCoordinateTapRoutingPenalizesTheIdentityMainSettledOnWhileTheWriteWasPending,passed,7.069,passed,7.002 +AgentDeviceRunnerUITests/RunnerTests/testCrowdedScreenWithAWideSliderPreservesItsSiblingButtons,passed,4.913,passed,3.954 +AgentDeviceRunnerUITests/RunnerTests/testEmptyReplacementWithoutResolvableTargetFailsClosed,passed,0.015,passed,0.013 +AgentDeviceRunnerUITests/RunnerTests/testExecuteDispatchedReturnsBusyBeforeBlockingSystemModalProbeDrains,passed,7.082,passed,6.989 +AgentDeviceRunnerUITests/RunnerTests/testFileScreenshotResponseWritesTheImageAndStillReportsItsDisplay,passed,0.018,passed,0.021 +AgentDeviceRunnerUITests/RunnerTests/testFillDoesNotRepairIntoAnIndistinguishableSuccessorInput,passed,8.144,passed,5.931 +AgentDeviceRunnerUITests/RunnerTests/testFillFailsWhenAppRemovesInputBeforeTextIsDelivered,passed,7.006,passed,4.808 +AgentDeviceRunnerUITests/RunnerTests/testFillFailsWhenAppReplacesInputBeforeTextIsDelivered,passed,7.507,passed,5.039 +AgentDeviceRunnerUITests/RunnerTests/testFillIntoAutoSubmittingDigitCountFieldReportsNoEvidenceFromItsSuccessor,passed,11.775,passed,9.577 +AgentDeviceRunnerUITests/RunnerTests/testFillIntoDigitCountFieldReportsUnconfirmedEvidence,passed,8.818,passed,6.972 +AgentDeviceRunnerUITests/RunnerTests/testFillIntoDigitGroupingFieldReportsUnconfirmedEvidenceAndDoesNotRepair,passed,6.718,passed,4.734 +AgentDeviceRunnerUITests/RunnerTests/testFillIntoMaskedTextFieldIsUnverifiedRatherThanAMismatch,passed,14.441,passed,5.654 +AgentDeviceRunnerUITests/RunnerTests/testFillIntoTemplateCurrencyFieldReportsUnconfirmedWithoutRetyping,passed,8.776,passed,6.958 +AgentDeviceRunnerUITests/RunnerTests/testFillPinsKnownLimitWhenAnIndistinguishableInputReplacesItMidDelivery,passed,7.773,passed,5.585 +AgentDeviceRunnerUITests/RunnerTests/testFillSucceedsWhenAppRemovesInputAfterLastCharacter,passed,9.036,passed,6.958 +AgentDeviceRunnerUITests/RunnerTests/testFillSucceedsWhenAppReplacesInputAfterLastCharacter,passed,9.728,passed,7.333 +AgentDeviceRunnerUITests/RunnerTests/testFillThatClearsOnSubmitIsUnverifiedAndSubmitsOnce,passed,6.616,passed,4.716 +AgentDeviceRunnerUITests/RunnerTests/testFillVerifiesAnUnidentifiedFieldThatMovesWhileTyping,passed,7.041,passed,4.962 +AgentDeviceRunnerUITests/RunnerTests/testFillWithAppWriteBackFailsWithoutRetyping,passed,6.681,passed,4.742 +AgentDeviceRunnerUITests/RunnerTests/testFreshCoordinateTapContainsUnavailableTextInputProbe,passed,8.796,passed,6.873 +AgentDeviceRunnerUITests/RunnerTests/testHardwareKeyboardResponderConfirmsItsOwnKeyboardFocus,passed,11.568,passed,3.58 +AgentDeviceRunnerUITests/RunnerTests/testHealthyCoordinateTapPreservesBareTypingWitness,passed,5.592,passed,4.522 +AgentDeviceRunnerUITests/RunnerTests/testMissingBundleCommandInvalidatesCompleteCachedTargetState,passed,4.009,passed,4.114 +AgentDeviceRunnerUITests/RunnerTests/testNoAppCommandStillServesAPresentedSurfaceInPlaceAndOtherwiseTheStandingTarget,passed,0.016,passed,0.017 +AgentDeviceRunnerUITests/RunnerTests/testNonInteractiveQuerySweepStopsAtTheSliceItsCallerWaitsFor,passed,3.946,passed,4.043 +AgentDeviceRunnerUITests/RunnerTests/testNonReadCommandStillLaunchesANotRunningSessionApp,passed,3.927,passed,4.105 +AgentDeviceRunnerUITests/RunnerTests/testObserveOnlySnapshotRefusesBackgroundWithoutActivationOrDisclosure,passed,3.969,passed,3.99 +AgentDeviceRunnerUITests/RunnerTests/testObserveOnlySnapshotReportsStateWithoutBindingOrActivatingTarget,passed,2.928,passed,2.945 +AgentDeviceRunnerUITests/RunnerTests/testObservedScreenCaptureAsksTheSystemSurfaceWhenTheSessionWindowIsUnresolved,passed,0.018,passed,0.009 +AgentDeviceRunnerUITests/RunnerTests/testObservedScreenCaptureKeepsTheSessionAppCaptureItResolved,passed,0.013,passed,0.009 +AgentDeviceRunnerUITests/RunnerTests/testObservedScreenCaptureReportsADisplayRefusalWithoutAskingTheSystemSurface,passed,0.009,passed,0.008 +AgentDeviceRunnerUITests/RunnerTests/testOverBudgetTypeWithoutResolvableElementTypesApplicationWide,passed,14.43,passed,7.606 +AgentDeviceRunnerUITests/RunnerTests/testPasteboardWriteLeavesTheTextOnTheGeneralPasteboard,passed,0.073,passed,0.035 +AgentDeviceRunnerUITests/RunnerTests/testPenalizedCoordinateFillAtAPointWithNoTextInputRefusesBeforeItsTapAndKeepsTheFocusedField,passed,13.952,passed,5.256 +AgentDeviceRunnerUITests/RunnerTests/testPenalizedCoordinateTapOnNonTextControlDoesNotAuthorizeBareType,passed,13.468,passed,5.887 +AgentDeviceRunnerUITests/RunnerTests/testPenalizedPlanRecoversThroughTheBoundedTreeWhenPrivateAXReadsNothing,passed,3.074,passed,3.103 +AgentDeviceRunnerUITests/RunnerTests/testPenalizedPlanThatEndsSparseAfterTheBoundedTreeReportsTheBudget,passed,4.008,passed,3.996 +AgentDeviceRunnerUITests/RunnerTests/testPerformedActivationReplacesAnElapsedAnnouncedWindow,passed,0.272,passed,0.266 +AgentDeviceRunnerUITests/RunnerTests/testPlanMovesPastATreeTierWhoseCollapsedTabQueryCannotStart,passed,3.024,passed,3.025 +AgentDeviceRunnerUITests/RunnerTests/testPrepareActiveCommandContextRoutesBlockingSystemModalToSpringboard,passed,0.016,passed,0.022 +AgentDeviceRunnerUITests/RunnerTests/testPresentedSurfaceCommandLeavesAStoppedAppStoppedAndUnbound,passed,0.02,passed,0.033 +AgentDeviceRunnerUITests/RunnerTests/testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation,passed,3.061,passed,3.018 +AgentDeviceRunnerUITests/RunnerTests/testQuerySelectorPrefersHittableMatchOverNonHittableDuplicate,passed,4.176,passed,3.128 +AgentDeviceRunnerUITests/RunnerTests/testQuerySweepThatEndsOnItsSliceDeadlinePenalizesChannelAndReachesPrivateAX,passed,4.622,passed,4.489 +AgentDeviceRunnerUITests/RunnerTests/testReadRefusesToLaunchANotRunningSessionApp,passed,0.068,passed,0.068 +AgentDeviceRunnerUITests/RunnerTests/testRecordStartThrowsTheCaptureRefusalItReceived,passed,2.041,passed,2.059 +AgentDeviceRunnerUITests/RunnerTests/testRegularSnapshotStillRepairsBackgroundAndDisclosesPriorState,passed,4.146,passed,4.062 +AgentDeviceRunnerUITests/RunnerTests/testRunnerWindowResolutionReadsOnlyThroughTheFirstUsableWindow,passed,4.069,passed,3.057 +AgentDeviceRunnerUITests/RunnerTests/testSelectorTapFallsBackToXCTestCoordinateWhenPrivateSynthesisFails,passed,3.636,passed,3.41 +AgentDeviceRunnerUITests/RunnerTests/testSelectorTapOnInputThatLeavesTheTreeOnFocusRecordsNoFailure,passed,9.656,passed,7.479 +AgentDeviceRunnerUITests/RunnerTests/testSingleDigitFillIntoDigitCountFieldIsTypedOnce,passed,7.953,passed,6.152 +AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails,passed,3.712,passed,3.62 +AgentDeviceRunnerUITests/RunnerTests/testSkipAppActivationPreflightRequiresCachedForegroundCoordinateOnlyTap,passed,3.018,passed,2.94 +AgentDeviceRunnerUITests/RunnerTests/testSkippedActivationAfterAnnouncedRelaunchKeepsAnElapsedWindow,passed,0.011,passed,0.019 +AgentDeviceRunnerUITests/RunnerTests/testSkippedActivationAfterAnnouncedRelaunchWaitsOnlyForTheRemainingWindow,passed,0.261,passed,0.263 +AgentDeviceRunnerUITests/RunnerTests/testSkippedActivationOfAnotherBundleThanTheAnnouncedRelaunchStampsFresh,passed,0.263,passed,0.267 +AgentDeviceRunnerUITests/RunnerTests/testSkippedActivationWithoutAWindowStampsOne,passed,0.262,passed,0.265 +AgentDeviceRunnerUITests/RunnerTests/testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds,passed,0.011,passed,0.01 +AgentDeviceRunnerUITests/RunnerTests/testStaleActivationWindowSurvivingTerminateInvalidationStampsFreshOnSkippedActivation,passed,0.263,passed,0.264 +AgentDeviceRunnerUITests/RunnerTests/testSuppressedAxIssueMakesTextInputProbeUnavailable,passed,4.493,passed,3.074 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedClearEmptiesTheFieldThatMovedOnFocus,passed,8.365,passed,5.168 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedClearFailsClosedWhenTheInputFoundUnderThePointCannotBeRead,passed,6.152,passed,3.118 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedClearFailsClosedWhenTheInputFoundUnderThePointIsGone,passed,6.16,passed,3.042 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedClearFindsAnIdentifiedInputWhoseHandleReBound,passed,7.552,passed,4.408 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedGesturePolicyMarkerWritesOncePerKindUntilTheDecisionChanges,passed,0.011,passed,0.013 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedReplacementClearsPastDroppedSelectAlls,passed,26.412,passed,12.641 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedReplacementCommitSkipsSubmitKeyText,passed,0.013,passed,0.015 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedReplacementConfirmsAFieldThatMovedOnFocus,passed,8.443,passed,5.236 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedReplacementFailsClosedOnAFailedLookupOfTheInputUnderThePoint,passed,10.018,passed,3.426 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedReplacementLeavesAMaskedFieldUnverified,passed,30.185,passed,14.583 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedReplacementPacesAnAppOwnedFieldAtItsAcknowledgeWindow,passed,14.347,passed,8.059 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedReplacementRefusesTextBeyondTheDeliveryBudget,passed,10.542,passed,4.007 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedReplacementTypesNothingWhenItCannotReadTheFieldBack,passed,0.138,passed,0.16 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedSequenceTapFallsBackToXCTestCoordinateTapWhenAccessibilityIsUnavailable,passed,4.418,passed,3.421 +AgentDeviceRunnerUITests/RunnerTests/testSynthesizedTextEntryFallsBackOnlyWhenPrivateSynthesisIsUnavailable,passed,0.009,passed,0.014 +AgentDeviceRunnerUITests/RunnerTests/testTapDeadlineExpiringDuringEventPreparationDoesNotSynthesize,passed,1.013,passed,1.017 +AgentDeviceRunnerUITests/RunnerTests/testTextInputProbeContainmentExcludesRequiredReadsAndLaterIssues,passed,4.288,passed,3.214 +AgentDeviceRunnerUITests/RunnerTests/testTextInputProbeIssueScopeIsThreadBound,passed,0.056,passed,0.055 +AgentDeviceRunnerUITests/RunnerTests/testTextInputProbePreservesEnclosingRunnerWait,passed,4.293,passed,3.034 +AgentDeviceRunnerUITests/RunnerTests/testTypeTextReliablyPacesSynthesizedReplacementThroughProductionCaller,passed,13.454,passed,6.586 +AgentDeviceRunnerUITests/RunnerTests/testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText,passed,10.092,passed,10.113 diff --git a/docs/evidence/ios-xctest-cost-profile-2026-10-19.md b/docs/evidence/ios-xctest-cost-profile-2026-10-19.md new file mode 100644 index 0000000000..2b0e4b734b --- /dev/null +++ b/docs/evidence/ios-xctest-cost-profile-2026-10-19.md @@ -0,0 +1,153 @@ +# XCTest cost profile and lane policy + +This evidence follows the [initial fixture reuse experiment](ios-xctest-fixture-reuse-2026-10-19.md). +It records a full local run of the selected native regressions and the subsequent scheduling change. + +## Attribution + +The first complete baseline is `431a94cb96014cb4bdf91f51a5a1eb76440f6e96`. +It executes all 118 then-selected methods. +While this work was running, main merged `fa4890aac6` through #3820. +That commit fixes the same always-nil iOS focus polling found by this profile. +The rebased candidate preserves that upstream production implementation without a diff. +The full comparison therefore includes upstream readiness improvements as well as this PR's fixture changes. +It does not isolate this PR's incremental speedup. + +## Setup + +The device is a dedicated iPhone 17 Pro simulator running iOS 26.2. +Its UDID is `74DBE486-DB09-47B3-B3C5-0637D740930D`. +The toolchain is `/Applications/Xcode-26.2.0.app/Contents/Developer`. +The package manager is pnpm 11.17.0. +The candidate has its own frozen installation and package build in a disposable checkout. +The baseline reuses the clean native build previously verified at its exact SHA. +Both native builds include `AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS=1`. +Both full invocations run `test-without-building` with parallel testing disabled. +They select the same 118 identifiers and run sequentially on the same simulator. +The candidate starts after notification authorization and simulator caches are warm. +No broad gate or other test process runs beside either full native invocation. + +## Full selected-suite comparison + +The candidate source is `19573d42b0` on base `a661e43d83735fc74a02118b5698e5d45e03ee21`. +Both logs contain the exact same 118 identifiers once, with zero unexpected failures or skips. +Each xcresult summary records 117 passed methods and one existing expected-failure method. + +| Measurement | Baseline | Rebased candidate | +| --- | ---: | ---: | +| Command wall time | 718.284 s | 484.579 s | +| Sum of method durations | 680.147 s | 464.813 s | +| Fixture launches | 81 | 81 | +| Observed existence-poll gaps | 111 / 113.340 s | 1 / 1.010 s | + +Wall time falls by 32.5%; method time falls by 31.7%. +These are local results, with the attribution and warm-state limits above. +They do not predict a hosted Apple job speedup. +[Every matched method](ios-xctest-cost-profile-2026-10-19.csv) retains its individual status and duration. + +## Findings + +The baseline completes 118 methods with no unexpected failures or skips. +Their durations sum to 680.147 seconds. +The command takes 718.284 seconds wall time. +There are 81 fixture launches. +XCTest activity timestamps expose 111 existence-poll gaps totalling 113.340 seconds. +Those gaps include all observed existence waits; they are not a count of modified call sites. +An already-focused selector tap still waits twice for two seconds and dispatches a second tap. +The polled app-wide focus lookup always returns nil on iOS. +Upstream #3820 fixes that route by corroborating the chosen input through safe snapshots. + +This PR checks `exists` before calling the same bounded fixture existence wait. +It retains the original timeout and fallback wait when the fixture is not present. +It does not change app-launch behavior or disable animations, diagnostics, caches or failure deadlines. +The identical-replacement alert still waits for its full uncertainty deadline. +Those expensive native failure contracts remain valuable and run nightly. + +## Isolated fixture timing and regression proof + +The disposable source is `7bb347f771`. +With only the fixture fast check removed, the same three methods pass in 14.081 seconds of method time. +With the fast check present, they pass in 9.909 seconds. +The difference is 4.172 seconds in this single local comparison. +The hardware-keyboard method changes from 6.937 to 3.706 seconds. +Window resolution changes from 4.181 to 3.072 seconds. +The unchanged activation-predicate control changes from 2.963 to 3.131 seconds. +The patch changes only `exists || waitForExistence(...)` to `waitForExistence(...)`. +Both variants retain the same timeout and pass all three methods. + +A six-method unmodified control passes focus, recovery, both disappearing-input tap routes, +moving-input replacement and upstream latency coverage. +The following five production mutants each build successfully and fail their selected method with exit 65. +Each mutant executes one method, with one failure and no skips. + +| Mutant | Observed detection | +| --- | --- | +| Focus poll never confirms | The real tap observer counts two dispatches instead of one. | +| Readiness claims focus before delivery | A dropped first tap is never recovered; count is one and the field lacks focus. | +| Remove the recovery tap | The dropped first tap leaves the field unfocused and the count at one. | +| Accept another element's focus | The unrelated-element negative assertion fails. | +| Admit non-tap commands to the activation bypass | Retained drag, scroll and sequence negatives fail after the duplicate drag row is removed. | + +The observer forwards to the original coordinate-tap implementation, except for the deliberately dropped first event. +It restores the method and removes its implementation block with deferred cleanup. +The focus implementation is inherited from #3820; these mutations establish the retained tests' value. +Every source patch is restored before the next experiment. +The final clean rebuild passes hardware focus, dropped-tap recovery and activation preflight, three of three. +Intentional failing invocations disable diagnostic collection only, to avoid XCTest's 600-second simulator diagnosis. +CI diagnostics and assertion failures remain enabled. + +## Native reach verification + +The 118-method comparison uses `19573d42b0`. +A complementary 331-method invocation uses `7bb347f771` after the duplicate drag row was removed. +Those runs cover all 449 nightly identifiers exactly once across their disjoint selections. +Their xcresult summaries total 448 passed, one expected failure, zero unexpected failures and zero skips. +The expected failure belongs to the existing optional-issue containment test. +The four PR methods are included in the 118-method run. +A separate final-code PR invocation passes all four methods in 4.436 seconds wall time. +Its xcresult summary passes the new source-derived count check: four executed of four expected. +This is coverage across two invocations, not a claim that the whole nightly workflow ran locally. +The changed preflight test also runs in the final-source controls and mutation below. + +## Xcode 27 verification + +Xcode 27.1 builds the final code at `7bb347f771` and passes all 15 selected methods with no skips. +The selection includes focus/tap-count recovery, disappearing and moving inputs, masked-input failures, +upstream latency, the three original fixture consolidations and the four PR platform checks. +It runs on the same iOS 26.2 simulator. + +## Coverage policy + +At the rebased source, discovery reaches 338 host methods, four PR simulator methods and 449 nightly methods. +No method is dark, and no lane reaches the command server entry point. + +PRs retain the macOS host decision suite, generated shared platform checks on the simulator, and public CLI E2E smoke. +Nightly and manual dispatch retain the full native iOS bundle, excluding only the command server entry point. +The optional-probe completion marker moves with its test to nightly. +The PR result summary now checks the source-derived execution count. +The selector still rejects dark tests, unknown skips and a reachable server entry point. +Native injected failures can consequently be discovered after merge rather than on every PR. + +Existing E2E does not establish equivalence for hidden-keyboard one-shot witnesses, penalized replacement, +dropped Select All, identical replacement alerts, blocked queries or optional XCTest issue containment. +Ordinary successful fill/alert scenarios are insufficient deletion proof. +These cases are retained, rather than removed as duplicates. + +## Raw evidence + +The local bundle is `/private/tmp/pr3824-full-profile`. +It contains commands, logs, result bundles, per-method profiles, mutation patches and build logs. +The comparison CSV preserves every matched method and its before/after status and duration. +To reproduce the full comparison selection, read its first column into `-only-testing:` flags. +Run `xcodebuild test-without-building -xctestrun "$XCTESTRUN_PATH"` with those flags, +`-destination "platform=iOS Simulator,id=$TEST_SIMULATOR_UDID"`, `-parallel-testing-enabled NO` +and a fresh `-resultBundlePath`. +The current `--ios-pr-tests` selector intentionally produces the smaller PR lane instead. + +Historical focused measurements retain their original source SHAs in the initial experiment report. + +## Cleanup + +The owned simulator and disposable checkout are removed after verification. +The implementation checkout and its existing edits are preserved. +The raw logs, result bundles, comparison commands and mutation patches remain in the local evidence bundle. diff --git a/docs/evidence/ios-xctest-fixture-reuse-2026-10-19.md b/docs/evidence/ios-xctest-fixture-reuse-2026-10-19.md new file mode 100644 index 0000000000..76cce2a9ac --- /dev/null +++ b/docs/evidence/ios-xctest-fixture-reuse-2026-10-19.md @@ -0,0 +1,137 @@ +# iOS XCTest fixture reuse + +This report records the initial fixture reuse batch at its named historical SHAs. +The [full-suite profile and nightly policy](ios-xctest-cost-profile-2026-10-19.md) record the later work. + +## Scope + +The base is `b5f47e7542e274fc84a193c4e111e6c73ac3b1f3`. +The test-source commit is `cefec4dd1ffe62537991f124ac8d26e94fa5505a`. +The change groups ten native methods into three methods. +It changes no production runner code, deadline, retry, or workflow selection policy. + +| Retained method | Preserved contracts | +| --- | --- | +| `testAlertActivationConfirmsBeforeDeadlineWithoutHandlingTheNotificationBanner` | A real notification banner exists. Activation consults no interruption monitor. The response succeeds before its deadline. The original action occurs once. The replacement action never occurs. | +| `testSkipAppActivationPreflightRequiresCachedForegroundCoordinateOnlyTap` | Missing cached targets refuse tap and scroll. A foreground cached coordinate tap bypasses activation. Selector tap, drag, scroll and mixed or coordinate gesture sequences retain activation. A changed bundle refuses the bypass. A terminated cached target refuses the bypass. | +| `testRunnerWindowResolutionReadsOnlyThroughTheFirstUsableWindow` | Two real windows exist before the frame probe is installed. The first usable window costs one frame read. An empty first window costs two reads. All empty windows produce no selected window and a zero app-frame fallback after three reads. The probe resets between rows. | + +The activation predicate runs its destructive termination input last. +Its former “background” method used `terminate()`. +The retained method names that state accurately. +The retained banner test also observes the interruption opt-out during native dispatch. +It uses the existing synthesis stub for a nonperforming probe. +It restores the original implementation before the real activation. +The process-wide frame implementation is restored with `defer`. +Each method terminates its fixture and clears the state that it owns. + +## Selection + +Source-derived PR selection changes from 125 to 118 methods. +All 115 unaffected identifiers remain. +The ten superseded identifiers are absent. +All three retained identifiers are present. +No declaration becomes unreachable. +The production `RunnerTests/testCommand` entrypoint remains excluded. +The macOS host selection remains 338 methods. + +## Local measurement + +The runs use Xcode 26.2, iOS 26.2 and a dedicated iPhone 17 Pro simulator. +The simulator UDID is `DB573A6D-FA35-4516-B5BB-478D8EFC8C5F`. +Dependencies use pnpm 11.17.0 with the frozen lockfile. +Each checkout is installed and built separately. +Native builds include `AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS=1`. +All test invocations use `test-without-building` and disable parallel testing. +The base and candidate run in sequence on the same simulator. +No broad gate runs during the native experiments. + +| Sample | Executed / passed / failed / skipped | Sum of method durations | +| --- | --- | ---: | +| First base run | 10 / 10 / 0 / 0 | 58.044 s | +| Warm base run | 10 / 10 / 0 / 0 | 38.213 s | +| Warm candidate run with the strengthened interruption oracle | 3 / 3 / 0 / 0 | 18.523 s | + +The warm comparison reduces method time by 19.690 s, or 51.5%. + +| Contract group | Warm base | Warm candidate | +| --- | ---: | ---: | +| Banner activation | 13.813 s | 11.450 s | +| Window resolution | 12.425 s | 4.089 s | +| Activation preflight | 11.975 s | 2.984 s | + +Observed fixture launches change from nine to three. +The first base run includes fresh simulator and notification setup. +Use the warm comparison for the headline result. +These are local focused measurements. +They do not predict the complete hosted Apple job duration. +Build, simulator boot and Xcode invocation overhead are outside the method sum. + +The command shape is: + +```sh +DEVELOPER_DIR=/Applications/Xcode-26.2.0.app/Contents/Developer \ +xcodebuild test-without-building \ + -xctestrun "$XCTESTRUN_PATH" \ + -destination "platform=iOS Simulator,id=$TEST_SIMULATOR_UDID" \ + -parallel-testing-enabled NO \ + -resultBundlePath "$RESULT_BUNDLE" \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertActivationConfirmsBeforeDeadlineWithoutHandlingTheNotificationBanner \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSkipAppActivationPreflightRequiresCachedForegroundCoordinateOnlyTap \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerWindowResolutionReadsOnlyThroughTheFirstUsableWindow +``` + +The base uses the ten original identifiers from the diff. +Each log is checked for the exact expected identifier set. +Count equality alone does not establish discovery. + +## Interruption oracle + +At the first test-source commit, `cb5360a045c1f8b4061bc01750e5bc6c22d264ed`, +a mutation that enabled interruption handling passed. +The empty callback list did not establish that the opt-out flag was set. +That survivor was not accepted as detection evidence. +The current test observes the flag inside the existing native synthesis stub. +It still requires the real banner, monitor, successful activation, deadline and action counts. +The current timing includes that additional probe. + +An initial deliberately failing run started a simulator diagnostic collection with a 600-second timeout. +That collection was interrupted after the assertion failures were recorded. +That interrupted run is excluded from mutation completion counts. +Subsequent mutation invocations add `-collect-test-diagnostics never`. +They retain test logs, result bundles, exact identifier checks and failure exit status. +CI diagnostics remain unchanged. + +## Mutation proof + +The disposable checkout starts at the test-source commit named above. +The superseded methods are absent from that checkout. +The unmodified retained suite passes all three selected methods. +Each production mutation is then built and tested separately. +Each mutant runs only its relevant retained method. +All nine builds succeed. +Each test run reports zero passed, one failed and zero skipped methods, with exit 65. +The source edit is restored before the next mutation. + +| Production mutation | Intended retained assertion | +| --- | --- | +| Return no activation outcome for alert accept | Native dispatch observation is absent. The real response fails. The original action count remains zero. | +| Set `doesNotHandleUIInterruptions` to false | The native dispatch probe observes false instead of true. | +| Read past all windows and return only the app-frame fallback | First usable-window selection and one-read witness fail. | +| Accept an empty window frame | Empty-first and all-empty controls fail their read count, frame or window-presence assertions. | +| Reject every command after the cached-target guard | The foreground cached coordinate-tap positive fails. | +| Accept every command after the cached-target guard | Drag, scroll and gesture-sequence negatives fail. | +| Remove the cached-target guard | Missing, changed-bundle and terminated-target negatives fail. | +| Accept every requested bundle after the foreground check | The changed-bundle negative fails. | +| Remove the foreground-state check | The terminated cached-target negative fails. | + +The implementation checkout contains none of these production edits. +The disposable checkout is clean after the experiments. +The raw logs, mutation patches and result bundles are retained locally. +This report records completed runs, not predicted detection. + +## Publication obligations + +The local measurements do not establish a hosted speedup. +Apple smoke, Integration Tests and Coverage remain CI obligations at the PR head. +The source-derived selection check and packaged-runner check remain required local gates. diff --git a/scripts/__tests__/apple-ci-impact.test.ts b/scripts/__tests__/apple-ci-impact.test.ts index e8f1025c5f..f1ac6980b1 100644 --- a/scripts/__tests__/apple-ci-impact.test.ts +++ b/scripts/__tests__/apple-ci-impact.test.ts @@ -433,7 +433,7 @@ test('the PR workflow applies the impact decision to the XCTest step', () => { ); expect(workflow).toContain('node --experimental-strip-types scripts/apple-ci-impact.ts xctest'); expect(workflow).toMatch( - /- name: Run targeted iOS runner XCTest regressions\n\s+id: ios-xctest\n\s+if: steps\.xctest-impact\.outputs\.run != 'false'/, + /- name: Run iOS runner platform checks\n\s+id: ios-xctest\n\s+if: steps\.xctest-impact\.outputs\.run != 'false'/, ); expect(workflow).toContain('if [ "$SELECTED" = \'true\' ] && [ "$OUTCOME" = \'skipped\' ]'); }); diff --git a/scripts/__tests__/xctest-selection.test.ts b/scripts/__tests__/xctest-selection.test.ts index 40c56ec7b8..4f066032fe 100644 --- a/scripts/__tests__/xctest-selection.test.ts +++ b/scripts/__tests__/xctest-selection.test.ts @@ -76,9 +76,8 @@ describe('the real tree', () => { const { declared, host, pr, nightly, dark } = counts(report); // Not pinned to today's exact numbers; the invariants are the shape. The host lane // (macOS) and the nightly (iOS) both skip only the entry point, so together with the - // simulator-only guard they cover everything else; the PR lane runs iOS-only methods - // and shared methods with platform-dependent bodies. The entry point is the only method - // outside every lane. + // simulator-only guard they cover everything else. The PR lane runs shared methods + // with platform-dependent bodies. Only the server entry point stays outside every lane. expect(host).toBeGreaterThan(0); expect(nightly).toBeGreaterThan(pr); expect(pr).toBeGreaterThan(0); @@ -87,6 +86,7 @@ describe('the real tree', () => { expect(reachedAnywhere.size).toBe(declared - 1); expect(reachedAnywhere.has(ENTRY_POINT)).toBe(false); for (const id of report.reach.pr) expect(report.reach.nightly.has(id)).toBe(true); + expect([...report.reach.pr].sort()).toEqual([...report.sharedPlatformBranchIds].sort()); expect(iosPrTestIdentifiers(report.declaredTests, report.sharedPlatformBranchIds)).toEqual([ ...report.reach.pr, ]); @@ -126,6 +126,7 @@ describe('the real tree', () => { for (const test of simulatorOnly) { expect(report.reach.host.has(test.identifier)).toBe(false); expect(report.reach.nightly.has(test.identifier)).toBe(true); + expect(report.reach.pr.has(test.identifier)).toBe(false); } }); @@ -245,12 +246,12 @@ describe('a planted typo', () => { }, ); - test('adding or removing an iOS-only method changes the PR selection automatically', () => { + test('an iOS-only addition enters nightly automatically without expanding the PR lane', () => { const iosMethod = '#if os(iOS)\nextension RunnerTests {\n func testIosOnly() {}\n}\n#endif\n'; const id = `${TARGET}/RunnerTests/testIosOnly`; - expect( - buildReport(TARGET, source(`${ENTRY_SOURCE}${iosMethod}`), laneWorkflows()).reach.pr.has(id), - ).toBe(true); + const report = buildReport(TARGET, source(`${ENTRY_SOURCE}${iosMethod}`), laneWorkflows()); + expect(report.reach.nightly.has(id)).toBe(true); + expect(report.reach.pr.has(id)).toBe(false); expect(buildReport(TARGET, source(ENTRY_SOURCE), laneWorkflows()).reach.pr.has(id)).toBe(false); }); @@ -351,7 +352,7 @@ describe('a planted guard', () => { const id = `${TARGET}/RunnerTests/testLaunchesApp`; expect(report.reach.host.has(id)).toBe(false); expect(report.reach.nightly.has(id)).toBe(true); - expect(report.reach.pr.has(id)).toBe(true); + expect(report.reach.pr.has(id)).toBe(false); expect(reportFailures(report)).toEqual([]); }); @@ -490,7 +491,7 @@ describe('the summary line', () => { const { declared, host, pr, nightly, dark } = counts(report); const summary = formatSummary(report); expect(summary).toContain(`${declared} declared`); - expect(summary).toContain(`reaches ${host}, PR iOS-specific lane`); + expect(summary).toContain(`reaches ${host}, PR iOS branch lane`); expect(summary).toContain(`selects ${pr}, nightly`); expect(summary).toContain(`reaches ${nightly};`); expect(summary).toContain(`${dark} reachable by no lane`); diff --git a/scripts/check-xctest-selection.ts b/scripts/check-xctest-selection.ts index 4723e0a56e..89ad42762b 100644 --- a/scripts/check-xctest-selection.ts +++ b/scripts/check-xctest-selection.ts @@ -8,7 +8,7 @@ // - host smoke-macos/action.yml, macOS host, every PR: the whole bundle as compiled for macOS, minus // `-skip-testing:` — the pure runner-decision tests, whose guard is // `#if AGENT_DEVICE_RUNNER_UNIT_TESTS` alone. -// - pr smoke-ios-run/action.yml, iOS Simulator: tests with iOS-specific Swift bodies, when selected. +// - pr smoke-ios-run/action.yml, iOS Simulator: shared tests with platform-dependent Swift bodies, when selected. // - nightly xctest-nightly.yml, iOS Simulator, scheduled: the whole bundle as compiled for // iOS, minus `-skip-testing:` — includes the simulator-only tests, whose guard is // `… && os(iOS)` (they launch the host app, route through SpringBoard, or assert an @@ -58,8 +58,8 @@ export type Lane = { readonly id: LaneId; readonly workflow: string; readonly platform: Platform; - /** `whole`: everything compiled minus `-skip-testing:`; `ios-specific`: derived from guards. */ - readonly selection: 'whole' | 'ios-specific'; + /** `whole`: everything compiled minus `-skip-testing:`; `platform-branch`: derived from guards. */ + readonly selection: 'whole' | 'platform-branch'; /** The job-summary heading the lane's reporter prints. */ readonly title: string; }; @@ -76,8 +76,8 @@ export const LANES: readonly Lane[] = [ id: 'pr', workflow: PR_XCTEST_SOURCE_FILE, platform: 'iOS', - selection: 'ios-specific', - title: 'iOS-specific runner PR XCTest lane', + selection: 'platform-branch', + title: 'iOS runner platform branch checks', }, { id: 'nightly', @@ -193,7 +193,7 @@ function laneReach( flagged: readonly FlaggedTest[], sharedPlatformBranchIds: readonly string[], ): Set { - if (entry.selection === 'ios-specific') + if (entry.selection === 'platform-branch') return new Set(iosPrTestIdentifiers(declaredTests, sharedPlatformBranchIds)); const skipped = identifiers(flagged, entry.workflow, 'skip-testing'); return new Set( @@ -213,7 +213,8 @@ export function iosPrTestIdentifiers( .filter( (test) => test.platforms.includes('iOS') && - (!test.platforms.includes('macOS') || branchSensitive.has(test.identifier)), + test.platforms.includes('macOS') && + branchSensitive.has(test.identifier), ) .map((test) => test.identifier) .filter((id) => !id.endsWith(`/${ENTRY_POINT_METHOD}`)) @@ -307,7 +308,7 @@ function prWorkflowWiringFailures(text: string | null): string[] { .join('\n'); const failures: string[] = []; if (!active.includes('scripts/check-xctest-selection.ts --ios-pr-tests')) { - failures.push('PR workflow does not invoke the generated iOS-specific XCTest selector.'); + failures.push('PR workflow does not invoke the generated iOS branch XCTest selector.'); } if (!active.includes('-only-testing:$test_id') || !active.includes('"${IOS_PR_TEST_ARGS[@]}"')) { failures.push('PR workflow does not pass each generated XCTest identifier to xcodebuild.'); @@ -448,7 +449,7 @@ export function reportFailures(report: SelectionReport): string[] { `${report.dark.length} declared XCTest method(s) are reachable by no lane:`, ...report.dark.map((identifier) => ` - ${identifier}`), 'The host lane runs everything the macOS build compiles, the nightly everything the iOS', - 'build compiles, and the PR lane selects iOS-specific methods; a method outside all three — usually', + 'build compiles, and the PR lane selects shared iOS branch methods; a method outside all three — usually', 'a guard naming a platform no lane runs — is dark from the day it is written. Widen the', 'guard or delete it.', ); @@ -469,7 +470,7 @@ export function formatSummary(report: SelectionReport): string { const { declared, host, pr, nightly, dark } = counts(report); return ( `xctest selection: ${declared} declared ${report.target} methods — host lane ` + - `(${HOST_XCTEST_SOURCE_FILE}, macOS, every PR) reaches ${host}, PR iOS-specific lane (${PR_XCTEST_SOURCE_FILE}, ` + + `(${HOST_XCTEST_SOURCE_FILE}, macOS, every PR) reaches ${host}, PR iOS branch lane (${PR_XCTEST_SOURCE_FILE}, ` + `iOS Simulator, selected PRs and main) selects ${pr}, nightly (${NIGHTLY_WORKFLOW_FILE}, iOS Simulator) ` + `reaches ${nightly}; ${dark} reachable by no lane; ${ENTRY_POINT_METHOD} skipped everywhere.\n` ); @@ -492,7 +493,7 @@ function main(): number { if (process.argv.slice(2).includes('--ios-pr-tests')) { const failures = reportFailures(report); if (report.reach.pr.size === 0) - failures.push('The PR lane selected no iOS-specific XCTest methods.'); + failures.push('The PR lane selected no shared iOS branch XCTest methods.'); if (failures.length > 0) { process.stderr.write(`${failures.join('\n')}\n`); return 1;