From 7606f85d717f32011c92f981008be05754d1a175 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 19 Oct 2026 20:10:38 +0200 Subject: [PATCH] perf(ci): overlap iOS simulator boot with the native build --- .../boot-ios-test-simulator/action.yml | 26 +-- .github/actions/smoke-ios-run/action.yml | 19 ++- .../ios-boot-build-overlap-2026-10-19.md | 57 +++++++ scripts/__tests__/apple-ci-impact.test.ts | 155 ++++++++++++++++++ scripts/wait-for-ios-simulator.sh | 19 +++ 5 files changed, 260 insertions(+), 16 deletions(-) create mode 100644 docs/evidence/ios-boot-build-overlap-2026-10-19.md create mode 100644 scripts/wait-for-ios-simulator.sh diff --git a/.github/actions/boot-ios-test-simulator/action.yml b/.github/actions/boot-ios-test-simulator/action.yml index 684ed38f72..953a1e2460 100644 --- a/.github/actions/boot-ios-test-simulator/action.yml +++ b/.github/actions/boot-ios-test-simulator/action.yml @@ -14,7 +14,15 @@ inputs: required: false default: '300' + wait-for-boot: + description: 'Wait for boot completion before returning' + required: false + default: 'true' + outputs: + boot-timeout-seconds: + description: 'Boot wait budget for a deferred readiness check' + value: ${{ inputs.boot-timeout-seconds }} simulator-udid: description: 'UDID of the booted iOS simulator' value: ${{ steps.boot.outputs.simulator-udid }} @@ -24,8 +32,14 @@ runs: steps: - name: Resolve and boot iOS test simulator id: boot + env: + WAIT_FOR_BOOT: ${{ inputs.wait-for-boot }} run: | set -euo pipefail + if [[ "$WAIT_FOR_BOOT" != 'true' && "$WAIT_FOR_BOOT" != 'false' ]]; then + echo 'wait-for-boot must be true or false' >&2 + exit 1 + fi RUNTIME_VERSION="${{ inputs.runtime-version }}" RUNTIME_TOKEN="SimRuntime.iOS-${RUNTIME_VERSION//./-}" export RUNTIME_TOKEN @@ -58,15 +72,7 @@ runs: echo "simulator-udid=$UDID" >> "$GITHUB_OUTPUT" xcrun simctl shutdown all || true xcrun simctl boot "$UDID" || true - TIMEOUT_SECONDS="${{ inputs.boot-timeout-seconds }}" - set +e - perl -e 'alarm shift; exec @ARGV' "$TIMEOUT_SECONDS" xcrun simctl bootstatus "$UDID" -b - BOOTSTATUS_EXIT="$?" - set -e - if [ "$BOOTSTATUS_EXIT" -eq 142 ]; then - echo "Timed out waiting ${TIMEOUT_SECONDS}s for simulator $UDID to boot" >&2 - xcrun simctl list devices || true - exit 1 + if [ "$WAIT_FOR_BOOT" = 'true' ]; then + bash scripts/wait-for-ios-simulator.sh "$UDID" "${{ inputs.boot-timeout-seconds }}" fi - exit "$BOOTSTATUS_EXIT" shell: bash diff --git a/.github/actions/smoke-ios-run/action.yml b/.github/actions/smoke-ios-run/action.yml index c11a117407..768bce5cb0 100644 --- a/.github/actions/smoke-ios-run/action.yml +++ b/.github/actions/smoke-ios-run/action.yml @@ -32,6 +32,13 @@ runs: mkdir -p .tmp echo "$FRONTMOST" > .tmp/ios-host-focus-before shell: bash + - name: Boot iOS test simulator + id: ios-simulator + uses: ./.github/actions/boot-ios-test-simulator + with: + runtime-version: ${{ env.IOS_RUNTIME_VERSION }} + preferred-device-name: iPhone 17 Pro + wait-for-boot: 'false' - name: Restore and build iOS XCTest runner uses: ./.github/actions/setup-apple-runner-build with: @@ -41,12 +48,12 @@ runs: gate: swift-runner-ios xcuitest-platform: ios xcuitest-destination: generic/platform=iOS Simulator - - name: Boot iOS test simulator - id: ios-simulator - uses: ./.github/actions/boot-ios-test-simulator - with: - runtime-version: ${{ env.IOS_RUNTIME_VERSION }} - preferred-device-name: iPhone 17 Pro + - name: Wait for iOS test simulator + run: | + bash scripts/wait-for-ios-simulator.sh \ + "${{ steps.ios-simulator.outputs.simulator-udid }}" \ + "${{ steps.ios-simulator.outputs.boot-timeout-seconds }}" + shell: bash - name: Run iOS runner platform checks id: ios-xctest if: steps.xctest-impact.outputs.run != 'false' diff --git a/docs/evidence/ios-boot-build-overlap-2026-10-19.md b/docs/evidence/ios-boot-build-overlap-2026-10-19.md new file mode 100644 index 0000000000..e63a126988 --- /dev/null +++ b/docs/evidence/ios-boot-build-overlap-2026-10-19.md @@ -0,0 +1,57 @@ +# iOS simulator boot and native build overlap + +## Change + +The iOS smoke lane starts its selected simulator before the generic native build. +The build runs while CoreSimulator completes boot. +A required boot-status barrier runs before the XCTest platform checks and public CLI preflight. +The barrier keeps the existing 300-second wait limit and propagates native failures. +Other users of the boot action still wait before the action returns. +No test selection or runtime policy changes. + +## Hosted baseline + +[Apple run 39287002947](https://github.com/callstack/agent-device/actions/runs/39287002947/job/118176526594), source `14f32fb218ac9e5608012bf6a5805652dc61f7fc`, passed in 23m49s. +The native runner preparation took 74.238s. +The subsequent simulator preparation took 109.032s. +These phases ran serially. +Their times show an overlap opportunity, not a promised hosted saving. + +## Local comparison + +The native sources are from base `f8eddcbda35b4b1ee4e28de963925c77d77eab56`. +Only workflow ordering and the shared wait script differ. +The host uses Xcode 26.2 and iOS 26.2 on arm64. +Each sample creates a fresh iPhone 17 Pro simulator and a separate DerivedData directory. +The build uses `scripts/build-xcuitest-apple.sh`, unit-test inclusion, arm64, and `generic/platform=iOS Simulator`. +The serial sample builds before it starts boot. +The overlap sample starts boot before it builds. +Both samples then require `scripts/wait-for-ios-simulator.sh 300` to succeed. +Creation, shutdown and deletion are outside the measured preparation interval. +All four native builds and boot barriers pass. + +| Order | Native build | Preparation wall time | +| --- | ---: | ---: | +| 0-serial | 12.834s | 41.556s | +| 1-overlap | 17.644s | 29.006s | +| 2-overlap | 17.687s | 30.299s | +| 3-serial | 15.872s | 39.922s | + +The two-sample means are 40.739s serial and 29.653s overlapping. +The observed saving is 11.086s (27.2%) for preparation on this host. +Overlap makes each build slower, but reduces the combined preparation time in both comparisons. +These are local samples with evolving system caches, not controlled hosted speed evidence. +The hosted lane uses a different Xcode and simulator image. +Hosted Apple CI must confirm correctness and total job cost. + +## Failure controls and cleanup + +The focused workflow suite passes 40 tests. +It executes the boot action with fake platform tools. +It verifies deferred boot, default blocking behavior, invalid-policy rejection, native failure propagation, and the mandatory readiness barrier before XCTest. +An injected alarm exit verifies the timeout branch without sleeping through a timeout in unit tests. +The real native comparisons above use the actual bounded wait. + +All four owned simulators were shut down and deleted. +Other devices were not changed. +Commands, native build logs, boot logs and results remain under `/private/tmp/pr-apple-boot-benchmark`. diff --git a/scripts/__tests__/apple-ci-impact.test.ts b/scripts/__tests__/apple-ci-impact.test.ts index f1ac6980b1..ea1d8eca06 100644 --- a/scripts/__tests__/apple-ci-impact.test.ts +++ b/scripts/__tests__/apple-ci-impact.test.ts @@ -547,3 +547,158 @@ test('a shallow PR merge still yields a known change set for all Apple selectors fs.rmSync(root, { recursive: true, force: true }); } }); + +test('iOS boot overlaps the native build and readiness remains mandatory before XCTest', () => { + const document = parse( + fs.readFileSync(path.join(repoRoot, '.github/actions/smoke-ios-run/action.yml'), 'utf8'), + ) as { runs: { steps: AppleStep[] } }; + const steps = document.runs.steps; + const boot = steps.findIndex((step) => step.id === 'ios-simulator'); + const build = steps.findIndex( + (step) => step.uses === './.github/actions/setup-apple-runner-build', + ); + const wait = steps.findIndex((step) => step.run?.includes('scripts/wait-for-ios-simulator.sh')); + const tests = steps.findIndex((step) => step.id === 'ios-xctest'); + expect(boot).toBeGreaterThanOrEqual(0); + expect(build).toBeGreaterThan(boot); + expect(wait).toBeGreaterThan(build); + expect(tests).toBeGreaterThan(wait); + expect(steps[boot]!.with?.['wait-for-boot']).toBe('false'); + expect(steps[wait]!.run).toContain('steps.ios-simulator.outputs.simulator-udid'); + expect(steps[wait]!.run).toContain('steps.ios-simulator.outputs.boot-timeout-seconds'); + expect(steps[wait]!.if).toBeUndefined(); + expect(steps[wait]!['continue-on-error']).not.toBe(true); +}); + +function simulatorBootFixture(mode: 'ready' | 'failed' | 'stalled') { + const directory = mkdtempForTestSync('ios-boot-wait-'); + const calls = path.join(directory, 'calls.ndjson'); + fs.writeFileSync( + path.join(directory, 'xcrun'), + `#!${process.execPath} +const fs = require('node:fs'); +const args = process.argv.slice(2); +fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify(args) + '\\n'); +if (args.includes('list')) console.log(JSON.stringify({ devices: { 'com.apple.CoreSimulator.SimRuntime.iOS-26-2': [{ udid: 'owned-ios', name: 'iPhone 17 Pro', isAvailable: true, state: 'Shutdown' }] } })); +if (args.includes('bootstatus')) { + if (process.env.FIXTURE_BOOT_MODE === 'failed') process.exit(7); + if (process.env.FIXTURE_BOOT_MODE === 'stalled') setInterval(() => {}, 1000); +} +`, + { mode: 0o755 }, + ); + if (mode === 'stalled') + fs.writeFileSync( + path.join(directory, 'perl'), + `#!${process.execPath} +const fs = require('node:fs'); +fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify(['alarm', ...process.argv.slice(2)]) + '\\n'); +process.exit(142); +`, + { mode: 0o755 }, + ); + return { + directory, + env: { + ...process.env, + PATH: `${directory}:${process.env.PATH}`, + FIXTURE_CALLS: calls, + FIXTURE_BOOT_MODE: mode, + }, + calls: () => + fs.existsSync(calls) + ? fs + .readFileSync(calls, 'utf8') + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as string[]) + : [], + }; +} + +function bootActionRun(waitForBoot?: string, mode: 'ready' | 'failed' = 'ready') { + const fixture = simulatorBootFixture(mode); + const doc = parse( + fs.readFileSync( + path.join(repoRoot, '.github/actions/boot-ios-test-simulator/action.yml'), + 'utf8', + ), + ) as { inputs: Record; runs: { steps: AppleStep[] } }; + const inputs = { + 'runtime-version': '26.2', + 'preferred-device-name': 'iPhone 17 Pro', + 'boot-timeout-seconds': '2', + }; + const script = doc.runs.steps[0]!.run!.replaceAll( + /\$\{\{ inputs\.([\w-]+) \}\}/g, + (_whole, key: keyof typeof inputs) => inputs[key], + ); + const result = spawnSync('/bin/bash', ['-c', script], { + cwd: repoRoot, + env: { + ...fixture.env, + WAIT_FOR_BOOT: waitForBoot ?? doc.inputs['wait-for-boot']!.default, + GITHUB_OUTPUT: path.join(fixture.directory, 'outputs'), + }, + encoding: 'utf8', + timeout: 10_000, + }); + return { result, calls: fixture.calls() }; +} + +test('deferred boot starts the selected simulator without waiting on its readiness', () => { + const { result, calls } = bootActionRun('false', 'failed'); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + expect(calls).toContainEqual(['simctl', 'boot', 'owned-ios']); + expect(calls.some((args) => args.includes('bootstatus'))).toBe(false); +}); + +test('existing boot action callers still wait and propagate a bootstatus failure', () => { + const { result, calls } = bootActionRun(undefined, 'failed'); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe(7); + expect(calls).toContainEqual(['simctl', 'bootstatus', 'owned-ios', '-b']); +}); + +test('invalid deferred-boot policy fails before changing simulator state', () => { + const { result, calls } = bootActionRun('unexpected'); + expect(result.status).toBe(1); + expect(calls).toEqual([]); +}); + +test.each([ + ['ready', '2', 0], + ['failed', '2', 7], + ['stalled', '1', 1], + ['ready', '0', 1], +] as const)( + 'deferred readiness barrier keeps native outcome and deadline: %s, %s seconds', + (mode, timeout, expected) => { + const fixture = simulatorBootFixture(mode); + const result = spawnSync( + '/bin/bash', + [path.join(repoRoot, 'scripts/wait-for-ios-simulator.sh'), 'owned-ios', timeout], + { env: fixture.env, encoding: 'utf8', timeout: 5000 }, + ); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe(expected); + if (mode === 'stalled') + expect(result.stderr).toContain('Timed out waiting 1s for simulator owned-ios'); + if (timeout === '0') expect(fixture.calls()).toEqual([]); + else if (mode === 'stalled') + expect(fixture.calls()).toContainEqual([ + 'alarm', + '-e', + 'alarm shift; exec @ARGV', + timeout, + 'xcrun', + 'simctl', + 'bootstatus', + 'owned-ios', + '-b', + ]); + else expect(fixture.calls()).toContainEqual(['simctl', 'bootstatus', 'owned-ios', '-b']); + }, +); diff --git a/scripts/wait-for-ios-simulator.sh b/scripts/wait-for-ios-simulator.sh new file mode 100644 index 0000000000..d90fe20b2a --- /dev/null +++ b/scripts/wait-for-ios-simulator.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +UDID="${1:?simulator UDID is required}" +TIMEOUT_SECONDS="${2:?boot timeout is required}" +if [[ ! "$TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then + echo 'Boot timeout must be a positive integer' >&2 + exit 1 +fi +set +e +perl -e 'alarm shift; exec @ARGV' "$TIMEOUT_SECONDS" xcrun simctl bootstatus "$UDID" -b +BOOTSTATUS_EXIT="$?" +set -e +if [ "$BOOTSTATUS_EXIT" -eq 142 ]; then + echo "Timed out waiting ${TIMEOUT_SECONDS}s for simulator $UDID to boot" >&2 + xcrun simctl list devices || true + exit 1 +fi +exit "$BOOTSTATUS_EXIT"