From d12e3e551b7d0a8bda9f485beb3f16af805be520 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Wed, 19 Aug 2026 21:54:58 +0000 Subject: [PATCH 1/5] Add iOS Simulator volume HID support --- .../src/platform/ios_simulator.rs | 4 +- .../src/platform/ios_simulator/input.rs | 52 +++++++++++++++---- .../src/platform/ios_simulator/session.rs | 12 ++++- .../accessibility-ios-sys/src/macos/hid.rs | 42 +++++++++++++++ 4 files changed, 97 insertions(+), 13 deletions(-) diff --git a/packages/accessibility-core/src/platform/ios_simulator.rs b/packages/accessibility-core/src/platform/ios_simulator.rs index f0a38a6..6739377 100644 --- a/packages/accessibility-core/src/platform/ios_simulator.rs +++ b/packages/accessibility-core/src/platform/ios_simulator.rs @@ -32,8 +32,8 @@ pub mod settings; pub use ax::{AxCommand, AxSnapshot, Discovery, ElementDetail, NormalizedRect, spawn_ax_worker}; pub use input::{ - HOME_INDICATOR_BAND, HardwareButton as InputHardwareButton, InputCommand, Orientation, - TouchEdge, TouchPhase, spawn_input_worker, + HOME_INDICATOR_BAND, HardwareButton as InputHardwareButton, InputCapabilities, InputCommand, + Orientation, TouchEdge, TouchPhase, spawn_input_worker, spawn_input_worker_with_capabilities, }; pub use keymap::{KeyStroke, keystroke_for, keystrokes_for}; pub use session::{DeviceInfo, SimSession, StatsReport, StreamStats}; diff --git a/packages/accessibility-core/src/platform/ios_simulator/input.rs b/packages/accessibility-core/src/platform/ios_simulator/input.rs index 7ea8a68..e92990b 100644 --- a/packages/accessibility-core/src/platform/ios_simulator/input.rs +++ b/packages/accessibility-core/src/platform/ios_simulator/input.rs @@ -59,6 +59,9 @@ pub enum TouchEdge { pub enum HardwareButton { Home, Lock, + VolumeUp, + VolumeDown, + Mute, Siri, SideButton, ApplePay, @@ -151,12 +154,26 @@ impl ScrollGesture { /// HID sends block on a dispatch queue round trip. It wakes periodically even /// when idle so a scroll gesture can be lifted after the wheel stops. pub fn spawn_input_worker(udid: &str) -> Result> { + Ok(spawn_input_worker_with_capabilities(udid)?.0) +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct InputCapabilities { + pub arbitrary_hid: bool, +} + +pub fn spawn_input_worker_with_capabilities( + udid: &str, +) -> Result<(Sender, InputCapabilities)> { use accessibility_ios_sys::{ HardwareButton as SysButton, Orientation as SysOrientation, SimulatorHID, TouchEdge as SysEdge, TouchPhase as SysPhase, }; let hid = SimulatorHID::for_device(Some(udid))?; + let capabilities = InputCapabilities { + arbitrary_hid: hid.supports_hid_arbitrary(), + }; let (tx, rx) = mpsc::channel::(); std::thread::Builder::new() @@ -191,14 +208,31 @@ pub fn spawn_input_worker(udid: &str) -> Result> { hid.touch_normalized_edge(x, y, phase, edge) } InputCommand::Button { button } => { - let button = match button { - HardwareButton::Home => SysButton::Home, - HardwareButton::Lock => SysButton::Lock, - HardwareButton::Siri => SysButton::Siri, - HardwareButton::SideButton => SysButton::SideButton, - HardwareButton::ApplePay => SysButton::ApplePay, - }; - hid.press_button(button, 0) + const CONSUMER_PAGE: u32 = 0x0c; + const CONSUMER_MUTE: u32 = 0xe2; + const CONSUMER_VOLUME_UP: u32 = 0xe9; + const CONSUMER_VOLUME_DOWN: u32 = 0xea; + + match button { + HardwareButton::Home => hid.press_button(SysButton::Home, 0), + HardwareButton::Lock => hid.press_button(SysButton::Lock, 0), + HardwareButton::VolumeUp => { + hid.press_hid_button(CONSUMER_PAGE, CONSUMER_VOLUME_UP, 0) + } + HardwareButton::VolumeDown => { + hid.press_hid_button(CONSUMER_PAGE, CONSUMER_VOLUME_DOWN, 0) + } + HardwareButton::Mute => { + hid.press_hid_button(CONSUMER_PAGE, CONSUMER_MUTE, 0) + } + HardwareButton::Siri => hid.press_button(SysButton::Siri, 0), + HardwareButton::SideButton => { + hid.press_button(SysButton::SideButton, 0) + } + HardwareButton::ApplePay => { + hid.press_button(SysButton::ApplePay, 0) + } + } } InputCommand::Key { key_code, @@ -245,7 +279,7 @@ pub fn spawn_input_worker(udid: &str) -> Result> { } })?; - Ok(tx) + Ok((tx, capabilities)) } /// Expand text into key presses and send them. diff --git a/packages/accessibility-core/src/platform/ios_simulator/session.rs b/packages/accessibility-core/src/platform/ios_simulator/session.rs index ffcd8a3..77c5a57 100644 --- a/packages/accessibility-core/src/platform/ios_simulator/session.rs +++ b/packages/accessibility-core/src/platform/ios_simulator/session.rs @@ -21,7 +21,9 @@ use crate::video::{ use super::SimulatorVideoCapture; use super::ax::{AxCommand, AxSnapshot, ElementDetail, spawn_ax_worker}; -use super::input::{InputCommand, Orientation, spawn_input_worker}; +use super::input::{ + InputCapabilities, InputCommand, Orientation, spawn_input_worker_with_capabilities, +}; use super::settings::{self, Setting, SettingKey}; /// How many encoded frames to buffer per subscriber. @@ -93,6 +95,7 @@ pub struct SimSession { stats: Arc, started: Instant, input: std::sync::mpsc::Sender, + input_capabilities: InputCapabilities, ax: mpsc::UnboundedSender, /// Last orientation we asked for. /// @@ -130,7 +133,7 @@ impl SimSession { }; let (capture, resolved_udid) = start_capture(udid, &config, sink)?; - let input = spawn_input_worker(&resolved_udid)?; + let (input, input_capabilities) = spawn_input_worker_with_capabilities(&resolved_udid)?; let ax = spawn_ax_worker(&resolved_udid)?; Ok(Arc::new(Self { @@ -141,6 +144,7 @@ impl SimSession { stats, started: Instant::now(), input, + input_capabilities, ax, orientation: std::sync::Mutex::new(Orientation::Portrait), })) @@ -218,6 +222,10 @@ impl SimSession { } } + pub fn input_capabilities(&self) -> InputCapabilities { + self.input_capabilities + } + pub fn orientation(&self) -> Orientation { *self.orientation.lock().unwrap() } diff --git a/packages/accessibility-ios-sys/src/macos/hid.rs b/packages/accessibility-ios-sys/src/macos/hid.rs index 4ee3c5b..bfa0d24 100644 --- a/packages/accessibility-ios-sys/src/macos/hid.rs +++ b/packages/accessibility-ios-sys/src/macos/hid.rs @@ -11,6 +11,8 @@ use super::*; /// Function pointer types for Indigo message creation (loaded from SimulatorKit via dlsym). type IndigoMessageForButtonFn = unsafe extern "C" fn(source: i32, action: i32, target: i32) -> *mut c_void; +type IndigoMessageForHIDArbitraryFn = + unsafe extern "C" fn(target: u32, page: u32, usage: u32, action: u32) -> *mut c_void; /// `IndigoHIDMessageForMouseNSEvent(CGPoint*, CGPoint*, IndigoHIDTarget, /// NSEventType, NSSize, IndigoHIDEdge)` /// @@ -81,6 +83,7 @@ pub struct SimulatorHID { screen_scale: f64, // Function pointers for message creation msg_for_button: IndigoMessageForButtonFn, + msg_for_hid_arbitrary: Option, msg_for_touch: IndigoMessageForTouchFn, msg_for_keyboard: IndigoMessageForKeyboardFn, } @@ -104,6 +107,11 @@ impl SimulatorHID { std::mem::transmute(sym) }; + let msg_for_hid_arbitrary = unsafe { + let sym = libc::dlsym(handle, c"IndigoHIDMessageForHIDArbitrary".as_ptr()); + (!sym.is_null()).then(|| std::mem::transmute(sym)) + }; + let msg_for_touch: IndigoMessageForTouchFn = unsafe { let sym = libc::dlsym(handle, c"IndigoHIDMessageForMouseNSEvent".as_ptr()); if sym.is_null() { @@ -176,6 +184,7 @@ impl SimulatorHID { screen_size, screen_scale, msg_for_button, + msg_for_hid_arbitrary, msg_for_touch, msg_for_keyboard, }) @@ -383,6 +392,22 @@ impl SimulatorHID { Ok(()) } + pub fn supports_hid_arbitrary(&self) -> bool { + self.msg_for_hid_arbitrary.is_some() + } + + pub fn press_hid_button(&self, page: u32, usage: u32, hold_ms: u64) -> Result<()> { + self.send_hid_button(page, usage, ButtonDirection::Down)?; + + std::thread::sleep(std::time::Duration::from_millis(if hold_ms > 0 { + hold_ms + } else { + 50 + })); + + self.send_hid_button(page, usage, ButtonDirection::Up) + } + /// Send a keyboard key press. /// /// # Arguments @@ -493,6 +518,23 @@ impl SimulatorHID { self.send_message(message, true) } + fn send_hid_button(&self, page: u32, usage: u32, direction: ButtonDirection) -> Result<()> { + const HID_ARBITRARY_BUTTON_TARGET: u32 = 0x32; + + let msg_for_hid_arbitrary = self + .msg_for_hid_arbitrary + .ok_or_else(|| anyhow!("IndigoHIDMessageForHIDArbitrary is unavailable"))?; + let message = unsafe { + msg_for_hid_arbitrary(HID_ARBITRARY_BUTTON_TARGET, page, usage, direction as u32) + }; + + if message.is_null() { + return Err(anyhow!("Failed to create arbitrary HID message")); + } + + self.send_message(message, true) + } + /// Send a keyboard event. fn send_keyboard(&self, key_code: u32, direction: ButtonDirection) -> Result<()> { let message = unsafe { (self.msg_for_keyboard)(key_code as i32, direction as i32) }; From 65306004530a75fb58b22e860fda37896038afdc Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Wed, 19 Aug 2026 22:16:26 +0000 Subject: [PATCH 2/5] Bound simulator settings subprocesses --- .../src/platform/ios_simulator/settings.rs | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/accessibility-core/src/platform/ios_simulator/settings.rs b/packages/accessibility-core/src/platform/ios_simulator/settings.rs index a3bcd47..9056f8e 100644 --- a/packages/accessibility-core/src/platform/ios_simulator/settings.rs +++ b/packages/accessibility-core/src/platform/ios_simulator/settings.rs @@ -10,9 +10,18 @@ //! libAccessibility setters. That is a meaningfully larger piece of work and //! is deliberately not attempted here. +use std::{ + process::{Command, Stdio}, + thread, + time::{Duration, Instant}, +}; + use anyhow::{Context, Result, anyhow}; use serde::{Deserialize, Serialize}; +const SIMCTL_TIMEOUT: Duration = Duration::from_secs(2); +const SIMCTL_POLL_INTERVAL: Duration = Duration::from_millis(10); + /// Content size categories, smallest to largest. /// /// The five `accessibility-*` entries are the extended range that only appears @@ -117,12 +126,40 @@ pub fn write(udid: &str, key: SettingKey, value: &str) -> Result { } fn simctl(args: &[&str]) -> Result { - let output = std::process::Command::new("xcrun") + let mut child = Command::new("xcrun") .arg("simctl") .args(args) - .output() + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() .context("failed to run xcrun simctl")?; + let deadline = Instant::now() + SIMCTL_TIMEOUT; + loop { + if child + .try_wait() + .context("failed to wait for xcrun simctl")? + .is_some() + { + break; + } + if Instant::now() >= deadline { + child.kill().context("failed to terminate xcrun simctl")?; + child + .wait() + .context("failed to reap timed out xcrun simctl")?; + return Err(anyhow!( + "simctl {} timed out after {} seconds", + args.join(" "), + SIMCTL_TIMEOUT.as_secs() + )); + } + thread::sleep(SIMCTL_POLL_INTERVAL); + } + + let output = child + .wait_with_output() + .context("failed to collect xcrun simctl output")?; if !output.status.success() { return Err(anyhow!( "simctl {} failed: {}", From fe840cf79618e44c498e909f15f7edac6e94bb0b Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Fri, 21 Aug 2026 21:09:56 +0000 Subject: [PATCH 3/5] Keep simulator capture responsive during interaction --- .../src/platform/android/video.rs | 3 +- .../src/platform/ios_simulator.rs | 5 ++ .../src/platform/ios_simulator/session.rs | 1 + packages/accessibility-core/src/video/mod.rs | 6 ++ .../src/macos/encoder.rs | 16 ++++- .../src/macos/framebuffer.rs | 60 +++++++++++++++---- .../accessibility-ios-sys/src/macos/stream.rs | 12 +++- 7 files changed, 86 insertions(+), 17 deletions(-) diff --git a/packages/accessibility-core/src/platform/android/video.rs b/packages/accessibility-core/src/platform/android/video.rs index ca71f7f..7186867 100644 --- a/packages/accessibility-core/src/platform/android/video.rs +++ b/packages/accessibility-core/src/platform/android/video.rs @@ -1,7 +1,7 @@ use std::io::Read; use std::sync::mpsc::{self, Receiver, SyncSender}; use std::thread::JoinHandle; -use std::time::Duration; +use std::time::{Duration, Instant}; use accessibility_android_sys::AdbClient; use accessibility_android_sys::emulator::screenrecord::{ @@ -247,6 +247,7 @@ fn emit( } else { FrameKind::Delta }, + captured_at: Instant::now(), }); } } diff --git a/packages/accessibility-core/src/platform/ios_simulator.rs b/packages/accessibility-core/src/platform/ios_simulator.rs index 6739377..5827a7f 100644 --- a/packages/accessibility-core/src/platform/ios_simulator.rs +++ b/packages/accessibility-core/src/platform/ios_simulator.rs @@ -365,6 +365,7 @@ impl SimulatorVideoCapture { sys::ChunkKind::Keyframe => FrameKind::Keyframe, sys::ChunkKind::Delta => FrameKind::Delta, }, + captured_at: chunk.captured_at, }); }); @@ -395,6 +396,10 @@ impl VideoCapture for SimulatorVideoCapture { self.inner.request_keyframe(); } + fn note_interaction(&self) { + self.inner.note_interaction(); + } + fn start_recording(&self, path: &std::path::Path, config: &RecordingConfig) -> Result<()> { self.inner.start_recording( path, diff --git a/packages/accessibility-core/src/platform/ios_simulator/session.rs b/packages/accessibility-core/src/platform/ios_simulator/session.rs index 77c5a57..c2eed27 100644 --- a/packages/accessibility-core/src/platform/ios_simulator/session.rs +++ b/packages/accessibility-core/src/platform/ios_simulator/session.rs @@ -275,6 +275,7 @@ impl SimSession { if let InputCommand::Rotate { orientation } = command { *self.orientation.lock().unwrap() = orientation; } + self.capture.note_interaction(); let _ = self.input.send(command); } diff --git a/packages/accessibility-core/src/video/mod.rs b/packages/accessibility-core/src/video/mod.rs index a739a46..d04ad7f 100644 --- a/packages/accessibility-core/src/video/mod.rs +++ b/packages/accessibility-core/src/video/mod.rs @@ -13,6 +13,7 @@ //! reports [`VideoCapture`] as unsupported. use std::sync::Arc; +use std::time::Instant; use anyhow::Result; use bytes::Bytes; @@ -83,6 +84,7 @@ impl FrameKind { pub struct EncodedFrame { pub data: Bytes, pub kind: FrameKind, + pub captured_at: Instant, } /// Pixel dimensions of the captured display. @@ -192,6 +194,10 @@ pub trait VideoCapture: Send + Sync { /// from a WebRTC receiver. fn request_keyframe(&self); + /// Keep capture at its configured cadence while interaction or resulting + /// animation is active. + fn note_interaction(&self) {} + /// Stop capturing and release platform resources. fn stop(&mut self); diff --git a/packages/accessibility-ios-sys/src/macos/encoder.rs b/packages/accessibility-ios-sys/src/macos/encoder.rs index 2f6d2b1..0bd18cd 100644 --- a/packages/accessibility-ios-sys/src/macos/encoder.rs +++ b/packages/accessibility-ios-sys/src/macos/encoder.rs @@ -14,6 +14,7 @@ use std::ffi::c_void; use std::ptr::NonNull; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; +use std::time::Instant; use anyhow::{Result, anyhow}; use block2::RcBlock; @@ -58,6 +59,7 @@ pub enum ChunkKind { pub struct EncodedChunk { pub data: Bytes, pub kind: ChunkKind, + pub captured_at: Instant, } /// Bits per pixel per frame to aim for when no explicit bitrate is given. @@ -214,7 +216,13 @@ impl H264Encoder { /// /// `width`/`height` describe the *source*; the session may be smaller if /// the config caps the long edge, in which case VideoToolbox scales. - pub fn encode(&mut self, image: &CVImageBuffer, width: i32, height: i32) -> Result<()> { + pub fn encode( + &mut self, + image: &CVImageBuffer, + width: i32, + height: i32, + captured_at: Instant, + ) -> Result<()> { if self.session.is_none() || self.source_dimensions != (width, height) { self.source_dimensions = (width, height); self.dimensions = self.config.encode_size(width, height); @@ -255,7 +263,7 @@ impl H264Encoder { return; } let sample = unsafe { &*sample }; - for chunk in package_sample(sample, nal_format, &emitted) { + for chunk in package_sample(sample, nal_format, &emitted, captured_at) { sink(chunk); } }, @@ -395,6 +403,7 @@ fn package_sample( sample: &CMSampleBuffer, format: NalFormat, emitted_parameter_set: &Mutex, + captured_at: Instant, ) -> Vec { let is_keyframe = sample_is_keyframe(sample); let Some(avcc) = sample_bytes(sample) else { @@ -418,6 +427,7 @@ fn package_sample( chunks.push(EncodedChunk { data: Bytes::from(build_avcc_record(sps, pps)), kind: ChunkKind::ParameterSet, + captured_at, }); } } @@ -428,6 +438,7 @@ fn package_sample( } else { ChunkKind::Delta }, + captured_at, }); } NalFormat::AnnexB => { @@ -448,6 +459,7 @@ fn package_sample( } else { ChunkKind::Delta }, + captured_at, }); } } diff --git a/packages/accessibility-ios-sys/src/macos/framebuffer.rs b/packages/accessibility-ios-sys/src/macos/framebuffer.rs index 5cfc4a8..7b28af4 100644 --- a/packages/accessibility-ios-sys/src/macos/framebuffer.rs +++ b/packages/accessibility-ios-sys/src/macos/framebuffer.rs @@ -24,7 +24,7 @@ use std::ffi::c_void; use std::ptr::NonNull; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; use anyhow::{Result, anyhow}; @@ -62,12 +62,14 @@ const FRAMEBUFFER_PORT_ID: &str = "com.apple.framebuffer.display"; /// re-emits keeps both honest. const IDLE_INTERVAL: Duration = Duration::from_millis(200); -/// Ticks between re-wire attempts while no frame has ever been captured. -/// +/// Keep polling briefly after input stops so animations and inertial scrolling +/// do not fall back to the idle cadence between input events. +const INTERACTION_HOLD: Duration = Duration::from_secs(1); + /// Descriptors are sometimes created lazily, so a registration that happened /// too early silently yields nothing. Rebuilding the port graph roughly once a /// second recovers from that. -const REWIRE_TICKS: u64 = 5; +const REWIRE_INTERVAL: Duration = Duration::from_secs(1); /// A frame handed to the sink, valid only for the duration of the call. /// @@ -135,6 +137,9 @@ struct CaptureState { blocks: Mutex>, sink: Mutex>, last_capture: Mutex, + active_until: Mutex, + activity: Condvar, + active_interval_micros: AtomicU64, frame_count: AtomicU64, rewire_count: AtomicU64, width: AtomicUsize, @@ -234,7 +239,8 @@ impl CaptureState { return; }; - *self.last_capture.lock().unwrap() = Instant::now(); + let captured_at = Instant::now(); + *self.last_capture.lock().unwrap() = captured_at; self.frame_count.fetch_add(1, Ordering::Relaxed); @@ -247,7 +253,7 @@ impl CaptureState { pixel_buffer: &live, width: CVPixelBufferGetWidth(&live) as u32, height: CVPixelBufferGetHeight(&live) as u32, - captured_at: Instant::now(), + captured_at, }); } } @@ -369,6 +375,9 @@ impl SimFramebuffer { blocks: Mutex::new(Vec::new()), sink: Mutex::new(None), last_capture: Mutex::new(Instant::now()), + active_until: Mutex::new(Instant::now()), + activity: Condvar::new(), + active_interval_micros: AtomicU64::new(1_000_000 / 60), frame_count: AtomicU64::new(0), rewire_count: AtomicU64::new(0), width: AtomicUsize::new(0), @@ -392,6 +401,17 @@ impl SimFramebuffer { *self.state.sink.lock().unwrap() = sink; } + pub fn set_active_frame_rate(&self, fps: u32) { + self.state + .active_interval_micros + .store(1_000_000 / u64::from(fps.max(1)), Ordering::Relaxed); + } + + pub fn note_interaction(&self) { + *self.state.active_until.lock().unwrap() = Instant::now() + INTERACTION_HOLD; + self.state.activity.notify_one(); + } + /// Begin capturing. Calling twice rebuilds the pipeline. pub fn start(&mut self) -> Result<()> { self.state.running.store(true, Ordering::Relaxed); @@ -407,24 +427,37 @@ impl SimFramebuffer { } let state = Arc::clone(&self.state); self.idle_thread = Some(std::thread::spawn(move || { - let mut tick: u64 = 0; + let mut next_rewire = Instant::now() + REWIRE_INTERVAL; while state.running.load(Ordering::Relaxed) { - std::thread::sleep(IDLE_INTERVAL); + let active_until = state.active_until.lock().unwrap(); + let active = *active_until > Instant::now(); + let interval = if active { + Duration::from_micros(state.active_interval_micros.load(Ordering::Relaxed)) + } else { + IDLE_INTERVAL + }; + let (active_until, _) = + state.activity.wait_timeout(active_until, interval).unwrap(); + drop(active_until); if !state.running.load(Ordering::Relaxed) { break; } - tick += 1; + let active = *state.active_until.lock().unwrap() > Instant::now(); + let interval = if active { + Duration::from_micros(state.active_interval_micros.load(Ordering::Relaxed)) + } else { + IDLE_INTERVAL + }; let idle_for = state.last_capture.lock().unwrap().elapsed(); - if idle_for >= IDLE_INTERVAL { + if idle_for >= interval { state.capture(true); } // Self-heal only until the first frame ever lands. After that // a silent pipeline means an idle screen, not a broken graph. - if state.frame_count.load(Ordering::Relaxed) == 0 - && tick.is_multiple_of(REWIRE_TICKS) - { + if state.frame_count.load(Ordering::Relaxed) == 0 && Instant::now() >= next_rewire { + next_rewire = Instant::now() + REWIRE_INTERVAL; state.rewire_count.fetch_add(1, Ordering::Relaxed); // Descriptors are sometimes created lazily, so a // registration that happened too early yields nothing. @@ -437,6 +470,7 @@ impl SimFramebuffer { pub fn stop(&mut self) { self.state.running.store(false, Ordering::Relaxed); + self.state.activity.notify_one(); if let Some(handle) = self.idle_thread.take() { let _ = handle.join(); } diff --git a/packages/accessibility-ios-sys/src/macos/stream.rs b/packages/accessibility-ios-sys/src/macos/stream.rs index da7ddf3..075dc91 100644 --- a/packages/accessibility-ios-sys/src/macos/stream.rs +++ b/packages/accessibility-ios-sys/src/macos/stream.rs @@ -38,6 +38,7 @@ impl SimVideoStream { /// use is a bounded channel that drops on overflow. pub fn start(udid: Option<&str>, config: EncoderConfig, sink: ChunkSink) -> Result { let mut framebuffer = SimFramebuffer::new(udid)?; + framebuffer.set_active_frame_rate(config.fps); let force_keyframe = Arc::new(AtomicBool::new(false)); let mut encoder = H264Encoder::new(config, Arc::clone(&force_keyframe), sink)?; @@ -48,7 +49,12 @@ impl SimVideoStream { // `CVPixelBuffer` derefs to `CVImageBuffer`, which is what // VideoToolbox wants. let image: &CVImageBuffer = frame.pixel_buffer; - if let Err(error) = encoder.encode(image, frame.width as i32, frame.height as i32) { + if let Err(error) = encoder.encode( + image, + frame.width as i32, + frame.height as i32, + frame.captured_at, + ) { eprintln!("[capture] encode failed: {error}"); } @@ -111,6 +117,10 @@ impl SimVideoStream { self.force_keyframe.store(true, Ordering::Relaxed); } + pub fn note_interaction(&self) { + self.framebuffer.note_interaction(); + } + /// Begin recording to `path`. Fails if one is already running. pub fn start_recording(&self, path: &Path, config: RecordingConfig) -> Result<()> { let geometry = self.geometry(); From 61872239c0560dc07b547ab58abbc3fd0d46776e Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Fri, 21 Aug 2026 23:05:01 +0000 Subject: [PATCH 4/5] Collapse simulator input worker capability API Fold spawn_input_worker_with_capabilities back into spawn_input_worker, and share the framebuffer tick interval between the wait and the capture check. --- .../src/platform/ios_simulator.rs | 2 +- .../src/platform/ios_simulator/input.rs | 21 +++++++------ .../src/platform/ios_simulator/session.rs | 6 ++-- .../src/macos/framebuffer.rs | 30 +++++++++++-------- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/packages/accessibility-core/src/platform/ios_simulator.rs b/packages/accessibility-core/src/platform/ios_simulator.rs index 5827a7f..b3265ba 100644 --- a/packages/accessibility-core/src/platform/ios_simulator.rs +++ b/packages/accessibility-core/src/platform/ios_simulator.rs @@ -33,7 +33,7 @@ pub mod settings; pub use ax::{AxCommand, AxSnapshot, Discovery, ElementDetail, NormalizedRect, spawn_ax_worker}; pub use input::{ HOME_INDICATOR_BAND, HardwareButton as InputHardwareButton, InputCapabilities, InputCommand, - Orientation, TouchEdge, TouchPhase, spawn_input_worker, spawn_input_worker_with_capabilities, + Orientation, TouchEdge, TouchPhase, spawn_input_worker, }; pub use keymap::{KeyStroke, keystroke_for, keystrokes_for}; pub use session::{DeviceInfo, SimSession, StatsReport, StreamStats}; diff --git a/packages/accessibility-core/src/platform/ios_simulator/input.rs b/packages/accessibility-core/src/platform/ios_simulator/input.rs index e92990b..47cbe0a 100644 --- a/packages/accessibility-core/src/platform/ios_simulator/input.rs +++ b/packages/accessibility-core/src/platform/ios_simulator/input.rs @@ -148,23 +148,22 @@ impl ScrollGesture { } } -/// Start the HID worker thread and return its command channel. +/// What the simulator's HID interface supports on this host. /// -/// The worker owns the `SimulatorHID` because it is not `Sync`, and because -/// HID sends block on a dispatch queue round trip. It wakes periodically even -/// when idle so a scroll gesture can be lifted after the wheel stops. -pub fn spawn_input_worker(udid: &str) -> Result> { - Ok(spawn_input_worker_with_capabilities(udid)?.0) -} - +/// `IndigoHIDMessageForHIDArbitrary` is resolved at runtime, so buttons that +/// ride on it (volume, mute) are unavailable on Xcode versions without it. #[derive(Debug, Clone, Copy, Default)] pub struct InputCapabilities { pub arbitrary_hid: bool, } -pub fn spawn_input_worker_with_capabilities( - udid: &str, -) -> Result<(Sender, InputCapabilities)> { +/// Start the HID worker thread and return its command channel plus the +/// capabilities the worker's `SimulatorHID` resolved. +/// +/// The worker owns the `SimulatorHID` because it is not `Sync`, and because +/// HID sends block on a dispatch queue round trip. It wakes periodically even +/// when idle so a scroll gesture can be lifted after the wheel stops. +pub fn spawn_input_worker(udid: &str) -> Result<(Sender, InputCapabilities)> { use accessibility_ios_sys::{ HardwareButton as SysButton, Orientation as SysOrientation, SimulatorHID, TouchEdge as SysEdge, TouchPhase as SysPhase, diff --git a/packages/accessibility-core/src/platform/ios_simulator/session.rs b/packages/accessibility-core/src/platform/ios_simulator/session.rs index c2eed27..6c1bc32 100644 --- a/packages/accessibility-core/src/platform/ios_simulator/session.rs +++ b/packages/accessibility-core/src/platform/ios_simulator/session.rs @@ -21,9 +21,7 @@ use crate::video::{ use super::SimulatorVideoCapture; use super::ax::{AxCommand, AxSnapshot, ElementDetail, spawn_ax_worker}; -use super::input::{ - InputCapabilities, InputCommand, Orientation, spawn_input_worker_with_capabilities, -}; +use super::input::{InputCapabilities, InputCommand, Orientation, spawn_input_worker}; use super::settings::{self, Setting, SettingKey}; /// How many encoded frames to buffer per subscriber. @@ -133,7 +131,7 @@ impl SimSession { }; let (capture, resolved_udid) = start_capture(udid, &config, sink)?; - let (input, input_capabilities) = spawn_input_worker_with_capabilities(&resolved_udid)?; + let (input, input_capabilities) = spawn_input_worker(&resolved_udid)?; let ax = spawn_ax_worker(&resolved_udid)?; Ok(Arc::new(Self { diff --git a/packages/accessibility-ios-sys/src/macos/framebuffer.rs b/packages/accessibility-ios-sys/src/macos/framebuffer.rs index 7b28af4..45f48b9 100644 --- a/packages/accessibility-ios-sys/src/macos/framebuffer.rs +++ b/packages/accessibility-ios-sys/src/macos/framebuffer.rs @@ -66,6 +66,9 @@ const IDLE_INTERVAL: Duration = Duration::from_millis(200); /// do not fall back to the idle cadence between input events. const INTERACTION_HOLD: Duration = Duration::from_secs(1); +/// How long to wait between re-wire attempts while no frame has ever been +/// captured. +/// /// Descriptors are sometimes created lazily, so a registration that happened /// too early silently yields nothing. Rebuilding the port graph roughly once a /// second recovers from that. @@ -258,6 +261,17 @@ impl CaptureState { } } + /// How long the idle thread should wait between forced re-emits: the + /// configured capture cadence while interaction is still recent, the idle + /// heartbeat once it has lapsed. + fn tick_interval(&self, active_until: Instant) -> Duration { + if active_until > Instant::now() { + Duration::from_micros(self.active_interval_micros.load(Ordering::Relaxed)) + } else { + IDLE_INTERVAL + } + } + /// Walk the IO port graph and register screen callbacks on every /// framebuffer descriptor, replacing any existing registration. fn wire_up(self: &Arc) -> Result<()> { @@ -430,12 +444,7 @@ impl SimFramebuffer { let mut next_rewire = Instant::now() + REWIRE_INTERVAL; while state.running.load(Ordering::Relaxed) { let active_until = state.active_until.lock().unwrap(); - let active = *active_until > Instant::now(); - let interval = if active { - Duration::from_micros(state.active_interval_micros.load(Ordering::Relaxed)) - } else { - IDLE_INTERVAL - }; + let interval = state.tick_interval(*active_until); let (active_until, _) = state.activity.wait_timeout(active_until, interval).unwrap(); drop(active_until); @@ -443,12 +452,9 @@ impl SimFramebuffer { break; } - let active = *state.active_until.lock().unwrap() > Instant::now(); - let interval = if active { - Duration::from_micros(state.active_interval_micros.load(Ordering::Relaxed)) - } else { - IDLE_INTERVAL - }; + // Re-read after the wait: an interaction may have arrived while + // the thread was parked, which shortens the interval. + let interval = state.tick_interval(*state.active_until.lock().unwrap()); let idle_for = state.last_capture.lock().unwrap().elapsed(); if idle_for >= interval { state.capture(true); From c752d1b4068657f1b68c40de45490bb46a10a5c1 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Fri, 21 Aug 2026 17:27:19 -0700 Subject: [PATCH 5/5] Replace simctl subprocess calls with direct SimDevice API for settings Move iOS simulator settings from spawning `xcrun simctl ui` subprocesses to calling CoreSimulator's `SimDevice` getters and setters directly. Run the synchronous SimDevice calls on a dedicated worker thread and expose async methods on SimulatorControl. Add timeout handling and proper error propagation. --- .../ios-simulator-accessibility/SKILL.md | 13 + .devin/skills/ios-simulator-input/SKILL.md | 5 + .../src/platform/ios_simulator.rs | 22 +- .../src/platform/ios_simulator/session.rs | 27 +- .../src/platform/ios_simulator/settings.rs | 281 ++++++----- packages/accessibility-ios-sys/src/macos.rs | 4 + .../accessibility-ios-sys/src/macos/common.rs | 4 + .../src/macos/control.rs | 441 ++++++++++++++++++ .../src/macos/encoder.rs | 4 +- .../src/macos/framebuffer.rs | 8 + .../accessibility-ios-sys/src/macos/hid.rs | 5 + .../accessibility-ios-sys/src/macos/reader.rs | 80 ++-- .../src/macos/reader/actions.rs | 183 +++++--- .../src/macos/recorder.rs | 1 + .../accessibility-ios-sys/src/macos/stream.rs | 9 +- packages/accessibility-serve/src/http.rs | 19 +- 16 files changed, 871 insertions(+), 235 deletions(-) create mode 100644 packages/accessibility-ios-sys/src/macos/control.rs diff --git a/.devin/skills/ios-simulator-accessibility/SKILL.md b/.devin/skills/ios-simulator-accessibility/SKILL.md index d29197f..2f0647e 100644 --- a/.devin/skills/ios-simulator-accessibility/SKILL.md +++ b/.devin/skills/ios-simulator-accessibility/SKILL.md @@ -23,6 +23,19 @@ Miss that and attribute reads do not fail, they silently return an empty label and a zero frame. `get_tree` already does this; see the matching step in `get_element_at_point`. +## SpringBoard crash remediation + +A dead SpringBoard can leave CoreSimulatorBridge serving a stale root with a +zero frame. Spawn the runtime's `bin/launchctl` through +`SimDevice.spawnAsyncWithPath:...` and run `stop com.apple.CoreSimulator.bridge`. +The service is kept alive and respawns automatically; after it exits, retry the +accessibility query once. This was verified on Xcode 26.6 by killing SpringBoard: +the retry returned a healthy tree from the replacement SpringBoard PID. + +The spawn completion and termination callbacks share one `Condvar`, so their +results must also share one mutex. macOS rejects using one condition variable +with two mutexes and panics before the CoreSimulator call can complete. + ## Backdrops swallow hit tests Every app has full-screen backdrops: the Application node plus one or more diff --git a/.devin/skills/ios-simulator-input/SKILL.md b/.devin/skills/ios-simulator-input/SKILL.md index 3a3bc67..54e1fb8 100644 --- a/.devin/skills/ios-simulator-input/SKILL.md +++ b/.devin/skills/ios-simulator-input/SKILL.md @@ -82,3 +82,8 @@ not publish that port. colour filters, transparency, VoiceOver) have no simctl verb and need a helper binary spawned inside the simulator that drives the private libAccessibility setters. + +The direct `SimDevice` getters return raw integers. Measured on Xcode 26.6: +appearance is 1 light / 2 dark, content size is 1 through 12, and increase +contrast is **1 disabled / 2 enabled** — it is not a 0/1 boolean even though +the setter accepts a BOOL. diff --git a/packages/accessibility-core/src/platform/ios_simulator.rs b/packages/accessibility-core/src/platform/ios_simulator.rs index b3265ba..88f788f 100644 --- a/packages/accessibility-core/src/platform/ios_simulator.rs +++ b/packages/accessibility-core/src/platform/ios_simulator.rs @@ -49,6 +49,10 @@ pub fn load_frameworks() -> Result<()> { /// /// This is a safe core wrapper around `accessibility-ios-sys`; it does not expose /// Objective-C, CoreFoundation, or libc handles outside the sys crate. +/// +/// Its inherent accessibility and HID methods are synchronous and may block on +/// simulator IPC or deliberate input pacing. Session users should prefer the +/// dedicated async worker APIs on [`SimSession`]. pub struct IOSSimulatorAccessibility { inner: sys::IOSSimulatorAccessibility, cache: ElementCache, @@ -75,6 +79,8 @@ impl IOSSimulatorAccessibility { } /// Get the accessibility tree from the frontmost app in the simulator. + /// + /// This synchronously waits on recursive accessibility bridge queries. pub fn get_tree(&mut self, filter: &TreeFilter) -> Result { self.clear_local_cache(); @@ -169,6 +175,9 @@ impl IOSSimulatorAccessibility { } /// Capture a screenshot of the entire simulator screen. + /// + /// This blocks while waiting for a framebuffer and encoding PNG. Async + /// callers should use the [`AccessibilityReader`] implementation instead. pub fn capture_screen(&self) -> Result { self.inner.capture_screen().map(from_sys_screenshot) } @@ -181,6 +190,9 @@ impl IOSSimulatorAccessibility { } /// Capture a screenshot of a specific element. + /// + /// This blocks for full-screen capture, image decoding, cropping, and PNG + /// re-encoding. pub fn capture_element(&mut self, id: ElementKey) -> Result { let sys_id = self.sys_id(id)?; self.inner.capture_element(sys_id).map(from_sys_screenshot) @@ -300,7 +312,15 @@ impl AccessibilityReader for IOSSimulatorAccessibility { &self, _target: &Target, ) -> impl std::future::Future> { - async move { IOSSimulatorAccessibility::capture_screen(self) } + let udid = self.device_udid().to_string(); + async move { + tokio::task::spawn_blocking(move || { + sys::IOSSimulatorAccessibility::capture_screen_for_device(Some(&udid)) + .map(from_sys_screenshot) + }) + .await + .map_err(|error| anyhow!("simulator screenshot worker failed: {error}"))? + } } fn get_screen_bounds( diff --git a/packages/accessibility-core/src/platform/ios_simulator/session.rs b/packages/accessibility-core/src/platform/ios_simulator/session.rs index 6c1bc32..9c2ed5e 100644 --- a/packages/accessibility-core/src/platform/ios_simulator/session.rs +++ b/packages/accessibility-core/src/platform/ios_simulator/session.rs @@ -22,7 +22,7 @@ use crate::video::{ use super::SimulatorVideoCapture; use super::ax::{AxCommand, AxSnapshot, ElementDetail, spawn_ax_worker}; use super::input::{InputCapabilities, InputCommand, Orientation, spawn_input_worker}; -use super::settings::{self, Setting, SettingKey}; +use super::settings::{Setting, SettingKey, SimulatorControl}; /// How many encoded frames to buffer per subscriber. /// @@ -95,6 +95,7 @@ pub struct SimSession { input: std::sync::mpsc::Sender, input_capabilities: InputCapabilities, ax: mpsc::UnboundedSender, + control: SimulatorControl, /// Last orientation we asked for. /// /// The framebuffer is always portrait-native — rotating the device @@ -104,7 +105,17 @@ pub struct SimSession { } impl SimSession { - /// Attach to a booted simulator and start capturing. + /// Attach to a booted simulator and start its independent runtime lanes. + /// + /// Starts: + /// + /// 1. Framebuffer capture and video encoding. + /// 2. Low-latency HID input forwarding. + /// 3. Accessibility tree and hit-test handling. + /// 4. Direct CoreSimulator settings control. + /// + /// Framework loading and lane initialization are synchronous and may block + /// before the session is returned. pub fn start(udid: Option<&str>, config: VideoConfig) -> Result> { let (frames, _) = broadcast::channel(FRAME_BUFFER); let latest_parameter_set = Arc::new(std::sync::Mutex::new(None)); @@ -133,6 +144,7 @@ impl SimSession { let (capture, resolved_udid) = start_capture(udid, &config, sink)?; let (input, input_capabilities) = spawn_input_worker(&resolved_udid)?; let ax = spawn_ax_worker(&resolved_udid)?; + let control = SimulatorControl::start(&resolved_udid)?; Ok(Arc::new(Self { device_udid: resolved_udid, @@ -144,6 +156,7 @@ impl SimSession { input, input_capabilities, ax, + control, orientation: std::sync::Mutex::new(Orientation::Portrait), })) } @@ -251,6 +264,8 @@ impl SimSession { Ok(path) } + /// Finalize the active recording, blocking for up to 30 seconds while the + /// writer flushes the MP4 index. pub fn stop_recording(&self) -> Result { self.capture.stop_recording() } @@ -259,12 +274,12 @@ impl SimSession { self.capture.recording_frames() } - pub fn settings(&self) -> Vec { - settings::read_all(&self.device_udid) + pub async fn settings(&self) -> Result> { + self.control.read_all().await } - pub fn set_setting(&self, key: SettingKey, value: &str) -> Result { - settings::write(&self.device_udid, key, value) + pub async fn set_setting(&self, key: SettingKey, value: &str) -> Result { + self.control.write(key, value).await } /// Queue an input event. Fire-and-forget: pointer events must never block diff --git a/packages/accessibility-core/src/platform/ios_simulator/settings.rs b/packages/accessibility-core/src/platform/ios_simulator/settings.rs index 9056f8e..a980bec 100644 --- a/packages/accessibility-core/src/platform/ios_simulator/settings.rs +++ b/packages/accessibility-core/src/platform/ios_simulator/settings.rs @@ -1,26 +1,27 @@ //! Simulator-wide UI settings. //! -//! These go through `simctl ui`, which is the same mechanism Xcode's Devices -//! window uses. Only the three options simctl actually implements are exposed: -//! appearance, increase contrast, and content size. +//! These go through CoreSimulator's direct `SimDevice` interface, which is the +//! same mechanism Xcode's Devices window uses. Only the three options exposed +//! by that interface are supported: appearance, increase contrast, and content +//! size. //! //! The Devices window also offers reduce-motion, colour filters, transparency -//! and VoiceOver, but simctl has no verb for those — they require a helper -//! binary spawned *inside* the simulator that drives the private +//! and VoiceOver, but `SimDevice` has no selector for those — they require a +//! helper binary spawned *inside* the simulator that drives the private //! libAccessibility setters. That is a meaningfully larger piece of work and //! is deliberately not attempted here. -use std::{ - process::{Command, Stdio}, - thread, - time::{Duration, Instant}, -}; +use std::time::Duration; -use anyhow::{Context, Result, anyhow}; +use accessibility_ios_sys::{ + SimulatorAppearance, SimulatorContentSize, SimulatorDevice, SimulatorIncreaseContrast, +}; +use anyhow::{Result, anyhow}; use serde::{Deserialize, Serialize}; +use tokio::sync::{mpsc, oneshot}; -const SIMCTL_TIMEOUT: Duration = Duration::from_secs(2); -const SIMCTL_POLL_INTERVAL: Duration = Duration::from_millis(10); +const CONTROL_TIMEOUT: Duration = Duration::from_secs(2); +const CONTROL_QUEUE_CAPACITY: usize = 16; /// Content size categories, smallest to largest. /// @@ -53,15 +54,6 @@ pub enum SettingKey { } impl SettingKey { - /// The `simctl ui` subcommand for this setting. - fn verb(self) -> &'static str { - match self { - SettingKey::Appearance => "appearance", - SettingKey::IncreaseContrast => "increase_contrast", - SettingKey::ContentSize => "content_size", - } - } - pub fn allowed_values(self) -> &'static [&'static str] { match self { SettingKey::Appearance => APPEARANCES, @@ -77,6 +69,73 @@ impl SettingKey { SettingKey::ContentSize, ] } + + fn read(self, device: &SimulatorDevice) -> Result { + let value = match self { + Self::Appearance => match device.appearance()? { + SimulatorAppearance::Light => "light", + SimulatorAppearance::Dark => "dark", + }, + Self::IncreaseContrast => match device.increase_contrast()? { + SimulatorIncreaseContrast::Disabled => "disabled", + SimulatorIncreaseContrast::Enabled => "enabled", + }, + Self::ContentSize => CONTENT_SIZES[device.content_size()?.index()], + }; + Ok(value.to_string()) + } + + fn write(self, device: &SimulatorDevice, value: &str) -> Result { + match self { + Self::Appearance => { + let appearance = match value { + "light" => SimulatorAppearance::Light, + "dark" => SimulatorAppearance::Dark, + _ => unreachable!("validated appearance"), + }; + device.set_appearance(appearance)?; + } + Self::IncreaseContrast => { + let contrast = match value { + "disabled" => SimulatorIncreaseContrast::Disabled, + "enabled" => SimulatorIncreaseContrast::Enabled, + _ => unreachable!("validated increase contrast"), + }; + device.set_increase_contrast(contrast)?; + } + Self::ContentSize => { + let size = match value { + "increment" => device.content_size()?.step(1), + "decrement" => device.content_size()?.step(-1), + value => { + let index = CONTENT_SIZES + .iter() + .position(|candidate| *candidate == value) + .expect("validated content size"); + SimulatorContentSize::try_from(index as i64 + 1)? + } + }; + device.set_content_size(size)?; + } + } + self.read(device) + } + + fn validate(self, value: &str) -> Result<()> { + // `content_size` also accepts increment/decrement, which are not in the + // reported value set but are the ergonomic way to drive it from a UI. + let stepping = + matches!(self, Self::ContentSize) && matches!(value, "increment" | "decrement"); + + if !stepping && !self.allowed_values().contains(&value) { + return Err(anyhow!( + "'{value}' is not valid for {:?}; expected one of {}", + self, + self.allowed_values().join(", ") + )); + } + Ok(()) + } } /// One setting and its current value, as reported by the simulator. @@ -88,86 +147,107 @@ pub struct Setting { pub allowed: &'static [&'static str], } -/// Read every supported setting from the device. -pub fn read_all(udid: &str) -> Vec { - SettingKey::all() - .into_iter() - .map(|key| Setting { - key, - // A failed read is reported as unknown rather than failing the - // whole request; one unsupported option should not blank the UI. - value: read(udid, key).unwrap_or_else(|_| "unknown".to_string()), - allowed: key.allowed_values(), - }) - .collect() +impl Setting { + fn read_all(device: &SimulatorDevice) -> Vec { + SettingKey::all() + .into_iter() + .map(|key| Self { + key, + // A failed read is reported as unknown rather than failing the + // whole request; one unsupported option should not blank the UI. + value: key.read(device).unwrap_or_else(|_| "unknown".to_string()), + allowed: key.allowed_values(), + }) + .collect() + } } -pub fn read(udid: &str, key: SettingKey) -> Result { - let output = simctl(&["ui", udid, key.verb()])?; - Ok(output.trim().to_string()) +enum ControlCommand { + ReadAll { + reply: oneshot::Sender>, + }, + Write { + key: SettingKey, + value: String, + reply: oneshot::Sender>, + }, } -pub fn write(udid: &str, key: SettingKey, value: &str) -> Result { - // `content_size` also accepts increment/decrement, which are not in the - // reported value set but are the ergonomic way to drive it from a UI. - let stepping = - matches!(key, SettingKey::ContentSize) && matches!(value, "increment" | "decrement"); - - if !stepping && !key.allowed_values().contains(&value) { - return Err(anyhow!( - "'{value}' is not valid for {:?}; expected one of {}", - key, - key.allowed_values().join(", ") - )); +impl ControlCommand { + fn execute(self, device: &SimulatorDevice) { + match self { + Self::ReadAll { reply } => { + let _ = reply.send(Setting::read_all(device)); + } + Self::Write { key, value, reply } => { + let _ = reply.send(key.write(device, &value)); + } + } } +} - simctl(&["ui", udid, key.verb(), value])?; - read(udid, key) +#[derive(Clone)] +pub(super) struct SimulatorControl { + commands: mpsc::Sender, } -fn simctl(args: &[&str]) -> Result { - let mut child = Command::new("xcrun") - .arg("simctl") - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .context("failed to run xcrun simctl")?; - - let deadline = Instant::now() + SIMCTL_TIMEOUT; - loop { - if child - .try_wait() - .context("failed to wait for xcrun simctl")? - .is_some() - { - break; - } - if Instant::now() >= deadline { - child.kill().context("failed to terminate xcrun simctl")?; - child - .wait() - .context("failed to reap timed out xcrun simctl")?; - return Err(anyhow!( - "simctl {} timed out after {} seconds", - args.join(" "), - SIMCTL_TIMEOUT.as_secs() - )); - } - thread::sleep(SIMCTL_POLL_INTERVAL); +impl SimulatorControl { + /// Attach the direct CoreSimulator control lane. + /// + /// Starts: + /// + /// 1. A dedicated worker thread that owns the `SimDevice` and serializes + /// its synchronous settings round trips. + pub(super) fn start(udid: &str) -> Result { + let device = SimulatorDevice::for_device(Some(udid))?; + let (commands, mut receiver) = mpsc::channel::(CONTROL_QUEUE_CAPACITY); + std::thread::Builder::new() + .name("sim-control".into()) + .spawn(move || { + while let Some(command) = receiver.blocking_recv() { + command.execute(&device); + } + })?; + Ok(Self { commands }) + } + + /// Read every supported setting from the device. + pub(super) async fn read_all(&self) -> Result> { + let (reply, response) = oneshot::channel(); + let request = async { + self.commands + .send(ControlCommand::ReadAll { reply }) + .await + .map_err(|_| anyhow!("simulator control worker stopped"))?; + response + .await + .map_err(|_| anyhow!("simulator control worker stopped")) + }; + tokio::time::timeout(CONTROL_TIMEOUT, request) + .await + .map_err(|_| anyhow!("simulator settings read timed out"))? } - let output = child - .wait_with_output() - .context("failed to collect xcrun simctl output")?; - if !output.status.success() { - return Err(anyhow!( - "simctl {} failed: {}", - args.join(" "), - String::from_utf8_lossy(&output.stderr).trim() - )); + pub(super) async fn write(&self, key: SettingKey, value: &str) -> Result { + key.validate(value)?; + let (reply, response) = oneshot::channel(); + let request = async { + self.commands + .send(ControlCommand::Write { + key, + value: value.to_string(), + reply, + }) + .await + .map_err(|_| anyhow!("simulator control worker stopped"))?; + response + .await + .map_err(|_| anyhow!("simulator control worker stopped"))? + }; + tokio::time::timeout(CONTROL_TIMEOUT, request) + .await + .map_err(|_| anyhow!("simulator setting write timed out"))? } - Ok(String::from_utf8_lossy(&output.stdout).to_string()) } #[cfg(test)] @@ -176,9 +256,10 @@ mod tests { #[test] fn rejects_values_outside_the_allowed_set() { - let error = write("no-such-device", SettingKey::Appearance, "chartreuse") + let error = SettingKey::Appearance + .validate("chartreuse") .expect_err("invalid appearance should be rejected"); - // Rejected locally, without ever shelling out to simctl. + // Rejected locally, without sending anything to CoreSimulator. assert!(error.to_string().contains("chartreuse")); } @@ -187,24 +268,14 @@ mod tests { // These are not reported values, so they must be allowed explicitly. assert!(!CONTENT_SIZES.contains(&"increment")); for value in ["increment", "decrement"] { - let error = write("no-such-device", SettingKey::ContentSize, value) - .expect_err("no such device"); - assert!( - !error.to_string().contains("is not valid"), - "{value} should reach simctl rather than being rejected" - ); + assert!(SettingKey::ContentSize.validate(value).is_ok()); } } #[test] - fn every_key_has_values_and_a_distinct_verb() { - let mut verbs = Vec::new(); + fn every_key_has_values() { for key in SettingKey::all() { assert!(!key.allowed_values().is_empty()); - verbs.push(key.verb()); } - verbs.sort_unstable(); - verbs.dedup(); - assert_eq!(verbs.len(), SettingKey::all().len()); } } diff --git a/packages/accessibility-ios-sys/src/macos.rs b/packages/accessibility-ios-sys/src/macos.rs index 883dd24..9214520 100644 --- a/packages/accessibility-ios-sys/src/macos.rs +++ b/packages/accessibility-ios-sys/src/macos.rs @@ -55,6 +55,7 @@ use objc2_foundation::{NSString, NSUUID}; use slotmap::SecondaryMap; mod common; +mod control; mod dispatcher; mod dynamic; mod encoder; @@ -70,6 +71,9 @@ pub use common::{ BootedSimulator, ButtonDirection, Element, ElementKey, ElementTree, HardwareButton, Point, Rect, ScreenSpace, Screenshot, Size, TreeFilter, booted_simulators, load_frameworks, }; +pub use control::{ + SimulatorAppearance, SimulatorContentSize, SimulatorDevice, SimulatorIncreaseContrast, +}; pub use encoder::{ ChunkKind, ChunkSink, EncodedChunk, EncoderConfig, H264Encoder, NalFormat, Tuning, }; diff --git a/packages/accessibility-ios-sys/src/macos/common.rs b/packages/accessibility-ios-sys/src/macos/common.rs index 938f9d3..0c4e2bd 100644 --- a/packages/accessibility-ios-sys/src/macos/common.rs +++ b/packages/accessibility-ios-sys/src/macos/common.rs @@ -53,6 +53,9 @@ pub struct Screenshot { } impl Screenshot { + /// Crop and re-encode this screenshot as PNG. + /// + /// This decodes and encodes the image synchronously on the calling thread. pub fn crop(&self, bounds: &Rect, screen_bounds: &Rect) -> Result { use image::ImageReader; use std::io::Cursor; @@ -504,6 +507,7 @@ pub(super) unsafe fn nsstring_to_string_static(ns_string: *mut AnyObject) -> Opt /// /// Devices are returned in CoreSimulator's order. Each entry includes the /// stable UDID used by the rest of this crate and the user-visible device name. +/// Framework loading and device enumeration happen synchronously. pub fn booted_simulators() -> Result> { crate::frameworks::load_coresimulator_framework()?; diff --git a/packages/accessibility-ios-sys/src/macos/control.rs b/packages/accessibility-ios-sys/src/macos/control.rs new file mode 100644 index 0000000..b2c29f9 --- /dev/null +++ b/packages/accessibility-ios-sys/src/macos/control.rs @@ -0,0 +1,441 @@ +//! Typed direct controls for a booted iOS Simulator. + +use std::path::PathBuf; +use std::sync::{Arc, Condvar, Mutex, OnceLock}; +use std::time::Duration; + +use anyhow::{Result, anyhow}; +use block2::RcBlock; +use dispatch2::{DispatchQueue, DispatchQueueAttr, DispatchRetained}; +use objc2::msg_send; +use objc2::runtime::{AnyObject, Bool, Sel}; +use objc2_foundation::{NSMutableArray, NSMutableDictionary, NSNumber, NSString}; + +use super::common::{find_booted_device, nsstring_to_string_static}; + +/// Typed direct control handle for one booted simulator. +/// +/// Setting getters and setters perform synchronous CoreSimulator IPC and may +/// block. Keep this handle on a dedicated worker rather than an async runtime +/// thread. +pub struct SimulatorDevice { + device: *mut AnyObject, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i64)] +pub enum SimulatorAppearance { + Light = 1, + Dark = 2, +} + +impl TryFrom for SimulatorAppearance { + type Error = anyhow::Error; + + fn try_from(value: i64) -> Result { + match value { + 1 => Ok(Self::Light), + 2 => Ok(Self::Dark), + _ => Err(anyhow!("unknown simulator appearance value {value}")), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i64)] +pub enum SimulatorIncreaseContrast { + Disabled = 1, + Enabled = 2, +} + +impl TryFrom for SimulatorIncreaseContrast { + type Error = anyhow::Error; + + fn try_from(value: i64) -> Result { + match value { + 1 => Ok(Self::Disabled), + 2 => Ok(Self::Enabled), + _ => Err(anyhow!("unknown simulator contrast value {value}")), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i64)] +pub enum SimulatorContentSize { + ExtraSmall = 1, + Small = 2, + Medium = 3, + Large = 4, + ExtraLarge = 5, + ExtraExtraLarge = 6, + ExtraExtraExtraLarge = 7, + AccessibilityMedium = 8, + AccessibilityLarge = 9, + AccessibilityExtraLarge = 10, + AccessibilityExtraExtraLarge = 11, + AccessibilityExtraExtraExtraLarge = 12, +} + +impl SimulatorContentSize { + pub fn index(self) -> usize { + self as usize - 1 + } + + pub fn step(self, delta: i64) -> Self { + Self::try_from((self as i64 + delta).clamp(1, 12)) + .expect("clamped content size should be valid") + } +} + +impl TryFrom for SimulatorContentSize { + type Error = anyhow::Error; + + fn try_from(value: i64) -> Result { + match value { + 1 => Ok(Self::ExtraSmall), + 2 => Ok(Self::Small), + 3 => Ok(Self::Medium), + 4 => Ok(Self::Large), + 5 => Ok(Self::ExtraLarge), + 6 => Ok(Self::ExtraExtraLarge), + 7 => Ok(Self::ExtraExtraExtraLarge), + 8 => Ok(Self::AccessibilityMedium), + 9 => Ok(Self::AccessibilityLarge), + 10 => Ok(Self::AccessibilityExtraLarge), + 11 => Ok(Self::AccessibilityExtraExtraLarge), + 12 => Ok(Self::AccessibilityExtraExtraExtraLarge), + _ => Err(anyhow!("unknown simulator content size value {value}")), + } + } +} + +unsafe impl Send for SimulatorDevice {} + +impl Drop for SimulatorDevice { + fn drop(&mut self) { + unsafe { objc2::ffi::objc_release(self.device) }; + } +} + +impl SimulatorDevice { + pub fn for_device(udid: Option<&str>) -> Result { + crate::frameworks::load_coresimulator_framework()?; + let device = unsafe { find_booted_device(udid)? }; + Ok(unsafe { Self::retaining(device) }) + } + + pub(super) unsafe fn retaining(device: *mut AnyObject) -> Self { + let device = unsafe { objc2::ffi::objc_retain(device) }; + Self { device } + } + + pub fn appearance(&self) -> Result { + self.require_selector(objc2::sel!(currentUIInterfaceStyle))?; + let value: i64 = unsafe { msg_send![self.device, currentUIInterfaceStyle] }; + SimulatorAppearance::try_from(value) + } + + pub fn set_appearance(&self, appearance: SimulatorAppearance) -> Result<()> { + self.require_selector(objc2::sel!(setUIInterfaceStyle:error:))?; + let mut error = std::ptr::null_mut(); + let success: Bool = unsafe { + msg_send![self.device, setUIInterfaceStyle: appearance as i64, error: &mut error] + }; + Self::bool_result(success, error, "set simulator appearance") + } + + pub fn content_size(&self) -> Result { + self.require_selector(objc2::sel!(currentContentSizeCategory))?; + let value: i64 = unsafe { msg_send![self.device, currentContentSizeCategory] }; + SimulatorContentSize::try_from(value) + } + + pub fn set_content_size(&self, size: SimulatorContentSize) -> Result<()> { + self.require_selector(objc2::sel!(setContentSizeCategory:error:))?; + let mut error = std::ptr::null_mut(); + let success: Bool = unsafe { + msg_send![self.device, setContentSizeCategory: size as i64, error: &mut error] + }; + Self::bool_result(success, error, "set simulator content size") + } + + pub fn increase_contrast(&self) -> Result { + self.require_selector(objc2::sel!(currentIncreaseContrastMode))?; + let value: i64 = unsafe { msg_send![self.device, currentIncreaseContrastMode] }; + SimulatorIncreaseContrast::try_from(value) + } + + pub fn set_increase_contrast(&self, contrast: SimulatorIncreaseContrast) -> Result<()> { + self.require_selector(objc2::sel!(setIncreaseContrastEnabled:error:))?; + let mut error = std::ptr::null_mut(); + let enabled = contrast == SimulatorIncreaseContrast::Enabled; + let success: Bool = unsafe { + msg_send![ + self.device, + setIncreaseContrastEnabled: Bool::from(enabled), + error: &mut error + ] + }; + Self::bool_result(success, error, "set simulator increase contrast") + } + + /// Restart the guest accessibility bridge and wait for launchd to stop it. + /// + /// Starts: + /// + /// 1. One guest `launchctl stop` process through CoreSimulator. + /// 2. Spawn-completion and process-termination callbacks on the control queue. + pub(super) fn restart_accessibility_bridge(&self) -> Result<()> { + const TIMEOUT: Duration = Duration::from_secs(2); + + let selector = objc2::sel!(spawnAsyncWithPath:options:terminationQueue:terminationHandler:completionQueue:completionHandler:); + let responds: Bool = unsafe { msg_send![self.device, respondsToSelector: selector] }; + if !responds.as_bool() { + return Err(anyhow!( + "CoreSimulator does not support asynchronous process spawning" + )); + } + + let runtime: *mut AnyObject = unsafe { msg_send![self.device, runtime] }; + if runtime.is_null() { + return Err(anyhow!("Simulator runtime is unavailable")); + } + let root: *mut AnyObject = unsafe { msg_send![runtime, root] }; + let root = unsafe { nsstring_to_string_static(root) } + .ok_or_else(|| anyhow!("Simulator runtime root is unavailable"))?; + let launch_path = PathBuf::from(root).join("bin/launchctl"); + let launch_path = launch_path + .to_str() + .ok_or_else(|| anyhow!("Simulator launchctl path is not valid UTF-8"))?; + + let arguments: objc2::rc::Retained> = NSMutableArray::new(); + for argument in [launch_path, "stop", "com.apple.CoreSimulator.bridge"] { + arguments.addObject(&NSString::from_str(argument)); + } + let options: objc2::rc::Retained> = + NSMutableDictionary::new(); + let arguments_key = NSString::from_str("arguments"); + let standalone_key = NSString::from_str("standalone"); + unsafe { + let _: () = msg_send![ + &*options, + setObject: &*arguments, + forKey: &*arguments_key + ]; + let standalone = NSNumber::new_bool(false); + let _: () = msg_send![ + &*options, + setObject: &*standalone, + forKey: &*standalone_key + ]; + } + + let process = ProcessSpawn::new(); + let completion = Arc::clone(&process); + let completion_block = RcBlock::new(move |error: *mut AnyObject, pid: i32| { + let result = if error.is_null() { + Ok(pid) + } else { + let description: *mut AnyObject = unsafe { msg_send![error, localizedDescription] }; + Err(unsafe { nsstring_to_string_static(description) } + .unwrap_or_else(|| "unknown CoreSimulator spawn error".to_string())) + }; + completion.complete(result); + }); + + let termination = Arc::clone(&process); + let termination_block = RcBlock::new(move |status: i32| termination.terminate(status)); + + let queue = Self::callback_queue(); + let launch_path = NSString::from_str(launch_path); + unsafe { + let _: () = msg_send![ + self.device, + spawnAsyncWithPath: &*launch_path, + options: &*options, + terminationQueue: queue, + terminationHandler: &*termination_block, + completionQueue: queue, + completionHandler: &*completion_block + ]; + } + + process.wait_for_spawn(TIMEOUT)?; + let status = process.wait_for_termination(TIMEOUT)?; + if libc::WIFEXITED(status) { + let code = libc::WEXITSTATUS(status); + if code == 0 || code == libc::ESRCH { + return Ok(()); + } + return Err(anyhow!("Simulator launchctl exited with status {code}")); + } + if libc::WIFSIGNALED(status) { + return Err(anyhow!( + "Simulator launchctl terminated by signal {}", + libc::WTERMSIG(status) + )); + } + Err(anyhow!("Simulator launchctl returned wait status {status}")) + } + + fn callback_queue() -> *mut AnyObject { + static QUEUE: OnceLock> = OnceLock::new(); + let queue = QUEUE.get_or_init(|| { + DispatchQueue::new( + "com.accessibility_cli.simulator.control", + DispatchQueueAttr::SERIAL, + ) + }); + DispatchRetained::as_ptr(queue).as_ptr().cast::() + } + + fn require_selector(&self, selector: Sel) -> Result<()> { + let responds: Bool = unsafe { msg_send![self.device, respondsToSelector: selector] }; + if responds.as_bool() { + Ok(()) + } else { + Err(anyhow!( + "CoreSimulator does not support selector {}", + selector.name().to_string_lossy() + )) + } + } + + fn bool_result(success: Bool, error: *mut AnyObject, operation: &str) -> Result<()> { + if success.as_bool() { + return Ok(()); + } + let detail = unsafe { + (!error.is_null()) + .then(|| { + let description: *mut AnyObject = msg_send![error, localizedDescription]; + nsstring_to_string_static(description) + }) + .flatten() + }; + Err(anyhow!( + "Failed to {operation}: {}", + detail.as_deref().unwrap_or("no error detail") + )) + } +} + +#[derive(Debug, Default)] +struct ProcessState { + result: Option>, + status: Option, +} + +#[derive(Debug)] +struct ProcessSpawn { + state: Mutex, + changed: Condvar, +} + +impl ProcessSpawn { + fn new() -> Arc { + Arc::new(Self { + state: Mutex::new(ProcessState::default()), + changed: Condvar::new(), + }) + } + + fn complete(&self, result: std::result::Result) { + self.state.lock().unwrap().result = Some(result); + self.changed.notify_all(); + } + + fn terminate(&self, status: i32) { + self.state.lock().unwrap().status = Some(status); + self.changed.notify_all(); + } + + fn wait_for_spawn(&self, timeout: Duration) -> Result { + let state = self.state.lock().unwrap(); + let (state, wait) = self + .changed + .wait_timeout_while(state, timeout, |state| state.result.is_none()) + .unwrap(); + if wait.timed_out() { + return Err(anyhow!("CoreSimulator process spawn timed out")); + } + state + .result + .as_ref() + .expect("spawn completion should be populated") + .as_ref() + .copied() + .map_err(|error| anyhow!(error.clone())) + } + + fn wait_for_termination(&self, timeout: Duration) -> Result { + let state = self.state.lock().unwrap(); + let (state, wait) = self + .changed + .wait_timeout_while(state, timeout, |state| state.status.is_none()) + .unwrap(); + if wait.timed_out() { + return Err(anyhow!("Simulator process did not exit")); + } + Ok(state + .status + .expect("termination status should be populated")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn setting_values_reject_unknown_raw_values() { + assert!(SimulatorAppearance::try_from(0).is_err()); + assert!(SimulatorAppearance::try_from(3).is_err()); + assert!(SimulatorIncreaseContrast::try_from(0).is_err()); + assert!(SimulatorIncreaseContrast::try_from(3).is_err()); + assert!(SimulatorContentSize::try_from(0).is_err()); + assert!(SimulatorContentSize::try_from(13).is_err()); + } + + #[test] + fn content_size_steps_stay_in_range() { + assert_eq!( + SimulatorContentSize::ExtraSmall.step(-1), + SimulatorContentSize::ExtraSmall + ); + assert_eq!( + SimulatorContentSize::ExtraSmall.step(1), + SimulatorContentSize::Small + ); + assert_eq!( + SimulatorContentSize::AccessibilityExtraExtraExtraLarge.step(1), + SimulatorContentSize::AccessibilityExtraExtraExtraLarge + ); + } + + #[test] + fn process_callbacks_share_one_wait_state() { + let process = ProcessSpawn::new(); + let callback = Arc::clone(&process); + let spawn = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(10)); + callback.complete(Ok(42)); + }); + assert_eq!(process.wait_for_spawn(Duration::from_secs(1)).unwrap(), 42); + spawn.join().unwrap(); + + let callback = Arc::clone(&process); + let terminate = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(10)); + callback.terminate(0); + }); + assert_eq!( + process + .wait_for_termination(Duration::from_secs(1)) + .unwrap(), + 0 + ); + terminate.join().unwrap(); + } +} diff --git a/packages/accessibility-ios-sys/src/macos/encoder.rs b/packages/accessibility-ios-sys/src/macos/encoder.rs index 0bd18cd..e419afa 100644 --- a/packages/accessibility-ios-sys/src/macos/encoder.rs +++ b/packages/accessibility-ios-sys/src/macos/encoder.rs @@ -215,7 +215,9 @@ impl H264Encoder { /// Encode one frame, rebuilding the session if the source resized. /// /// `width`/`height` describe the *source*; the session may be smaller if - /// the config caps the long edge, in which case VideoToolbox scales. + /// the config caps the long edge, in which case VideoToolbox scales. The + /// pixel transfer is synchronous and blocks until the encoder owns a copy + /// of the recycled simulator surface. pub fn encode( &mut self, image: &CVImageBuffer, diff --git a/packages/accessibility-ios-sys/src/macos/framebuffer.rs b/packages/accessibility-ios-sys/src/macos/framebuffer.rs index 45f48b9..cf7eeab 100644 --- a/packages/accessibility-ios-sys/src/macos/framebuffer.rs +++ b/packages/accessibility-ios-sys/src/macos/framebuffer.rs @@ -362,6 +362,10 @@ impl CaptureState { } /// Live framebuffer capture session for one simulator device. +/// +/// Starting synchronously discovers and registers SimulatorKit IO ports. +/// Stopping, including through `Drop`, blocks while the idle worker exits and +/// callbacks are unregistered. pub struct SimFramebuffer { device_udid: String, state: Arc, @@ -427,6 +431,8 @@ impl SimFramebuffer { } /// Begin capturing. Calling twice rebuilds the pipeline. + /// + /// IO-port discovery and callback registration happen synchronously. pub fn start(&mut self) -> Result<()> { self.state.running.store(true, Ordering::Relaxed); self.state.wire_up()?; @@ -474,6 +480,8 @@ impl SimFramebuffer { })); } + /// Stop capture, blocking until the idle worker exits and callbacks are + /// unregistered. pub fn stop(&mut self) { self.state.running.store(false, Ordering::Relaxed); self.state.activity.notify_one(); diff --git a/packages/accessibility-ios-sys/src/macos/hid.rs b/packages/accessibility-ios-sys/src/macos/hid.rs index bfa0d24..1b4d285 100644 --- a/packages/accessibility-ios-sys/src/macos/hid.rs +++ b/packages/accessibility-ios-sys/src/macos/hid.rs @@ -75,6 +75,11 @@ pub enum Orientation { LandscapeLeft = 4, } +/// Direct simulator HID client. +/// +/// Sends synchronously wait for a dispatch-queue round trip. Composite taps, +/// swipes, buttons, and key presses also sleep to preserve event timing, so +/// callers should keep this client off async runtime threads. pub struct SimulatorHID { client: *mut AnyObject, // SimDeviceLegacyHIDClient device: *mut AnyObject, // SimDevice, retained for GSEvent port lookup diff --git a/packages/accessibility-ios-sys/src/macos/reader.rs b/packages/accessibility-ios-sys/src/macos/reader.rs index 95ac843..3542a71 100644 --- a/packages/accessibility-ios-sys/src/macos/reader.rs +++ b/packages/accessibility-ios-sys/src/macos/reader.rs @@ -1,4 +1,5 @@ use super::common::{ElementCache, find_booted_device, get_translator, map_ax_role_ios}; +use super::control::SimulatorDevice; use super::dispatcher::{ CFRetain, ensure_dispatcher_registered, generate_token, get_dispatcher_state, }; @@ -10,6 +11,10 @@ mod actions; /// iOS Simulator accessibility reader. /// /// Provides access to the accessibility tree of iOS apps running in the iOS Simulator. +/// +/// Tree queries, hit tests, and actions synchronously wait on the simulator's +/// accessibility bridge, which currently has no response timeout. Call these +/// methods from a dedicated worker thread. pub struct IOSSimulatorAccessibility { translator: *mut AnyObject, device: *mut AnyObject, @@ -74,6 +79,9 @@ impl IOSSimulatorAccessibility { } /// Get the accessibility tree from the frontmost app in the simulator. + /// + /// This recursively performs synchronous bridge queries with no response + /// timeout and may also wait for accessibility remediation. pub fn get_tree(&mut self, filter: &TreeFilter) -> Result { // Clear previous cache self.clear_cache(); @@ -159,8 +167,8 @@ impl IOSSimulatorAccessibility { } // Store the app bounds for screenshot coordinate conversion. - // iOS accessibility coordinates are in macOS screen space, but xcrun simctl screenshot - // captures device-local coordinates starting at (0,0). We need to subtract the app's + // iOS accessibility coordinates are in macOS screen space, but framebuffer screenshots + // use device-local coordinates starting at (0,0). We need to subtract the app's // origin to convert accessibility bounds to device-local coordinates. self.app_bounds = Some(Rect::new( Point::new(frame.origin.x, frame.origin.y), @@ -198,61 +206,25 @@ impl IOSSimulatorAccessibility { "[WARN] This usually means SpringBoard crashed and CoreSimulatorBridge needs restart" ); - // Get the device UDID for the launchctl command + // Get the device UDID for diagnostics if remediation fails. let udid = &self.device_udid; - // Restart CoreSimulatorBridge via launchctl - // The service name pattern is: com.apple.CoreSimulator.bridge. - let service_name = format!("com.apple.CoreSimulator.bridge.{}", udid); - - // Use xcrun simctl to stop and restart the bridge - // This is safer than directly calling launchctl - let output = std::process::Command::new("xcrun") - .args([ - "simctl", - "spawn", - udid, - "launchctl", - "kickstart", - "-k", - &format!("system/{}", service_name), - ]) - .output(); - - match output { - Ok(output) => { - if output.status.success() { - eprintln!("[INFO] Successfully restarted CoreSimulatorBridge"); - // Give the bridge a moment to restart - std::thread::sleep(std::time::Duration::from_millis(500)); - Ok(true) - } else { - // If kickstart fails, try using simctl directly - let stderr = String::from_utf8_lossy(&output.stderr); - eprintln!( - "[WARN] Failed to restart via launchctl ({}), trying alternative...", - stderr.trim() - ); - - // Alternative: use simctl shutdown and boot - // This is more disruptive but more reliable - // For now, just return an error with instructions - Err(anyhow!( - "Accessibility subsystem appears to be in a bad state (zero-sized frame). \ - This typically happens when SpringBoard has crashed. \ - Try restarting the simulator or running: \ - xcrun simctl shutdown {} && xcrun simctl boot {}", - udid, - udid - )) - } - } - Err(e) => Err(anyhow!( - "Failed to restart CoreSimulatorBridge: {}. \ + // Restart CoreSimulatorBridge through the simulator's launchd domain. + // The guest service name is com.apple.CoreSimulator.bridge. + let device = unsafe { SimulatorDevice::retaining(self.device) }; + device.restart_accessibility_bridge().map_err(|error| { + anyhow!( + "Failed to restart CoreSimulatorBridge for {}: {}. \ Try restarting the simulator manually.", - e - )), - } + udid, + error + ) + })?; + + eprintln!("[INFO] Successfully restarted CoreSimulatorBridge"); + // Give the bridge a moment to restart + std::thread::sleep(std::time::Duration::from_millis(500)); + Ok(true) } /// Build an Element from an AXPMacPlatformElement. diff --git a/packages/accessibility-ios-sys/src/macos/reader/actions.rs b/packages/accessibility-ios-sys/src/macos/reader/actions.rs index 0e18f87..a1c9f5b 100644 --- a/packages/accessibility-ios-sys/src/macos/reader/actions.rs +++ b/packages/accessibility-ios-sys/src/macos/reader/actions.rs @@ -1,5 +1,11 @@ +use std::sync::mpsc::RecvTimeoutError; + use super::*; use crate::macos::dispatcher::{CFRelease, get_dispatcher_state}; +use objc2_core_video::{ + CVPixelBufferGetBaseAddress, CVPixelBufferGetBytesPerRow, CVPixelBufferLockBaseAddress, + CVPixelBufferLockFlags, CVPixelBufferUnlockBaseAddress, +}; impl IOSSimulatorAccessibility { /// Clear the element cache and release retained element pointers. @@ -367,63 +373,40 @@ impl IOSSimulatorAccessibility { /// Capture a screenshot of the entire simulator screen. /// - /// Uses `xcrun simctl io` to capture the screenshot as PNG. + /// Uses SimulatorKit's live framebuffer and encodes a copied frame as PNG. + /// This blocks the current thread while waiting for a frame and encoding it. pub fn capture_screen(&self) -> Result { - use std::io::Read; - - // Create a temporary file for the screenshot - let temp_dir = std::env::temp_dir(); - let screenshot_path = temp_dir.join(format!( - "accessibility_cli_screenshot_{}.png", - std::process::id() - )); - - // Run xcrun simctl io screenshot - let output = std::process::Command::new("xcrun") - .args([ - "simctl", - "io", - &self.device_udid, - "screenshot", - "--type=png", - screenshot_path.to_str().unwrap(), - ]) - .output() - .map_err(|e| anyhow!("Failed to execute xcrun simctl: {}", e))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - // Clean up temp file if it exists - let _ = std::fs::remove_file(&screenshot_path); - return Err(anyhow!("Screenshot capture failed: {}", stderr.trim())); - } + Self::capture_screen_for_device(Some(&self.device_udid)) + } - // Read the PNG file - let mut file = std::fs::File::open(&screenshot_path) - .map_err(|e| anyhow!("Failed to open screenshot file: {}", e))?; - let mut data = Vec::new(); - file.read_to_end(&mut data) - .map_err(|e| anyhow!("Failed to read screenshot file: {}", e))?; - - // Clean up temp file - let _ = std::fs::remove_file(&screenshot_path); - - // Decode PNG to get dimensions - let (width, height) = { - use image::ImageReader; - use std::io::Cursor; - let img = ImageReader::new(Cursor::new(&data)) - .with_guessed_format()? - .decode() - .map_err(|e| anyhow!("Failed to decode screenshot: {}", e))?; - (img.width(), img.height()) + /// Capture a PNG screenshot directly from a booted simulator's framebuffer. + /// + /// This is a blocking API: it waits up to three seconds for a frame, copies + /// the full framebuffer before SimulatorKit recycles it, and encodes the + /// copy as PNG on the calling thread. Async callers should run it on a + /// blocking worker. + pub fn capture_screen_for_device(udid: Option<&str>) -> Result { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let mut sender = Some(sender); + let mut framebuffer = SimFramebuffer::new(udid)?; + framebuffer.set_sink(Some(Box::new(move |frame| { + if let Some(sender) = sender.take() { + let _ = sender.send(RawScreenshot::copy(frame)); + } + }))); + framebuffer.start()?; + let frame = match receiver.recv_timeout(std::time::Duration::from_secs(3)) { + Ok(frame) => frame?, + Err(RecvTimeoutError::Timeout) => { + return Err(anyhow!("Timed out waiting for a simulator framebuffer")); + } + Err(RecvTimeoutError::Disconnected) => { + return Err(anyhow!("Simulator framebuffer capture stopped")); + } }; + drop(framebuffer); - Ok(Screenshot { - data, - width, - height, - }) + frame.encode() } /// Get the screen bounds for the simulator. @@ -438,7 +421,8 @@ impl IOSSimulatorAccessibility { /// Capture a screenshot of a specific element. /// - /// This captures the full screen and crops to the element's bounds. + /// This captures the full screen and crops to the element's bounds. It + /// blocks while waiting for a frame and while decoding and re-encoding PNG. pub fn capture_element(&mut self, id: ElementKey) -> Result { // Get element bounds from cache let element_ptr = @@ -463,3 +447,94 @@ impl IOSSimulatorAccessibility { screenshot.crop(&bounds, &screen_bounds) } } + +struct RawScreenshot { + rgba: Vec, + width: u32, + height: u32, +} + +impl RawScreenshot { + fn copy(frame: CapturedFrame<'_>) -> Result { + let flags = CVPixelBufferLockFlags::ReadOnly; + let status = unsafe { CVPixelBufferLockBaseAddress(frame.pixel_buffer, flags) }; + if status != 0 { + return Err(anyhow!("Failed to lock simulator framebuffer: {status}")); + } + + let result = (|| { + let base = CVPixelBufferGetBaseAddress(frame.pixel_buffer).cast::(); + if base.is_null() { + return Err(anyhow!("Simulator framebuffer has no base address")); + } + let row_bytes = CVPixelBufferGetBytesPerRow(frame.pixel_buffer); + let packed_row_bytes = usize::try_from(frame.width)? + .checked_mul(4) + .ok_or_else(|| anyhow!("Simulator framebuffer row size overflow"))?; + if row_bytes < packed_row_bytes { + return Err(anyhow!( + "Simulator framebuffer row is shorter than its width" + )); + } + let height = usize::try_from(frame.height)?; + let length = packed_row_bytes + .checked_mul(height) + .ok_or_else(|| anyhow!("Simulator framebuffer size overflow"))?; + let mut rgba = vec![0; length]; + for row in 0..height { + unsafe { + std::ptr::copy_nonoverlapping( + base.add(row * row_bytes), + rgba.as_mut_ptr().add(row * packed_row_bytes), + packed_row_bytes, + ); + } + } + Self::convert_bgra(&mut rgba); + Ok(Self { + rgba, + width: frame.width, + height: frame.height, + }) + })(); + + let unlock = unsafe { CVPixelBufferUnlockBaseAddress(frame.pixel_buffer, flags) }; + match result { + Err(error) => Err(error), + Ok(_) if unlock != 0 => { + Err(anyhow!("Failed to unlock simulator framebuffer: {unlock}")) + } + Ok(frame) => Ok(frame), + } + } + + fn convert_bgra(pixels: &mut [u8]) { + for pixel in pixels.chunks_exact_mut(4) { + pixel.swap(0, 2); + } + } + + fn encode(self) -> Result { + let image = image::RgbaImage::from_raw(self.width, self.height, self.rgba) + .ok_or_else(|| anyhow!("Failed to construct simulator screenshot image"))?; + let mut data = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(image).write_to(&mut data, image::ImageFormat::Png)?; + Ok(Screenshot { + data: data.into_inner(), + width: self.width, + height: self.height, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn converts_bgra_pixels_to_rgba() { + let mut pixels = [1, 2, 3, 4, 5, 6, 7, 8]; + RawScreenshot::convert_bgra(&mut pixels); + assert_eq!(pixels, [3, 2, 1, 4, 7, 6, 5, 8]); + } +} diff --git a/packages/accessibility-ios-sys/src/macos/recorder.rs b/packages/accessibility-ios-sys/src/macos/recorder.rs index 179ba21..3e45642 100644 --- a/packages/accessibility-ios-sys/src/macos/recorder.rs +++ b/packages/accessibility-ios-sys/src/macos/recorder.rs @@ -159,6 +159,7 @@ impl Recorder { /// /// Frames the writer is not ready for are dropped rather than queued: a /// recording that falls behind should lose frames, not unbounded memory. + /// Pixel transfer and writer submission run synchronously on the caller. pub fn append(&mut self, source: &CVPixelBuffer) -> Result<()> { let started_at = *self.started_at.get_or_insert_with(std::time::Instant::now); diff --git a/packages/accessibility-ios-sys/src/macos/stream.rs b/packages/accessibility-ios-sys/src/macos/stream.rs index 075dc91..9a26d49 100644 --- a/packages/accessibility-ios-sys/src/macos/stream.rs +++ b/packages/accessibility-ios-sys/src/macos/stream.rs @@ -35,7 +35,9 @@ impl SimVideoStream { /// Start capturing `udid` and pushing encoded chunks to `sink`. /// /// `sink` runs on the capture queue, so it must not block; the intended - /// use is a bounded channel that drops on overflow. + /// use is a bounded channel that drops on overflow. Framework loading, + /// encoder creation, and framebuffer registration happen synchronously + /// before this function returns. pub fn start(udid: Option<&str>, config: EncoderConfig, sink: ChunkSink) -> Result { let mut framebuffer = SimFramebuffer::new(udid)?; framebuffer.set_active_frame_rate(config.fps); @@ -142,6 +144,8 @@ impl SimVideoStream { } /// Stop the running recording and finalize the file. + /// + /// This blocks while `AVAssetWriter` flushes, for up to 30 seconds. pub fn stop_recording(&self) -> Result { let recorder = self .recorder @@ -162,6 +166,9 @@ impl SimVideoStream { .map(|recorder| recorder.frames()) } + /// Stop capture and finalize any active recording. + /// + /// This may block on recording finalization and framebuffer worker shutdown. pub fn stop(&mut self) { // Finalize before tearing the capture down, so an in-flight recording // is left playable rather than truncated. diff --git a/packages/accessibility-serve/src/http.rs b/packages/accessibility-serve/src/http.rs index 99758b3..637d22a 100644 --- a/packages/accessibility-serve/src/http.rs +++ b/packages/accessibility-serve/src/http.rs @@ -81,15 +81,12 @@ async fn stats(State(state): State) -> Json) -> Json> { - // Each read shells out to simctl, so keep it off the async worker threads. + // Each read goes through the simulator control worker, keeping synchronous + // CoreSimulator round trips off the async worker threads. let Some(session) = state.session.ios_session().cloned() else { return Json(Vec::new()); }; - Json( - tokio::task::spawn_blocking(move || session.settings()) - .await - .unwrap_or_default(), - ) + Json(session.settings().await.unwrap_or_default()) } #[cfg(not(target_os = "macos"))] @@ -123,13 +120,9 @@ async fn set_setting( ) .into_response(); }; - let result = - tokio::task::spawn_blocking(move || session.set_setting(request.key, &request.value)).await; - - match result { - Ok(Ok(value)) => Json(serde_json::json!({ "value": value })).into_response(), - Ok(Err(error)) => (StatusCode::BAD_REQUEST, error.to_string()).into_response(), - Err(error) => internal_error(anyhow::anyhow!(error)), + match session.set_setting(request.key, &request.value).await { + Ok(value) => Json(serde_json::json!({ "value": value })).into_response(), + Err(error) => (StatusCode::BAD_REQUEST, error.to_string()).into_response(), } }