Volume HID, bounded simctl, and interaction-paced simulator capture

.devin/skills/ios-simulator-accessibility/SKILL.md

@@ -23,6 +23,19 @@ Miss that and attribute reads do not fail, they silently return an empty label
 and a zero frame. `get_tree` already does this; see the matching step in
 `get_element_at_point`.
 
+## SpringBoard crash remediation
+
+A dead SpringBoard can leave CoreSimulatorBridge serving a stale root with a
+zero frame. Spawn the runtime's `bin/launchctl` through
+`SimDevice.spawnAsyncWithPath:...` and run `stop com.apple.CoreSimulator.bridge`.
+The service is kept alive and respawns automatically; after it exits, retry the
+accessibility query once. This was verified on Xcode 26.6 by killing SpringBoard:
+the retry returned a healthy tree from the replacement SpringBoard PID.
+
+The spawn completion and termination callbacks share one `Condvar`, so their
+results must also share one mutex. macOS rejects using one condition variable
+with two mutexes and panics before the CoreSimulator call can complete.
+
 ## Backdrops swallow hit tests
 
 Every app has full-screen backdrops: the Application node plus one or more

.devin/skills/ios-simulator-input/SKILL.md

@@ -82,3 +82,8 @@ not publish that port.
 colour filters, transparency, VoiceOver) have no simctl verb and need a helper
 binary spawned inside the simulator that drives the private libAccessibility
 setters.
+
+The direct `SimDevice` getters return raw integers. Measured on Xcode 26.6:
+appearance is 1 light / 2 dark, content size is 1 through 12, and increase
+contrast is **1 disabled / 2 enabled** — it is not a 0/1 boolean even though
+the setter accepts a BOOL.

packages/accessibility-core/src/platform/android/video.rs

@@ -1,7 +1,7 @@
 use std::io::Read;
 use std::sync::mpsc::{self, Receiver, SyncSender};
 use std::thread::JoinHandle;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use accessibility_android_sys::AdbClient;
 use accessibility_android_sys::emulator::screenrecord::{
@@ -247,6 +247,7 @@ fn emit(
             } else {
                 FrameKind::Delta
             },
+            captured_at: Instant::now(),
         });
     }
 }

packages/accessibility-core/src/platform/ios_simulator.rs

@@ -32,8 +32,8 @@ pub mod settings;
 
 pub use ax::{AxCommand, AxSnapshot, Discovery, ElementDetail, NormalizedRect, spawn_ax_worker};
 pub use input::{
-    HOME_INDICATOR_BAND, HardwareButton as InputHardwareButton, InputCommand, Orientation,
-    TouchEdge, TouchPhase, spawn_input_worker,
+    HOME_INDICATOR_BAND, HardwareButton as InputHardwareButton, InputCapabilities, InputCommand,
+    Orientation, TouchEdge, TouchPhase, spawn_input_worker,
 };
 pub use keymap::{KeyStroke, keystroke_for, keystrokes_for};
 pub use session::{DeviceInfo, SimSession, StatsReport, StreamStats};
@@ -49,6 +49,10 @@ pub fn load_frameworks() -> Result<()> {
 ///
 /// This is a safe core wrapper around `accessibility-ios-sys`; it does not expose
 /// Objective-C, CoreFoundation, or libc handles outside the sys crate.
+///
+/// Its inherent accessibility and HID methods are synchronous and may block on
+/// simulator IPC or deliberate input pacing. Session users should prefer the
+/// dedicated async worker APIs on [`SimSession`].
 pub struct IOSSimulatorAccessibility {
     inner: sys::IOSSimulatorAccessibility,
     cache: ElementCache,
@@ -75,6 +79,8 @@ impl IOSSimulatorAccessibility {
     }
 
     /// Get the accessibility tree from the frontmost app in the simulator.
+    ///
+    /// This synchronously waits on recursive accessibility bridge queries.
     pub fn get_tree(&mut self, filter: &TreeFilter) -> Result<ElementTree> {
         self.clear_local_cache();
 
@@ -169,6 +175,9 @@ impl IOSSimulatorAccessibility {
     }
 
     /// Capture a screenshot of the entire simulator screen.
+    ///
+    /// This blocks while waiting for a framebuffer and encoding PNG. Async
+    /// callers should use the [`AccessibilityReader`] implementation instead.
     pub fn capture_screen(&self) -> Result<Screenshot> {
         self.inner.capture_screen().map(from_sys_screenshot)
     }
@@ -181,6 +190,9 @@ impl IOSSimulatorAccessibility {
     }
 
     /// Capture a screenshot of a specific element.
+    ///
+    /// This blocks for full-screen capture, image decoding, cropping, and PNG
+    /// re-encoding.
     pub fn capture_element(&mut self, id: ElementKey) -> Result<Screenshot> {
         let sys_id = self.sys_id(id)?;
         self.inner.capture_element(sys_id).map(from_sys_screenshot)
@@ -300,7 +312,15 @@ impl AccessibilityReader for IOSSimulatorAccessibility {
         &self,
         _target: &Target,
     ) -> impl std::future::Future<Output = Result<Screenshot>> {
-        async move { IOSSimulatorAccessibility::capture_screen(self) }
+        let udid = self.device_udid().to_string();
+        async move {
+            tokio::task::spawn_blocking(move || {
+                sys::IOSSimulatorAccessibility::capture_screen_for_device(Some(&udid))
+                    .map(from_sys_screenshot)
+            })
+            .await
+            .map_err(|error| anyhow!("simulator screenshot worker failed: {error}"))?
+        }
     }
 
     fn get_screen_bounds(
@@ -365,6 +385,7 @@ impl SimulatorVideoCapture {
                     sys::ChunkKind::Keyframe => FrameKind::Keyframe,
                     sys::ChunkKind::Delta => FrameKind::Delta,
                 },
+                captured_at: chunk.captured_at,
             });
         });
 
@@ -395,6 +416,10 @@ impl VideoCapture for SimulatorVideoCapture {
         self.inner.request_keyframe();
     }
 
+    fn note_interaction(&self) {
+        self.inner.note_interaction();
+    }
+
     fn start_recording(&self, path: &std::path::Path, config: &RecordingConfig) -> Result<()> {
         self.inner.start_recording(
             path,

packages/accessibility-core/src/platform/ios_simulator/input.rs

@@ -59,6 +59,9 @@ pub enum TouchEdge {
 pub enum HardwareButton {
     Home,
     Lock,
+    VolumeUp,
+    VolumeDown,
+    Mute,
     Siri,
     SideButton,
     ApplePay,
@@ -145,18 +148,31 @@ impl ScrollGesture {
     }
 }
 
-/// Start the HID worker thread and return its command channel.
+/// What the simulator's HID interface supports on this host.
+///
+/// `IndigoHIDMessageForHIDArbitrary` is resolved at runtime, so buttons that
+/// ride on it (volume, mute) are unavailable on Xcode versions without it.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct InputCapabilities {
+    pub arbitrary_hid: bool,
+}
+
+/// Start the HID worker thread and return its command channel plus the
+/// capabilities the worker's `SimulatorHID` resolved.
 ///
 /// The worker owns the `SimulatorHID` because it is not `Sync`, and because
 /// HID sends block on a dispatch queue round trip. It wakes periodically even
 /// when idle so a scroll gesture can be lifted after the wheel stops.
-pub fn spawn_input_worker(udid: &str) -> Result<Sender<InputCommand>> {
+pub fn spawn_input_worker(udid: &str) -> Result<(Sender<InputCommand>, InputCapabilities)> {
     use accessibility_ios_sys::{
         HardwareButton as SysButton, Orientation as SysOrientation, SimulatorHID,
         TouchEdge as SysEdge, TouchPhase as SysPhase,
     };
 
     let hid = SimulatorHID::for_device(Some(udid))?;
+    let capabilities = InputCapabilities {
+        arbitrary_hid: hid.supports_hid_arbitrary(),
+    };
     let (tx, rx) = mpsc::channel::<InputCommand>();
 
     std::thread::Builder::new()
@@ -191,14 +207,31 @@ pub fn spawn_input_worker(udid: &str) -> Result<Sender<InputCommand>> {
                                 hid.touch_normalized_edge(x, y, phase, edge)
                             }
                             InputCommand::Button { button } => {
-                                let button = match button {
-                                    HardwareButton::Home => SysButton::Home,
-                                    HardwareButton::Lock => SysButton::Lock,
-                                    HardwareButton::Siri => SysButton::Siri,
-                                    HardwareButton::SideButton => SysButton::SideButton,
-                                    HardwareButton::ApplePay => SysButton::ApplePay,
-                                };
-                                hid.press_button(button, 0)
+                                const CONSUMER_PAGE: u32 = 0x0c;
+                                const CONSUMER_MUTE: u32 = 0xe2;
+                                const CONSUMER_VOLUME_UP: u32 = 0xe9;
+                                const CONSUMER_VOLUME_DOWN: u32 = 0xea;
+
+                                match button {
+                                    HardwareButton::Home => hid.press_button(SysButton::Home, 0),
+                                    HardwareButton::Lock => hid.press_button(SysButton::Lock, 0),
+                                    HardwareButton::VolumeUp => {
+                                        hid.press_hid_button(CONSUMER_PAGE, CONSUMER_VOLUME_UP, 0)
+                                    }
+                                    HardwareButton::VolumeDown => {
+                                        hid.press_hid_button(CONSUMER_PAGE, CONSUMER_VOLUME_DOWN, 0)
+                                    }
+                                    HardwareButton::Mute => {
+                                        hid.press_hid_button(CONSUMER_PAGE, CONSUMER_MUTE, 0)
+                                    }
+                                    HardwareButton::Siri => hid.press_button(SysButton::Siri, 0),
+                                    HardwareButton::SideButton => {
+                                        hid.press_button(SysButton::SideButton, 0)
+                                    }
+                                    HardwareButton::ApplePay => {
+                                        hid.press_button(SysButton::ApplePay, 0)
+                                    }
+                                }
                             }
                             InputCommand::Key {
                                 key_code,
@@ -245,7 +278,7 @@ pub fn spawn_input_worker(udid: &str) -> Result<Sender<InputCommand>> {
             }
         })?;
 
-    Ok(tx)
+    Ok((tx, capabilities))
 }
 
 /// Expand text into key presses and send them.

packages/accessibility-core/src/platform/ios_simulator/session.rs

@@ -21,8 +21,8 @@ use crate::video::{
 
 use super::SimulatorVideoCapture;
 use super::ax::{AxCommand, AxSnapshot, ElementDetail, spawn_ax_worker};
-use super::input::{InputCommand, Orientation, spawn_input_worker};
-use super::settings::{self, Setting, SettingKey};
+use super::input::{InputCapabilities, InputCommand, Orientation, spawn_input_worker};
+use super::settings::{Setting, SettingKey, SimulatorControl};
 
 /// How many encoded frames to buffer per subscriber.
 ///
@@ -93,7 +93,9 @@ pub struct SimSession {
     stats: Arc<StreamStats>,
     started: Instant,
     input: std::sync::mpsc::Sender<InputCommand>,
+    input_capabilities: InputCapabilities,
     ax: mpsc::UnboundedSender<AxCommand>,
+    control: SimulatorControl,
     /// Last orientation we asked for.
     ///
     /// The framebuffer is always portrait-native — rotating the device
@@ -103,7 +105,17 @@ pub struct SimSession {
 }
 
 impl SimSession {
-    /// Attach to a booted simulator and start capturing.
+    /// Attach to a booted simulator and start its independent runtime lanes.
+    ///
+    /// Starts:
+    ///
+    /// 1. Framebuffer capture and video encoding.
+    /// 2. Low-latency HID input forwarding.
+    /// 3. Accessibility tree and hit-test handling.
+    /// 4. Direct CoreSimulator settings control.
+    ///
+    /// Framework loading and lane initialization are synchronous and may block
+    /// before the session is returned.
     pub fn start(udid: Option<&str>, config: VideoConfig) -> Result<Arc<Self>> {
         let (frames, _) = broadcast::channel(FRAME_BUFFER);
         let latest_parameter_set = Arc::new(std::sync::Mutex::new(None));
@@ -130,8 +142,9 @@ impl SimSession {
         };
 
         let (capture, resolved_udid) = start_capture(udid, &config, sink)?;
-        let input = spawn_input_worker(&resolved_udid)?;
+        let (input, input_capabilities) = spawn_input_worker(&resolved_udid)?;
         let ax = spawn_ax_worker(&resolved_udid)?;
+        let control = SimulatorControl::start(&resolved_udid)?;
 
         Ok(Arc::new(Self {
             device_udid: resolved_udid,
@@ -141,7 +154,9 @@ impl SimSession {
             stats,
             started: Instant::now(),
             input,
+            input_capabilities,
             ax,
+            control,
             orientation: std::sync::Mutex::new(Orientation::Portrait),
         }))
     }
@@ -218,6 +233,10 @@ impl SimSession {
         }
     }
 
+    pub fn input_capabilities(&self) -> InputCapabilities {
+        self.input_capabilities
+    }
+
     pub fn orientation(&self) -> Orientation {
         *self.orientation.lock().unwrap()
     }
@@ -245,6 +264,8 @@ impl SimSession {
         Ok(path)
     }
 
+    /// Finalize the active recording, blocking for up to 30 seconds while the
+    /// writer flushes the MP4 index.
     pub fn stop_recording(&self) -> Result<Recording> {
         self.capture.stop_recording()
     }
@@ -253,12 +274,12 @@ impl SimSession {
         self.capture.recording_frames()
     }
 
-    pub fn settings(&self) -> Vec<Setting> {
-        settings::read_all(&self.device_udid)
+    pub async fn settings(&self) -> Result<Vec<Setting>> {
+        self.control.read_all().await
     }
 
-    pub fn set_setting(&self, key: SettingKey, value: &str) -> Result<String> {
-        settings::write(&self.device_udid, key, value)
+    pub async fn set_setting(&self, key: SettingKey, value: &str) -> Result<String> {
+        self.control.write(key, value).await
     }
 
     /// Queue an input event. Fire-and-forget: pointer events must never block
@@ -267,6 +288,7 @@ impl SimSession {
         if let InputCommand::Rotate { orientation } = command {
             *self.orientation.lock().unwrap() = orientation;
         }
+        self.capture.note_interaction();
         let _ = self.input.send(command);
     }