From 7f67873d412964e7d5ed867c484c250d05492698 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Fri, 21 Aug 2026 01:33:59 +0000 Subject: [PATCH 1/3] Make Android ADB operations async --- .../examples/emulator_raw_capture_probe.rs | 12 +- .../examples/emulator_webrtc_probe.rs | 2 +- .../examples/screenrecord_probe.rs | 15 +- .../accessibility-android-sys/src/emulator.rs | 47 +++-- .../src/emulator/raw.rs | 2 +- packages/accessibility-android-sys/src/lib.rs | 195 +++++++++++------- packages/accessibility-cli/src/lib.rs | 18 +- .../examples/android_session_probe.rs | 49 +++-- .../src/accessibility/mod.rs | 7 +- .../src/accessibility/targeted.rs | 27 ++- packages/accessibility-core/src/api/app.rs | 2 + .../src/platform/android.rs | 142 +++++++------ .../src/platform/android/ax.rs | 4 +- .../src/platform/android/input.rs | 24 ++- .../src/platform/android/session.rs | 18 +- .../src/platform/ios_simulator.rs | 7 +- .../accessibility-core/src/platform/macos.rs | 45 ++-- .../accessibility-core/src/platform/msft.rs | 19 +- .../accessibility-core/src/platform/x11.rs | 28 ++- .../tests/calculator_e2e.rs | 1 + .../tests/settings_android_e2e.rs | 89 ++++---- packages/accessibility-serve/src/http.rs | 4 +- packages/accessibility-serve/src/lib.rs | 1 + packages/accessibility-serve/src/session.rs | 18 +- 24 files changed, 460 insertions(+), 316 deletions(-) diff --git a/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs b/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs index 89ee8c9..145813c 100644 --- a/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs +++ b/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs @@ -17,7 +17,7 @@ const PROBE_DURATION: Duration = Duration::from_secs(5); #[tokio::main] async fn main() -> Result<()> { let selector = std::env::args().nth(1); - let discovery = discover_emulator(selector.as_deref())?; + let discovery = discover_emulator(selector.as_deref()).await?; let serial = discovery .properties .get("port.serial") @@ -34,15 +34,19 @@ async fn main() -> Result<()> { let stimulus_serial = serial.clone(); let stimulus = std::thread::spawn(move || { let adb = AdbClient::discover(Some(&stimulus_serial)); - let Ok((width, height)) = adb.get_screen_size() else { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let Ok((width, height)) = runtime.block_on(adb.get_screen_size()) else { return; }; while !stimulus_stop.load(Ordering::Relaxed) { - let _ = adb.swipe( + let _ = runtime.block_on(adb.swipe( (width as f64 * 0.5, height as f64 * 0.75), (width as f64 * 0.5, height as f64 * 0.25), 250, - ); + )); std::thread::sleep(Duration::from_millis(150)); } }); diff --git a/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs b/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs index 70d1a81..57b242e 100644 --- a/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs +++ b/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs @@ -31,7 +31,7 @@ const RECOVERY_TIMEOUT: Duration = Duration::from_secs(5); #[tokio::main] async fn main() -> Result<()> { let selector = std::env::args().nth(1); - let discovery = discover_emulator(selector.as_deref())?; + let discovery = discover_emulator(selector.as_deref()).await?; println!("discovery : {}", discovery.path.display()); println!("endpoint : {}", discovery.endpoint()); diff --git a/packages/accessibility-android-sys/examples/screenrecord_probe.rs b/packages/accessibility-android-sys/examples/screenrecord_probe.rs index 3df6839..7713424 100644 --- a/packages/accessibility-android-sys/examples/screenrecord_probe.rs +++ b/packages/accessibility-android-sys/examples/screenrecord_probe.rs @@ -11,13 +11,14 @@ use anyhow::{Context, Result, bail}; const IDLE_FLUSH: Duration = Duration::from_millis(75); const PROBE_DURATION: Duration = Duration::from_secs(5); -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { let serial = std::env::args() .nth(1) .unwrap_or_else(|| "emulator-5554".to_string()); let adb = AdbClient::discover(Some(&serial)); - adb.check_connection()?; - let (width, height) = adb.get_screen_size()?; + adb.check_connection().await?; + let (width, height) = adb.get_screen_size().await?; let config = ScreenRecordConfig::for_max_dimension(width, height, Some(1280), 5_000_000); println!("device : {serial}"); println!("source : {width}x{height}"); @@ -25,12 +26,16 @@ fn main() -> Result<()> { let stimulus = adb.clone(); std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); std::thread::sleep(Duration::from_secs(1)); - let _ = stimulus.swipe( + let _ = runtime.block_on(stimulus.swipe( (width as f64 * 0.5, height as f64 * 0.75), (width as f64 * 0.5, height as f64 * 0.25), 800, - ); + )); }); let started = Instant::now(); diff --git a/packages/accessibility-android-sys/src/emulator.rs b/packages/accessibility-android-sys/src/emulator.rs index c6a4050..c009a54 100644 --- a/packages/accessibility-android-sys/src/emulator.rs +++ b/packages/accessibility-android-sys/src/emulator.rs @@ -38,9 +38,9 @@ pub struct EmulatorDiscovery { } impl EmulatorDiscovery { - pub fn from_file(path: impl AsRef) -> Result { + pub async fn from_file(path: impl AsRef) -> Result { let path = path.as_ref(); - let contents = std::fs::read_to_string(path).with_context(|| { + let contents = tokio::fs::read_to_string(path).await.with_context(|| { format!("failed to read emulator discovery file {}", path.display()) })?; let properties = parse_properties(&contents); @@ -246,17 +246,17 @@ impl EmulatorGrpcClient { } } -pub fn discover_emulator(selector: Option<&str>) -> Result { - discover_emulator_in(&discovery_directories(), selector) +pub async fn discover_emulator(selector: Option<&str>) -> Result { + discover_emulator_in(&discovery_directories(), selector).await } -pub fn discover_emulator_in( +pub async fn discover_emulator_in( directories: &[PathBuf], selector: Option<&str>, ) -> Result { let mut paths = BTreeSet::new(); for directory in directories { - let entries = match std::fs::read_dir(directory) { + let mut entries = match tokio::fs::read_dir(directory).await { Ok(entries) => entries, Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, Err(error) => { @@ -268,8 +268,8 @@ pub fn discover_emulator_in( }); } }; - for entry in entries { - let path = entry?.path(); + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); let Some(name) = path.file_name().and_then(|name| name.to_str()) else { continue; }; @@ -279,10 +279,12 @@ pub fn discover_emulator_in( } } - let mut discoveries = paths - .into_iter() - .filter_map(|path| EmulatorDiscovery::from_file(path).ok()) - .collect::>(); + let mut discoveries = Vec::new(); + for path in paths { + if let Ok(discovery) = EmulatorDiscovery::from_file(path).await { + discoveries.push(discovery); + } + } if let Some(selector) = selector { discoveries.retain(|discovery| discovery.matches(selector)); } @@ -351,9 +353,9 @@ mod tests { path } - #[test] + #[tokio::test] #[cfg_attr(miri, ignore)] - fn parses_discovery_file() { + async fn parses_discovery_file() { let directory = test_directory("parse"); let path = directory.join("pid_1234.ini"); std::fs::write( @@ -361,7 +363,7 @@ mod tests { "grpc.port = 8554\ngrpc.token = secret\navd.name = Pixel_8\nport.serial = 5554\n", ) .unwrap(); - let discovery = EmulatorDiscovery::from_file(&path).unwrap(); + let discovery = EmulatorDiscovery::from_file(&path).await.unwrap(); assert_eq!(discovery.pid, Some(1234)); assert_eq!(discovery.grpc_port, 8554); assert_eq!(discovery.grpc_token.as_deref(), Some("secret")); @@ -370,9 +372,9 @@ mod tests { std::fs::remove_dir_all(directory).unwrap(); } - #[test] + #[tokio::test] #[cfg_attr(miri, ignore)] - fn selects_one_emulator() { + async fn selects_one_emulator() { let directory = test_directory("select"); std::fs::write( directory.join("pid_1.ini"), @@ -384,10 +386,15 @@ mod tests { "grpc.port=8555\navd.name=tablet\n", ) .unwrap(); - let discovery = - discover_emulator_in(std::slice::from_ref(&directory), Some("tablet")).unwrap(); + let discovery = discover_emulator_in(std::slice::from_ref(&directory), Some("tablet")) + .await + .unwrap(); assert_eq!(discovery.grpc_port, 8555); - assert!(discover_emulator_in(std::slice::from_ref(&directory), None).is_err()); + assert!( + discover_emulator_in(std::slice::from_ref(&directory), None) + .await + .is_err() + ); std::fs::remove_dir_all(directory).unwrap(); } } diff --git a/packages/accessibility-android-sys/src/emulator/raw.rs b/packages/accessibility-android-sys/src/emulator/raw.rs index 3b64c5a..1fd8013 100644 --- a/packages/accessibility-android-sys/src/emulator/raw.rs +++ b/packages/accessibility-android-sys/src/emulator/raw.rs @@ -48,7 +48,7 @@ impl RawFrameStream { if config.width == 0 || config.height == 0 { bail!("raw Android capture requires non-zero dimensions"); } - let discovery = discover_emulator(selector)?; + let discovery = discover_emulator(selector).await?; Self::start_with_discovery(discovery, config).await } diff --git a/packages/accessibility-android-sys/src/lib.rs b/packages/accessibility-android-sys/src/lib.rs index 1e2f9e3..445b503 100644 --- a/packages/accessibility-android-sys/src/lib.rs +++ b/packages/accessibility-android-sys/src/lib.rs @@ -2,14 +2,16 @@ pub mod emulator; -use std::process::{Command, Output}; +use std::process::{Output, Stdio}; use std::time::Duration; use anyhow::{Context, Result, bail}; use keyboard_types::Code; +use tokio::process::Command; const UI_DUMP_ATTEMPTS: usize = 3; const UI_DUMP_RETRY_DELAY: Duration = Duration::from_millis(500); +pub const DEFAULT_ADB_TIMEOUT: Duration = Duration::from_secs(30); /// Android key codes for `input keyevent` command. /// @@ -435,6 +437,8 @@ pub struct AdbClient { pub serial: Option, /// Path to the ADB binary. pub adb_path: String, + /// Maximum time to wait for an ADB command. + pub timeout: Duration, } impl Default for AdbClient { @@ -442,6 +446,7 @@ impl Default for AdbClient { Self { serial: None, adb_path: "adb".to_string(), + timeout: DEFAULT_ADB_TIMEOUT, } } } @@ -452,6 +457,7 @@ impl AdbClient { Self { serial: serial.map(String::from), adb_path: "adb".to_string(), + timeout: DEFAULT_ADB_TIMEOUT, } } @@ -492,9 +498,16 @@ impl AdbClient { Self { serial: serial.map(String::from), adb_path: adb_path.to_string(), + timeout: DEFAULT_ADB_TIMEOUT, } } + /// Set the maximum time to wait for an ADB command. + pub fn with_timeout(mut self, timeout: Duration) -> Self { + self.timeout = timeout; + self + } + /// Build base ADB command with optional device serial. fn base_command(&self) -> Command { let mut cmd = Command::new(&self.adb_path); @@ -504,52 +517,62 @@ impl AdbClient { cmd } - /// Execute an ADB shell command. - pub fn shell(&self, args: &[&str]) -> Result { + async fn run(&self, kind: &str, args: &[&str], leading: Option<&str>) -> Result { let mut cmd = self.base_command(); - cmd.arg("shell").args(args); - - let output = cmd - .output() - .context("Failed to execute adb shell command")?; + if let Some(leading) = leading { + cmd.arg(leading); + } + cmd.args(args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let child = cmd.spawn().with_context(|| { + if kind == "devices" { + format!( + "ADB binary not found at '{}'. Install Android SDK Platform Tools.", + self.adb_path + ) + } else { + format!("Failed to execute adb {kind} command") + } + })?; + tokio::time::timeout(self.timeout, child.wait_with_output()) + .await + .map_err(|_| { + anyhow::anyhow!( + "ADB binary '{}' {kind} command timed out after {:?}", + self.adb_path, + self.timeout + ) + })? + .with_context(|| format!("Failed to execute adb {kind} command")) + } + /// Execute an ADB shell command. + pub async fn shell(&self, args: &[&str]) -> Result { + let output = self.run("shell", args, Some("shell")).await?; Self::check_output(&output, "shell")?; Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } /// Execute an ADB shell command and return raw bytes. - pub fn shell_raw(&self, args: &[&str]) -> Result> { - let mut cmd = self.base_command(); - cmd.arg("shell").args(args); - - let output = cmd - .output() - .context("Failed to execute adb shell command")?; - + pub async fn shell_raw(&self, args: &[&str]) -> Result> { + let output = self.run("shell", args, Some("shell")).await?; Self::check_output(&output, "shell")?; Ok(output.stdout) } /// Execute `adb exec-out` for efficient binary output. - pub fn exec_out(&self, args: &[&str]) -> Result> { - let mut cmd = self.base_command(); - cmd.arg("exec-out").args(args); - - let output = cmd - .output() - .context("Failed to execute adb exec-out command")?; - + pub async fn exec_out(&self, args: &[&str]) -> Result> { + let output = self.run("exec-out", args, Some("exec-out")).await?; Self::check_output(&output, "exec-out")?; Ok(output.stdout) } /// Execute a general ADB command (not shell). - pub fn command(&self, args: &[&str]) -> Result { - let mut cmd = self.base_command(); - cmd.args(args); - - let output = cmd.output().context("Failed to execute adb command")?; - + pub async fn command(&self, args: &[&str]) -> Result { + let output = self.run("adb", args, None).await?; Self::check_output(&output, "adb")?; Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } @@ -570,8 +593,8 @@ impl AdbClient { } /// Check if ADB is available and a device is connected. - pub fn check_connection(&self) -> Result<()> { - let devices = self.connected_devices()?; + pub async fn check_connection(&self) -> Result<()> { + let devices = self.connected_devices().await?; if devices.is_empty() { bail!("No Android devices connected. Connect a device or start an emulator."); } @@ -587,16 +610,8 @@ impl AdbClient { Ok(()) } - pub fn connected_devices(&self) -> Result> { - let output = Command::new(&self.adb_path) - .arg("devices") - .output() - .with_context(|| { - format!( - "ADB binary not found at '{}'. Install Android SDK Platform Tools.", - self.adb_path - ) - })?; + pub async fn connected_devices(&self) -> Result> { + let output = self.run("devices", &[], Some("devices")).await?; Self::check_output(&output, "devices")?; let stdout = String::from_utf8_lossy(&output.stdout); Ok(stdout @@ -609,8 +624,8 @@ impl AdbClient { .collect()) } - pub fn resolved_serial(&self) -> Result { - let devices = self.connected_devices()?; + pub async fn resolved_serial(&self) -> Result { + let devices = self.connected_devices().await?; if let Some(serial) = &self.serial { if devices.contains(serial) { return Ok(serial.clone()); @@ -632,8 +647,8 @@ impl AdbClient { } /// Get the screen size in pixels. - pub fn get_screen_size(&self) -> Result<(u32, u32)> { - let output = self.shell(&["wm", "size"])?; + pub async fn get_screen_size(&self) -> Result<(u32, u32)> { + let output = self.shell(&["wm", "size"]).await?; for line in output.lines() { if let Some(size_str) = line.strip_prefix("Physical size:") { let size_str = size_str.trim(); @@ -653,23 +668,24 @@ impl AdbClient { } /// Capture a screenshot as PNG bytes. - pub fn screenshot(&self) -> Result> { - self.exec_out(&["screencap", "-p"]) + pub async fn screenshot(&self) -> Result> { + self.exec_out(&["screencap", "-p"]).await } /// Tap at screen coordinates. - pub fn tap(&self, x: f64, y: f64) -> Result<()> { + pub async fn tap(&self, x: f64, y: f64) -> Result<()> { self.shell(&[ "input", "tap", &x.round().to_string(), &y.round().to_string(), - ])?; + ]) + .await?; Ok(()) } /// Swipe from one point to another. - pub fn swipe(&self, start: (f64, f64), end: (f64, f64), duration_ms: u64) -> Result<()> { + pub async fn swipe(&self, start: (f64, f64), end: (f64, f64), duration_ms: u64) -> Result<()> { self.shell(&[ "input", "swipe", @@ -678,34 +694,36 @@ impl AdbClient { &end.0.round().to_string(), &end.1.round().to_string(), &duration_ms.to_string(), - ])?; + ]) + .await?; Ok(()) } /// Send a key event. - pub fn key_event(&self, keycode: u32) -> Result<()> { - self.shell(&["input", "keyevent", &keycode.to_string()])?; + pub async fn key_event(&self, keycode: u32) -> Result<()> { + self.shell(&["input", "keyevent", &keycode.to_string()]) + .await?; Ok(()) } /// Send text input. - pub fn input_text(&self, text: &str) -> Result<()> { + pub async fn input_text(&self, text: &str) -> Result<()> { let escaped = escape_shell_text(text); - self.shell(&["input", "text", &escaped])?; + self.shell(&["input", "text", &escaped]).await?; Ok(()) } /// Dump the UI hierarchy as XML. - pub fn dump_ui(&self) -> Result { + pub async fn dump_ui(&self) -> Result { let mut last_error = None; for attempt in 1..=UI_DUMP_ATTEMPTS { - match self.dump_ui_once() { + match self.dump_ui_once().await { Ok(xml) => return Ok(xml), Err(error) => { last_error = Some(error); if attempt < UI_DUMP_ATTEMPTS { - std::thread::sleep(UI_DUMP_RETRY_DELAY); + tokio::time::sleep(UI_DUMP_RETRY_DELAY).await; } } } @@ -716,13 +734,13 @@ impl AdbClient { )) } - fn dump_ui_once(&self) -> Result { - let result = self.shell(&["uiautomator", "dump", "/dev/tty"]); + async fn dump_ui_once(&self) -> Result { + let result = self.shell(&["uiautomator", "dump", "/dev/tty"]).await; match result { Ok(output) => match extract_ui_xml(&output) { Some(xml) => Ok(xml), - None => self.dump_ui_via_file().with_context(|| { + None => self.dump_ui_via_file().await.with_context(|| { format!( "direct uiautomator dump did not contain XML: {}", truncate_for_error(&output) @@ -731,27 +749,28 @@ impl AdbClient { }, Err(error) => self .dump_ui_via_file() + .await .with_context(|| format!("direct uiautomator dump failed: {error}")), } } - fn dump_ui_via_file(&self) -> Result { + async fn dump_ui_via_file(&self) -> Result { let tmp_path = "/data/local/tmp/window_dump.xml"; - let _ = self.shell(&["rm", "-f", tmp_path]); - let dump_output = self.shell(&["uiautomator", "dump", tmp_path])?; + let _ = self.shell(&["rm", "-f", tmp_path]).await; + let dump_output = self.shell(&["uiautomator", "dump", tmp_path]).await?; if let Some(xml) = extract_ui_xml(&dump_output) { - let _ = self.shell(&["rm", "-f", tmp_path]); + let _ = self.shell(&["rm", "-f", tmp_path]).await; return Ok(xml); } - let xml = self.shell(&["cat", tmp_path]).with_context(|| { + let xml = self.shell(&["cat", tmp_path]).await.with_context(|| { format!( "uiautomator dump did not create readable file at {tmp_path}; dump output: {}", truncate_for_error(&dump_output) ) })?; - let _ = self.shell(&["rm", "-f", tmp_path]); + let _ = self.shell(&["rm", "-f", tmp_path]).await; if let Some(xml) = extract_ui_xml(&xml) { Ok(xml) @@ -761,11 +780,11 @@ impl AdbClient { } /// Launch an app by package name and optional activity. - pub fn launch_app(&self, package: &str, activity: Option<&str>) -> Result<()> { + pub async fn launch_app(&self, package: &str, activity: Option<&str>) -> Result<()> { match activity { Some(act) => { let component = format!("{}/{}", package, act); - self.shell(&["am", "start", "-n", &component])?; + self.shell(&["am", "start", "-n", &component]).await?; } None => { self.shell(&[ @@ -775,21 +794,22 @@ impl AdbClient { "-c", "android.intent.category.LAUNCHER", "1", - ])?; + ]) + .await?; } } Ok(()) } /// Force stop an app. - pub fn stop_app(&self, package: &str) -> Result<()> { - self.shell(&["am", "force-stop", package])?; + pub async fn stop_app(&self, package: &str) -> Result<()> { + self.shell(&["am", "force-stop", package]).await?; Ok(()) } /// Get the current foreground activity. - pub fn get_current_activity(&self) -> Result { - let output = self.shell(&["dumpsys", "activity", "activities"])?; + pub async fn get_current_activity(&self) -> Result { + let output = self.shell(&["dumpsys", "activity", "activities"]).await?; for line in output.lines() { let trimmed = line.trim(); @@ -799,7 +819,7 @@ impl AdbClient { } } - let output = self.shell(&["dumpsys", "window", "windows"])?; + let output = self.shell(&["dumpsys", "window", "windows"]).await?; for line in output.lines() { if line.contains("mCurrentFocus") || line.contains("mFocusedApp") { return Ok(line.trim().to_string()); @@ -921,4 +941,27 @@ mod tests { Some(AndroidKeyCode::F1) ); } + + #[cfg(unix)] + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn adb_command_times_out_and_kills_child() { + let adb = + AdbClient::with_adb_path(None, "/bin/sleep").with_timeout(Duration::from_millis(100)); + let started = std::time::Instant::now(); + let error = adb.command(&["5"]).await.unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(2)); + assert!(error.to_string().contains("timed out after")); + } + + #[cfg(unix)] + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn missing_adb_binary_has_install_context() { + let adb = AdbClient::with_adb_path(None, "/no/such/adb"); + let error = adb.connected_devices().await.unwrap_err(); + assert!(error.to_string().contains( + "ADB binary not found at '/no/such/adb'. Install Android SDK Platform Tools." + )); + } } diff --git a/packages/accessibility-cli/src/lib.rs b/packages/accessibility-cli/src/lib.rs index 2bf75f7..0e23f84 100644 --- a/packages/accessibility-cli/src/lib.rs +++ b/packages/accessibility-cli/src/lib.rs @@ -79,7 +79,7 @@ fn screenshot_path() -> std::path::PathBuf { /// Handle screenshot-screen command. async fn handle_screenshot_screen(adapter: &TargetedAccessibility, args: &CommonArgs) { println!("Capturing full screen screenshot..."); - match adapter.capture_screen() { + match adapter.capture_screen().await { Ok(screenshot) => { if args.overlay { match adapter.get_screen_bounds().await { @@ -109,7 +109,7 @@ async fn handle_screenshot_screen(adapter: &TargetedAccessibility, args: &Common eprintln!("Failed to get screen bounds: {}", e); // Save raw screenshot without overlay as fallback let filename = screenshot_path(); - if let Err(e) = std::fs::write(&filename, &screenshot.data) { + if let Err(e) = tokio::fs::write(&filename, &screenshot.data).await { eprintln!("Failed to save screenshot to {}: {}", filename.display(), e); std::process::exit(1); } @@ -123,7 +123,7 @@ async fn handle_screenshot_screen(adapter: &TargetedAccessibility, args: &Common } } else { let filename = screenshot_path(); - if let Err(e) = std::fs::write(&filename, &screenshot.data) { + if let Err(e) = tokio::fs::write(&filename, &screenshot.data).await { eprintln!("Failed to write {}: {}", filename.display(), e); std::process::exit(1); } @@ -190,7 +190,7 @@ async fn handle_annotate( } // Capture screenshot - let screenshot = match adapter.capture_screen() { + let screenshot = match adapter.capture_screen().await { Ok(s) => s, Err(e) => { eprintln!("Failed to capture screenshot: {}", e); @@ -836,7 +836,7 @@ async fn handle_screenshot_elements( std::process::exit(1); } }; - let screenshot = match adapter.capture_screen() { + let screenshot = match adapter.capture_screen().await { Ok(s) => s, Err(e) => { eprintln!("Failed to capture screen: {}", e); @@ -876,7 +876,7 @@ async fn handle_screenshot_elements( match screenshot.crop(bounds, &screen_bounds) { Ok(cropped) => { let filename = screenshot_path(); - if let Err(e) = std::fs::write(&filename, &cropped.data) { + if let Err(e) = tokio::fs::write(&filename, &cropped.data).await { eprintln!("Failed to write {}: {}", filename.display(), e); std::process::exit(1); } @@ -2274,7 +2274,7 @@ pub async fn run_cli(cli: &Cli) { // Android works on all host platforms via ADB PlatformType::Android => { // Create raw AndroidAccessibility for Android-specific commands - let mut android_adapter = match AndroidAccessibility::new(cli.serial.as_deref()) { + let mut android_adapter = match AndroidAccessibility::new(cli.serial.as_deref()).await { Ok(a) => a, Err(e) => { eprintln!("Failed to create Android adapter: {}", e); @@ -2307,7 +2307,7 @@ pub async fn run_cli(cli: &Cli) { Some(serial) => AndroidTarget::Serial(serial.to_owned()), None => AndroidTarget::DefaultDevice, }; - let mut adapter = match TargetedAccessibility::new_android(android_target) { + let mut adapter = match TargetedAccessibility::new_android(android_target).await { Ok(a) => a, Err(e) => { eprintln!("Failed to create Android adapter: {}", e); @@ -2570,7 +2570,7 @@ async fn handle_android_specific(adapter: &mut AndroidAccessibility, cli: &Cli) // Handle ADB tap if let Some((x, y)) = cli.adb.adb_tap { println!("Tapping at ({}, {})...", x, y); - match adapter.adb().tap(x, y) { + match adapter.adb().tap(x, y).await { Ok(()) => println!("Tap successful!"), Err(e) => { eprintln!("Tap failed: {}", e); diff --git a/packages/accessibility-core/examples/android_session_probe.rs b/packages/accessibility-core/examples/android_session_probe.rs index 13eb895..73a7cdc 100644 --- a/packages/accessibility-core/examples/android_session_probe.rs +++ b/packages/accessibility-core/examples/android_session_probe.rs @@ -12,12 +12,15 @@ use anyhow::{Context, Result}; async fn main() -> Result<()> { let requested_serial = std::env::args().nth(1); let adb = AdbClient::discover(requested_serial.as_deref()); - let serial = adb.resolved_serial()?; - let _ = adb.stop_app("com.google.android.settings.intelligence"); - let _ = adb.stop_app("com.android.settings"); - adb.launch_app("com.android.settings", Some(".Settings"))?; + let serial = adb.resolved_serial().await?; + let _ = adb + .stop_app("com.google.android.settings.intelligence") + .await; + let _ = adb.stop_app("com.android.settings").await; + adb.launch_app("com.android.settings", Some(".Settings")) + .await?; tokio::time::sleep(Duration::from_millis(500)).await; - let session = EmulatorSession::start(Some(&serial), VideoConfig::default())?; + let session = EmulatorSession::start(Some(&serial), VideoConfig::default()).await?; session.seed_orientation().await; let device = session.device_info(); println!("device : {}", device.serial); @@ -42,16 +45,20 @@ async fn main() -> Result<()> { let tap_x = bounds.x + bounds.width / 2.0; let tap_y = bounds.y + bounds.height / 2.0; println!("tap : {}", tappable.selector); - session.send_input(InputCommand::Touch { - phase: TouchPhase::Begin, - x: tap_x, - y: tap_y, - }); - session.send_input(InputCommand::Touch { - phase: TouchPhase::End, - x: tap_x, - y: tap_y, - }); + session + .send_input(InputCommand::Touch { + phase: TouchPhase::Begin, + x: tap_x, + y: tap_y, + }) + .await; + session + .send_input(InputCommand::Touch { + phase: TouchPhase::End, + x: tap_x, + y: tap_y, + }) + .await; tokio::time::sleep(Duration::from_millis(700)).await; let tapped = session.ax_snapshot(false).await?; if tapped @@ -63,9 +70,11 @@ async fn main() -> Result<()> { anyhow::bail!("gRPC touch input did not change the Settings tree"); } - session.send_input(InputCommand::Button { - button: HardwareButton::Home, - }); + session + .send_input(InputCommand::Button { + button: HardwareButton::Home, + }) + .await; tokio::time::sleep(Duration::from_millis(500)).await; let home = session.ax_snapshot(false).await?; if home.app_name == tapped.app_name { @@ -98,7 +107,7 @@ async fn main() -> Result<()> { println!("received : {received} frames, {keyframes} key, {bytes} bytes"); println!("session : {:.1} fps, {:.2} Mbps", stats.fps, stats.mbps); - session.set_orientation(Orientation::LandscapeRight)?; + session.set_orientation(Orientation::LandscapeRight).await?; let landscape = session.ax_snapshot(false).await?; let landscape_stats = session.stats(); if !landscape.is_landscape || landscape_stats.encoded_width <= landscape_stats.encoded_height { @@ -108,7 +117,7 @@ async fn main() -> Result<()> { "landscape : {}x{}", landscape_stats.encoded_width, landscape_stats.encoded_height ); - session.set_orientation(Orientation::Portrait)?; + session.set_orientation(Orientation::Portrait).await?; let portrait = session.ax_snapshot(false).await?; let portrait_stats = session.stats(); if portrait.is_landscape || portrait_stats.encoded_width >= portrait_stats.encoded_height { diff --git a/packages/accessibility-core/src/accessibility/mod.rs b/packages/accessibility-core/src/accessibility/mod.rs index ec3e53c..b6f973c 100644 --- a/packages/accessibility-core/src/accessibility/mod.rs +++ b/packages/accessibility-core/src/accessibility/mod.rs @@ -94,8 +94,11 @@ pub trait AccessibilityReader { // Platform adapter methods (merged from PlatformAdapter trait) /// Capture a screenshot for a target. - fn capture_screen(&self, _target: &Target) -> Result { - anyhow::bail!("Screenshot not supported on this platform") + fn capture_screen( + &self, + _target: &Target, + ) -> impl std::future::Future> { + async { anyhow::bail!("Screenshot not supported on this platform") } } /// Get bounds for coordinate conversion. diff --git a/packages/accessibility-core/src/accessibility/targeted.rs b/packages/accessibility-core/src/accessibility/targeted.rs index 04441bf..8058907 100644 --- a/packages/accessibility-core/src/accessibility/targeted.rs +++ b/packages/accessibility-core/src/accessibility/targeted.rs @@ -76,6 +76,23 @@ macro_rules! dispatch_mut_async { }; } +/// Macro to dispatch immutable async method calls to the inner reader implementation. +macro_rules! dispatch_async { + ($self:expr, $method:ident $(, $arg:expr)*) => { + match &$self.inner { + #[cfg(target_os = "macos")] + AccessibilityReaderImpl::MacOS(r) => AccessibilityReader::$method(r $(, $arg)*).await, + #[cfg(target_os = "macos")] + AccessibilityReaderImpl::IOSSimulator(r) => AccessibilityReader::$method(r $(, $arg)*).await, + #[cfg(target_os = "windows")] + AccessibilityReaderImpl::Windows(r) => AccessibilityReader::$method(r $(, $arg)*).await, + #[cfg(target_os = "linux")] + AccessibilityReaderImpl::Linux(r) => AccessibilityReader::$method(r $(, $arg)*).await, + AccessibilityReaderImpl::Android(r) => AccessibilityReader::$method(r $(, $arg)*).await, + } + }; +} + /// Wrapper that stores a target and provides convenience methods. /// /// This wrapper holds an underlying `AccessibilityReader` implementation and @@ -94,7 +111,7 @@ macro_rules! dispatch_mut_async { /// /// // No need to pass pid on every call /// let tree = reader.get_tree(&TreeFilter::default())?; -/// let screenshot = reader.capture_screen()?; +/// let screenshot = reader.capture_screen().await?; /// reader.keystroke(Code::Enter, Modifiers::empty())?; /// ``` pub struct TargetedAccessibility { @@ -252,10 +269,10 @@ impl TargetedAccessibility { } /// Create a new Android accessibility reader. - pub fn new_android(target: AndroidTarget) -> Result { + pub async fn new_android(target: AndroidTarget) -> Result { Ok(Self { inner: AccessibilityReaderImpl::Android( - crate::platform::android::AndroidAccessibility::new(target.serial())?, + crate::platform::android::AndroidAccessibility::new(target.serial()).await?, ), target: Target::Android(target), }) @@ -279,8 +296,8 @@ impl TargetedAccessibility { /// Capture a screenshot of the target window. /// /// Uses the stored target automatically. - pub fn capture_screen(&self) -> Result { - dispatch!(self, capture_screen, &self.target) + pub async fn capture_screen(&self) -> Result { + dispatch_async!(self, capture_screen, &self.target) } /// Get the bounds of the target window. diff --git a/packages/accessibility-core/src/api/app.rs b/packages/accessibility-core/src/api/app.rs index ad8f1e1..c2da9c9 100644 --- a/packages/accessibility-core/src/api/app.rs +++ b/packages/accessibility-core/src/api/app.rs @@ -157,6 +157,7 @@ impl App { }, Platform::Android => match &config.target { Target::Android(target) => TargetedAccessibility::new_android(target.clone()) + .await .map_err(|e| Error::ConnectionFailed { message: format!("Failed to create Android adapter: {}", e), }), @@ -286,6 +287,7 @@ impl App { let inner = self.inner.lock().await; inner .capture_screen() + .await .map_err(|e: anyhow::Error| Error::ScreenshotFailed { message: e.to_string(), }) diff --git a/packages/accessibility-core/src/platform/android.rs b/packages/accessibility-core/src/platform/android.rs index 8d4062a..4c05e3e 100644 --- a/packages/accessibility-core/src/platform/android.rs +++ b/packages/accessibility-core/src/platform/android.rs @@ -13,7 +13,7 @@ //! //! ```text //! Rust (AndroidAccessibility) -//! ↓ std::process::Command +//! ↓ tokio::process::Command //! adb shell / adb exec-out //! ↓ //! Android Device/Emulator @@ -32,7 +32,7 @@ //! use accessibility_core::accessibility::{AccessibilityReader, TreeFilter}; //! //! // Connect to the default device -//! let mut reader = AndroidAccessibility::new(None)?; +//! let mut reader = AndroidAccessibility::new(None).await?; //! //! // Get the UI tree //! let tree = reader.get_tree(&Target::Android(AndroidTarget::DefaultDevice), &TreeFilter::default()).await?; @@ -501,17 +501,17 @@ impl AndroidAccessibility { /// # Example /// ```ignore /// // Connect to default device - /// let reader = AndroidAccessibility::new(None)?; + /// let reader = AndroidAccessibility::new(None).await?; /// /// // Connect to specific device - /// let reader = AndroidAccessibility::new(Some("emulator-5554"))?; + /// let reader = AndroidAccessibility::new(Some("emulator-5554")).await?; /// ``` - pub fn new(serial: Option<&str>) -> Result { + pub async fn new(serial: Option<&str>) -> Result { let adb = AdbClient::new(serial); - adb.check_connection()?; + adb.check_connection().await?; // Get initial screen size - let screen_size = adb.get_screen_size().ok(); + let screen_size = adb.get_screen_size().await.ok(); Ok(Self { adb, @@ -523,11 +523,11 @@ impl AndroidAccessibility { } /// Create a new Android accessibility reader with a custom ADB path. - pub fn with_adb_path(serial: Option<&str>, adb_path: &str) -> Result { + pub async fn with_adb_path(serial: Option<&str>, adb_path: &str) -> Result { let adb = AdbClient::with_adb_path(serial, adb_path); - adb.check_connection()?; + adb.check_connection().await?; - let screen_size = adb.get_screen_size().ok(); + let screen_size = adb.get_screen_size().await.ok(); Ok(Self { adb, @@ -549,8 +549,8 @@ impl AndroidAccessibility { } /// Refresh the cached screen size. - pub fn refresh_screen_size(&mut self) -> Result<(u32, u32)> { - let size = self.adb.get_screen_size()?; + pub async fn refresh_screen_size(&mut self) -> Result<(u32, u32)> { + let size = self.adb.get_screen_size().await?; self.screen_size = Some(size); Ok(size) } @@ -575,7 +575,7 @@ impl AccessibilityReader for AndroidAccessibility { self.element_bounds.clear(); // Dump UI hierarchy - let xml = self.adb.dump_ui()?; + let xml = self.adb.dump_ui().await?; // Parse XML into node tree let root_node = parse_ui_xml(&xml)?; @@ -622,7 +622,7 @@ impl AccessibilityReader for AndroidAccessibility { let center = self .get_element_center(id) .ok_or_else(|| anyhow!("Element {} not found or has no bounds", id))?; - self.adb.tap(center.x, center.y)?; + self.adb.tap(center.x, center.y).await?; Ok(()) } Action::Focus => { @@ -630,7 +630,7 @@ impl AccessibilityReader for AndroidAccessibility { let center = self .get_element_center(id) .ok_or_else(|| anyhow!("Element {} not found or has no bounds", id))?; - self.adb.tap(center.x, center.y)?; + self.adb.tap(center.x, center.y).await?; Ok(()) } Action::ScrollIntoView => { @@ -640,7 +640,8 @@ impl AccessibilityReader for AndroidAccessibility { let start_y = size.1 as f64 * 0.7; let end_y = size.1 as f64 * 0.3; self.adb - .swipe((center_x, start_y), (center_x, end_y), 300)?; + .swipe((center_x, start_y), (center_x, end_y), 300) + .await?; } Ok(()) } @@ -657,19 +658,19 @@ impl AccessibilityReader for AndroidAccessibility { let center = self .get_element_center(id) .ok_or_else(|| anyhow!("Element {} not found or has no bounds", id))?; - self.adb.tap(center.x, center.y)?; + self.adb.tap(center.x, center.y).await?; // Small delay to ensure focus tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; // Clear existing text (select all and delete) - self.adb.key_event(AndroidKeyCode::CtrlLeft as u32)?; - self.adb.key_event(AndroidKeyCode::A as u32)?; - self.adb.key_event(AndroidKeyCode::Del as u32)?; + self.adb.key_event(AndroidKeyCode::CtrlLeft as u32).await?; + self.adb.key_event(AndroidKeyCode::A as u32).await?; + self.adb.key_event(AndroidKeyCode::Del as u32).await?; // Type the new value if !value.is_empty() { - self.adb.input_text(value)?; + self.adb.input_text(value).await?; } Ok(()) @@ -709,25 +710,27 @@ impl AccessibilityReader for AndroidAccessibility { self.cache.version() } - fn capture_screen(&self, _target: &Target) -> Result { - let data = self.adb.screenshot()?; - - // Get image dimensions from PNG header - let (width, height) = if data.len() > 24 { - // PNG header: 8 bytes signature, then IHDR chunk - // IHDR starts at byte 8, width at 16, height at 20 (both big-endian u32) - let width = u32::from_be_bytes([data[16], data[17], data[18], data[19]]); - let height = u32::from_be_bytes([data[20], data[21], data[22], data[23]]); - (width, height) - } else { - self.screen_size.unwrap_or_default() - }; - - Ok(Screenshot { - data, - width, - height, - }) + fn capture_screen(&self, _target: &Target) -> impl Future> { + async move { + let data = self.adb.screenshot().await?; + + // Get image dimensions from PNG header + let (width, height) = if data.len() > 24 { + // PNG header: 8 bytes signature, then IHDR chunk + // IHDR starts at byte 8, width at 16, height at 20 (both big-endian u32) + let width = u32::from_be_bytes([data[16], data[17], data[18], data[19]]); + let height = u32::from_be_bytes([data[20], data[21], data[22], data[23]]); + (width, height) + } else { + self.screen_size.unwrap_or_default() + }; + + Ok(Screenshot { + data, + width, + height, + }) + } } fn get_screen_bounds(&self, _target: &Target) -> impl Future> { @@ -755,21 +758,21 @@ impl AccessibilityReader for AndroidAccessibility { async move { // Press modifiers if modifiers.contains(Modifiers::SHIFT) { - self.adb.key_event(AndroidKeyCode::ShiftLeft as u32)?; + self.adb.key_event(AndroidKeyCode::ShiftLeft as u32).await?; } if modifiers.contains(Modifiers::CONTROL) { - self.adb.key_event(AndroidKeyCode::CtrlLeft as u32)?; + self.adb.key_event(AndroidKeyCode::CtrlLeft as u32).await?; } if modifiers.contains(Modifiers::ALT) { - self.adb.key_event(AndroidKeyCode::AltLeft as u32)?; + self.adb.key_event(AndroidKeyCode::AltLeft as u32).await?; } if modifiers.contains(Modifiers::META) { - self.adb.key_event(AndroidKeyCode::MetaLeft as u32)?; + self.adb.key_event(AndroidKeyCode::MetaLeft as u32).await?; } // Press main key if let Some(keycode) = AndroidKeyCode::from_code(key) { - self.adb.key_event(keycode as u32)?; + self.adb.key_event(keycode as u32).await?; } else { bail!("Unsupported key code: {:?}", key); } @@ -780,7 +783,7 @@ impl AccessibilityReader for AndroidAccessibility { fn type_raw(&mut self, _target: &Target, text: &str) -> impl Future> { async move { - self.adb.input_text(text)?; + self.adb.input_text(text).await?; Ok(()) } } @@ -794,7 +797,7 @@ impl AccessibilityReader for AndroidAccessibility { ) -> impl Future> { async move { // Android only supports single tap (no right-click) - self.adb.tap(x, y)?; + self.adb.tap(x, y).await?; Ok(()) } } @@ -821,7 +824,9 @@ impl AccessibilityReader for AndroidAccessibility { let end_x = center_x + delta_x * swipe_distance / 2.0; let end_y = center_y - delta_y * swipe_distance / 2.0; - self.adb.swipe((start_x, start_y), (end_x, end_y), 100)?; + self.adb + .swipe((start_x, start_y), (end_x, end_y), 100) + .await?; Ok(()) } } @@ -990,84 +995,88 @@ pub trait AndroidExtensions { impl AndroidExtensions for AndroidAccessibility { fn press_back(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::Back as u32)?; + self.adb.key_event(AndroidKeyCode::Back as u32).await?; Ok(()) } } fn press_home(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::Home as u32)?; + self.adb.key_event(AndroidKeyCode::Home as u32).await?; Ok(()) } } fn press_recent_apps(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::AppSwitch as u32)?; + self.adb.key_event(AndroidKeyCode::AppSwitch as u32).await?; Ok(()) } } fn press_menu(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::Menu as u32)?; + self.adb.key_event(AndroidKeyCode::Menu as u32).await?; Ok(()) } } fn volume_up(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::VolumeUp as u32)?; + self.adb.key_event(AndroidKeyCode::VolumeUp as u32).await?; Ok(()) } } fn volume_down(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::VolumeDown as u32)?; + self.adb + .key_event(AndroidKeyCode::VolumeDown as u32) + .await?; Ok(()) } } fn volume_mute(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::VolumeMute as u32)?; + self.adb + .key_event(AndroidKeyCode::VolumeMute as u32) + .await?; Ok(()) } } fn press_power(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::Power as u32)?; + self.adb.key_event(AndroidKeyCode::Power as u32).await?; Ok(()) } } fn wake_up(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::Wakeup as u32)?; + self.adb.key_event(AndroidKeyCode::Wakeup as u32).await?; Ok(()) } } fn sleep(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::Sleep as u32)?; + self.adb.key_event(AndroidKeyCode::Sleep as u32).await?; Ok(()) } } fn launch_app(&mut self, package: &str) -> impl Future> { async move { - self.adb.launch_app(package, None)?; + self.adb.launch_app(package, None).await?; Ok(()) } } fn stop_app(&mut self, package: &str) -> impl Future> { async move { - self.adb.stop_app(package)?; + self.adb.stop_app(package).await?; Ok(()) } } @@ -1079,7 +1088,7 @@ impl AndroidExtensions for AndroidAccessibility { duration_ms: u64, ) -> impl Future> { async move { - self.adb.swipe(start, end, duration_ms)?; + self.adb.swipe(start, end, duration_ms).await?; Ok(()) } } @@ -1087,26 +1096,29 @@ impl AndroidExtensions for AndroidAccessibility { fn long_press(&mut self, x: f64, y: f64, duration_ms: u64) -> impl Future> { async move { // Long press is a swipe with same start and end - self.adb.swipe((x, y), (x, y), duration_ms)?; + self.adb.swipe((x, y), (x, y), duration_ms).await?; Ok(()) } } fn get_current_activity(&self) -> impl Future> { - async move { self.adb.get_current_activity() } + async move { self.adb.get_current_activity().await } } fn open_notifications(&mut self) -> impl Future> { async move { self.adb - .shell(&["cmd", "statusbar", "expand-notifications"])?; + .shell(&["cmd", "statusbar", "expand-notifications"]) + .await?; Ok(()) } } fn open_quick_settings(&mut self) -> impl Future> { async move { - self.adb.shell(&["cmd", "statusbar", "expand-settings"])?; + self.adb + .shell(&["cmd", "statusbar", "expand-settings"]) + .await?; Ok(()) } } diff --git a/packages/accessibility-core/src/platform/android/ax.rs b/packages/accessibility-core/src/platform/android/ax.rs index 5c975d9..3edd731 100644 --- a/packages/accessibility-core/src/platform/android/ax.rs +++ b/packages/accessibility-core/src/platform/android/ax.rs @@ -84,9 +84,9 @@ pub enum AxCommand { }, } -pub fn spawn_ax_worker(serial: &str) -> Result> { +pub async fn spawn_ax_worker(serial: &str) -> Result> { let adb = super::AdbClient::discover(Some(serial)); - let mut reader = AndroidAccessibility::with_adb_path(Some(serial), &adb.adb_path)?; + let mut reader = AndroidAccessibility::with_adb_path(Some(serial), &adb.adb_path).await?; let target = Target::Android(AndroidTarget::Serial(serial.to_string())); let (commands, mut command_rx) = mpsc::unbounded_channel(); std::thread::Builder::new() diff --git a/packages/accessibility-core/src/platform/android/input.rs b/packages/accessibility-core/src/platform/android/input.rs index 9be2b6c..0191e7f 100644 --- a/packages/accessibility-core/src/platform/android/input.rs +++ b/packages/accessibility-core/src/platform/android/input.rs @@ -89,11 +89,11 @@ pub enum InputCommand { }, } -pub fn spawn_input_worker( +pub async fn spawn_input_worker( serial: &str, geometry: ScreenGeometry, ) -> Result> { - let discovery = discover_emulator(Some(serial))?; + let discovery = discover_emulator(Some(serial)).await?; let adb = AdbClient::discover(Some(serial)); let (commands, mut command_rx) = tokio::sync::mpsc::unbounded_channel(); let (ready_tx, ready_rx) = mpsc::sync_channel(1); @@ -122,11 +122,11 @@ pub fn spawn_input_worker( while let Some(command) = command_rx.recv().await { let command = match command { InputCommand::Rotate { orientation } => { - let _ = set_device_orientation(&adb, orientation); + let _ = set_device_orientation(&adb, orientation).await; continue; } InputCommand::Button { button } => { - apply_hardware_button(&adb, button); + apply_hardware_button(&adb, button).await; continue; } command => command, @@ -236,26 +236,28 @@ fn normalized_coordinate(value: f64, dimension: u32) -> i32 { (value.clamp(0.0, 1.0) * dimension.saturating_sub(1) as f64).round() as i32 } -fn apply_hardware_button(adb: &AdbClient, button: HardwareButton) { +async fn apply_hardware_button(adb: &AdbClient, button: HardwareButton) { let key = match button { HardwareButton::Home => AndroidKeyCode::Home, HardwareButton::Back => AndroidKeyCode::Back, HardwareButton::Lock => AndroidKeyCode::Power, HardwareButton::AppSwitch => AndroidKeyCode::AppSwitch, }; - let _ = adb.key_event(key as u32); + let _ = adb.key_event(key as u32).await; } -pub fn set_device_orientation(adb: &AdbClient, orientation: Orientation) -> Result<()> { - adb.shell(&["wm", "fixed-to-user-rotation", "enabled"])?; +pub async fn set_device_orientation(adb: &AdbClient, orientation: Orientation) -> Result<()> { + adb.shell(&["wm", "fixed-to-user-rotation", "enabled"]) + .await?; let target = orientation.android_rotation(); - adb.shell(&["wm", "user-rotation", "lock", &target.to_string()])?; + adb.shell(&["wm", "user-rotation", "lock", &target.to_string()]) + .await?; for _ in 0..20 { - let output = adb.shell(&["dumpsys", "display"])?; + let output = adb.shell(&["dumpsys", "display"]).await?; if display_rotation(&output) == Some(target) { return Ok(()); } - std::thread::sleep(std::time::Duration::from_millis(100)); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; } bail!("Android display did not reach rotation {target}") } diff --git a/packages/accessibility-core/src/platform/android/session.rs b/packages/accessibility-core/src/platform/android/session.rs index cac3eda..f5f9c36 100644 --- a/packages/accessibility-core/src/platform/android/session.rs +++ b/packages/accessibility-core/src/platform/android/session.rs @@ -63,14 +63,14 @@ pub struct EmulatorSession { } impl EmulatorSession { - pub fn start(serial: Option<&str>, config: VideoConfig) -> Result> { + pub async fn start(serial: Option<&str>, config: VideoConfig) -> Result> { let adb = AdbClient::discover(serial); - let serial = adb.resolved_serial()?; + let serial = adb.resolved_serial().await?; if !serial.starts_with("emulator-") { bail!("Android Emulator streaming requires an emulator serial, got '{serial}'"); } let adb = AdbClient::discover(Some(&serial)); - let (width, height) = adb.get_screen_size()?; + let (width, height) = adb.get_screen_size().await?; let geometry = ScreenGeometry { width, height }; let (frames, _) = broadcast::channel(FRAME_BUFFER); let stats = Arc::new(StreamStats::default()); @@ -89,8 +89,8 @@ impl EmulatorSession { }) }; let capture = AndroidVideoCapture::start(adb.clone(), geometry, &config, sink)?; - let input = spawn_input_worker(&serial, geometry)?; - let ax = spawn_ax_worker(&serial)?; + let input = spawn_input_worker(&serial, geometry).await?; + let ax = spawn_ax_worker(&serial).await?; Ok(Arc::new(Self { serial, adb, @@ -133,16 +133,16 @@ impl EmulatorSession { *self.orientation.lock().unwrap() } - pub fn set_orientation(&self, orientation: Orientation) -> Result<()> { - set_device_orientation(&self.adb, orientation)?; + pub async fn set_orientation(&self, orientation: Orientation) -> Result<()> { + set_device_orientation(&self.adb, orientation).await?; self.capture.set_landscape(orientation.is_landscape())?; *self.orientation.lock().unwrap() = orientation; Ok(()) } - pub fn send_input(&self, command: InputCommand) { + pub async fn send_input(&self, command: InputCommand) { if let InputCommand::Rotate { orientation } = command { - let _ = self.set_orientation(orientation); + let _ = self.set_orientation(orientation).await; return; } let _ = self.input.send(command); diff --git a/packages/accessibility-core/src/platform/ios_simulator.rs b/packages/accessibility-core/src/platform/ios_simulator.rs index 8a15bd7..f0a38a6 100644 --- a/packages/accessibility-core/src/platform/ios_simulator.rs +++ b/packages/accessibility-core/src/platform/ios_simulator.rs @@ -296,8 +296,11 @@ impl AccessibilityReader for IOSSimulatorAccessibility { IOSSimulatorAccessibility::snapshot_version(self) } - fn capture_screen(&self, _target: &Target) -> Result { - IOSSimulatorAccessibility::capture_screen(self) + fn capture_screen( + &self, + _target: &Target, + ) -> impl std::future::Future> { + async move { IOSSimulatorAccessibility::capture_screen(self) } } fn get_screen_bounds( diff --git a/packages/accessibility-core/src/platform/macos.rs b/packages/accessibility-core/src/platform/macos.rs index 5772821..50201be 100644 --- a/packages/accessibility-core/src/platform/macos.rs +++ b/packages/accessibility-core/src/platform/macos.rs @@ -1793,31 +1793,36 @@ impl AccessibilityReader for MacOSAccessibility { true } - fn capture_screen(&self, target: &Target) -> Result { - let pid = match target { - Target::Pid(pid) => Some(*pid), - Target::System => None, - _ => bail!("macOS screenshot requires Target::Pid or Target::System"), - }; + fn capture_screen( + &self, + target: &Target, + ) -> impl std::future::Future> { + async move { + let pid = match target { + Target::Pid(pid) => Some(*pid), + Target::System => None, + _ => bail!("macOS screenshot requires Target::Pid or Target::System"), + }; - if let Some(pid) = pid - && let Ok(Some(screenshot)) = Self::capture_window_for_pid(pid) - { - return Ok(screenshot); - } + if let Some(pid) = pid + && let Ok(Some(screenshot)) = Self::capture_window_for_pid(pid) + { + return Ok(screenshot); + } - let screenshot = Self::capture_main_display()?; + let screenshot = Self::capture_main_display()?; - if let Some(pid) = pid - && let Some(window_bounds) = Self::get_window_bounds_for_pid(pid) - { - let screen_bounds = Self::main_display_bounds(); - if let Ok(cropped) = screenshot.crop(&window_bounds, &screen_bounds) { - return Ok(cropped); + if let Some(pid) = pid + && let Some(window_bounds) = Self::get_window_bounds_for_pid(pid) + { + let screen_bounds = Self::main_display_bounds(); + if let Ok(cropped) = screenshot.crop(&window_bounds, &screen_bounds) { + return Ok(cropped); + } } - } - Ok(screenshot) + Ok(screenshot) + } } async fn get_screen_bounds(&self, target: &Target) -> Result { diff --git a/packages/accessibility-core/src/platform/msft.rs b/packages/accessibility-core/src/platform/msft.rs index 902d600..342de23 100644 --- a/packages/accessibility-core/src/platform/msft.rs +++ b/packages/accessibility-core/src/platform/msft.rs @@ -189,13 +189,18 @@ impl AccessibilityReader for WindowsAccessibility { self.cache.version() } - fn capture_screen(&self, target: &Target) -> Result { - let screenshot = match target { - Target::Pid(pid) => self.inner.capture_screen_for_pid(*pid), - Target::System => self.inner.capture_screen(), - _ => bail!("Windows screenshot requires Target::Pid or Target::System"), - }; - screenshot.map(from_sys_screenshot) + fn capture_screen( + &self, + target: &Target, + ) -> impl std::future::Future> { + async move { + let screenshot = match target { + Target::Pid(pid) => self.inner.capture_screen_for_pid(*pid), + Target::System => self.inner.capture_screen(), + _ => bail!("Windows screenshot requires Target::Pid or Target::System"), + }; + screenshot.map(from_sys_screenshot) + } } async fn get_screen_bounds(&self, target: &Target) -> Result { diff --git a/packages/accessibility-core/src/platform/x11.rs b/packages/accessibility-core/src/platform/x11.rs index 2a72911..8a38812 100644 --- a/packages/accessibility-core/src/platform/x11.rs +++ b/packages/accessibility-core/src/platform/x11.rs @@ -1063,18 +1063,24 @@ impl AccessibilityReader for LinuxAccessibility { // Platform adapter methods (merged from LinuxAdapter) - fn capture_screen(&self, target: &Target) -> Result { - let pid = match target { - Target::Pid(pid) => Some(*pid), - Target::System => None, - _ => bail!("Linux screenshot requires Target::Pid or Target::System"), - }; - if let Some(pid) = pid - && let Ok(screenshot) = self.capture_window(pid) - { - return Ok(screenshot); + #[allow(clippy::manual_async_fn)] + fn capture_screen( + &self, + target: &Target, + ) -> impl std::future::Future> { + async move { + let pid = match target { + Target::Pid(pid) => Some(*pid), + Target::System => None, + _ => bail!("Linux screenshot requires Target::Pid or Target::System"), + }; + if let Some(pid) = pid + && let Ok(screenshot) = self.capture_window(pid) + { + return Ok(screenshot); + } + LinuxAccessibility::capture_screen(self) } - LinuxAccessibility::capture_screen(self) } async fn get_screen_bounds(&self, target: &Target) -> Result { diff --git a/packages/accessibility-core/tests/calculator_e2e.rs b/packages/accessibility-core/tests/calculator_e2e.rs index 25fd94d..0762517 100644 --- a/packages/accessibility-core/tests/calculator_e2e.rs +++ b/packages/accessibility-core/tests/calculator_e2e.rs @@ -506,6 +506,7 @@ async fn test_screen_screenshot() { let screenshot = accessibility .capture_screen(&Target::System) + .await .expect("Failed to capture screen"); // Screen should have reasonable dimensions (at least 800x600) diff --git a/packages/accessibility-core/tests/settings_android_e2e.rs b/packages/accessibility-core/tests/settings_android_e2e.rs index 62aefa2..e39cab2 100644 --- a/packages/accessibility-core/tests/settings_android_e2e.rs +++ b/packages/accessibility-core/tests/settings_android_e2e.rs @@ -31,21 +31,22 @@ struct DeviceGuard { } impl DeviceGuard { - fn new() -> Result { + async fn new() -> Result { let adb = AdbClient::new(None); adb.command(&["wait-for-device"]) + .await .context("Failed waiting for Android device")?; - adb.check_connection()?; - wait_for_boot(&adb)?; - stabilize_device(&adb); + adb.check_connection().await?; + wait_for_boot(&adb).await?; + stabilize_device(&adb).await; Ok(Self { adb }) } } -fn wait_for_boot(adb: &AdbClient) -> Result<()> { +async fn wait_for_boot(adb: &AdbClient) -> Result<()> { let start = Instant::now(); loop { - if let Ok(output) = adb.shell(&["getprop", "sys.boot_completed"]) + if let Ok(output) = adb.shell(&["getprop", "sys.boot_completed"]).await && output.trim() == "1" { return Ok(()); @@ -58,34 +59,38 @@ fn wait_for_boot(adb: &AdbClient) -> Result<()> { ); } - std::thread::sleep(Duration::from_secs(2)); + tokio::time::sleep(Duration::from_secs(2)).await; } } -fn stabilize_device(adb: &AdbClient) { - let _ = adb.shell(&["input", "keyevent", "224"]); - let _ = adb.shell(&["wm", "dismiss-keyguard"]); +async fn stabilize_device(adb: &AdbClient) { + let _ = adb.shell(&["input", "keyevent", "224"]).await; + let _ = adb.shell(&["wm", "dismiss-keyguard"]).await; for setting in [ "window_animation_scale", "transition_animation_scale", "animator_duration_scale", ] { - let _ = adb.shell(&["settings", "put", "global", setting, "0"]); + let _ = adb + .shell(&["settings", "put", "global", setting, "0"]) + .await; } } -fn launch_settings(adb: &AdbClient) -> Result<()> { - let launcher_result = adb.launch_app(SETTINGS_PACKAGE, None); - let settings_result = adb.shell(&[ - "am", - "start", - "-W", - "-a", - SETTINGS_ACTION, - "-p", - SETTINGS_PACKAGE, - ]); +async fn launch_settings(adb: &AdbClient) -> Result<()> { + let launcher_result = adb.launch_app(SETTINGS_PACKAGE, None).await; + let settings_result = adb + .shell(&[ + "am", + "start", + "-W", + "-a", + SETTINGS_ACTION, + "-p", + SETTINGS_PACKAGE, + ]) + .await; match (launcher_result, settings_result) { (Ok(()), _) | (_, Ok(_)) => {} @@ -99,11 +104,11 @@ fn launch_settings(adb: &AdbClient) -> Result<()> { Ok(()) } -fn wait_for_settings_process(adb: &AdbClient, timeout: Duration) -> Result<()> { +async fn wait_for_settings_process(adb: &AdbClient, timeout: Duration) -> Result<()> { let start = Instant::now(); loop { - let observation = match adb.shell(&["pidof", SETTINGS_PACKAGE]) { + let observation = match adb.shell(&["pidof", SETTINGS_PACKAGE]).await { Ok(pid) => { let pid = pid.trim(); if !pid.is_empty() { @@ -111,7 +116,7 @@ fn wait_for_settings_process(adb: &AdbClient, timeout: Duration) -> Result<()> { } "pidof returned no Settings process".to_string() } - Err(pidof_error) => match adb.shell(&["ps", "-A"]) { + Err(pidof_error) => match adb.shell(&["ps", "-A"]).await { Ok(processes) => { if processes .lines() @@ -133,7 +138,7 @@ fn wait_for_settings_process(adb: &AdbClient, timeout: Duration) -> Result<()> { ); } - std::thread::sleep(POLL_INTERVAL); + tokio::time::sleep(POLL_INTERVAL).await; } } @@ -144,8 +149,9 @@ struct AndroidSettingsGuard { impl AndroidSettingsGuard { async fn launch() -> Result { - let device = DeviceGuard::new()?; + let device = DeviceGuard::new().await?; let mut accessibility = AndroidAccessibility::new(None) + .await .context("Failed to create Android accessibility reader")?; reset_settings(&mut accessibility).await?; @@ -172,7 +178,12 @@ impl AndroidSettingsGuard { impl Drop for AndroidSettingsGuard { fn drop(&mut self) { - let _ = self.device.adb.stop_app(SETTINGS_PACKAGE); + if let Ok(handle) = tokio::runtime::Handle::try_current() { + let adb = self.device.adb.clone(); + handle.spawn(async move { + let _ = adb.stop_app(SETTINGS_PACKAGE).await; + }); + } } } @@ -185,13 +196,13 @@ impl Deref for AndroidSettingsGuard { } async fn reset_settings(accessibility: &mut AndroidAccessibility) -> Result<()> { - let _ = accessibility.adb().stop_app(SETTINGS_PACKAGE); + let _ = accessibility.adb().stop_app(SETTINGS_PACKAGE).await; let _ = accessibility.wake_up().await; let _ = accessibility.press_home().await; tokio::time::sleep(Duration::from_secs(1)).await; - launch_settings(accessibility.adb())?; - wait_for_settings_process(accessibility.adb(), SETTINGS_PROCESS_TIMEOUT)?; + launch_settings(accessibility.adb()).await?; + wait_for_settings_process(accessibility.adb(), SETTINGS_PROCESS_TIMEOUT).await?; tokio::time::sleep(Duration::from_secs(3)).await; Ok(()) } @@ -231,7 +242,7 @@ async fn wait_for_settings_tree( } if start.elapsed() >= next_relaunch { - let _ = launch_settings(adb); + let _ = launch_settings(adb).await; next_relaunch = start.elapsed() + SETTINGS_RELAUNCH_INTERVAL; } @@ -252,9 +263,10 @@ fn count_role(element: &Element, role: Role) -> usize { #[serial] #[ignore = "Requires Android device/emulator with ADB"] async fn test_android_device_input_smoke() -> Result<()> { - let device = DeviceGuard::new()?; - let mut accessibility = - AndroidAccessibility::new(None).context("Failed to create Android accessibility reader")?; + let device = DeviceGuard::new().await?; + let mut accessibility = AndroidAccessibility::new(None) + .await + .context("Failed to create Android accessibility reader")?; accessibility.wake_up().await?; accessibility.press_home().await?; @@ -266,12 +278,13 @@ async fn test_android_device_input_smoke() -> Result<()> { let (width, height) = accessibility .refresh_screen_size() + .await .context("Failed to get Android screen size")?; assert!(width > 0); assert!(height > 0); - launch_settings(accessibility.adb())?; - wait_for_settings_process(accessibility.adb(), SETTINGS_PROCESS_TIMEOUT)?; + launch_settings(accessibility.adb()).await?; + wait_for_settings_process(accessibility.adb(), SETTINGS_PROCESS_TIMEOUT).await?; tokio::time::sleep(Duration::from_secs(2)).await; let center_x = width as f64 / 2.0; @@ -285,7 +298,7 @@ async fn test_android_device_input_smoke() -> Result<()> { .swipe((center_x, end_y), (center_x, start_y), 300) .await?; - let _ = device.adb.stop_app(SETTINGS_PACKAGE); + let _ = device.adb.stop_app(SETTINGS_PACKAGE).await; Ok(()) } diff --git a/packages/accessibility-serve/src/http.rs b/packages/accessibility-serve/src/http.rs index 3e24dc6..99758b3 100644 --- a/packages/accessibility-serve/src/http.rs +++ b/packages/accessibility-serve/src/http.rs @@ -195,7 +195,7 @@ async fn set_orientation( State(state): State, Json(request): Json, ) -> Response { - match state.session.set_orientation(request.orientation) { + match state.session.set_orientation(request.orientation).await { Ok(()) => Json(serde_json::json!({ "orientation": request.orientation })).into_response(), Err(error) => (StatusCode::BAD_REQUEST, error.to_string()).into_response(), } @@ -336,7 +336,7 @@ async fn pump_input(state: AppState, mut socket: WebSocket) { _ => continue, }; - if let Err(error) = state.session.send_input_json(&payload) { + if let Err(error) = state.session.send_input_json(&payload).await { tracing::debug!("ignoring malformed input event: {error}"); } } diff --git a/packages/accessibility-serve/src/lib.rs b/packages/accessibility-serve/src/lib.rs index 7e4cc5e..67e6eb4 100644 --- a/packages/accessibility-serve/src/lib.rs +++ b/packages/accessibility-serve/src/lib.rs @@ -133,6 +133,7 @@ pub async fn serve(_config: ServeConfig) -> Result<()> { pub async fn serve_emulator(config: ServeEmulatorConfig) -> Result<()> { let session = session::EmulatorSession::start(config.serial.as_deref(), config.video) + .await .context("failed to start Android Emulator capture")?; let session = session::Session::android(session); session.seed_orientation().await; diff --git a/packages/accessibility-serve/src/session.rs b/packages/accessibility-serve/src/session.rs index 6bd90e3..5a4940d 100644 --- a/packages/accessibility-serve/src/session.rs +++ b/packages/accessibility-serve/src/session.rs @@ -136,29 +136,35 @@ impl Session { } } - pub fn send_input_json(&self, payload: &str) -> Result<()> { + pub async fn send_input_json(&self, payload: &str) -> Result<()> { match self { #[cfg(target_os = "macos")] Self::Ios(session) => { session.send_input(serde_json::from_str::(payload)?); } Self::Android(session) => { - session.send_input(serde_json::from_str::( - payload, - )?); + session + .send_input(serde_json::from_str::( + payload, + )?) + .await; } } Ok(()) } - pub fn set_orientation(&self, orientation: Orientation) -> Result<()> { + pub async fn set_orientation(&self, orientation: Orientation) -> Result<()> { match self { #[cfg(target_os = "macos")] Self::Ios(session) => { session.set_orientation(to_ios_orientation(orientation)); Ok(()) } - Self::Android(session) => session.set_orientation(to_android_orientation(orientation)), + Self::Android(session) => { + session + .set_orientation(to_android_orientation(orientation)) + .await + } } } From fc94b0714da2b43dc2dd05679d544621a01d912b Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Fri, 21 Aug 2026 01:35:50 +0000 Subject: [PATCH 2/3] Fix ADB command construction --- packages/accessibility-android-sys/src/lib.rs | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/packages/accessibility-android-sys/src/lib.rs b/packages/accessibility-android-sys/src/lib.rs index 445b503..b77bd6d 100644 --- a/packages/accessibility-android-sys/src/lib.rs +++ b/packages/accessibility-android-sys/src/lib.rs @@ -517,33 +517,29 @@ impl AdbClient { cmd } - async fn run(&self, kind: &str, args: &[&str], leading: Option<&str>) -> Result { - let mut cmd = self.base_command(); - if let Some(leading) = leading { - cmd.arg(leading); - } - cmd.args(args) - .stdin(Stdio::null()) + /// Build an ADB command without an optional device serial. + fn command_without_serial(&self) -> Command { + Command::new(&self.adb_path) + } + + async fn run(&self, kind: &str, mut cmd: Command) -> Result { + cmd.stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .kill_on_drop(true); let child = cmd.spawn().with_context(|| { - if kind == "devices" { - format!( - "ADB binary not found at '{}'. Install Android SDK Platform Tools.", - self.adb_path - ) - } else { - format!("Failed to execute adb {kind} command") - } + format!( + "ADB binary not found at '{}'. Install Android SDK Platform Tools.", + self.adb_path + ) })?; tokio::time::timeout(self.timeout, child.wait_with_output()) .await .map_err(|_| { anyhow::anyhow!( - "ADB binary '{}' {kind} command timed out after {:?}", - self.adb_path, - self.timeout + "adb {kind} command timed out after {}s (adb path: '{}')", + self.timeout.as_secs_f64(), + self.adb_path ) })? .with_context(|| format!("Failed to execute adb {kind} command")) @@ -551,28 +547,36 @@ impl AdbClient { /// Execute an ADB shell command. pub async fn shell(&self, args: &[&str]) -> Result { - let output = self.run("shell", args, Some("shell")).await?; + let mut cmd = self.base_command(); + cmd.arg("shell").args(args); + let output = self.run("shell", cmd).await?; Self::check_output(&output, "shell")?; Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } /// Execute an ADB shell command and return raw bytes. pub async fn shell_raw(&self, args: &[&str]) -> Result> { - let output = self.run("shell", args, Some("shell")).await?; + let mut cmd = self.base_command(); + cmd.arg("shell").args(args); + let output = self.run("shell", cmd).await?; Self::check_output(&output, "shell")?; Ok(output.stdout) } /// Execute `adb exec-out` for efficient binary output. pub async fn exec_out(&self, args: &[&str]) -> Result> { - let output = self.run("exec-out", args, Some("exec-out")).await?; + let mut cmd = self.base_command(); + cmd.arg("exec-out").args(args); + let output = self.run("exec-out", cmd).await?; Self::check_output(&output, "exec-out")?; Ok(output.stdout) } /// Execute a general ADB command (not shell). pub async fn command(&self, args: &[&str]) -> Result { - let output = self.run("adb", args, None).await?; + let mut cmd = self.base_command(); + cmd.args(args); + let output = self.run("adb", cmd).await?; Self::check_output(&output, "adb")?; Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } @@ -611,7 +615,9 @@ impl AdbClient { } pub async fn connected_devices(&self) -> Result> { - let output = self.run("devices", &[], Some("devices")).await?; + let mut cmd = self.command_without_serial(); + cmd.arg("devices"); + let output = self.run("devices", cmd).await?; Self::check_output(&output, "devices")?; let stdout = String::from_utf8_lossy(&output.stdout); Ok(stdout From 5595190edbce3736cd334174fad292fb6c69890d Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Fri, 21 Aug 2026 01:57:15 +0000 Subject: [PATCH 3/3] Allow large generated tonic errors --- packages/accessibility-android-sys/src/emulator.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/accessibility-android-sys/src/emulator.rs b/packages/accessibility-android-sys/src/emulator.rs index c009a54..2899967 100644 --- a/packages/accessibility-android-sys/src/emulator.rs +++ b/packages/accessibility-android-sys/src/emulator.rs @@ -11,6 +11,8 @@ pub mod raw; pub mod screenrecord; pub mod protocol { + #![allow(clippy::result_large_err)] + pub mod controller { tonic::include_proto!("android.emulation.control"); }