diff --git a/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs b/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs index 89ee8c9..a7cad61 100644 --- a/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs +++ b/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs @@ -17,7 +17,7 @@ const PROBE_DURATION: Duration = Duration::from_secs(5); #[tokio::main] async fn main() -> Result<()> { let selector = std::env::args().nth(1); - let discovery = discover_emulator(selector.as_deref())?; + let discovery = discover_emulator(selector.as_deref()).await?; let serial = discovery .properties .get("port.serial") @@ -32,18 +32,20 @@ async fn main() -> Result<()> { let stop = Arc::new(AtomicBool::new(false)); let stimulus_stop = Arc::clone(&stop); let stimulus_serial = serial.clone(); - let stimulus = std::thread::spawn(move || { + let stimulus = tokio::spawn(async move { let adb = AdbClient::discover(Some(&stimulus_serial)); - let Ok((width, height)) = adb.get_screen_size() else { + let Ok((width, height)) = adb.get_screen_size().await else { return; }; while !stimulus_stop.load(Ordering::Relaxed) { - let _ = adb.swipe( - (width as f64 * 0.5, height as f64 * 0.75), - (width as f64 * 0.5, height as f64 * 0.25), - 250, - ); - std::thread::sleep(Duration::from_millis(150)); + let _ = adb + .swipe( + (width as f64 * 0.5, height as f64 * 0.75), + (width as f64 * 0.5, height as f64 * 0.25), + 250, + ) + .await; + tokio::time::sleep(Duration::from_millis(150)).await; } }); @@ -52,7 +54,7 @@ async fn main() -> Result<()> { let mmap = probe_mmap(client).await?; print_report("mmap", &mmap); stop.store(true, Ordering::Relaxed); - let _ = stimulus.join(); + let _ = stimulus.await; println!("probe passed"); Ok(()) } diff --git a/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs b/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs index 70d1a81..57b242e 100644 --- a/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs +++ b/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs @@ -31,7 +31,7 @@ const RECOVERY_TIMEOUT: Duration = Duration::from_secs(5); #[tokio::main] async fn main() -> Result<()> { let selector = std::env::args().nth(1); - let discovery = discover_emulator(selector.as_deref())?; + let discovery = discover_emulator(selector.as_deref()).await?; println!("discovery : {}", discovery.path.display()); println!("endpoint : {}", discovery.endpoint()); diff --git a/packages/accessibility-android-sys/examples/screenrecord_probe.rs b/packages/accessibility-android-sys/examples/screenrecord_probe.rs index 3df6839..4d861d9 100644 --- a/packages/accessibility-android-sys/examples/screenrecord_probe.rs +++ b/packages/accessibility-android-sys/examples/screenrecord_probe.rs @@ -11,26 +11,29 @@ use anyhow::{Context, Result, bail}; const IDLE_FLUSH: Duration = Duration::from_millis(75); const PROBE_DURATION: Duration = Duration::from_secs(5); -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { let serial = std::env::args() .nth(1) .unwrap_or_else(|| "emulator-5554".to_string()); let adb = AdbClient::discover(Some(&serial)); - adb.check_connection()?; - let (width, height) = adb.get_screen_size()?; + adb.check_connection().await?; + let (width, height) = adb.get_screen_size().await?; let config = ScreenRecordConfig::for_max_dimension(width, height, Some(1280), 5_000_000); println!("device : {serial}"); println!("source : {width}x{height}"); println!("encoded : {}x{}", config.width, config.height); let stimulus = adb.clone(); - std::thread::spawn(move || { - std::thread::sleep(Duration::from_secs(1)); - let _ = stimulus.swipe( - (width as f64 * 0.5, height as f64 * 0.75), - (width as f64 * 0.5, height as f64 * 0.25), - 800, - ); + let stimulus = tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(1)).await; + let _ = stimulus + .swipe( + (width as f64 * 0.5, height as f64 * 0.75), + (width as f64 * 0.5, height as f64 * 0.25), + 800, + ) + .await; }); let started = Instant::now(); @@ -67,6 +70,7 @@ fn main() -> Result<()> { restart.elapsed().as_secs_f64() * 1000.0 ); println!("entry NALs: {:?}", nal_types(&first.data)); + let _ = stimulus.await; println!("probe passed"); Ok(()) } diff --git a/packages/accessibility-android-sys/src/emulator.rs b/packages/accessibility-android-sys/src/emulator.rs index c6a4050..2899967 100644 --- a/packages/accessibility-android-sys/src/emulator.rs +++ b/packages/accessibility-android-sys/src/emulator.rs @@ -11,6 +11,8 @@ pub mod raw; pub mod screenrecord; pub mod protocol { + #![allow(clippy::result_large_err)] + pub mod controller { tonic::include_proto!("android.emulation.control"); } @@ -38,9 +40,9 @@ pub struct EmulatorDiscovery { } impl EmulatorDiscovery { - pub fn from_file(path: impl AsRef) -> Result { + pub async fn from_file(path: impl AsRef) -> Result { let path = path.as_ref(); - let contents = std::fs::read_to_string(path).with_context(|| { + let contents = tokio::fs::read_to_string(path).await.with_context(|| { format!("failed to read emulator discovery file {}", path.display()) })?; let properties = parse_properties(&contents); @@ -246,17 +248,17 @@ impl EmulatorGrpcClient { } } -pub fn discover_emulator(selector: Option<&str>) -> Result { - discover_emulator_in(&discovery_directories(), selector) +pub async fn discover_emulator(selector: Option<&str>) -> Result { + discover_emulator_in(&discovery_directories(), selector).await } -pub fn discover_emulator_in( +pub async fn discover_emulator_in( directories: &[PathBuf], selector: Option<&str>, ) -> Result { let mut paths = BTreeSet::new(); for directory in directories { - let entries = match std::fs::read_dir(directory) { + let mut entries = match tokio::fs::read_dir(directory).await { Ok(entries) => entries, Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, Err(error) => { @@ -268,8 +270,8 @@ pub fn discover_emulator_in( }); } }; - for entry in entries { - let path = entry?.path(); + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); let Some(name) = path.file_name().and_then(|name| name.to_str()) else { continue; }; @@ -279,10 +281,12 @@ pub fn discover_emulator_in( } } - let mut discoveries = paths - .into_iter() - .filter_map(|path| EmulatorDiscovery::from_file(path).ok()) - .collect::>(); + let mut discoveries = Vec::new(); + for path in paths { + if let Ok(discovery) = EmulatorDiscovery::from_file(path).await { + discoveries.push(discovery); + } + } if let Some(selector) = selector { discoveries.retain(|discovery| discovery.matches(selector)); } @@ -351,9 +355,9 @@ mod tests { path } - #[test] + #[tokio::test] #[cfg_attr(miri, ignore)] - fn parses_discovery_file() { + async fn parses_discovery_file() { let directory = test_directory("parse"); let path = directory.join("pid_1234.ini"); std::fs::write( @@ -361,7 +365,7 @@ mod tests { "grpc.port = 8554\ngrpc.token = secret\navd.name = Pixel_8\nport.serial = 5554\n", ) .unwrap(); - let discovery = EmulatorDiscovery::from_file(&path).unwrap(); + let discovery = EmulatorDiscovery::from_file(&path).await.unwrap(); assert_eq!(discovery.pid, Some(1234)); assert_eq!(discovery.grpc_port, 8554); assert_eq!(discovery.grpc_token.as_deref(), Some("secret")); @@ -370,9 +374,9 @@ mod tests { std::fs::remove_dir_all(directory).unwrap(); } - #[test] + #[tokio::test] #[cfg_attr(miri, ignore)] - fn selects_one_emulator() { + async fn selects_one_emulator() { let directory = test_directory("select"); std::fs::write( directory.join("pid_1.ini"), @@ -384,10 +388,15 @@ mod tests { "grpc.port=8555\navd.name=tablet\n", ) .unwrap(); - let discovery = - discover_emulator_in(std::slice::from_ref(&directory), Some("tablet")).unwrap(); + let discovery = discover_emulator_in(std::slice::from_ref(&directory), Some("tablet")) + .await + .unwrap(); assert_eq!(discovery.grpc_port, 8555); - assert!(discover_emulator_in(std::slice::from_ref(&directory), None).is_err()); + assert!( + discover_emulator_in(std::slice::from_ref(&directory), None) + .await + .is_err() + ); std::fs::remove_dir_all(directory).unwrap(); } } diff --git a/packages/accessibility-android-sys/src/emulator/raw.rs b/packages/accessibility-android-sys/src/emulator/raw.rs index 3b64c5a..1fd8013 100644 --- a/packages/accessibility-android-sys/src/emulator/raw.rs +++ b/packages/accessibility-android-sys/src/emulator/raw.rs @@ -48,7 +48,7 @@ impl RawFrameStream { if config.width == 0 || config.height == 0 { bail!("raw Android capture requires non-zero dimensions"); } - let discovery = discover_emulator(selector)?; + let discovery = discover_emulator(selector).await?; Self::start_with_discovery(discovery, config).await } diff --git a/packages/accessibility-android-sys/src/lib.rs b/packages/accessibility-android-sys/src/lib.rs index 1e2f9e3..45fa586 100644 --- a/packages/accessibility-android-sys/src/lib.rs +++ b/packages/accessibility-android-sys/src/lib.rs @@ -1,8 +1,9 @@ //! Low-level ADB wrappers used by accessibility-cli's Android backend. pub mod emulator; +mod transport; -use std::process::{Command, Output}; +use std::net::SocketAddr; use std::time::Duration; use anyhow::{Context, Result, bail}; @@ -10,6 +11,7 @@ use keyboard_types::Code; const UI_DUMP_ATTEMPTS: usize = 3; const UI_DUMP_RETRY_DELAY: Duration = Duration::from_millis(500); +pub const DEFAULT_ADB_TIMEOUT: Duration = Duration::from_secs(30); /// Android key codes for `input keyevent` command. /// @@ -435,14 +437,14 @@ pub struct AdbClient { pub serial: Option, /// Path to the ADB binary. pub adb_path: String, + /// Maximum time to wait for an ADB command. + pub timeout: Duration, + server_addr: SocketAddr, } impl Default for AdbClient { fn default() -> Self { - Self { - serial: None, - adb_path: "adb".to_string(), - } + Self::new(None) } } @@ -452,6 +454,8 @@ impl AdbClient { Self { serial: serial.map(String::from), adb_path: "adb".to_string(), + timeout: DEFAULT_ADB_TIMEOUT, + server_addr: server_addr_from_environment(), } } @@ -492,86 +496,111 @@ impl AdbClient { Self { serial: serial.map(String::from), adb_path: adb_path.to_string(), + timeout: DEFAULT_ADB_TIMEOUT, + server_addr: server_addr_from_environment(), } } - /// Build base ADB command with optional device serial. - fn base_command(&self) -> Command { - let mut cmd = Command::new(&self.adb_path); - if let Some(ref serial) = self.serial { - cmd.arg("-s").arg(serial); - } - cmd + /// Set the maximum time to wait for an ADB command. + pub fn with_timeout(mut self, timeout: Duration) -> Self { + self.timeout = timeout; + self } - /// Execute an ADB shell command. - pub fn shell(&self, args: &[&str]) -> Result { - let mut cmd = self.base_command(); - cmd.arg("shell").args(args); + /// Set the ADB server address. + pub fn with_server_addr(mut self, server_addr: SocketAddr) -> Self { + self.server_addr = server_addr; + self + } - let output = cmd - .output() - .context("Failed to execute adb shell command")?; + async fn run( + &self, + kind: &str, + operation: impl std::future::Future>, + ) -> Result { + tokio::time::timeout(self.timeout, operation) + .await + .map_err(|_| { + anyhow::anyhow!( + "adb {kind} request timed out after {}s (adb server: {})", + self.timeout.as_secs_f64(), + self.server_addr + ) + })? + } - Self::check_output(&output, "shell")?; + /// Create the socket transport for this client. + fn transport(&self) -> transport::AdbTransport { + transport::AdbTransport::new(self.server_addr, &self.adb_path) + } + + /// Execute an ADB shell command. + /// + /// The shell-v2 exit status is propagated: a non-zero device command + /// returns an error, unlike the previous `adb shell` process path. + pub async fn shell(&self, args: &[&str]) -> Result { + let transport = self.transport(); + let output = self + .run("shell", transport.shell(self.serial.as_deref(), args)) + .await?; + Self::check_shell_output(&output)?; Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } /// Execute an ADB shell command and return raw bytes. - pub fn shell_raw(&self, args: &[&str]) -> Result> { - let mut cmd = self.base_command(); - cmd.arg("shell").args(args); - - let output = cmd - .output() - .context("Failed to execute adb shell command")?; - - Self::check_output(&output, "shell")?; + /// + /// The shell-v2 exit status is propagated: a non-zero device command + /// returns an error, unlike the previous `adb shell` process path. + pub async fn shell_raw(&self, args: &[&str]) -> Result> { + let transport = self.transport(); + let output = self + .run("shell", transport.shell(self.serial.as_deref(), args)) + .await?; + Self::check_shell_output(&output)?; Ok(output.stdout) } - /// Execute `adb exec-out` for efficient binary output. - pub fn exec_out(&self, args: &[&str]) -> Result> { - let mut cmd = self.base_command(); - cmd.arg("exec-out").args(args); - - let output = cmd - .output() - .context("Failed to execute adb exec-out command")?; - - Self::check_output(&output, "exec-out")?; - Ok(output.stdout) + /// Execute a device command and return its binary output. + /// + /// The underlying `exec:` service provides no exit status; this method + /// returns bytes until the server closes the stream. + pub async fn exec_out(&self, args: &[&str]) -> Result> { + let transport = self.transport(); + self.run("exec", transport.exec(self.serial.as_deref(), args)) + .await } - /// Execute a general ADB command (not shell). - pub fn command(&self, args: &[&str]) -> Result { - let mut cmd = self.base_command(); - cmd.args(args); - - let output = cmd.output().context("Failed to execute adb command")?; + /// Query the ADB server version. + pub async fn server_version(&self) -> Result { + let transport = self.transport(); + let output = self + .run("host:version", transport.host_query("host:version")) + .await?; + Ok(String::from_utf8_lossy(&output).into_owned()) + } - Self::check_output(&output, "adb")?; - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + /// Query the feature list reported by the ADB server. + pub async fn server_features(&self) -> Result { + let transport = self.transport(); + let output = self + .run("host:features", transport.host_query("host:features")) + .await?; + Ok(String::from_utf8_lossy(&output).into_owned()) } - fn check_output(output: &Output, cmd_type: &str) -> Result<()> { - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - let stdout = String::from_utf8_lossy(&output.stdout); - bail!( - "ADB {} command failed (exit code {}): stdout={}, stderr={}", - cmd_type, - output.status.code().unwrap_or(-1), - stdout.trim(), - stderr.trim() - ); - } - Ok(()) + /// Wait until the selected device is available. + pub async fn wait_for_device(&self) -> Result<()> { + let transport = self.transport(); + self.run( + "wait-for-device", + transport.wait_for_device(self.serial.as_deref()), + ) + .await } /// Check if ADB is available and a device is connected. - pub fn check_connection(&self) -> Result<()> { - let devices = self.connected_devices()?; + pub async fn check_connection(&self) -> Result<()> { + let devices = self.connected_devices().await?; if devices.is_empty() { bail!("No Android devices connected. Connect a device or start an emulator."); } @@ -587,21 +616,14 @@ impl AdbClient { Ok(()) } - pub fn connected_devices(&self) -> Result> { - let output = Command::new(&self.adb_path) - .arg("devices") - .output() - .with_context(|| { - format!( - "ADB binary not found at '{}'. Install Android SDK Platform Tools.", - self.adb_path - ) - })?; - Self::check_output(&output, "devices")?; - let stdout = String::from_utf8_lossy(&output.stdout); + pub async fn connected_devices(&self) -> Result> { + let transport = self.transport(); + let output = self + .run("host:devices", transport.host_query("host:devices")) + .await?; + let stdout = String::from_utf8_lossy(&output); Ok(stdout .lines() - .skip(1) .filter_map(|line| { let (serial, state) = line.split_once('\t')?; (state.split_whitespace().next() == Some("device")).then(|| serial.to_string()) @@ -609,8 +631,20 @@ impl AdbClient { .collect()) } - pub fn resolved_serial(&self) -> Result { - let devices = self.connected_devices()?; + fn check_shell_output(output: &transport::ShellOutput) -> Result<()> { + if output.exit_code != 0 { + bail!( + "ADB shell command failed (exit code {}): stdout={}, stderr={}", + output.exit_code, + String::from_utf8_lossy(&output.stdout).trim(), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(()) + } + + pub async fn resolved_serial(&self) -> Result { + let devices = self.connected_devices().await?; if let Some(serial) = &self.serial { if devices.contains(serial) { return Ok(serial.clone()); @@ -632,8 +666,8 @@ impl AdbClient { } /// Get the screen size in pixels. - pub fn get_screen_size(&self) -> Result<(u32, u32)> { - let output = self.shell(&["wm", "size"])?; + pub async fn get_screen_size(&self) -> Result<(u32, u32)> { + let output = self.shell(&["wm", "size"]).await?; for line in output.lines() { if let Some(size_str) = line.strip_prefix("Physical size:") { let size_str = size_str.trim(); @@ -653,23 +687,24 @@ impl AdbClient { } /// Capture a screenshot as PNG bytes. - pub fn screenshot(&self) -> Result> { - self.exec_out(&["screencap", "-p"]) + pub async fn screenshot(&self) -> Result> { + self.exec_out(&["screencap", "-p"]).await } /// Tap at screen coordinates. - pub fn tap(&self, x: f64, y: f64) -> Result<()> { + pub async fn tap(&self, x: f64, y: f64) -> Result<()> { self.shell(&[ "input", "tap", &x.round().to_string(), &y.round().to_string(), - ])?; + ]) + .await?; Ok(()) } /// Swipe from one point to another. - pub fn swipe(&self, start: (f64, f64), end: (f64, f64), duration_ms: u64) -> Result<()> { + pub async fn swipe(&self, start: (f64, f64), end: (f64, f64), duration_ms: u64) -> Result<()> { self.shell(&[ "input", "swipe", @@ -678,34 +713,36 @@ impl AdbClient { &end.0.round().to_string(), &end.1.round().to_string(), &duration_ms.to_string(), - ])?; + ]) + .await?; Ok(()) } /// Send a key event. - pub fn key_event(&self, keycode: u32) -> Result<()> { - self.shell(&["input", "keyevent", &keycode.to_string()])?; + pub async fn key_event(&self, keycode: u32) -> Result<()> { + self.shell(&["input", "keyevent", &keycode.to_string()]) + .await?; Ok(()) } /// Send text input. - pub fn input_text(&self, text: &str) -> Result<()> { + pub async fn input_text(&self, text: &str) -> Result<()> { let escaped = escape_shell_text(text); - self.shell(&["input", "text", &escaped])?; + self.shell(&["input", "text", &escaped]).await?; Ok(()) } /// Dump the UI hierarchy as XML. - pub fn dump_ui(&self) -> Result { + pub async fn dump_ui(&self) -> Result { let mut last_error = None; for attempt in 1..=UI_DUMP_ATTEMPTS { - match self.dump_ui_once() { + match self.dump_ui_once().await { Ok(xml) => return Ok(xml), Err(error) => { last_error = Some(error); if attempt < UI_DUMP_ATTEMPTS { - std::thread::sleep(UI_DUMP_RETRY_DELAY); + tokio::time::sleep(UI_DUMP_RETRY_DELAY).await; } } } @@ -716,13 +753,13 @@ impl AdbClient { )) } - fn dump_ui_once(&self) -> Result { - let result = self.shell(&["uiautomator", "dump", "/dev/tty"]); + async fn dump_ui_once(&self) -> Result { + let result = self.shell(&["uiautomator", "dump", "/dev/tty"]).await; match result { Ok(output) => match extract_ui_xml(&output) { Some(xml) => Ok(xml), - None => self.dump_ui_via_file().with_context(|| { + None => self.dump_ui_via_file().await.with_context(|| { format!( "direct uiautomator dump did not contain XML: {}", truncate_for_error(&output) @@ -731,27 +768,28 @@ impl AdbClient { }, Err(error) => self .dump_ui_via_file() + .await .with_context(|| format!("direct uiautomator dump failed: {error}")), } } - fn dump_ui_via_file(&self) -> Result { + async fn dump_ui_via_file(&self) -> Result { let tmp_path = "/data/local/tmp/window_dump.xml"; - let _ = self.shell(&["rm", "-f", tmp_path]); - let dump_output = self.shell(&["uiautomator", "dump", tmp_path])?; + let _ = self.shell(&["rm", "-f", tmp_path]).await; + let dump_output = self.shell(&["uiautomator", "dump", tmp_path]).await?; if let Some(xml) = extract_ui_xml(&dump_output) { - let _ = self.shell(&["rm", "-f", tmp_path]); + let _ = self.shell(&["rm", "-f", tmp_path]).await; return Ok(xml); } - let xml = self.shell(&["cat", tmp_path]).with_context(|| { + let xml = self.shell(&["cat", tmp_path]).await.with_context(|| { format!( "uiautomator dump did not create readable file at {tmp_path}; dump output: {}", truncate_for_error(&dump_output) ) })?; - let _ = self.shell(&["rm", "-f", tmp_path]); + let _ = self.shell(&["rm", "-f", tmp_path]).await; if let Some(xml) = extract_ui_xml(&xml) { Ok(xml) @@ -761,11 +799,11 @@ impl AdbClient { } /// Launch an app by package name and optional activity. - pub fn launch_app(&self, package: &str, activity: Option<&str>) -> Result<()> { + pub async fn launch_app(&self, package: &str, activity: Option<&str>) -> Result<()> { match activity { Some(act) => { let component = format!("{}/{}", package, act); - self.shell(&["am", "start", "-n", &component])?; + self.shell(&["am", "start", "-n", &component]).await?; } None => { self.shell(&[ @@ -775,21 +813,22 @@ impl AdbClient { "-c", "android.intent.category.LAUNCHER", "1", - ])?; + ]) + .await?; } } Ok(()) } /// Force stop an app. - pub fn stop_app(&self, package: &str) -> Result<()> { - self.shell(&["am", "force-stop", package])?; + pub async fn stop_app(&self, package: &str) -> Result<()> { + self.shell(&["am", "force-stop", package]).await?; Ok(()) } /// Get the current foreground activity. - pub fn get_current_activity(&self) -> Result { - let output = self.shell(&["dumpsys", "activity", "activities"])?; + pub async fn get_current_activity(&self) -> Result { + let output = self.shell(&["dumpsys", "activity", "activities"]).await?; for line in output.lines() { let trimmed = line.trim(); @@ -799,7 +838,7 @@ impl AdbClient { } } - let output = self.shell(&["dumpsys", "window", "windows"])?; + let output = self.shell(&["dumpsys", "window", "windows"]).await?; for line in output.lines() { if line.contains("mCurrentFocus") || line.contains("mFocusedApp") { return Ok(line.trim().to_string()); @@ -818,6 +857,14 @@ fn adb_binary_name() -> &'static str { } } +fn server_addr_from_environment() -> SocketAddr { + let port = std::env::var("ANDROID_ADB_SERVER_PORT") + .ok() + .and_then(|port| port.parse().ok()) + .unwrap_or(transport::DEFAULT_SERVER_ADDR.port()); + SocketAddr::new(transport::DEFAULT_SERVER_ADDR.ip(), port) +} + /// Escape text for ADB shell input command. pub fn escape_shell_text(text: &str) -> String { let mut result = String::with_capacity(text.len() * 2); @@ -921,4 +968,16 @@ mod tests { Some(AndroidKeyCode::F1) ); } + + #[cfg(unix)] + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn missing_adb_binary_has_install_context() { + let adb = AdbClient::with_adb_path(None, "/no/such/adb") + .with_server_addr(SocketAddr::from(([127, 0, 0, 1], 1))); + let error = adb.connected_devices().await.unwrap_err(); + assert!(error.to_string().contains( + "ADB binary not found at '/no/such/adb'. Install Android SDK Platform Tools." + )); + } } diff --git a/packages/accessibility-android-sys/src/transport.rs b/packages/accessibility-android-sys/src/transport.rs new file mode 100644 index 0000000..855b568 --- /dev/null +++ b/packages/accessibility-android-sys/src/transport.rs @@ -0,0 +1,569 @@ +//! A narrow implementation of the ADB server smartsocket protocol. +//! +//! The workspace deliberately hand-rolls this small protocol slice instead of +//! adding `droidrun-adb` or `adbutils-rs`: the client needs only a stable set of +//! host, shell-v2, and exec services without expanding the dependency surface. + +use std::net::SocketAddr; + +use anyhow::{Context, Result, bail}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; +use tokio::process::Command; + +#[derive(Clone)] +pub(crate) struct AdbTransport { + server_addr: SocketAddr, + adb_path: String, +} + +impl AdbTransport { + pub(crate) fn new(server_addr: SocketAddr, adb_path: &str) -> Self { + Self { + server_addr, + adb_path: adb_path.to_string(), + } + } + + pub(crate) async fn host_query(&self, service: &str) -> Result> { + let mut stream = self.connect().await?; + write_service(&mut stream, service).await?; + read_status(&mut stream).await?; + read_length_prefixed(&mut stream, MAX_OUTPUT_LENGTH, "host response").await + } + + pub(crate) async fn wait_for_device(&self, serial: Option<&str>) -> Result<()> { + let service = match serial { + Some(serial) => format!("host-serial:{serial}:wait-for-any-device"), + None => "host:wait-for-any-device".to_string(), + }; + let mut stream = self.connect().await?; + write_service(&mut stream, &service).await?; + read_status(&mut stream).await + } + + pub(crate) async fn shell(&self, serial: Option<&str>, args: &[&str]) -> Result { + let service = format!("shell,v2,raw:{}", args.join(" ")); + let mut stream = self.switch_to_device(serial).await?; + write_service(&mut stream, &service).await?; + read_status(&mut stream) + .await + .context("shell-v2-capable device/adb is required")?; + read_shell_output(&mut stream).await + } + + pub(crate) async fn exec(&self, serial: Option<&str>, args: &[&str]) -> Result> { + let service = format!("exec:{}", args.join(" ")); + let mut stream = self.switch_to_device(serial).await?; + write_service(&mut stream, &service).await?; + read_status(&mut stream).await?; + read_to_end_limited(&mut stream, MAX_OUTPUT_LENGTH, "exec output").await + } + + async fn connect(&self) -> Result { + match TcpStream::connect(self.server_addr).await { + Ok(stream) => Ok(stream), + Err(error) if is_bootstrap_error(&error) => { + self.start_server().await?; + TcpStream::connect(self.server_addr).await.with_context(|| { + format!("failed to connect to ADB server at {}", self.server_addr) + }) + } + Err(error) => Err(error).with_context(|| { + format!("failed to connect to ADB server at {}", self.server_addr) + }), + } + } + + async fn start_server(&self) -> Result<()> { + let child = Command::new(&self.adb_path) + .arg("start-server") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true) + .spawn() + .with_context(|| { + format!( + "no ADB server at {} and ADB binary not found at '{}'. Install Android SDK Platform Tools.", + self.server_addr, self.adb_path + ) + })?; + let output = child + .wait_with_output() + .await + .context("failed to wait for adb start-server")?; + if !output.status.success() { + bail!( + "adb start-server failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(()) + } + + async fn switch_to_device(&self, serial: Option<&str>) -> Result { + let service = match serial { + Some(serial) => format!("host:tport:serial:{serial}"), + None => "host:tport:any".to_string(), + }; + let mut stream = self.connect().await?; + write_service(&mut stream, &service).await?; + read_status(&mut stream).await?; + let mut transport_id = [0; 8]; + stream + .read_exact(&mut transport_id) + .await + .context("truncated ADB transport id")?; + Ok(stream) + } +} + +#[derive(Debug, Default)] +pub(crate) struct ShellOutput { + pub(crate) stdout: Vec, + pub(crate) stderr: Vec, + pub(crate) exit_code: u8, +} + +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ShellPacketId { + Stdin = 0, + Stdout = 1, + Stderr = 2, + Exit = 3, + CloseStdin = 4, + WindowSizeChange = 5, + Invalid = 255, +} + +impl ShellPacketId { + fn decode(value: u8) -> Result { + match value { + 0 => Ok(Self::Stdin), + 1 => Ok(Self::Stdout), + 2 => Ok(Self::Stderr), + 3 => Ok(Self::Exit), + 4 => Ok(Self::CloseStdin), + 5 => Ok(Self::WindowSizeChange), + 255 => Ok(Self::Invalid), + value => bail!("unknown shell-v2 packet id {value}"), + } + } +} + +async fn write_service(stream: &mut TcpStream, service: &str) -> Result<()> { + let length = service.len(); + if length > MAX_SERVICE_LENGTH { + bail!("ADB service string exceeds {MAX_SERVICE_LENGTH} bytes"); + } + let header = format!("{length:04x}"); + stream.write_all(header.as_bytes()).await?; + stream.write_all(service.as_bytes()).await?; + Ok(()) +} + +async fn read_status(stream: &mut TcpStream) -> Result<()> { + let mut status = [0; 4]; + stream + .read_exact(&mut status) + .await + .context("truncated ADB response status")?; + match &status { + b"OKAY" => Ok(()), + b"FAIL" => { + let message = read_length_prefixed(stream, MAX_PACKET_LENGTH, "ADB failure").await?; + bail!("ADB server failure: {}", String::from_utf8_lossy(&message)); + } + _ => bail!("malformed ADB response status"), + } +} + +async fn read_length_prefixed( + reader: &mut R, + max_length: usize, + kind: &str, +) -> Result> { + let length = read_hex_length(reader, kind).await?; + if length > max_length { + bail!("{kind} length {length} exceeds maximum {max_length}"); + } + let mut payload = vec![0; length]; + reader + .read_exact(&mut payload) + .await + .with_context(|| format!("truncated {kind} payload"))?; + Ok(payload) +} + +async fn read_shell_output(stream: &mut TcpStream) -> Result { + let mut output = ShellOutput::default(); + loop { + let mut header = [0; 5]; + stream + .read_exact(&mut header) + .await + .context("truncated shell-v2 packet header")?; + let packet_id = ShellPacketId::decode(header[0])?; + let length = u32::from_le_bytes(header[1..].try_into().unwrap()) as usize; + if length > MAX_PACKET_LENGTH { + bail!("shell-v2 packet length {length} exceeds maximum {MAX_PACKET_LENGTH}"); + } + match packet_id { + ShellPacketId::Stdout => { + if length + > MAX_OUTPUT_LENGTH.saturating_sub(output.stdout.len() + output.stderr.len()) + { + bail!("shell-v2 output exceeds maximum {MAX_OUTPUT_LENGTH} bytes"); + } + let mut payload = vec![0; length]; + stream.read_exact(&mut payload).await?; + output.stdout.extend(payload); + } + ShellPacketId::Stderr => { + if length + > MAX_OUTPUT_LENGTH.saturating_sub(output.stdout.len() + output.stderr.len()) + { + bail!("shell-v2 output exceeds maximum {MAX_OUTPUT_LENGTH} bytes"); + } + let mut payload = vec![0; length]; + stream.read_exact(&mut payload).await?; + output.stderr.extend(payload); + } + ShellPacketId::Exit => { + if length != 1 { + bail!("shell-v2 exit packet must contain one byte"); + } + let mut exit_code = [0]; + stream.read_exact(&mut exit_code).await?; + output.exit_code = exit_code[0]; + let mut trailing = [0]; + if stream.read(&mut trailing).await? != 0 { + bail!("shell-v2 stream has data after the exit packet"); + } + return Ok(output); + } + ShellPacketId::Stdin + | ShellPacketId::CloseStdin + | ShellPacketId::WindowSizeChange + | ShellPacketId::Invalid => bail!("unexpected shell-v2 packet id {}", packet_id as u8), + } + } +} + +async fn read_to_end_limited( + reader: &mut R, + max_length: usize, + kind: &str, +) -> Result> { + let mut output = Vec::new(); + let mut buffer = [0; IO_BUFFER_LENGTH]; + loop { + let read = reader.read(&mut buffer).await?; + if read == 0 { + return Ok(output); + } + if read > max_length.saturating_sub(output.len()) { + bail!("{kind} exceeds maximum {max_length} bytes"); + } + output.extend_from_slice(&buffer[..read]); + } +} + +async fn read_hex_length(reader: &mut R, kind: &str) -> Result { + let mut header = [0; 4]; + reader + .read_exact(&mut header) + .await + .with_context(|| format!("truncated {kind} length"))?; + let text = std::str::from_utf8(&header).with_context(|| format!("invalid {kind} length"))?; + usize::from_str_radix(text, 16).with_context(|| format!("invalid {kind} length")) +} + +fn is_bootstrap_error(error: &std::io::Error) -> bool { + error.kind() == std::io::ErrorKind::ConnectionRefused +} + +/// The default local ADB server endpoint. +pub(crate) const DEFAULT_SERVER_ADDR: SocketAddr = + SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 5037); + +const MAX_SERVICE_LENGTH: usize = 1024; + +/// The maximum payload accepted in one shell-v2 packet. +const MAX_PACKET_LENGTH: usize = 1024 * 1024; + +/// The maximum output accumulated from one shell or exec request. +const MAX_OUTPUT_LENGTH: usize = 64 * 1024 * 1024; + +const IO_BUFFER_LENGTH: usize = 8192; + +#[cfg(test)] +mod tests { + use super::*; + use crate::AdbClient; + use std::time::Duration; + use tokio::net::{TcpListener, TcpStream}; + + async fn bind_listener() -> (TcpListener, SocketAddr) { + let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let address = listener.local_addr().unwrap(); + (listener, address) + } + + async fn read_service(stream: &mut TcpStream) -> String { + let mut header = [0; 4]; + stream.read_exact(&mut header).await.unwrap(); + let length = usize::from_str_radix(std::str::from_utf8(&header).unwrap(), 16).unwrap(); + let mut service = vec![0; length]; + stream.read_exact(&mut service).await.unwrap(); + String::from_utf8(service).unwrap() + } + + async fn write_host_response(stream: &mut TcpStream, payload: &[u8]) { + stream.write_all(b"OKAY").await.unwrap(); + stream + .write_all(format!("{:04x}", payload.len()).as_bytes()) + .await + .unwrap(); + stream.write_all(payload).await.unwrap(); + } + + async fn write_shell_packet(stream: &mut TcpStream, id: u8, payload: &[u8]) { + stream.write_all(&[id]).await.unwrap(); + stream + .write_all(&(payload.len() as u32).to_le_bytes()) + .await + .unwrap(); + stream.write_all(payload).await.unwrap(); + } + + async fn switch_to_shell( + stream: &mut TcpStream, + expected_switch: &str, + expected_command: &str, + ) { + assert_eq!(read_service(stream).await, expected_switch); + stream.write_all(b"OKAYtid-1234").await.unwrap(); + assert_eq!( + read_service(stream).await, + format!("shell,v2,raw:{expected_command}") + ); + stream.write_all(b"OKAY").await.unwrap(); + } + + // Miri's default isolation does not support socket syscalls; these tests + // still run under the normal test harness. + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn host_devices_filters_non_devices() { + let (listener, address) = bind_listener().await; + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + assert_eq!(read_service(&mut stream).await, "host:devices"); + write_host_response( + &mut stream, + b"phone\tdevice\noffline\toffline\nunauthorized\tunauthorized\n", + ) + .await; + }); + let adb = AdbClient::new(Some("ignored")).with_server_addr(address); + assert_eq!(adb.connected_devices().await.unwrap(), ["phone"]); + server.await.unwrap(); + } + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn host_devices_parses_single_device_payload() { + let (listener, address) = bind_listener().await; + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + assert_eq!(read_service(&mut stream).await, "host:devices"); + write_host_response(&mut stream, b"emulator-5554\tdevice\n").await; + }); + let adb = AdbClient::new(None).with_server_addr(address); + assert_eq!(adb.connected_devices().await.unwrap(), ["emulator-5554"]); + server.await.unwrap(); + } + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn transport_switch_reads_tid_and_shell_request() { + let (listener, address) = bind_listener().await; + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + switch_to_shell(&mut stream, "host:tport:serial:emulator-5554", "echo ok").await; + write_shell_packet(&mut stream, ShellPacketId::Exit as u8, &[0]).await; + }); + let adb = AdbClient::new(Some("emulator-5554")).with_server_addr(address); + assert_eq!(adb.shell(&["echo", "ok"]).await.unwrap(), ""); + server.await.unwrap(); + } + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn shell_collects_split_stdout_stderr_and_exit() { + let (listener, address) = bind_listener().await; + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + switch_to_shell(&mut stream, "host:tport:any", "echo ok").await; + write_shell_packet(&mut stream, 1, b"hello ").await; + write_shell_packet(&mut stream, 2, b"warning").await; + write_shell_packet(&mut stream, 1, b"world").await; + write_shell_packet(&mut stream, 3, &[0]).await; + }); + let adb = AdbClient::new(None).with_server_addr(address); + let output = adb.shell_raw(&["echo", "ok"]).await.unwrap(); + assert_eq!(output, b"hello world"); + server.await.unwrap(); + } + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn nonzero_shell_exit_includes_stderr() { + let (listener, address) = bind_listener().await; + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + switch_to_shell(&mut stream, "host:tport:any", "false").await; + write_shell_packet(&mut stream, 2, b"bad command").await; + write_shell_packet(&mut stream, 3, &[7]).await; + }); + let adb = AdbClient::new(None).with_server_addr(address); + let error = adb.shell(&["false"]).await.unwrap_err(); + assert!(error.to_string().contains("exit code 7")); + assert!(error.to_string().contains("bad command")); + server.await.unwrap(); + } + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn fail_response_surfaces_server_message() { + let (listener, address) = bind_listener().await; + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + assert_eq!(read_service(&mut stream).await, "host:version"); + stream.write_all(b"FAIL000eserver says no").await.unwrap(); + }); + let adb = AdbClient::new(None).with_server_addr(address); + let error = adb.server_version().await.unwrap_err(); + assert!(error.to_string().contains("server says no")); + server.await.unwrap(); + } + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn truncated_status_and_payload_error() { + let (listener, address) = bind_listener().await; + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + assert_eq!(read_service(&mut stream).await, "host:version"); + stream.write_all(b"OK").await.unwrap(); + }); + let adb = AdbClient::new(None).with_server_addr(address); + assert!(adb.server_version().await.is_err()); + server.await.unwrap(); + + let (listener, address) = bind_listener().await; + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + assert_eq!(read_service(&mut stream).await, "host:devices"); + stream.write_all(b"OKAY0004ab").await.unwrap(); + }); + let adb = AdbClient::new(None).with_server_addr(address); + assert!(adb.connected_devices().await.is_err()); + server.await.unwrap(); + } + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn shell_eof_before_exit_is_an_error() { + let (listener, address) = bind_listener().await; + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + switch_to_shell(&mut stream, "host:tport:any", "echo").await; + write_shell_packet(&mut stream, 1, b"partial output").await; + }); + let adb = AdbClient::new(None).with_server_addr(address); + assert!(adb.shell_raw(&["echo"]).await.is_err()); + server.await.unwrap(); + } + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn oversized_and_unknown_shell_packets_are_rejected() { + let (listener, address) = bind_listener().await; + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + switch_to_shell(&mut stream, "host:tport:any", "echo").await; + stream.write_all(&[1]).await.unwrap(); + stream + .write_all(&((MAX_PACKET_LENGTH as u32) + 1).to_le_bytes()) + .await + .unwrap(); + }); + let adb = AdbClient::new(None).with_server_addr(address); + assert!(adb.shell_raw(&["echo"]).await.is_err()); + server.await.unwrap(); + + let (listener, address) = bind_listener().await; + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + switch_to_shell(&mut stream, "host:tport:any", "echo").await; + stream.write_all(&[99, 0, 0, 0, 0]).await.unwrap(); + }); + let adb = AdbClient::new(None).with_server_addr(address); + assert!(adb.shell_raw(&["echo"]).await.is_err()); + server.await.unwrap(); + } + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn exec_preserves_binary_output() { + let (listener, address) = bind_listener().await; + let expected = b"\0png\r\n\xff"; + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + assert_eq!(read_service(&mut stream).await, "host:tport:any"); + stream.write_all(b"OKAYtid-1234").await.unwrap(); + assert_eq!(read_service(&mut stream).await, "exec:screencap -p"); + stream.write_all(b"OKAY").await.unwrap(); + stream.write_all(expected).await.unwrap(); + }); + let adb = AdbClient::new(None).with_server_addr(address); + assert_eq!(adb.exec_out(&["screencap", "-p"]).await.unwrap(), expected); + server.await.unwrap(); + } + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn refused_server_reports_missing_server_and_binary() { + let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); + let adb = AdbClient::with_adb_path(None, "/no/such/adb").with_server_addr(address); + let error = adb.server_version().await.unwrap_err().to_string(); + assert!(error.contains("no ADB server")); + assert!(error.contains("ADB binary not found")); + } + + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn request_timeout_drops_stalled_socket() { + let (listener, address) = bind_listener().await; + let server = tokio::spawn(async move { + let (_stream, _) = listener.accept().await.unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + }); + let adb = AdbClient::new(None) + .with_server_addr(address) + .with_timeout(Duration::from_millis(50)); + let error = adb.server_version().await.unwrap_err(); + assert!(error.to_string().contains("timed out")); + server.abort(); + } +} diff --git a/packages/accessibility-cli/src/lib.rs b/packages/accessibility-cli/src/lib.rs index 2bf75f7..0e23f84 100644 --- a/packages/accessibility-cli/src/lib.rs +++ b/packages/accessibility-cli/src/lib.rs @@ -79,7 +79,7 @@ fn screenshot_path() -> std::path::PathBuf { /// Handle screenshot-screen command. async fn handle_screenshot_screen(adapter: &TargetedAccessibility, args: &CommonArgs) { println!("Capturing full screen screenshot..."); - match adapter.capture_screen() { + match adapter.capture_screen().await { Ok(screenshot) => { if args.overlay { match adapter.get_screen_bounds().await { @@ -109,7 +109,7 @@ async fn handle_screenshot_screen(adapter: &TargetedAccessibility, args: &Common eprintln!("Failed to get screen bounds: {}", e); // Save raw screenshot without overlay as fallback let filename = screenshot_path(); - if let Err(e) = std::fs::write(&filename, &screenshot.data) { + if let Err(e) = tokio::fs::write(&filename, &screenshot.data).await { eprintln!("Failed to save screenshot to {}: {}", filename.display(), e); std::process::exit(1); } @@ -123,7 +123,7 @@ async fn handle_screenshot_screen(adapter: &TargetedAccessibility, args: &Common } } else { let filename = screenshot_path(); - if let Err(e) = std::fs::write(&filename, &screenshot.data) { + if let Err(e) = tokio::fs::write(&filename, &screenshot.data).await { eprintln!("Failed to write {}: {}", filename.display(), e); std::process::exit(1); } @@ -190,7 +190,7 @@ async fn handle_annotate( } // Capture screenshot - let screenshot = match adapter.capture_screen() { + let screenshot = match adapter.capture_screen().await { Ok(s) => s, Err(e) => { eprintln!("Failed to capture screenshot: {}", e); @@ -836,7 +836,7 @@ async fn handle_screenshot_elements( std::process::exit(1); } }; - let screenshot = match adapter.capture_screen() { + let screenshot = match adapter.capture_screen().await { Ok(s) => s, Err(e) => { eprintln!("Failed to capture screen: {}", e); @@ -876,7 +876,7 @@ async fn handle_screenshot_elements( match screenshot.crop(bounds, &screen_bounds) { Ok(cropped) => { let filename = screenshot_path(); - if let Err(e) = std::fs::write(&filename, &cropped.data) { + if let Err(e) = tokio::fs::write(&filename, &cropped.data).await { eprintln!("Failed to write {}: {}", filename.display(), e); std::process::exit(1); } @@ -2274,7 +2274,7 @@ pub async fn run_cli(cli: &Cli) { // Android works on all host platforms via ADB PlatformType::Android => { // Create raw AndroidAccessibility for Android-specific commands - let mut android_adapter = match AndroidAccessibility::new(cli.serial.as_deref()) { + let mut android_adapter = match AndroidAccessibility::new(cli.serial.as_deref()).await { Ok(a) => a, Err(e) => { eprintln!("Failed to create Android adapter: {}", e); @@ -2307,7 +2307,7 @@ pub async fn run_cli(cli: &Cli) { Some(serial) => AndroidTarget::Serial(serial.to_owned()), None => AndroidTarget::DefaultDevice, }; - let mut adapter = match TargetedAccessibility::new_android(android_target) { + let mut adapter = match TargetedAccessibility::new_android(android_target).await { Ok(a) => a, Err(e) => { eprintln!("Failed to create Android adapter: {}", e); @@ -2570,7 +2570,7 @@ async fn handle_android_specific(adapter: &mut AndroidAccessibility, cli: &Cli) // Handle ADB tap if let Some((x, y)) = cli.adb.adb_tap { println!("Tapping at ({}, {})...", x, y); - match adapter.adb().tap(x, y) { + match adapter.adb().tap(x, y).await { Ok(()) => println!("Tap successful!"), Err(e) => { eprintln!("Tap failed: {}", e); diff --git a/packages/accessibility-cli/tests/cli_smoke.rs b/packages/accessibility-cli/tests/cli_smoke.rs index f03a3c5..b804747 100644 --- a/packages/accessibility-cli/tests/cli_smoke.rs +++ b/packages/accessibility-cli/tests/cli_smoke.rs @@ -96,6 +96,7 @@ fn operational_flags_parse_before_backend_startup() { for args in cases { let mut cmd = Command::cargo_bin("accessibility-cli").unwrap(); cmd.env("PATH", &no_adb_path) + .env("ANDROID_ADB_SERVER_PORT", "1") .args(*args) .assert() .failure() @@ -195,6 +196,7 @@ fn press_accepts_query_on_non_ios_platforms() { .join("test-no-adb"); let mut cmd = Command::cargo_bin("accessibility-cli").unwrap(); cmd.env("PATH", &no_adb_path) + .env("ANDROID_ADB_SERVER_PORT", "1") .args([ "--platform", "android", diff --git a/packages/accessibility-core/examples/android_session_probe.rs b/packages/accessibility-core/examples/android_session_probe.rs index 13eb895..73a7cdc 100644 --- a/packages/accessibility-core/examples/android_session_probe.rs +++ b/packages/accessibility-core/examples/android_session_probe.rs @@ -12,12 +12,15 @@ use anyhow::{Context, Result}; async fn main() -> Result<()> { let requested_serial = std::env::args().nth(1); let adb = AdbClient::discover(requested_serial.as_deref()); - let serial = adb.resolved_serial()?; - let _ = adb.stop_app("com.google.android.settings.intelligence"); - let _ = adb.stop_app("com.android.settings"); - adb.launch_app("com.android.settings", Some(".Settings"))?; + let serial = adb.resolved_serial().await?; + let _ = adb + .stop_app("com.google.android.settings.intelligence") + .await; + let _ = adb.stop_app("com.android.settings").await; + adb.launch_app("com.android.settings", Some(".Settings")) + .await?; tokio::time::sleep(Duration::from_millis(500)).await; - let session = EmulatorSession::start(Some(&serial), VideoConfig::default())?; + let session = EmulatorSession::start(Some(&serial), VideoConfig::default()).await?; session.seed_orientation().await; let device = session.device_info(); println!("device : {}", device.serial); @@ -42,16 +45,20 @@ async fn main() -> Result<()> { let tap_x = bounds.x + bounds.width / 2.0; let tap_y = bounds.y + bounds.height / 2.0; println!("tap : {}", tappable.selector); - session.send_input(InputCommand::Touch { - phase: TouchPhase::Begin, - x: tap_x, - y: tap_y, - }); - session.send_input(InputCommand::Touch { - phase: TouchPhase::End, - x: tap_x, - y: tap_y, - }); + session + .send_input(InputCommand::Touch { + phase: TouchPhase::Begin, + x: tap_x, + y: tap_y, + }) + .await; + session + .send_input(InputCommand::Touch { + phase: TouchPhase::End, + x: tap_x, + y: tap_y, + }) + .await; tokio::time::sleep(Duration::from_millis(700)).await; let tapped = session.ax_snapshot(false).await?; if tapped @@ -63,9 +70,11 @@ async fn main() -> Result<()> { anyhow::bail!("gRPC touch input did not change the Settings tree"); } - session.send_input(InputCommand::Button { - button: HardwareButton::Home, - }); + session + .send_input(InputCommand::Button { + button: HardwareButton::Home, + }) + .await; tokio::time::sleep(Duration::from_millis(500)).await; let home = session.ax_snapshot(false).await?; if home.app_name == tapped.app_name { @@ -98,7 +107,7 @@ async fn main() -> Result<()> { println!("received : {received} frames, {keyframes} key, {bytes} bytes"); println!("session : {:.1} fps, {:.2} Mbps", stats.fps, stats.mbps); - session.set_orientation(Orientation::LandscapeRight)?; + session.set_orientation(Orientation::LandscapeRight).await?; let landscape = session.ax_snapshot(false).await?; let landscape_stats = session.stats(); if !landscape.is_landscape || landscape_stats.encoded_width <= landscape_stats.encoded_height { @@ -108,7 +117,7 @@ async fn main() -> Result<()> { "landscape : {}x{}", landscape_stats.encoded_width, landscape_stats.encoded_height ); - session.set_orientation(Orientation::Portrait)?; + session.set_orientation(Orientation::Portrait).await?; let portrait = session.ax_snapshot(false).await?; let portrait_stats = session.stats(); if portrait.is_landscape || portrait_stats.encoded_width >= portrait_stats.encoded_height { diff --git a/packages/accessibility-core/src/accessibility/mod.rs b/packages/accessibility-core/src/accessibility/mod.rs index ec3e53c..b6f973c 100644 --- a/packages/accessibility-core/src/accessibility/mod.rs +++ b/packages/accessibility-core/src/accessibility/mod.rs @@ -94,8 +94,11 @@ pub trait AccessibilityReader { // Platform adapter methods (merged from PlatformAdapter trait) /// Capture a screenshot for a target. - fn capture_screen(&self, _target: &Target) -> Result { - anyhow::bail!("Screenshot not supported on this platform") + fn capture_screen( + &self, + _target: &Target, + ) -> impl std::future::Future> { + async { anyhow::bail!("Screenshot not supported on this platform") } } /// Get bounds for coordinate conversion. diff --git a/packages/accessibility-core/src/accessibility/targeted.rs b/packages/accessibility-core/src/accessibility/targeted.rs index 04441bf..8058907 100644 --- a/packages/accessibility-core/src/accessibility/targeted.rs +++ b/packages/accessibility-core/src/accessibility/targeted.rs @@ -76,6 +76,23 @@ macro_rules! dispatch_mut_async { }; } +/// Macro to dispatch immutable async method calls to the inner reader implementation. +macro_rules! dispatch_async { + ($self:expr, $method:ident $(, $arg:expr)*) => { + match &$self.inner { + #[cfg(target_os = "macos")] + AccessibilityReaderImpl::MacOS(r) => AccessibilityReader::$method(r $(, $arg)*).await, + #[cfg(target_os = "macos")] + AccessibilityReaderImpl::IOSSimulator(r) => AccessibilityReader::$method(r $(, $arg)*).await, + #[cfg(target_os = "windows")] + AccessibilityReaderImpl::Windows(r) => AccessibilityReader::$method(r $(, $arg)*).await, + #[cfg(target_os = "linux")] + AccessibilityReaderImpl::Linux(r) => AccessibilityReader::$method(r $(, $arg)*).await, + AccessibilityReaderImpl::Android(r) => AccessibilityReader::$method(r $(, $arg)*).await, + } + }; +} + /// Wrapper that stores a target and provides convenience methods. /// /// This wrapper holds an underlying `AccessibilityReader` implementation and @@ -94,7 +111,7 @@ macro_rules! dispatch_mut_async { /// /// // No need to pass pid on every call /// let tree = reader.get_tree(&TreeFilter::default())?; -/// let screenshot = reader.capture_screen()?; +/// let screenshot = reader.capture_screen().await?; /// reader.keystroke(Code::Enter, Modifiers::empty())?; /// ``` pub struct TargetedAccessibility { @@ -252,10 +269,10 @@ impl TargetedAccessibility { } /// Create a new Android accessibility reader. - pub fn new_android(target: AndroidTarget) -> Result { + pub async fn new_android(target: AndroidTarget) -> Result { Ok(Self { inner: AccessibilityReaderImpl::Android( - crate::platform::android::AndroidAccessibility::new(target.serial())?, + crate::platform::android::AndroidAccessibility::new(target.serial()).await?, ), target: Target::Android(target), }) @@ -279,8 +296,8 @@ impl TargetedAccessibility { /// Capture a screenshot of the target window. /// /// Uses the stored target automatically. - pub fn capture_screen(&self) -> Result { - dispatch!(self, capture_screen, &self.target) + pub async fn capture_screen(&self) -> Result { + dispatch_async!(self, capture_screen, &self.target) } /// Get the bounds of the target window. diff --git a/packages/accessibility-core/src/api/app.rs b/packages/accessibility-core/src/api/app.rs index ad8f1e1..c2da9c9 100644 --- a/packages/accessibility-core/src/api/app.rs +++ b/packages/accessibility-core/src/api/app.rs @@ -157,6 +157,7 @@ impl App { }, Platform::Android => match &config.target { Target::Android(target) => TargetedAccessibility::new_android(target.clone()) + .await .map_err(|e| Error::ConnectionFailed { message: format!("Failed to create Android adapter: {}", e), }), @@ -286,6 +287,7 @@ impl App { let inner = self.inner.lock().await; inner .capture_screen() + .await .map_err(|e: anyhow::Error| Error::ScreenshotFailed { message: e.to_string(), }) diff --git a/packages/accessibility-core/src/platform/android.rs b/packages/accessibility-core/src/platform/android.rs index 8d4062a..1474af9 100644 --- a/packages/accessibility-core/src/platform/android.rs +++ b/packages/accessibility-core/src/platform/android.rs @@ -7,14 +7,14 @@ //! //! This module provides accessibility support for Android devices and emulators through //! the Android Debug Bridge (ADB). Unlike other platforms that use native accessibility APIs, -//! Android support works via shell commands executed through ADB. +//! Android support works through the ADB server smartsocket protocol. //! //! # Architecture //! //! ```text //! Rust (AndroidAccessibility) -//! ↓ std::process::Command -//! adb shell / adb exec-out +//! ↓ TCP smartsocket +//! ADB server //! ↓ //! Android Device/Emulator //! ``` @@ -32,7 +32,7 @@ //! use accessibility_core::accessibility::{AccessibilityReader, TreeFilter}; //! //! // Connect to the default device -//! let mut reader = AndroidAccessibility::new(None)?; +//! let mut reader = AndroidAccessibility::new(None).await?; //! //! // Get the UI tree //! let tree = reader.get_tree(&Target::Android(AndroidTarget::DefaultDevice), &TreeFilter::default()).await?; @@ -501,17 +501,17 @@ impl AndroidAccessibility { /// # Example /// ```ignore /// // Connect to default device - /// let reader = AndroidAccessibility::new(None)?; + /// let reader = AndroidAccessibility::new(None).await?; /// /// // Connect to specific device - /// let reader = AndroidAccessibility::new(Some("emulator-5554"))?; + /// let reader = AndroidAccessibility::new(Some("emulator-5554")).await?; /// ``` - pub fn new(serial: Option<&str>) -> Result { + pub async fn new(serial: Option<&str>) -> Result { let adb = AdbClient::new(serial); - adb.check_connection()?; + adb.check_connection().await?; // Get initial screen size - let screen_size = adb.get_screen_size().ok(); + let screen_size = adb.get_screen_size().await.ok(); Ok(Self { adb, @@ -523,11 +523,11 @@ impl AndroidAccessibility { } /// Create a new Android accessibility reader with a custom ADB path. - pub fn with_adb_path(serial: Option<&str>, adb_path: &str) -> Result { + pub async fn with_adb_path(serial: Option<&str>, adb_path: &str) -> Result { let adb = AdbClient::with_adb_path(serial, adb_path); - adb.check_connection()?; + adb.check_connection().await?; - let screen_size = adb.get_screen_size().ok(); + let screen_size = adb.get_screen_size().await.ok(); Ok(Self { adb, @@ -549,8 +549,8 @@ impl AndroidAccessibility { } /// Refresh the cached screen size. - pub fn refresh_screen_size(&mut self) -> Result<(u32, u32)> { - let size = self.adb.get_screen_size()?; + pub async fn refresh_screen_size(&mut self) -> Result<(u32, u32)> { + let size = self.adb.get_screen_size().await?; self.screen_size = Some(size); Ok(size) } @@ -575,7 +575,7 @@ impl AccessibilityReader for AndroidAccessibility { self.element_bounds.clear(); // Dump UI hierarchy - let xml = self.adb.dump_ui()?; + let xml = self.adb.dump_ui().await?; // Parse XML into node tree let root_node = parse_ui_xml(&xml)?; @@ -622,7 +622,7 @@ impl AccessibilityReader for AndroidAccessibility { let center = self .get_element_center(id) .ok_or_else(|| anyhow!("Element {} not found or has no bounds", id))?; - self.adb.tap(center.x, center.y)?; + self.adb.tap(center.x, center.y).await?; Ok(()) } Action::Focus => { @@ -630,7 +630,7 @@ impl AccessibilityReader for AndroidAccessibility { let center = self .get_element_center(id) .ok_or_else(|| anyhow!("Element {} not found or has no bounds", id))?; - self.adb.tap(center.x, center.y)?; + self.adb.tap(center.x, center.y).await?; Ok(()) } Action::ScrollIntoView => { @@ -640,7 +640,8 @@ impl AccessibilityReader for AndroidAccessibility { let start_y = size.1 as f64 * 0.7; let end_y = size.1 as f64 * 0.3; self.adb - .swipe((center_x, start_y), (center_x, end_y), 300)?; + .swipe((center_x, start_y), (center_x, end_y), 300) + .await?; } Ok(()) } @@ -657,19 +658,19 @@ impl AccessibilityReader for AndroidAccessibility { let center = self .get_element_center(id) .ok_or_else(|| anyhow!("Element {} not found or has no bounds", id))?; - self.adb.tap(center.x, center.y)?; + self.adb.tap(center.x, center.y).await?; // Small delay to ensure focus tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; // Clear existing text (select all and delete) - self.adb.key_event(AndroidKeyCode::CtrlLeft as u32)?; - self.adb.key_event(AndroidKeyCode::A as u32)?; - self.adb.key_event(AndroidKeyCode::Del as u32)?; + self.adb.key_event(AndroidKeyCode::CtrlLeft as u32).await?; + self.adb.key_event(AndroidKeyCode::A as u32).await?; + self.adb.key_event(AndroidKeyCode::Del as u32).await?; // Type the new value if !value.is_empty() { - self.adb.input_text(value)?; + self.adb.input_text(value).await?; } Ok(()) @@ -709,25 +710,27 @@ impl AccessibilityReader for AndroidAccessibility { self.cache.version() } - fn capture_screen(&self, _target: &Target) -> Result { - let data = self.adb.screenshot()?; - - // Get image dimensions from PNG header - let (width, height) = if data.len() > 24 { - // PNG header: 8 bytes signature, then IHDR chunk - // IHDR starts at byte 8, width at 16, height at 20 (both big-endian u32) - let width = u32::from_be_bytes([data[16], data[17], data[18], data[19]]); - let height = u32::from_be_bytes([data[20], data[21], data[22], data[23]]); - (width, height) - } else { - self.screen_size.unwrap_or_default() - }; - - Ok(Screenshot { - data, - width, - height, - }) + fn capture_screen(&self, _target: &Target) -> impl Future> { + async move { + let data = self.adb.screenshot().await?; + + // Get image dimensions from PNG header + let (width, height) = if data.len() > 24 { + // PNG header: 8 bytes signature, then IHDR chunk + // IHDR starts at byte 8, width at 16, height at 20 (both big-endian u32) + let width = u32::from_be_bytes([data[16], data[17], data[18], data[19]]); + let height = u32::from_be_bytes([data[20], data[21], data[22], data[23]]); + (width, height) + } else { + self.screen_size.unwrap_or_default() + }; + + Ok(Screenshot { + data, + width, + height, + }) + } } fn get_screen_bounds(&self, _target: &Target) -> impl Future> { @@ -755,21 +758,21 @@ impl AccessibilityReader for AndroidAccessibility { async move { // Press modifiers if modifiers.contains(Modifiers::SHIFT) { - self.adb.key_event(AndroidKeyCode::ShiftLeft as u32)?; + self.adb.key_event(AndroidKeyCode::ShiftLeft as u32).await?; } if modifiers.contains(Modifiers::CONTROL) { - self.adb.key_event(AndroidKeyCode::CtrlLeft as u32)?; + self.adb.key_event(AndroidKeyCode::CtrlLeft as u32).await?; } if modifiers.contains(Modifiers::ALT) { - self.adb.key_event(AndroidKeyCode::AltLeft as u32)?; + self.adb.key_event(AndroidKeyCode::AltLeft as u32).await?; } if modifiers.contains(Modifiers::META) { - self.adb.key_event(AndroidKeyCode::MetaLeft as u32)?; + self.adb.key_event(AndroidKeyCode::MetaLeft as u32).await?; } // Press main key if let Some(keycode) = AndroidKeyCode::from_code(key) { - self.adb.key_event(keycode as u32)?; + self.adb.key_event(keycode as u32).await?; } else { bail!("Unsupported key code: {:?}", key); } @@ -780,7 +783,7 @@ impl AccessibilityReader for AndroidAccessibility { fn type_raw(&mut self, _target: &Target, text: &str) -> impl Future> { async move { - self.adb.input_text(text)?; + self.adb.input_text(text).await?; Ok(()) } } @@ -794,7 +797,7 @@ impl AccessibilityReader for AndroidAccessibility { ) -> impl Future> { async move { // Android only supports single tap (no right-click) - self.adb.tap(x, y)?; + self.adb.tap(x, y).await?; Ok(()) } } @@ -821,7 +824,9 @@ impl AccessibilityReader for AndroidAccessibility { let end_x = center_x + delta_x * swipe_distance / 2.0; let end_y = center_y - delta_y * swipe_distance / 2.0; - self.adb.swipe((start_x, start_y), (end_x, end_y), 100)?; + self.adb + .swipe((start_x, start_y), (end_x, end_y), 100) + .await?; Ok(()) } } @@ -990,84 +995,88 @@ pub trait AndroidExtensions { impl AndroidExtensions for AndroidAccessibility { fn press_back(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::Back as u32)?; + self.adb.key_event(AndroidKeyCode::Back as u32).await?; Ok(()) } } fn press_home(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::Home as u32)?; + self.adb.key_event(AndroidKeyCode::Home as u32).await?; Ok(()) } } fn press_recent_apps(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::AppSwitch as u32)?; + self.adb.key_event(AndroidKeyCode::AppSwitch as u32).await?; Ok(()) } } fn press_menu(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::Menu as u32)?; + self.adb.key_event(AndroidKeyCode::Menu as u32).await?; Ok(()) } } fn volume_up(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::VolumeUp as u32)?; + self.adb.key_event(AndroidKeyCode::VolumeUp as u32).await?; Ok(()) } } fn volume_down(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::VolumeDown as u32)?; + self.adb + .key_event(AndroidKeyCode::VolumeDown as u32) + .await?; Ok(()) } } fn volume_mute(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::VolumeMute as u32)?; + self.adb + .key_event(AndroidKeyCode::VolumeMute as u32) + .await?; Ok(()) } } fn press_power(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::Power as u32)?; + self.adb.key_event(AndroidKeyCode::Power as u32).await?; Ok(()) } } fn wake_up(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::Wakeup as u32)?; + self.adb.key_event(AndroidKeyCode::Wakeup as u32).await?; Ok(()) } } fn sleep(&mut self) -> impl Future> { async move { - self.adb.key_event(AndroidKeyCode::Sleep as u32)?; + self.adb.key_event(AndroidKeyCode::Sleep as u32).await?; Ok(()) } } fn launch_app(&mut self, package: &str) -> impl Future> { async move { - self.adb.launch_app(package, None)?; + self.adb.launch_app(package, None).await?; Ok(()) } } fn stop_app(&mut self, package: &str) -> impl Future> { async move { - self.adb.stop_app(package)?; + self.adb.stop_app(package).await?; Ok(()) } } @@ -1079,7 +1088,7 @@ impl AndroidExtensions for AndroidAccessibility { duration_ms: u64, ) -> impl Future> { async move { - self.adb.swipe(start, end, duration_ms)?; + self.adb.swipe(start, end, duration_ms).await?; Ok(()) } } @@ -1087,26 +1096,29 @@ impl AndroidExtensions for AndroidAccessibility { fn long_press(&mut self, x: f64, y: f64, duration_ms: u64) -> impl Future> { async move { // Long press is a swipe with same start and end - self.adb.swipe((x, y), (x, y), duration_ms)?; + self.adb.swipe((x, y), (x, y), duration_ms).await?; Ok(()) } } fn get_current_activity(&self) -> impl Future> { - async move { self.adb.get_current_activity() } + async move { self.adb.get_current_activity().await } } fn open_notifications(&mut self) -> impl Future> { async move { self.adb - .shell(&["cmd", "statusbar", "expand-notifications"])?; + .shell(&["cmd", "statusbar", "expand-notifications"]) + .await?; Ok(()) } } fn open_quick_settings(&mut self) -> impl Future> { async move { - self.adb.shell(&["cmd", "statusbar", "expand-settings"])?; + self.adb + .shell(&["cmd", "statusbar", "expand-settings"]) + .await?; Ok(()) } } diff --git a/packages/accessibility-core/src/platform/android/ax.rs b/packages/accessibility-core/src/platform/android/ax.rs index 5c975d9..347d1ec 100644 --- a/packages/accessibility-core/src/platform/android/ax.rs +++ b/packages/accessibility-core/src/platform/android/ax.rs @@ -84,35 +84,29 @@ pub enum AxCommand { }, } -pub fn spawn_ax_worker(serial: &str) -> Result> { +pub async fn spawn_ax_worker(serial: &str) -> Result> { let adb = super::AdbClient::discover(Some(serial)); - let mut reader = AndroidAccessibility::with_adb_path(Some(serial), &adb.adb_path)?; + let mut reader = AndroidAccessibility::with_adb_path(Some(serial), &adb.adb_path).await?; let target = Target::Android(AndroidTarget::Serial(serial.to_string())); let (commands, mut command_rx) = mpsc::unbounded_channel(); - std::thread::Builder::new() - .name("android-emulator-ax".into()) - .spawn(move || { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build(); - let Ok(runtime) = runtime else { return }; - let mut screen = None; - while let Some(command) = command_rx.blocking_recv() { - match command { - AxCommand::Snapshot { scan, reply } => { - let result = runtime.block_on(snapshot(&mut reader, &target, scan)); - if let Ok((_, bounds)) = &result { - screen = Some(*bounds); - } - let _ = reply.send(result.map(|(snapshot, _)| snapshot)); - } - AxCommand::HitTest { x, y, reply } => { - let result = runtime.block_on(hit_test(&mut reader, screen.as_ref(), x, y)); - let _ = reply.send(result); + tokio::spawn(async move { + let mut screen = None; + while let Some(command) = command_rx.recv().await { + match command { + AxCommand::Snapshot { scan, reply } => { + let result = snapshot(&mut reader, &target, scan).await; + if let Ok((_, bounds)) = &result { + screen = Some(*bounds); } + let _ = reply.send(result.map(|(snapshot, _)| snapshot)); + } + AxCommand::HitTest { x, y, reply } => { + let result = hit_test(&mut reader, screen.as_ref(), x, y).await; + let _ = reply.send(result); } } - })?; + } + }); Ok(commands) } diff --git a/packages/accessibility-core/src/platform/android/input.rs b/packages/accessibility-core/src/platform/android/input.rs index 9be2b6c..ca1f801 100644 --- a/packages/accessibility-core/src/platform/android/input.rs +++ b/packages/accessibility-core/src/platform/android/input.rs @@ -1,5 +1,3 @@ -use std::sync::mpsc; - use accessibility_android_sys::emulator::protocol::controller::input_event; use accessibility_android_sys::emulator::protocol::controller::keyboard_event::{ KeyCodeType, KeyEventType, @@ -11,7 +9,7 @@ use accessibility_android_sys::emulator::{EmulatorGrpcClient, discover_emulator} use accessibility_android_sys::{AdbClient, AndroidKeyCode}; use anyhow::{Result, anyhow, bail}; use serde::{Deserialize, Serialize}; -use tokio::sync::mpsc::UnboundedSender; +use tokio::sync::{mpsc::UnboundedSender, oneshot}; use crate::video::ScreenGeometry; @@ -89,58 +87,44 @@ pub enum InputCommand { }, } -pub fn spawn_input_worker( +pub async fn spawn_input_worker( serial: &str, geometry: ScreenGeometry, ) -> Result> { - let discovery = discover_emulator(Some(serial))?; + let discovery = discover_emulator(Some(serial)).await?; let adb = AdbClient::discover(Some(serial)); let (commands, mut command_rx) = tokio::sync::mpsc::unbounded_channel(); - let (ready_tx, ready_rx) = mpsc::sync_channel(1); - std::thread::Builder::new() - .name("android-emulator-input".into()) - .spawn(move || { - let runtime = match tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - { - Ok(runtime) => runtime, - Err(error) => { - let _ = ready_tx.send(Err(error.to_string())); - return; + let (ready_tx, ready_rx) = oneshot::channel(); + tokio::spawn(async move { + let mut client = match EmulatorGrpcClient::connect(discovery).await { + Ok(client) => client, + Err(error) => { + let _ = ready_tx.send(Err(error.to_string())); + return; + } + }; + let _ = ready_tx.send(Ok(())); + while let Some(command) = command_rx.recv().await { + let command = match command { + InputCommand::Rotate { orientation } => { + let _ = set_device_orientation(&adb, orientation).await; + continue; + } + InputCommand::Button { button } => { + apply_hardware_button(&adb, button).await; + continue; } + command => command, }; - runtime.block_on(async move { - let mut client = match EmulatorGrpcClient::connect(discovery).await { - Ok(client) => client, - Err(error) => { - let _ = ready_tx.send(Err(error.to_string())); - return; - } - }; - let _ = ready_tx.send(Ok(())); - while let Some(command) = command_rx.recv().await { - let command = match command { - InputCommand::Rotate { orientation } => { - let _ = set_device_orientation(&adb, orientation); - continue; - } - InputCommand::Button { button } => { - apply_hardware_button(&adb, button); - continue; - } - command => command, - }; - for event in to_events(command, geometry) { - if let Err(error) = client.send_input(event).await { - eprintln!("Android Emulator input failed: {error:#}"); - return; - } - } + for event in to_events(command, geometry) { + if let Err(error) = client.send_input(event).await { + eprintln!("Android Emulator input failed: {error:#}"); + return; } - }); - })?; - match ready_rx.recv() { + } + } + }); + match ready_rx.await { Ok(Ok(())) => Ok(commands), Ok(Err(error)) => Err(anyhow!(error)), Err(_) => Err(anyhow!( @@ -236,26 +220,28 @@ fn normalized_coordinate(value: f64, dimension: u32) -> i32 { (value.clamp(0.0, 1.0) * dimension.saturating_sub(1) as f64).round() as i32 } -fn apply_hardware_button(adb: &AdbClient, button: HardwareButton) { +async fn apply_hardware_button(adb: &AdbClient, button: HardwareButton) { let key = match button { HardwareButton::Home => AndroidKeyCode::Home, HardwareButton::Back => AndroidKeyCode::Back, HardwareButton::Lock => AndroidKeyCode::Power, HardwareButton::AppSwitch => AndroidKeyCode::AppSwitch, }; - let _ = adb.key_event(key as u32); + let _ = adb.key_event(key as u32).await; } -pub fn set_device_orientation(adb: &AdbClient, orientation: Orientation) -> Result<()> { - adb.shell(&["wm", "fixed-to-user-rotation", "enabled"])?; +pub async fn set_device_orientation(adb: &AdbClient, orientation: Orientation) -> Result<()> { + adb.shell(&["wm", "fixed-to-user-rotation", "enabled"]) + .await?; let target = orientation.android_rotation(); - adb.shell(&["wm", "user-rotation", "lock", &target.to_string()])?; + adb.shell(&["wm", "user-rotation", "lock", &target.to_string()]) + .await?; for _ in 0..20 { - let output = adb.shell(&["dumpsys", "display"])?; + let output = adb.shell(&["dumpsys", "display"]).await?; if display_rotation(&output) == Some(target) { return Ok(()); } - std::thread::sleep(std::time::Duration::from_millis(100)); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; } bail!("Android display did not reach rotation {target}") } diff --git a/packages/accessibility-core/src/platform/android/session.rs b/packages/accessibility-core/src/platform/android/session.rs index cac3eda..f5f9c36 100644 --- a/packages/accessibility-core/src/platform/android/session.rs +++ b/packages/accessibility-core/src/platform/android/session.rs @@ -63,14 +63,14 @@ pub struct EmulatorSession { } impl EmulatorSession { - pub fn start(serial: Option<&str>, config: VideoConfig) -> Result> { + pub async fn start(serial: Option<&str>, config: VideoConfig) -> Result> { let adb = AdbClient::discover(serial); - let serial = adb.resolved_serial()?; + let serial = adb.resolved_serial().await?; if !serial.starts_with("emulator-") { bail!("Android Emulator streaming requires an emulator serial, got '{serial}'"); } let adb = AdbClient::discover(Some(&serial)); - let (width, height) = adb.get_screen_size()?; + let (width, height) = adb.get_screen_size().await?; let geometry = ScreenGeometry { width, height }; let (frames, _) = broadcast::channel(FRAME_BUFFER); let stats = Arc::new(StreamStats::default()); @@ -89,8 +89,8 @@ impl EmulatorSession { }) }; let capture = AndroidVideoCapture::start(adb.clone(), geometry, &config, sink)?; - let input = spawn_input_worker(&serial, geometry)?; - let ax = spawn_ax_worker(&serial)?; + let input = spawn_input_worker(&serial, geometry).await?; + let ax = spawn_ax_worker(&serial).await?; Ok(Arc::new(Self { serial, adb, @@ -133,16 +133,16 @@ impl EmulatorSession { *self.orientation.lock().unwrap() } - pub fn set_orientation(&self, orientation: Orientation) -> Result<()> { - set_device_orientation(&self.adb, orientation)?; + pub async fn set_orientation(&self, orientation: Orientation) -> Result<()> { + set_device_orientation(&self.adb, orientation).await?; self.capture.set_landscape(orientation.is_landscape())?; *self.orientation.lock().unwrap() = orientation; Ok(()) } - pub fn send_input(&self, command: InputCommand) { + pub async fn send_input(&self, command: InputCommand) { if let InputCommand::Rotate { orientation } = command { - let _ = self.set_orientation(orientation); + let _ = self.set_orientation(orientation).await; return; } let _ = self.input.send(command); diff --git a/packages/accessibility-core/src/platform/ios_simulator.rs b/packages/accessibility-core/src/platform/ios_simulator.rs index 8a15bd7..f0a38a6 100644 --- a/packages/accessibility-core/src/platform/ios_simulator.rs +++ b/packages/accessibility-core/src/platform/ios_simulator.rs @@ -296,8 +296,11 @@ impl AccessibilityReader for IOSSimulatorAccessibility { IOSSimulatorAccessibility::snapshot_version(self) } - fn capture_screen(&self, _target: &Target) -> Result { - IOSSimulatorAccessibility::capture_screen(self) + fn capture_screen( + &self, + _target: &Target, + ) -> impl std::future::Future> { + async move { IOSSimulatorAccessibility::capture_screen(self) } } fn get_screen_bounds( diff --git a/packages/accessibility-core/src/platform/macos.rs b/packages/accessibility-core/src/platform/macos.rs index 5772821..50201be 100644 --- a/packages/accessibility-core/src/platform/macos.rs +++ b/packages/accessibility-core/src/platform/macos.rs @@ -1793,31 +1793,36 @@ impl AccessibilityReader for MacOSAccessibility { true } - fn capture_screen(&self, target: &Target) -> Result { - let pid = match target { - Target::Pid(pid) => Some(*pid), - Target::System => None, - _ => bail!("macOS screenshot requires Target::Pid or Target::System"), - }; + fn capture_screen( + &self, + target: &Target, + ) -> impl std::future::Future> { + async move { + let pid = match target { + Target::Pid(pid) => Some(*pid), + Target::System => None, + _ => bail!("macOS screenshot requires Target::Pid or Target::System"), + }; - if let Some(pid) = pid - && let Ok(Some(screenshot)) = Self::capture_window_for_pid(pid) - { - return Ok(screenshot); - } + if let Some(pid) = pid + && let Ok(Some(screenshot)) = Self::capture_window_for_pid(pid) + { + return Ok(screenshot); + } - let screenshot = Self::capture_main_display()?; + let screenshot = Self::capture_main_display()?; - if let Some(pid) = pid - && let Some(window_bounds) = Self::get_window_bounds_for_pid(pid) - { - let screen_bounds = Self::main_display_bounds(); - if let Ok(cropped) = screenshot.crop(&window_bounds, &screen_bounds) { - return Ok(cropped); + if let Some(pid) = pid + && let Some(window_bounds) = Self::get_window_bounds_for_pid(pid) + { + let screen_bounds = Self::main_display_bounds(); + if let Ok(cropped) = screenshot.crop(&window_bounds, &screen_bounds) { + return Ok(cropped); + } } - } - Ok(screenshot) + Ok(screenshot) + } } async fn get_screen_bounds(&self, target: &Target) -> Result { diff --git a/packages/accessibility-core/src/platform/msft.rs b/packages/accessibility-core/src/platform/msft.rs index 902d600..342de23 100644 --- a/packages/accessibility-core/src/platform/msft.rs +++ b/packages/accessibility-core/src/platform/msft.rs @@ -189,13 +189,18 @@ impl AccessibilityReader for WindowsAccessibility { self.cache.version() } - fn capture_screen(&self, target: &Target) -> Result { - let screenshot = match target { - Target::Pid(pid) => self.inner.capture_screen_for_pid(*pid), - Target::System => self.inner.capture_screen(), - _ => bail!("Windows screenshot requires Target::Pid or Target::System"), - }; - screenshot.map(from_sys_screenshot) + fn capture_screen( + &self, + target: &Target, + ) -> impl std::future::Future> { + async move { + let screenshot = match target { + Target::Pid(pid) => self.inner.capture_screen_for_pid(*pid), + Target::System => self.inner.capture_screen(), + _ => bail!("Windows screenshot requires Target::Pid or Target::System"), + }; + screenshot.map(from_sys_screenshot) + } } async fn get_screen_bounds(&self, target: &Target) -> Result { diff --git a/packages/accessibility-core/src/platform/x11.rs b/packages/accessibility-core/src/platform/x11.rs index 2a72911..8a38812 100644 --- a/packages/accessibility-core/src/platform/x11.rs +++ b/packages/accessibility-core/src/platform/x11.rs @@ -1063,18 +1063,24 @@ impl AccessibilityReader for LinuxAccessibility { // Platform adapter methods (merged from LinuxAdapter) - fn capture_screen(&self, target: &Target) -> Result { - let pid = match target { - Target::Pid(pid) => Some(*pid), - Target::System => None, - _ => bail!("Linux screenshot requires Target::Pid or Target::System"), - }; - if let Some(pid) = pid - && let Ok(screenshot) = self.capture_window(pid) - { - return Ok(screenshot); + #[allow(clippy::manual_async_fn)] + fn capture_screen( + &self, + target: &Target, + ) -> impl std::future::Future> { + async move { + let pid = match target { + Target::Pid(pid) => Some(*pid), + Target::System => None, + _ => bail!("Linux screenshot requires Target::Pid or Target::System"), + }; + if let Some(pid) = pid + && let Ok(screenshot) = self.capture_window(pid) + { + return Ok(screenshot); + } + LinuxAccessibility::capture_screen(self) } - LinuxAccessibility::capture_screen(self) } async fn get_screen_bounds(&self, target: &Target) -> Result { diff --git a/packages/accessibility-core/tests/calculator_e2e.rs b/packages/accessibility-core/tests/calculator_e2e.rs index 25fd94d..0762517 100644 --- a/packages/accessibility-core/tests/calculator_e2e.rs +++ b/packages/accessibility-core/tests/calculator_e2e.rs @@ -506,6 +506,7 @@ async fn test_screen_screenshot() { let screenshot = accessibility .capture_screen(&Target::System) + .await .expect("Failed to capture screen"); // Screen should have reasonable dimensions (at least 800x600) diff --git a/packages/accessibility-core/tests/settings_android_e2e.rs b/packages/accessibility-core/tests/settings_android_e2e.rs index 62aefa2..ba3ad0e 100644 --- a/packages/accessibility-core/tests/settings_android_e2e.rs +++ b/packages/accessibility-core/tests/settings_android_e2e.rs @@ -31,21 +31,22 @@ struct DeviceGuard { } impl DeviceGuard { - fn new() -> Result { + async fn new() -> Result { let adb = AdbClient::new(None); - adb.command(&["wait-for-device"]) + adb.wait_for_device() + .await .context("Failed waiting for Android device")?; - adb.check_connection()?; - wait_for_boot(&adb)?; - stabilize_device(&adb); + adb.check_connection().await?; + wait_for_boot(&adb).await?; + stabilize_device(&adb).await; Ok(Self { adb }) } } -fn wait_for_boot(adb: &AdbClient) -> Result<()> { +async fn wait_for_boot(adb: &AdbClient) -> Result<()> { let start = Instant::now(); loop { - if let Ok(output) = adb.shell(&["getprop", "sys.boot_completed"]) + if let Ok(output) = adb.shell(&["getprop", "sys.boot_completed"]).await && output.trim() == "1" { return Ok(()); @@ -58,34 +59,38 @@ fn wait_for_boot(adb: &AdbClient) -> Result<()> { ); } - std::thread::sleep(Duration::from_secs(2)); + tokio::time::sleep(Duration::from_secs(2)).await; } } -fn stabilize_device(adb: &AdbClient) { - let _ = adb.shell(&["input", "keyevent", "224"]); - let _ = adb.shell(&["wm", "dismiss-keyguard"]); +async fn stabilize_device(adb: &AdbClient) { + let _ = adb.shell(&["input", "keyevent", "224"]).await; + let _ = adb.shell(&["wm", "dismiss-keyguard"]).await; for setting in [ "window_animation_scale", "transition_animation_scale", "animator_duration_scale", ] { - let _ = adb.shell(&["settings", "put", "global", setting, "0"]); + let _ = adb + .shell(&["settings", "put", "global", setting, "0"]) + .await; } } -fn launch_settings(adb: &AdbClient) -> Result<()> { - let launcher_result = adb.launch_app(SETTINGS_PACKAGE, None); - let settings_result = adb.shell(&[ - "am", - "start", - "-W", - "-a", - SETTINGS_ACTION, - "-p", - SETTINGS_PACKAGE, - ]); +async fn launch_settings(adb: &AdbClient) -> Result<()> { + let launcher_result = adb.launch_app(SETTINGS_PACKAGE, None).await; + let settings_result = adb + .shell(&[ + "am", + "start", + "-W", + "-a", + SETTINGS_ACTION, + "-p", + SETTINGS_PACKAGE, + ]) + .await; match (launcher_result, settings_result) { (Ok(()), _) | (_, Ok(_)) => {} @@ -99,11 +104,11 @@ fn launch_settings(adb: &AdbClient) -> Result<()> { Ok(()) } -fn wait_for_settings_process(adb: &AdbClient, timeout: Duration) -> Result<()> { +async fn wait_for_settings_process(adb: &AdbClient, timeout: Duration) -> Result<()> { let start = Instant::now(); loop { - let observation = match adb.shell(&["pidof", SETTINGS_PACKAGE]) { + let observation = match adb.shell(&["pidof", SETTINGS_PACKAGE]).await { Ok(pid) => { let pid = pid.trim(); if !pid.is_empty() { @@ -111,7 +116,7 @@ fn wait_for_settings_process(adb: &AdbClient, timeout: Duration) -> Result<()> { } "pidof returned no Settings process".to_string() } - Err(pidof_error) => match adb.shell(&["ps", "-A"]) { + Err(pidof_error) => match adb.shell(&["ps", "-A"]).await { Ok(processes) => { if processes .lines() @@ -133,7 +138,7 @@ fn wait_for_settings_process(adb: &AdbClient, timeout: Duration) -> Result<()> { ); } - std::thread::sleep(POLL_INTERVAL); + tokio::time::sleep(POLL_INTERVAL).await; } } @@ -144,8 +149,9 @@ struct AndroidSettingsGuard { impl AndroidSettingsGuard { async fn launch() -> Result { - let device = DeviceGuard::new()?; + let device = DeviceGuard::new().await?; let mut accessibility = AndroidAccessibility::new(None) + .await .context("Failed to create Android accessibility reader")?; reset_settings(&mut accessibility).await?; @@ -172,7 +178,12 @@ impl AndroidSettingsGuard { impl Drop for AndroidSettingsGuard { fn drop(&mut self) { - let _ = self.device.adb.stop_app(SETTINGS_PACKAGE); + if let Ok(handle) = tokio::runtime::Handle::try_current() { + let adb = self.device.adb.clone(); + handle.spawn(async move { + let _ = adb.stop_app(SETTINGS_PACKAGE).await; + }); + } } } @@ -185,13 +196,13 @@ impl Deref for AndroidSettingsGuard { } async fn reset_settings(accessibility: &mut AndroidAccessibility) -> Result<()> { - let _ = accessibility.adb().stop_app(SETTINGS_PACKAGE); + let _ = accessibility.adb().stop_app(SETTINGS_PACKAGE).await; let _ = accessibility.wake_up().await; let _ = accessibility.press_home().await; tokio::time::sleep(Duration::from_secs(1)).await; - launch_settings(accessibility.adb())?; - wait_for_settings_process(accessibility.adb(), SETTINGS_PROCESS_TIMEOUT)?; + launch_settings(accessibility.adb()).await?; + wait_for_settings_process(accessibility.adb(), SETTINGS_PROCESS_TIMEOUT).await?; tokio::time::sleep(Duration::from_secs(3)).await; Ok(()) } @@ -231,7 +242,7 @@ async fn wait_for_settings_tree( } if start.elapsed() >= next_relaunch { - let _ = launch_settings(adb); + let _ = launch_settings(adb).await; next_relaunch = start.elapsed() + SETTINGS_RELAUNCH_INTERVAL; } @@ -252,9 +263,10 @@ fn count_role(element: &Element, role: Role) -> usize { #[serial] #[ignore = "Requires Android device/emulator with ADB"] async fn test_android_device_input_smoke() -> Result<()> { - let device = DeviceGuard::new()?; - let mut accessibility = - AndroidAccessibility::new(None).context("Failed to create Android accessibility reader")?; + let device = DeviceGuard::new().await?; + let mut accessibility = AndroidAccessibility::new(None) + .await + .context("Failed to create Android accessibility reader")?; accessibility.wake_up().await?; accessibility.press_home().await?; @@ -266,12 +278,13 @@ async fn test_android_device_input_smoke() -> Result<()> { let (width, height) = accessibility .refresh_screen_size() + .await .context("Failed to get Android screen size")?; assert!(width > 0); assert!(height > 0); - launch_settings(accessibility.adb())?; - wait_for_settings_process(accessibility.adb(), SETTINGS_PROCESS_TIMEOUT)?; + launch_settings(accessibility.adb()).await?; + wait_for_settings_process(accessibility.adb(), SETTINGS_PROCESS_TIMEOUT).await?; tokio::time::sleep(Duration::from_secs(2)).await; let center_x = width as f64 / 2.0; @@ -285,7 +298,7 @@ async fn test_android_device_input_smoke() -> Result<()> { .swipe((center_x, end_y), (center_x, start_y), 300) .await?; - let _ = device.adb.stop_app(SETTINGS_PACKAGE); + let _ = device.adb.stop_app(SETTINGS_PACKAGE).await; Ok(()) } diff --git a/packages/accessibility-serve/src/http.rs b/packages/accessibility-serve/src/http.rs index 3e24dc6..99758b3 100644 --- a/packages/accessibility-serve/src/http.rs +++ b/packages/accessibility-serve/src/http.rs @@ -195,7 +195,7 @@ async fn set_orientation( State(state): State, Json(request): Json, ) -> Response { - match state.session.set_orientation(request.orientation) { + match state.session.set_orientation(request.orientation).await { Ok(()) => Json(serde_json::json!({ "orientation": request.orientation })).into_response(), Err(error) => (StatusCode::BAD_REQUEST, error.to_string()).into_response(), } @@ -336,7 +336,7 @@ async fn pump_input(state: AppState, mut socket: WebSocket) { _ => continue, }; - if let Err(error) = state.session.send_input_json(&payload) { + if let Err(error) = state.session.send_input_json(&payload).await { tracing::debug!("ignoring malformed input event: {error}"); } } diff --git a/packages/accessibility-serve/src/lib.rs b/packages/accessibility-serve/src/lib.rs index 7e4cc5e..67e6eb4 100644 --- a/packages/accessibility-serve/src/lib.rs +++ b/packages/accessibility-serve/src/lib.rs @@ -133,6 +133,7 @@ pub async fn serve(_config: ServeConfig) -> Result<()> { pub async fn serve_emulator(config: ServeEmulatorConfig) -> Result<()> { let session = session::EmulatorSession::start(config.serial.as_deref(), config.video) + .await .context("failed to start Android Emulator capture")?; let session = session::Session::android(session); session.seed_orientation().await; diff --git a/packages/accessibility-serve/src/session.rs b/packages/accessibility-serve/src/session.rs index 6bd90e3..5a4940d 100644 --- a/packages/accessibility-serve/src/session.rs +++ b/packages/accessibility-serve/src/session.rs @@ -136,29 +136,35 @@ impl Session { } } - pub fn send_input_json(&self, payload: &str) -> Result<()> { + pub async fn send_input_json(&self, payload: &str) -> Result<()> { match self { #[cfg(target_os = "macos")] Self::Ios(session) => { session.send_input(serde_json::from_str::(payload)?); } Self::Android(session) => { - session.send_input(serde_json::from_str::( - payload, - )?); + session + .send_input(serde_json::from_str::( + payload, + )?) + .await; } } Ok(()) } - pub fn set_orientation(&self, orientation: Orientation) -> Result<()> { + pub async fn set_orientation(&self, orientation: Orientation) -> Result<()> { match self { #[cfg(target_os = "macos")] Self::Ios(session) => { session.set_orientation(to_ios_orientation(orientation)); Ok(()) } - Self::Android(session) => session.set_orientation(to_android_orientation(orientation)), + Self::Android(session) => { + session + .set_orientation(to_android_orientation(orientation)) + .await + } } }