diff --git a/Cargo.lock b/Cargo.lock index 8cb7562..ee04799 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,17 @@ name = "accessibility-android-sys" version = "0.1.0" dependencies = [ "anyhow", + "bytes", "keyboard-types", + "memmap2", + "prost", + "protoc-bin-vendored", + "rtp", + "serde_json", + "tokio", + "tonic", + "tonic-build", + "webrtc", ] [[package]] @@ -1531,6 +1541,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.9" @@ -1550,6 +1566,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -1775,6 +1797,25 @@ dependencies = [ "subtle", ] +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -1892,6 +1933,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -1900,6 +1942,20 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", ] [[package]] @@ -1909,12 +1965,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "bytes", + "futures-channel", + "futures-util", "http", "http-body", "hyper", + "libc", "pin-project-lite", + "socket2 0.6.3", "tokio", "tower-service", + "tracing", ] [[package]] @@ -2331,6 +2392,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -2392,6 +2462,12 @@ dependencies = [ "pxfm", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "nalgebra" version = "0.32.6" @@ -3026,6 +3102,16 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "phf" version = "0.10.1" @@ -3106,6 +3192,26 @@ dependencies = [ "siphasher 1.0.2", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -3311,6 +3417,122 @@ dependencies = [ "syn", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "protoc-bin-vendored" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" + [[package]] name = "pxfm" version = "0.1.29" @@ -4348,6 +4570,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.29.0" @@ -4369,6 +4602,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -4403,6 +4637,46 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "base64", + "bytes", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn", +] + [[package]] name = "tower" version = "0.5.3" @@ -4411,9 +4685,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -4477,6 +4754,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "ttf-parser" version = "0.25.1" @@ -4664,6 +4947,15 @@ dependencies = [ "atomic-waker", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/packages/accessibility-android-sys/Cargo.toml b/packages/accessibility-android-sys/Cargo.toml index 12fc874..c7d657c 100644 --- a/packages/accessibility-android-sys/Cargo.toml +++ b/packages/accessibility-android-sys/Cargo.toml @@ -12,4 +12,18 @@ categories = ["accessibility", "api-bindings", "os"] [dependencies] anyhow.workspace = true +bytes.workspace = true keyboard-types.workspace = true +memmap2 = "0.9" +prost = "0.13" +tokio.workspace = true +tonic = { version = "0.13", default-features = false, features = ["channel", "codegen", "prost"] } + +[dev-dependencies] +rtp = "0.14" +serde_json.workspace = true +webrtc.workspace = true + +[build-dependencies] +protoc-bin-vendored = "3" +tonic-build = { version = "0.13", default-features = false, features = ["prost"] } diff --git a/packages/accessibility-android-sys/build.rs b/packages/accessibility-android-sys/build.rs new file mode 100644 index 0000000..9384d36 --- /dev/null +++ b/packages/accessibility-android-sys/build.rs @@ -0,0 +1,16 @@ +fn main() -> Result<(), Box> { + let protoc = protoc_bin_vendored::protoc_bin_path()?; + let mut prost = tonic_build::Config::new(); + prost.protoc_executable(protoc); + tonic_build::configure() + .build_server(false) + .compile_protos_with_config( + prost, + &[ + "proto/emulator_controller.proto", + "proto/rtc_service_v2.proto", + ], + &["proto"], + )?; + Ok(()) +} diff --git a/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs b/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs new file mode 100644 index 0000000..89ee8c9 --- /dev/null +++ b/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs @@ -0,0 +1,196 @@ +use std::fs::OpenOptions; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use accessibility_android_sys::AdbClient; +use accessibility_android_sys::emulator::protocol::controller::image_format::ImgFormat; +use accessibility_android_sys::emulator::protocol::controller::image_transport::TransportChannel; +use accessibility_android_sys::emulator::protocol::controller::{ImageFormat, ImageTransport}; +use accessibility_android_sys::emulator::{EmulatorGrpcClient, discover_emulator}; +use anyhow::{Context, Result, bail}; +use memmap2::MmapOptions; + +const MAX_DIMENSION: u32 = 1280; +const PROBE_DURATION: Duration = Duration::from_secs(5); + +#[tokio::main] +async fn main() -> Result<()> { + let selector = std::env::args().nth(1); + let discovery = discover_emulator(selector.as_deref())?; + let serial = discovery + .properties + .get("port.serial") + .map(|port| format!("emulator-{port}")) + .context("discovery file has no port.serial")?; + let mut client = EmulatorGrpcClient::connect(discovery.clone()).await?; + let status = client.status().await?; + println!("emulator : {}", status.version); + println!("endpoint : {}", discovery.endpoint()); + println!("serial : {serial}"); + + let stop = Arc::new(AtomicBool::new(false)); + let stimulus_stop = Arc::clone(&stop); + let stimulus_serial = serial.clone(); + let stimulus = std::thread::spawn(move || { + let adb = AdbClient::discover(Some(&stimulus_serial)); + let Ok((width, height)) = adb.get_screen_size() else { + return; + }; + while !stimulus_stop.load(Ordering::Relaxed) { + let _ = adb.swipe( + (width as f64 * 0.5, height as f64 * 0.75), + (width as f64 * 0.5, height as f64 * 0.25), + 250, + ); + std::thread::sleep(Duration::from_millis(150)); + } + }); + + let bytes = probe_bytes(client.clone()).await?; + print_report("grpc", &bytes); + let mmap = probe_mmap(client).await?; + print_report("mmap", &mmap); + stop.store(true, Ordering::Relaxed); + let _ = stimulus.join(); + println!("probe passed"); + Ok(()) +} + +#[derive(Default)] +struct Report { + frames: u64, + bytes: u64, + gaps: u64, + tears: u64, + latency_us: Vec, + width: u32, + height: u32, + elapsed: Duration, +} + +async fn probe_bytes(mut client: EmulatorGrpcClient) -> Result { + let format = request_format(None); + let mut stream = client.stream_screenshots(format).await?; + collect(&mut stream, None).await +} + +async fn probe_mmap(mut client: EmulatorGrpcClient) -> Result { + let path = std::env::temp_dir().join(format!( + "accessibility-emulator-raw-{}.rgba", + std::process::id() + )); + let capacity = MAX_DIMENSION as u64 * MAX_DIMENSION as u64 * 4; + let file = OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(&path)?; + file.set_len(capacity)?; + let mut mapping = unsafe { MmapOptions::new().len(capacity as usize).map_mut(&file)? }; + let handle = format!("file://{}", path.display()); + let format = request_format(Some(ImageTransport { + channel: TransportChannel::Mmap as i32, + handle, + })); + let mut stream = client.stream_screenshots(format).await?; + let report = collect(&mut stream, Some(&mut mapping)).await; + drop(mapping); + drop(file); + let _ = std::fs::remove_file(path); + report +} + +fn request_format(transport: Option) -> ImageFormat { + ImageFormat { + format: ImgFormat::Rgba8888 as i32, + rotation: None, + width: MAX_DIMENSION, + height: MAX_DIMENSION, + display: 0, + transport, + } +} + +async fn collect( + stream: &mut tonic::Streaming, + mut mapping: Option<&mut memmap2::MmapMut>, +) -> Result { + let started = Instant::now(); + let mut report = Report::default(); + let mut previous_seq = None; + while started.elapsed() < PROBE_DURATION { + let image = match tokio::time::timeout(Duration::from_secs(2), stream.message()).await { + Ok(Ok(Some(image))) => image, + Ok(Ok(None)) => break, + Ok(Err(error)) => return Err(error.into()), + Err(_) if report.frames > 0 => continue, + Err(error) => return Err(error).context("timed out waiting for emulator frame"), + }; + let format = image.format.context("emulator frame has no format")?; + let width = format.width.max(image.width); + let height = format.height.max(image.height); + if width == 0 || height == 0 { + continue; + } + let expected = width as usize * height as usize * 4; + if let Some(mapping) = mapping.as_deref_mut() { + if expected > mapping.len() { + bail!("mapped frame {width}x{height} exceeds mapping"); + } + let first = checksum(&mapping[..expected]); + let owned = mapping[..expected].to_vec(); + let second = checksum(&mapping[..expected]); + report.tears += u64::from(first != second || first != checksum(&owned)); + report.bytes += owned.len() as u64; + } else { + if image.image.len() != expected { + bail!( + "RGBA frame is {} bytes, expected {expected} for {width}x{height}", + image.image.len() + ); + } + report.bytes += image.image.len() as u64; + } + if let Some(previous) = previous_seq { + report.gaps += image.seq.wrapping_sub(previous).saturating_sub(1) as u64; + } + previous_seq = Some(image.seq); + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_micros() as u64; + if image.timestamp_us > 0 && now >= image.timestamp_us { + report.latency_us.push(now - image.timestamp_us); + } + report.frames += 1; + report.width = width; + report.height = height; + } + report.elapsed = started.elapsed(); + Ok(report) +} + +fn checksum(bytes: &[u8]) -> u64 { + bytes + .chunks(4096) + .map(|chunk| chunk.first().copied().unwrap_or_default() as u64) + .sum() +} + +fn print_report(name: &str, report: &Report) { + let mean_latency = if report.latency_us.is_empty() { + 0.0 + } else { + report.latency_us.iter().sum::() as f64 / report.latency_us.len() as f64 + }; + println!("{name}:"); + println!(" geometry : {}x{}", report.width, report.height); + println!(" frames : {}", report.frames); + println!( + " fps : {:.1}", + report.frames as f64 / report.elapsed.as_secs_f64() + ); + println!(" copied : {} bytes", report.bytes); + println!(" seq gaps : {}", report.gaps); + println!(" tears : {}", report.tears); + println!(" latency : {:.1} ms", mean_latency / 1000.0); +} diff --git a/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs b/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs new file mode 100644 index 0000000..70d1a81 --- /dev/null +++ b/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs @@ -0,0 +1,282 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use accessibility_android_sys::emulator::protocol::rtc::Id; +use accessibility_android_sys::emulator::{EmulatorGrpcClient, discover_emulator}; +use anyhow::{Context, Result, anyhow, bail}; +use rtp::codecs::h264::H264Packet; +use serde_json::{Value, json}; +use tokio::sync::mpsc; +use webrtc::api::APIBuilder; +use webrtc::api::interceptor_registry::register_default_interceptors; +use webrtc::api::media_engine::{MIME_TYPE_H264, MediaEngine}; +use webrtc::ice_transport::ice_candidate::RTCIceCandidateInit; +use webrtc::interceptor::registry::Registry; +use webrtc::media::io::sample_builder::SampleBuilder; +use webrtc::peer_connection::RTCPeerConnection; +use webrtc::peer_connection::configuration::RTCConfiguration; +use webrtc::peer_connection::sdp::sdp_type::RTCSdpType; +use webrtc::peer_connection::sdp::session_description::RTCSessionDescription; +use webrtc::rtcp::payload_feedbacks::picture_loss_indication::PictureLossIndication; +use webrtc::rtp_transceiver::RTCPFeedback; +use webrtc::rtp_transceiver::rtp_codec::{ + RTCRtpCodecCapability, RTCRtpCodecParameters, RTPCodecType, +}; +use webrtc::track::track_remote::TrackRemote; + +const TRACK_TIMEOUT: Duration = Duration::from_secs(15); +const KEYFRAME_TIMEOUT: Duration = Duration::from_secs(10); +const RECOVERY_TIMEOUT: Duration = Duration::from_secs(5); + +#[tokio::main] +async fn main() -> Result<()> { + let selector = std::env::args().nth(1); + let discovery = discover_emulator(selector.as_deref())?; + println!("discovery : {}", discovery.path.display()); + println!("endpoint : {}", discovery.endpoint()); + + let mut grpc = EmulatorGrpcClient::connect(discovery).await?; + let status = grpc.status().await?; + println!("emulator : {}", status.version); + println!("booted : {}", status.booted); + if !status.booted { + bail!("Android Emulator has not finished booting"); + } + if let (Some(width), Some(height)) = ( + status.platform_config.get("hw.lcd.width"), + status.platform_config.get("hw.lcd.height"), + ) { + println!("display : {width}x{height}"); + } + + let id = grpc.begin_rtc_stream().await?; + println!("rtc id : {}", id.guid); + let messages = grpc.receive_jsep_stream(id.clone()).await?; + let peer = create_peer().await?; + let (track_tx, mut track_rx) = mpsc::channel(1); + peer.on_track(Box::new(move |track, _, _| { + let track_tx = track_tx.clone(); + Box::pin(async move { + let _ = track_tx.send(track).await; + }) + })); + + let signaling_peer = Arc::clone(&peer); + let signaling_grpc = grpc.clone(); + let signaling_id = id.clone(); + let mut signaling = tokio::spawn(async move { + run_signaling(signaling_peer, signaling_grpc, signaling_id, messages).await + }); + + let track = tokio::select! { + track = track_rx.recv() => track.ok_or_else(|| anyhow!("WebRTC peer closed before receiving a track"))?, + result = &mut signaling => return result.context("signaling task panicked")?, + _ = tokio::time::sleep(TRACK_TIMEOUT) => bail!("timed out waiting for the emulator video track"), + }; + let codec = track.codec().capability; + println!("codec : {} {}", codec.mime_type, codec.sdp_fmtp_line); + if !codec.mime_type.eq_ignore_ascii_case(MIME_TYPE_H264) { + bail!("emulator negotiated {}, not H.264", codec.mime_type); + } + + let report = read_frames(&peer, &track).await?; + println!("frames : {}", report.frames); + println!("keyframes : {}", report.keyframes); + println!("bytes : {}", report.bytes); + println!( + "fps : {:.1}", + report.frames as f64 / report.elapsed.as_secs_f64() + ); + println!( + "bitrate : {:.2} Mbps", + report.bytes as f64 * 8.0 / report.elapsed.as_secs_f64() / 1_000_000.0 + ); + println!( + "pli->idr : {:.1} ms", + report.recovery.as_secs_f64() * 1000.0 + ); + + grpc.end_rtc_stream(id).await?; + peer.close().await?; + signaling.abort(); + println!("probe passed"); + Ok(()) +} + +async fn create_peer() -> Result> { + let mut media_engine = MediaEngine::default(); + let feedback = vec![ + RTCPFeedback { + typ: "nack".to_owned(), + parameter: "".to_owned(), + }, + RTCPFeedback { + typ: "nack".to_owned(), + parameter: "pli".to_owned(), + }, + RTCPFeedback { + typ: "ccm".to_owned(), + parameter: "fir".to_owned(), + }, + ]; + for (payload_type, profile) in [(102, "42001f"), (125, "42e01f"), (123, "640032")] { + media_engine.register_codec( + RTCRtpCodecParameters { + capability: RTCRtpCodecCapability { + mime_type: MIME_TYPE_H264.to_owned(), + clock_rate: 90000, + channels: 0, + sdp_fmtp_line: format!( + "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id={profile}" + ), + rtcp_feedback: feedback.clone(), + }, + payload_type, + ..Default::default() + }, + RTPCodecType::Video, + )?; + } + let registry = register_default_interceptors(Registry::new(), &mut media_engine)?; + let api = APIBuilder::new() + .with_media_engine(media_engine) + .with_interceptor_registry(registry) + .build(); + Ok(Arc::new( + api.new_peer_connection(RTCConfiguration::default()).await?, + )) +} + +async fn run_signaling( + peer: Arc, + mut grpc: EmulatorGrpcClient, + id: Id, + mut messages: tonic::Streaming< + accessibility_android_sys::emulator::protocol::rtc::ReceiveJsepMessageResponse, + >, +) -> Result<()> { + let mut remote_description_set = false; + let mut pending_candidates = Vec::new(); + while let Some(response) = messages.message().await? { + let Some(jsep) = response.jsep_msg else { + continue; + }; + if jsep.message.is_empty() { + continue; + } + let message: Value = serde_json::from_str(&jsep.message) + .with_context(|| format!("invalid emulator JSEP message: {}", jsep.message))?; + if message.get("bye").is_some() { + bail!("emulator ended the RTC stream before the probe completed"); + } + if let Some(value) = message.get("candidate") { + let candidate: RTCIceCandidateInit = serde_json::from_value(value.clone())?; + if remote_description_set { + peer.add_ice_candidate(candidate).await?; + } else { + pending_candidates.push(candidate); + } + } + let Some(value) = message.get("sdp") else { + continue; + }; + let description: RTCSessionDescription = serde_json::from_value(value.clone())?; + if description.sdp_type != RTCSdpType::Offer { + continue; + } + peer.set_remote_description(description).await?; + remote_description_set = true; + for candidate in pending_candidates.drain(..) { + peer.add_ice_candidate(candidate).await?; + } + let answer = peer.create_answer(None).await?; + let mut gathering_complete = peer.gathering_complete_promise().await; + peer.set_local_description(answer).await?; + let _ = gathering_complete.recv().await; + let answer = peer + .local_description() + .await + .ok_or_else(|| anyhow!("WebRTC peer produced no local answer"))?; + grpc.send_jsep(id.clone(), json!({ "sdp": answer }).to_string()) + .await?; + } + bail!("emulator JSEP stream closed") +} + +struct ProbeReport { + frames: u64, + keyframes: u64, + bytes: u64, + elapsed: Duration, + recovery: Duration, +} + +async fn read_frames(peer: &RTCPeerConnection, track: &TrackRemote) -> Result { + let started = Instant::now(); + let mut builder = SampleBuilder::new(16, H264Packet::default(), 90000) + .with_max_time_delay(Duration::from_millis(250)); + let mut frames = 0; + let mut keyframes = 0; + let mut bytes = 0; + let mut requested_at: Option = None; + + loop { + let timeout = if requested_at.is_some() { + RECOVERY_TIMEOUT + } else { + KEYFRAME_TIMEOUT + }; + let (packet, _) = tokio::time::timeout(timeout, track.read_rtp()) + .await + .context("timed out waiting for H.264 RTP")??; + builder.push(packet); + while let Some(sample) = builder.pop() { + let types = annex_b_nal_types(&sample.data); + let keyframe = types.contains(&5); + if keyframe && (!types.contains(&7) || !types.contains(&8)) { + bail!("H.264 keyframe did not carry SPS and PPS"); + } + frames += 1; + bytes += sample.data.len() as u64; + if !keyframe { + continue; + } + keyframes += 1; + if let Some(requested_at) = requested_at { + return Ok(ProbeReport { + frames, + keyframes, + bytes, + elapsed: started.elapsed(), + recovery: requested_at.elapsed(), + }); + } + peer.write_rtcp(&[Box::new(PictureLossIndication { + sender_ssrc: 0, + media_ssrc: track.ssrc(), + })]) + .await?; + requested_at = Some(Instant::now()); + } + } +} + +fn annex_b_nal_types(data: &[u8]) -> Vec { + let mut types = Vec::new(); + let mut index = 0; + while index + 3 < data.len() { + let start_len = if data[index..].starts_with(&[0, 0, 0, 1]) { + 4 + } else if data[index..].starts_with(&[0, 0, 1]) { + 3 + } else { + index += 1; + continue; + }; + if let Some(header) = data.get(index + start_len) { + types.push(header & 0x1f); + } + index += start_len + 1; + } + types +} diff --git a/packages/accessibility-android-sys/examples/screenrecord_probe.rs b/packages/accessibility-android-sys/examples/screenrecord_probe.rs new file mode 100644 index 0000000..3df6839 --- /dev/null +++ b/packages/accessibility-android-sys/examples/screenrecord_probe.rs @@ -0,0 +1,140 @@ +use std::io::Read; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +use accessibility_android_sys::AdbClient; +use accessibility_android_sys::emulator::screenrecord::{ + AnnexBAccessUnitParser, H264AccessUnit, ScreenRecordConfig, spawn_screenrecord, +}; +use anyhow::{Context, Result, bail}; + +const IDLE_FLUSH: Duration = Duration::from_millis(75); +const PROBE_DURATION: Duration = Duration::from_secs(5); + +fn main() -> Result<()> { + let serial = std::env::args() + .nth(1) + .unwrap_or_else(|| "emulator-5554".to_string()); + let adb = AdbClient::discover(Some(&serial)); + adb.check_connection()?; + let (width, height) = adb.get_screen_size()?; + let config = ScreenRecordConfig::for_max_dimension(width, height, Some(1280), 5_000_000); + println!("device : {serial}"); + println!("source : {width}x{height}"); + println!("encoded : {}x{}", config.width, config.height); + + let stimulus = adb.clone(); + std::thread::spawn(move || { + std::thread::sleep(Duration::from_secs(1)); + let _ = stimulus.swipe( + (width as f64 * 0.5, height as f64 * 0.75), + (width as f64 * 0.5, height as f64 * 0.25), + 800, + ); + }); + + let started = Instant::now(); + let frames = capture(&adb, config, PROBE_DURATION, false)?; + let elapsed = started.elapsed(); + let keyframes = frames.iter().filter(|frame| frame.keyframe).count(); + let bytes = frames.iter().map(|frame| frame.data.len()).sum::(); + if frames.is_empty() { + bail!("screenrecord produced no complete H.264 frames"); + } + if keyframes == 0 { + bail!("screenrecord produced no H.264 keyframe"); + } + println!("frames : {}", frames.len()); + println!("keyframes : {keyframes}"); + println!("bytes : {bytes}"); + println!( + "fps : {:.1}", + frames.len() as f64 / elapsed.as_secs_f64() + ); + println!( + "bitrate : {:.2} Mbps", + bytes as f64 * 8.0 / elapsed.as_secs_f64() / 1_000_000.0 + ); + + let restart = Instant::now(); + let restarted = capture(&adb, config, Duration::from_secs(2), true)?; + let first = restarted + .iter() + .find(|frame| frame.keyframe) + .context("fresh screenrecord session produced no keyframe")?; + println!( + "restart : {:.1} ms", + restart.elapsed().as_secs_f64() * 1000.0 + ); + println!("entry NALs: {:?}", nal_types(&first.data)); + println!("probe passed"); + Ok(()) +} + +fn capture( + adb: &AdbClient, + config: ScreenRecordConfig, + duration: Duration, + stop_on_keyframe: bool, +) -> Result> { + let mut child = spawn_screenrecord(adb, config)?; + let mut stdout = child + .stdout + .take() + .context("screenrecord stdout was not piped")?; + let (tx, rx) = mpsc::sync_channel::>(8); + let reader = std::thread::spawn(move || { + let mut buffer = vec![0u8; 64 * 1024]; + loop { + match stdout.read(&mut buffer) { + Ok(0) | Err(_) => break, + Ok(read) => { + if tx.send(buffer[..read].to_vec()).is_err() { + break; + } + } + } + } + }); + + let started = Instant::now(); + let mut parser = AnnexBAccessUnitParser::default(); + let mut frames = Vec::new(); + while started.elapsed() < duration { + match rx.recv_timeout(IDLE_FLUSH) { + Ok(chunk) => frames.extend(parser.push(&chunk)), + Err(mpsc::RecvTimeoutError::Timeout) => frames.extend(parser.flush_idle()), + Err(mpsc::RecvTimeoutError::Disconnected) => break, + } + if stop_on_keyframe && frames.iter().any(|frame| frame.keyframe) { + break; + } + } + frames.extend(parser.flush_idle()); + child.kill().context("failed to stop screenrecord")?; + let _ = child.wait(); + reader + .join() + .map_err(|_| anyhow::anyhow!("screenrecord reader panicked"))?; + Ok(frames) +} + +fn nal_types(data: &[u8]) -> Vec { + let mut types = Vec::new(); + let mut index = 0; + while index + 3 < data.len() { + let code_len = if data[index..].starts_with(&[0, 0, 0, 1]) { + 4 + } else if data[index..].starts_with(&[0, 0, 1]) { + 3 + } else { + index += 1; + continue; + }; + if let Some(header) = data.get(index + code_len) { + types.push(header & 0x1f); + } + index += code_len + 1; + } + types +} diff --git a/packages/accessibility-android-sys/proto/emulator_controller.proto b/packages/accessibility-android-sys/proto/emulator_controller.proto new file mode 100644 index 0000000..61ac2c4 --- /dev/null +++ b/packages/accessibility-android-sys/proto/emulator_controller.proto @@ -0,0 +1,135 @@ +syntax = "proto3"; + +package android.emulation.control; + +import "google/protobuf/empty.proto"; + +service EmulatorController { + rpc getStatus(google.protobuf.Empty) returns (EmulatorStatus) {} + rpc sendKey(KeyboardEvent) returns (google.protobuf.Empty) {} + rpc sendTouch(TouchEvent) returns (google.protobuf.Empty) {} + rpc sendMouse(MouseEvent) returns (google.protobuf.Empty) {} + rpc streamScreenshot(ImageFormat) returns (stream Image) {} +} + +message EmulatorStatus { + string version = 1; + uint64 uptime = 2; + bool booted = 3; + uint64 heartbeat = 6; + map guestConfig = 7; + map platformConfig = 8; +} + +message Touch { + int32 x = 1; + int32 y = 2; + int32 identifier = 3; + int32 pressure = 4; + int32 touch_major = 5; + int32 touch_minor = 6; + EventExpiration expiration = 7; + int32 orientation = 8; + + enum EventExpiration { + EVENT_EXPIRATION_UNSPECIFIED = 0; + NEVER_EXPIRE = 1; + } +} + +message TouchEvent { + repeated Touch touches = 1; + int32 display = 2; +} + +message MouseEvent { + int32 x = 1; + int32 y = 2; + int32 buttons = 3; + int32 display = 4; +} + +message WheelEvent { + int32 dx = 1; + int32 dy = 2; + int32 display = 3; +} + +message KeyboardEvent { + KeyCodeType codeType = 1; + KeyEventType eventType = 2; + int32 keyCode = 3; + string key = 4; + string text = 5; + + enum KeyCodeType { + Usb = 0; + Evdev = 1; + XKB = 2; + Win = 3; + Mac = 4; + } + + enum KeyEventType { + keydown = 0; + keyup = 1; + keypress = 2; + } +} + +message ImageTransport { + enum TransportChannel { + TRANSPORT_CHANNEL_UNSPECIFIED = 0; + MMAP = 1; + } + + TransportChannel channel = 1; + string handle = 2; +} + +message Rotation { + enum SkinRotation { + PORTRAIT = 0; + LANDSCAPE = 1; + REVERSE_PORTRAIT = 2; + REVERSE_LANDSCAPE = 3; + } + + SkinRotation rotation = 1; + double xAxis = 2; + double yAxis = 3; + double zAxis = 4; +} + +message ImageFormat { + enum ImgFormat { + PNG = 0; + RGBA8888 = 1; + RGB888 = 2; + } + + ImgFormat format = 1; + Rotation rotation = 2; + uint32 width = 3; + uint32 height = 4; + uint32 display = 5; + ImageTransport transport = 6; +} + +message Image { + ImageFormat format = 1; + uint32 width = 2; + uint32 height = 3; + bytes image = 4; + uint32 seq = 5; + uint64 timestampUs = 6; +} + +message InputEvent { + oneof type { + KeyboardEvent key_event = 1; + TouchEvent touch_event = 2; + MouseEvent mouse_event = 3; + WheelEvent wheel_event = 6; + } +} diff --git a/packages/accessibility-android-sys/proto/rtc_service_v2.proto b/packages/accessibility-android-sys/proto/rtc_service_v2.proto new file mode 100644 index 0000000..d654a12 --- /dev/null +++ b/packages/accessibility-android-sys/proto/rtc_service_v2.proto @@ -0,0 +1,38 @@ +syntax = "proto3"; + +package android.emulation.control.v2; + +service Rtc { + rpc RequestRtcStream(RtcStreamRequest) returns (RtcStreamResponse) {} + rpc SendJsepMessage(SendJsepMessageRequest) returns (SendJsepMessageResponse) {} + rpc ReceiveJsepMessageStream(ReceiveJsepMessageRequest) returns (stream ReceiveJsepMessageResponse) {} +} + +message RtcStreamRequest {} + +message RtcStreamResponse { + Id id = 1; +} + +message SendJsepMessageRequest { + JsepMsg jsep_msg = 1; +} + +message SendJsepMessageResponse {} + +message ReceiveJsepMessageRequest { + Id id = 1; +} + +message ReceiveJsepMessageResponse { + JsepMsg jsep_msg = 1; +} + +message Id { + string guid = 1; +} + +message JsepMsg { + Id id = 1; + string message = 2; +} diff --git a/packages/accessibility-android-sys/src/emulator.rs b/packages/accessibility-android-sys/src/emulator.rs new file mode 100644 index 0000000..c6a4050 --- /dev/null +++ b/packages/accessibility-android-sys/src/emulator.rs @@ -0,0 +1,393 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail}; +use tonic::metadata::MetadataValue; +use tonic::transport::{Channel, Endpoint}; +use tonic::{Request, Streaming}; + +pub mod raw; +pub mod screenrecord; + +pub mod protocol { + pub mod controller { + tonic::include_proto!("android.emulation.control"); + } + + pub mod rtc { + tonic::include_proto!("android.emulation.control.v2"); + } +} + +use protocol::controller::emulator_controller_client::EmulatorControllerClient; +use protocol::controller::{EmulatorStatus, Image, ImageFormat, InputEvent, input_event}; +use protocol::rtc::rtc_client::RtcClient; +use protocol::rtc::{ + Id, JsepMsg, ReceiveJsepMessageRequest, ReceiveJsepMessageResponse, RtcStreamRequest, + SendJsepMessageRequest, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EmulatorDiscovery { + pub path: PathBuf, + pub pid: Option, + pub grpc_port: u16, + pub grpc_token: Option, + pub properties: BTreeMap, +} + +impl EmulatorDiscovery { + pub fn from_file(path: impl AsRef) -> Result { + let path = path.as_ref(); + let contents = std::fs::read_to_string(path).with_context(|| { + format!("failed to read emulator discovery file {}", path.display()) + })?; + let properties = parse_properties(&contents); + let grpc_port = properties + .get("grpc.port") + .ok_or_else(|| anyhow!("{} has no grpc.port", path.display()))? + .parse::() + .with_context(|| format!("{} has an invalid grpc.port", path.display()))?; + let grpc_token = properties + .get("grpc.token") + .filter(|token| !token.is_empty()) + .cloned(); + let pid = path + .file_stem() + .and_then(|name| name.to_str()) + .and_then(|name| name.strip_prefix("pid_")) + .and_then(|pid| pid.parse().ok()); + Ok(Self { + path: path.to_path_buf(), + pid, + grpc_port, + grpc_token, + properties, + }) + } + + pub fn matches(&self, selector: &str) -> bool { + if self + .properties + .values() + .any(|value| value.eq_ignore_ascii_case(selector)) + { + return true; + } + if self.pid.is_some_and(|pid| selector == pid.to_string()) { + return true; + } + self.properties.get("port.serial").is_some_and(|port| { + selector == port || selector.eq_ignore_ascii_case(&format!("emulator-{port}")) + }) + } + + pub fn endpoint(&self) -> String { + format!("http://127.0.0.1:{}", self.grpc_port) + } +} + +#[derive(Debug, Clone)] +pub struct EmulatorCapabilities { + pub discovery: EmulatorDiscovery, + pub version: String, + pub booted: bool, + pub rtc_v2: bool, +} + +#[derive(Clone)] +pub struct EmulatorGrpcClient { + discovery: EmulatorDiscovery, + controller: EmulatorControllerClient, + rtc: RtcClient, +} + +impl EmulatorGrpcClient { + pub async fn connect(discovery: EmulatorDiscovery) -> Result { + let channel = Endpoint::from_shared(discovery.endpoint())? + .connect_timeout(Duration::from_secs(3)) + .timeout(Duration::from_secs(10)) + .connect() + .await + .with_context(|| { + format!( + "failed to connect to Android Emulator gRPC endpoint {}", + discovery.endpoint() + ) + })?; + Ok(Self { + discovery, + controller: EmulatorControllerClient::new(channel.clone()), + rtc: RtcClient::new(channel), + }) + } + + pub fn discovery(&self) -> &EmulatorDiscovery { + &self.discovery + } + + pub async fn status(&mut self) -> Result { + let request = self.authorized(())?; + self.controller + .get_status(request) + .await + .map(|response| response.into_inner()) + .context("Android Emulator getStatus failed") + } + + pub async fn stream_screenshots(&mut self, format: ImageFormat) -> Result> { + let request = self.authorized(format)?; + self.controller + .stream_screenshot(request) + .await + .map(|response| response.into_inner()) + .context("Android Emulator streamScreenshot failed") + } + + pub async fn send_input(&mut self, event: InputEvent) -> Result<()> { + match event + .r#type + .ok_or_else(|| anyhow!("Android input event has no type"))? + { + input_event::Type::KeyEvent(event) => { + let request = self.authorized(event)?; + self.controller + .send_key(request) + .await + .context("Android Emulator sendKey failed")?; + } + input_event::Type::TouchEvent(event) => { + let request = self.authorized(event)?; + self.controller + .send_touch(request) + .await + .context("Android Emulator sendTouch failed")?; + } + input_event::Type::MouseEvent(event) => { + let request = self.authorized(event)?; + self.controller + .send_mouse(request) + .await + .context("Android Emulator sendMouse failed")?; + } + input_event::Type::WheelEvent(_) => { + bail!("Android Emulator wheel input requires a streaming RPC") + } + } + Ok(()) + } + + pub async fn begin_rtc_stream(&mut self) -> Result { + let request = self.authorized(RtcStreamRequest {})?; + let response = self + .rtc + .request_rtc_stream(request) + .await + .context("Android Emulator RTC v2 RequestRtcStream failed")? + .into_inner(); + let id = response + .id + .filter(|id| !id.guid.is_empty()) + .ok_or_else(|| anyhow!("Android Emulator RTC v2 returned no stream id"))?; + Ok(id) + } + + pub async fn receive_jsep_stream( + &mut self, + id: Id, + ) -> Result> { + let request = self.authorized(ReceiveJsepMessageRequest { id: Some(id) })?; + self.rtc + .receive_jsep_message_stream(request) + .await + .map(|response| response.into_inner()) + .context("Android Emulator RTC v2 ReceiveJsepMessageStream failed") + } + + pub async fn send_jsep(&mut self, id: Id, message: impl Into) -> Result<()> { + let request = self.authorized(SendJsepMessageRequest { + jsep_msg: Some(JsepMsg { + id: Some(id), + message: message.into(), + }), + })?; + self.rtc + .send_jsep_message(request) + .await + .context("Android Emulator RTC v2 SendJsepMessage failed")?; + Ok(()) + } + + pub async fn end_rtc_stream(&mut self, id: Id) -> Result<()> { + self.send_jsep(id, r#"{"bye":true}"#).await + } + + pub async fn probe_capabilities(&mut self) -> Result { + let status = self.status().await?; + let id = self.begin_rtc_stream().await?; + self.end_rtc_stream(id).await?; + Ok(EmulatorCapabilities { + discovery: self.discovery.clone(), + version: status.version, + booted: status.booted, + rtc_v2: true, + }) + } + + fn authorized(&self, value: T) -> Result> { + let mut request = Request::new(value); + if let Some(token) = &self.discovery.grpc_token { + let value = MetadataValue::try_from(format!("Bearer {token}")) + .context("emulator gRPC token is not valid HTTP metadata")?; + request.metadata_mut().insert("authorization", value); + } + Ok(request) + } +} + +pub fn discover_emulator(selector: Option<&str>) -> Result { + discover_emulator_in(&discovery_directories(), selector) +} + +pub fn discover_emulator_in( + directories: &[PathBuf], + selector: Option<&str>, +) -> Result { + let mut paths = BTreeSet::new(); + for directory in directories { + let entries = match std::fs::read_dir(directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to read emulator discovery directory {}", + directory.display() + ) + }); + } + }; + for entry in entries { + let path = entry?.path(); + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if name.starts_with("pid_") && name.ends_with(".ini") { + paths.insert(path); + } + } + } + + let mut discoveries = paths + .into_iter() + .filter_map(|path| EmulatorDiscovery::from_file(path).ok()) + .collect::>(); + if let Some(selector) = selector { + discoveries.retain(|discovery| discovery.matches(selector)); + } + + match discoveries.len() { + 0 => match selector { + Some(selector) => bail!("no running Android Emulator matches '{selector}'"), + None => bail!("no running Android Emulator with a gRPC discovery file was found"), + }, + 1 => Ok(discoveries.remove(0)), + count => match selector { + Some(selector) => bail!("{count} running Android Emulators match '{selector}'"), + None => bail!( + "{count} Android Emulators are running; specify an AVD name, PID, serial, or gRPC port" + ), + }, + } +} + +pub fn discovery_directories() -> Vec { + let mut directories = BTreeSet::new(); + directories.insert(std::env::temp_dir().join("avd/running")); + if let Some(path) = std::env::var_os("ANDROID_EMULATOR_DISCOVERY") { + directories.insert(PathBuf::from(path)); + } + if let Some(path) = std::env::var_os("ANDROID_AVD_HOME") { + directories.insert(PathBuf::from(path).join("running")); + } + if let Some(home) = std::env::var_os("HOME") { + let home = PathBuf::from(home); + directories.insert(home.join(".android/avd/running")); + if cfg!(target_os = "macos") { + directories.insert(home.join("Library/Android/avd/running")); + directories.insert(home.join("Library/Caches/TemporaryItems/avd/running")); + } + } + directories.into_iter().collect() +} + +fn parse_properties(contents: &str) -> BTreeMap { + contents + .lines() + .filter_map(|line| { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') || line.starts_with(';') { + return None; + } + let (key, value) = line.split_once('=')?; + Some((key.trim().to_string(), value.trim().to_string())) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicU64, Ordering}; + + use super::*; + + static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + + fn test_directory(name: &str) -> PathBuf { + let nonce = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("accessibility-android-{name}-{nonce}")); + std::fs::create_dir_all(&path).unwrap(); + path + } + + #[test] + #[cfg_attr(miri, ignore)] + fn parses_discovery_file() { + let directory = test_directory("parse"); + let path = directory.join("pid_1234.ini"); + std::fs::write( + &path, + "grpc.port = 8554\ngrpc.token = secret\navd.name = Pixel_8\nport.serial = 5554\n", + ) + .unwrap(); + let discovery = EmulatorDiscovery::from_file(&path).unwrap(); + assert_eq!(discovery.pid, Some(1234)); + assert_eq!(discovery.grpc_port, 8554); + assert_eq!(discovery.grpc_token.as_deref(), Some("secret")); + assert!(discovery.matches("Pixel_8")); + assert!(discovery.matches("emulator-5554")); + std::fs::remove_dir_all(directory).unwrap(); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn selects_one_emulator() { + let directory = test_directory("select"); + std::fs::write( + directory.join("pid_1.ini"), + "grpc.port=8554\navd.name=phone\n", + ) + .unwrap(); + std::fs::write( + directory.join("pid_2.ini"), + "grpc.port=8555\navd.name=tablet\n", + ) + .unwrap(); + let discovery = + discover_emulator_in(std::slice::from_ref(&directory), Some("tablet")).unwrap(); + assert_eq!(discovery.grpc_port, 8555); + assert!(discover_emulator_in(std::slice::from_ref(&directory), None).is_err()); + std::fs::remove_dir_all(directory).unwrap(); + } +} diff --git a/packages/accessibility-android-sys/src/emulator/raw.rs b/packages/accessibility-android-sys/src/emulator/raw.rs new file mode 100644 index 0000000..20e5be5 --- /dev/null +++ b/packages/accessibility-android-sys/src/emulator/raw.rs @@ -0,0 +1,226 @@ +use std::fs::{File, OpenOptions}; +use std::path::PathBuf; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, anyhow, bail}; +use memmap2::{MmapMut, MmapOptions}; +use tonic::Streaming; + +use super::protocol::controller::image_format::ImgFormat; +use super::protocol::controller::image_transport::TransportChannel; +use super::protocol::controller::{Image, ImageFormat, ImageTransport}; +use super::{EmulatorDiscovery, EmulatorGrpcClient, discover_emulator}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RawFrameTransport { + Grpc, + Mmap, +} + +#[derive(Debug, Clone, Copy)] +pub struct RawFrameConfig { + pub width: u32, + pub height: u32, + pub display: u32, + pub transport: RawFrameTransport, +} + +#[derive(Debug)] +pub struct RawFrame { + pub pixels: Vec, + pub width: u32, + pub height: u32, + pub sequence: u32, + pub timestamp_us: u64, + pub rotation: i32, +} + +pub struct RawFrameStream { + discovery: EmulatorDiscovery, + stream: Streaming, + mapping: Option, + mapping_file: Option, + mapping_path: Option, +} + +impl RawFrameStream { + pub async fn start(selector: Option<&str>, config: RawFrameConfig) -> Result { + if config.width == 0 || config.height == 0 { + bail!("raw Android capture requires non-zero dimensions"); + } + let discovery = discover_emulator(selector)?; + Self::start_with_discovery(discovery, config).await + } + + pub async fn start_with_discovery( + discovery: EmulatorDiscovery, + config: RawFrameConfig, + ) -> Result { + let mut client = EmulatorGrpcClient::connect(discovery.clone()).await?; + let capacity = rgba_len(config.width, config.height)?; + let (mapping, mapping_file, mapping_path, transport) = match config.transport { + RawFrameTransport::Grpc => (None, None, None, None), + RawFrameTransport::Mmap => { + let path = mapping_path(); + let file = OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(&path) + .with_context(|| { + format!("creating emulator frame mapping {}", path.display()) + })?; + file.set_len(capacity as u64)?; + let mapping = unsafe { MmapOptions::new().len(capacity).map_mut(&file)? }; + let transport = ImageTransport { + channel: TransportChannel::Mmap as i32, + handle: file_uri(&path), + }; + (Some(mapping), Some(file), Some(path), Some(transport)) + } + }; + let stream = client + .stream_screenshots(ImageFormat { + format: ImgFormat::Rgba8888 as i32, + rotation: None, + width: config.width, + height: config.height, + display: config.display, + transport, + }) + .await?; + Ok(Self { + discovery, + stream, + mapping, + mapping_file, + mapping_path, + }) + } + + pub fn discovery(&self) -> &EmulatorDiscovery { + &self.discovery + } + + pub async fn next_frame(&mut self) -> Result> { + loop { + let Some(image) = self.stream.message().await? else { + return Ok(None); + }; + let format = image.format.context("emulator frame has no format")?; + let width = format.width.max(image.width); + let height = format.height.max(image.height); + if width == 0 || height == 0 { + continue; + } + let expected = rgba_len(width, height)?; + let mut pixels = if let Some(mapping) = &self.mapping { + let source = mapping + .get(..expected) + .ok_or_else(|| anyhow!("emulator frame exceeds its memory mapping"))?; + copy_bottom_up_rgba(source, width, height) + } else { + if image.image.len() != expected { + bail!( + "RGBA frame is {} bytes, expected {expected} for {width}x{height}", + image.image.len() + ); + } + image.image + }; + if self.mapping.is_none() { + flip_vertical_rgba(&mut pixels, width, height); + } + return Ok(Some(RawFrame { + pixels, + width, + height, + sequence: image.seq, + timestamp_us: image.timestamp_us, + rotation: format.rotation.map_or(0, |rotation| rotation.rotation), + })); + } + } +} + +impl Drop for RawFrameStream { + fn drop(&mut self) { + self.mapping.take(); + self.mapping_file.take(); + if let Some(path) = self.mapping_path.take() { + let _ = std::fs::remove_file(path); + } + } +} + +fn rgba_len(width: u32, height: u32) -> Result { + width + .checked_mul(height) + .and_then(|pixels| pixels.checked_mul(4)) + .map(|bytes| bytes as usize) + .context("Android frame dimensions overflow") +} + +fn copy_bottom_up_rgba(source: &[u8], width: u32, height: u32) -> Vec { + let stride = width as usize * 4; + let mut pixels = vec![0; source.len()]; + for row in 0..height as usize { + let source_start = (height as usize - 1 - row) * stride; + let target_start = row * stride; + pixels[target_start..target_start + stride] + .copy_from_slice(&source[source_start..source_start + stride]); + } + pixels +} + +fn flip_vertical_rgba(pixels: &mut [u8], width: u32, height: u32) { + let stride = width as usize * 4; + for row in 0..height as usize / 2 { + let opposite = height as usize - 1 - row; + let (before, after) = pixels.split_at_mut(opposite * stride); + before[row * stride..(row + 1) * stride].swap_with_slice(&mut after[..stride]); + } +} + +fn mapping_path() -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or_default(); + std::env::temp_dir().join(format!( + "accessibility-emulator-raw-{}-{nonce}.rgba", + std::process::id() + )) +} + +fn file_uri(path: &std::path::Path) -> String { + #[cfg(target_os = "windows")] + { + format!("file:///{}", path.display().to_string().replace('\\', "/")) + } + #[cfg(not(target_os = "windows"))] + { + format!("file://{}", path.display()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn flips_bottom_up_rgba() { + let mut pixels = vec![1, 2, 3, 4, 5, 6, 7, 8]; + flip_vertical_rgba(&mut pixels, 1, 2); + assert_eq!(pixels, vec![5, 6, 7, 8, 1, 2, 3, 4]); + } + + #[test] + fn copies_bottom_up_rgba() { + assert_eq!( + copy_bottom_up_rgba(&[1, 2, 3, 4, 5, 6, 7, 8], 1, 2), + vec![5, 6, 7, 8, 1, 2, 3, 4] + ); + } +} diff --git a/packages/accessibility-android-sys/src/emulator/screenrecord.rs b/packages/accessibility-android-sys/src/emulator/screenrecord.rs new file mode 100644 index 0000000..7e5f086 --- /dev/null +++ b/packages/accessibility-android-sys/src/emulator/screenrecord.rs @@ -0,0 +1,368 @@ +use std::process::{Child, Command, Stdio}; + +use anyhow::{Context, Result}; +use bytes::Bytes; + +use crate::AdbClient; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ScreenRecordConfig { + pub width: u32, + pub height: u32, + pub bit_rate: u32, + pub display_id: Option, +} + +impl ScreenRecordConfig { + pub fn for_max_dimension( + width: u32, + height: u32, + max_dimension: Option, + bit_rate: u32, + ) -> Self { + let (width, height) = scaled_even_size(width, height, max_dimension); + Self { + width, + height, + bit_rate, + display_id: None, + } + } +} + +pub fn spawn_screenrecord(adb: &AdbClient, config: ScreenRecordConfig) -> Result { + let mut command = Command::new(&adb.adb_path); + if let Some(serial) = &adb.serial { + command.arg("-s").arg(serial); + } + command.args([ + "exec-out", + "screenrecord", + "--output-format=h264", + "--time-limit", + "0", + "--size", + &format!("{}x{}", config.width, config.height), + "--bit-rate", + &config.bit_rate.to_string(), + ]); + if let Some(display_id) = config.display_id { + command.args(["--display-id", &display_id.to_string()]); + } + command + .arg("-") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .context("failed to start adb screenrecord") +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct H264AccessUnit { + pub data: Bytes, + pub keyframe: bool, +} + +#[derive(Default)] +pub struct AnnexBAccessUnitParser { + stream: Vec, + assembler: AccessUnitAssembler, +} + +impl AnnexBAccessUnitParser { + pub fn push(&mut self, chunk: &[u8]) -> Vec { + self.stream.extend_from_slice(chunk); + let starts = start_codes(&self.stream); + if starts.is_empty() { + if self.stream.len() > 4 { + let keep = self.stream.split_off(self.stream.len() - 4); + self.stream = keep; + } + return Vec::new(); + } + if starts[0].0 > 0 { + self.stream.drain(..starts[0].0); + } + let starts = start_codes(&self.stream); + if starts.len() < 2 { + return Vec::new(); + } + + let mut out = Vec::new(); + for pair in starts.windows(2) { + let (start, code_len) = pair[0]; + let end = pair[1].0; + if let Some(frame) = self + .assembler + .push_nalu(trim_trailing_zeroes(&self.stream[start + code_len..end])) + { + out.push(frame); + } + } + self.stream.drain(..starts.last().unwrap().0); + out + } + + pub fn flush_idle(&mut self) -> Vec { + let mut out = Vec::new(); + if let Some((start, code_len)) = start_codes(&self.stream).first().copied() { + let nalu = trim_trailing_zeroes(&self.stream[start + code_len..]); + if !nalu.is_empty() + && let Some(frame) = self.assembler.push_nalu(nalu) + { + out.push(frame); + } + } + self.stream.clear(); + if let Some(frame) = self.assembler.flush() { + out.push(frame); + } + out + } +} + +#[derive(Default)] +struct AccessUnitAssembler { + data: Vec, + has_vcl: bool, + keyframe: bool, + has_sps: bool, + has_pps: bool, + latest_sps: Option>, + latest_pps: Option>, +} + +impl AccessUnitAssembler { + fn push_nalu(&mut self, nalu: &[u8]) -> Option { + let nalu_type = nalu.first().map(|byte| byte & 0x1f)?; + let starts_new_picture = + matches!(nalu_type, 1 | 5) && self.has_vcl && first_mb_in_slice(nalu) == Some(0); + let starts_new_prefix = self.has_vcl && matches!(nalu_type, 6..=9); + let completed = (starts_new_picture || starts_new_prefix) + .then(|| self.take_frame()) + .flatten(); + + match nalu_type { + 1 | 5 => { + self.has_vcl = true; + self.keyframe |= nalu_type == 5; + } + 7 => { + self.has_sps = true; + self.latest_sps = Some(nalu.to_vec()); + } + 8 => { + self.has_pps = true; + self.latest_pps = Some(nalu.to_vec()); + } + _ => {} + } + append_nalu(&mut self.data, nalu); + completed + } + + fn flush(&mut self) -> Option { + self.take_frame() + } + + fn take_frame(&mut self) -> Option { + if !self.has_vcl { + return None; + } + let mut data = Vec::new(); + if self.keyframe { + if !self.has_sps + && let Some(sps) = &self.latest_sps + { + append_nalu(&mut data, sps); + } + if !self.has_pps + && let Some(pps) = &self.latest_pps + { + append_nalu(&mut data, pps); + } + } + data.append(&mut self.data); + let frame = H264AccessUnit { + data: Bytes::from(data), + keyframe: self.keyframe, + }; + self.has_vcl = false; + self.keyframe = false; + self.has_sps = false; + self.has_pps = false; + Some(frame) + } +} + +fn append_nalu(out: &mut Vec, nalu: &[u8]) { + out.extend_from_slice(&[0, 0, 0, 1]); + out.extend_from_slice(nalu); +} + +fn start_codes(data: &[u8]) -> Vec<(usize, usize)> { + let mut starts = Vec::new(); + let mut index = 0; + while index + 3 <= data.len() { + if index + 4 <= data.len() && data[index..index + 4] == [0, 0, 0, 1] { + starts.push((index, 4)); + index += 4; + } else if data[index..index + 3] == [0, 0, 1] { + starts.push((index, 3)); + index += 3; + } else { + index += 1; + } + } + starts +} + +fn trim_trailing_zeroes(mut data: &[u8]) -> &[u8] { + while data.last() == Some(&0) { + data = &data[..data.len() - 1]; + } + data +} + +fn first_mb_in_slice(nalu: &[u8]) -> Option { + if !matches!(nalu.first().map(|byte| byte & 0x1f), Some(1 | 5)) { + return None; + } + let mut rbsp = Vec::with_capacity(nalu.len().saturating_sub(1)); + let mut zeroes = 0; + for &byte in nalu.get(1..)? { + if zeroes >= 2 && byte == 3 { + zeroes = 0; + continue; + } + rbsp.push(byte); + if byte == 0 { + zeroes += 1; + } else { + zeroes = 0; + } + } + read_unsigned_exp_golomb(&rbsp) +} + +fn read_unsigned_exp_golomb(data: &[u8]) -> Option { + let mut bit = 0usize; + let mut leading_zeroes = 0u32; + while bit < data.len() * 8 && !read_bit(data, bit)? { + leading_zeroes += 1; + bit += 1; + if leading_zeroes >= 32 { + return None; + } + } + bit += 1; + let mut suffix = 0u32; + for _ in 0..leading_zeroes { + suffix = (suffix << 1) | u32::from(read_bit(data, bit)?); + bit += 1; + } + Some((1u32 << leading_zeroes) - 1 + suffix) +} + +fn read_bit(data: &[u8], bit: usize) -> Option { + let byte = *data.get(bit / 8)?; + Some(byte & (1 << (7 - bit % 8)) != 0) +} + +fn scaled_even_size(width: u32, height: u32, max_dimension: Option) -> (u32, u32) { + let Some(max_dimension) = max_dimension else { + return (width & !1, height & !1); + }; + let longest = width.max(height); + if longest <= max_dimension { + return (width & !1, height & !1); + } + let max_dimension = max_dimension.max(2) & !1; + if width >= height { + let scaled_height = + ((height as u64 * max_dimension as u64 / width as u64) as u32).max(2) & !1; + (max_dimension, scaled_height) + } else { + let scaled_width = + ((width as u64 * max_dimension as u64 / height as u64) as u32).max(2) & !1; + (scaled_width, max_dimension) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn annex_b(nalus: &[&[u8]]) -> Vec { + let mut out = Vec::new(); + for nalu in nalus { + append_nalu(&mut out, nalu); + } + out + } + + fn types(data: &[u8]) -> Vec { + let starts = start_codes(data); + starts + .iter() + .filter_map(|(start, code_len)| data.get(start + code_len).map(|byte| byte & 0x1f)) + .collect() + } + + #[test] + fn parses_every_byte_boundary() { + let stream = annex_b(&[ + &[0x67, 0x42, 0x00, 0x1f], + &[0x68, 0xce], + &[0x65, 0x80, 0xaa], + &[0x41, 0x80, 0xbb], + ]); + let mut parser = AnnexBAccessUnitParser::default(); + let mut frames = Vec::new(); + for byte in stream { + frames.extend(parser.push(&[byte])); + } + frames.extend(parser.flush_idle()); + assert_eq!(frames.len(), 2); + assert!(frames[0].keyframe); + assert_eq!(types(&frames[0].data), vec![7, 8, 5]); + assert!(!frames[1].keyframe); + assert_eq!(types(&frames[1].data), vec![1]); + } + + #[test] + fn keeps_multiple_slices_in_one_picture() { + let stream = annex_b(&[ + &[0x41, 0x80, 0xaa], + &[0x41, 0x40, 0xbb], + &[0x41, 0x80, 0xcc], + ]); + let mut parser = AnnexBAccessUnitParser::default(); + let mut frames = parser.push(&stream); + frames.extend(parser.flush_idle()); + assert_eq!(frames.len(), 2); + assert_eq!(types(&frames[0].data), vec![1, 1]); + assert_eq!(types(&frames[1].data), vec![1]); + } + + #[test] + fn prepends_cached_parameter_sets_to_later_keyframes() { + let stream = annex_b(&[ + &[0x67, 0x42, 0x00, 0x1f], + &[0x68, 0xce], + &[0x65, 0x80, 0xaa], + &[0x41, 0x80, 0xbb], + &[0x65, 0x80, 0xcc], + ]); + let mut parser = AnnexBAccessUnitParser::default(); + let mut frames = parser.push(&stream); + frames.extend(parser.flush_idle()); + assert_eq!(types(&frames[2].data), vec![7, 8, 5]); + } + + #[test] + fn scales_to_even_dimensions() { + assert_eq!(scaled_even_size(1080, 2424, Some(1280)), (570, 1280)); + assert_eq!(scaled_even_size(1081, 2425, None), (1080, 2424)); + } +} diff --git a/packages/accessibility-android-sys/src/lib.rs b/packages/accessibility-android-sys/src/lib.rs index 01b1013..1e2f9e3 100644 --- a/packages/accessibility-android-sys/src/lib.rs +++ b/packages/accessibility-android-sys/src/lib.rs @@ -1,5 +1,7 @@ //! Low-level ADB wrappers used by accessibility-cli's Android backend. +pub mod emulator; + use std::process::{Command, Output}; use std::time::Duration; @@ -453,6 +455,38 @@ impl AdbClient { } } + pub fn discover(serial: Option<&str>) -> Self { + for root in [ + std::env::var_os("ANDROID_SDK_ROOT"), + std::env::var_os("ANDROID_HOME"), + ] + .into_iter() + .flatten() + { + let path = std::path::PathBuf::from(root) + .join("platform-tools") + .join(adb_binary_name()); + if path.is_file() { + return Self::with_adb_path(serial, &path.to_string_lossy()); + } + } + if let Some(home) = std::env::var_os("HOME") { + let home = std::path::PathBuf::from(home); + let roots = if cfg!(target_os = "macos") { + vec![home.join("Library/Android/sdk"), home.join("Android/Sdk")] + } else { + vec![home.join("Android/Sdk")] + }; + for root in roots { + let path = root.join("platform-tools").join(adb_binary_name()); + if path.is_file() { + return Self::with_adb_path(serial, &path.to_string_lossy()); + } + } + } + Self::new(serial) + } + /// Create a new ADB client with a custom ADB path. pub fn with_adb_path(serial: Option<&str>, adb_path: &str) -> Self { Self { @@ -537,44 +571,64 @@ impl AdbClient { /// Check if ADB is available and a device is connected. pub fn check_connection(&self) -> Result<()> { - let version_result = Command::new(&self.adb_path).arg("version").output(); - - match version_result { - Ok(output) if output.status.success() => {} - Ok(_) => bail!("ADB binary found but returned error"), - Err(e) => bail!( - "ADB binary not found at '{}': {}. Install Android SDK Platform Tools.", - self.adb_path, - e - ), + let devices = self.connected_devices()?; + if devices.is_empty() { + bail!("No Android devices connected. Connect a device or start an emulator."); + } + if let Some(serial) = &self.serial + && !devices.contains(serial) + { + bail!( + "Device '{}' is not connected. Available devices: {}", + serial, + devices.join(", ") + ); } + Ok(()) + } - let devices = self.command(&["devices"])?; - let device_count = devices + pub fn connected_devices(&self) -> Result> { + let output = Command::new(&self.adb_path) + .arg("devices") + .output() + .with_context(|| { + format!( + "ADB binary not found at '{}'. Install Android SDK Platform Tools.", + self.adb_path + ) + })?; + Self::check_output(&output, "devices")?; + let stdout = String::from_utf8_lossy(&output.stdout); + Ok(stdout .lines() .skip(1) - .filter(|line| { - let trimmed = line.trim(); - !trimmed.is_empty() && trimmed.contains('\t') + .filter_map(|line| { + let (serial, state) = line.split_once('\t')?; + (state.split_whitespace().next() == Some("device")).then(|| serial.to_string()) }) - .count(); - - if device_count == 0 { - bail!("No Android devices connected. Connect a device or start an emulator."); - } + .collect()) + } - if let Some(ref serial) = self.serial { - let found = devices.lines().skip(1).any(|line| line.starts_with(serial)); - if !found { - bail!( - "Device '{}' not found. Available devices:\n{}", - serial, - devices - ); + pub fn resolved_serial(&self) -> Result { + let devices = self.connected_devices()?; + if let Some(serial) = &self.serial { + if devices.contains(serial) { + return Ok(serial.clone()); } + bail!( + "Device '{}' is not connected. Available devices: {}", + serial, + devices.join(", ") + ); + } + match devices.as_slice() { + [serial] => Ok(serial.clone()), + [] => bail!("No Android devices connected. Connect a device or start an emulator."), + _ => bail!( + "Multiple Android devices are connected; specify one of: {}", + devices.join(", ") + ), } - - Ok(()) } /// Get the screen size in pixels. @@ -756,6 +810,14 @@ impl AdbClient { } } +fn adb_binary_name() -> &'static str { + if cfg!(target_os = "windows") { + "adb.exe" + } else { + "adb" + } +} + /// Escape text for ADB shell input command. pub fn escape_shell_text(text: &str) -> String { let mut result = String::with_capacity(text.len() * 2); diff --git a/packages/accessibility-cli/Cargo.toml b/packages/accessibility-cli/Cargo.toml index d1cf5ea..b9bb3b5 100644 --- a/packages/accessibility-cli/Cargo.toml +++ b/packages/accessibility-cli/Cargo.toml @@ -9,6 +9,7 @@ homepage.workspace = true [dependencies] accessibility-core.workspace = true +accessibility-serve.workspace = true anyhow.workspace = true clap.workspace = true ctrlc.workspace = true @@ -23,6 +24,3 @@ serial_test.workspace = true [[bin]] name = "accessibility-cli" path = "src/main.rs" - -[target.'cfg(target_os = "macos")'.dependencies] -accessibility-serve.workspace = true diff --git a/packages/accessibility-cli/src/lib.rs b/packages/accessibility-cli/src/lib.rs index 21cd5c5..2bf75f7 100644 --- a/packages/accessibility-cli/src/lib.rs +++ b/packages/accessibility-cli/src/lib.rs @@ -1852,6 +1852,10 @@ pub enum Command { /// Serve the iOS Simulator in a browser: live video, input, and element /// inspection. ServeSim(ServeSimArgs), + #[command( + about = "Serve an Android Emulator in a browser with live video, input, and element inspection" + )] + ServeEmulator(ServeEmulatorArgs), } #[derive(clap::Args)] @@ -1912,6 +1916,45 @@ pub struct ServeSimArgs { pub ice_servers: Vec, } +#[derive(clap::Args)] +pub struct ServeEmulatorArgs { + #[arg( + long, + help = "Emulator serial from adb devices; defaults to the only connected device" + )] + pub serial: Option, + #[arg(long, default_value_t = 3200, help = "Port to listen on")] + pub port: u16, + #[arg(long, default_value = "127.0.0.1", help = "Address to bind")] + pub bind: std::net::IpAddr, + #[arg( + long, + default_value = "webrtc", + help = "Preferred transport: webrtc or h264" + )] + pub transport: String, + #[arg( + long, + default_value_t = 60, + help = "Frame rate used to derive the bitrate" + )] + pub fps: u32, + #[arg(long, help = "screenrecord bitrate in bits per second")] + pub bitrate: Option, + #[arg(long, default_value_t = 1280, help = "Longest encoded edge")] + pub max_dimension: u32, + #[arg(long, help = "Capture at native display resolution")] + pub native_resolution: bool, + #[arg( + long, + default_value_t = 2, + help = "Requested keyframe recovery interval" + )] + pub keyframe_interval: u32, + #[arg(long = "ice-server", help = "ICE server URL, repeatable")] + pub ice_servers: Vec, +} + /// Run the CLI using process arguments. pub fn run() { let cli = Cli::parse(); @@ -1934,6 +1977,7 @@ pub fn run() { async fn run_command(command: &Command) { let result = match command { Command::ServeSim(args) => run_serve_sim(args).await, + Command::ServeEmulator(args) => run_serve_emulator(args).await, }; if let Err(error) = result { eprintln!("Error: {error:#}"); @@ -1981,6 +2025,30 @@ async fn run_serve_sim(_args: &ServeSimArgs) -> anyhow::Result<()> { anyhow::bail!("serve-sim requires macOS with Xcode and a booted iOS Simulator") } +async fn run_serve_emulator(args: &ServeEmulatorArgs) -> anyhow::Result<()> { + use accessibility_core::video::{NalFormat, Tuning, VideoConfig}; + use accessibility_serve::{ServeEmulatorConfig, Transport, serve_emulator}; + + let transport: Transport = args.transport.parse()?; + serve_emulator(ServeEmulatorConfig { + serial: args.serial.clone(), + address: std::net::SocketAddr::new(args.bind, args.port), + transport, + video: VideoConfig { + fps: args.fps, + tuning: Tuning::Interactive { + bitrate: args.bitrate, + }, + max_dimension: (!args.native_resolution).then_some(args.max_dimension), + keyframe_interval_secs: args.keyframe_interval, + nal_format: NalFormat::AnnexB, + ..Default::default() + }, + ice_servers: args.ice_servers.clone(), + }) + .await +} + /// Build a TreeFilter from CommonArgs fn build_filter(common: &CommonArgs) -> TreeFilter { TreeFilter { diff --git a/packages/accessibility-core/examples/android_session_probe.rs b/packages/accessibility-core/examples/android_session_probe.rs new file mode 100644 index 0000000..13eb895 --- /dev/null +++ b/packages/accessibility-core/examples/android_session_probe.rs @@ -0,0 +1,123 @@ +use std::time::{Duration, Instant}; + +use accessibility_core::platform::android::AdbClient; +use accessibility_core::platform::android::input::{ + HardwareButton, InputCommand, Orientation, TouchPhase, +}; +use accessibility_core::platform::android::session::EmulatorSession; +use accessibility_core::video::{FrameKind, VideoConfig}; +use anyhow::{Context, Result}; + +#[tokio::main] +async fn main() -> Result<()> { + let requested_serial = std::env::args().nth(1); + let adb = AdbClient::discover(requested_serial.as_deref()); + let serial = adb.resolved_serial()?; + let _ = adb.stop_app("com.google.android.settings.intelligence"); + let _ = adb.stop_app("com.android.settings"); + adb.launch_app("com.android.settings", Some(".Settings"))?; + tokio::time::sleep(Duration::from_millis(500)).await; + let session = EmulatorSession::start(Some(&serial), VideoConfig::default())?; + session.seed_orientation().await; + let device = session.device_info(); + println!("device : {}", device.serial); + println!("geometry : {}x{}", device.width, device.height); + println!("orientation: {:?}", device.orientation); + + let snapshot = session.ax_snapshot(false).await?; + println!("app : {:?}", snapshot.app_name); + println!("elements : {}", snapshot.elements.len()); + println!("coverage : {:.1}%", snapshot.coverage * 100.0); + if let Some(element) = session.ax_hit_test(0.5, 0.5).await? { + println!("center : {} {}", element.role, element.selector); + } + let tappable = snapshot + .elements + .iter() + .find(|element| { + element.actions.iter().any(|action| action == "click") && element.bounds.is_some() + }) + .context("Settings tree has no tappable element")?; + let bounds = tappable.bounds.unwrap(); + let tap_x = bounds.x + bounds.width / 2.0; + let tap_y = bounds.y + bounds.height / 2.0; + println!("tap : {}", tappable.selector); + session.send_input(InputCommand::Touch { + phase: TouchPhase::Begin, + x: tap_x, + y: tap_y, + }); + session.send_input(InputCommand::Touch { + phase: TouchPhase::End, + x: tap_x, + y: tap_y, + }); + tokio::time::sleep(Duration::from_millis(700)).await; + let tapped = session.ax_snapshot(false).await?; + if tapped + .elements + .iter() + .map(|element| &element.selector) + .eq(snapshot.elements.iter().map(|element| &element.selector)) + { + anyhow::bail!("gRPC touch input did not change the Settings tree"); + } + + session.send_input(InputCommand::Button { + button: HardwareButton::Home, + }); + tokio::time::sleep(Duration::from_millis(500)).await; + let home = session.ax_snapshot(false).await?; + if home.app_name == tapped.app_name { + anyhow::bail!("gRPC Home input did not leave the tapped Settings surface"); + } + println!("after Home : {:?}", home.app_name); + let mut frames = session.subscribe(); + let started = Instant::now(); + let mut received = 0; + let mut keyframes = 0; + let mut bytes = 0; + while started.elapsed() < Duration::from_secs(5) { + let frame = match tokio::time::timeout(Duration::from_secs(2), frames.recv()).await { + Ok(frame) => frame?, + Err(_) if received > 0 => break, + Err(error) => { + return Err(error).context("timed out waiting for an Android Emulator frame"); + } + }; + received += 1; + bytes += frame.data.len(); + if frame.kind == FrameKind::Keyframe { + keyframes += 1; + } + if received >= 30 { + break; + } + } + let stats = session.stats(); + println!("received : {received} frames, {keyframes} key, {bytes} bytes"); + println!("session : {:.1} fps, {:.2} Mbps", stats.fps, stats.mbps); + + session.set_orientation(Orientation::LandscapeRight)?; + let landscape = session.ax_snapshot(false).await?; + let landscape_stats = session.stats(); + if !landscape.is_landscape || landscape_stats.encoded_width <= landscape_stats.encoded_height { + anyhow::bail!("Android Emulator did not reconfigure for landscape"); + } + println!( + "landscape : {}x{}", + landscape_stats.encoded_width, landscape_stats.encoded_height + ); + session.set_orientation(Orientation::Portrait)?; + let portrait = session.ax_snapshot(false).await?; + let portrait_stats = session.stats(); + if portrait.is_landscape || portrait_stats.encoded_width >= portrait_stats.encoded_height { + anyhow::bail!("Android Emulator did not reconfigure for portrait"); + } + println!( + "portrait : {}x{}", + portrait_stats.encoded_width, portrait_stats.encoded_height + ); + println!("probe passed"); + Ok(()) +} diff --git a/packages/accessibility-core/src/platform/android.rs b/packages/accessibility-core/src/platform/android.rs index d26a195..8d4062a 100644 --- a/packages/accessibility-core/src/platform/android.rs +++ b/packages/accessibility-core/src/platform/android.rs @@ -56,8 +56,15 @@ use crate::accessibility::{ TreeFilter, }; use crate::input::{Code, Modifiers, MouseButton}; +use crate::video::{FrameSink, ScreenGeometry, VideoCapture, VideoConfig}; +pub mod ax; +pub mod input; +pub mod session; +pub mod video; pub use accessibility_android_sys::{AdbClient, AndroidKeyCode}; +pub use input::{HardwareButton, InputCommand, Orientation, TouchPhase, spawn_input_worker}; +pub use video::AndroidVideoCapture; /// Parse Android bounds string like "[0,0][1080,1920]" into a Rect. fn parse_bounds(bounds_str: &str) -> Option { @@ -831,6 +838,27 @@ impl AccessibilityReader for AndroidAccessibility { true } + fn start_video_capture( + &self, + config: &VideoConfig, + sink: FrameSink, + ) -> Result> { + let (width, height) = self + .screen_size + .ok_or_else(|| anyhow!("Android screen size is unavailable"))?; + let capture = AndroidVideoCapture::start( + self.adb.clone(), + ScreenGeometry { width, height }, + config, + sink, + )?; + Ok(Box::new(capture)) + } + + fn supports_video_capture(&self) -> bool { + true + } + fn supports_terminal_display(&self) -> bool { true } diff --git a/packages/accessibility-core/src/platform/android/ax.rs b/packages/accessibility-core/src/platform/android/ax.rs new file mode 100644 index 0000000..5c975d9 --- /dev/null +++ b/packages/accessibility-core/src/platform/android/ax.rs @@ -0,0 +1,271 @@ +use anyhow::{Result, anyhow}; +use serde::Serialize; +use tokio::sync::{mpsc, oneshot}; + +use crate::accessibility::{ + AccessibilityReader, AndroidTarget, Element, ElementTree, Point, Rect, Size, Target, TreeFilter, +}; + +use super::AndroidAccessibility; + +const COVERAGE_COLUMNS: usize = 32; +const COVERAGE_ROWS: usize = 64; +const BACKDROP_AREA: f64 = 0.9; + +#[derive(Debug, Clone, Copy, Serialize)] +pub struct NormalizedRect { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} + +impl NormalizedRect { + fn from_screen(rect: &Rect, screen: &Rect) -> Option { + if screen.size.width <= 0.0 || screen.size.height <= 0.0 { + return None; + } + Some(Self { + x: (rect.origin.x - screen.origin.x) / screen.size.width, + y: (rect.origin.y - screen.origin.y) / screen.size.height, + width: rect.size.width / screen.size.width, + height: rect.size.height / screen.size.height, + }) + } + + fn is_backdrop(self) -> bool { + self.width * self.height >= BACKDROP_AREA + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Discovery { + Recursive, + PointGrid, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ElementDetail { + pub id: String, + pub role: String, + pub label: Option, + pub value: Option, + pub identifier: Option, + pub enabled: bool, + pub focused: bool, + pub actions: Vec, + pub bounds: Option, + pub depth: u32, + pub selector: String, + pub discovery: Discovery, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AxSnapshot { + pub app_name: Option, + pub pid: Option, + pub elements: Vec, + pub coverage: f64, + pub coverage_after_scan: Option, + pub probes: Option, + pub is_landscape: bool, +} + +pub enum AxCommand { + Snapshot { + scan: bool, + reply: oneshot::Sender>, + }, + HitTest { + x: f64, + y: f64, + reply: oneshot::Sender>>, + }, +} + +pub fn spawn_ax_worker(serial: &str) -> Result> { + let adb = super::AdbClient::discover(Some(serial)); + let mut reader = AndroidAccessibility::with_adb_path(Some(serial), &adb.adb_path)?; + let target = Target::Android(AndroidTarget::Serial(serial.to_string())); + let (commands, mut command_rx) = mpsc::unbounded_channel(); + std::thread::Builder::new() + .name("android-emulator-ax".into()) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build(); + let Ok(runtime) = runtime else { return }; + let mut screen = None; + while let Some(command) = command_rx.blocking_recv() { + match command { + AxCommand::Snapshot { scan, reply } => { + let result = runtime.block_on(snapshot(&mut reader, &target, scan)); + if let Ok((_, bounds)) = &result { + screen = Some(*bounds); + } + let _ = reply.send(result.map(|(snapshot, _)| snapshot)); + } + AxCommand::HitTest { x, y, reply } => { + let result = runtime.block_on(hit_test(&mut reader, screen.as_ref(), x, y)); + let _ = reply.send(result); + } + } + } + })?; + Ok(commands) +} + +async fn snapshot( + reader: &mut AndroidAccessibility, + target: &Target, + _scan: bool, +) -> Result<(AxSnapshot, Rect)> { + let tree = reader.get_tree(target, &TreeFilter::default()).await?; + let screen = + screen_bounds(&tree).ok_or_else(|| anyhow!("Android tree has no screen bounds"))?; + let mut elements = Vec::with_capacity(tree.element_count); + flatten(&tree.root, &screen, 0, &mut elements); + elements.retain(|element| { + element.bounds.is_some_and(|bounds| { + !bounds.is_backdrop() && bounds.width > 0.0 && bounds.height > 0.0 + }) + }); + let coverage = coverage(&elements); + Ok(( + AxSnapshot { + app_name: tree.app_name, + pid: tree.pid, + elements, + coverage, + coverage_after_scan: None, + probes: None, + is_landscape: screen.size.width > screen.size.height, + }, + screen, + )) +} + +async fn hit_test( + reader: &mut AndroidAccessibility, + screen: Option<&Rect>, + x: f64, + y: f64, +) -> Result> { + let screen = + screen.ok_or_else(|| anyhow!("no accessibility snapshot yet; fetch the tree first"))?; + let point_x = screen.origin.x + x * screen.size.width; + let point_y = screen.origin.y + y * screen.size.height; + let Some(id) = reader.hit_test(point_x, point_y).await? else { + return Ok(None); + }; + Ok(reader + .get_element(id) + .map(|element| to_detail(element, screen, 0))) +} + +fn screen_bounds(tree: &ElementTree) -> Option { + let mut max_x = 0.0f64; + let mut max_y = 0.0f64; + let mut stack = vec![&tree.root]; + while let Some(element) = stack.pop() { + if let Some(bounds) = &element.bounds { + max_x = max_x.max(bounds.origin.x + bounds.size.width); + max_y = max_y.max(bounds.origin.y + bounds.size.height); + } + stack.extend(element.children.iter()); + } + (max_x > 0.0 && max_y > 0.0).then(|| Rect::new(Point::new(0.0, 0.0), Size::new(max_x, max_y))) +} + +fn flatten(element: &Element, screen: &Rect, depth: u32, out: &mut Vec) { + out.push(to_detail(element, screen, depth)); + for child in &element.children { + flatten(child, screen, depth + 1, out); + } +} + +fn to_detail(element: &Element, screen: &Rect, depth: u32) -> ElementDetail { + let role = format!("{:?}", element.role); + ElementDetail { + id: element.id.to_string(), + role: role.clone(), + label: element.title.clone().filter(|value| !value.is_empty()), + value: element.value.clone().filter(|value| !value.is_empty()), + identifier: element.identifier.clone().filter(|value| !value.is_empty()), + enabled: element.enabled, + focused: element.focused, + actions: element.actions.clone(), + bounds: element + .bounds + .as_ref() + .and_then(|bounds| NormalizedRect::from_screen(bounds, screen)), + depth, + selector: selector_for(element, &role), + discovery: Discovery::Recursive, + } +} + +fn selector_for(element: &Element, role: &str) -> String { + if let Some(identifier) = element + .identifier + .as_deref() + .filter(|value| !value.is_empty()) + { + return format!("#{identifier}"); + } + if let Some(label) = element.title.as_deref().filter(|value| !value.is_empty()) { + return format!("{role}[label=\"{}\"]", label.replace('"', "\\\"")); + } + role.to_string() +} + +fn coverage(elements: &[ElementDetail]) -> f64 { + let mut cells = [false; COVERAGE_COLUMNS * COVERAGE_ROWS]; + for bounds in elements.iter().filter_map(|element| element.bounds) { + let left = (bounds.x.clamp(0.0, 1.0) * COVERAGE_COLUMNS as f64).floor() as usize; + let right = (((bounds.x + bounds.width).clamp(0.0, 1.0) * COVERAGE_COLUMNS as f64).ceil() + as usize) + .min(COVERAGE_COLUMNS); + let top = (bounds.y.clamp(0.0, 1.0) * COVERAGE_ROWS as f64).floor() as usize; + let bottom = (((bounds.y + bounds.height).clamp(0.0, 1.0) * COVERAGE_ROWS as f64).ceil() + as usize) + .min(COVERAGE_ROWS); + for row in top..bottom { + for column in left..right { + cells[row * COVERAGE_COLUMNS + column] = true; + } + } + } + cells.iter().filter(|filled| **filled).count() as f64 / cells.len() as f64 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::accessibility::Element; + use accesskit::Role; + + #[test] + fn normalizes_android_pixel_bounds() { + let screen = Rect::new(Point::new(0.0, 0.0), Size::new(1080.0, 2400.0)); + let bounds = Rect::new(Point::new(108.0, 240.0), Size::new(540.0, 1200.0)); + let normalized = NormalizedRect::from_screen(&bounds, &screen).unwrap(); + assert_eq!(normalized.x, 0.1); + assert_eq!(normalized.y, 0.1); + assert_eq!(normalized.width, 0.5); + assert_eq!(normalized.height, 0.5); + } + + #[test] + fn coverage_counts_element_area() { + let mut element = Element::new(crate::accessibility::ElementKey::default(), Role::Button); + element.bounds = Some(Rect::new(Point::new(0.0, 0.0), Size::new(100.0, 100.0))); + let detail = to_detail( + &element, + &Rect::new(Point::new(0.0, 0.0), Size::new(200.0, 200.0)), + 0, + ); + assert_eq!(coverage(&[detail]), 0.25); + } +} diff --git a/packages/accessibility-core/src/platform/android/input.rs b/packages/accessibility-core/src/platform/android/input.rs new file mode 100644 index 0000000..9be2b6c --- /dev/null +++ b/packages/accessibility-core/src/platform/android/input.rs @@ -0,0 +1,332 @@ +use std::sync::mpsc; + +use accessibility_android_sys::emulator::protocol::controller::input_event; +use accessibility_android_sys::emulator::protocol::controller::keyboard_event::{ + KeyCodeType, KeyEventType, +}; +use accessibility_android_sys::emulator::protocol::controller::{ + InputEvent, KeyboardEvent, Touch, TouchEvent, +}; +use accessibility_android_sys::emulator::{EmulatorGrpcClient, discover_emulator}; +use accessibility_android_sys::{AdbClient, AndroidKeyCode}; +use anyhow::{Result, anyhow, bail}; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc::UnboundedSender; + +use crate::video::ScreenGeometry; + +const ACTIVE_PRESSURE: i32 = 0x7fff; +const TOUCH_SIZE: i32 = 8; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum TouchPhase { + Begin, + Move, + End, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HardwareButton { + Home, + Back, + Lock, + AppSwitch, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Orientation { + Portrait, + PortraitUpsideDown, + LandscapeLeft, + LandscapeRight, +} + +impl Orientation { + pub fn is_landscape(self) -> bool { + matches!(self, Self::LandscapeLeft | Self::LandscapeRight) + } + + fn android_rotation(self) -> u8 { + match self { + Self::Portrait => 0, + Self::LandscapeLeft => 1, + Self::PortraitUpsideDown => 2, + Self::LandscapeRight => 3, + } + } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum InputCommand { + Touch { + phase: TouchPhase, + x: f64, + y: f64, + }, + Key { + key_code: u32, + #[serde(default)] + modifiers: Vec, + }, + Text { + text: String, + }, + Scroll { + dx: f64, + dy: f64, + x: f64, + y: f64, + }, + Button { + button: HardwareButton, + }, + Rotate { + orientation: Orientation, + }, +} + +pub fn spawn_input_worker( + serial: &str, + geometry: ScreenGeometry, +) -> Result> { + let discovery = discover_emulator(Some(serial))?; + let adb = AdbClient::discover(Some(serial)); + let (commands, mut command_rx) = tokio::sync::mpsc::unbounded_channel(); + let (ready_tx, ready_rx) = mpsc::sync_channel(1); + std::thread::Builder::new() + .name("android-emulator-input".into()) + .spawn(move || { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = ready_tx.send(Err(error.to_string())); + return; + } + }; + runtime.block_on(async move { + let mut client = match EmulatorGrpcClient::connect(discovery).await { + Ok(client) => client, + Err(error) => { + let _ = ready_tx.send(Err(error.to_string())); + return; + } + }; + let _ = ready_tx.send(Ok(())); + while let Some(command) = command_rx.recv().await { + let command = match command { + InputCommand::Rotate { orientation } => { + let _ = set_device_orientation(&adb, orientation); + continue; + } + InputCommand::Button { button } => { + apply_hardware_button(&adb, button); + continue; + } + command => command, + }; + for event in to_events(command, geometry) { + if let Err(error) = client.send_input(event).await { + eprintln!("Android Emulator input failed: {error:#}"); + return; + } + } + } + }); + })?; + match ready_rx.recv() { + Ok(Ok(())) => Ok(commands), + Ok(Err(error)) => Err(anyhow!(error)), + Err(_) => Err(anyhow!( + "Android Emulator input worker stopped during startup" + )), + } +} + +fn to_events(command: InputCommand, geometry: ScreenGeometry) -> Vec { + match command { + InputCommand::Touch { phase, x, y } => vec![touch_event(phase, x, y, geometry)], + InputCommand::Key { + key_code, + modifiers, + } => { + let mut events = Vec::with_capacity(modifiers.len() * 2 + 2); + events.extend( + modifiers + .iter() + .map(|modifier| usb_key(*modifier, KeyEventType::Keydown)), + ); + events.push(usb_key(key_code, KeyEventType::Keydown)); + events.push(usb_key(key_code, KeyEventType::Keyup)); + events.extend( + modifiers + .iter() + .rev() + .map(|modifier| usb_key(*modifier, KeyEventType::Keyup)), + ); + events + } + InputCommand::Text { text } => vec![keyboard_event(KeyboardEvent { + text, + ..Default::default() + })], + InputCommand::Scroll { dx, dy, x, y } => { + let end_x = (x - dx).clamp(0.0, 1.0); + let end_y = (y - dy).clamp(0.0, 1.0); + vec![ + touch_event(TouchPhase::Begin, x, y, geometry), + touch_event(TouchPhase::Move, end_x, end_y, geometry), + touch_event(TouchPhase::End, end_x, end_y, geometry), + ] + } + InputCommand::Button { .. } => Vec::new(), + InputCommand::Rotate { .. } => Vec::new(), + } +} + +fn touch_event(phase: TouchPhase, x: f64, y: f64, geometry: ScreenGeometry) -> InputEvent { + InputEvent { + r#type: Some(input_event::Type::TouchEvent(TouchEvent { + touches: vec![Touch { + x: normalized_coordinate(x, geometry.width), + y: normalized_coordinate(y, geometry.height), + identifier: 0, + pressure: if phase == TouchPhase::End { + 0 + } else { + ACTIVE_PRESSURE + }, + touch_major: TOUCH_SIZE, + touch_minor: TOUCH_SIZE, + expiration: 1, + orientation: 0, + }], + display: 0, + })), + } +} + +fn usb_key(key_code: u32, event_type: KeyEventType) -> InputEvent { + let key_code = if key_code <= 0xffff { + 0x070000 | key_code + } else { + key_code + }; + keyboard_event(KeyboardEvent { + code_type: KeyCodeType::Usb as i32, + event_type: event_type as i32, + key_code: key_code as i32, + ..Default::default() + }) +} + +fn keyboard_event(event: KeyboardEvent) -> InputEvent { + InputEvent { + r#type: Some(input_event::Type::KeyEvent(event)), + } +} + +fn normalized_coordinate(value: f64, dimension: u32) -> i32 { + (value.clamp(0.0, 1.0) * dimension.saturating_sub(1) as f64).round() as i32 +} + +fn apply_hardware_button(adb: &AdbClient, button: HardwareButton) { + let key = match button { + HardwareButton::Home => AndroidKeyCode::Home, + HardwareButton::Back => AndroidKeyCode::Back, + HardwareButton::Lock => AndroidKeyCode::Power, + HardwareButton::AppSwitch => AndroidKeyCode::AppSwitch, + }; + let _ = adb.key_event(key as u32); +} + +pub fn set_device_orientation(adb: &AdbClient, orientation: Orientation) -> Result<()> { + adb.shell(&["wm", "fixed-to-user-rotation", "enabled"])?; + let target = orientation.android_rotation(); + adb.shell(&["wm", "user-rotation", "lock", &target.to_string()])?; + for _ in 0..20 { + let output = adb.shell(&["dumpsys", "display"])?; + if display_rotation(&output) == Some(target) { + return Ok(()); + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + bail!("Android display did not reach rotation {target}") +} + +fn display_rotation(output: &str) -> Option { + output.lines().find_map(|line| { + line.trim() + .strip_prefix("mCurrentOrientation=")? + .parse() + .ok() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn geometry() -> ScreenGeometry { + ScreenGeometry { + width: 1080, + height: 2424, + } + } + + #[test] + fn touch_maps_normalized_coordinates_and_pressure() { + let events = to_events( + InputCommand::Touch { + phase: TouchPhase::Begin, + x: 0.5, + y: 1.0, + }, + geometry(), + ); + let Some(input_event::Type::TouchEvent(event)) = &events[0].r#type else { + panic!("expected touch event"); + }; + assert_eq!(event.touches[0].x, 540); + assert_eq!(event.touches[0].y, 2423); + assert_eq!(event.touches[0].pressure, ACTIVE_PRESSURE); + } + + #[test] + fn parses_display_rotation() { + assert_eq!( + display_rotation("other\n mCurrentOrientation=3\nmore"), + Some(3) + ); + } + + #[test] + fn modifiers_are_held_around_key() { + let events = to_events( + InputCommand::Key { + key_code: 4, + modifiers: vec![225], + }, + geometry(), + ); + let kinds = events + .iter() + .map(|event| { + let Some(input_event::Type::KeyEvent(key)) = &event.r#type else { + panic!("expected key event"); + }; + (key.key_code, key.event_type) + }) + .collect::>(); + assert_eq!( + kinds, + vec![(0x0700e1, 0), (0x070004, 0), (0x070004, 1), (0x0700e1, 1),] + ); + } +} diff --git a/packages/accessibility-core/src/platform/android/session.rs b/packages/accessibility-core/src/platform/android/session.rs new file mode 100644 index 0000000..cac3eda --- /dev/null +++ b/packages/accessibility-core/src/platform/android/session.rs @@ -0,0 +1,233 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; + +use anyhow::{Result, anyhow, bail}; +use tokio::sync::{broadcast, mpsc, oneshot}; + +use crate::video::{EncodedFrame, FrameKind, ScreenGeometry, VideoCapture, VideoConfig}; + +use super::ax::{AxCommand, AxSnapshot, ElementDetail, spawn_ax_worker}; +use super::input::{InputCommand, Orientation, set_device_orientation, spawn_input_worker}; +use super::{AdbClient, AndroidVideoCapture}; + +const FRAME_BUFFER: usize = 16; + +#[derive(Default)] +pub struct StreamStats { + pub frames: AtomicU64, + pub keyframes: AtomicU64, + pub bytes: AtomicU64, + pub keyframe_requests: AtomicU64, + pub lag_events: AtomicU64, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct StatsReport { + pub uptime_secs: f64, + pub frames: u64, + pub keyframes: u64, + pub bytes: u64, + pub fps: f64, + pub mbps: f64, + pub bits_per_pixel: f64, + pub mean_frame_kb: f64, + pub keyframe_requests: u64, + pub lag_events: u64, + pub subscribers: usize, + pub recording_frames: Option, + pub width: u32, + pub height: u32, + pub encoded_width: u32, + pub encoded_height: u32, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct DeviceInfo { + pub serial: String, + pub width: u32, + pub height: u32, + pub orientation: Orientation, +} + +pub struct EmulatorSession { + serial: String, + adb: AdbClient, + capture: AndroidVideoCapture, + frames: broadcast::Sender, + stats: Arc, + started: Instant, + input: tokio::sync::mpsc::UnboundedSender, + ax: mpsc::UnboundedSender, + orientation: std::sync::Mutex, +} + +impl EmulatorSession { + pub fn start(serial: Option<&str>, config: VideoConfig) -> Result> { + let adb = AdbClient::discover(serial); + let serial = adb.resolved_serial()?; + if !serial.starts_with("emulator-") { + bail!("Android Emulator streaming requires an emulator serial, got '{serial}'"); + } + let adb = AdbClient::discover(Some(&serial)); + let (width, height) = adb.get_screen_size()?; + let geometry = ScreenGeometry { width, height }; + let (frames, _) = broadcast::channel(FRAME_BUFFER); + let stats = Arc::new(StreamStats::default()); + let sink = { + let frames = frames.clone(); + let stats = Arc::clone(&stats); + Arc::new(move |frame: EncodedFrame| { + stats.frames.fetch_add(1, Ordering::Relaxed); + stats + .bytes + .fetch_add(frame.data.len() as u64, Ordering::Relaxed); + if frame.kind == FrameKind::Keyframe { + stats.keyframes.fetch_add(1, Ordering::Relaxed); + } + let _ = frames.send(frame); + }) + }; + let capture = AndroidVideoCapture::start(adb.clone(), geometry, &config, sink)?; + let input = spawn_input_worker(&serial, geometry)?; + let ax = spawn_ax_worker(&serial)?; + Ok(Arc::new(Self { + serial, + adb, + capture, + frames, + stats, + started: Instant::now(), + input, + ax, + orientation: std::sync::Mutex::new(Orientation::Portrait), + })) + } + + pub fn subscribe(&self) -> broadcast::Receiver { + let receiver = self.frames.subscribe(); + self.capture.request_keyframe(); + receiver + } + + pub fn request_keyframe(&self) { + self.stats.keyframe_requests.fetch_add(1, Ordering::Relaxed); + self.capture.request_keyframe(); + } + + pub fn note_lag(&self) { + self.stats.lag_events.fetch_add(1, Ordering::Relaxed); + } + + pub fn device_info(&self) -> DeviceInfo { + let geometry = self.capture.geometry(); + DeviceInfo { + serial: self.serial.clone(), + width: geometry.width, + height: geometry.height, + orientation: self.orientation(), + } + } + + pub fn orientation(&self) -> Orientation { + *self.orientation.lock().unwrap() + } + + pub fn set_orientation(&self, orientation: Orientation) -> Result<()> { + set_device_orientation(&self.adb, orientation)?; + self.capture.set_landscape(orientation.is_landscape())?; + *self.orientation.lock().unwrap() = orientation; + Ok(()) + } + + pub fn send_input(&self, command: InputCommand) { + if let InputCommand::Rotate { orientation } = command { + let _ = self.set_orientation(orientation); + return; + } + let _ = self.input.send(command); + } + + pub fn stats(&self) -> StatsReport { + let elapsed = self.started.elapsed().as_secs_f64().max(1e-6); + let frames = self.stats.frames.load(Ordering::Relaxed); + let bytes = self.stats.bytes.load(Ordering::Relaxed); + let geometry = self.capture.geometry(); + let encoded = self.capture.encoded_geometry(); + let pixels = encoded.width as f64 * encoded.height as f64; + let fps = frames as f64 / elapsed; + StatsReport { + uptime_secs: (elapsed * 10.0).round() / 10.0, + frames, + keyframes: self.stats.keyframes.load(Ordering::Relaxed), + bytes, + fps: (fps * 10.0).round() / 10.0, + mbps: ((bytes as f64 * 8.0 / elapsed / 1e6) * 100.0).round() / 100.0, + bits_per_pixel: if pixels > 0.0 && fps > 0.0 { + ((bytes as f64 * 8.0 / elapsed) / (pixels * fps) * 10000.0).round() / 10000.0 + } else { + 0.0 + }, + mean_frame_kb: if frames > 0 { + ((bytes as f64 / frames as f64 / 1024.0) * 100.0).round() / 100.0 + } else { + 0.0 + }, + keyframe_requests: self.stats.keyframe_requests.load(Ordering::Relaxed), + lag_events: self.stats.lag_events.load(Ordering::Relaxed), + subscribers: self.frames.receiver_count(), + recording_frames: None, + width: geometry.width, + height: geometry.height, + encoded_width: encoded.width, + encoded_height: encoded.height, + } + } + + pub async fn ax_snapshot(&self, scan: bool) -> Result { + let (reply, response) = oneshot::channel(); + self.ax + .send(AxCommand::Snapshot { scan, reply }) + .map_err(|_| anyhow!("Android accessibility worker stopped"))?; + let snapshot = response + .await + .map_err(|_| anyhow!("Android accessibility worker stopped"))??; + self.reconcile_orientation(snapshot.is_landscape); + Ok(snapshot) + } + + pub async fn ax_hit_test(&self, x: f64, y: f64) -> Result> { + let (reply, response) = oneshot::channel(); + self.ax + .send(AxCommand::HitTest { x, y, reply }) + .map_err(|_| anyhow!("Android accessibility worker stopped"))?; + response + .await + .map_err(|_| anyhow!("Android accessibility worker stopped"))? + } + + pub async fn seed_orientation(&self) { + if let Ok(snapshot) = self.ax_snapshot(false).await { + self.reconcile_orientation(snapshot.is_landscape); + } + } + + fn reconcile_orientation(&self, is_landscape: bool) { + let changed = { + let mut orientation = self.orientation.lock().unwrap(); + if orientation.is_landscape() == is_landscape { + false + } else { + *orientation = if is_landscape { + Orientation::LandscapeLeft + } else { + Orientation::Portrait + }; + true + } + }; + if changed { + let _ = self.capture.set_landscape(is_landscape); + } + } +} diff --git a/packages/accessibility-core/src/platform/android/video.rs b/packages/accessibility-core/src/platform/android/video.rs new file mode 100644 index 0000000..ca71f7f --- /dev/null +++ b/packages/accessibility-core/src/platform/android/video.rs @@ -0,0 +1,277 @@ +use std::io::Read; +use std::sync::mpsc::{self, Receiver, SyncSender}; +use std::thread::JoinHandle; +use std::time::Duration; + +use accessibility_android_sys::AdbClient; +use accessibility_android_sys::emulator::screenrecord::{ + AnnexBAccessUnitParser, ScreenRecordConfig, spawn_screenrecord, +}; +use anyhow::{Result, anyhow, bail}; + +use crate::video::{ + EncodedFrame, FrameKind, FrameSink, NalFormat, ScreenGeometry, Tuning, VideoCapture, + VideoConfig, +}; + +const IDLE_FLUSH: Duration = Duration::from_millis(75); +const RESTART_DELAY: Duration = Duration::from_millis(100); + +#[derive(Debug, Clone, Copy)] +enum CaptureControl { + Restart(ScreenRecordConfig), + Stop, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CaptureOutcome { + Restart(Option), + Stop, +} + +pub struct AndroidVideoCapture { + geometry: ScreenGeometry, + encoded_geometry: std::sync::Mutex, + capture_config: std::sync::Mutex, + max_dimension: Option, + bit_rate: u32, + control: SyncSender, + worker: Option>, +} + +impl AndroidVideoCapture { + pub fn start( + adb: AdbClient, + geometry: ScreenGeometry, + config: &VideoConfig, + sink: FrameSink, + ) -> Result { + if !geometry.is_valid() { + bail!("Android Emulator screen geometry is unavailable"); + } + if config.nal_format != NalFormat::AnnexB { + bail!("Android Emulator screenrecord capture requires Annex-B H.264"); + } + let bit_rate = match config.tuning { + Tuning::Interactive { bitrate } => bitrate + .unwrap_or_else(|| derived_bit_rate(geometry, config.max_dimension, config.fps)), + Tuning::Recording { .. } => { + bail!("Android Emulator live capture does not support recording tuning") + } + }; + let capture_config = ScreenRecordConfig::for_max_dimension( + geometry.width, + geometry.height, + config.max_dimension, + bit_rate, + ); + let encoded_geometry = ScreenGeometry { + width: capture_config.width, + height: capture_config.height, + }; + let first_child = spawn_screenrecord(&adb, capture_config)?; + let (control, commands) = mpsc::sync_channel(4); + let worker = std::thread::Builder::new() + .name("android-screenrecord".into()) + .spawn(move || run_worker(adb, capture_config, first_child, commands, sink))?; + Ok(Self { + geometry, + encoded_geometry: std::sync::Mutex::new(encoded_geometry), + capture_config: std::sync::Mutex::new(capture_config), + max_dimension: config.max_dimension, + bit_rate, + control, + worker: Some(worker), + }) + } + + pub fn set_landscape(&self, landscape: bool) -> Result<()> { + let (width, height) = if landscape { + (self.geometry.height, self.geometry.width) + } else { + (self.geometry.width, self.geometry.height) + }; + let config = + ScreenRecordConfig::for_max_dimension(width, height, self.max_dimension, self.bit_rate); + self.control + .send(CaptureControl::Restart(config)) + .map_err(|_| anyhow!("Android screenrecord worker stopped"))?; + *self.capture_config.lock().unwrap() = config; + *self.encoded_geometry.lock().unwrap() = ScreenGeometry { + width: config.width, + height: config.height, + }; + Ok(()) + } + + fn stop_worker(&mut self) { + let _ = self.control.send(CaptureControl::Stop); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +impl VideoCapture for AndroidVideoCapture { + fn geometry(&self) -> ScreenGeometry { + self.geometry + } + + fn encoded_geometry(&self) -> ScreenGeometry { + *self.encoded_geometry.lock().unwrap() + } + + fn request_keyframe(&self) { + let config = *self.capture_config.lock().unwrap(); + let _ = self.control.try_send(CaptureControl::Restart(config)); + } + + fn stop(&mut self) { + self.stop_worker(); + } +} + +impl Drop for AndroidVideoCapture { + fn drop(&mut self) { + self.stop_worker(); + } +} + +fn run_worker( + adb: AdbClient, + mut config: ScreenRecordConfig, + first_child: std::process::Child, + commands: Receiver, + sink: FrameSink, +) { + let mut next_child = Some(first_child); + loop { + let child = match next_child.take() { + Some(child) => child, + None => match spawn_screenrecord(&adb, config) { + Ok(child) => child, + Err(_) => { + match commands.recv_timeout(RESTART_DELAY) { + Ok(CaptureControl::Stop) | Err(mpsc::RecvTimeoutError::Disconnected) => { + return; + } + Ok(CaptureControl::Restart(next)) => config = next, + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + continue; + } + }, + }; + match pump_child(child, &commands, &sink) { + CaptureOutcome::Stop => return, + CaptureOutcome::Restart(Some(next)) => config = next, + CaptureOutcome::Restart(None) => {} + } + match commands.recv_timeout(RESTART_DELAY) { + Ok(CaptureControl::Stop) | Err(mpsc::RecvTimeoutError::Disconnected) => return, + Ok(CaptureControl::Restart(next)) => config = next, + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + } +} + +fn pump_child( + mut child: std::process::Child, + commands: &Receiver, + sink: &FrameSink, +) -> CaptureOutcome { + let Some(mut stdout) = child.stdout.take() else { + return CaptureOutcome::Restart(None); + }; + let stderr = child.stderr.take(); + let (chunks_tx, chunks_rx) = mpsc::sync_channel::>(8); + let reader = std::thread::spawn(move || { + let mut buffer = vec![0u8; 64 * 1024]; + loop { + match stdout.read(&mut buffer) { + Ok(0) | Err(_) => break, + Ok(read) => { + if chunks_tx.send(buffer[..read].to_vec()).is_err() { + break; + } + } + } + } + }); + let stderr_reader = stderr.map(|mut stderr| { + std::thread::spawn(move || { + let mut output = Vec::new(); + let _ = stderr.read_to_end(&mut output); + output + }) + }); + + let mut parser = AnnexBAccessUnitParser::default(); + let outcome = loop { + match commands.try_recv() { + Ok(CaptureControl::Stop) | Err(mpsc::TryRecvError::Disconnected) => { + break CaptureOutcome::Stop; + } + Ok(CaptureControl::Restart(next)) => break CaptureOutcome::Restart(Some(next)), + Err(mpsc::TryRecvError::Empty) => {} + } + match chunks_rx.recv_timeout(IDLE_FLUSH) { + Ok(chunk) => emit(parser.push(&chunk), sink), + Err(mpsc::RecvTimeoutError::Timeout) => emit(parser.flush_idle(), sink), + Err(mpsc::RecvTimeoutError::Disconnected) => { + emit(parser.flush_idle(), sink); + break CaptureOutcome::Restart(None); + } + } + }; + + drop(chunks_rx); + let _ = child.kill(); + let _ = child.wait(); + let _ = reader.join(); + if let Some(stderr_reader) = stderr_reader { + let _ = stderr_reader.join(); + } + outcome +} + +fn emit( + frames: Vec, + sink: &FrameSink, +) { + for frame in frames { + sink(EncodedFrame { + data: frame.data, + kind: if frame.keyframe { + FrameKind::Keyframe + } else { + FrameKind::Delta + }, + }); + } +} + +fn derived_bit_rate(geometry: ScreenGeometry, max_dimension: Option, fps: u32) -> u32 { + let size = + ScreenRecordConfig::for_max_dimension(geometry.width, geometry.height, max_dimension, 1); + let bits = size.width as f64 * size.height as f64 * fps.max(1) as f64 * 0.15; + bits.round().clamp(1_000_000.0, 24_000_000.0) as u32 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn derives_bitrate_from_encoded_geometry() { + let bitrate = derived_bit_rate( + ScreenGeometry { + width: 1080, + height: 2424, + }, + Some(1280), + 60, + ); + assert_eq!(bitrate, 6_566_400); + } +} diff --git a/packages/accessibility-serve/src/http.rs b/packages/accessibility-serve/src/http.rs index cd3506a..3e24dc6 100644 --- a/packages/accessibility-serve/src/http.rs +++ b/packages/accessibility-serve/src/http.rs @@ -13,8 +13,8 @@ use serde::{Deserialize, Serialize}; use accessibility_core::video::FrameKind; use crate::avcc; -use crate::input::{InputCommand, Orientation}; -use crate::session::SimSession; +use crate::session::{Orientation, Session}; +#[cfg(target_os = "macos")] use crate::settings::{Setting, SettingKey}; use crate::webrtc_stream::WebRtcEngine; @@ -22,7 +22,7 @@ const INDEX_HTML: &str = include_str!("../static/index.html"); #[derive(Clone)] pub struct AppState { - pub session: Arc, + pub session: Session, pub webrtc: Arc, pub default_transport: String, } @@ -51,6 +51,7 @@ async fn index() -> Html<&'static str> { #[derive(Serialize)] struct ConfigResponse { udid: String, + platform: &'static str, /// Raw framebuffer size. Constant regardless of orientation. width: u32, height: u32, @@ -63,13 +64,14 @@ struct ConfigResponse { async fn config(State(state): State) -> Json { let device = state.session.device_info(); Json(ConfigResponse { - udid: device.udid, + udid: device.id, + platform: device.platform, width: device.width, height: device.height, orientation: device.orientation, default_transport: state.default_transport.clone(), transports: vec!["webrtc", "h264"], - home_indicator_band: crate::input::HOME_INDICATOR_BAND, + home_indicator_band: state.session.home_indicator_band(), }) } @@ -77,9 +79,12 @@ async fn stats(State(state): State) -> Json) -> Json> { // Each read shells out to simctl, so keep it off the async worker threads. - let session = Arc::clone(&state.session); + let Some(session) = state.session.ios_session().cloned() else { + return Json(Vec::new()); + }; Json( tokio::task::spawn_blocking(move || session.settings()) .await @@ -87,17 +92,37 @@ async fn settings(State(state): State) -> Json> { ) } +#[cfg(not(target_os = "macos"))] +async fn settings(State(_state): State) -> Json> { + Json(Vec::new()) +} + +#[cfg(target_os = "macos")] #[derive(Deserialize)] struct SettingRequest { key: SettingKey, value: String, } +#[cfg(not(target_os = "macos"))] +#[derive(Deserialize)] +struct SettingRequest { + key: String, + value: String, +} + +#[cfg(target_os = "macos")] async fn set_setting( State(state): State, Json(request): Json, ) -> Response { - let session = Arc::clone(&state.session); + let Some(session) = state.session.ios_session().cloned() else { + return ( + StatusCode::BAD_REQUEST, + "settings are not supported for Android Emulator streams", + ) + .into_response(); + }; let result = tokio::task::spawn_blocking(move || session.set_setting(request.key, &request.value)).await; @@ -108,6 +133,19 @@ async fn set_setting( } } +#[cfg(not(target_os = "macos"))] +async fn set_setting( + State(_state): State, + Json(request): Json, +) -> Response { + let _ = (request.key, request.value); + ( + StatusCode::BAD_REQUEST, + "settings are not supported for Android Emulator streams", + ) + .into_response() +} + #[derive(Deserialize, Default)] struct RecordingRequest { /// Quality from 0 to 1. @@ -130,7 +168,7 @@ async fn start_recording( // Starting touches AVFoundation and the filesystem, so keep it off the // async worker threads. - let session = Arc::clone(&state.session); + let session = state.session.clone(); match tokio::task::spawn_blocking(move || session.start_recording(config)).await { Ok(Ok(path)) => Json(serde_json::json!({ "path": path })).into_response(), Ok(Err(error)) => (StatusCode::CONFLICT, error.to_string()).into_response(), @@ -140,7 +178,7 @@ async fn start_recording( async fn stop_recording(State(state): State) -> Response { // Finalizing blocks until the writer has flushed the file's index. - let session = Arc::clone(&state.session); + let session = state.session.clone(); match tokio::task::spawn_blocking(move || session.stop_recording()).await { Ok(Ok(recording)) => Json(recording).into_response(), Ok(Err(error)) => (StatusCode::CONFLICT, error.to_string()).into_response(), @@ -157,8 +195,10 @@ async fn set_orientation( State(state): State, Json(request): Json, ) -> Response { - state.session.set_orientation(request.orientation); - Json(serde_json::json!({ "orientation": request.orientation })).into_response() + match state.session.set_orientation(request.orientation) { + Ok(()) => Json(serde_json::json!({ "orientation": request.orientation })).into_response(), + Err(error) => (StatusCode::BAD_REQUEST, error.to_string()).into_response(), + } } /// Map an `anyhow` error onto a 500 with the message preserved. @@ -207,11 +247,7 @@ struct AnswerResponse { } async fn webrtc_offer(State(state): State, Json(offer): Json) -> Response { - match state - .webrtc - .answer(Arc::clone(&state.session), offer.sdp) - .await - { + match state.webrtc.answer(state.session.clone(), offer.sdp).await { Ok(sdp) => Json(AnswerResponse { sdp }).into_response(), Err(error) => internal_error(error), } @@ -300,9 +336,8 @@ async fn pump_input(state: AppState, mut socket: WebSocket) { _ => continue, }; - match serde_json::from_str::(&payload) { - Ok(command) => state.session.send_input(command), - Err(error) => tracing::debug!("ignoring malformed input event: {error}"), + if let Err(error) = state.session.send_input_json(&payload) { + tracing::debug!("ignoring malformed input event: {error}"); } } } diff --git a/packages/accessibility-serve/src/lib.rs b/packages/accessibility-serve/src/lib.rs index 01350c8..7e4cc5e 100644 --- a/packages/accessibility-serve/src/lib.rs +++ b/packages/accessibility-serve/src/lib.rs @@ -10,26 +10,20 @@ pub mod avcc; pub mod ax; #[cfg(target_os = "macos")] pub mod coverage; -#[cfg(target_os = "macos")] pub mod http; #[cfg(target_os = "macos")] pub mod input; #[cfg(target_os = "macos")] pub mod keymap; -#[cfg(target_os = "macos")] pub mod session; #[cfg(target_os = "macos")] pub mod settings; -#[cfg(target_os = "macos")] pub mod webrtc_stream; use std::net::SocketAddr; -#[cfg(target_os = "macos")] use std::sync::Arc; -#[cfg(target_os = "macos")] -use anyhow::Context; -use anyhow::Result; +use anyhow::{Context, Result}; use accessibility_core::video::VideoConfig; @@ -47,7 +41,6 @@ pub enum Transport { H264, } -#[cfg(target_os = "macos")] impl Transport { fn as_str(self) -> &'static str { match self { @@ -93,6 +86,27 @@ impl Default for ServeConfig { } } +#[derive(Debug, Clone)] +pub struct ServeEmulatorConfig { + pub serial: Option, + pub address: SocketAddr, + pub transport: Transport, + pub video: VideoConfig, + pub ice_servers: Vec, +} + +impl Default for ServeEmulatorConfig { + fn default() -> Self { + Self { + serial: None, + address: SocketAddr::from(([127, 0, 0, 1], 3200)), + transport: Transport::default(), + video: VideoConfig::default(), + ice_servers: Vec::new(), + } + } +} + /// Start capturing and serve until the process is interrupted. #[cfg(target_os = "macos")] pub async fn serve(config: ServeConfig) -> Result<()> { @@ -100,36 +114,60 @@ pub async fn serve(config: ServeConfig) -> Result<()> { .context("failed to start simulator capture")?; // The framebuffer cannot reveal orientation, so ask accessibility once // before serving; otherwise an already-rotated device renders sideways. + let session = session::Session::ios(session); session.seed_orientation().await; - let device = session.device_info(); + serve_session( + session, + config.address, + config.transport, + config.ice_servers, + ) + .await +} + +/// iOS Simulator serving is only available on macOS. +#[cfg(not(target_os = "macos"))] +pub async fn serve(_config: ServeConfig) -> Result<()> { + anyhow::bail!("Serving an iOS Simulator requires macOS") +} + +pub async fn serve_emulator(config: ServeEmulatorConfig) -> Result<()> { + let session = session::EmulatorSession::start(config.serial.as_deref(), config.video) + .context("failed to start Android Emulator capture")?; + let session = session::Session::android(session); + session.seed_orientation().await; + serve_session( + session, + config.address, + config.transport, + config.ice_servers, + ) + .await +} +async fn serve_session( + session: session::Session, + address: SocketAddr, + transport: Transport, + ice_servers: Vec, +) -> Result<()> { + let device = session.device_info(); let webrtc = Arc::new( - webrtc_stream::WebRtcEngine::new(config.ice_servers.clone()) - .context("failed to initialize WebRTC")?, + webrtc_stream::WebRtcEngine::new(ice_servers).context("failed to initialize WebRTC")?, ); - let state = http::AppState { session, webrtc, - default_transport: config.transport.as_str().to_string(), + default_transport: transport.as_str().to_string(), }; - - let listener = tokio::net::TcpListener::bind(config.address) + let listener = tokio::net::TcpListener::bind(address) .await - .with_context(|| format!("failed to bind {}", config.address))?; + .with_context(|| format!("failed to bind {address}"))?; let bound = listener.local_addr()?; - - println!("serving simulator {}", device.udid); - println!(" transport : {}", config.transport.as_str()); + println!("serving {} {}", device.platform, device.id); + println!(" transport : {}", transport.as_str()); println!(" preview : http://{bound}"); - axum::serve(listener, http::router(state)) .await .context("server error") } - -/// iOS Simulator serving is only available on macOS. -#[cfg(not(target_os = "macos"))] -pub async fn serve(_config: ServeConfig) -> Result<()> { - anyhow::bail!("Serving an iOS Simulator requires macOS") -} diff --git a/packages/accessibility-serve/src/session.rs b/packages/accessibility-serve/src/session.rs index 5c62903..6bd90e3 100644 --- a/packages/accessibility-serve/src/session.rs +++ b/packages/accessibility-serve/src/session.rs @@ -1,3 +1,305 @@ //! Compatibility re-exports for reusable iOS Simulator sessions. -pub use accessibility_core::platform::ios_simulator::session::*; +use std::sync::Arc; + +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use tokio::sync::broadcast; + +use accessibility_core::platform::android::input as android_input; +use accessibility_core::platform::android::session as android_session; +use accessibility_core::video::{EncodedFrame, Recording, RecordingConfig}; + +#[cfg(target_os = "macos")] +pub use accessibility_core::platform::ios_simulator::session::SimSession; +#[cfg(target_os = "macos")] +use accessibility_core::platform::ios_simulator::{input as ios_input, session as ios_session}; + +pub use accessibility_core::platform::android::session::EmulatorSession; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Orientation { + Portrait, + PortraitUpsideDown, + LandscapeLeft, + LandscapeRight, +} + +impl Orientation { + pub fn is_landscape(self) -> bool { + matches!(self, Self::LandscapeLeft | Self::LandscapeRight) + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct DeviceInfo { + pub id: String, + pub width: u32, + pub height: u32, + pub orientation: Orientation, + pub platform: &'static str, +} + +#[derive(Debug, Clone, Serialize)] +pub struct StatsReport { + pub uptime_secs: f64, + pub frames: u64, + pub keyframes: u64, + pub bytes: u64, + pub fps: f64, + pub mbps: f64, + pub bits_per_pixel: f64, + pub mean_frame_kb: f64, + pub keyframe_requests: u64, + pub lag_events: u64, + pub subscribers: usize, + pub recording_frames: Option, + pub width: u32, + pub height: u32, + pub encoded_width: u32, + pub encoded_height: u32, +} + +#[derive(Clone)] +pub enum Session { + #[cfg(target_os = "macos")] + Ios(Arc), + Android(Arc), +} + +impl Session { + pub fn android(session: Arc) -> Self { + Self::Android(session) + } + + #[cfg(target_os = "macos")] + pub fn ios(session: Arc) -> Self { + Self::Ios(session) + } + + pub fn subscribe(&self) -> broadcast::Receiver { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => session.subscribe(), + Self::Android(session) => session.subscribe(), + } + } + + pub fn request_keyframe(&self) { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => session.request_keyframe(), + Self::Android(session) => session.request_keyframe(), + } + } + + pub fn note_lag(&self) { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => session.note_lag(), + Self::Android(session) => session.note_lag(), + } + } + + pub fn device_info(&self) -> DeviceInfo { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => { + let device = session.device_info(); + DeviceInfo { + id: device.udid, + width: device.width, + height: device.height, + orientation: from_ios_orientation(device.orientation), + platform: "ios_simulator", + } + } + Self::Android(session) => { + let device = session.device_info(); + DeviceInfo { + id: device.serial, + width: device.width, + height: device.height, + orientation: from_android_orientation(device.orientation), + platform: "android_emulator", + } + } + } + } + + pub fn stats(&self) -> StatsReport { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => from_ios_stats(session.stats()), + Self::Android(session) => from_android_stats(session.stats()), + } + } + + pub fn send_input_json(&self, payload: &str) -> Result<()> { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => { + session.send_input(serde_json::from_str::(payload)?); + } + Self::Android(session) => { + session.send_input(serde_json::from_str::( + payload, + )?); + } + } + Ok(()) + } + + pub fn set_orientation(&self, orientation: Orientation) -> Result<()> { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => { + session.set_orientation(to_ios_orientation(orientation)); + Ok(()) + } + Self::Android(session) => session.set_orientation(to_android_orientation(orientation)), + } + } + + pub async fn seed_orientation(&self) { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => session.seed_orientation().await, + Self::Android(session) => session.seed_orientation().await, + } + } + + pub async fn ax_snapshot(&self, scan: bool) -> Result { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => Ok(serde_json::to_value(session.ax_snapshot(scan).await?)?), + Self::Android(session) => Ok(serde_json::to_value(session.ax_snapshot(scan).await?)?), + } + } + + pub async fn ax_hit_test(&self, x: f64, y: f64) -> Result { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => Ok(serde_json::to_value(session.ax_hit_test(x, y).await?)?), + Self::Android(session) => Ok(serde_json::to_value(session.ax_hit_test(x, y).await?)?), + } + } + + pub fn start_recording(&self, _config: RecordingConfig) -> Result { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => session.start_recording(_config), + Self::Android(_) => { + anyhow::bail!("recording is not supported for Android Emulator streams") + } + } + } + + pub fn stop_recording(&self) -> Result { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => session.stop_recording(), + Self::Android(_) => { + anyhow::bail!("recording is not supported for Android Emulator streams") + } + } + } + + pub fn home_indicator_band(&self) -> f64 { + match self { + #[cfg(target_os = "macos")] + Self::Ios(_) => ios_input::HOME_INDICATOR_BAND, + Self::Android(_) => 1.0, + } + } + + #[cfg(target_os = "macos")] + pub fn ios_session(&self) -> Option<&Arc> { + match self { + Self::Ios(session) => Some(session), + Self::Android(_) => None, + } + } +} + +fn from_android_orientation(orientation: android_input::Orientation) -> Orientation { + match orientation { + android_input::Orientation::Portrait => Orientation::Portrait, + android_input::Orientation::PortraitUpsideDown => Orientation::PortraitUpsideDown, + android_input::Orientation::LandscapeLeft => Orientation::LandscapeLeft, + android_input::Orientation::LandscapeRight => Orientation::LandscapeRight, + } +} + +fn to_android_orientation(orientation: Orientation) -> android_input::Orientation { + match orientation { + Orientation::Portrait => android_input::Orientation::Portrait, + Orientation::PortraitUpsideDown => android_input::Orientation::PortraitUpsideDown, + Orientation::LandscapeLeft => android_input::Orientation::LandscapeLeft, + Orientation::LandscapeRight => android_input::Orientation::LandscapeRight, + } +} + +fn from_android_stats(stats: android_session::StatsReport) -> StatsReport { + StatsReport { + uptime_secs: stats.uptime_secs, + frames: stats.frames, + keyframes: stats.keyframes, + bytes: stats.bytes, + fps: stats.fps, + mbps: stats.mbps, + bits_per_pixel: stats.bits_per_pixel, + mean_frame_kb: stats.mean_frame_kb, + keyframe_requests: stats.keyframe_requests, + lag_events: stats.lag_events, + subscribers: stats.subscribers, + recording_frames: stats.recording_frames, + width: stats.width, + height: stats.height, + encoded_width: stats.encoded_width, + encoded_height: stats.encoded_height, + } +} + +#[cfg(target_os = "macos")] +fn from_ios_orientation(orientation: ios_input::Orientation) -> Orientation { + match orientation { + ios_input::Orientation::Portrait => Orientation::Portrait, + ios_input::Orientation::PortraitUpsideDown => Orientation::PortraitUpsideDown, + ios_input::Orientation::LandscapeLeft => Orientation::LandscapeLeft, + ios_input::Orientation::LandscapeRight => Orientation::LandscapeRight, + } +} + +#[cfg(target_os = "macos")] +fn to_ios_orientation(orientation: Orientation) -> ios_input::Orientation { + match orientation { + Orientation::Portrait => ios_input::Orientation::Portrait, + Orientation::PortraitUpsideDown => ios_input::Orientation::PortraitUpsideDown, + Orientation::LandscapeLeft => ios_input::Orientation::LandscapeLeft, + Orientation::LandscapeRight => ios_input::Orientation::LandscapeRight, + } +} + +#[cfg(target_os = "macos")] +fn from_ios_stats(stats: ios_session::StatsReport) -> StatsReport { + StatsReport { + uptime_secs: stats.uptime_secs, + frames: stats.frames, + keyframes: stats.keyframes, + bytes: stats.bytes, + fps: stats.fps, + mbps: stats.mbps, + bits_per_pixel: stats.bits_per_pixel, + mean_frame_kb: stats.mean_frame_kb, + keyframe_requests: stats.keyframe_requests, + lag_events: stats.lag_events, + subscribers: stats.subscribers, + recording_frames: stats.recording_frames, + width: stats.width, + height: stats.height, + encoded_width: stats.encoded_width, + encoded_height: stats.encoded_height, + } +} diff --git a/packages/accessibility-serve/src/webrtc_stream.rs b/packages/accessibility-serve/src/webrtc_stream.rs index e5da645..33e8e8c 100644 --- a/packages/accessibility-serve/src/webrtc_stream.rs +++ b/packages/accessibility-serve/src/webrtc_stream.rs @@ -28,7 +28,7 @@ use webrtc::track::track_local::track_local_static_sample::TrackLocalStaticSampl use accessibility_core::video::FrameKind; -use crate::session::SimSession; +use crate::session::Session; pub struct WebRtcEngine { api: API, @@ -65,7 +65,7 @@ impl WebRtcEngine { } /// Answer a browser offer, wiring a fresh track to the capture stream. - pub async fn answer(&self, session: Arc, offer_sdp: String) -> Result { + pub async fn answer(&self, session: Session, offer_sdp: String) -> Result { let peer = Arc::new(self.api.new_peer_connection(self.config.clone()).await?); let track = Arc::new(TrackLocalStaticSample::new( @@ -74,7 +74,7 @@ impl WebRtcEngine { ..Default::default() }, "video".to_owned(), - format!("sim-{}", session.device_info().udid), + format!("sim-{}", session.device_info().id), )); let sender = peer @@ -84,7 +84,7 @@ impl WebRtcEngine { // Sender RTCP has to be drained or feedback never gets processed. PLI // and FIR both mean "I cannot decode, send me a fresh IDR". { - let session = Arc::clone(&session); + let session = session.clone(); tokio::spawn(async move { let mut buffer = vec![0u8; 1500]; while let Ok((packets, _)) = sender.read(&mut buffer).await { @@ -100,7 +100,7 @@ impl WebRtcEngine { }); } - let forwarder = spawn_forwarder(Arc::clone(&session), Arc::clone(&track)); + let forwarder = spawn_forwarder(session.clone(), Arc::clone(&track)); // Tear the forwarding task down when the viewer goes away, otherwise // every reconnect would leak a subscriber on the broadcast channel. @@ -137,7 +137,7 @@ impl WebRtcEngine { /// Pump encoded frames from the capture broadcast onto a viewer's track. fn spawn_forwarder( - session: Arc, + session: Session, track: Arc, ) -> tokio::task::JoinHandle<()> { let mut frames = session.subscribe(); diff --git a/packages/accessibility-serve/static/index.html b/packages/accessibility-serve/static/index.html index 82efd5e..bc1e9d2 100644 --- a/packages/accessibility-serve/static/index.html +++ b/packages/accessibility-serve/static/index.html @@ -255,9 +255,11 @@

Device settings

const isLandscape = () => orientation === "landscape_left" || orientation === "landscape_right"; +const isAndroid = () => config?.platform === "android_emulator"; /** Upright display point (0..1) to raw framebuffer point. */ function toRawPoint(x, y) { + if (isAndroid()) return { x, y }; switch (orientation) { case "landscape_left": return { x: y, y: 1 - x }; case "landscape_right": return { x: 1 - y, y: x }; @@ -280,12 +282,14 @@

Device settings

/** The raw edge for a display point, or "none" if it is not near one. */ function edgeForDisplayPoint(y) { + if (isAndroid()) return "none"; if (y < config.home_indicator_band) return "none"; return BOTTOM_EDGE_BY_ORIENTATION[orientation] || "bottom"; } /** Rotate a delta, which has direction but no origin. */ function toRawDelta(dx, dy) { + if (isAndroid()) return { dx, dy }; switch (orientation) { case "landscape_left": return { dx: dy, dy: -dx }; case "landscape_right": return { dx: -dy, dy: dx }; @@ -327,10 +331,16 @@

Device settings

stage.style.height = `${uprightH * scale}px`; // The media keeps the framebuffer's own aspect and is rotated on top of it. - media.style.width = `${fbW * scale}px`; - media.style.height = `${fbH * scale}px`; - media.style.transform = - `translate(-50%, -50%) rotate(${ROTATION[orientation] || 0}deg)`; + if (isAndroid()) { + media.style.width = `${uprightW * scale}px`; + media.style.height = `${uprightH * scale}px`; + media.style.transform = "translate(-50%, -50%)"; + } else { + media.style.width = `${fbW * scale}px`; + media.style.height = `${fbH * scale}px`; + media.style.transform = + `translate(-50%, -50%) rotate(${ROTATION[orientation] || 0}deg)`; + } } window.addEventListener("resize", () => config && layout()); @@ -494,12 +504,17 @@

Device settings

async function rotate(map) { const next = map[orientation] || "portrait"; - await fetch("/api/orientation", { + const response = await fetch("/api/orientation", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ orientation: next }), }); - orientation = next; + if (!response.ok) { + setStatus(`rotation failed: ${await response.text()}`, "error"); + return; + } + const result = await response.json(); + orientation = result.orientation || next; layout(); if (selected) drawSelection(selected); }