From a0bd2375c9148316237cced279e7146ad65f47bc Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Mon, 17 Aug 2026 17:47:21 -0700 Subject: [PATCH 1/4] Add Android Emulator streaming backend Build a reusable emulator session around raw H.264 capture, gRPC input, orientation, and accessibility so Android displays can use the same serving contract as iOS. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <166158716+staging-devin-ai-integration[bot]@users.noreply.github.com> --- Cargo.lock | 292 +++++++++++++ docs/ANDROID_EMULATOR_RAW_CAPTURE_PLAN.md | 249 +++++++++++ docs/ANDROID_EMULATOR_STREAMING_PLAN.md | 250 +++++++++++ packages/accessibility-android-sys/Cargo.toml | 14 + packages/accessibility-android-sys/build.rs | 16 + .../examples/emulator_raw_capture_probe.rs | 196 +++++++++ .../examples/emulator_webrtc_probe.rs | 282 +++++++++++++ .../examples/screenrecord_probe.rs | 140 +++++++ .../proto/emulator_controller.proto | 135 ++++++ .../proto/rtc_service_v2.proto | 38 ++ .../accessibility-android-sys/src/emulator.rs | 390 ++++++++++++++++++ .../src/emulator/raw.rs | 226 ++++++++++ .../src/emulator/screenrecord.rs | 368 +++++++++++++++++ packages/accessibility-android-sys/src/lib.rs | 122 ++++-- packages/accessibility-cli/src/lib.rs | 68 +++ packages/accessibility-cli/tests/cli_smoke.rs | 11 + .../examples/android_session_probe.rs | 123 ++++++ .../src/platform/android.rs | 28 ++ .../src/platform/android/ax.rs | 271 ++++++++++++ .../src/platform/android/input.rs | 332 +++++++++++++++ .../src/platform/android/session.rs | 233 +++++++++++ .../src/platform/android/video.rs | 277 +++++++++++++ packages/accessibility-serve/src/http.rs | 73 +++- packages/accessibility-serve/src/lib.rs | 90 ++-- packages/accessibility-serve/src/session.rs | 304 +++++++++++++- .../accessibility-serve/src/webrtc_stream.rs | 12 +- .../accessibility-serve/static/index.html | 27 +- 27 files changed, 4479 insertions(+), 88 deletions(-) create mode 100644 docs/ANDROID_EMULATOR_RAW_CAPTURE_PLAN.md create mode 100644 docs/ANDROID_EMULATOR_STREAMING_PLAN.md create mode 100644 packages/accessibility-android-sys/build.rs create mode 100644 packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs create mode 100644 packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs create mode 100644 packages/accessibility-android-sys/examples/screenrecord_probe.rs create mode 100644 packages/accessibility-android-sys/proto/emulator_controller.proto create mode 100644 packages/accessibility-android-sys/proto/rtc_service_v2.proto create mode 100644 packages/accessibility-android-sys/src/emulator.rs create mode 100644 packages/accessibility-android-sys/src/emulator/raw.rs create mode 100644 packages/accessibility-android-sys/src/emulator/screenrecord.rs create mode 100644 packages/accessibility-core/examples/android_session_probe.rs create mode 100644 packages/accessibility-core/src/platform/android/ax.rs create mode 100644 packages/accessibility-core/src/platform/android/input.rs create mode 100644 packages/accessibility-core/src/platform/android/session.rs create mode 100644 packages/accessibility-core/src/platform/android/video.rs diff --git a/Cargo.lock b/Cargo.lock index 8cb7562..f59648a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,17 @@ name = "accessibility-android-sys" version = "0.1.0" dependencies = [ "anyhow", + "bytes", "keyboard-types", + "memmap2", + "prost", + "protoc-bin-vendored", + "rtp", + "serde_json", + "tokio", + "tonic", + "tonic-build", + "webrtc", ] [[package]] @@ -1531,6 +1541,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.9" @@ -1550,6 +1566,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -1775,6 +1797,25 @@ dependencies = [ "subtle", ] +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -1892,6 +1933,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -1900,6 +1942,20 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", ] [[package]] @@ -1909,12 +1965,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "bytes", + "futures-channel", + "futures-util", "http", "http-body", "hyper", + "libc", "pin-project-lite", + "socket2 0.6.3", "tokio", "tower-service", + "tracing", ] [[package]] @@ -2331,6 +2392,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843a98750cd611cc2965a8213b53b43e715f13c37a9e096c6408e69990961db7" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -2392,6 +2462,12 @@ dependencies = [ "pxfm", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "nalgebra" version = "0.32.6" @@ -3026,6 +3102,16 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "phf" version = "0.10.1" @@ -3106,6 +3192,26 @@ dependencies = [ "siphasher 1.0.2", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -3311,6 +3417,122 @@ dependencies = [ "syn", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "protoc-bin-vendored" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" + [[package]] name = "pxfm" version = "0.1.29" @@ -4348,6 +4570,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.29.0" @@ -4369,6 +4602,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -4403,6 +4637,46 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "base64", + "bytes", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn", +] + [[package]] name = "tower" version = "0.5.3" @@ -4411,9 +4685,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -4477,6 +4754,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "ttf-parser" version = "0.25.1" @@ -4664,6 +4947,15 @@ dependencies = [ "atomic-waker", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/docs/ANDROID_EMULATOR_RAW_CAPTURE_PLAN.md b/docs/ANDROID_EMULATOR_RAW_CAPTURE_PLAN.md new file mode 100644 index 0000000..1afd7f1 --- /dev/null +++ b/docs/ANDROID_EMULATOR_RAW_CAPTURE_PLAN.md @@ -0,0 +1,249 @@ +# Android Emulator Raw Capture and Devin Remote Integration + +## Goal + +Use the Android Emulator's raw framebuffer stream as an input to `devin-remote`'s existing software H.264 encoder and congestion controller. Keep the screenrecord backend as the standalone and compatibility fallback. + +The production path must preserve: + +- emulator source timestamps and frame sequence numbers; +- dynamic bitrate, resolution, and frame-rate control; +- explicit keyframe requests; +- bounded queues and stale-frame dropping; +- the existing QUIC/WebSocket transport adaptation boundary; +- Android touch, keyboard, rotation, and accessibility inspection. + +## Architecture + +```text +Android Emulator renderer + | + | EmulatorController.streamScreenshot + | RGBA8888 + sequence + timestampUs + | normal gRPC bytes or local MMAP + v +Android raw capture source + | + | Arc + v +devin-remote desktop_stream + | + +-- RGBA -> I420 + +-- x264 or OpenH264 + +-- force keyframe + +-- bitrate / pixel / fps ladder + +-- capture and encode pacing + v +existing StreamEvent::Frame + | + v +existing congestion controller + | + v +QUIC or WebSocket transport +``` + +Input remains independent: + +```text +browser input + -> devin-remote + -> EmulatorController gRPC for touch, text, and ordinary keys + -> ADB fallback for Android hardware buttons +``` + +Accessibility remains ADB and `uiautomator` through `accessibility-core`. + +## Why not goldfish-webrtc-bridge + +`goldfish-webrtc-bridge` is a conditionally built, emulator-bundled host executable. Newer source also exposes the implementation as the `android-webrtc` shared library. It exists to connect the emulator framebuffer to Chromium libwebrtc encoding and WebRTC transport. + +It is not a separately supported SDK Manager package, and the standard macOS Emulator 37.1.11 package does not ship it. Building it requires the emulator source tree and Chromium's libwebrtc dependencies. + +`devin-remote` already has the parts the bridge would add: software H.264 encoding, keyframe control, pacing, link measurements, and congestion adaptation. Only the raw emulator capture source is needed. + +## Phase 1: protocol and capture probes + +Extend the minimal EmulatorController protocol with: + +- `streamScreenshot(ImageFormat) returns (stream Image)`; +- `ImageFormat` with RGBA8888, requested dimensions, orientation, display id, and optional transport; +- `ImageTransport` with MMAP channel and handle; +- `Image` with pixels, sequence number, and timestamp. + +Add a probe that measures both modes against a running emulator: + +1. RGBA8888 in ordinary gRPC messages. +2. RGBA8888 through a client-owned memory-mapped file. + +For each mode report: + +- negotiated dimensions and orientation; +- frame count and cadence under motion; +- capture-to-receive latency from `timestampUs`; +- sequence gaps; +- bytes copied per frame; +- MMAP tearing or inconsistent-frame observations; +- behavior across rotation and reconnect. + +MMAP is accepted only if it is supported by the standard emulator package and frame ownership can be made safe before the next write. Otherwise use ordinary local gRPC at the congestion controller's requested resolution. + +### Measured result + +Validated against Android Emulator 37.1.11 on macOS with a Pixel 10 API 36 AVD at a 1280-pixel capture bound: + +| Transport | Negotiated frame | Frames | Cadence | Mean source-to-receive latency | Sequence gaps | Detected tearing | +|---|---:|---:|---:|---:|---:|---:| +| gRPC bytes | 570x1280 RGBA | 287 / 5 s | 57.3 fps | 5.1 ms | 0 | n/a | +| MMAP | 570x1280 RGBA | 302 / 5 s | 60.2 fps | 2.1 ms | 0 | 0 | + +MMAP is the production default. The capture task copies each announced frame into an owned buffer immediately and publishes it through a one-frame overwrite slot. If the encoder runs below 60 fps, obsolete raw frames are dropped instead of queueing latency. + +The API treats requested dimensions as an aspect-ratio bounding box. A request for 534x1200 negotiated 534x1198, so encoder geometry is always created from the first returned frame rather than the request. + +## Phase 2: reusable raw frame source + +Implement an Android raw frame stream below `accessibility-core`'s platform layer: + +- discover and authenticate to the selected emulator; +- request RGBA8888 at an even target size; +- return owned frames with width, height, sequence, timestamp, and orientation; +- detect dropped source frames through sequence gaps; +- copy mapped bytes before acknowledging/awaiting the next frame; +- reconnect on gRPC stream termination; +- expose current source geometry and orientation; +- keep capture independent from accessibility and input workers. + +The capture source must not encode video and must not use the encoded `VideoCapture` trait. The screenrecord implementation remains separate. + +## Phase 3: devin-remote encoder integration + +Add Android as a software-encoded source in `desktop_stream`: + +1. Resolve Android display geometry before creating the encoder. +2. Request emulator frames at the geometry selected by the existing viewport and headroom policy. +3. Wrap each owned RGBA frame as the existing `capture::Frame`. +4. Feed it through `H264Encoder` and the existing `stream_software_encoded` loop. +5. Use the emulator's `timestampUs` as the capture timestamp rather than assigning it after receipt. +6. Treat the source as damage-driven because `streamScreenshot` emits on emulator frame production. +7. On forced refresh, reuse the last owned raw frame if the emulator has not produced another one. +8. On bitrate-only congestion changes, rebuild only the encoder as today. +9. On pixel-step changes or orientation changes, restart the screenshot stream and encoder with new geometry. +10. Preserve source sequence gaps and capture latency in telemetry. + +Do not forward screenrecord H.264 into the remote production path. That bypasses the existing encoder and congestion controller. + +## Phase 4: display source and control protocol + +Extend `DisplaySource` with `android_emulator` and an optional ADB serial. + +The source must send `source_config` before media frames with: + +- platform and serial; +- source geometry; +- encoded geometry; +- orientation; +- capabilities for touch, keyboard, hardware buttons, rotation, and accessibility. + +Reuse the existing simulator request/response shapes where they are genuinely common: + +- stats; +- accessibility snapshot; +- accessibility hit test; +- orientation; +- error responses with request ids. + +Keep Android-specific input translation and coordinate semantics out of the iOS implementation. + +## Phase 5: congestion and QUIC adaptation + +Continue using the current `Congestion` and `Headroom` controller. Feed it transport-native QUIC observations when the QUIC leg is enabled: + +- ACKed goodput; +- smoothed RTT and RTT variance; +- congestion window and in-flight bytes; +- sender queue age; +- datagram loss; +- viewer decode queue and paint delay. + +The existing adaptation order remains: + +1. bitrate share; +2. capture pixel steps; +3. frame-rate steps. + +A keyframe request continues through `StreamerControl::Keyframe` to the existing encoder. No emulator capture restart is required for decoder recovery. + +## Phase 6: verification + +Unit tests: + +- protobuf wire shapes; +- screenshot metadata conversion; +- sequence-gap detection; +- source timestamp conversion; +- RGBA ownership and dimensions; +- MMAP bounds and copy behavior; +- source selection parsing; +- Android geometry and orientation transitions; +- controller-triggered capture restarts at pixel-step boundaries; +- iOS and desktop source compatibility. + +End-to-end probes: + +- raw gRPC capture; +- MMAP capture when supported; +- x264 and OpenH264 encoding; +- forced keyframe; +- bitrate backoff and recovery; +- pixel and FPS backoff; +- portrait and landscape; +- touch, text, hardware buttons, and accessibility; +- stream reconnect; +- sustained motion and an extended idle/motion soak. + +Repository verification: + +```sh +# accessibility-cli +cargo fmt --all +cargo build --workspace +cargo clippy --workspace --all-targets +cargo test --workspace --lib +cargo test -p accessibility-cli --test cli_smoke + +# devin-webapp/apps/devin/devin-rs +cargo fmt +cargo check -p devin-remote +cargo test -p devin-remote +``` + +## Implementation status + +Implemented and validated: + +- EmulatorController screenshot protocol, ordinary gRPC probe, and MMAP probe. +- Reusable owned RGBA stream with sequence, timestamp, rotation, bottom-up correction, and mapping cleanup. +- `devin-remote` `android_emulator` display-source selection. +- MMAP capture with ordinary gRPC frame fallback. +- A dedicated 60 fps capture task and one-frame overwrite slot so encoder backpressure drops stale frames. +- Encoder geometry derived from the first negotiated frame. +- Existing RGBA-to-I420 and x264/OpenH264 encoder path. +- Existing keyframe, bitrate, frame-rate, queue-lag, goodput, RTT, and jitter controls. +- Pixel-step and orientation changes through a controlled display reconnect. +- EmulatorController gRPC touch, scroll, text, and USB keyboard input. +- ADB Home, Back, Lock, and app-switch buttons. +- Android accessibility snapshots and cached hit testing through the existing accessibility-core ADB backend. +- Source stats and source configuration messages. +- Real-emulator tests for raw frame to IDR and full WebSocket source configuration, encoded keyframe, and accessibility snapshot. + +Remaining integration work: + +- Add a dedicated Android Emulator workspace panel or source picker in the webapp. The backend is reachable now through `source=android_emulator&source_id=`. +- Feed transport-native QUIC ACK, congestion-window, in-flight, and datagram-loss metrics into the existing controller when the QUIC leg exposes them. The current controller continues to use drained goodput, socket queue age, viewer RTT, and jitter. +- Update `devin-remote` to the new accessibility-cli revision after these cross-repository changes are committed, then remove its temporary pinned-protocol capture client so the shared crate is the sole implementation. +- Run the extended motion/idle soak and validate the production Linux emulator image. + +## Fallback + +If raw screenshot streaming is unavailable or unstable on a target emulator package, use the existing screenrecord backend for that source. The fallback has coarser congestion adaptation because bitrate, size, and keyframe recovery require restarting screenrecord, but it preserves functional streaming and control. diff --git a/docs/ANDROID_EMULATOR_STREAMING_PLAN.md b/docs/ANDROID_EMULATOR_STREAMING_PLAN.md new file mode 100644 index 0000000..8af9496 --- /dev/null +++ b/docs/ANDROID_EMULATOR_STREAMING_PLAN.md @@ -0,0 +1,250 @@ +# Android Emulator Streaming Plan + +## Goal + +Add live Android Emulator display streaming, input, and accessibility inspection through the same encoded-frame and serving abstractions used by the iOS Simulator backend. The first implementation targets the official Android Emulator and arbitrary unmodified user applications. Physical Android devices and the Devin webapp UI are deferred. + +## Chosen approach + +Use a hybrid of Android's built-in interfaces: + +- `adb exec-out screenrecord --output-format=h264 --time-limit 0 -` for display video; +- `EmulatorController` gRPC for persistent input and device control; +- ADB and `uiautomator` for accessibility. + +`screenrecord` encodes the composed display inside Android and emits Annex-B H.264 without host-side decoding or re-encoding. Rust will split the byte stream into access units, classify keyframes, cache SPS and PPS NAL units, and publish frames through the existing `VideoCapture` interface. + +```text +Android composed display + | + v +screenrecord / MediaCodec + | + | ADB exec-out, Annex-B H.264 + v +Rust access-unit parser + | + v +VideoCapture / session broadcast + | + +-- WebRTC serving + +-- raw H.264 WebSocket serving + +Browser input + | + v +EmulatorController streaming gRPC + | + v +Android virtual input devices +``` + +This design does not inject code into the foreground application. Capture and input operate at the emulator or Android system level, so applications do not require modification or instrumentation. + +## Compatibility + +The requirement applies to the Android Emulator package, not the Android API level of the virtual device. + +- Initial supported emulator floor: Android Emulator 35.1.21, September 2024. +- Initial supported guest floor: Android 7.0 / API 24, where raw H.264 `screenrecord` output is available. +- Recommended version: latest stable Android Emulator. +- Runtime behavior: detect the required gRPC services and `screenrecord` options rather than trusting version strings alone. + +Virtual devices may use any Android system image at or above the guest floor that is supported by the installed emulator. + +## Phase 1: discovery and capability detection + +Implement host-side discovery in `accessibility-android-sys`: + +1. Locate active emulator discovery files: + - Linux: `~/.android/avd/running/pid_*.ini` + - macOS: `~/Library/Android/avd/running/pid_*.ini` +2. Parse the gRPC port, token, AVD identity, and related metadata. +3. Resolve an explicitly requested emulator when possible. +4. Require one unambiguous running emulator when no identifier is supplied. +5. Connect to `EmulatorController` and `android.emulation.control.v2.Rtc`. +6. Return actionable errors for missing discovery data, authentication failure, unsupported services, and ambiguous selection. + +Use minimal protocol definitions for only the services and messages this project needs. Keep the definitions versioned with the implementation and test their wire-facing serialization. + +## Phase 2: measured RTC result and video probe + +The RTC experiment was implemented as: + +```sh +cargo run -p accessibility-android-sys --example emulator_webrtc_probe +``` + +Measured on the standard macOS SDK Android Emulator 37.1.11: + +1. Emulator discovery, bearer authentication, and `EmulatorController.getStatus` succeeded. +2. Android Studio's default JWT allowlist rejected `Rtc.RequestRtcStream` because the RTC service is not listed. +3. A separate emulator launched with explicit insecure `-grpc 8556`, matching Google's container launcher, removed the authentication restriction. +4. `Rtc.RequestRtcStream` then returned gRPC `UNIMPLEMENTED`. +5. The installed macOS emulator package contains no WebRTC module or video bridge executable. + +The RTC v2 definitions exist in the emulator source tree but are not a portable capability of standard SDK emulator packages. Version checks alone cannot make this backend reliable. The implementation therefore uses raw H.264 `screenrecord` video with gRPC input. + +The capture probe must now verify: + +1. `screenrecord --output-format=h264 --time-limit 0 -` produces a sustained byte stream. +2. The parser handles three- and four-byte Annex-B start codes across arbitrary read boundaries. +3. NAL units are grouped into complete access units using access-unit delimiters, slice headers, timestamps when available, and encoder behavior observed from supported emulator builds. +4. SPS and PPS are cached and prepended to keyframes. +5. New clients and lag recovery restart `screenrecord` to force a new encoder session and decoder entry point. +6. Capture restart latency is measured and bounded. +7. Size, bitrate, frame rate, geometry, and long-running behavior are reported. + +The supported Android 36 image reports screenrecord v1.4, accepts `--time-limit 0`, and successfully emitted raw H.264 through `adb exec-out`. + +## Phase 3: reusable Android emulator session + +Implement an Android emulator session in `accessibility-core` with the same responsibilities as the iOS simulator session: + +- own the capture, input, and accessibility resources for one emulator; +- publish encoded frames through a small bounded broadcast channel; +- request an immediate keyframe for new subscribers; +- track lag and request recovery after dropped frames; +- expose raw and encoded geometry; +- collect common stream statistics; +- keep input independent from slow accessibility tree operations; +- stop and release the `screenrecord` child and gRPC streams deterministically. + +Implement `AndroidEmulatorVideoCapture` as `VideoCapture`: + +- `geometry()` returns source display geometry; +- `encoded_geometry()` reflects the configured screenrecord size; +- `request_keyframe()` replaces the screenrecord process with a fresh encoder session; +- `stop()` terminates capture and joins the parser worker; +- recording remains unsupported until a separate recording design is chosen. + +Move genuinely shared stream statistics and browser-facing accessibility response types out of the iOS-only module rather than duplicating their definitions. + +## Phase 4: input + +Use `EmulatorController.streamInputEvent` or the corresponding direct RPCs for: + +- touch begin, move, and end; +- multiple pointer identifiers where supported; +- keyboard down and up; +- UTF-8 text; +- scrolling and wheel events; +- Home, Back, Recents, Power, and other Android hardware keys; +- orientation changes. + +Normalize coordinates at the session boundary and map them against current oriented display geometry. Preserve ordering on a persistent stream instead of spawning an ADB process per event. + +Where the existing client protocol supplies USB HID usage codes, use the emulator protocol's USB code type rather than maintaining an unnecessary second key map. + +## Phase 5: accessibility inspection + +Reuse `AndroidAccessibility` and its ADB client for tree collection: + +1. Fetch the hierarchy through `uiautomator dump`. +2. Flatten it into the shared `ElementDetail` representation. +3. Normalize element bounds against current display geometry. +4. Preserve labels, values, resource identifiers, roles, states, and actions. +5. Compute tree coverage through the shared coverage grid. +6. Implement hit testing against the latest cached tree, preferring the smallest containing element. + +Android hit testing is cache-based and must not be presented as the same authoritative live operation available through the iOS accessibility bridge. Point-grid discovery remains iOS-specific unless an Android need is demonstrated. + +## Phase 6: standalone serving interface + +Add a command parallel to `serve-sim`: + +```sh +cargo run -p accessibility-cli -- serve-emulator --serial emulator-5554 --port 3200 +``` + +It should support the existing video options where the emulator RTC sender can honor them and expose the same core surface: + +- source configuration and geometry; +- live encoded video; +- input WebSocket; +- accessibility snapshot; +- accessibility hit test; +- orientation; +- stream statistics. + +Refactor `accessibility-serve` around a platform-neutral session boundary rather than copying the HTTP, WebSocket, AVCC, and WebRTC transport implementations. + +## Phase 7: tests and verification + +Add tests at representation boundaries: + +- discovery file parsing and emulator selection; +- missing, ambiguous, and unsupported emulator errors; +- protobuf request serialization and response decoding; +- Annex-B stream splitting across arbitrary read boundaries; +- H.264 access-unit assembly; +- SPS/PPS caching and keyframe reconstruction; +- screenrecord restart recovery; +- coordinate mapping under each orientation; +- Android tree flattening, normalization, coverage, and hit testing; +- session lag and keyframe-request accounting. + +Run: + +```sh +cargo fmt --all +cargo build --workspace +cargo clippy --workspace --all-targets +cargo test --workspace --lib +cargo test -p accessibility-cli --test cli_smoke +``` + +End-to-end verification requires a supported running emulator. A successful run must cover: + +- arbitrary APK installation and launch; +- sustained video under motion; +- touch drag and scrolling; +- text and hardware keys; +- rotation and geometry updates; +- accessibility tree and hit testing; +- decoder recovery after capture restart; +- disconnect and reconnect. + +Do not claim end-to-end validation if only compile and unit verification were possible. + +## Validation record + +Validated on macOS against Android Emulator 37.1.11 running an Android 36 arm64 image: + +- raw screenrecord H.264 at 570x1280 and 1280x570; +- Constrained Baseline SPS, PPS, IDR, and delta-frame parsing; +- 56-70 frames during a five-second gesture stimulus; +- approximately 0.9-1.2 Mbps during that stimulus; +- fresh SPS/PPS+IDR approximately 170-470 ms after capture restart; +- gRPC touch injection against the Settings UI; +- ADB hardware Home fallback after the emulator's successful `sendKey` RPC produced no action; +- accessibility tree, normalized coverage, cached hover picking, and live hit testing; +- browser WebCodecs rendering through the raw H.264 WebSocket; +- exact portrait/landscape rotation using `wm fixed-to-user-rotation` and orientation-aware capture restart; +- return from landscape to portrait with encoded geometry updated in both directions. + +The standard macOS emulator package did not provide RTC v2 media streaming even at Emulator 37.1.11, so RTC source presence is not used as a compatibility claim. + +## Deferred integration + +Do not modify `devin-webapp` until the standalone backend is validated. + +The follow-up integration will: + +1. update the `accessibility-core` revision used by `devin-remote`; +2. add `source=android_emulator` to the display source selection; +3. forward Android frames through the existing port 6080 writer; +4. translate the shared request/response protocol for Android input and inspection; +5. add a separate Android Emulator tab while sharing transport, inspector, and device-frame UI internals with iOS; +6. add the Android quote surface and platform availability gating. + +## Explicit non-goals for the first implementation + +- Physical Android devices. +- Target-app instrumentation or injection. +- A scrcpy server dependency. +- An RTC/WebRTC capture backend in standard SDK emulator packages. +- Audio streaming. +- MP4 recording. +- Multi-emulator simultaneous serving from one process. +- Devin webapp UI changes. diff --git a/packages/accessibility-android-sys/Cargo.toml b/packages/accessibility-android-sys/Cargo.toml index 12fc874..a43de6a 100644 --- a/packages/accessibility-android-sys/Cargo.toml +++ b/packages/accessibility-android-sys/Cargo.toml @@ -12,4 +12,18 @@ categories = ["accessibility", "api-bindings", "os"] [dependencies] anyhow.workspace = true +bytes.workspace = true keyboard-types.workspace = true +memmap2 = "=0.9.8" +prost = "=0.13.5" +tokio.workspace = true +tonic = { version = "=0.13.1", default-features = false, features = ["channel", "codegen", "prost"] } + +[dev-dependencies] +rtp = "=0.14.0" +serde_json.workspace = true +webrtc.workspace = true + +[build-dependencies] +protoc-bin-vendored = "=3.2.0" +tonic-build = { version = "=0.13.1", default-features = false, features = ["prost"] } diff --git a/packages/accessibility-android-sys/build.rs b/packages/accessibility-android-sys/build.rs new file mode 100644 index 0000000..9384d36 --- /dev/null +++ b/packages/accessibility-android-sys/build.rs @@ -0,0 +1,16 @@ +fn main() -> Result<(), Box> { + let protoc = protoc_bin_vendored::protoc_bin_path()?; + let mut prost = tonic_build::Config::new(); + prost.protoc_executable(protoc); + tonic_build::configure() + .build_server(false) + .compile_protos_with_config( + prost, + &[ + "proto/emulator_controller.proto", + "proto/rtc_service_v2.proto", + ], + &["proto"], + )?; + Ok(()) +} diff --git a/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs b/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs new file mode 100644 index 0000000..89ee8c9 --- /dev/null +++ b/packages/accessibility-android-sys/examples/emulator_raw_capture_probe.rs @@ -0,0 +1,196 @@ +use std::fs::OpenOptions; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use accessibility_android_sys::AdbClient; +use accessibility_android_sys::emulator::protocol::controller::image_format::ImgFormat; +use accessibility_android_sys::emulator::protocol::controller::image_transport::TransportChannel; +use accessibility_android_sys::emulator::protocol::controller::{ImageFormat, ImageTransport}; +use accessibility_android_sys::emulator::{EmulatorGrpcClient, discover_emulator}; +use anyhow::{Context, Result, bail}; +use memmap2::MmapOptions; + +const MAX_DIMENSION: u32 = 1280; +const PROBE_DURATION: Duration = Duration::from_secs(5); + +#[tokio::main] +async fn main() -> Result<()> { + let selector = std::env::args().nth(1); + let discovery = discover_emulator(selector.as_deref())?; + let serial = discovery + .properties + .get("port.serial") + .map(|port| format!("emulator-{port}")) + .context("discovery file has no port.serial")?; + let mut client = EmulatorGrpcClient::connect(discovery.clone()).await?; + let status = client.status().await?; + println!("emulator : {}", status.version); + println!("endpoint : {}", discovery.endpoint()); + println!("serial : {serial}"); + + let stop = Arc::new(AtomicBool::new(false)); + let stimulus_stop = Arc::clone(&stop); + let stimulus_serial = serial.clone(); + let stimulus = std::thread::spawn(move || { + let adb = AdbClient::discover(Some(&stimulus_serial)); + let Ok((width, height)) = adb.get_screen_size() else { + return; + }; + while !stimulus_stop.load(Ordering::Relaxed) { + let _ = adb.swipe( + (width as f64 * 0.5, height as f64 * 0.75), + (width as f64 * 0.5, height as f64 * 0.25), + 250, + ); + std::thread::sleep(Duration::from_millis(150)); + } + }); + + let bytes = probe_bytes(client.clone()).await?; + print_report("grpc", &bytes); + let mmap = probe_mmap(client).await?; + print_report("mmap", &mmap); + stop.store(true, Ordering::Relaxed); + let _ = stimulus.join(); + println!("probe passed"); + Ok(()) +} + +#[derive(Default)] +struct Report { + frames: u64, + bytes: u64, + gaps: u64, + tears: u64, + latency_us: Vec, + width: u32, + height: u32, + elapsed: Duration, +} + +async fn probe_bytes(mut client: EmulatorGrpcClient) -> Result { + let format = request_format(None); + let mut stream = client.stream_screenshots(format).await?; + collect(&mut stream, None).await +} + +async fn probe_mmap(mut client: EmulatorGrpcClient) -> Result { + let path = std::env::temp_dir().join(format!( + "accessibility-emulator-raw-{}.rgba", + std::process::id() + )); + let capacity = MAX_DIMENSION as u64 * MAX_DIMENSION as u64 * 4; + let file = OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(&path)?; + file.set_len(capacity)?; + let mut mapping = unsafe { MmapOptions::new().len(capacity as usize).map_mut(&file)? }; + let handle = format!("file://{}", path.display()); + let format = request_format(Some(ImageTransport { + channel: TransportChannel::Mmap as i32, + handle, + })); + let mut stream = client.stream_screenshots(format).await?; + let report = collect(&mut stream, Some(&mut mapping)).await; + drop(mapping); + drop(file); + let _ = std::fs::remove_file(path); + report +} + +fn request_format(transport: Option) -> ImageFormat { + ImageFormat { + format: ImgFormat::Rgba8888 as i32, + rotation: None, + width: MAX_DIMENSION, + height: MAX_DIMENSION, + display: 0, + transport, + } +} + +async fn collect( + stream: &mut tonic::Streaming, + mut mapping: Option<&mut memmap2::MmapMut>, +) -> Result { + let started = Instant::now(); + let mut report = Report::default(); + let mut previous_seq = None; + while started.elapsed() < PROBE_DURATION { + let image = match tokio::time::timeout(Duration::from_secs(2), stream.message()).await { + Ok(Ok(Some(image))) => image, + Ok(Ok(None)) => break, + Ok(Err(error)) => return Err(error.into()), + Err(_) if report.frames > 0 => continue, + Err(error) => return Err(error).context("timed out waiting for emulator frame"), + }; + let format = image.format.context("emulator frame has no format")?; + let width = format.width.max(image.width); + let height = format.height.max(image.height); + if width == 0 || height == 0 { + continue; + } + let expected = width as usize * height as usize * 4; + if let Some(mapping) = mapping.as_deref_mut() { + if expected > mapping.len() { + bail!("mapped frame {width}x{height} exceeds mapping"); + } + let first = checksum(&mapping[..expected]); + let owned = mapping[..expected].to_vec(); + let second = checksum(&mapping[..expected]); + report.tears += u64::from(first != second || first != checksum(&owned)); + report.bytes += owned.len() as u64; + } else { + if image.image.len() != expected { + bail!( + "RGBA frame is {} bytes, expected {expected} for {width}x{height}", + image.image.len() + ); + } + report.bytes += image.image.len() as u64; + } + if let Some(previous) = previous_seq { + report.gaps += image.seq.wrapping_sub(previous).saturating_sub(1) as u64; + } + previous_seq = Some(image.seq); + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_micros() as u64; + if image.timestamp_us > 0 && now >= image.timestamp_us { + report.latency_us.push(now - image.timestamp_us); + } + report.frames += 1; + report.width = width; + report.height = height; + } + report.elapsed = started.elapsed(); + Ok(report) +} + +fn checksum(bytes: &[u8]) -> u64 { + bytes + .chunks(4096) + .map(|chunk| chunk.first().copied().unwrap_or_default() as u64) + .sum() +} + +fn print_report(name: &str, report: &Report) { + let mean_latency = if report.latency_us.is_empty() { + 0.0 + } else { + report.latency_us.iter().sum::() as f64 / report.latency_us.len() as f64 + }; + println!("{name}:"); + println!(" geometry : {}x{}", report.width, report.height); + println!(" frames : {}", report.frames); + println!( + " fps : {:.1}", + report.frames as f64 / report.elapsed.as_secs_f64() + ); + println!(" copied : {} bytes", report.bytes); + println!(" seq gaps : {}", report.gaps); + println!(" tears : {}", report.tears); + println!(" latency : {:.1} ms", mean_latency / 1000.0); +} diff --git a/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs b/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs new file mode 100644 index 0000000..70d1a81 --- /dev/null +++ b/packages/accessibility-android-sys/examples/emulator_webrtc_probe.rs @@ -0,0 +1,282 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use accessibility_android_sys::emulator::protocol::rtc::Id; +use accessibility_android_sys::emulator::{EmulatorGrpcClient, discover_emulator}; +use anyhow::{Context, Result, anyhow, bail}; +use rtp::codecs::h264::H264Packet; +use serde_json::{Value, json}; +use tokio::sync::mpsc; +use webrtc::api::APIBuilder; +use webrtc::api::interceptor_registry::register_default_interceptors; +use webrtc::api::media_engine::{MIME_TYPE_H264, MediaEngine}; +use webrtc::ice_transport::ice_candidate::RTCIceCandidateInit; +use webrtc::interceptor::registry::Registry; +use webrtc::media::io::sample_builder::SampleBuilder; +use webrtc::peer_connection::RTCPeerConnection; +use webrtc::peer_connection::configuration::RTCConfiguration; +use webrtc::peer_connection::sdp::sdp_type::RTCSdpType; +use webrtc::peer_connection::sdp::session_description::RTCSessionDescription; +use webrtc::rtcp::payload_feedbacks::picture_loss_indication::PictureLossIndication; +use webrtc::rtp_transceiver::RTCPFeedback; +use webrtc::rtp_transceiver::rtp_codec::{ + RTCRtpCodecCapability, RTCRtpCodecParameters, RTPCodecType, +}; +use webrtc::track::track_remote::TrackRemote; + +const TRACK_TIMEOUT: Duration = Duration::from_secs(15); +const KEYFRAME_TIMEOUT: Duration = Duration::from_secs(10); +const RECOVERY_TIMEOUT: Duration = Duration::from_secs(5); + +#[tokio::main] +async fn main() -> Result<()> { + let selector = std::env::args().nth(1); + let discovery = discover_emulator(selector.as_deref())?; + println!("discovery : {}", discovery.path.display()); + println!("endpoint : {}", discovery.endpoint()); + + let mut grpc = EmulatorGrpcClient::connect(discovery).await?; + let status = grpc.status().await?; + println!("emulator : {}", status.version); + println!("booted : {}", status.booted); + if !status.booted { + bail!("Android Emulator has not finished booting"); + } + if let (Some(width), Some(height)) = ( + status.platform_config.get("hw.lcd.width"), + status.platform_config.get("hw.lcd.height"), + ) { + println!("display : {width}x{height}"); + } + + let id = grpc.begin_rtc_stream().await?; + println!("rtc id : {}", id.guid); + let messages = grpc.receive_jsep_stream(id.clone()).await?; + let peer = create_peer().await?; + let (track_tx, mut track_rx) = mpsc::channel(1); + peer.on_track(Box::new(move |track, _, _| { + let track_tx = track_tx.clone(); + Box::pin(async move { + let _ = track_tx.send(track).await; + }) + })); + + let signaling_peer = Arc::clone(&peer); + let signaling_grpc = grpc.clone(); + let signaling_id = id.clone(); + let mut signaling = tokio::spawn(async move { + run_signaling(signaling_peer, signaling_grpc, signaling_id, messages).await + }); + + let track = tokio::select! { + track = track_rx.recv() => track.ok_or_else(|| anyhow!("WebRTC peer closed before receiving a track"))?, + result = &mut signaling => return result.context("signaling task panicked")?, + _ = tokio::time::sleep(TRACK_TIMEOUT) => bail!("timed out waiting for the emulator video track"), + }; + let codec = track.codec().capability; + println!("codec : {} {}", codec.mime_type, codec.sdp_fmtp_line); + if !codec.mime_type.eq_ignore_ascii_case(MIME_TYPE_H264) { + bail!("emulator negotiated {}, not H.264", codec.mime_type); + } + + let report = read_frames(&peer, &track).await?; + println!("frames : {}", report.frames); + println!("keyframes : {}", report.keyframes); + println!("bytes : {}", report.bytes); + println!( + "fps : {:.1}", + report.frames as f64 / report.elapsed.as_secs_f64() + ); + println!( + "bitrate : {:.2} Mbps", + report.bytes as f64 * 8.0 / report.elapsed.as_secs_f64() / 1_000_000.0 + ); + println!( + "pli->idr : {:.1} ms", + report.recovery.as_secs_f64() * 1000.0 + ); + + grpc.end_rtc_stream(id).await?; + peer.close().await?; + signaling.abort(); + println!("probe passed"); + Ok(()) +} + +async fn create_peer() -> Result> { + let mut media_engine = MediaEngine::default(); + let feedback = vec![ + RTCPFeedback { + typ: "nack".to_owned(), + parameter: "".to_owned(), + }, + RTCPFeedback { + typ: "nack".to_owned(), + parameter: "pli".to_owned(), + }, + RTCPFeedback { + typ: "ccm".to_owned(), + parameter: "fir".to_owned(), + }, + ]; + for (payload_type, profile) in [(102, "42001f"), (125, "42e01f"), (123, "640032")] { + media_engine.register_codec( + RTCRtpCodecParameters { + capability: RTCRtpCodecCapability { + mime_type: MIME_TYPE_H264.to_owned(), + clock_rate: 90000, + channels: 0, + sdp_fmtp_line: format!( + "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id={profile}" + ), + rtcp_feedback: feedback.clone(), + }, + payload_type, + ..Default::default() + }, + RTPCodecType::Video, + )?; + } + let registry = register_default_interceptors(Registry::new(), &mut media_engine)?; + let api = APIBuilder::new() + .with_media_engine(media_engine) + .with_interceptor_registry(registry) + .build(); + Ok(Arc::new( + api.new_peer_connection(RTCConfiguration::default()).await?, + )) +} + +async fn run_signaling( + peer: Arc, + mut grpc: EmulatorGrpcClient, + id: Id, + mut messages: tonic::Streaming< + accessibility_android_sys::emulator::protocol::rtc::ReceiveJsepMessageResponse, + >, +) -> Result<()> { + let mut remote_description_set = false; + let mut pending_candidates = Vec::new(); + while let Some(response) = messages.message().await? { + let Some(jsep) = response.jsep_msg else { + continue; + }; + if jsep.message.is_empty() { + continue; + } + let message: Value = serde_json::from_str(&jsep.message) + .with_context(|| format!("invalid emulator JSEP message: {}", jsep.message))?; + if message.get("bye").is_some() { + bail!("emulator ended the RTC stream before the probe completed"); + } + if let Some(value) = message.get("candidate") { + let candidate: RTCIceCandidateInit = serde_json::from_value(value.clone())?; + if remote_description_set { + peer.add_ice_candidate(candidate).await?; + } else { + pending_candidates.push(candidate); + } + } + let Some(value) = message.get("sdp") else { + continue; + }; + let description: RTCSessionDescription = serde_json::from_value(value.clone())?; + if description.sdp_type != RTCSdpType::Offer { + continue; + } + peer.set_remote_description(description).await?; + remote_description_set = true; + for candidate in pending_candidates.drain(..) { + peer.add_ice_candidate(candidate).await?; + } + let answer = peer.create_answer(None).await?; + let mut gathering_complete = peer.gathering_complete_promise().await; + peer.set_local_description(answer).await?; + let _ = gathering_complete.recv().await; + let answer = peer + .local_description() + .await + .ok_or_else(|| anyhow!("WebRTC peer produced no local answer"))?; + grpc.send_jsep(id.clone(), json!({ "sdp": answer }).to_string()) + .await?; + } + bail!("emulator JSEP stream closed") +} + +struct ProbeReport { + frames: u64, + keyframes: u64, + bytes: u64, + elapsed: Duration, + recovery: Duration, +} + +async fn read_frames(peer: &RTCPeerConnection, track: &TrackRemote) -> Result { + let started = Instant::now(); + let mut builder = SampleBuilder::new(16, H264Packet::default(), 90000) + .with_max_time_delay(Duration::from_millis(250)); + let mut frames = 0; + let mut keyframes = 0; + let mut bytes = 0; + let mut requested_at: Option = None; + + loop { + let timeout = if requested_at.is_some() { + RECOVERY_TIMEOUT + } else { + KEYFRAME_TIMEOUT + }; + let (packet, _) = tokio::time::timeout(timeout, track.read_rtp()) + .await + .context("timed out waiting for H.264 RTP")??; + builder.push(packet); + while let Some(sample) = builder.pop() { + let types = annex_b_nal_types(&sample.data); + let keyframe = types.contains(&5); + if keyframe && (!types.contains(&7) || !types.contains(&8)) { + bail!("H.264 keyframe did not carry SPS and PPS"); + } + frames += 1; + bytes += sample.data.len() as u64; + if !keyframe { + continue; + } + keyframes += 1; + if let Some(requested_at) = requested_at { + return Ok(ProbeReport { + frames, + keyframes, + bytes, + elapsed: started.elapsed(), + recovery: requested_at.elapsed(), + }); + } + peer.write_rtcp(&[Box::new(PictureLossIndication { + sender_ssrc: 0, + media_ssrc: track.ssrc(), + })]) + .await?; + requested_at = Some(Instant::now()); + } + } +} + +fn annex_b_nal_types(data: &[u8]) -> Vec { + let mut types = Vec::new(); + let mut index = 0; + while index + 3 < data.len() { + let start_len = if data[index..].starts_with(&[0, 0, 0, 1]) { + 4 + } else if data[index..].starts_with(&[0, 0, 1]) { + 3 + } else { + index += 1; + continue; + }; + if let Some(header) = data.get(index + start_len) { + types.push(header & 0x1f); + } + index += start_len + 1; + } + types +} diff --git a/packages/accessibility-android-sys/examples/screenrecord_probe.rs b/packages/accessibility-android-sys/examples/screenrecord_probe.rs new file mode 100644 index 0000000..3df6839 --- /dev/null +++ b/packages/accessibility-android-sys/examples/screenrecord_probe.rs @@ -0,0 +1,140 @@ +use std::io::Read; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +use accessibility_android_sys::AdbClient; +use accessibility_android_sys::emulator::screenrecord::{ + AnnexBAccessUnitParser, H264AccessUnit, ScreenRecordConfig, spawn_screenrecord, +}; +use anyhow::{Context, Result, bail}; + +const IDLE_FLUSH: Duration = Duration::from_millis(75); +const PROBE_DURATION: Duration = Duration::from_secs(5); + +fn main() -> Result<()> { + let serial = std::env::args() + .nth(1) + .unwrap_or_else(|| "emulator-5554".to_string()); + let adb = AdbClient::discover(Some(&serial)); + adb.check_connection()?; + let (width, height) = adb.get_screen_size()?; + let config = ScreenRecordConfig::for_max_dimension(width, height, Some(1280), 5_000_000); + println!("device : {serial}"); + println!("source : {width}x{height}"); + println!("encoded : {}x{}", config.width, config.height); + + let stimulus = adb.clone(); + std::thread::spawn(move || { + std::thread::sleep(Duration::from_secs(1)); + let _ = stimulus.swipe( + (width as f64 * 0.5, height as f64 * 0.75), + (width as f64 * 0.5, height as f64 * 0.25), + 800, + ); + }); + + let started = Instant::now(); + let frames = capture(&adb, config, PROBE_DURATION, false)?; + let elapsed = started.elapsed(); + let keyframes = frames.iter().filter(|frame| frame.keyframe).count(); + let bytes = frames.iter().map(|frame| frame.data.len()).sum::(); + if frames.is_empty() { + bail!("screenrecord produced no complete H.264 frames"); + } + if keyframes == 0 { + bail!("screenrecord produced no H.264 keyframe"); + } + println!("frames : {}", frames.len()); + println!("keyframes : {keyframes}"); + println!("bytes : {bytes}"); + println!( + "fps : {:.1}", + frames.len() as f64 / elapsed.as_secs_f64() + ); + println!( + "bitrate : {:.2} Mbps", + bytes as f64 * 8.0 / elapsed.as_secs_f64() / 1_000_000.0 + ); + + let restart = Instant::now(); + let restarted = capture(&adb, config, Duration::from_secs(2), true)?; + let first = restarted + .iter() + .find(|frame| frame.keyframe) + .context("fresh screenrecord session produced no keyframe")?; + println!( + "restart : {:.1} ms", + restart.elapsed().as_secs_f64() * 1000.0 + ); + println!("entry NALs: {:?}", nal_types(&first.data)); + println!("probe passed"); + Ok(()) +} + +fn capture( + adb: &AdbClient, + config: ScreenRecordConfig, + duration: Duration, + stop_on_keyframe: bool, +) -> Result> { + let mut child = spawn_screenrecord(adb, config)?; + let mut stdout = child + .stdout + .take() + .context("screenrecord stdout was not piped")?; + let (tx, rx) = mpsc::sync_channel::>(8); + let reader = std::thread::spawn(move || { + let mut buffer = vec![0u8; 64 * 1024]; + loop { + match stdout.read(&mut buffer) { + Ok(0) | Err(_) => break, + Ok(read) => { + if tx.send(buffer[..read].to_vec()).is_err() { + break; + } + } + } + } + }); + + let started = Instant::now(); + let mut parser = AnnexBAccessUnitParser::default(); + let mut frames = Vec::new(); + while started.elapsed() < duration { + match rx.recv_timeout(IDLE_FLUSH) { + Ok(chunk) => frames.extend(parser.push(&chunk)), + Err(mpsc::RecvTimeoutError::Timeout) => frames.extend(parser.flush_idle()), + Err(mpsc::RecvTimeoutError::Disconnected) => break, + } + if stop_on_keyframe && frames.iter().any(|frame| frame.keyframe) { + break; + } + } + frames.extend(parser.flush_idle()); + child.kill().context("failed to stop screenrecord")?; + let _ = child.wait(); + reader + .join() + .map_err(|_| anyhow::anyhow!("screenrecord reader panicked"))?; + Ok(frames) +} + +fn nal_types(data: &[u8]) -> Vec { + let mut types = Vec::new(); + let mut index = 0; + while index + 3 < data.len() { + let code_len = if data[index..].starts_with(&[0, 0, 0, 1]) { + 4 + } else if data[index..].starts_with(&[0, 0, 1]) { + 3 + } else { + index += 1; + continue; + }; + if let Some(header) = data.get(index + code_len) { + types.push(header & 0x1f); + } + index += code_len + 1; + } + types +} diff --git a/packages/accessibility-android-sys/proto/emulator_controller.proto b/packages/accessibility-android-sys/proto/emulator_controller.proto new file mode 100644 index 0000000..61ac2c4 --- /dev/null +++ b/packages/accessibility-android-sys/proto/emulator_controller.proto @@ -0,0 +1,135 @@ +syntax = "proto3"; + +package android.emulation.control; + +import "google/protobuf/empty.proto"; + +service EmulatorController { + rpc getStatus(google.protobuf.Empty) returns (EmulatorStatus) {} + rpc sendKey(KeyboardEvent) returns (google.protobuf.Empty) {} + rpc sendTouch(TouchEvent) returns (google.protobuf.Empty) {} + rpc sendMouse(MouseEvent) returns (google.protobuf.Empty) {} + rpc streamScreenshot(ImageFormat) returns (stream Image) {} +} + +message EmulatorStatus { + string version = 1; + uint64 uptime = 2; + bool booted = 3; + uint64 heartbeat = 6; + map guestConfig = 7; + map platformConfig = 8; +} + +message Touch { + int32 x = 1; + int32 y = 2; + int32 identifier = 3; + int32 pressure = 4; + int32 touch_major = 5; + int32 touch_minor = 6; + EventExpiration expiration = 7; + int32 orientation = 8; + + enum EventExpiration { + EVENT_EXPIRATION_UNSPECIFIED = 0; + NEVER_EXPIRE = 1; + } +} + +message TouchEvent { + repeated Touch touches = 1; + int32 display = 2; +} + +message MouseEvent { + int32 x = 1; + int32 y = 2; + int32 buttons = 3; + int32 display = 4; +} + +message WheelEvent { + int32 dx = 1; + int32 dy = 2; + int32 display = 3; +} + +message KeyboardEvent { + KeyCodeType codeType = 1; + KeyEventType eventType = 2; + int32 keyCode = 3; + string key = 4; + string text = 5; + + enum KeyCodeType { + Usb = 0; + Evdev = 1; + XKB = 2; + Win = 3; + Mac = 4; + } + + enum KeyEventType { + keydown = 0; + keyup = 1; + keypress = 2; + } +} + +message ImageTransport { + enum TransportChannel { + TRANSPORT_CHANNEL_UNSPECIFIED = 0; + MMAP = 1; + } + + TransportChannel channel = 1; + string handle = 2; +} + +message Rotation { + enum SkinRotation { + PORTRAIT = 0; + LANDSCAPE = 1; + REVERSE_PORTRAIT = 2; + REVERSE_LANDSCAPE = 3; + } + + SkinRotation rotation = 1; + double xAxis = 2; + double yAxis = 3; + double zAxis = 4; +} + +message ImageFormat { + enum ImgFormat { + PNG = 0; + RGBA8888 = 1; + RGB888 = 2; + } + + ImgFormat format = 1; + Rotation rotation = 2; + uint32 width = 3; + uint32 height = 4; + uint32 display = 5; + ImageTransport transport = 6; +} + +message Image { + ImageFormat format = 1; + uint32 width = 2; + uint32 height = 3; + bytes image = 4; + uint32 seq = 5; + uint64 timestampUs = 6; +} + +message InputEvent { + oneof type { + KeyboardEvent key_event = 1; + TouchEvent touch_event = 2; + MouseEvent mouse_event = 3; + WheelEvent wheel_event = 6; + } +} diff --git a/packages/accessibility-android-sys/proto/rtc_service_v2.proto b/packages/accessibility-android-sys/proto/rtc_service_v2.proto new file mode 100644 index 0000000..d654a12 --- /dev/null +++ b/packages/accessibility-android-sys/proto/rtc_service_v2.proto @@ -0,0 +1,38 @@ +syntax = "proto3"; + +package android.emulation.control.v2; + +service Rtc { + rpc RequestRtcStream(RtcStreamRequest) returns (RtcStreamResponse) {} + rpc SendJsepMessage(SendJsepMessageRequest) returns (SendJsepMessageResponse) {} + rpc ReceiveJsepMessageStream(ReceiveJsepMessageRequest) returns (stream ReceiveJsepMessageResponse) {} +} + +message RtcStreamRequest {} + +message RtcStreamResponse { + Id id = 1; +} + +message SendJsepMessageRequest { + JsepMsg jsep_msg = 1; +} + +message SendJsepMessageResponse {} + +message ReceiveJsepMessageRequest { + Id id = 1; +} + +message ReceiveJsepMessageResponse { + JsepMsg jsep_msg = 1; +} + +message Id { + string guid = 1; +} + +message JsepMsg { + Id id = 1; + string message = 2; +} diff --git a/packages/accessibility-android-sys/src/emulator.rs b/packages/accessibility-android-sys/src/emulator.rs new file mode 100644 index 0000000..b0292a2 --- /dev/null +++ b/packages/accessibility-android-sys/src/emulator.rs @@ -0,0 +1,390 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail}; +use tonic::metadata::MetadataValue; +use tonic::transport::{Channel, Endpoint}; +use tonic::{Request, Streaming}; + +pub mod raw; +pub mod screenrecord; + +pub mod protocol { + pub mod controller { + tonic::include_proto!("android.emulation.control"); + } + + pub mod rtc { + tonic::include_proto!("android.emulation.control.v2"); + } +} + +use protocol::controller::emulator_controller_client::EmulatorControllerClient; +use protocol::controller::{EmulatorStatus, Image, ImageFormat, InputEvent, input_event}; +use protocol::rtc::rtc_client::RtcClient; +use protocol::rtc::{ + Id, JsepMsg, ReceiveJsepMessageRequest, ReceiveJsepMessageResponse, RtcStreamRequest, + SendJsepMessageRequest, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EmulatorDiscovery { + pub path: PathBuf, + pub pid: Option, + pub grpc_port: u16, + pub grpc_token: Option, + pub properties: BTreeMap, +} + +impl EmulatorDiscovery { + pub fn from_file(path: impl AsRef) -> Result { + let path = path.as_ref(); + let contents = std::fs::read_to_string(path).with_context(|| { + format!("failed to read emulator discovery file {}", path.display()) + })?; + let properties = parse_properties(&contents); + let grpc_port = properties + .get("grpc.port") + .ok_or_else(|| anyhow!("{} has no grpc.port", path.display()))? + .parse::() + .with_context(|| format!("{} has an invalid grpc.port", path.display()))?; + let grpc_token = properties + .get("grpc.token") + .filter(|token| !token.is_empty()) + .cloned(); + let pid = path + .file_stem() + .and_then(|name| name.to_str()) + .and_then(|name| name.strip_prefix("pid_")) + .and_then(|pid| pid.parse().ok()); + Ok(Self { + path: path.to_path_buf(), + pid, + grpc_port, + grpc_token, + properties, + }) + } + + pub fn matches(&self, selector: &str) -> bool { + if self + .properties + .values() + .any(|value| value.eq_ignore_ascii_case(selector)) + { + return true; + } + if self.pid.is_some_and(|pid| selector == pid.to_string()) { + return true; + } + self.properties.get("port.serial").is_some_and(|port| { + selector == port || selector.eq_ignore_ascii_case(&format!("emulator-{port}")) + }) + } + + pub fn endpoint(&self) -> String { + format!("http://127.0.0.1:{}", self.grpc_port) + } +} + +#[derive(Debug, Clone)] +pub struct EmulatorCapabilities { + pub discovery: EmulatorDiscovery, + pub version: String, + pub booted: bool, + pub rtc_v2: bool, +} + +#[derive(Clone)] +pub struct EmulatorGrpcClient { + discovery: EmulatorDiscovery, + controller: EmulatorControllerClient, + rtc: RtcClient, +} + +impl EmulatorGrpcClient { + pub async fn connect(discovery: EmulatorDiscovery) -> Result { + let channel = Endpoint::from_shared(discovery.endpoint())? + .connect_timeout(Duration::from_secs(3)) + .timeout(Duration::from_secs(10)) + .connect() + .await + .with_context(|| { + format!( + "failed to connect to Android Emulator gRPC endpoint {}", + discovery.endpoint() + ) + })?; + Ok(Self { + discovery, + controller: EmulatorControllerClient::new(channel.clone()), + rtc: RtcClient::new(channel), + }) + } + + pub fn discovery(&self) -> &EmulatorDiscovery { + &self.discovery + } + + pub async fn status(&mut self) -> Result { + let request = self.authorized(())?; + self.controller + .get_status(request) + .await + .map(|response| response.into_inner()) + .context("Android Emulator getStatus failed") + } + + pub async fn stream_screenshots(&mut self, format: ImageFormat) -> Result> { + let request = self.authorized(format)?; + self.controller + .stream_screenshot(request) + .await + .map(|response| response.into_inner()) + .context("Android Emulator streamScreenshot failed") + } + + pub async fn send_input(&mut self, event: InputEvent) -> Result<()> { + match event + .r#type + .ok_or_else(|| anyhow!("Android input event has no type"))? + { + input_event::Type::KeyEvent(event) => { + let request = self.authorized(event)?; + self.controller + .send_key(request) + .await + .context("Android Emulator sendKey failed")?; + } + input_event::Type::TouchEvent(event) => { + let request = self.authorized(event)?; + self.controller + .send_touch(request) + .await + .context("Android Emulator sendTouch failed")?; + } + input_event::Type::MouseEvent(event) => { + let request = self.authorized(event)?; + self.controller + .send_mouse(request) + .await + .context("Android Emulator sendMouse failed")?; + } + input_event::Type::WheelEvent(_) => { + bail!("Android Emulator wheel input requires a streaming RPC") + } + } + Ok(()) + } + + pub async fn begin_rtc_stream(&mut self) -> Result { + let request = self.authorized(RtcStreamRequest {})?; + let response = self + .rtc + .request_rtc_stream(request) + .await + .context("Android Emulator RTC v2 RequestRtcStream failed")? + .into_inner(); + let id = response + .id + .filter(|id| !id.guid.is_empty()) + .ok_or_else(|| anyhow!("Android Emulator RTC v2 returned no stream id"))?; + Ok(id) + } + + pub async fn receive_jsep_stream( + &mut self, + id: Id, + ) -> Result> { + let request = self.authorized(ReceiveJsepMessageRequest { id: Some(id) })?; + self.rtc + .receive_jsep_message_stream(request) + .await + .map(|response| response.into_inner()) + .context("Android Emulator RTC v2 ReceiveJsepMessageStream failed") + } + + pub async fn send_jsep(&mut self, id: Id, message: impl Into) -> Result<()> { + let request = self.authorized(SendJsepMessageRequest { + jsep_msg: Some(JsepMsg { + id: Some(id), + message: message.into(), + }), + })?; + self.rtc + .send_jsep_message(request) + .await + .context("Android Emulator RTC v2 SendJsepMessage failed")?; + Ok(()) + } + + pub async fn end_rtc_stream(&mut self, id: Id) -> Result<()> { + self.send_jsep(id, r#"{"bye":true}"#).await + } + + pub async fn probe_capabilities(&mut self) -> Result { + let status = self.status().await?; + let id = self.begin_rtc_stream().await?; + self.end_rtc_stream(id).await?; + Ok(EmulatorCapabilities { + discovery: self.discovery.clone(), + version: status.version, + booted: status.booted, + rtc_v2: true, + }) + } + + fn authorized(&self, value: T) -> Result> { + let mut request = Request::new(value); + if let Some(token) = &self.discovery.grpc_token { + let value = MetadataValue::try_from(format!("Bearer {token}")) + .context("emulator gRPC token is not valid HTTP metadata")?; + request.metadata_mut().insert("authorization", value); + } + Ok(request) + } +} + +pub fn discover_emulator(selector: Option<&str>) -> Result { + discover_emulator_in(&discovery_directories(), selector) +} + +pub fn discover_emulator_in( + directories: &[PathBuf], + selector: Option<&str>, +) -> Result { + let mut paths = BTreeSet::new(); + for directory in directories { + let entries = match std::fs::read_dir(directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to read emulator discovery directory {}", + directory.display() + ) + }); + } + }; + for entry in entries { + let path = entry?.path(); + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if name.starts_with("pid_") && name.ends_with(".ini") { + paths.insert(path); + } + } + } + + let mut discoveries = paths + .into_iter() + .filter_map(|path| EmulatorDiscovery::from_file(path).ok()) + .collect::>(); + if let Some(selector) = selector { + discoveries.retain(|discovery| discovery.matches(selector)); + } + + match discoveries.len() { + 0 => match selector { + Some(selector) => bail!("no running Android Emulator matches '{selector}'"), + None => bail!("no running Android Emulator with a gRPC discovery file was found"), + }, + 1 => Ok(discoveries.remove(0)), + count => match selector { + Some(selector) => bail!("{count} running Android Emulators match '{selector}'"), + None => bail!( + "{count} Android Emulators are running; specify an AVD name, PID, serial, or gRPC port" + ), + }, + } +} + +pub fn discovery_directories() -> Vec { + let mut directories = BTreeSet::new(); + directories.insert(std::env::temp_dir().join("avd/running")); + if let Some(path) = std::env::var_os("ANDROID_EMULATOR_DISCOVERY") { + directories.insert(PathBuf::from(path)); + } + if let Some(path) = std::env::var_os("ANDROID_AVD_HOME") { + directories.insert(PathBuf::from(path).join("running")); + } + if let Some(home) = std::env::var_os("HOME") { + let home = PathBuf::from(home); + directories.insert(home.join(".android/avd/running")); + if cfg!(target_os = "macos") { + directories.insert(home.join("Library/Android/avd/running")); + directories.insert(home.join("Library/Caches/TemporaryItems/avd/running")); + } + } + directories.into_iter().collect() +} + +fn parse_properties(contents: &str) -> BTreeMap { + contents + .lines() + .filter_map(|line| { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') || line.starts_with(';') { + return None; + } + let (key, value) = line.split_once('=')?; + Some((key.trim().to_string(), value.trim().to_string())) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_directory(name: &str) -> PathBuf { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!("accessibility-android-{name}-{nonce}")); + std::fs::create_dir_all(&path).unwrap(); + path + } + + #[test] + fn parses_discovery_file() { + let directory = test_directory("parse"); + let path = directory.join("pid_1234.ini"); + std::fs::write( + &path, + "grpc.port = 8554\ngrpc.token = secret\navd.name = Pixel_8\nport.serial = 5554\n", + ) + .unwrap(); + let discovery = EmulatorDiscovery::from_file(&path).unwrap(); + assert_eq!(discovery.pid, Some(1234)); + assert_eq!(discovery.grpc_port, 8554); + assert_eq!(discovery.grpc_token.as_deref(), Some("secret")); + assert!(discovery.matches("Pixel_8")); + assert!(discovery.matches("emulator-5554")); + std::fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn selects_one_emulator() { + let directory = test_directory("select"); + std::fs::write( + directory.join("pid_1.ini"), + "grpc.port=8554\navd.name=phone\n", + ) + .unwrap(); + std::fs::write( + directory.join("pid_2.ini"), + "grpc.port=8555\navd.name=tablet\n", + ) + .unwrap(); + let discovery = + discover_emulator_in(std::slice::from_ref(&directory), Some("tablet")).unwrap(); + assert_eq!(discovery.grpc_port, 8555); + assert!(discover_emulator_in(std::slice::from_ref(&directory), None).is_err()); + std::fs::remove_dir_all(directory).unwrap(); + } +} diff --git a/packages/accessibility-android-sys/src/emulator/raw.rs b/packages/accessibility-android-sys/src/emulator/raw.rs new file mode 100644 index 0000000..20e5be5 --- /dev/null +++ b/packages/accessibility-android-sys/src/emulator/raw.rs @@ -0,0 +1,226 @@ +use std::fs::{File, OpenOptions}; +use std::path::PathBuf; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, anyhow, bail}; +use memmap2::{MmapMut, MmapOptions}; +use tonic::Streaming; + +use super::protocol::controller::image_format::ImgFormat; +use super::protocol::controller::image_transport::TransportChannel; +use super::protocol::controller::{Image, ImageFormat, ImageTransport}; +use super::{EmulatorDiscovery, EmulatorGrpcClient, discover_emulator}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RawFrameTransport { + Grpc, + Mmap, +} + +#[derive(Debug, Clone, Copy)] +pub struct RawFrameConfig { + pub width: u32, + pub height: u32, + pub display: u32, + pub transport: RawFrameTransport, +} + +#[derive(Debug)] +pub struct RawFrame { + pub pixels: Vec, + pub width: u32, + pub height: u32, + pub sequence: u32, + pub timestamp_us: u64, + pub rotation: i32, +} + +pub struct RawFrameStream { + discovery: EmulatorDiscovery, + stream: Streaming, + mapping: Option, + mapping_file: Option, + mapping_path: Option, +} + +impl RawFrameStream { + pub async fn start(selector: Option<&str>, config: RawFrameConfig) -> Result { + if config.width == 0 || config.height == 0 { + bail!("raw Android capture requires non-zero dimensions"); + } + let discovery = discover_emulator(selector)?; + Self::start_with_discovery(discovery, config).await + } + + pub async fn start_with_discovery( + discovery: EmulatorDiscovery, + config: RawFrameConfig, + ) -> Result { + let mut client = EmulatorGrpcClient::connect(discovery.clone()).await?; + let capacity = rgba_len(config.width, config.height)?; + let (mapping, mapping_file, mapping_path, transport) = match config.transport { + RawFrameTransport::Grpc => (None, None, None, None), + RawFrameTransport::Mmap => { + let path = mapping_path(); + let file = OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(&path) + .with_context(|| { + format!("creating emulator frame mapping {}", path.display()) + })?; + file.set_len(capacity as u64)?; + let mapping = unsafe { MmapOptions::new().len(capacity).map_mut(&file)? }; + let transport = ImageTransport { + channel: TransportChannel::Mmap as i32, + handle: file_uri(&path), + }; + (Some(mapping), Some(file), Some(path), Some(transport)) + } + }; + let stream = client + .stream_screenshots(ImageFormat { + format: ImgFormat::Rgba8888 as i32, + rotation: None, + width: config.width, + height: config.height, + display: config.display, + transport, + }) + .await?; + Ok(Self { + discovery, + stream, + mapping, + mapping_file, + mapping_path, + }) + } + + pub fn discovery(&self) -> &EmulatorDiscovery { + &self.discovery + } + + pub async fn next_frame(&mut self) -> Result> { + loop { + let Some(image) = self.stream.message().await? else { + return Ok(None); + }; + let format = image.format.context("emulator frame has no format")?; + let width = format.width.max(image.width); + let height = format.height.max(image.height); + if width == 0 || height == 0 { + continue; + } + let expected = rgba_len(width, height)?; + let mut pixels = if let Some(mapping) = &self.mapping { + let source = mapping + .get(..expected) + .ok_or_else(|| anyhow!("emulator frame exceeds its memory mapping"))?; + copy_bottom_up_rgba(source, width, height) + } else { + if image.image.len() != expected { + bail!( + "RGBA frame is {} bytes, expected {expected} for {width}x{height}", + image.image.len() + ); + } + image.image + }; + if self.mapping.is_none() { + flip_vertical_rgba(&mut pixels, width, height); + } + return Ok(Some(RawFrame { + pixels, + width, + height, + sequence: image.seq, + timestamp_us: image.timestamp_us, + rotation: format.rotation.map_or(0, |rotation| rotation.rotation), + })); + } + } +} + +impl Drop for RawFrameStream { + fn drop(&mut self) { + self.mapping.take(); + self.mapping_file.take(); + if let Some(path) = self.mapping_path.take() { + let _ = std::fs::remove_file(path); + } + } +} + +fn rgba_len(width: u32, height: u32) -> Result { + width + .checked_mul(height) + .and_then(|pixels| pixels.checked_mul(4)) + .map(|bytes| bytes as usize) + .context("Android frame dimensions overflow") +} + +fn copy_bottom_up_rgba(source: &[u8], width: u32, height: u32) -> Vec { + let stride = width as usize * 4; + let mut pixels = vec![0; source.len()]; + for row in 0..height as usize { + let source_start = (height as usize - 1 - row) * stride; + let target_start = row * stride; + pixels[target_start..target_start + stride] + .copy_from_slice(&source[source_start..source_start + stride]); + } + pixels +} + +fn flip_vertical_rgba(pixels: &mut [u8], width: u32, height: u32) { + let stride = width as usize * 4; + for row in 0..height as usize / 2 { + let opposite = height as usize - 1 - row; + let (before, after) = pixels.split_at_mut(opposite * stride); + before[row * stride..(row + 1) * stride].swap_with_slice(&mut after[..stride]); + } +} + +fn mapping_path() -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or_default(); + std::env::temp_dir().join(format!( + "accessibility-emulator-raw-{}-{nonce}.rgba", + std::process::id() + )) +} + +fn file_uri(path: &std::path::Path) -> String { + #[cfg(target_os = "windows")] + { + format!("file:///{}", path.display().to_string().replace('\\', "/")) + } + #[cfg(not(target_os = "windows"))] + { + format!("file://{}", path.display()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn flips_bottom_up_rgba() { + let mut pixels = vec![1, 2, 3, 4, 5, 6, 7, 8]; + flip_vertical_rgba(&mut pixels, 1, 2); + assert_eq!(pixels, vec![5, 6, 7, 8, 1, 2, 3, 4]); + } + + #[test] + fn copies_bottom_up_rgba() { + assert_eq!( + copy_bottom_up_rgba(&[1, 2, 3, 4, 5, 6, 7, 8], 1, 2), + vec![5, 6, 7, 8, 1, 2, 3, 4] + ); + } +} diff --git a/packages/accessibility-android-sys/src/emulator/screenrecord.rs b/packages/accessibility-android-sys/src/emulator/screenrecord.rs new file mode 100644 index 0000000..7e5f086 --- /dev/null +++ b/packages/accessibility-android-sys/src/emulator/screenrecord.rs @@ -0,0 +1,368 @@ +use std::process::{Child, Command, Stdio}; + +use anyhow::{Context, Result}; +use bytes::Bytes; + +use crate::AdbClient; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ScreenRecordConfig { + pub width: u32, + pub height: u32, + pub bit_rate: u32, + pub display_id: Option, +} + +impl ScreenRecordConfig { + pub fn for_max_dimension( + width: u32, + height: u32, + max_dimension: Option, + bit_rate: u32, + ) -> Self { + let (width, height) = scaled_even_size(width, height, max_dimension); + Self { + width, + height, + bit_rate, + display_id: None, + } + } +} + +pub fn spawn_screenrecord(adb: &AdbClient, config: ScreenRecordConfig) -> Result { + let mut command = Command::new(&adb.adb_path); + if let Some(serial) = &adb.serial { + command.arg("-s").arg(serial); + } + command.args([ + "exec-out", + "screenrecord", + "--output-format=h264", + "--time-limit", + "0", + "--size", + &format!("{}x{}", config.width, config.height), + "--bit-rate", + &config.bit_rate.to_string(), + ]); + if let Some(display_id) = config.display_id { + command.args(["--display-id", &display_id.to_string()]); + } + command + .arg("-") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .context("failed to start adb screenrecord") +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct H264AccessUnit { + pub data: Bytes, + pub keyframe: bool, +} + +#[derive(Default)] +pub struct AnnexBAccessUnitParser { + stream: Vec, + assembler: AccessUnitAssembler, +} + +impl AnnexBAccessUnitParser { + pub fn push(&mut self, chunk: &[u8]) -> Vec { + self.stream.extend_from_slice(chunk); + let starts = start_codes(&self.stream); + if starts.is_empty() { + if self.stream.len() > 4 { + let keep = self.stream.split_off(self.stream.len() - 4); + self.stream = keep; + } + return Vec::new(); + } + if starts[0].0 > 0 { + self.stream.drain(..starts[0].0); + } + let starts = start_codes(&self.stream); + if starts.len() < 2 { + return Vec::new(); + } + + let mut out = Vec::new(); + for pair in starts.windows(2) { + let (start, code_len) = pair[0]; + let end = pair[1].0; + if let Some(frame) = self + .assembler + .push_nalu(trim_trailing_zeroes(&self.stream[start + code_len..end])) + { + out.push(frame); + } + } + self.stream.drain(..starts.last().unwrap().0); + out + } + + pub fn flush_idle(&mut self) -> Vec { + let mut out = Vec::new(); + if let Some((start, code_len)) = start_codes(&self.stream).first().copied() { + let nalu = trim_trailing_zeroes(&self.stream[start + code_len..]); + if !nalu.is_empty() + && let Some(frame) = self.assembler.push_nalu(nalu) + { + out.push(frame); + } + } + self.stream.clear(); + if let Some(frame) = self.assembler.flush() { + out.push(frame); + } + out + } +} + +#[derive(Default)] +struct AccessUnitAssembler { + data: Vec, + has_vcl: bool, + keyframe: bool, + has_sps: bool, + has_pps: bool, + latest_sps: Option>, + latest_pps: Option>, +} + +impl AccessUnitAssembler { + fn push_nalu(&mut self, nalu: &[u8]) -> Option { + let nalu_type = nalu.first().map(|byte| byte & 0x1f)?; + let starts_new_picture = + matches!(nalu_type, 1 | 5) && self.has_vcl && first_mb_in_slice(nalu) == Some(0); + let starts_new_prefix = self.has_vcl && matches!(nalu_type, 6..=9); + let completed = (starts_new_picture || starts_new_prefix) + .then(|| self.take_frame()) + .flatten(); + + match nalu_type { + 1 | 5 => { + self.has_vcl = true; + self.keyframe |= nalu_type == 5; + } + 7 => { + self.has_sps = true; + self.latest_sps = Some(nalu.to_vec()); + } + 8 => { + self.has_pps = true; + self.latest_pps = Some(nalu.to_vec()); + } + _ => {} + } + append_nalu(&mut self.data, nalu); + completed + } + + fn flush(&mut self) -> Option { + self.take_frame() + } + + fn take_frame(&mut self) -> Option { + if !self.has_vcl { + return None; + } + let mut data = Vec::new(); + if self.keyframe { + if !self.has_sps + && let Some(sps) = &self.latest_sps + { + append_nalu(&mut data, sps); + } + if !self.has_pps + && let Some(pps) = &self.latest_pps + { + append_nalu(&mut data, pps); + } + } + data.append(&mut self.data); + let frame = H264AccessUnit { + data: Bytes::from(data), + keyframe: self.keyframe, + }; + self.has_vcl = false; + self.keyframe = false; + self.has_sps = false; + self.has_pps = false; + Some(frame) + } +} + +fn append_nalu(out: &mut Vec, nalu: &[u8]) { + out.extend_from_slice(&[0, 0, 0, 1]); + out.extend_from_slice(nalu); +} + +fn start_codes(data: &[u8]) -> Vec<(usize, usize)> { + let mut starts = Vec::new(); + let mut index = 0; + while index + 3 <= data.len() { + if index + 4 <= data.len() && data[index..index + 4] == [0, 0, 0, 1] { + starts.push((index, 4)); + index += 4; + } else if data[index..index + 3] == [0, 0, 1] { + starts.push((index, 3)); + index += 3; + } else { + index += 1; + } + } + starts +} + +fn trim_trailing_zeroes(mut data: &[u8]) -> &[u8] { + while data.last() == Some(&0) { + data = &data[..data.len() - 1]; + } + data +} + +fn first_mb_in_slice(nalu: &[u8]) -> Option { + if !matches!(nalu.first().map(|byte| byte & 0x1f), Some(1 | 5)) { + return None; + } + let mut rbsp = Vec::with_capacity(nalu.len().saturating_sub(1)); + let mut zeroes = 0; + for &byte in nalu.get(1..)? { + if zeroes >= 2 && byte == 3 { + zeroes = 0; + continue; + } + rbsp.push(byte); + if byte == 0 { + zeroes += 1; + } else { + zeroes = 0; + } + } + read_unsigned_exp_golomb(&rbsp) +} + +fn read_unsigned_exp_golomb(data: &[u8]) -> Option { + let mut bit = 0usize; + let mut leading_zeroes = 0u32; + while bit < data.len() * 8 && !read_bit(data, bit)? { + leading_zeroes += 1; + bit += 1; + if leading_zeroes >= 32 { + return None; + } + } + bit += 1; + let mut suffix = 0u32; + for _ in 0..leading_zeroes { + suffix = (suffix << 1) | u32::from(read_bit(data, bit)?); + bit += 1; + } + Some((1u32 << leading_zeroes) - 1 + suffix) +} + +fn read_bit(data: &[u8], bit: usize) -> Option { + let byte = *data.get(bit / 8)?; + Some(byte & (1 << (7 - bit % 8)) != 0) +} + +fn scaled_even_size(width: u32, height: u32, max_dimension: Option) -> (u32, u32) { + let Some(max_dimension) = max_dimension else { + return (width & !1, height & !1); + }; + let longest = width.max(height); + if longest <= max_dimension { + return (width & !1, height & !1); + } + let max_dimension = max_dimension.max(2) & !1; + if width >= height { + let scaled_height = + ((height as u64 * max_dimension as u64 / width as u64) as u32).max(2) & !1; + (max_dimension, scaled_height) + } else { + let scaled_width = + ((width as u64 * max_dimension as u64 / height as u64) as u32).max(2) & !1; + (scaled_width, max_dimension) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn annex_b(nalus: &[&[u8]]) -> Vec { + let mut out = Vec::new(); + for nalu in nalus { + append_nalu(&mut out, nalu); + } + out + } + + fn types(data: &[u8]) -> Vec { + let starts = start_codes(data); + starts + .iter() + .filter_map(|(start, code_len)| data.get(start + code_len).map(|byte| byte & 0x1f)) + .collect() + } + + #[test] + fn parses_every_byte_boundary() { + let stream = annex_b(&[ + &[0x67, 0x42, 0x00, 0x1f], + &[0x68, 0xce], + &[0x65, 0x80, 0xaa], + &[0x41, 0x80, 0xbb], + ]); + let mut parser = AnnexBAccessUnitParser::default(); + let mut frames = Vec::new(); + for byte in stream { + frames.extend(parser.push(&[byte])); + } + frames.extend(parser.flush_idle()); + assert_eq!(frames.len(), 2); + assert!(frames[0].keyframe); + assert_eq!(types(&frames[0].data), vec![7, 8, 5]); + assert!(!frames[1].keyframe); + assert_eq!(types(&frames[1].data), vec![1]); + } + + #[test] + fn keeps_multiple_slices_in_one_picture() { + let stream = annex_b(&[ + &[0x41, 0x80, 0xaa], + &[0x41, 0x40, 0xbb], + &[0x41, 0x80, 0xcc], + ]); + let mut parser = AnnexBAccessUnitParser::default(); + let mut frames = parser.push(&stream); + frames.extend(parser.flush_idle()); + assert_eq!(frames.len(), 2); + assert_eq!(types(&frames[0].data), vec![1, 1]); + assert_eq!(types(&frames[1].data), vec![1]); + } + + #[test] + fn prepends_cached_parameter_sets_to_later_keyframes() { + let stream = annex_b(&[ + &[0x67, 0x42, 0x00, 0x1f], + &[0x68, 0xce], + &[0x65, 0x80, 0xaa], + &[0x41, 0x80, 0xbb], + &[0x65, 0x80, 0xcc], + ]); + let mut parser = AnnexBAccessUnitParser::default(); + let mut frames = parser.push(&stream); + frames.extend(parser.flush_idle()); + assert_eq!(types(&frames[2].data), vec![7, 8, 5]); + } + + #[test] + fn scales_to_even_dimensions() { + assert_eq!(scaled_even_size(1080, 2424, Some(1280)), (570, 1280)); + assert_eq!(scaled_even_size(1081, 2425, None), (1080, 2424)); + } +} diff --git a/packages/accessibility-android-sys/src/lib.rs b/packages/accessibility-android-sys/src/lib.rs index 01b1013..1e2f9e3 100644 --- a/packages/accessibility-android-sys/src/lib.rs +++ b/packages/accessibility-android-sys/src/lib.rs @@ -1,5 +1,7 @@ //! Low-level ADB wrappers used by accessibility-cli's Android backend. +pub mod emulator; + use std::process::{Command, Output}; use std::time::Duration; @@ -453,6 +455,38 @@ impl AdbClient { } } + pub fn discover(serial: Option<&str>) -> Self { + for root in [ + std::env::var_os("ANDROID_SDK_ROOT"), + std::env::var_os("ANDROID_HOME"), + ] + .into_iter() + .flatten() + { + let path = std::path::PathBuf::from(root) + .join("platform-tools") + .join(adb_binary_name()); + if path.is_file() { + return Self::with_adb_path(serial, &path.to_string_lossy()); + } + } + if let Some(home) = std::env::var_os("HOME") { + let home = std::path::PathBuf::from(home); + let roots = if cfg!(target_os = "macos") { + vec![home.join("Library/Android/sdk"), home.join("Android/Sdk")] + } else { + vec![home.join("Android/Sdk")] + }; + for root in roots { + let path = root.join("platform-tools").join(adb_binary_name()); + if path.is_file() { + return Self::with_adb_path(serial, &path.to_string_lossy()); + } + } + } + Self::new(serial) + } + /// Create a new ADB client with a custom ADB path. pub fn with_adb_path(serial: Option<&str>, adb_path: &str) -> Self { Self { @@ -537,44 +571,64 @@ impl AdbClient { /// Check if ADB is available and a device is connected. pub fn check_connection(&self) -> Result<()> { - let version_result = Command::new(&self.adb_path).arg("version").output(); - - match version_result { - Ok(output) if output.status.success() => {} - Ok(_) => bail!("ADB binary found but returned error"), - Err(e) => bail!( - "ADB binary not found at '{}': {}. Install Android SDK Platform Tools.", - self.adb_path, - e - ), + let devices = self.connected_devices()?; + if devices.is_empty() { + bail!("No Android devices connected. Connect a device or start an emulator."); + } + if let Some(serial) = &self.serial + && !devices.contains(serial) + { + bail!( + "Device '{}' is not connected. Available devices: {}", + serial, + devices.join(", ") + ); } + Ok(()) + } - let devices = self.command(&["devices"])?; - let device_count = devices + pub fn connected_devices(&self) -> Result> { + let output = Command::new(&self.adb_path) + .arg("devices") + .output() + .with_context(|| { + format!( + "ADB binary not found at '{}'. Install Android SDK Platform Tools.", + self.adb_path + ) + })?; + Self::check_output(&output, "devices")?; + let stdout = String::from_utf8_lossy(&output.stdout); + Ok(stdout .lines() .skip(1) - .filter(|line| { - let trimmed = line.trim(); - !trimmed.is_empty() && trimmed.contains('\t') + .filter_map(|line| { + let (serial, state) = line.split_once('\t')?; + (state.split_whitespace().next() == Some("device")).then(|| serial.to_string()) }) - .count(); - - if device_count == 0 { - bail!("No Android devices connected. Connect a device or start an emulator."); - } + .collect()) + } - if let Some(ref serial) = self.serial { - let found = devices.lines().skip(1).any(|line| line.starts_with(serial)); - if !found { - bail!( - "Device '{}' not found. Available devices:\n{}", - serial, - devices - ); + pub fn resolved_serial(&self) -> Result { + let devices = self.connected_devices()?; + if let Some(serial) = &self.serial { + if devices.contains(serial) { + return Ok(serial.clone()); } + bail!( + "Device '{}' is not connected. Available devices: {}", + serial, + devices.join(", ") + ); + } + match devices.as_slice() { + [serial] => Ok(serial.clone()), + [] => bail!("No Android devices connected. Connect a device or start an emulator."), + _ => bail!( + "Multiple Android devices are connected; specify one of: {}", + devices.join(", ") + ), } - - Ok(()) } /// Get the screen size in pixels. @@ -756,6 +810,14 @@ impl AdbClient { } } +fn adb_binary_name() -> &'static str { + if cfg!(target_os = "windows") { + "adb.exe" + } else { + "adb" + } +} + /// Escape text for ADB shell input command. pub fn escape_shell_text(text: &str) -> String { let mut result = String::with_capacity(text.len() * 2); diff --git a/packages/accessibility-cli/src/lib.rs b/packages/accessibility-cli/src/lib.rs index 21cd5c5..2bf75f7 100644 --- a/packages/accessibility-cli/src/lib.rs +++ b/packages/accessibility-cli/src/lib.rs @@ -1852,6 +1852,10 @@ pub enum Command { /// Serve the iOS Simulator in a browser: live video, input, and element /// inspection. ServeSim(ServeSimArgs), + #[command( + about = "Serve an Android Emulator in a browser with live video, input, and element inspection" + )] + ServeEmulator(ServeEmulatorArgs), } #[derive(clap::Args)] @@ -1912,6 +1916,45 @@ pub struct ServeSimArgs { pub ice_servers: Vec, } +#[derive(clap::Args)] +pub struct ServeEmulatorArgs { + #[arg( + long, + help = "Emulator serial from adb devices; defaults to the only connected device" + )] + pub serial: Option, + #[arg(long, default_value_t = 3200, help = "Port to listen on")] + pub port: u16, + #[arg(long, default_value = "127.0.0.1", help = "Address to bind")] + pub bind: std::net::IpAddr, + #[arg( + long, + default_value = "webrtc", + help = "Preferred transport: webrtc or h264" + )] + pub transport: String, + #[arg( + long, + default_value_t = 60, + help = "Frame rate used to derive the bitrate" + )] + pub fps: u32, + #[arg(long, help = "screenrecord bitrate in bits per second")] + pub bitrate: Option, + #[arg(long, default_value_t = 1280, help = "Longest encoded edge")] + pub max_dimension: u32, + #[arg(long, help = "Capture at native display resolution")] + pub native_resolution: bool, + #[arg( + long, + default_value_t = 2, + help = "Requested keyframe recovery interval" + )] + pub keyframe_interval: u32, + #[arg(long = "ice-server", help = "ICE server URL, repeatable")] + pub ice_servers: Vec, +} + /// Run the CLI using process arguments. pub fn run() { let cli = Cli::parse(); @@ -1934,6 +1977,7 @@ pub fn run() { async fn run_command(command: &Command) { let result = match command { Command::ServeSim(args) => run_serve_sim(args).await, + Command::ServeEmulator(args) => run_serve_emulator(args).await, }; if let Err(error) = result { eprintln!("Error: {error:#}"); @@ -1981,6 +2025,30 @@ async fn run_serve_sim(_args: &ServeSimArgs) -> anyhow::Result<()> { anyhow::bail!("serve-sim requires macOS with Xcode and a booted iOS Simulator") } +async fn run_serve_emulator(args: &ServeEmulatorArgs) -> anyhow::Result<()> { + use accessibility_core::video::{NalFormat, Tuning, VideoConfig}; + use accessibility_serve::{ServeEmulatorConfig, Transport, serve_emulator}; + + let transport: Transport = args.transport.parse()?; + serve_emulator(ServeEmulatorConfig { + serial: args.serial.clone(), + address: std::net::SocketAddr::new(args.bind, args.port), + transport, + video: VideoConfig { + fps: args.fps, + tuning: Tuning::Interactive { + bitrate: args.bitrate, + }, + max_dimension: (!args.native_resolution).then_some(args.max_dimension), + keyframe_interval_secs: args.keyframe_interval, + nal_format: NalFormat::AnnexB, + ..Default::default() + }, + ice_servers: args.ice_servers.clone(), + }) + .await +} + /// Build a TreeFilter from CommonArgs fn build_filter(common: &CommonArgs) -> TreeFilter { TreeFilter { diff --git a/packages/accessibility-cli/tests/cli_smoke.rs b/packages/accessibility-cli/tests/cli_smoke.rs index f03a3c5..e7df4a2 100644 --- a/packages/accessibility-cli/tests/cli_smoke.rs +++ b/packages/accessibility-cli/tests/cli_smoke.rs @@ -12,6 +12,17 @@ fn help_mentions_primary_command() { .stdout(predicate::str::contains("--platform")); } +#[test] +fn serve_emulator_help_exposes_stream_options() { + let mut cmd = Command::cargo_bin("accessibility-cli").unwrap(); + cmd.args(["serve-emulator", "--help"]) + .assert() + .success() + .stdout(predicate::str::contains("--serial")) + .stdout(predicate::str::contains("--transport")) + .stdout(predicate::str::contains("--max-dimension")); +} + #[test] fn invalid_platform_fails_before_touching_accessibility_backend() { let mut cmd = Command::cargo_bin("accessibility-cli").unwrap(); diff --git a/packages/accessibility-core/examples/android_session_probe.rs b/packages/accessibility-core/examples/android_session_probe.rs new file mode 100644 index 0000000..13eb895 --- /dev/null +++ b/packages/accessibility-core/examples/android_session_probe.rs @@ -0,0 +1,123 @@ +use std::time::{Duration, Instant}; + +use accessibility_core::platform::android::AdbClient; +use accessibility_core::platform::android::input::{ + HardwareButton, InputCommand, Orientation, TouchPhase, +}; +use accessibility_core::platform::android::session::EmulatorSession; +use accessibility_core::video::{FrameKind, VideoConfig}; +use anyhow::{Context, Result}; + +#[tokio::main] +async fn main() -> Result<()> { + let requested_serial = std::env::args().nth(1); + let adb = AdbClient::discover(requested_serial.as_deref()); + let serial = adb.resolved_serial()?; + let _ = adb.stop_app("com.google.android.settings.intelligence"); + let _ = adb.stop_app("com.android.settings"); + adb.launch_app("com.android.settings", Some(".Settings"))?; + tokio::time::sleep(Duration::from_millis(500)).await; + let session = EmulatorSession::start(Some(&serial), VideoConfig::default())?; + session.seed_orientation().await; + let device = session.device_info(); + println!("device : {}", device.serial); + println!("geometry : {}x{}", device.width, device.height); + println!("orientation: {:?}", device.orientation); + + let snapshot = session.ax_snapshot(false).await?; + println!("app : {:?}", snapshot.app_name); + println!("elements : {}", snapshot.elements.len()); + println!("coverage : {:.1}%", snapshot.coverage * 100.0); + if let Some(element) = session.ax_hit_test(0.5, 0.5).await? { + println!("center : {} {}", element.role, element.selector); + } + let tappable = snapshot + .elements + .iter() + .find(|element| { + element.actions.iter().any(|action| action == "click") && element.bounds.is_some() + }) + .context("Settings tree has no tappable element")?; + let bounds = tappable.bounds.unwrap(); + let tap_x = bounds.x + bounds.width / 2.0; + let tap_y = bounds.y + bounds.height / 2.0; + println!("tap : {}", tappable.selector); + session.send_input(InputCommand::Touch { + phase: TouchPhase::Begin, + x: tap_x, + y: tap_y, + }); + session.send_input(InputCommand::Touch { + phase: TouchPhase::End, + x: tap_x, + y: tap_y, + }); + tokio::time::sleep(Duration::from_millis(700)).await; + let tapped = session.ax_snapshot(false).await?; + if tapped + .elements + .iter() + .map(|element| &element.selector) + .eq(snapshot.elements.iter().map(|element| &element.selector)) + { + anyhow::bail!("gRPC touch input did not change the Settings tree"); + } + + session.send_input(InputCommand::Button { + button: HardwareButton::Home, + }); + tokio::time::sleep(Duration::from_millis(500)).await; + let home = session.ax_snapshot(false).await?; + if home.app_name == tapped.app_name { + anyhow::bail!("gRPC Home input did not leave the tapped Settings surface"); + } + println!("after Home : {:?}", home.app_name); + let mut frames = session.subscribe(); + let started = Instant::now(); + let mut received = 0; + let mut keyframes = 0; + let mut bytes = 0; + while started.elapsed() < Duration::from_secs(5) { + let frame = match tokio::time::timeout(Duration::from_secs(2), frames.recv()).await { + Ok(frame) => frame?, + Err(_) if received > 0 => break, + Err(error) => { + return Err(error).context("timed out waiting for an Android Emulator frame"); + } + }; + received += 1; + bytes += frame.data.len(); + if frame.kind == FrameKind::Keyframe { + keyframes += 1; + } + if received >= 30 { + break; + } + } + let stats = session.stats(); + println!("received : {received} frames, {keyframes} key, {bytes} bytes"); + println!("session : {:.1} fps, {:.2} Mbps", stats.fps, stats.mbps); + + session.set_orientation(Orientation::LandscapeRight)?; + let landscape = session.ax_snapshot(false).await?; + let landscape_stats = session.stats(); + if !landscape.is_landscape || landscape_stats.encoded_width <= landscape_stats.encoded_height { + anyhow::bail!("Android Emulator did not reconfigure for landscape"); + } + println!( + "landscape : {}x{}", + landscape_stats.encoded_width, landscape_stats.encoded_height + ); + session.set_orientation(Orientation::Portrait)?; + let portrait = session.ax_snapshot(false).await?; + let portrait_stats = session.stats(); + if portrait.is_landscape || portrait_stats.encoded_width >= portrait_stats.encoded_height { + anyhow::bail!("Android Emulator did not reconfigure for portrait"); + } + println!( + "portrait : {}x{}", + portrait_stats.encoded_width, portrait_stats.encoded_height + ); + println!("probe passed"); + Ok(()) +} diff --git a/packages/accessibility-core/src/platform/android.rs b/packages/accessibility-core/src/platform/android.rs index d26a195..8d4062a 100644 --- a/packages/accessibility-core/src/platform/android.rs +++ b/packages/accessibility-core/src/platform/android.rs @@ -56,8 +56,15 @@ use crate::accessibility::{ TreeFilter, }; use crate::input::{Code, Modifiers, MouseButton}; +use crate::video::{FrameSink, ScreenGeometry, VideoCapture, VideoConfig}; +pub mod ax; +pub mod input; +pub mod session; +pub mod video; pub use accessibility_android_sys::{AdbClient, AndroidKeyCode}; +pub use input::{HardwareButton, InputCommand, Orientation, TouchPhase, spawn_input_worker}; +pub use video::AndroidVideoCapture; /// Parse Android bounds string like "[0,0][1080,1920]" into a Rect. fn parse_bounds(bounds_str: &str) -> Option { @@ -831,6 +838,27 @@ impl AccessibilityReader for AndroidAccessibility { true } + fn start_video_capture( + &self, + config: &VideoConfig, + sink: FrameSink, + ) -> Result> { + let (width, height) = self + .screen_size + .ok_or_else(|| anyhow!("Android screen size is unavailable"))?; + let capture = AndroidVideoCapture::start( + self.adb.clone(), + ScreenGeometry { width, height }, + config, + sink, + )?; + Ok(Box::new(capture)) + } + + fn supports_video_capture(&self) -> bool { + true + } + fn supports_terminal_display(&self) -> bool { true } diff --git a/packages/accessibility-core/src/platform/android/ax.rs b/packages/accessibility-core/src/platform/android/ax.rs new file mode 100644 index 0000000..5c975d9 --- /dev/null +++ b/packages/accessibility-core/src/platform/android/ax.rs @@ -0,0 +1,271 @@ +use anyhow::{Result, anyhow}; +use serde::Serialize; +use tokio::sync::{mpsc, oneshot}; + +use crate::accessibility::{ + AccessibilityReader, AndroidTarget, Element, ElementTree, Point, Rect, Size, Target, TreeFilter, +}; + +use super::AndroidAccessibility; + +const COVERAGE_COLUMNS: usize = 32; +const COVERAGE_ROWS: usize = 64; +const BACKDROP_AREA: f64 = 0.9; + +#[derive(Debug, Clone, Copy, Serialize)] +pub struct NormalizedRect { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} + +impl NormalizedRect { + fn from_screen(rect: &Rect, screen: &Rect) -> Option { + if screen.size.width <= 0.0 || screen.size.height <= 0.0 { + return None; + } + Some(Self { + x: (rect.origin.x - screen.origin.x) / screen.size.width, + y: (rect.origin.y - screen.origin.y) / screen.size.height, + width: rect.size.width / screen.size.width, + height: rect.size.height / screen.size.height, + }) + } + + fn is_backdrop(self) -> bool { + self.width * self.height >= BACKDROP_AREA + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Discovery { + Recursive, + PointGrid, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ElementDetail { + pub id: String, + pub role: String, + pub label: Option, + pub value: Option, + pub identifier: Option, + pub enabled: bool, + pub focused: bool, + pub actions: Vec, + pub bounds: Option, + pub depth: u32, + pub selector: String, + pub discovery: Discovery, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AxSnapshot { + pub app_name: Option, + pub pid: Option, + pub elements: Vec, + pub coverage: f64, + pub coverage_after_scan: Option, + pub probes: Option, + pub is_landscape: bool, +} + +pub enum AxCommand { + Snapshot { + scan: bool, + reply: oneshot::Sender>, + }, + HitTest { + x: f64, + y: f64, + reply: oneshot::Sender>>, + }, +} + +pub fn spawn_ax_worker(serial: &str) -> Result> { + let adb = super::AdbClient::discover(Some(serial)); + let mut reader = AndroidAccessibility::with_adb_path(Some(serial), &adb.adb_path)?; + let target = Target::Android(AndroidTarget::Serial(serial.to_string())); + let (commands, mut command_rx) = mpsc::unbounded_channel(); + std::thread::Builder::new() + .name("android-emulator-ax".into()) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build(); + let Ok(runtime) = runtime else { return }; + let mut screen = None; + while let Some(command) = command_rx.blocking_recv() { + match command { + AxCommand::Snapshot { scan, reply } => { + let result = runtime.block_on(snapshot(&mut reader, &target, scan)); + if let Ok((_, bounds)) = &result { + screen = Some(*bounds); + } + let _ = reply.send(result.map(|(snapshot, _)| snapshot)); + } + AxCommand::HitTest { x, y, reply } => { + let result = runtime.block_on(hit_test(&mut reader, screen.as_ref(), x, y)); + let _ = reply.send(result); + } + } + } + })?; + Ok(commands) +} + +async fn snapshot( + reader: &mut AndroidAccessibility, + target: &Target, + _scan: bool, +) -> Result<(AxSnapshot, Rect)> { + let tree = reader.get_tree(target, &TreeFilter::default()).await?; + let screen = + screen_bounds(&tree).ok_or_else(|| anyhow!("Android tree has no screen bounds"))?; + let mut elements = Vec::with_capacity(tree.element_count); + flatten(&tree.root, &screen, 0, &mut elements); + elements.retain(|element| { + element.bounds.is_some_and(|bounds| { + !bounds.is_backdrop() && bounds.width > 0.0 && bounds.height > 0.0 + }) + }); + let coverage = coverage(&elements); + Ok(( + AxSnapshot { + app_name: tree.app_name, + pid: tree.pid, + elements, + coverage, + coverage_after_scan: None, + probes: None, + is_landscape: screen.size.width > screen.size.height, + }, + screen, + )) +} + +async fn hit_test( + reader: &mut AndroidAccessibility, + screen: Option<&Rect>, + x: f64, + y: f64, +) -> Result> { + let screen = + screen.ok_or_else(|| anyhow!("no accessibility snapshot yet; fetch the tree first"))?; + let point_x = screen.origin.x + x * screen.size.width; + let point_y = screen.origin.y + y * screen.size.height; + let Some(id) = reader.hit_test(point_x, point_y).await? else { + return Ok(None); + }; + Ok(reader + .get_element(id) + .map(|element| to_detail(element, screen, 0))) +} + +fn screen_bounds(tree: &ElementTree) -> Option { + let mut max_x = 0.0f64; + let mut max_y = 0.0f64; + let mut stack = vec![&tree.root]; + while let Some(element) = stack.pop() { + if let Some(bounds) = &element.bounds { + max_x = max_x.max(bounds.origin.x + bounds.size.width); + max_y = max_y.max(bounds.origin.y + bounds.size.height); + } + stack.extend(element.children.iter()); + } + (max_x > 0.0 && max_y > 0.0).then(|| Rect::new(Point::new(0.0, 0.0), Size::new(max_x, max_y))) +} + +fn flatten(element: &Element, screen: &Rect, depth: u32, out: &mut Vec) { + out.push(to_detail(element, screen, depth)); + for child in &element.children { + flatten(child, screen, depth + 1, out); + } +} + +fn to_detail(element: &Element, screen: &Rect, depth: u32) -> ElementDetail { + let role = format!("{:?}", element.role); + ElementDetail { + id: element.id.to_string(), + role: role.clone(), + label: element.title.clone().filter(|value| !value.is_empty()), + value: element.value.clone().filter(|value| !value.is_empty()), + identifier: element.identifier.clone().filter(|value| !value.is_empty()), + enabled: element.enabled, + focused: element.focused, + actions: element.actions.clone(), + bounds: element + .bounds + .as_ref() + .and_then(|bounds| NormalizedRect::from_screen(bounds, screen)), + depth, + selector: selector_for(element, &role), + discovery: Discovery::Recursive, + } +} + +fn selector_for(element: &Element, role: &str) -> String { + if let Some(identifier) = element + .identifier + .as_deref() + .filter(|value| !value.is_empty()) + { + return format!("#{identifier}"); + } + if let Some(label) = element.title.as_deref().filter(|value| !value.is_empty()) { + return format!("{role}[label=\"{}\"]", label.replace('"', "\\\"")); + } + role.to_string() +} + +fn coverage(elements: &[ElementDetail]) -> f64 { + let mut cells = [false; COVERAGE_COLUMNS * COVERAGE_ROWS]; + for bounds in elements.iter().filter_map(|element| element.bounds) { + let left = (bounds.x.clamp(0.0, 1.0) * COVERAGE_COLUMNS as f64).floor() as usize; + let right = (((bounds.x + bounds.width).clamp(0.0, 1.0) * COVERAGE_COLUMNS as f64).ceil() + as usize) + .min(COVERAGE_COLUMNS); + let top = (bounds.y.clamp(0.0, 1.0) * COVERAGE_ROWS as f64).floor() as usize; + let bottom = (((bounds.y + bounds.height).clamp(0.0, 1.0) * COVERAGE_ROWS as f64).ceil() + as usize) + .min(COVERAGE_ROWS); + for row in top..bottom { + for column in left..right { + cells[row * COVERAGE_COLUMNS + column] = true; + } + } + } + cells.iter().filter(|filled| **filled).count() as f64 / cells.len() as f64 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::accessibility::Element; + use accesskit::Role; + + #[test] + fn normalizes_android_pixel_bounds() { + let screen = Rect::new(Point::new(0.0, 0.0), Size::new(1080.0, 2400.0)); + let bounds = Rect::new(Point::new(108.0, 240.0), Size::new(540.0, 1200.0)); + let normalized = NormalizedRect::from_screen(&bounds, &screen).unwrap(); + assert_eq!(normalized.x, 0.1); + assert_eq!(normalized.y, 0.1); + assert_eq!(normalized.width, 0.5); + assert_eq!(normalized.height, 0.5); + } + + #[test] + fn coverage_counts_element_area() { + let mut element = Element::new(crate::accessibility::ElementKey::default(), Role::Button); + element.bounds = Some(Rect::new(Point::new(0.0, 0.0), Size::new(100.0, 100.0))); + let detail = to_detail( + &element, + &Rect::new(Point::new(0.0, 0.0), Size::new(200.0, 200.0)), + 0, + ); + assert_eq!(coverage(&[detail]), 0.25); + } +} diff --git a/packages/accessibility-core/src/platform/android/input.rs b/packages/accessibility-core/src/platform/android/input.rs new file mode 100644 index 0000000..9be2b6c --- /dev/null +++ b/packages/accessibility-core/src/platform/android/input.rs @@ -0,0 +1,332 @@ +use std::sync::mpsc; + +use accessibility_android_sys::emulator::protocol::controller::input_event; +use accessibility_android_sys::emulator::protocol::controller::keyboard_event::{ + KeyCodeType, KeyEventType, +}; +use accessibility_android_sys::emulator::protocol::controller::{ + InputEvent, KeyboardEvent, Touch, TouchEvent, +}; +use accessibility_android_sys::emulator::{EmulatorGrpcClient, discover_emulator}; +use accessibility_android_sys::{AdbClient, AndroidKeyCode}; +use anyhow::{Result, anyhow, bail}; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc::UnboundedSender; + +use crate::video::ScreenGeometry; + +const ACTIVE_PRESSURE: i32 = 0x7fff; +const TOUCH_SIZE: i32 = 8; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum TouchPhase { + Begin, + Move, + End, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HardwareButton { + Home, + Back, + Lock, + AppSwitch, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Orientation { + Portrait, + PortraitUpsideDown, + LandscapeLeft, + LandscapeRight, +} + +impl Orientation { + pub fn is_landscape(self) -> bool { + matches!(self, Self::LandscapeLeft | Self::LandscapeRight) + } + + fn android_rotation(self) -> u8 { + match self { + Self::Portrait => 0, + Self::LandscapeLeft => 1, + Self::PortraitUpsideDown => 2, + Self::LandscapeRight => 3, + } + } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum InputCommand { + Touch { + phase: TouchPhase, + x: f64, + y: f64, + }, + Key { + key_code: u32, + #[serde(default)] + modifiers: Vec, + }, + Text { + text: String, + }, + Scroll { + dx: f64, + dy: f64, + x: f64, + y: f64, + }, + Button { + button: HardwareButton, + }, + Rotate { + orientation: Orientation, + }, +} + +pub fn spawn_input_worker( + serial: &str, + geometry: ScreenGeometry, +) -> Result> { + let discovery = discover_emulator(Some(serial))?; + let adb = AdbClient::discover(Some(serial)); + let (commands, mut command_rx) = tokio::sync::mpsc::unbounded_channel(); + let (ready_tx, ready_rx) = mpsc::sync_channel(1); + std::thread::Builder::new() + .name("android-emulator-input".into()) + .spawn(move || { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = ready_tx.send(Err(error.to_string())); + return; + } + }; + runtime.block_on(async move { + let mut client = match EmulatorGrpcClient::connect(discovery).await { + Ok(client) => client, + Err(error) => { + let _ = ready_tx.send(Err(error.to_string())); + return; + } + }; + let _ = ready_tx.send(Ok(())); + while let Some(command) = command_rx.recv().await { + let command = match command { + InputCommand::Rotate { orientation } => { + let _ = set_device_orientation(&adb, orientation); + continue; + } + InputCommand::Button { button } => { + apply_hardware_button(&adb, button); + continue; + } + command => command, + }; + for event in to_events(command, geometry) { + if let Err(error) = client.send_input(event).await { + eprintln!("Android Emulator input failed: {error:#}"); + return; + } + } + } + }); + })?; + match ready_rx.recv() { + Ok(Ok(())) => Ok(commands), + Ok(Err(error)) => Err(anyhow!(error)), + Err(_) => Err(anyhow!( + "Android Emulator input worker stopped during startup" + )), + } +} + +fn to_events(command: InputCommand, geometry: ScreenGeometry) -> Vec { + match command { + InputCommand::Touch { phase, x, y } => vec![touch_event(phase, x, y, geometry)], + InputCommand::Key { + key_code, + modifiers, + } => { + let mut events = Vec::with_capacity(modifiers.len() * 2 + 2); + events.extend( + modifiers + .iter() + .map(|modifier| usb_key(*modifier, KeyEventType::Keydown)), + ); + events.push(usb_key(key_code, KeyEventType::Keydown)); + events.push(usb_key(key_code, KeyEventType::Keyup)); + events.extend( + modifiers + .iter() + .rev() + .map(|modifier| usb_key(*modifier, KeyEventType::Keyup)), + ); + events + } + InputCommand::Text { text } => vec![keyboard_event(KeyboardEvent { + text, + ..Default::default() + })], + InputCommand::Scroll { dx, dy, x, y } => { + let end_x = (x - dx).clamp(0.0, 1.0); + let end_y = (y - dy).clamp(0.0, 1.0); + vec![ + touch_event(TouchPhase::Begin, x, y, geometry), + touch_event(TouchPhase::Move, end_x, end_y, geometry), + touch_event(TouchPhase::End, end_x, end_y, geometry), + ] + } + InputCommand::Button { .. } => Vec::new(), + InputCommand::Rotate { .. } => Vec::new(), + } +} + +fn touch_event(phase: TouchPhase, x: f64, y: f64, geometry: ScreenGeometry) -> InputEvent { + InputEvent { + r#type: Some(input_event::Type::TouchEvent(TouchEvent { + touches: vec![Touch { + x: normalized_coordinate(x, geometry.width), + y: normalized_coordinate(y, geometry.height), + identifier: 0, + pressure: if phase == TouchPhase::End { + 0 + } else { + ACTIVE_PRESSURE + }, + touch_major: TOUCH_SIZE, + touch_minor: TOUCH_SIZE, + expiration: 1, + orientation: 0, + }], + display: 0, + })), + } +} + +fn usb_key(key_code: u32, event_type: KeyEventType) -> InputEvent { + let key_code = if key_code <= 0xffff { + 0x070000 | key_code + } else { + key_code + }; + keyboard_event(KeyboardEvent { + code_type: KeyCodeType::Usb as i32, + event_type: event_type as i32, + key_code: key_code as i32, + ..Default::default() + }) +} + +fn keyboard_event(event: KeyboardEvent) -> InputEvent { + InputEvent { + r#type: Some(input_event::Type::KeyEvent(event)), + } +} + +fn normalized_coordinate(value: f64, dimension: u32) -> i32 { + (value.clamp(0.0, 1.0) * dimension.saturating_sub(1) as f64).round() as i32 +} + +fn apply_hardware_button(adb: &AdbClient, button: HardwareButton) { + let key = match button { + HardwareButton::Home => AndroidKeyCode::Home, + HardwareButton::Back => AndroidKeyCode::Back, + HardwareButton::Lock => AndroidKeyCode::Power, + HardwareButton::AppSwitch => AndroidKeyCode::AppSwitch, + }; + let _ = adb.key_event(key as u32); +} + +pub fn set_device_orientation(adb: &AdbClient, orientation: Orientation) -> Result<()> { + adb.shell(&["wm", "fixed-to-user-rotation", "enabled"])?; + let target = orientation.android_rotation(); + adb.shell(&["wm", "user-rotation", "lock", &target.to_string()])?; + for _ in 0..20 { + let output = adb.shell(&["dumpsys", "display"])?; + if display_rotation(&output) == Some(target) { + return Ok(()); + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + bail!("Android display did not reach rotation {target}") +} + +fn display_rotation(output: &str) -> Option { + output.lines().find_map(|line| { + line.trim() + .strip_prefix("mCurrentOrientation=")? + .parse() + .ok() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn geometry() -> ScreenGeometry { + ScreenGeometry { + width: 1080, + height: 2424, + } + } + + #[test] + fn touch_maps_normalized_coordinates_and_pressure() { + let events = to_events( + InputCommand::Touch { + phase: TouchPhase::Begin, + x: 0.5, + y: 1.0, + }, + geometry(), + ); + let Some(input_event::Type::TouchEvent(event)) = &events[0].r#type else { + panic!("expected touch event"); + }; + assert_eq!(event.touches[0].x, 540); + assert_eq!(event.touches[0].y, 2423); + assert_eq!(event.touches[0].pressure, ACTIVE_PRESSURE); + } + + #[test] + fn parses_display_rotation() { + assert_eq!( + display_rotation("other\n mCurrentOrientation=3\nmore"), + Some(3) + ); + } + + #[test] + fn modifiers_are_held_around_key() { + let events = to_events( + InputCommand::Key { + key_code: 4, + modifiers: vec![225], + }, + geometry(), + ); + let kinds = events + .iter() + .map(|event| { + let Some(input_event::Type::KeyEvent(key)) = &event.r#type else { + panic!("expected key event"); + }; + (key.key_code, key.event_type) + }) + .collect::>(); + assert_eq!( + kinds, + vec![(0x0700e1, 0), (0x070004, 0), (0x070004, 1), (0x0700e1, 1),] + ); + } +} diff --git a/packages/accessibility-core/src/platform/android/session.rs b/packages/accessibility-core/src/platform/android/session.rs new file mode 100644 index 0000000..cac3eda --- /dev/null +++ b/packages/accessibility-core/src/platform/android/session.rs @@ -0,0 +1,233 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; + +use anyhow::{Result, anyhow, bail}; +use tokio::sync::{broadcast, mpsc, oneshot}; + +use crate::video::{EncodedFrame, FrameKind, ScreenGeometry, VideoCapture, VideoConfig}; + +use super::ax::{AxCommand, AxSnapshot, ElementDetail, spawn_ax_worker}; +use super::input::{InputCommand, Orientation, set_device_orientation, spawn_input_worker}; +use super::{AdbClient, AndroidVideoCapture}; + +const FRAME_BUFFER: usize = 16; + +#[derive(Default)] +pub struct StreamStats { + pub frames: AtomicU64, + pub keyframes: AtomicU64, + pub bytes: AtomicU64, + pub keyframe_requests: AtomicU64, + pub lag_events: AtomicU64, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct StatsReport { + pub uptime_secs: f64, + pub frames: u64, + pub keyframes: u64, + pub bytes: u64, + pub fps: f64, + pub mbps: f64, + pub bits_per_pixel: f64, + pub mean_frame_kb: f64, + pub keyframe_requests: u64, + pub lag_events: u64, + pub subscribers: usize, + pub recording_frames: Option, + pub width: u32, + pub height: u32, + pub encoded_width: u32, + pub encoded_height: u32, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct DeviceInfo { + pub serial: String, + pub width: u32, + pub height: u32, + pub orientation: Orientation, +} + +pub struct EmulatorSession { + serial: String, + adb: AdbClient, + capture: AndroidVideoCapture, + frames: broadcast::Sender, + stats: Arc, + started: Instant, + input: tokio::sync::mpsc::UnboundedSender, + ax: mpsc::UnboundedSender, + orientation: std::sync::Mutex, +} + +impl EmulatorSession { + pub fn start(serial: Option<&str>, config: VideoConfig) -> Result> { + let adb = AdbClient::discover(serial); + let serial = adb.resolved_serial()?; + if !serial.starts_with("emulator-") { + bail!("Android Emulator streaming requires an emulator serial, got '{serial}'"); + } + let adb = AdbClient::discover(Some(&serial)); + let (width, height) = adb.get_screen_size()?; + let geometry = ScreenGeometry { width, height }; + let (frames, _) = broadcast::channel(FRAME_BUFFER); + let stats = Arc::new(StreamStats::default()); + let sink = { + let frames = frames.clone(); + let stats = Arc::clone(&stats); + Arc::new(move |frame: EncodedFrame| { + stats.frames.fetch_add(1, Ordering::Relaxed); + stats + .bytes + .fetch_add(frame.data.len() as u64, Ordering::Relaxed); + if frame.kind == FrameKind::Keyframe { + stats.keyframes.fetch_add(1, Ordering::Relaxed); + } + let _ = frames.send(frame); + }) + }; + let capture = AndroidVideoCapture::start(adb.clone(), geometry, &config, sink)?; + let input = spawn_input_worker(&serial, geometry)?; + let ax = spawn_ax_worker(&serial)?; + Ok(Arc::new(Self { + serial, + adb, + capture, + frames, + stats, + started: Instant::now(), + input, + ax, + orientation: std::sync::Mutex::new(Orientation::Portrait), + })) + } + + pub fn subscribe(&self) -> broadcast::Receiver { + let receiver = self.frames.subscribe(); + self.capture.request_keyframe(); + receiver + } + + pub fn request_keyframe(&self) { + self.stats.keyframe_requests.fetch_add(1, Ordering::Relaxed); + self.capture.request_keyframe(); + } + + pub fn note_lag(&self) { + self.stats.lag_events.fetch_add(1, Ordering::Relaxed); + } + + pub fn device_info(&self) -> DeviceInfo { + let geometry = self.capture.geometry(); + DeviceInfo { + serial: self.serial.clone(), + width: geometry.width, + height: geometry.height, + orientation: self.orientation(), + } + } + + pub fn orientation(&self) -> Orientation { + *self.orientation.lock().unwrap() + } + + pub fn set_orientation(&self, orientation: Orientation) -> Result<()> { + set_device_orientation(&self.adb, orientation)?; + self.capture.set_landscape(orientation.is_landscape())?; + *self.orientation.lock().unwrap() = orientation; + Ok(()) + } + + pub fn send_input(&self, command: InputCommand) { + if let InputCommand::Rotate { orientation } = command { + let _ = self.set_orientation(orientation); + return; + } + let _ = self.input.send(command); + } + + pub fn stats(&self) -> StatsReport { + let elapsed = self.started.elapsed().as_secs_f64().max(1e-6); + let frames = self.stats.frames.load(Ordering::Relaxed); + let bytes = self.stats.bytes.load(Ordering::Relaxed); + let geometry = self.capture.geometry(); + let encoded = self.capture.encoded_geometry(); + let pixels = encoded.width as f64 * encoded.height as f64; + let fps = frames as f64 / elapsed; + StatsReport { + uptime_secs: (elapsed * 10.0).round() / 10.0, + frames, + keyframes: self.stats.keyframes.load(Ordering::Relaxed), + bytes, + fps: (fps * 10.0).round() / 10.0, + mbps: ((bytes as f64 * 8.0 / elapsed / 1e6) * 100.0).round() / 100.0, + bits_per_pixel: if pixels > 0.0 && fps > 0.0 { + ((bytes as f64 * 8.0 / elapsed) / (pixels * fps) * 10000.0).round() / 10000.0 + } else { + 0.0 + }, + mean_frame_kb: if frames > 0 { + ((bytes as f64 / frames as f64 / 1024.0) * 100.0).round() / 100.0 + } else { + 0.0 + }, + keyframe_requests: self.stats.keyframe_requests.load(Ordering::Relaxed), + lag_events: self.stats.lag_events.load(Ordering::Relaxed), + subscribers: self.frames.receiver_count(), + recording_frames: None, + width: geometry.width, + height: geometry.height, + encoded_width: encoded.width, + encoded_height: encoded.height, + } + } + + pub async fn ax_snapshot(&self, scan: bool) -> Result { + let (reply, response) = oneshot::channel(); + self.ax + .send(AxCommand::Snapshot { scan, reply }) + .map_err(|_| anyhow!("Android accessibility worker stopped"))?; + let snapshot = response + .await + .map_err(|_| anyhow!("Android accessibility worker stopped"))??; + self.reconcile_orientation(snapshot.is_landscape); + Ok(snapshot) + } + + pub async fn ax_hit_test(&self, x: f64, y: f64) -> Result> { + let (reply, response) = oneshot::channel(); + self.ax + .send(AxCommand::HitTest { x, y, reply }) + .map_err(|_| anyhow!("Android accessibility worker stopped"))?; + response + .await + .map_err(|_| anyhow!("Android accessibility worker stopped"))? + } + + pub async fn seed_orientation(&self) { + if let Ok(snapshot) = self.ax_snapshot(false).await { + self.reconcile_orientation(snapshot.is_landscape); + } + } + + fn reconcile_orientation(&self, is_landscape: bool) { + let changed = { + let mut orientation = self.orientation.lock().unwrap(); + if orientation.is_landscape() == is_landscape { + false + } else { + *orientation = if is_landscape { + Orientation::LandscapeLeft + } else { + Orientation::Portrait + }; + true + } + }; + if changed { + let _ = self.capture.set_landscape(is_landscape); + } + } +} diff --git a/packages/accessibility-core/src/platform/android/video.rs b/packages/accessibility-core/src/platform/android/video.rs new file mode 100644 index 0000000..ca71f7f --- /dev/null +++ b/packages/accessibility-core/src/platform/android/video.rs @@ -0,0 +1,277 @@ +use std::io::Read; +use std::sync::mpsc::{self, Receiver, SyncSender}; +use std::thread::JoinHandle; +use std::time::Duration; + +use accessibility_android_sys::AdbClient; +use accessibility_android_sys::emulator::screenrecord::{ + AnnexBAccessUnitParser, ScreenRecordConfig, spawn_screenrecord, +}; +use anyhow::{Result, anyhow, bail}; + +use crate::video::{ + EncodedFrame, FrameKind, FrameSink, NalFormat, ScreenGeometry, Tuning, VideoCapture, + VideoConfig, +}; + +const IDLE_FLUSH: Duration = Duration::from_millis(75); +const RESTART_DELAY: Duration = Duration::from_millis(100); + +#[derive(Debug, Clone, Copy)] +enum CaptureControl { + Restart(ScreenRecordConfig), + Stop, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CaptureOutcome { + Restart(Option), + Stop, +} + +pub struct AndroidVideoCapture { + geometry: ScreenGeometry, + encoded_geometry: std::sync::Mutex, + capture_config: std::sync::Mutex, + max_dimension: Option, + bit_rate: u32, + control: SyncSender, + worker: Option>, +} + +impl AndroidVideoCapture { + pub fn start( + adb: AdbClient, + geometry: ScreenGeometry, + config: &VideoConfig, + sink: FrameSink, + ) -> Result { + if !geometry.is_valid() { + bail!("Android Emulator screen geometry is unavailable"); + } + if config.nal_format != NalFormat::AnnexB { + bail!("Android Emulator screenrecord capture requires Annex-B H.264"); + } + let bit_rate = match config.tuning { + Tuning::Interactive { bitrate } => bitrate + .unwrap_or_else(|| derived_bit_rate(geometry, config.max_dimension, config.fps)), + Tuning::Recording { .. } => { + bail!("Android Emulator live capture does not support recording tuning") + } + }; + let capture_config = ScreenRecordConfig::for_max_dimension( + geometry.width, + geometry.height, + config.max_dimension, + bit_rate, + ); + let encoded_geometry = ScreenGeometry { + width: capture_config.width, + height: capture_config.height, + }; + let first_child = spawn_screenrecord(&adb, capture_config)?; + let (control, commands) = mpsc::sync_channel(4); + let worker = std::thread::Builder::new() + .name("android-screenrecord".into()) + .spawn(move || run_worker(adb, capture_config, first_child, commands, sink))?; + Ok(Self { + geometry, + encoded_geometry: std::sync::Mutex::new(encoded_geometry), + capture_config: std::sync::Mutex::new(capture_config), + max_dimension: config.max_dimension, + bit_rate, + control, + worker: Some(worker), + }) + } + + pub fn set_landscape(&self, landscape: bool) -> Result<()> { + let (width, height) = if landscape { + (self.geometry.height, self.geometry.width) + } else { + (self.geometry.width, self.geometry.height) + }; + let config = + ScreenRecordConfig::for_max_dimension(width, height, self.max_dimension, self.bit_rate); + self.control + .send(CaptureControl::Restart(config)) + .map_err(|_| anyhow!("Android screenrecord worker stopped"))?; + *self.capture_config.lock().unwrap() = config; + *self.encoded_geometry.lock().unwrap() = ScreenGeometry { + width: config.width, + height: config.height, + }; + Ok(()) + } + + fn stop_worker(&mut self) { + let _ = self.control.send(CaptureControl::Stop); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +impl VideoCapture for AndroidVideoCapture { + fn geometry(&self) -> ScreenGeometry { + self.geometry + } + + fn encoded_geometry(&self) -> ScreenGeometry { + *self.encoded_geometry.lock().unwrap() + } + + fn request_keyframe(&self) { + let config = *self.capture_config.lock().unwrap(); + let _ = self.control.try_send(CaptureControl::Restart(config)); + } + + fn stop(&mut self) { + self.stop_worker(); + } +} + +impl Drop for AndroidVideoCapture { + fn drop(&mut self) { + self.stop_worker(); + } +} + +fn run_worker( + adb: AdbClient, + mut config: ScreenRecordConfig, + first_child: std::process::Child, + commands: Receiver, + sink: FrameSink, +) { + let mut next_child = Some(first_child); + loop { + let child = match next_child.take() { + Some(child) => child, + None => match spawn_screenrecord(&adb, config) { + Ok(child) => child, + Err(_) => { + match commands.recv_timeout(RESTART_DELAY) { + Ok(CaptureControl::Stop) | Err(mpsc::RecvTimeoutError::Disconnected) => { + return; + } + Ok(CaptureControl::Restart(next)) => config = next, + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + continue; + } + }, + }; + match pump_child(child, &commands, &sink) { + CaptureOutcome::Stop => return, + CaptureOutcome::Restart(Some(next)) => config = next, + CaptureOutcome::Restart(None) => {} + } + match commands.recv_timeout(RESTART_DELAY) { + Ok(CaptureControl::Stop) | Err(mpsc::RecvTimeoutError::Disconnected) => return, + Ok(CaptureControl::Restart(next)) => config = next, + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + } +} + +fn pump_child( + mut child: std::process::Child, + commands: &Receiver, + sink: &FrameSink, +) -> CaptureOutcome { + let Some(mut stdout) = child.stdout.take() else { + return CaptureOutcome::Restart(None); + }; + let stderr = child.stderr.take(); + let (chunks_tx, chunks_rx) = mpsc::sync_channel::>(8); + let reader = std::thread::spawn(move || { + let mut buffer = vec![0u8; 64 * 1024]; + loop { + match stdout.read(&mut buffer) { + Ok(0) | Err(_) => break, + Ok(read) => { + if chunks_tx.send(buffer[..read].to_vec()).is_err() { + break; + } + } + } + } + }); + let stderr_reader = stderr.map(|mut stderr| { + std::thread::spawn(move || { + let mut output = Vec::new(); + let _ = stderr.read_to_end(&mut output); + output + }) + }); + + let mut parser = AnnexBAccessUnitParser::default(); + let outcome = loop { + match commands.try_recv() { + Ok(CaptureControl::Stop) | Err(mpsc::TryRecvError::Disconnected) => { + break CaptureOutcome::Stop; + } + Ok(CaptureControl::Restart(next)) => break CaptureOutcome::Restart(Some(next)), + Err(mpsc::TryRecvError::Empty) => {} + } + match chunks_rx.recv_timeout(IDLE_FLUSH) { + Ok(chunk) => emit(parser.push(&chunk), sink), + Err(mpsc::RecvTimeoutError::Timeout) => emit(parser.flush_idle(), sink), + Err(mpsc::RecvTimeoutError::Disconnected) => { + emit(parser.flush_idle(), sink); + break CaptureOutcome::Restart(None); + } + } + }; + + drop(chunks_rx); + let _ = child.kill(); + let _ = child.wait(); + let _ = reader.join(); + if let Some(stderr_reader) = stderr_reader { + let _ = stderr_reader.join(); + } + outcome +} + +fn emit( + frames: Vec, + sink: &FrameSink, +) { + for frame in frames { + sink(EncodedFrame { + data: frame.data, + kind: if frame.keyframe { + FrameKind::Keyframe + } else { + FrameKind::Delta + }, + }); + } +} + +fn derived_bit_rate(geometry: ScreenGeometry, max_dimension: Option, fps: u32) -> u32 { + let size = + ScreenRecordConfig::for_max_dimension(geometry.width, geometry.height, max_dimension, 1); + let bits = size.width as f64 * size.height as f64 * fps.max(1) as f64 * 0.15; + bits.round().clamp(1_000_000.0, 24_000_000.0) as u32 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn derives_bitrate_from_encoded_geometry() { + let bitrate = derived_bit_rate( + ScreenGeometry { + width: 1080, + height: 2424, + }, + Some(1280), + 60, + ); + assert_eq!(bitrate, 6_566_400); + } +} diff --git a/packages/accessibility-serve/src/http.rs b/packages/accessibility-serve/src/http.rs index cd3506a..3e24dc6 100644 --- a/packages/accessibility-serve/src/http.rs +++ b/packages/accessibility-serve/src/http.rs @@ -13,8 +13,8 @@ use serde::{Deserialize, Serialize}; use accessibility_core::video::FrameKind; use crate::avcc; -use crate::input::{InputCommand, Orientation}; -use crate::session::SimSession; +use crate::session::{Orientation, Session}; +#[cfg(target_os = "macos")] use crate::settings::{Setting, SettingKey}; use crate::webrtc_stream::WebRtcEngine; @@ -22,7 +22,7 @@ const INDEX_HTML: &str = include_str!("../static/index.html"); #[derive(Clone)] pub struct AppState { - pub session: Arc, + pub session: Session, pub webrtc: Arc, pub default_transport: String, } @@ -51,6 +51,7 @@ async fn index() -> Html<&'static str> { #[derive(Serialize)] struct ConfigResponse { udid: String, + platform: &'static str, /// Raw framebuffer size. Constant regardless of orientation. width: u32, height: u32, @@ -63,13 +64,14 @@ struct ConfigResponse { async fn config(State(state): State) -> Json { let device = state.session.device_info(); Json(ConfigResponse { - udid: device.udid, + udid: device.id, + platform: device.platform, width: device.width, height: device.height, orientation: device.orientation, default_transport: state.default_transport.clone(), transports: vec!["webrtc", "h264"], - home_indicator_band: crate::input::HOME_INDICATOR_BAND, + home_indicator_band: state.session.home_indicator_band(), }) } @@ -77,9 +79,12 @@ async fn stats(State(state): State) -> Json) -> Json> { // Each read shells out to simctl, so keep it off the async worker threads. - let session = Arc::clone(&state.session); + let Some(session) = state.session.ios_session().cloned() else { + return Json(Vec::new()); + }; Json( tokio::task::spawn_blocking(move || session.settings()) .await @@ -87,17 +92,37 @@ async fn settings(State(state): State) -> Json> { ) } +#[cfg(not(target_os = "macos"))] +async fn settings(State(_state): State) -> Json> { + Json(Vec::new()) +} + +#[cfg(target_os = "macos")] #[derive(Deserialize)] struct SettingRequest { key: SettingKey, value: String, } +#[cfg(not(target_os = "macos"))] +#[derive(Deserialize)] +struct SettingRequest { + key: String, + value: String, +} + +#[cfg(target_os = "macos")] async fn set_setting( State(state): State, Json(request): Json, ) -> Response { - let session = Arc::clone(&state.session); + let Some(session) = state.session.ios_session().cloned() else { + return ( + StatusCode::BAD_REQUEST, + "settings are not supported for Android Emulator streams", + ) + .into_response(); + }; let result = tokio::task::spawn_blocking(move || session.set_setting(request.key, &request.value)).await; @@ -108,6 +133,19 @@ async fn set_setting( } } +#[cfg(not(target_os = "macos"))] +async fn set_setting( + State(_state): State, + Json(request): Json, +) -> Response { + let _ = (request.key, request.value); + ( + StatusCode::BAD_REQUEST, + "settings are not supported for Android Emulator streams", + ) + .into_response() +} + #[derive(Deserialize, Default)] struct RecordingRequest { /// Quality from 0 to 1. @@ -130,7 +168,7 @@ async fn start_recording( // Starting touches AVFoundation and the filesystem, so keep it off the // async worker threads. - let session = Arc::clone(&state.session); + let session = state.session.clone(); match tokio::task::spawn_blocking(move || session.start_recording(config)).await { Ok(Ok(path)) => Json(serde_json::json!({ "path": path })).into_response(), Ok(Err(error)) => (StatusCode::CONFLICT, error.to_string()).into_response(), @@ -140,7 +178,7 @@ async fn start_recording( async fn stop_recording(State(state): State) -> Response { // Finalizing blocks until the writer has flushed the file's index. - let session = Arc::clone(&state.session); + let session = state.session.clone(); match tokio::task::spawn_blocking(move || session.stop_recording()).await { Ok(Ok(recording)) => Json(recording).into_response(), Ok(Err(error)) => (StatusCode::CONFLICT, error.to_string()).into_response(), @@ -157,8 +195,10 @@ async fn set_orientation( State(state): State, Json(request): Json, ) -> Response { - state.session.set_orientation(request.orientation); - Json(serde_json::json!({ "orientation": request.orientation })).into_response() + match state.session.set_orientation(request.orientation) { + Ok(()) => Json(serde_json::json!({ "orientation": request.orientation })).into_response(), + Err(error) => (StatusCode::BAD_REQUEST, error.to_string()).into_response(), + } } /// Map an `anyhow` error onto a 500 with the message preserved. @@ -207,11 +247,7 @@ struct AnswerResponse { } async fn webrtc_offer(State(state): State, Json(offer): Json) -> Response { - match state - .webrtc - .answer(Arc::clone(&state.session), offer.sdp) - .await - { + match state.webrtc.answer(state.session.clone(), offer.sdp).await { Ok(sdp) => Json(AnswerResponse { sdp }).into_response(), Err(error) => internal_error(error), } @@ -300,9 +336,8 @@ async fn pump_input(state: AppState, mut socket: WebSocket) { _ => continue, }; - match serde_json::from_str::(&payload) { - Ok(command) => state.session.send_input(command), - Err(error) => tracing::debug!("ignoring malformed input event: {error}"), + if let Err(error) = state.session.send_input_json(&payload) { + tracing::debug!("ignoring malformed input event: {error}"); } } } diff --git a/packages/accessibility-serve/src/lib.rs b/packages/accessibility-serve/src/lib.rs index 01350c8..7e4cc5e 100644 --- a/packages/accessibility-serve/src/lib.rs +++ b/packages/accessibility-serve/src/lib.rs @@ -10,26 +10,20 @@ pub mod avcc; pub mod ax; #[cfg(target_os = "macos")] pub mod coverage; -#[cfg(target_os = "macos")] pub mod http; #[cfg(target_os = "macos")] pub mod input; #[cfg(target_os = "macos")] pub mod keymap; -#[cfg(target_os = "macos")] pub mod session; #[cfg(target_os = "macos")] pub mod settings; -#[cfg(target_os = "macos")] pub mod webrtc_stream; use std::net::SocketAddr; -#[cfg(target_os = "macos")] use std::sync::Arc; -#[cfg(target_os = "macos")] -use anyhow::Context; -use anyhow::Result; +use anyhow::{Context, Result}; use accessibility_core::video::VideoConfig; @@ -47,7 +41,6 @@ pub enum Transport { H264, } -#[cfg(target_os = "macos")] impl Transport { fn as_str(self) -> &'static str { match self { @@ -93,6 +86,27 @@ impl Default for ServeConfig { } } +#[derive(Debug, Clone)] +pub struct ServeEmulatorConfig { + pub serial: Option, + pub address: SocketAddr, + pub transport: Transport, + pub video: VideoConfig, + pub ice_servers: Vec, +} + +impl Default for ServeEmulatorConfig { + fn default() -> Self { + Self { + serial: None, + address: SocketAddr::from(([127, 0, 0, 1], 3200)), + transport: Transport::default(), + video: VideoConfig::default(), + ice_servers: Vec::new(), + } + } +} + /// Start capturing and serve until the process is interrupted. #[cfg(target_os = "macos")] pub async fn serve(config: ServeConfig) -> Result<()> { @@ -100,36 +114,60 @@ pub async fn serve(config: ServeConfig) -> Result<()> { .context("failed to start simulator capture")?; // The framebuffer cannot reveal orientation, so ask accessibility once // before serving; otherwise an already-rotated device renders sideways. + let session = session::Session::ios(session); session.seed_orientation().await; - let device = session.device_info(); + serve_session( + session, + config.address, + config.transport, + config.ice_servers, + ) + .await +} + +/// iOS Simulator serving is only available on macOS. +#[cfg(not(target_os = "macos"))] +pub async fn serve(_config: ServeConfig) -> Result<()> { + anyhow::bail!("Serving an iOS Simulator requires macOS") +} + +pub async fn serve_emulator(config: ServeEmulatorConfig) -> Result<()> { + let session = session::EmulatorSession::start(config.serial.as_deref(), config.video) + .context("failed to start Android Emulator capture")?; + let session = session::Session::android(session); + session.seed_orientation().await; + serve_session( + session, + config.address, + config.transport, + config.ice_servers, + ) + .await +} +async fn serve_session( + session: session::Session, + address: SocketAddr, + transport: Transport, + ice_servers: Vec, +) -> Result<()> { + let device = session.device_info(); let webrtc = Arc::new( - webrtc_stream::WebRtcEngine::new(config.ice_servers.clone()) - .context("failed to initialize WebRTC")?, + webrtc_stream::WebRtcEngine::new(ice_servers).context("failed to initialize WebRTC")?, ); - let state = http::AppState { session, webrtc, - default_transport: config.transport.as_str().to_string(), + default_transport: transport.as_str().to_string(), }; - - let listener = tokio::net::TcpListener::bind(config.address) + let listener = tokio::net::TcpListener::bind(address) .await - .with_context(|| format!("failed to bind {}", config.address))?; + .with_context(|| format!("failed to bind {address}"))?; let bound = listener.local_addr()?; - - println!("serving simulator {}", device.udid); - println!(" transport : {}", config.transport.as_str()); + println!("serving {} {}", device.platform, device.id); + println!(" transport : {}", transport.as_str()); println!(" preview : http://{bound}"); - axum::serve(listener, http::router(state)) .await .context("server error") } - -/// iOS Simulator serving is only available on macOS. -#[cfg(not(target_os = "macos"))] -pub async fn serve(_config: ServeConfig) -> Result<()> { - anyhow::bail!("Serving an iOS Simulator requires macOS") -} diff --git a/packages/accessibility-serve/src/session.rs b/packages/accessibility-serve/src/session.rs index 5c62903..7c31f6d 100644 --- a/packages/accessibility-serve/src/session.rs +++ b/packages/accessibility-serve/src/session.rs @@ -1,3 +1,305 @@ //! Compatibility re-exports for reusable iOS Simulator sessions. -pub use accessibility_core::platform::ios_simulator::session::*; +use std::sync::Arc; + +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use tokio::sync::broadcast; + +use accessibility_core::platform::android::input as android_input; +use accessibility_core::platform::android::session as android_session; +use accessibility_core::video::{EncodedFrame, Recording, RecordingConfig}; + +#[cfg(target_os = "macos")] +pub use accessibility_core::platform::ios_simulator::session::SimSession; +#[cfg(target_os = "macos")] +use accessibility_core::platform::ios_simulator::{input as ios_input, session as ios_session}; + +pub use accessibility_core::platform::android::session::EmulatorSession; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Orientation { + Portrait, + PortraitUpsideDown, + LandscapeLeft, + LandscapeRight, +} + +impl Orientation { + pub fn is_landscape(self) -> bool { + matches!(self, Self::LandscapeLeft | Self::LandscapeRight) + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct DeviceInfo { + pub id: String, + pub width: u32, + pub height: u32, + pub orientation: Orientation, + pub platform: &'static str, +} + +#[derive(Debug, Clone, Serialize)] +pub struct StatsReport { + pub uptime_secs: f64, + pub frames: u64, + pub keyframes: u64, + pub bytes: u64, + pub fps: f64, + pub mbps: f64, + pub bits_per_pixel: f64, + pub mean_frame_kb: f64, + pub keyframe_requests: u64, + pub lag_events: u64, + pub subscribers: usize, + pub recording_frames: Option, + pub width: u32, + pub height: u32, + pub encoded_width: u32, + pub encoded_height: u32, +} + +#[derive(Clone)] +pub enum Session { + #[cfg(target_os = "macos")] + Ios(Arc), + Android(Arc), +} + +impl Session { + pub fn android(session: Arc) -> Self { + Self::Android(session) + } + + #[cfg(target_os = "macos")] + pub fn ios(session: Arc) -> Self { + Self::Ios(session) + } + + pub fn subscribe(&self) -> broadcast::Receiver { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => session.subscribe(), + Self::Android(session) => session.subscribe(), + } + } + + pub fn request_keyframe(&self) { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => session.request_keyframe(), + Self::Android(session) => session.request_keyframe(), + } + } + + pub fn note_lag(&self) { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => session.note_lag(), + Self::Android(session) => session.note_lag(), + } + } + + pub fn device_info(&self) -> DeviceInfo { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => { + let device = session.device_info(); + DeviceInfo { + id: device.udid, + width: device.width, + height: device.height, + orientation: from_ios_orientation(device.orientation), + platform: "ios_simulator", + } + } + Self::Android(session) => { + let device = session.device_info(); + DeviceInfo { + id: device.serial, + width: device.width, + height: device.height, + orientation: from_android_orientation(device.orientation), + platform: "android_emulator", + } + } + } + } + + pub fn stats(&self) -> StatsReport { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => from_ios_stats(session.stats()), + Self::Android(session) => from_android_stats(session.stats()), + } + } + + pub fn send_input_json(&self, payload: &str) -> Result<()> { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => { + session.send_input(serde_json::from_str::(payload)?); + } + Self::Android(session) => { + session.send_input(serde_json::from_str::( + payload, + )?); + } + } + Ok(()) + } + + pub fn set_orientation(&self, orientation: Orientation) -> Result<()> { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => { + session.set_orientation(to_ios_orientation(orientation)); + Ok(()) + } + Self::Android(session) => session.set_orientation(to_android_orientation(orientation)), + } + } + + pub async fn seed_orientation(&self) { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => session.seed_orientation().await, + Self::Android(session) => session.seed_orientation().await, + } + } + + pub async fn ax_snapshot(&self, scan: bool) -> Result { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => Ok(serde_json::to_value(session.ax_snapshot(scan).await?)?), + Self::Android(session) => Ok(serde_json::to_value(session.ax_snapshot(scan).await?)?), + } + } + + pub async fn ax_hit_test(&self, x: f64, y: f64) -> Result { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => Ok(serde_json::to_value(session.ax_hit_test(x, y).await?)?), + Self::Android(session) => Ok(serde_json::to_value(session.ax_hit_test(x, y).await?)?), + } + } + + pub fn start_recording(&self, config: RecordingConfig) -> Result { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => session.start_recording(config), + Self::Android(_) => { + anyhow::bail!("recording is not supported for Android Emulator streams") + } + } + } + + pub fn stop_recording(&self) -> Result { + match self { + #[cfg(target_os = "macos")] + Self::Ios(session) => session.stop_recording(), + Self::Android(_) => { + anyhow::bail!("recording is not supported for Android Emulator streams") + } + } + } + + pub fn home_indicator_band(&self) -> f64 { + match self { + #[cfg(target_os = "macos")] + Self::Ios(_) => ios_input::HOME_INDICATOR_BAND, + Self::Android(_) => 1.0, + } + } + + #[cfg(target_os = "macos")] + pub fn ios_session(&self) -> Option<&Arc> { + match self { + Self::Ios(session) => Some(session), + Self::Android(_) => None, + } + } +} + +fn from_android_orientation(orientation: android_input::Orientation) -> Orientation { + match orientation { + android_input::Orientation::Portrait => Orientation::Portrait, + android_input::Orientation::PortraitUpsideDown => Orientation::PortraitUpsideDown, + android_input::Orientation::LandscapeLeft => Orientation::LandscapeLeft, + android_input::Orientation::LandscapeRight => Orientation::LandscapeRight, + } +} + +fn to_android_orientation(orientation: Orientation) -> android_input::Orientation { + match orientation { + Orientation::Portrait => android_input::Orientation::Portrait, + Orientation::PortraitUpsideDown => android_input::Orientation::PortraitUpsideDown, + Orientation::LandscapeLeft => android_input::Orientation::LandscapeLeft, + Orientation::LandscapeRight => android_input::Orientation::LandscapeRight, + } +} + +fn from_android_stats(stats: android_session::StatsReport) -> StatsReport { + StatsReport { + uptime_secs: stats.uptime_secs, + frames: stats.frames, + keyframes: stats.keyframes, + bytes: stats.bytes, + fps: stats.fps, + mbps: stats.mbps, + bits_per_pixel: stats.bits_per_pixel, + mean_frame_kb: stats.mean_frame_kb, + keyframe_requests: stats.keyframe_requests, + lag_events: stats.lag_events, + subscribers: stats.subscribers, + recording_frames: stats.recording_frames, + width: stats.width, + height: stats.height, + encoded_width: stats.encoded_width, + encoded_height: stats.encoded_height, + } +} + +#[cfg(target_os = "macos")] +fn from_ios_orientation(orientation: ios_input::Orientation) -> Orientation { + match orientation { + ios_input::Orientation::Portrait => Orientation::Portrait, + ios_input::Orientation::PortraitUpsideDown => Orientation::PortraitUpsideDown, + ios_input::Orientation::LandscapeLeft => Orientation::LandscapeLeft, + ios_input::Orientation::LandscapeRight => Orientation::LandscapeRight, + } +} + +#[cfg(target_os = "macos")] +fn to_ios_orientation(orientation: Orientation) -> ios_input::Orientation { + match orientation { + Orientation::Portrait => ios_input::Orientation::Portrait, + Orientation::PortraitUpsideDown => ios_input::Orientation::PortraitUpsideDown, + Orientation::LandscapeLeft => ios_input::Orientation::LandscapeLeft, + Orientation::LandscapeRight => ios_input::Orientation::LandscapeRight, + } +} + +#[cfg(target_os = "macos")] +fn from_ios_stats(stats: ios_session::StatsReport) -> StatsReport { + StatsReport { + uptime_secs: stats.uptime_secs, + frames: stats.frames, + keyframes: stats.keyframes, + bytes: stats.bytes, + fps: stats.fps, + mbps: stats.mbps, + bits_per_pixel: stats.bits_per_pixel, + mean_frame_kb: stats.mean_frame_kb, + keyframe_requests: stats.keyframe_requests, + lag_events: stats.lag_events, + subscribers: stats.subscribers, + recording_frames: stats.recording_frames, + width: stats.width, + height: stats.height, + encoded_width: stats.encoded_width, + encoded_height: stats.encoded_height, + } +} diff --git a/packages/accessibility-serve/src/webrtc_stream.rs b/packages/accessibility-serve/src/webrtc_stream.rs index e5da645..33e8e8c 100644 --- a/packages/accessibility-serve/src/webrtc_stream.rs +++ b/packages/accessibility-serve/src/webrtc_stream.rs @@ -28,7 +28,7 @@ use webrtc::track::track_local::track_local_static_sample::TrackLocalStaticSampl use accessibility_core::video::FrameKind; -use crate::session::SimSession; +use crate::session::Session; pub struct WebRtcEngine { api: API, @@ -65,7 +65,7 @@ impl WebRtcEngine { } /// Answer a browser offer, wiring a fresh track to the capture stream. - pub async fn answer(&self, session: Arc, offer_sdp: String) -> Result { + pub async fn answer(&self, session: Session, offer_sdp: String) -> Result { let peer = Arc::new(self.api.new_peer_connection(self.config.clone()).await?); let track = Arc::new(TrackLocalStaticSample::new( @@ -74,7 +74,7 @@ impl WebRtcEngine { ..Default::default() }, "video".to_owned(), - format!("sim-{}", session.device_info().udid), + format!("sim-{}", session.device_info().id), )); let sender = peer @@ -84,7 +84,7 @@ impl WebRtcEngine { // Sender RTCP has to be drained or feedback never gets processed. PLI // and FIR both mean "I cannot decode, send me a fresh IDR". { - let session = Arc::clone(&session); + let session = session.clone(); tokio::spawn(async move { let mut buffer = vec![0u8; 1500]; while let Ok((packets, _)) = sender.read(&mut buffer).await { @@ -100,7 +100,7 @@ impl WebRtcEngine { }); } - let forwarder = spawn_forwarder(Arc::clone(&session), Arc::clone(&track)); + let forwarder = spawn_forwarder(session.clone(), Arc::clone(&track)); // Tear the forwarding task down when the viewer goes away, otherwise // every reconnect would leak a subscriber on the broadcast channel. @@ -137,7 +137,7 @@ impl WebRtcEngine { /// Pump encoded frames from the capture broadcast onto a viewer's track. fn spawn_forwarder( - session: Arc, + session: Session, track: Arc, ) -> tokio::task::JoinHandle<()> { let mut frames = session.subscribe(); diff --git a/packages/accessibility-serve/static/index.html b/packages/accessibility-serve/static/index.html index 82efd5e..bc1e9d2 100644 --- a/packages/accessibility-serve/static/index.html +++ b/packages/accessibility-serve/static/index.html @@ -255,9 +255,11 @@

Device settings

const isLandscape = () => orientation === "landscape_left" || orientation === "landscape_right"; +const isAndroid = () => config?.platform === "android_emulator"; /** Upright display point (0..1) to raw framebuffer point. */ function toRawPoint(x, y) { + if (isAndroid()) return { x, y }; switch (orientation) { case "landscape_left": return { x: y, y: 1 - x }; case "landscape_right": return { x: 1 - y, y: x }; @@ -280,12 +282,14 @@

Device settings

/** The raw edge for a display point, or "none" if it is not near one. */ function edgeForDisplayPoint(y) { + if (isAndroid()) return "none"; if (y < config.home_indicator_band) return "none"; return BOTTOM_EDGE_BY_ORIENTATION[orientation] || "bottom"; } /** Rotate a delta, which has direction but no origin. */ function toRawDelta(dx, dy) { + if (isAndroid()) return { dx, dy }; switch (orientation) { case "landscape_left": return { dx: dy, dy: -dx }; case "landscape_right": return { dx: -dy, dy: dx }; @@ -327,10 +331,16 @@

Device settings

stage.style.height = `${uprightH * scale}px`; // The media keeps the framebuffer's own aspect and is rotated on top of it. - media.style.width = `${fbW * scale}px`; - media.style.height = `${fbH * scale}px`; - media.style.transform = - `translate(-50%, -50%) rotate(${ROTATION[orientation] || 0}deg)`; + if (isAndroid()) { + media.style.width = `${uprightW * scale}px`; + media.style.height = `${uprightH * scale}px`; + media.style.transform = "translate(-50%, -50%)"; + } else { + media.style.width = `${fbW * scale}px`; + media.style.height = `${fbH * scale}px`; + media.style.transform = + `translate(-50%, -50%) rotate(${ROTATION[orientation] || 0}deg)`; + } } window.addEventListener("resize", () => config && layout()); @@ -494,12 +504,17 @@

Device settings

async function rotate(map) { const next = map[orientation] || "portrait"; - await fetch("/api/orientation", { + const response = await fetch("/api/orientation", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ orientation: next }), }); - orientation = next; + if (!response.ok) { + setStatus(`rotation failed: ${await response.text()}`, "error"); + return; + } + const result = await response.json(); + orientation = result.orientation || next; layout(); if (selected) drawSelection(selected); } From 73321b29843528f53491345db6d0d7b6679113c0 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Mon, 17 Aug 2026 17:53:44 -0700 Subject: [PATCH 2/4] Align raw capture memory mapping dependency Keep the shared Android capture crate on the same vetted memmap2 release already used by devin-remote so consumers do not resolve an unnecessary downgrade. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <166158716+staging-devin-ai-integration[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- packages/accessibility-android-sys/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f59648a..ee04799 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2394,9 +2394,9 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memmap2" -version = "0.9.8" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843a98750cd611cc2965a8213b53b43e715f13c37a9e096c6408e69990961db7" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" dependencies = [ "libc", ] diff --git a/packages/accessibility-android-sys/Cargo.toml b/packages/accessibility-android-sys/Cargo.toml index a43de6a..a1f35a1 100644 --- a/packages/accessibility-android-sys/Cargo.toml +++ b/packages/accessibility-android-sys/Cargo.toml @@ -14,7 +14,7 @@ categories = ["accessibility", "api-bindings", "os"] anyhow.workspace = true bytes.workspace = true keyboard-types.workspace = true -memmap2 = "=0.9.8" +memmap2 = "=0.9.10" prost = "=0.13.5" tokio.workspace = true tonic = { version = "=0.13.1", default-features = false, features = ["channel", "codegen", "prost"] } From 213a7c88231b0c4cb87af68413c15ebee078c951 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Mon, 17 Aug 2026 19:49:00 -0700 Subject: [PATCH 3/4] Fix Android backend portability checks Keep new dependencies on compatible semver ranges, make the serve backend available on every target, and avoid Miri-hostile test setup while trimming nonessential docs and smoke coverage. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <166158716+staging-devin-ai-integration[bot]@users.noreply.github.com> --- docs/ANDROID_EMULATOR_RAW_CAPTURE_PLAN.md | 249 ----------------- docs/ANDROID_EMULATOR_STREAMING_PLAN.md | 250 ------------------ packages/accessibility-android-sys/Cargo.toml | 12 +- .../accessibility-android-sys/src/emulator.rs | 9 +- packages/accessibility-cli/Cargo.toml | 4 +- packages/accessibility-cli/tests/cli_smoke.rs | 11 - 6 files changed, 12 insertions(+), 523 deletions(-) delete mode 100644 docs/ANDROID_EMULATOR_RAW_CAPTURE_PLAN.md delete mode 100644 docs/ANDROID_EMULATOR_STREAMING_PLAN.md diff --git a/docs/ANDROID_EMULATOR_RAW_CAPTURE_PLAN.md b/docs/ANDROID_EMULATOR_RAW_CAPTURE_PLAN.md deleted file mode 100644 index 1afd7f1..0000000 --- a/docs/ANDROID_EMULATOR_RAW_CAPTURE_PLAN.md +++ /dev/null @@ -1,249 +0,0 @@ -# Android Emulator Raw Capture and Devin Remote Integration - -## Goal - -Use the Android Emulator's raw framebuffer stream as an input to `devin-remote`'s existing software H.264 encoder and congestion controller. Keep the screenrecord backend as the standalone and compatibility fallback. - -The production path must preserve: - -- emulator source timestamps and frame sequence numbers; -- dynamic bitrate, resolution, and frame-rate control; -- explicit keyframe requests; -- bounded queues and stale-frame dropping; -- the existing QUIC/WebSocket transport adaptation boundary; -- Android touch, keyboard, rotation, and accessibility inspection. - -## Architecture - -```text -Android Emulator renderer - | - | EmulatorController.streamScreenshot - | RGBA8888 + sequence + timestampUs - | normal gRPC bytes or local MMAP - v -Android raw capture source - | - | Arc - v -devin-remote desktop_stream - | - +-- RGBA -> I420 - +-- x264 or OpenH264 - +-- force keyframe - +-- bitrate / pixel / fps ladder - +-- capture and encode pacing - v -existing StreamEvent::Frame - | - v -existing congestion controller - | - v -QUIC or WebSocket transport -``` - -Input remains independent: - -```text -browser input - -> devin-remote - -> EmulatorController gRPC for touch, text, and ordinary keys - -> ADB fallback for Android hardware buttons -``` - -Accessibility remains ADB and `uiautomator` through `accessibility-core`. - -## Why not goldfish-webrtc-bridge - -`goldfish-webrtc-bridge` is a conditionally built, emulator-bundled host executable. Newer source also exposes the implementation as the `android-webrtc` shared library. It exists to connect the emulator framebuffer to Chromium libwebrtc encoding and WebRTC transport. - -It is not a separately supported SDK Manager package, and the standard macOS Emulator 37.1.11 package does not ship it. Building it requires the emulator source tree and Chromium's libwebrtc dependencies. - -`devin-remote` already has the parts the bridge would add: software H.264 encoding, keyframe control, pacing, link measurements, and congestion adaptation. Only the raw emulator capture source is needed. - -## Phase 1: protocol and capture probes - -Extend the minimal EmulatorController protocol with: - -- `streamScreenshot(ImageFormat) returns (stream Image)`; -- `ImageFormat` with RGBA8888, requested dimensions, orientation, display id, and optional transport; -- `ImageTransport` with MMAP channel and handle; -- `Image` with pixels, sequence number, and timestamp. - -Add a probe that measures both modes against a running emulator: - -1. RGBA8888 in ordinary gRPC messages. -2. RGBA8888 through a client-owned memory-mapped file. - -For each mode report: - -- negotiated dimensions and orientation; -- frame count and cadence under motion; -- capture-to-receive latency from `timestampUs`; -- sequence gaps; -- bytes copied per frame; -- MMAP tearing or inconsistent-frame observations; -- behavior across rotation and reconnect. - -MMAP is accepted only if it is supported by the standard emulator package and frame ownership can be made safe before the next write. Otherwise use ordinary local gRPC at the congestion controller's requested resolution. - -### Measured result - -Validated against Android Emulator 37.1.11 on macOS with a Pixel 10 API 36 AVD at a 1280-pixel capture bound: - -| Transport | Negotiated frame | Frames | Cadence | Mean source-to-receive latency | Sequence gaps | Detected tearing | -|---|---:|---:|---:|---:|---:|---:| -| gRPC bytes | 570x1280 RGBA | 287 / 5 s | 57.3 fps | 5.1 ms | 0 | n/a | -| MMAP | 570x1280 RGBA | 302 / 5 s | 60.2 fps | 2.1 ms | 0 | 0 | - -MMAP is the production default. The capture task copies each announced frame into an owned buffer immediately and publishes it through a one-frame overwrite slot. If the encoder runs below 60 fps, obsolete raw frames are dropped instead of queueing latency. - -The API treats requested dimensions as an aspect-ratio bounding box. A request for 534x1200 negotiated 534x1198, so encoder geometry is always created from the first returned frame rather than the request. - -## Phase 2: reusable raw frame source - -Implement an Android raw frame stream below `accessibility-core`'s platform layer: - -- discover and authenticate to the selected emulator; -- request RGBA8888 at an even target size; -- return owned frames with width, height, sequence, timestamp, and orientation; -- detect dropped source frames through sequence gaps; -- copy mapped bytes before acknowledging/awaiting the next frame; -- reconnect on gRPC stream termination; -- expose current source geometry and orientation; -- keep capture independent from accessibility and input workers. - -The capture source must not encode video and must not use the encoded `VideoCapture` trait. The screenrecord implementation remains separate. - -## Phase 3: devin-remote encoder integration - -Add Android as a software-encoded source in `desktop_stream`: - -1. Resolve Android display geometry before creating the encoder. -2. Request emulator frames at the geometry selected by the existing viewport and headroom policy. -3. Wrap each owned RGBA frame as the existing `capture::Frame`. -4. Feed it through `H264Encoder` and the existing `stream_software_encoded` loop. -5. Use the emulator's `timestampUs` as the capture timestamp rather than assigning it after receipt. -6. Treat the source as damage-driven because `streamScreenshot` emits on emulator frame production. -7. On forced refresh, reuse the last owned raw frame if the emulator has not produced another one. -8. On bitrate-only congestion changes, rebuild only the encoder as today. -9. On pixel-step changes or orientation changes, restart the screenshot stream and encoder with new geometry. -10. Preserve source sequence gaps and capture latency in telemetry. - -Do not forward screenrecord H.264 into the remote production path. That bypasses the existing encoder and congestion controller. - -## Phase 4: display source and control protocol - -Extend `DisplaySource` with `android_emulator` and an optional ADB serial. - -The source must send `source_config` before media frames with: - -- platform and serial; -- source geometry; -- encoded geometry; -- orientation; -- capabilities for touch, keyboard, hardware buttons, rotation, and accessibility. - -Reuse the existing simulator request/response shapes where they are genuinely common: - -- stats; -- accessibility snapshot; -- accessibility hit test; -- orientation; -- error responses with request ids. - -Keep Android-specific input translation and coordinate semantics out of the iOS implementation. - -## Phase 5: congestion and QUIC adaptation - -Continue using the current `Congestion` and `Headroom` controller. Feed it transport-native QUIC observations when the QUIC leg is enabled: - -- ACKed goodput; -- smoothed RTT and RTT variance; -- congestion window and in-flight bytes; -- sender queue age; -- datagram loss; -- viewer decode queue and paint delay. - -The existing adaptation order remains: - -1. bitrate share; -2. capture pixel steps; -3. frame-rate steps. - -A keyframe request continues through `StreamerControl::Keyframe` to the existing encoder. No emulator capture restart is required for decoder recovery. - -## Phase 6: verification - -Unit tests: - -- protobuf wire shapes; -- screenshot metadata conversion; -- sequence-gap detection; -- source timestamp conversion; -- RGBA ownership and dimensions; -- MMAP bounds and copy behavior; -- source selection parsing; -- Android geometry and orientation transitions; -- controller-triggered capture restarts at pixel-step boundaries; -- iOS and desktop source compatibility. - -End-to-end probes: - -- raw gRPC capture; -- MMAP capture when supported; -- x264 and OpenH264 encoding; -- forced keyframe; -- bitrate backoff and recovery; -- pixel and FPS backoff; -- portrait and landscape; -- touch, text, hardware buttons, and accessibility; -- stream reconnect; -- sustained motion and an extended idle/motion soak. - -Repository verification: - -```sh -# accessibility-cli -cargo fmt --all -cargo build --workspace -cargo clippy --workspace --all-targets -cargo test --workspace --lib -cargo test -p accessibility-cli --test cli_smoke - -# devin-webapp/apps/devin/devin-rs -cargo fmt -cargo check -p devin-remote -cargo test -p devin-remote -``` - -## Implementation status - -Implemented and validated: - -- EmulatorController screenshot protocol, ordinary gRPC probe, and MMAP probe. -- Reusable owned RGBA stream with sequence, timestamp, rotation, bottom-up correction, and mapping cleanup. -- `devin-remote` `android_emulator` display-source selection. -- MMAP capture with ordinary gRPC frame fallback. -- A dedicated 60 fps capture task and one-frame overwrite slot so encoder backpressure drops stale frames. -- Encoder geometry derived from the first negotiated frame. -- Existing RGBA-to-I420 and x264/OpenH264 encoder path. -- Existing keyframe, bitrate, frame-rate, queue-lag, goodput, RTT, and jitter controls. -- Pixel-step and orientation changes through a controlled display reconnect. -- EmulatorController gRPC touch, scroll, text, and USB keyboard input. -- ADB Home, Back, Lock, and app-switch buttons. -- Android accessibility snapshots and cached hit testing through the existing accessibility-core ADB backend. -- Source stats and source configuration messages. -- Real-emulator tests for raw frame to IDR and full WebSocket source configuration, encoded keyframe, and accessibility snapshot. - -Remaining integration work: - -- Add a dedicated Android Emulator workspace panel or source picker in the webapp. The backend is reachable now through `source=android_emulator&source_id=`. -- Feed transport-native QUIC ACK, congestion-window, in-flight, and datagram-loss metrics into the existing controller when the QUIC leg exposes them. The current controller continues to use drained goodput, socket queue age, viewer RTT, and jitter. -- Update `devin-remote` to the new accessibility-cli revision after these cross-repository changes are committed, then remove its temporary pinned-protocol capture client so the shared crate is the sole implementation. -- Run the extended motion/idle soak and validate the production Linux emulator image. - -## Fallback - -If raw screenshot streaming is unavailable or unstable on a target emulator package, use the existing screenrecord backend for that source. The fallback has coarser congestion adaptation because bitrate, size, and keyframe recovery require restarting screenrecord, but it preserves functional streaming and control. diff --git a/docs/ANDROID_EMULATOR_STREAMING_PLAN.md b/docs/ANDROID_EMULATOR_STREAMING_PLAN.md deleted file mode 100644 index 8af9496..0000000 --- a/docs/ANDROID_EMULATOR_STREAMING_PLAN.md +++ /dev/null @@ -1,250 +0,0 @@ -# Android Emulator Streaming Plan - -## Goal - -Add live Android Emulator display streaming, input, and accessibility inspection through the same encoded-frame and serving abstractions used by the iOS Simulator backend. The first implementation targets the official Android Emulator and arbitrary unmodified user applications. Physical Android devices and the Devin webapp UI are deferred. - -## Chosen approach - -Use a hybrid of Android's built-in interfaces: - -- `adb exec-out screenrecord --output-format=h264 --time-limit 0 -` for display video; -- `EmulatorController` gRPC for persistent input and device control; -- ADB and `uiautomator` for accessibility. - -`screenrecord` encodes the composed display inside Android and emits Annex-B H.264 without host-side decoding or re-encoding. Rust will split the byte stream into access units, classify keyframes, cache SPS and PPS NAL units, and publish frames through the existing `VideoCapture` interface. - -```text -Android composed display - | - v -screenrecord / MediaCodec - | - | ADB exec-out, Annex-B H.264 - v -Rust access-unit parser - | - v -VideoCapture / session broadcast - | - +-- WebRTC serving - +-- raw H.264 WebSocket serving - -Browser input - | - v -EmulatorController streaming gRPC - | - v -Android virtual input devices -``` - -This design does not inject code into the foreground application. Capture and input operate at the emulator or Android system level, so applications do not require modification or instrumentation. - -## Compatibility - -The requirement applies to the Android Emulator package, not the Android API level of the virtual device. - -- Initial supported emulator floor: Android Emulator 35.1.21, September 2024. -- Initial supported guest floor: Android 7.0 / API 24, where raw H.264 `screenrecord` output is available. -- Recommended version: latest stable Android Emulator. -- Runtime behavior: detect the required gRPC services and `screenrecord` options rather than trusting version strings alone. - -Virtual devices may use any Android system image at or above the guest floor that is supported by the installed emulator. - -## Phase 1: discovery and capability detection - -Implement host-side discovery in `accessibility-android-sys`: - -1. Locate active emulator discovery files: - - Linux: `~/.android/avd/running/pid_*.ini` - - macOS: `~/Library/Android/avd/running/pid_*.ini` -2. Parse the gRPC port, token, AVD identity, and related metadata. -3. Resolve an explicitly requested emulator when possible. -4. Require one unambiguous running emulator when no identifier is supplied. -5. Connect to `EmulatorController` and `android.emulation.control.v2.Rtc`. -6. Return actionable errors for missing discovery data, authentication failure, unsupported services, and ambiguous selection. - -Use minimal protocol definitions for only the services and messages this project needs. Keep the definitions versioned with the implementation and test their wire-facing serialization. - -## Phase 2: measured RTC result and video probe - -The RTC experiment was implemented as: - -```sh -cargo run -p accessibility-android-sys --example emulator_webrtc_probe -``` - -Measured on the standard macOS SDK Android Emulator 37.1.11: - -1. Emulator discovery, bearer authentication, and `EmulatorController.getStatus` succeeded. -2. Android Studio's default JWT allowlist rejected `Rtc.RequestRtcStream` because the RTC service is not listed. -3. A separate emulator launched with explicit insecure `-grpc 8556`, matching Google's container launcher, removed the authentication restriction. -4. `Rtc.RequestRtcStream` then returned gRPC `UNIMPLEMENTED`. -5. The installed macOS emulator package contains no WebRTC module or video bridge executable. - -The RTC v2 definitions exist in the emulator source tree but are not a portable capability of standard SDK emulator packages. Version checks alone cannot make this backend reliable. The implementation therefore uses raw H.264 `screenrecord` video with gRPC input. - -The capture probe must now verify: - -1. `screenrecord --output-format=h264 --time-limit 0 -` produces a sustained byte stream. -2. The parser handles three- and four-byte Annex-B start codes across arbitrary read boundaries. -3. NAL units are grouped into complete access units using access-unit delimiters, slice headers, timestamps when available, and encoder behavior observed from supported emulator builds. -4. SPS and PPS are cached and prepended to keyframes. -5. New clients and lag recovery restart `screenrecord` to force a new encoder session and decoder entry point. -6. Capture restart latency is measured and bounded. -7. Size, bitrate, frame rate, geometry, and long-running behavior are reported. - -The supported Android 36 image reports screenrecord v1.4, accepts `--time-limit 0`, and successfully emitted raw H.264 through `adb exec-out`. - -## Phase 3: reusable Android emulator session - -Implement an Android emulator session in `accessibility-core` with the same responsibilities as the iOS simulator session: - -- own the capture, input, and accessibility resources for one emulator; -- publish encoded frames through a small bounded broadcast channel; -- request an immediate keyframe for new subscribers; -- track lag and request recovery after dropped frames; -- expose raw and encoded geometry; -- collect common stream statistics; -- keep input independent from slow accessibility tree operations; -- stop and release the `screenrecord` child and gRPC streams deterministically. - -Implement `AndroidEmulatorVideoCapture` as `VideoCapture`: - -- `geometry()` returns source display geometry; -- `encoded_geometry()` reflects the configured screenrecord size; -- `request_keyframe()` replaces the screenrecord process with a fresh encoder session; -- `stop()` terminates capture and joins the parser worker; -- recording remains unsupported until a separate recording design is chosen. - -Move genuinely shared stream statistics and browser-facing accessibility response types out of the iOS-only module rather than duplicating their definitions. - -## Phase 4: input - -Use `EmulatorController.streamInputEvent` or the corresponding direct RPCs for: - -- touch begin, move, and end; -- multiple pointer identifiers where supported; -- keyboard down and up; -- UTF-8 text; -- scrolling and wheel events; -- Home, Back, Recents, Power, and other Android hardware keys; -- orientation changes. - -Normalize coordinates at the session boundary and map them against current oriented display geometry. Preserve ordering on a persistent stream instead of spawning an ADB process per event. - -Where the existing client protocol supplies USB HID usage codes, use the emulator protocol's USB code type rather than maintaining an unnecessary second key map. - -## Phase 5: accessibility inspection - -Reuse `AndroidAccessibility` and its ADB client for tree collection: - -1. Fetch the hierarchy through `uiautomator dump`. -2. Flatten it into the shared `ElementDetail` representation. -3. Normalize element bounds against current display geometry. -4. Preserve labels, values, resource identifiers, roles, states, and actions. -5. Compute tree coverage through the shared coverage grid. -6. Implement hit testing against the latest cached tree, preferring the smallest containing element. - -Android hit testing is cache-based and must not be presented as the same authoritative live operation available through the iOS accessibility bridge. Point-grid discovery remains iOS-specific unless an Android need is demonstrated. - -## Phase 6: standalone serving interface - -Add a command parallel to `serve-sim`: - -```sh -cargo run -p accessibility-cli -- serve-emulator --serial emulator-5554 --port 3200 -``` - -It should support the existing video options where the emulator RTC sender can honor them and expose the same core surface: - -- source configuration and geometry; -- live encoded video; -- input WebSocket; -- accessibility snapshot; -- accessibility hit test; -- orientation; -- stream statistics. - -Refactor `accessibility-serve` around a platform-neutral session boundary rather than copying the HTTP, WebSocket, AVCC, and WebRTC transport implementations. - -## Phase 7: tests and verification - -Add tests at representation boundaries: - -- discovery file parsing and emulator selection; -- missing, ambiguous, and unsupported emulator errors; -- protobuf request serialization and response decoding; -- Annex-B stream splitting across arbitrary read boundaries; -- H.264 access-unit assembly; -- SPS/PPS caching and keyframe reconstruction; -- screenrecord restart recovery; -- coordinate mapping under each orientation; -- Android tree flattening, normalization, coverage, and hit testing; -- session lag and keyframe-request accounting. - -Run: - -```sh -cargo fmt --all -cargo build --workspace -cargo clippy --workspace --all-targets -cargo test --workspace --lib -cargo test -p accessibility-cli --test cli_smoke -``` - -End-to-end verification requires a supported running emulator. A successful run must cover: - -- arbitrary APK installation and launch; -- sustained video under motion; -- touch drag and scrolling; -- text and hardware keys; -- rotation and geometry updates; -- accessibility tree and hit testing; -- decoder recovery after capture restart; -- disconnect and reconnect. - -Do not claim end-to-end validation if only compile and unit verification were possible. - -## Validation record - -Validated on macOS against Android Emulator 37.1.11 running an Android 36 arm64 image: - -- raw screenrecord H.264 at 570x1280 and 1280x570; -- Constrained Baseline SPS, PPS, IDR, and delta-frame parsing; -- 56-70 frames during a five-second gesture stimulus; -- approximately 0.9-1.2 Mbps during that stimulus; -- fresh SPS/PPS+IDR approximately 170-470 ms after capture restart; -- gRPC touch injection against the Settings UI; -- ADB hardware Home fallback after the emulator's successful `sendKey` RPC produced no action; -- accessibility tree, normalized coverage, cached hover picking, and live hit testing; -- browser WebCodecs rendering through the raw H.264 WebSocket; -- exact portrait/landscape rotation using `wm fixed-to-user-rotation` and orientation-aware capture restart; -- return from landscape to portrait with encoded geometry updated in both directions. - -The standard macOS emulator package did not provide RTC v2 media streaming even at Emulator 37.1.11, so RTC source presence is not used as a compatibility claim. - -## Deferred integration - -Do not modify `devin-webapp` until the standalone backend is validated. - -The follow-up integration will: - -1. update the `accessibility-core` revision used by `devin-remote`; -2. add `source=android_emulator` to the display source selection; -3. forward Android frames through the existing port 6080 writer; -4. translate the shared request/response protocol for Android input and inspection; -5. add a separate Android Emulator tab while sharing transport, inspector, and device-frame UI internals with iOS; -6. add the Android quote surface and platform availability gating. - -## Explicit non-goals for the first implementation - -- Physical Android devices. -- Target-app instrumentation or injection. -- A scrcpy server dependency. -- An RTC/WebRTC capture backend in standard SDK emulator packages. -- Audio streaming. -- MP4 recording. -- Multi-emulator simultaneous serving from one process. -- Devin webapp UI changes. diff --git a/packages/accessibility-android-sys/Cargo.toml b/packages/accessibility-android-sys/Cargo.toml index a1f35a1..c7d657c 100644 --- a/packages/accessibility-android-sys/Cargo.toml +++ b/packages/accessibility-android-sys/Cargo.toml @@ -14,16 +14,16 @@ categories = ["accessibility", "api-bindings", "os"] anyhow.workspace = true bytes.workspace = true keyboard-types.workspace = true -memmap2 = "=0.9.10" -prost = "=0.13.5" +memmap2 = "0.9" +prost = "0.13" tokio.workspace = true -tonic = { version = "=0.13.1", default-features = false, features = ["channel", "codegen", "prost"] } +tonic = { version = "0.13", default-features = false, features = ["channel", "codegen", "prost"] } [dev-dependencies] -rtp = "=0.14.0" +rtp = "0.14" serde_json.workspace = true webrtc.workspace = true [build-dependencies] -protoc-bin-vendored = "=3.2.0" -tonic-build = { version = "=0.13.1", default-features = false, features = ["prost"] } +protoc-bin-vendored = "3" +tonic-build = { version = "0.13", default-features = false, features = ["prost"] } diff --git a/packages/accessibility-android-sys/src/emulator.rs b/packages/accessibility-android-sys/src/emulator.rs index b0292a2..1b14e25 100644 --- a/packages/accessibility-android-sys/src/emulator.rs +++ b/packages/accessibility-android-sys/src/emulator.rs @@ -338,13 +338,14 @@ fn parse_properties(contents: &str) -> BTreeMap { #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicU64, Ordering}; + use super::*; + static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + fn test_directory(name: &str) -> PathBuf { - let nonce = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos(); + let nonce = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed); let path = std::env::temp_dir().join(format!("accessibility-android-{name}-{nonce}")); std::fs::create_dir_all(&path).unwrap(); path diff --git a/packages/accessibility-cli/Cargo.toml b/packages/accessibility-cli/Cargo.toml index d1cf5ea..b9bb3b5 100644 --- a/packages/accessibility-cli/Cargo.toml +++ b/packages/accessibility-cli/Cargo.toml @@ -9,6 +9,7 @@ homepage.workspace = true [dependencies] accessibility-core.workspace = true +accessibility-serve.workspace = true anyhow.workspace = true clap.workspace = true ctrlc.workspace = true @@ -23,6 +24,3 @@ serial_test.workspace = true [[bin]] name = "accessibility-cli" path = "src/main.rs" - -[target.'cfg(target_os = "macos")'.dependencies] -accessibility-serve.workspace = true diff --git a/packages/accessibility-cli/tests/cli_smoke.rs b/packages/accessibility-cli/tests/cli_smoke.rs index e7df4a2..f03a3c5 100644 --- a/packages/accessibility-cli/tests/cli_smoke.rs +++ b/packages/accessibility-cli/tests/cli_smoke.rs @@ -12,17 +12,6 @@ fn help_mentions_primary_command() { .stdout(predicate::str::contains("--platform")); } -#[test] -fn serve_emulator_help_exposes_stream_options() { - let mut cmd = Command::cargo_bin("accessibility-cli").unwrap(); - cmd.args(["serve-emulator", "--help"]) - .assert() - .success() - .stdout(predicate::str::contains("--serial")) - .stdout(predicate::str::contains("--transport")) - .stdout(predicate::str::contains("--max-dimension")); -} - #[test] fn invalid_platform_fails_before_touching_accessibility_backend() { let mut cmd = Command::cargo_bin("accessibility-cli").unwrap(); From 0d6e4146acb23b4cc73d15b3683a997d128c6572 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Mon, 17 Aug 2026 19:55:13 -0700 Subject: [PATCH 4/4] Keep cross-platform CI checks portable Skip filesystem-backed discovery tests under isolated Miri and keep the recording parameter warning-free when the iOS variant is compiled out. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <166158716+staging-devin-ai-integration[bot]@users.noreply.github.com> --- packages/accessibility-android-sys/src/emulator.rs | 2 ++ packages/accessibility-serve/src/session.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/accessibility-android-sys/src/emulator.rs b/packages/accessibility-android-sys/src/emulator.rs index 1b14e25..c6a4050 100644 --- a/packages/accessibility-android-sys/src/emulator.rs +++ b/packages/accessibility-android-sys/src/emulator.rs @@ -352,6 +352,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn parses_discovery_file() { let directory = test_directory("parse"); let path = directory.join("pid_1234.ini"); @@ -370,6 +371,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] fn selects_one_emulator() { let directory = test_directory("select"); std::fs::write( diff --git a/packages/accessibility-serve/src/session.rs b/packages/accessibility-serve/src/session.rs index 7c31f6d..6bd90e3 100644 --- a/packages/accessibility-serve/src/session.rs +++ b/packages/accessibility-serve/src/session.rs @@ -186,10 +186,10 @@ impl Session { } } - pub fn start_recording(&self, config: RecordingConfig) -> Result { + pub fn start_recording(&self, _config: RecordingConfig) -> Result { match self { #[cfg(target_os = "macos")] - Self::Ios(session) => session.start_recording(config), + Self::Ios(session) => session.start_recording(_config), Self::Android(_) => { anyhow::bail!("recording is not supported for Android Emulator streams") }