diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27c17adb..ef0b9e10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,9 @@ jobs: with: tool: cargo-llvm-cov + - name: Install ALSA development files + run: sudo apt-get update && sudo apt-get install --yes libasound2-dev + - uses: Swatinem/rust-cache@v2 - name: Check formatting @@ -106,6 +109,9 @@ jobs: - uses: Swatinem/rust-cache@v2 + - name: Install ALSA development files + run: sudo apt-get update && sudo apt-get install --yes libasound2-dev + - name: Build documentation env: RUSTDOCFLAGS: -D warnings @@ -139,6 +145,9 @@ jobs: with: toolchain: ${{ steps.msrv.outputs.version }} + - name: Install ALSA development files + run: sudo apt-get update && sudo apt-get install --yes libasound2-dev + - uses: Swatinem/rust-cache@v2 - name: Build with the declared MSRV diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7d6cb1d3..d25b6716 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,6 +50,9 @@ jobs: with: tool: cargo-llvm-cov + - name: Install ALSA development files + run: sudo apt-get update && sudo apt-get install --yes libasound2-dev + - uses: Swatinem/rust-cache@v2 - name: Check formatting diff --git a/Cargo.lock b/Cargo.lock index df05affd..a7f61c62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,6 +144,28 @@ dependencies = [ "memchr", ] +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.13.1", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android_system_properties" version = "0.1.6" @@ -203,6 +225,30 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.13.1", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.1" @@ -273,7 +319,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", - "shlex", + "jobserver", + "libc", + "shlex 2.0.1", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", ] [[package]] @@ -322,6 +385,27 @@ dependencies = [ "inout", ] +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "const-oid" version = "0.10.2" @@ -350,7 +434,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "064badf302c3194842cf2c5d61f56cc88e54a759313879cdf03abdd27d0c3b97" dependencies = [ - "bitflags", + "bitflags 2.13.1", "core-foundation", "core-graphics-types", "foreign-types", @@ -363,11 +447,54 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" dependencies = [ - "bitflags", + "bitflags 2.13.1", "core-foundation", "libc", ] +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4739a805a62757a83e5654fa3faabec0442666b263bb2287d5a8185bfd953" +dependencies = [ + "bindgen", +] + +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs", + "dasp_sample", + "jni", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -430,6 +557,12 @@ dependencies = [ "cipher", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.11.1" @@ -502,7 +635,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags", + "bitflags 2.13.1", "objc2", ] @@ -517,6 +650,12 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + [[package]] name = "equivalent" version = "1.0.2" @@ -746,6 +885,12 @@ dependencies = [ "polyval", ] +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "hashbrown" version = "0.17.1" @@ -892,7 +1037,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.62.2", ] [[package]] @@ -1059,12 +1204,75 @@ version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.105" @@ -1082,6 +1290,16 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libredox" version = "0.1.25" @@ -1115,6 +1333,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.8.3" @@ -1137,6 +1364,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1168,12 +1401,62 @@ dependencies = [ "pxfm", ] +[[package]] +name = "ndk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" +dependencies = [ + "bitflags 2.13.1", + "jni-sys 0.3.1", + "log", + "ndk-sys", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys 0.3.1", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1183,6 +1466,28 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "objc2" version = "0.6.4" @@ -1198,7 +1503,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" dependencies = [ - "bitflags", + "bitflags 2.13.1", "objc2", "objc2-core-foundation", "objc2-foundation", @@ -1210,7 +1515,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.13.1", "dispatch2", "objc2", ] @@ -1227,12 +1532,35 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags", + "bitflags 2.13.1", "block2", "objc2", "objc2-core-foundation", ] +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1263,13 +1591,19 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "png" version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" dependencies = [ - "bitflags", + "bitflags 2.13.1", "crc32fast", "fdeflate", "flate2", @@ -1312,6 +1646,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -1602,7 +1945,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -1763,6 +2106,12 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "shlex" version = "2.0.1" @@ -2019,6 +2368,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tinycomputer-accessibility" +version = "0.7.0" +dependencies = [ + "cpal", + "log", + "serde", + "serde_json", + "thiserror 2.0.20", +] + [[package]] name = "tinycomputer-browser" version = "0.7.0" @@ -2130,6 +2490,7 @@ version = "0.3.0" dependencies = [ "anyhow", "httpdate", + "regex", "url", ] @@ -2292,6 +2653,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.25.15+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + [[package]] name = "toml_parser" version = "1.1.3+spec-1.1.0" @@ -2328,7 +2701,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.1", "bytes", "futures-util", "http", @@ -2671,6 +3044,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -2680,7 +3073,7 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link", - "windows-result", + "windows-result 0.4.1", "windows-strings", ] @@ -2712,6 +3105,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -2730,6 +3132,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -2757,6 +3168,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -2788,6 +3214,12 @@ dependencies = [ "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -2800,6 +3232,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -2812,6 +3250,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -2830,6 +3274,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -2842,6 +3292,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -2854,6 +3310,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -2866,6 +3328,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -2883,6 +3351,9 @@ name = "winnow" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] [[package]] name = "writeable" diff --git a/Cargo.toml b/Cargo.toml index 5d3b12e4..4dfabb71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,10 @@ tinycomputer-engine = { path = "crates/tinycomputer-engine" } # The agent's on-screen cursor: aim points and human glide paths a surface # draws over its content. Cosmetic; it sends no input. tinycomputer-cursor = { path = "crates/tinycomputer-cursor" } +# In-process desktop accessibility middleware for hosts: focused-text queries, +# permission detection, the Globe key listener. A plain library with no bus, so +# a host links it directly instead of calling the module. +tinycomputer-accessibility = { path = "crates/tinycomputer-accessibility" } # Native typed Jev transport used by the desktop-control policy: the decisions # crate of the vendored `vendor/tinyinference` submodule, pinned by its gitlink # so provider validation and retry behavior are reproducible. Only this crate @@ -85,6 +89,12 @@ thiserror = "2" serde = { version = "1", features = ["derive"] } # Positional argument arrays and the module configuration blob. serde_json = "1" +# Debug and warning logs from `tinycomputer-accessibility`: the facade only, the +# host installs the logger. +log = "0.4" +# `tinycomputer-accessibility`'s `microphone-probe` feature asks the default +# input device whether the app may record. Same major as the host's capture stack. +cpal = "0.15" # Unpredictable one-use handles bind host confirmation to a module-held action. getrandom = "0.4" # Held browser outputs travel as base64 chunks with a SHA-256 digest. Same diff --git a/README.md b/README.md index ca868489..e674646e 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,7 @@ scripts/docker-lab -- crates/tinycomputer-examples/fixtures/run task_fixture | `tinycomputer-desktop` | desktop apps, through agent-desktop | | `tinycomputer-browser` | web pages, through agent-browser | | `tinycomputer-cursor` | the cursor you can watch | +| `tinycomputer-accessibility` | in-process focus, permission and Globe-key answers for a host | | `tinycomputer-skills` | the guide and schemas for calling agents | | `tinycomputer-examples` | examples, the lab, saved plans | diff --git a/crates/tinycomputer-accessibility/Cargo.toml b/crates/tinycomputer-accessibility/Cargo.toml new file mode 100644 index 00000000..2f5eaa83 --- /dev/null +++ b/crates/tinycomputer-accessibility/Cargo.toml @@ -0,0 +1,62 @@ +[package] +name = "tinycomputer-accessibility" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "In-process desktop accessibility middleware for hosts: focused-text queries, paste validation, permission detection, the Globe key listener, and terminal heuristics, with the macOS FFI and Swift helper process behind it." +documentation = "https://docs.rs/tinycomputer-accessibility" +readme = "README.md" +keywords = ["accessibility", "macos", "permissions", "focus"] +categories = ["os"] +publish = false + +[features] +default = [] +# Probe the default input device with `cpal` to detect microphone permission. +# Without it `detect_microphone_permission` reports `PermissionState::Unknown`, +# so a host that never records audio does not compile the audio stack. +microphone-probe = ["dep:cpal"] + +[dependencies] +# Permission state and focus context cross into hosts as JSON. +serde = { workspace = true } +# The Swift helper speaks one JSON object per line over stdin/stdout. +serde_json = { workspace = true } +# Debug and warning logs for the helper process, focus queries, and probes. +# Only the facade: the host installs the logger. +log = { workspace = true } +# Typed errors returned by fallible public accessibility APIs. +thiserror = { workspace = true } +# Default-input-device probe behind `microphone-probe`. +cpal = { workspace = true, optional = true } + +# The workspace lint set, except that `unsafe_code` is `deny` rather than +# `forbid`: the macOS permission checks call ApplicationServices, CoreFoundation +# and IOKit directly (`src/permissions.rs`), so that module — and only it — +# allows `unsafe`, each block with a `// SAFETY:` comment. Keep the rest +# identical to the root `Cargo.toml`. +[lints.rust] +unsafe_code = "deny" +missing_docs = "warn" +missing_debug_implementations = "warn" +unreachable_pub = "warn" +rust_2018_idioms = { level = "warn", priority = -1 } + +[lints.clippy] +all = { level = "warn", priority = -1 } +pedantic = { level = "warn", priority = -1 } +unwrap_used = "warn" +expect_used = "warn" +panic = "warn" +todo = "warn" +unimplemented = "warn" +missing_errors_doc = "warn" +missing_panics_doc = "warn" +doc_markdown = "warn" +must_use_candidate = "warn" + +[lints.rustdoc] +broken_intra_doc_links = "warn" +private_intra_doc_links = "warn" diff --git a/crates/tinycomputer-accessibility/README.md b/crates/tinycomputer-accessibility/README.md new file mode 100644 index 00000000..5946a355 --- /dev/null +++ b/crates/tinycomputer-accessibility/README.md @@ -0,0 +1,46 @@ +# tinycomputer-accessibility + +Part of [tinycomputer](../../README.md), a decision model (Jev) based harness for +desktop and browser automation, written in Rust. Its user guide is +[`docs/crates/tinycomputer-accessibility/`](../../docs/crates/tinycomputer-accessibility/README.md). + +Desktop accessibility middleware for a host that needs answers **in-process and +synchronously**: which text field has focus, may this process see input events, +did the Globe key go down. It is a plain library with no TinyBus, no async +runtime, and no engine. It is not the agent-facing desktop surface: that is +[`tinycomputer-desktop`](../tinycomputer-desktop/README.md), served over the bus +by the `tinycomputer` module. + +| Module | Holds | +|---|---| +| `focus` | `focused_text_context[_verbose]` and `validate_focused_target`: query and re-check the focused text element | +| `permissions` | Accessibility, Input Monitoring and Microphone detection and requests; macOS `ApplicationServices` / `CoreFoundation` / `IOKit` FFI | +| `globe` | The macOS Globe/Fn key listener (a Swift process reporting `FN_DOWN` / `FN_UP`) | +| `helper/` | The persistent Swift helper process (focus, paste, overlay over stdin/stdout JSON) and its embedded Swift source | +| `automation_state` | Session flag for a denied "System Events" Apple Events grant | +| `terminal`, `text_util` | Terminal-window heuristics and AX string normalisation | +| `types` | `FocusedTextContext`, `ElementBounds`, `PermissionKind`, `PermissionState`, `PermissionStatus` | +| `error` | `Error` and the crate-wide `Result` used by fallible public operations | + +## Features + +| Feature | Effect | +|---|---| +| `microphone-probe` | Probe whether an input device is present with `cpal`. Device enumeration does not confirm recording authorization, so macOS and Windows return `Unknown` until the host opens a capture stream. Off by default. | + +## Platforms + +Everything real happens on macOS. Elsewhere focus queries return +`Error::UnsupportedPlatform`, `validate_focused_target` passes, the Globe listener reports +`supported: false`, and Accessibility / Input Monitoring are `Unsupported`. + +## Unsafe + +The workspace forbids `unsafe`; this crate lowers it to `deny` and only +`src/permissions.rs` allows it, for the macOS permission FFI, each block with a +`// SAFETY:` comment. Dependencies are `serde`, `serde_json`, `log`, and +`thiserror` (plus `cpal` behind `microphone-probe`). + +The macOS paths (FFI, helper process) cannot run in Linux CI: tests cover the +pure logic and the non-macOS fallbacks, and macOS code is checked with +`cargo check --target aarch64-apple-darwin`. diff --git a/crates/tinycomputer-accessibility/src/automation_state.rs b/crates/tinycomputer-accessibility/src/automation_state.rs new file mode 100644 index 00000000..8ec798d1 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/automation_state.rs @@ -0,0 +1,62 @@ +//! Session-local denial flag for macOS Apple Events automation. +//! +//! Captures the reactive signal that osascript returns +//! `errAEEventNotPermitted (-1743)` when the calling app lacks an +//! Automation grant for the target. After observation, gated osascript +//! call sites short-circuit until the flag is cleared. +//! +//! Why a reactive flag instead of an in-process probe: +//! `AEDeterminePermissionToAutomateTarget(askUserIfNeeded=false)` would +//! be the principled silent-probe API but it SIGBUSes inside +//! AE.framework's TCC client whenever called from any binary that links +//! the host process (PAC mismatch between arm64 Rust binaries and +//! arm64e Apple frameworks, mediated by `objc2-app-kit` transitive +//! deps). Verified across seven workarounds during #985 plan validation. +//! The osascript stderr `(-1743)` substring is a stable Apple-defined +//! error code that's already produced by the existing fallback path — +//! capturing it costs nothing extra and avoids the FFI entirely. +//! +//! The host can clear the flag after a user changes the Automation grant in +//! System Settings. Clearing only resets the remembered denial; it does not +//! request or grant permission. +//! +//! The host must observe `(-1743)` in an error returned by +//! `focused_text_context` or `focused_text_context_verbose` and call +//! `mark_system_events_denied` itself. This crate does not infer that denial +//! from a query result. + +use std::sync::atomic::{AtomicBool, Ordering}; + +static SYSTEM_EVENTS_DENIED: AtomicBool = AtomicBool::new(false); + +/// Mark that osascript has returned -1743 for `tell application "System +/// Events"` in this process. +/// +/// Hosts should call this after detecting `(-1743)` in a focus-query error; +/// this crate does not call the function automatically. +pub fn mark_system_events_denied() { + SYSTEM_EVENTS_DENIED.store(true, Ordering::Relaxed); +} + +/// True iff a -1743 has been observed in this process since the last +/// `clear_automation_denial`. The focus fallback checks this and short-circuits +/// before spawning osascript. +pub fn system_events_denied() -> bool { + SYSTEM_EVENTS_DENIED.load(Ordering::Relaxed) +} + +/// Reset the denial flag after a user changes the Automation grant, allowing +/// the next focus query to probe again. The host calls this explicitly. +pub fn clear() { + SYSTEM_EVENTS_DENIED.store(false, Ordering::Relaxed); +} + +#[cfg(test)] +pub(crate) fn test_lock() -> std::sync::MutexGuard<'static, ()> { + static M: std::sync::Mutex<()> = std::sync::Mutex::new(()); + M.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +#[cfg(test)] +#[path = "automation_state_tests.rs"] +mod tests; diff --git a/crates/tinycomputer-accessibility/src/automation_state_tests.rs b/crates/tinycomputer-accessibility/src/automation_state_tests.rs new file mode 100644 index 00000000..d6e34b48 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/automation_state_tests.rs @@ -0,0 +1,56 @@ +//! Tests for marking, clearing, and concurrently reading the System Events denial flag. + +#![allow(clippy::unwrap_used, clippy::expect_used)] +use super::*; + +#[test] +fn defaults_to_not_denied() { + let _g = test_lock(); + clear(); + assert!(!system_events_denied()); +} + +#[test] +fn mark_then_observe() { + let _g = test_lock(); + clear(); + assert!(!system_events_denied()); + mark_system_events_denied(); + assert!(system_events_denied()); + clear(); + assert!(!system_events_denied()); +} + +#[test] +fn idempotent_mark_and_clear() { + let _g = test_lock(); + clear(); + mark_system_events_denied(); + mark_system_events_denied(); + assert!(system_events_denied()); + clear(); + clear(); + assert!(!system_events_denied()); +} + +#[test] +fn concurrent_mark_and_read() { + let _g = test_lock(); + clear(); + let producers: Vec<_> = (0..8) + .map(|_| std::thread::spawn(mark_system_events_denied)) + .collect(); + let readers: Vec<_> = (0..8) + .map(|_| std::thread::spawn(system_events_denied)) + .collect(); + for h in producers { + h.join().unwrap(); + } + for h in readers { + // Read may race the marks — only the post-join state is + // load-bearing for correctness. + let _ = h.join().unwrap(); + } + assert!(system_events_denied()); + clear(); +} diff --git a/crates/tinycomputer-accessibility/src/error/error_tests.rs b/crates/tinycomputer-accessibility/src/error/error_tests.rs new file mode 100644 index 00000000..6fba2746 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/error/error_tests.rs @@ -0,0 +1,44 @@ +//! Tests for the public accessibility error categories and display messages. + +use super::Error; + +#[test] +fn displays_platform_and_operation_errors() { + assert_eq!( + Error::UnsupportedPlatform.to_string(), + "operation is unsupported on this platform" + ); + assert_eq!( + Error::FocusQuery("helper timed out".to_string()).to_string(), + "focused text query failed: helper timed out" + ); + assert_eq!( + Error::HelperTimeout("osascript exceeded 1500ms".to_string()).to_string(), + "accessibility helper timed out: osascript exceeded 1500ms" + ); + assert_eq!( + Error::GlobeListener("helper failed".to_string()).to_string(), + "Globe listener operation failed: helper failed" + ); +} + +#[test] +fn focus_identity_errors_preserve_expected_and_actual_values() { + let app = Error::FocusChanged { + expected: "Editor".to_string(), + actual: "Browser".to_string(), + }; + assert_eq!( + app.to_string(), + "focus shifted from 'Editor' to 'Browser', aborting insertion" + ); + + let role = Error::FocusRoleChanged { + expected: "AXTextArea".to_string(), + actual: "AXButton".to_string(), + }; + assert_eq!( + role.to_string(), + "focus role changed from 'AXTextArea' to 'AXButton', aborting insertion" + ); +} diff --git a/crates/tinycomputer-accessibility/src/error/mod.rs b/crates/tinycomputer-accessibility/src/error/mod.rs new file mode 100644 index 00000000..7d982cf5 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/error/mod.rs @@ -0,0 +1,44 @@ +//! Errors returned by fallible accessibility operations. + +/// Failure from a public accessibility operation. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The requested operation is unavailable on this operating system. + #[error("operation is unsupported on this platform")] + UnsupportedPlatform, + /// A focus query could not obtain focused text. + #[error("focused text query failed: {0}")] + FocusQuery(String), + /// A helper process or OS command did not answer before its deadline. + #[error("accessibility helper timed out: {0}")] + HelperTimeout(String), + /// Focus moved to a different application before the operation completed. + #[error("focus shifted from '{expected}' to '{actual}', aborting insertion")] + FocusChanged { + /// Application captured by the caller. + expected: String, + /// Application that currently owns focus. + actual: String, + }, + /// The focused element changed to an incompatible role. + #[error("focus role changed from '{expected}' to '{actual}', aborting insertion")] + FocusRoleChanged { + /// Element role captured by the caller. + expected: String, + /// Role of the currently focused element. + actual: String, + }, + /// The focused element moved from its captured bounds. + #[error("focused element bounds changed, aborting insertion")] + FocusTargetChanged, + /// The Globe listener could not be started, inspected, or stopped. + #[error("Globe listener operation failed: {0}")] + GlobeListener(String), +} + +/// Result type for public accessibility operations. +pub type Result = std::result::Result; + +#[cfg(test)] +#[path = "error_tests.rs"] +mod tests; diff --git a/crates/tinycomputer-accessibility/src/focus.rs b/crates/tinycomputer-accessibility/src/focus.rs new file mode 100644 index 00000000..7c0ec197 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/focus.rs @@ -0,0 +1,577 @@ +//! Accessibility focus queries. +//! +//! Primary path: unified Swift helper (native AX API, fast, persistent process). +//! Fallback: osascript subprocess (slower, but works without compiled helper). + +#[cfg(target_os = "macos")] +use super::terminal::{is_terminal_app, is_text_role}; +#[cfg(target_os = "macos")] +use super::text_util::{normalize_ax_value, parse_ax_number}; +#[cfg(target_os = "macos")] +use super::types::ElementBounds; +use super::types::FocusedTextContext; +use super::{Error, Result as AccessibilityResult}; +#[cfg(any(target_os = "macos", all(test, unix)))] +use std::{ + io::Read, + process::{Command, ExitStatus, Output, Stdio}, + time::{Duration, Instant}, +}; + +#[cfg(target_os = "macos")] +const FOCUS_COMMAND_TIMEOUT: Duration = Duration::from_millis(1_500); +#[cfg(any(target_os = "macos", all(test, unix)))] +const COMMAND_TIMEOUT_POLL_INTERVAL: Duration = Duration::from_millis(10); + +#[cfg(any(target_os = "macos", all(test, unix)))] +fn command_output_with_timeout( + command_name: &str, + command: &mut Command, + timeout: Duration, +) -> Result { + command.stdout(Stdio::piped()).stderr(Stdio::piped()); + let mut child = command + .spawn() + .map_err(|e| format!("failed to run {command_name}: {e}"))?; + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + let stdout_reader = std::thread::spawn(move || read_pipe(stdout)); + let stderr_reader = std::thread::spawn(move || read_pipe(stderr)); + let started_at = Instant::now(); + + loop { + match child.try_wait() { + Ok(Some(status)) => { + return collect_command_output(command_name, status, stdout_reader, stderr_reader); + } + Ok(None) if started_at.elapsed() >= timeout => { + let _ = child.kill(); + let _ = child.wait(); + return Err(format!( + "{command_name} timed out after {}ms", + timeout.as_millis() + )); + } + Ok(None) => std::thread::sleep(COMMAND_TIMEOUT_POLL_INTERVAL), + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(format!("failed to wait for {command_name}: {error}")); + } + } + } +} + +#[cfg(any(target_os = "macos", all(test, unix)))] +fn read_pipe(pipe: Option) -> Vec { + let mut bytes = Vec::new(); + if let Some(mut pipe) = pipe { + let _ = pipe.read_to_end(&mut bytes); + } + bytes +} + +#[cfg(any(target_os = "macos", all(test, unix)))] +fn collect_command_output( + command_name: &str, + status: ExitStatus, + stdout_reader: std::thread::JoinHandle>, + stderr_reader: std::thread::JoinHandle>, +) -> Result { + let stdout = stdout_reader + .join() + .map_err(|_| format!("failed to collect {command_name} stdout"))?; + let stderr = stderr_reader + .join() + .map_err(|_| format!("failed to collect {command_name} stderr"))?; + Ok(Output { + status, + stdout, + stderr, + }) +} + +// --------------------------------------------------------------------------- +// Focus query: unified helper → osascript fallback +// --------------------------------------------------------------------------- + +#[cfg(target_os = "macos")] +/// Query the OS for the focused text element, without verbose diagnostics. +/// +/// # Errors +/// +/// Returns a message when the focus query fails or is unsupported on this platform. +pub fn focused_text_context() -> AccessibilityResult { + let ctx = focused_text_context_verbose()?; + if let Some(err) = ctx.raw_error.as_ref() { + return Err(Error::FocusQuery(format!( + "focused text unavailable via accessibility api: {err}" + ))); + } + Ok(ctx) +} + +/// Query the focused text element. Tries the unified Swift helper first (native AX, ~5-15ms), +/// falls back to osascript (~50-100ms) if the helper is unavailable. +/// +/// # Errors +/// +/// Returns a message when both the helper and the osascript fallback fail. +#[cfg(target_os = "macos")] +pub fn focused_text_context_verbose() -> AccessibilityResult { + match focused_text_via_helper() { + Ok(mut ctx) if ctx.raw_error.is_some() => { + log::debug!( + "[accessibility] helper returned raw_error={:?}, falling back to osascript", + ctx.raw_error + ); + match focused_text_via_osascript() { + Ok(fallback) => Ok(fallback), + Err(fallback_err) => { + log::debug!( + "[accessibility] osascript fallback failed ({fallback_err}); keeping helper context" + ); + if let Some(helper_error) = ctx.raw_error.as_mut() { + helper_error + .push_str(&format!("; osascript fallback failed: {fallback_err}")); + } else { + ctx.raw_error = Some(fallback_err); + } + Ok(ctx) + } + } + } + Ok(ctx) => Ok(ctx), + Err(helper_err) => { + log::debug!( + "[accessibility] helper focus query failed ({helper_err}), falling back to osascript" + ); + focused_text_via_osascript().map_err(classify_focus_error) + } + } +} + +#[cfg(target_os = "macos")] +fn classify_focus_error(message: String) -> Error { + if message.contains("timed out") { + Error::HelperTimeout(message) + } else { + Error::FocusQuery(message) + } +} + +/// Focus query via the unified Swift helper. +#[cfg(target_os = "macos")] +// AX coordinates are screen points, well inside `i32`. +#[allow(clippy::cast_possible_truncation)] +fn focused_text_via_helper() -> Result { + let request = serde_json::json!({"type": "focus"}); + let resp = super::helper::helper_send_receive(&request)?; + + let app_name = resp + .get("app_name") + .and_then(|v| v.as_str()) + .map(std::string::ToString::to_string); + let role = resp + .get("role") + .and_then(|v| v.as_str()) + .map(std::string::ToString::to_string); + let text = resp + .get("text") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + let selected_text = resp + .get("selected_text") + .and_then(|v| v.as_str()) + .map(std::string::ToString::to_string); + let raw_error = resp + .get("error") + .and_then(|v| v.as_str()) + .map(std::string::ToString::to_string); + + let x = resp + .get("x") + .and_then(serde_json::Value::as_i64) + .map(|v| v as i32); + let y = resp + .get("y") + .and_then(serde_json::Value::as_i64) + .map(|v| v as i32); + let w = resp + .get("w") + .and_then(serde_json::Value::as_i64) + .map(|v| v as i32); + let h = resp + .get("h") + .and_then(serde_json::Value::as_i64) + .map(|v| v as i32); + + Ok(FocusedTextContext { + app_name, + role, + text, + selected_text, + raw_error, + bounds: match (x, y, w, h) { + (Some(x), Some(y), Some(width), Some(height)) if width > 0 && height > 0 => { + Some(ElementBounds { + x, + y, + width, + height, + }) + } + _ => None, + }, + }) +} + +/// Focus query via osascript (fallback when helper is unavailable). +/// +/// Short-circuits when `automation_state::system_events_denied()` is set +/// (the autocomplete refresh loop captured `(-1743)` from a prior +/// osascript invocation). This stops re-firing osascript — and the +/// macOS Apple Events consent popup — once we've observed the denial +/// within the current session. The flag clears on +/// `autocomplete::start_if_enabled` so a user-initiated re-engagement +/// after granting via System Settings re-probes naturally. +#[cfg(target_os = "macos")] +// One linear AppleScript round-trip with its error mapping; splitting it would +// only scatter the shared context. +#[allow(clippy::too_many_lines)] +fn focused_text_via_osascript() -> Result { + if super::automation_state::system_events_denied() { + return Err( + "focused_text_via_osascript skipped: System Events automation previously denied (-1743)" + .to_string(), + ); + } + + let script = r##" + tell application "System Events" + set sep to character id 31 + set frontApp to first application process whose frontmost is true + set appName to name of frontApp + set roleValue to "unknown" + set textValue to "" + set selectedValue to "" + set errValue to "" + set posX to "" + set posY to "" + set sizeW to "" + set sizeH to "" + set targetRoles to {"AXTextArea", "AXTextField", "AXSearchField", "AXComboBox", "AXEditableText"} + + try + set value of attribute "AXEnhancedUserInterface" of frontApp to true + end try + + try + set focusedElement to value of attribute "AXFocusedUIElement" of frontApp + try + set roleValue to value of attribute "AXRole" of focusedElement as text + end try + try + set textValue to value of attribute "AXValue" of focusedElement as text + end try + try + set p to value of attribute "AXPosition" of focusedElement + set posX to item 1 of p as text + set posY to item 2 of p as text + end try + try + set s to value of attribute "AXSize" of focusedElement + set sizeW to item 1 of s as text + set sizeH to item 2 of s as text + end try + if textValue is "missing value" then set textValue to "" + if textValue is "" then + try + set selectedValue to value of attribute "AXSelectedText" of focusedElement as text + end try + if selectedValue is "missing value" then set selectedValue to "" + if selectedValue is not "" then set textValue to selectedValue + end if + if textValue is "" then + try + set textValue to value of attribute "AXTitle" of focusedElement as text + end try + if textValue is "missing value" then set textValue to "" + end if + on error errMsg number errNum + set errValue to "ERROR:" & errNum & ":" & errMsg + end try + + if textValue is "" then + try + set focusedWindow to value of attribute "AXFocusedWindow" of frontApp + set childElems to entire contents of focusedWindow + set staticPromptValue to "" + set staticFallbackValue to "" + repeat with childElem in childElems + set childRole to "" + set childValue to "" + set childSelectedValue to "" + try + set childRole to value of attribute "AXRole" of childElem as text + end try + if childRole is in targetRoles then + try + set childValue to value of attribute "AXValue" of childElem as text + end try + set childPosX to "" + set childPosY to "" + set childSizeW to "" + set childSizeH to "" + try + set cp to value of attribute "AXPosition" of childElem + set childPosX to item 1 of cp as text + set childPosY to item 2 of cp as text + end try + try + set cs to value of attribute "AXSize" of childElem + set childSizeW to item 1 of cs as text + set childSizeH to item 2 of cs as text + end try + if childValue is "missing value" then set childValue to "" + if childValue is "" then + try + set childSelectedValue to value of attribute "AXSelectedText" of childElem as text + end try + if childSelectedValue is "missing value" then set childSelectedValue to "" + if childSelectedValue is not "" then set childValue to childSelectedValue + end if + if childValue is not "" then + set roleValue to childRole + set textValue to childValue + if childPosX is not "" then set posX to childPosX + if childPosY is not "" then set posY to childPosY + if childSizeW is not "" then set sizeW to childSizeW + if childSizeH is not "" then set sizeH to childSizeH + exit repeat + end if + end if + end repeat + if textValue is "" then + repeat with childElem in childElems + set childRole to "" + set childValue to "" + try + set childRole to value of attribute "AXRole" of childElem as text + end try + if childRole is "AXStaticText" then + try + set childValue to value of attribute "AXValue" of childElem as text + end try + if childValue is "missing value" then set childValue to "" + if childValue is not "" then + set staticFallbackValue to childValue + if childValue contains "$ " or childValue contains "# " or childValue contains "> " then + set staticPromptValue to childValue + end if + end if + end if + end repeat + if staticPromptValue is not "" then + set roleValue to "AXStaticText" + set textValue to staticPromptValue + else if staticFallbackValue is not "" then + set roleValue to "AXStaticText" + set textValue to staticFallbackValue + end if + end if + on error errMsg2 number errNum2 + if errValue is "" then set errValue to "ERROR:" & errNum2 & ":" & errMsg2 + end try + end if + + if textValue is "" and errValue is "" then + set errValue to "ERROR:no_text_candidate_found" + end if + + return appName & sep & roleValue & sep & textValue & sep & selectedValue & sep & errValue & sep & posX & sep & posY & sep & sizeW & sep & sizeH + end tell + "##; + + let mut command = Command::new("osascript"); + command.arg("-e").arg(script); + let output = command_output_with_timeout( + "osascript focused_text_via_osascript", + &mut command, + FOCUS_COMMAND_TIMEOUT, + )?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.is_empty() { + return Err("unable to query focused text context".to_string()); + } + return Err(format!("unable to query focused text context: {stderr}")); + } + + let text = String::from_utf8_lossy(&output.stdout); + let trimmed = text.trim_end_matches(['\r', '\n']); + let mut segments = trimmed.splitn(9, '\u{1f}'); + let app_name = segments + .next() + .map(|s| normalize_ax_value(s.trim())) + .filter(|s| !s.is_empty()); + let role = segments + .next() + .map(|s| normalize_ax_value(s.trim())) + .filter(|s| !s.is_empty()); + let mut value = segments.next().map(normalize_ax_value).unwrap_or_default(); + let mut selected_text = segments + .next() + .map(normalize_ax_value) + .filter(|s| !s.is_empty()); + let mut raw_error = segments + .next() + .map(|s| normalize_ax_value(s.trim())) + .filter(|s| !s.is_empty()); + let pos_x = segments.next().and_then(parse_ax_number); + let pos_y = segments.next().and_then(parse_ax_number); + let size_w = segments.next().and_then(parse_ax_number); + let size_h = segments.next().and_then(parse_ax_number); + + let allow_terminal_text_value = + is_terminal_app(app_name.as_deref()) && !value.trim().is_empty(); + if !is_text_role(role.as_deref()) && !allow_terminal_text_value { + value.clear(); + selected_text = None; + if raw_error.is_none() { + raw_error = Some("ERROR:no_text_candidate_found".to_string()); + } + } + + Ok(FocusedTextContext { + app_name, + role, + text: value, + selected_text, + raw_error, + bounds: match (pos_x, pos_y, size_w, size_h) { + (Some(x), Some(y), Some(width), Some(height)) if width > 0 && height > 0 => { + Some(ElementBounds { + x, + y, + width, + height, + }) + } + _ => None, + }, + }) +} + +#[cfg(not(target_os = "macos"))] +/// Query the OS for the focused text element, without verbose diagnostics. +/// +/// # Errors +/// +/// Returns a message when the focus query fails or is unsupported on this platform. +pub fn focused_text_context() -> AccessibilityResult { + Err(Error::UnsupportedPlatform) +} + +#[cfg(not(target_os = "macos"))] +/// Query the OS for the focused text element, keeping raw diagnostics. +/// +/// # Errors +/// +/// Returns a message when the focus query fails or is unsupported on this platform. +pub fn focused_text_context_verbose() -> AccessibilityResult { + Err(Error::UnsupportedPlatform) +} + +// --------------------------------------------------------------------------- +// Focus target validation +// --------------------------------------------------------------------------- + +#[cfg(target_os = "macos")] +fn is_text_editable_role(role: &str) -> bool { + matches!(role, "AXTextArea" | "AXTextField") +} + +#[cfg(target_os = "macos")] +/// Validate that the currently focused element still matches the target the +/// caller captured (`expected_app`, `expected_role`, and `expected_bounds` when given). +/// +/// Inconclusive checks pass: no expected app, a failed focus query, or an +/// unsupported platform. +/// +/// # Errors +/// +/// Returns a message when focus moved to a different application, or to a +/// role that is not an interchangeable text-editable role. +pub fn validate_focused_target( + expected_app: Option<&str>, + expected_role: Option<&str>, + expected_bounds: Option, +) -> AccessibilityResult<()> { + if expected_app.is_none() && expected_role.is_none() && expected_bounds.is_none() { + return Ok(()); + } + let current = focused_text_context_verbose(); + match current { + Ok(ctx) => { + if let (Some(expected), Some(actual)) = (expected_app, ctx.app_name.as_deref()) + && expected.to_lowercase() != actual.to_lowercase() + { + return Err(Error::FocusChanged { + expected: expected.to_string(), + actual: actual.to_string(), + }); + } + if let (Some(expected), Some(actual)) = (expected_role, ctx.role.as_deref()) + && expected != actual + { + if is_text_editable_role(expected) && is_text_editable_role(actual) { + log::debug!( + "[accessibility] validate_focused_target: role changed '{expected}' -> '{actual}'; proceeding" + ); + } else { + return Err(Error::FocusRoleChanged { + expected: expected.to_string(), + actual: actual.to_string(), + }); + } + } + if let (Some(expected), Some(actual)) = (expected_bounds, ctx.bounds) + && expected.x == actual.x + && expected.y == actual.y + && expected.width == actual.width + && expected.height == actual.height + { + // The captured and current element occupy the same bounds. + } else if expected_bounds.is_some() { + return Err(Error::FocusTargetChanged); + } + Ok(()) + } + Err(_) => Ok(()), + } +} + +#[cfg(not(target_os = "macos"))] +/// Validate that the currently focused element still matches the target the +/// caller captured (`expected_app`, and `expected_role` when given). +/// +/// Inconclusive checks pass: no expected app, a failed focus query, or an +/// unsupported platform. +/// +/// # Errors +/// +/// Returns a message when focus moved to a different application, or to a +/// role that is not an interchangeable text-editable role. +pub fn validate_focused_target( + _expected_app: Option<&str>, + _expected_role: Option<&str>, + _expected_bounds: Option, +) -> AccessibilityResult<()> { + Ok(()) +} + +#[cfg(test)] +#[path = "focus_tests.rs"] +mod tests; diff --git a/crates/tinycomputer-accessibility/src/focus_tests.rs b/crates/tinycomputer-accessibility/src/focus_tests.rs new file mode 100644 index 00000000..c5ec7880 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/focus_tests.rs @@ -0,0 +1,56 @@ +//! Unit tests for the focus module's bounded command runner. + +#![allow(clippy::unwrap_used, clippy::expect_used)] +#[cfg(unix)] +use super::*; + +#[cfg(unix)] +#[test] +fn command_output_with_timeout_returns_output_for_fast_command() { + let mut command = Command::new("sh"); + command.arg("-c").arg("printf ready"); + + let output = + command_output_with_timeout("test fast command", &mut command, Duration::from_secs(1)) + .expect("fast command should complete"); + + assert!(output.status.success()); + assert_eq!(String::from_utf8_lossy(&output.stdout), "ready"); +} + +#[cfg(unix)] +#[test] +fn command_output_with_timeout_drains_large_output_while_waiting() { + let mut command = Command::new("sh"); + command.arg("-c").arg("head -c 200000 /dev/zero"); + + let output = command_output_with_timeout( + "test large output command", + &mut command, + Duration::from_secs(2), + ) + .expect("large output should not fill the pipe and block the child"); + + assert!(output.status.success()); + assert_eq!(output.stdout.len(), 200_000); +} + +#[cfg(unix)] +#[test] +fn command_output_with_timeout_kills_slow_command() { + let mut command = Command::new("sh"); + command.arg("-c").arg("sleep 2; printf late"); + + let error = + command_output_with_timeout("test slow command", &mut command, Duration::from_millis(50)) + .expect_err("slow command should time out"); + + assert!(error.contains("timed out after")); +} + +#[cfg(not(target_os = "macos"))] +#[test] +fn public_focus_query_returns_typed_unsupported_error() { + let error = super::focused_text_context().expect_err("focus querying is macOS-only"); + assert!(matches!(error, super::Error::UnsupportedPlatform)); +} diff --git a/crates/tinycomputer-accessibility/src/globe.rs b/crates/tinycomputer-accessibility/src/globe.rs new file mode 100644 index 00000000..4aaacea9 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/globe.rs @@ -0,0 +1,515 @@ +//! macOS Globe/Fn key listener helper management. +//! +//! The listener runs as a tiny Swift process that monitors `flagsChanged` +//! events globally and reports `FN_DOWN` / `FN_UP` lines over stdout. + +#[cfg(target_os = "macos")] +use super::Error; +use super::{PermissionState, Result as AccessibilityResult}; +#[cfg(any(target_os = "macos", test))] +use std::collections::VecDeque; + +#[cfg(target_os = "macos")] +use std::fs; +#[cfg(target_os = "macos")] +use std::hash::{Hash, Hasher}; +#[cfg(target_os = "macos")] +use std::io::{BufRead, BufReader}; +#[cfg(target_os = "macos")] +use std::path::PathBuf; +#[cfg(target_os = "macos")] +use std::process::{Child, Command, Stdio}; +#[cfg(target_os = "macos")] +use std::sync::LazyLock; +#[cfg(target_os = "macos")] +use std::sync::{Arc, Mutex as StdMutex}; + +fn input_monitoring_permission() -> PermissionState { + #[cfg(target_os = "macos")] + { + super::detect_input_monitoring_permission() + } + #[cfg(not(target_os = "macos"))] + { + PermissionState::Unsupported + } +} + +#[cfg(target_os = "macos")] +const LOG_PREFIX: &str = "[globe_hotkey]"; +#[cfg(any(target_os = "macos", test))] +const MAX_PENDING_EVENTS: usize = 64; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +/// State of the Globe/Fn key listener. +pub struct GlobeHotkeyStatus { + /// Whether this platform supports the listener. + pub supported: bool, + /// Whether the listener process is alive. + pub running: bool, + /// Input Monitoring state the listener needs. + pub input_monitoring_permission: PermissionState, + /// Most recent listener error, if any. + pub last_error: Option, + /// Events queued and not yet polled. + pub events_pending: usize, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +/// Listener status plus the key events drained by one poll. +pub struct GlobeHotkeyPollResult { + /// Listener status at poll time. + pub status: GlobeHotkeyStatus, + /// Drained events, oldest first (`FN_DOWN` / `FN_UP`). + pub events: Vec, +} + +#[cfg(target_os = "macos")] +struct GlobeListenerProcess { + child: Child, + event_queue: Arc>>, + /// Most recent listener error, if any. + last_error: Arc>>, +} + +#[cfg(target_os = "macos")] +static GLOBE_LISTENER: LazyLock>> = + LazyLock::new(|| StdMutex::new(None)); + +#[cfg(target_os = "macos")] +fn push_event(queue: &Arc>>, event: String) { + let Ok(mut guard) = queue.lock() else { + log::warn!("{LOG_PREFIX} failed to lock queue for event"); + return; + }; + guard.push_back(event); + trim_event_queue(&mut guard); +} + +#[cfg(any(target_os = "macos", test))] +fn trim_event_queue(queue: &mut VecDeque) { + while queue.len() > MAX_PENDING_EVENTS { + let _ = queue.pop_front(); + } +} + +#[cfg(target_os = "macos")] +fn set_last_error(error_store: &Arc>>, message: Option) { + let Ok(mut guard) = error_store.lock() else { + log::warn!("{LOG_PREFIX} failed to lock last_error store"); + return; + }; + *guard = message; +} + +#[cfg(target_os = "macos")] +fn drain_events(queue: &Arc>>) -> Vec { + let Ok(mut guard) = queue.lock() else { + log::warn!("{LOG_PREFIX} failed to lock queue for drain"); + return Vec::new(); + }; + guard.drain(..).collect() +} + +#[cfg(target_os = "macos")] +fn queue_len(queue: &Arc>>) -> usize { + let Ok(guard) = queue.lock() else { + return 0; + }; + guard.len() +} + +#[cfg(target_os = "macos")] +fn current_error(error_store: &Arc>>) -> Option { + let Ok(guard) = error_store.lock() else { + return Some("failed to read globe listener error state".to_string()); + }; + guard.clone() +} + +#[cfg(target_os = "macos")] +// Spawn, wire the reader threads, and record the process as one unit so the +// lock is held across the whole transition. +#[allow(clippy::too_many_lines)] +fn ensure_running_locked( + state: &mut Option, +) -> Result { + let input_monitoring_permission = input_monitoring_permission(); + if input_monitoring_permission != PermissionState::Granted { + let message = + "input monitoring permission is required for the macOS Globe/Fn listener".to_string(); + log::warn!( + "{LOG_PREFIX} start skipped: input_monitoring_permission={input_monitoring_permission:?}" + ); + if let Some(mut process) = state.take() { + set_last_error(&process.last_error, Some(message.clone())); + let _ = process.child.kill(); + let _ = process.child.wait(); + let _ = drain_events(&process.event_queue); + } + return Ok(GlobeHotkeyStatus { + supported: true, + running: false, + input_monitoring_permission, + last_error: Some(message), + events_pending: 0, + }); + } + + if let Some(process) = state.as_mut() { + match process.child.try_wait() { + Ok(None) => { + return Ok(GlobeHotkeyStatus { + supported: true, + running: true, + input_monitoring_permission, + last_error: current_error(&process.last_error), + events_pending: queue_len(&process.event_queue), + }); + } + Ok(Some(status)) => { + let message = format!("globe listener exited unexpectedly: {status}"); + log::warn!("{LOG_PREFIX} {message}"); + set_last_error(&process.last_error, Some(message)); + *state = None; + } + Err(err) => { + let message = format!("failed to inspect globe listener state: {err}"); + log::warn!("{LOG_PREFIX} {message}"); + set_last_error(&process.last_error, Some(message)); + let _ = process.child.kill(); + let _ = process.child.wait(); + *state = None; + } + } + } + + let binary_path = ensure_globe_helper_binary()?; + log::info!("{LOG_PREFIX} starting helper {}", binary_path.display()); + let mut child = Command::new(&binary_path) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("failed to spawn globe listener helper: {e}"))?; + + let stdout = child + .stdout + .take() + .ok_or_else(|| "failed to capture globe listener stdout".to_string())?; + let stderr = child + .stderr + .take() + .ok_or_else(|| "failed to capture globe listener stderr".to_string())?; + + let event_queue = Arc::new(StdMutex::new(VecDeque::with_capacity(MAX_PENDING_EVENTS))); + let last_error = Arc::new(StdMutex::new(None)); + + { + let queue = event_queue.clone(); + let error_store = last_error.clone(); + std::thread::spawn(move || { + let reader = BufReader::new(stdout); + for line in reader.lines() { + match line { + Ok(line) => { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + log::debug!("{LOG_PREFIX} helper event={trimmed}"); + push_event(&queue, trimmed.to_string()); + set_last_error(&error_store, None); + } + Err(err) => { + let message = format!("failed reading globe listener stdout: {err}"); + log::warn!("{LOG_PREFIX} {message}"); + set_last_error(&error_store, Some(message)); + break; + } + } + } + log::debug!("{LOG_PREFIX} stdout reader exited"); + }); + } + + { + let error_store = last_error.clone(); + std::thread::spawn(move || { + let reader = BufReader::new(stderr); + for line in reader.lines() { + match line { + Ok(line) => { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + log::warn!("{LOG_PREFIX} helper stderr={trimmed}"); + set_last_error(&error_store, Some(trimmed.to_string())); + } + Err(err) => { + let message = format!("failed reading globe listener stderr: {err}"); + log::warn!("{LOG_PREFIX} {message}"); + set_last_error(&error_store, Some(message)); + break; + } + } + } + log::debug!("{LOG_PREFIX} stderr reader exited"); + }); + } + + *state = Some(GlobeListenerProcess { + child, + event_queue, + last_error, + }); + + let process = state + .as_ref() + .ok_or_else(|| "globe listener process missing after spawn".to_string())?; + Ok(GlobeHotkeyStatus { + supported: true, + running: true, + input_monitoring_permission, + last_error: current_error(&process.last_error), + events_pending: queue_len(&process.event_queue), + }) +} + +#[cfg(target_os = "macos")] +fn ensure_globe_helper_binary() -> Result { + let cache_dir = super::helper::private_cache_dir("openhuman-globe-listener")?; + + let source = globe_swift_source(); + let mut source_hasher = std::collections::hash_map::DefaultHasher::new(); + source.hash(&mut source_hasher); + let source_id = format!("{:016x}", source_hasher.finish()); + let source_path = cache_dir.join(format!("globe_listener_{source_id}.swift")); + let binary_path = cache_dir.join(format!("globe_listener_{source_id}")); + + let needs_write = match fs::read_to_string(&source_path) { + Ok(existing) => existing != source, + Err(_) => true, + }; + if needs_write { + fs::write(&source_path, &source) + .map_err(|e| format!("failed to write globe helper source: {e}"))?; + } + + let needs_compile = needs_write || !binary_path.exists(); + if needs_compile { + let temporary_binary = cache_dir.join(format!( + "globe_listener_{source_id}.tmp-{}", + std::process::id() + )); + log::debug!("{LOG_PREFIX} compiling Swift helper"); + let output = Command::new("xcrun") + .args(["swiftc", "-O", "-framework", "Cocoa"]) + .arg(&source_path) + .arg("-o") + .arg(&temporary_binary) + .output() + .or_else(|_| { + Command::new("swiftc") + .args(["-O", "-framework", "Cocoa"]) + .arg(&source_path) + .arg("-o") + .arg(&temporary_binary) + .output() + }) + .map_err(|e| format!("failed to invoke swiftc for globe listener: {e}"))?; + if !output.status.success() { + let _ = fs::remove_file(&temporary_binary); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + return Err(format!( + "failed to compile globe listener helper: {}", + if stderr.is_empty() { + "swiftc returned non-zero exit status".to_string() + } else { + stderr + } + )); + } + fs::rename(&temporary_binary, &binary_path) + .map_err(|e| format!("failed to install compiled globe listener helper: {e}"))?; + log::debug!("{LOG_PREFIX} Swift helper compiled successfully"); + } + + Ok(binary_path) +} + +#[cfg(target_os = "macos")] +fn globe_swift_source() -> String { + r#"import Cocoa +import Darwin + +var fnIsDown = false +func emit(_ message: String) { + FileHandle.standardOutput.write((message + "\n").data(using: .utf8)!) + fflush(stdout) +} + +guard let monitor = NSEvent.addGlobalMonitorForEvents(matching: .flagsChanged, handler: { event in + let flags = event.modifierFlags + let containsFn = flags.contains(.function) + + if containsFn && !fnIsDown { + fnIsDown = true + emit("FN_DOWN") + } else if !containsFn && fnIsDown { + fnIsDown = false + emit("FN_UP") + } + +}) else { + FileHandle.standardError.write("Failed to create event monitor\n".data(using: .utf8)!) + exit(1) +} + +let signalSource = DispatchSource.makeSignalSource(signal: SIGTERM, queue: .main) +signal(SIGTERM, SIG_IGN) +signalSource.setEventHandler { + NSEvent.removeMonitor(monitor) + exit(0) +} +signalSource.resume() + +let app = NSApplication.shared +app.setActivationPolicy(.accessory) +app.run() +"# + .to_string() +} + +#[cfg(target_os = "macos")] +/// Start the Globe/Fn key listener process; unsupported platforms report `supported: false`. +/// +/// # Errors +/// +/// Returns a message when the listener helper cannot be built or started. +pub fn globe_listener_start() -> AccessibilityResult { + let mut guard = GLOBE_LISTENER + .lock() + .map_err(|_| Error::GlobeListener("globe listener lock poisoned".to_string()))?; + ensure_running_locked(&mut guard).map_err(Error::GlobeListener) +} + +#[cfg(target_os = "macos")] +/// Drain pending Globe/Fn key events and report listener status. +/// +/// # Errors +/// +/// Returns a message when the listener state cannot be read. +pub fn globe_listener_poll() -> AccessibilityResult { + let mut guard = GLOBE_LISTENER + .lock() + .map_err(|_| Error::GlobeListener("globe listener lock poisoned".to_string()))?; + let status = if guard.is_some() { + ensure_running_locked(&mut guard).map_err(Error::GlobeListener)? + } else { + GlobeHotkeyStatus { + supported: true, + running: false, + input_monitoring_permission: input_monitoring_permission(), + last_error: None, + events_pending: 0, + } + }; + let events = guard + .as_ref() + .map(|process| drain_events(&process.event_queue)) + .unwrap_or_default(); + Ok(GlobeHotkeyPollResult { + status: GlobeHotkeyStatus { + events_pending: 0, + ..status + }, + events, + }) +} + +#[cfg(target_os = "macos")] +/// Stop the Globe/Fn key listener process. +/// +/// # Errors +/// +/// Returns a message when the listener state cannot be read. +pub fn globe_listener_stop() -> AccessibilityResult { + let mut guard = GLOBE_LISTENER + .lock() + .map_err(|_| Error::GlobeListener("globe listener lock poisoned".to_string()))?; + if let Some(mut process) = guard.take() { + log::info!("{LOG_PREFIX} stopping helper pid={}", process.child.id()); + let _ = process.child.kill(); + let _ = process.child.wait(); + let events = drain_events(&process.event_queue); + log::debug!( + "{LOG_PREFIX} drained {} queued events on stop", + events.len() + ); + } + + Ok(GlobeHotkeyStatus { + supported: true, + running: false, + input_monitoring_permission: input_monitoring_permission(), + last_error: None, + events_pending: 0, + }) +} + +#[cfg(not(target_os = "macos"))] +/// Start the Globe/Fn key listener process; unsupported platforms report `supported: false`. +/// +/// # Errors +/// +/// Returns a message when the listener helper cannot be built or started. +pub fn globe_listener_start() -> AccessibilityResult { + Ok(GlobeHotkeyStatus { + supported: false, + running: false, + input_monitoring_permission: input_monitoring_permission(), + last_error: Some("Globe/Fn hotkey listener is only supported on macOS".to_string()), + events_pending: 0, + }) +} + +#[cfg(not(target_os = "macos"))] +/// Drain pending Globe/Fn key events and report listener status. +/// +/// # Errors +/// +/// Returns a message when the listener state cannot be read. +pub fn globe_listener_poll() -> AccessibilityResult { + Ok(GlobeHotkeyPollResult { + status: GlobeHotkeyStatus { + supported: false, + running: false, + input_monitoring_permission: input_monitoring_permission(), + last_error: Some("Globe/Fn hotkey listener is only supported on macOS".to_string()), + events_pending: 0, + }, + events: Vec::new(), + }) +} + +#[cfg(not(target_os = "macos"))] +/// Stop the Globe/Fn key listener process. +/// +/// # Errors +/// +/// Returns a message when the listener state cannot be read. +pub fn globe_listener_stop() -> AccessibilityResult { + Ok(GlobeHotkeyStatus { + supported: false, + running: false, + input_monitoring_permission: input_monitoring_permission(), + last_error: Some("Globe/Fn hotkey listener is only supported on macOS".to_string()), + events_pending: 0, + }) +} + +#[cfg(test)] +#[path = "globe_tests.rs"] +mod tests; diff --git a/crates/tinycomputer-accessibility/src/globe_tests.rs b/crates/tinycomputer-accessibility/src/globe_tests.rs new file mode 100644 index 00000000..3c6a9313 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/globe_tests.rs @@ -0,0 +1,45 @@ +//! Unit tests for the Globe listener's bounded event queue and platform fallbacks. + +#![allow(clippy::expect_used)] + +use super::{MAX_PENDING_EVENTS, trim_event_queue}; +use std::collections::VecDeque; + +#[test] +fn event_queue_keeps_latest_events() { + let mut queue = VecDeque::new(); + for index in 0..(MAX_PENDING_EVENTS + 5) { + queue.push_back(format!("event-{index}")); + trim_event_queue(&mut queue); + } + + assert_eq!(queue.len(), MAX_PENDING_EVENTS); + assert_eq!(queue.front().map(String::as_str), Some("event-5")); + let expected_last = format!("event-{}", MAX_PENDING_EVENTS + 4); + assert_eq!( + queue.back().map(String::as_str), + Some(expected_last.as_str()) + ); +} + +#[cfg(not(target_os = "macos"))] +#[test] +fn non_macos_listener_entry_points_report_unsupported() { + let started = super::globe_listener_start().expect("fallback start returns status"); + assert!(!started.supported); + assert!(!started.running); + assert_eq!( + started.input_monitoring_permission, + super::PermissionState::Unsupported + ); + assert_eq!(started.events_pending, 0); + + let polled = super::globe_listener_poll().expect("fallback poll returns status"); + assert!(!polled.status.supported); + assert!(!polled.status.running); + assert!(polled.events.is_empty()); + + let stopped = super::globe_listener_stop().expect("fallback stop returns status"); + assert!(!stopped.supported); + assert!(!stopped.running); +} diff --git a/crates/tinycomputer-accessibility/src/helper.rs b/crates/tinycomputer-accessibility/src/helper.rs new file mode 100644 index 00000000..5f712b4f --- /dev/null +++ b/crates/tinycomputer-accessibility/src/helper.rs @@ -0,0 +1,79 @@ +//! Unified Swift helper process: focus queries, paste, and overlay in one native binary. +//! +//! Replaces the separate osascript subprocess spawns and standalone overlay binary +//! with a single persistent Swift process communicating via stdin/stdout JSON. +//! +//! ## Mutex architecture +//! +//! Three globals prevent deadlock between fire-and-forget (show/hide) and +//! request-response (focus/paste) callers: +//! +//! - `UNIFIED_HELPER`: guards the process handle + stdin writer. +//! Held only for the brief duration of a stdin write (~μs). +//! - `RESPONSE_RX`: guards the mpsc receiver that the background reader +//! thread populates. Held only for the duration of `recv_timeout`. +//! - `RECV_SERIALISER`: held for the entire send+receive round-trip so that +//! two callers cannot interleave their reads. +//! +//! Fire-and-forget callers never touch `RESPONSE_RX` or `RECV_SERIALISER`, +//! so `show`/`hide` can proceed while a `focus` query is in-flight. +//! +//! Split by responsibility: [`process`] owns the process lifecycle and the +//! send/receive paths, and the `swift_*` modules hold the embedded Swift +//! source (assembled by [`swift_source`]) that the helper binary compiles +//! from. + +mod process; +mod swift_ax_actions; +mod swift_focus; +mod swift_overlay; +mod swift_paste; +mod swift_source; + +#[cfg(target_os = "macos")] +pub(crate) fn private_cache_dir(name: &str) -> Result { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let temp_dir = std::fs::canonicalize(std::env::temp_dir()) + .map_err(|error| format!("failed to resolve temporary directory: {error}"))?; + let temp_metadata = std::fs::symlink_metadata(&temp_dir) + .map_err(|error| format!("failed to inspect temporary directory: {error}"))?; + if !temp_metadata.is_dir() + || temp_metadata.mode() & 0o077 != 0 + || temp_metadata.mode() & 0o700 != 0o700 + { + return Err("temporary directory is not private to its owner".to_string()); + } + + let cache_dir = temp_dir.join(name); + match std::fs::create_dir(&cache_dir) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(format!("failed to create helper cache directory: {error}")), + } + + let cache_metadata = std::fs::symlink_metadata(&cache_dir) + .map_err(|error| format!("failed to inspect helper cache directory: {error}"))?; + if cache_metadata.file_type().is_symlink() + || !cache_metadata.is_dir() + || cache_metadata.uid() != temp_metadata.uid() + { + return Err("helper cache directory has unexpected ownership or type".to_string()); + } + std::fs::set_permissions(&cache_dir, std::fs::Permissions::from_mode(0o700)) + .map_err(|error| format!("failed to secure helper cache directory: {error}"))?; + + Ok(cache_dir) +} + +#[cfg(test)] +#[path = "helper_tests.rs"] +mod tests; + +#[cfg(target_os = "macos")] +#[allow(unused_imports)] +pub(crate) use process::helper_send_receive; +pub use process::precompile_helper_background; +#[cfg(target_os = "macos")] +#[allow(unused_imports)] +pub(crate) use process::{helper_quit, helper_send_fire_and_forget}; diff --git a/crates/tinycomputer-accessibility/src/helper/process.rs b/crates/tinycomputer-accessibility/src/helper/process.rs new file mode 100644 index 00000000..77caef78 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/helper/process.rs @@ -0,0 +1,399 @@ +//! Unified helper process lifecycle: spawning and supervising the persistent +//! Swift binary, the request/response and fire-and-forget send paths, and +//! compiling/caching the helper binary from its embedded Swift source. + +#[cfg(target_os = "macos")] +use std::hash::{Hash, Hasher}; +#[cfg(target_os = "macos")] +use std::io::{BufRead, BufReader, Write}; +#[cfg(target_os = "macos")] +use std::sync::LazyLock; +#[cfg(target_os = "macos")] +use std::sync::atomic::{AtomicU64, Ordering}; +#[cfg(target_os = "macos")] +use std::sync::{Mutex as StdMutex, mpsc}; +#[cfg(target_os = "macos")] +use std::time::{Duration, Instant}; +#[cfg(target_os = "macos")] +use std::{ + fs, + path::PathBuf, + process::{Child, ChildStdin, Command, Stdio}, +}; + +#[cfg(target_os = "macos")] +use super::swift_source::unified_swift_source; +#[cfg(target_os = "macos")] +use serde_json::Value; + +/// Process handle + stdin writer. Held only briefly for writes. +#[cfg(target_os = "macos")] +struct UnifiedHelperProcess { + child: Child, + stdin: ChildStdin, +} + +/// Guards the process handle and stdin. +#[cfg(target_os = "macos")] +static UNIFIED_HELPER: LazyLock>> = + LazyLock::new(|| StdMutex::new(None)); + +/// Channel receiver fed by the background stdout-reader thread. +/// Separate from `UNIFIED_HELPER` so fire-and-forget callers never contend here. +#[cfg(target_os = "macos")] +static RESPONSE_RX: LazyLock>>> = + LazyLock::new(|| StdMutex::new(None)); + +/// Serialises request/response pairs so two callers cannot interleave reads. +/// Fire-and-forget callers never acquire this lock. +#[cfg(target_os = "macos")] +static RECV_SERIALISER: LazyLock> = LazyLock::new(|| StdMutex::new(())); + +/// Prevents concurrent Swift compiles from `ensure_helper_binary` vs background precompile. +#[cfg(target_os = "macos")] +static HELPER_COMPILE_LOCK: LazyLock> = LazyLock::new(|| StdMutex::new(())); + +/// Monotonic ids for `helper_send_receive` so a late line cannot be consumed as the wrong reply. +#[cfg(target_os = "macos")] +static HELPER_REQ_ID: AtomicU64 = AtomicU64::new(1); + +/// Timeout for a single request/response round-trip with the Swift helper. +#[cfg(target_os = "macos")] +const HELPER_RECV_TIMEOUT: Duration = Duration::from_secs(8); + +/// Send a JSON request and read a JSON response (one line each). +/// Used for `focus` and `paste` commands that produce a response. +/// +/// Holds `RECV_SERIALISER` for the full round-trip, but releases +/// `UNIFIED_HELPER` before blocking on the channel recv, so fire-and-forget +/// callers (`show`/`hide`) are never blocked by an in-flight focus query. +#[cfg(target_os = "macos")] +pub(crate) fn helper_send_receive( + request: &serde_json::Value, +) -> Result { + // Serialise request/response pairs — prevents interleaved reads. + let _rr_guard = RECV_SERIALISER + .lock() + .map_err(|_| "recv serialiser lock poisoned".to_string())?; + + ensure_helper_running()?; + + let id_num = HELPER_REQ_ID.fetch_add(1, Ordering::Relaxed); + let id_str = id_num.to_string(); + + let mut req = request.clone(); + let req_obj = req + .as_object_mut() + .ok_or_else(|| "helper request must be a JSON object".to_string())?; + req_obj.insert("id".to_string(), Value::String(id_str.clone())); + + // Write the request, holding UNIFIED_HELPER only for this brief write. + { + let mut guard = UNIFIED_HELPER + .lock() + .map_err(|_| "unified helper lock poisoned".to_string())?; + let helper = guard + .as_mut() + .ok_or_else(|| "unified helper unavailable".to_string())?; + let line = req.to_string(); + helper + .stdin + .write_all(line.as_bytes()) + .and_then(|()| helper.stdin.write_all(b"\n")) + .and_then(|()| helper.stdin.flush()) + .map_err(|e| format!("failed to write to helper stdin: {e}"))?; + } // UNIFIED_HELPER released here — fire-and-forget callers can proceed + + // Read until the line matches `id` (discards stale lines after a timeout or reordering). + let deadline = Instant::now() + HELPER_RECV_TIMEOUT; + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + reset_helper_process(); + return Err(format!( + "helper response timed out waiting for id {id_str} (stale lines may follow)" + )); + } + let chunk = remaining.min(Duration::from_millis(500)); + let response_line = { + let rx_guard = RESPONSE_RX + .lock() + .map_err(|_| "response rx lock poisoned".to_string())?; + let rx = rx_guard + .as_ref() + .ok_or_else(|| "response channel unavailable".to_string())?; + match rx.recv_timeout(chunk) { + Ok(line) => line, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + // Non-fatal: the outer loop will check `remaining` and + // either retry or surface the top-level timeout. + continue; + } + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + return Err("helper response channel disconnected".to_string()); + } + } + }; + + if response_line.trim().is_empty() { + log::debug!( + "[accessibility] helper skipped empty stdout line while waiting for id {id_str}" + ); + continue; + } + + let value: Value = serde_json::from_str(response_line.trim()) + .map_err(|e| format!("failed to parse helper response: {e}"))?; + + let matches = value + .get("id") + .and_then(|v| v.as_str()) + .is_some_and(|rid| rid == id_str.as_str()); + if matches { + return Ok(value); + } + log::debug!( + "[accessibility] discarding helper response with mismatched or missing id (want id={id_str})" + ); + } +} + +#[cfg(target_os = "macos")] +fn reset_helper_process() { + if let Ok(mut guard) = UNIFIED_HELPER.lock() { + if let Some(mut helper) = guard.take() { + let _ = helper.child.kill(); + let _ = helper.child.wait(); + } + if let Ok(mut rx_guard) = RESPONSE_RX.lock() { + rx_guard.take(); + } + } +} + +/// Send a JSON request without waiting for a response. +/// Used for `show`, `hide`, and `quit` commands. +/// Only acquires `UNIFIED_HELPER` (for the stdin write) — never blocks on I/O. +#[cfg(target_os = "macos")] +// Part of the helper protocol (`show`/`hide`/`quit`); the overlay is not driven +// from this crate yet. +#[allow(dead_code)] +pub(crate) fn helper_send_fire_and_forget(request: &serde_json::Value) -> Result<(), String> { + ensure_helper_running()?; + let mut guard = UNIFIED_HELPER + .lock() + .map_err(|_| "unified helper lock poisoned".to_string())?; + let helper = guard + .as_mut() + .ok_or_else(|| "unified helper unavailable".to_string())?; + + let line = request.to_string(); + helper + .stdin + .write_all(line.as_bytes()) + .and_then(|()| helper.stdin.write_all(b"\n")) + .and_then(|()| helper.stdin.flush()) + .map_err(|e| format!("failed to write to helper stdin: {e}"))?; + Ok(()) +} + +/// Quit and clean up the helper process. +#[cfg(target_os = "macos")] +#[allow(dead_code)] +pub(crate) fn helper_quit() -> Result<(), String> { + // Drop the response channel first so the reader thread exits cleanly. + { + let mut rx_guard = RESPONSE_RX + .lock() + .map_err(|_| "response rx lock poisoned".to_string())?; + rx_guard.take(); + } + let mut guard = UNIFIED_HELPER + .lock() + .map_err(|_| "unified helper lock poisoned".to_string())?; + if let Some(mut helper) = guard.take() { + let _ = helper.stdin.write_all(br#"{"type":"quit"}"#); + let _ = helper.stdin.write_all(b"\n"); + let _ = helper.stdin.flush(); + let _ = helper.child.kill(); + let _ = helper.child.wait(); + } + Ok(()) +} + +/// Ensure the helper process is running. Spawns it (and the stdout reader +/// thread) if not yet started or if it has exited unexpectedly. +#[cfg(target_os = "macos")] +fn ensure_helper_running() -> Result<(), String> { + let mut guard = UNIFIED_HELPER + .lock() + .map_err(|_| "unified helper lock poisoned".to_string())?; + + if let Some(helper) = guard.as_mut() { + if helper + .child + .try_wait() + .map_err(|e| format!("failed to query helper state: {e}"))? + .is_none() + { + return Ok(()); // Still running + } + log::debug!("[accessibility] unified helper exited, restarting"); + *guard = None; + // Also drop the stale receiver so a new one will be created below. + if let Ok(mut rx_guard) = RESPONSE_RX.lock() { + rx_guard.take(); + } + } + + let binary_path = ensure_helper_binary()?; + let mut child = Command::new(&binary_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| format!("failed to spawn unified helper: {e}"))?; + + let stdin = child + .stdin + .take() + .ok_or_else(|| "failed to capture helper stdin".to_string())?; + let stdout = child + .stdout + .take() + .ok_or_else(|| "failed to capture helper stdout".to_string())?; + + // Spawn a background thread that continuously reads lines from the helper's + // stdout and forwards them into the channel. The thread exits when the + // sender is dropped (i.e. when helper_quit drops RESPONSE_RX) or when the + // process closes its stdout. + let (tx, rx) = mpsc::channel::(); + std::thread::spawn(move || { + let mut reader = BufReader::new(stdout); + let mut line = String::new(); + loop { + line.clear(); + match reader.read_line(&mut line) { + Ok(0) => break, // EOF — helper exited + Ok(_) => { + let trimmed = line.trim().to_string(); + if !trimmed.is_empty() && tx.send(trimmed).is_err() { + break; // Receiver dropped — time to exit + } + } + Err(e) => { + log::debug!("[accessibility] helper stdout reader error: {e}"); + break; + } + } + } + log::debug!("[accessibility] helper stdout reader thread exiting"); + }); + + // Store the new receiver. + if let Ok(mut rx_guard) = RESPONSE_RX.lock() { + *rx_guard = Some(rx); + } + + *guard = Some(UnifiedHelperProcess { child, stdin }); + log::debug!("[accessibility] unified helper started"); + Ok(()) +} + +/// Compile the Swift helper binary in the background so the first overlay +/// request does not incur the compile latency. Safe to call multiple times; +/// subsequent calls are no-ops (the binary is cached by `ensure_helper_binary`). +#[cfg(target_os = "macos")] +pub fn precompile_helper_background() { + std::thread::spawn(|| { + log::debug!("[accessibility] precompile_helper_background: starting"); + match ensure_helper_binary() { + Ok(path) => log::debug!("[accessibility] helper binary ready: {}", path.display()), + Err(e) => log::warn!( + "[accessibility] helper precompile failed (will retry on first use): {e}" + ), + } + }); +} + +/// No-op on non-macOS platforms. +#[cfg(not(target_os = "macos"))] +pub fn precompile_helper_background() {} + +#[cfg(target_os = "macos")] +fn ensure_helper_binary() -> Result { + let _compile_guard = HELPER_COMPILE_LOCK + .lock() + .map_err(|_| "helper compile lock poisoned".to_string())?; + + let cache_dir = super::private_cache_dir("openhuman-accessibility-helper")?; + let source = unified_swift_source(); + let mut source_hasher = std::collections::hash_map::DefaultHasher::new(); + source.hash(&mut source_hasher); + let source_id = format!("{:016x}", source_hasher.finish()); + let source_path = cache_dir.join(format!("unified_helper_{source_id}.swift")); + let binary_path = cache_dir.join(format!("unified_helper_{source_id}")); + + let needs_write = match fs::read_to_string(&source_path) { + Ok(existing) => existing != source, + Err(_) => true, + }; + if needs_write { + fs::write(&source_path, &source) + .map_err(|e| format!("failed to write helper source: {e}"))?; + } + + let needs_compile = needs_write || !binary_path.exists(); + if needs_compile { + let temporary_binary = cache_dir.join(format!( + "unified_helper_{source_id}.tmp-{}", + std::process::id() + )); + log::debug!("[accessibility] compiling unified Swift helper"); + let output = Command::new("xcrun") + .args([ + "swiftc", + "-O", + "-framework", + "Cocoa", + "-framework", + "ApplicationServices", + ]) + .arg(&source_path) + .arg("-o") + .arg(&temporary_binary) + .output() + .or_else(|_| { + Command::new("swiftc") + .args([ + "-O", + "-framework", + "Cocoa", + "-framework", + "ApplicationServices", + ]) + .arg(&source_path) + .arg("-o") + .arg(&temporary_binary) + .output() + }) + .map_err(|e| format!("failed to invoke swiftc: {e}"))?; + if !output.status.success() { + let _ = fs::remove_file(&temporary_binary); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + return Err(format!( + "failed to compile unified helper: {}", + if stderr.is_empty() { + "swiftc returned non-zero exit status".to_string() + } else { + stderr + } + )); + } + fs::rename(&temporary_binary, &binary_path) + .map_err(|e| format!("failed to install compiled unified helper: {e}"))?; + log::debug!("[accessibility] unified helper compiled successfully"); + } + + Ok(binary_path) +} diff --git a/crates/tinycomputer-accessibility/src/helper/swift_ax_actions.rs b/crates/tinycomputer-accessibility/src/helper/swift_ax_actions.rs new file mode 100644 index 00000000..31b8766d --- /dev/null +++ b/crates/tinycomputer-accessibility/src/helper/swift_ax_actions.rs @@ -0,0 +1,148 @@ +//! Embedded Swift source fragment: `AXUIElement` app interaction — finding a +//! running app, walking its AX tree, listing elements, pressing controls, and +//! setting text-field values. +#[cfg(target_os = "macos")] +pub(super) const SWIFT_AX_ACTIONS: &str = r#"// MARK: - AXUIElement App Interaction + +/// Find a running application by display name (exact, prefix, or contains match). +func findRunningApp(named appName: String) -> NSRunningApplication? { + let apps = NSWorkspace.shared.runningApplications + let lower = appName.lowercased() + if let app = apps.first(where: { $0.localizedName?.lowercased() == lower }) { return app } + if let app = apps.first(where: { $0.localizedName?.lowercased().hasPrefix(lower) ?? false }) { return app } + return apps.first(where: { $0.localizedName?.lowercased().contains(lower) ?? false }) +} + +/// Walk the AX element tree depth-first. Visitor returns true to stop early. +func axWalk(_ element: AXUIElement, depth: Int = 0, maxDepth: Int = 12, + visitor: (AXUIElement, String, String) -> Bool) -> Bool { + if depth > maxDepth { return false } + var roleRef: AnyObject? + AXUIElementCopyAttributeValue(element, kAXRoleAttribute as CFString, &roleRef) + let role = (roleRef as? String) ?? "" + var labelRef: AnyObject? + AXUIElementCopyAttributeValue(element, kAXTitleAttribute as CFString, &labelRef) + var label = (labelRef as? String) ?? "" + if label.isEmpty { + var descRef: AnyObject? + AXUIElementCopyAttributeValue(element, kAXDescriptionAttribute as CFString, &descRef) + label = (descRef as? String) ?? "" + } + if visitor(element, role, label) { return true } + var childrenRef: AnyObject? + guard AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &childrenRef) == .success, + let children = childrenRef as? [AXUIElement] else { return false } + for child in children { + if axWalk(child, depth: depth + 1, maxDepth: maxDepth, visitor: visitor) { return true } + } + return false +} + +/// List interactive UI elements in the named app. +func axListElements(appName: String, id: String?) -> [String: Any] { + guard let app = findRunningApp(named: appName) else { + return ["type": "ax_list", "id": id ?? "", "ok": false, + "error": "App '\(appName)' not found or not running", "elements": [] as [Any]] + } + let axApp = AXUIElementCreateApplication(app.processIdentifier) + let interactiveRoles: Set = [ + "AXButton", "AXMenuItem", "AXMenuBarItem", "AXTextField", "AXTextArea", + "AXCheckBox", "AXRadioButton", "AXSlider", "AXPopUpButton", + "AXComboBox", "AXLink", "AXTab" + ] + var elements: [[String: Any]] = [] + axWalk(axApp, maxDepth: 10) { el, role, label in + if interactiveRoles.contains(role) && !label.isEmpty { + elements.append(["role": role, "label": label, "enabled": axEnabled(el)]) + } + return false + } + return ["type": "ax_list", "id": id ?? "", "ok": true, "error": NSNull(), "elements": elements] +} + +/// Read the AXEnabled attribute; default to `true` when the attribute is absent +/// (most static/text elements don't expose it, and we don't want to hide them). +func axEnabled(_ element: AXUIElement) -> Bool { + var ref: AnyObject? + if AXUIElementCopyAttributeValue(element, kAXEnabledAttribute as CFString, &ref) == .success, + let b = ref as? Bool { + return b + } + return true +} + +/// Collect all AX elements whose label contains `label` (case-insensitive). +/// Returns matches sorted exact-first so "Play" beats "Playlist". +struct AXCandidate { + var element: AXUIElement + var label: String + var exact: Bool +} + +func axCollectMatches(_ root: AXUIElement, label: String, depth: Int = 0, maxDepth: Int = 12) -> [AXCandidate] { + if depth > maxDepth { return [] } + var roleRef: AnyObject?; AXUIElementCopyAttributeValue(root, kAXRoleAttribute as CFString, &roleRef) + var titleRef: AnyObject?; AXUIElementCopyAttributeValue(root, kAXTitleAttribute as CFString, &titleRef) + var elemLabel = (titleRef as? String) ?? "" + if elemLabel.isEmpty { var d: AnyObject?; AXUIElementCopyAttributeValue(root, kAXDescriptionAttribute as CFString, &d); elemLabel = (d as? String) ?? "" } + let lower = label.lowercased(); let elLower = elemLabel.lowercased() + var results: [AXCandidate] = [] + if !elemLabel.isEmpty, elLower.contains(lower) { + results.append(AXCandidate(element: root, label: elemLabel, exact: elLower == lower)) + } + var childrenRef: AnyObject? + guard AXUIElementCopyAttributeValue(root, kAXChildrenAttribute as CFString, &childrenRef) == .success, + let children = childrenRef as? [AXUIElement] else { return results } + for child in children { results += axCollectMatches(child, label: label, depth: depth+1, maxDepth: maxDepth) } + return results +} + +/// Press a UI element by label — exact match preferred over contains match. +func axPress(appName: String, label: String, id: String?) -> [String: Any] { + guard let app = findRunningApp(named: appName) else { + return ["type": "ax_press", "id": id ?? "", "ok": false, + "error": "App '\(appName)' not found or not running"] + } + let axApp = AXUIElementCreateApplication(app.processIdentifier) + var matches = axCollectMatches(axApp, label: label) + // Exact matches first so "Play" beats "Playlist" + matches.sort { $0.exact && !$1.exact } + for match in matches { + if AXUIElementPerformAction(match.element, kAXPressAction as CFString) == .success { + return ["type": "ax_press", "id": id ?? "", "ok": true, "error": NSNull(), + "pressed": match.label] + } + } + return ["type": "ax_press", "id": id ?? "", "ok": false, + "error": "No pressable element matching '\(label)' found in '\(appName)' (\(matches.count) label match(es) not actionable)"] +} + +/// Set the value of a text field by partial label match (or first text field if label is empty). +func axSetValue(appName: String, label: String, value: String, id: String?) -> [String: Any] { + guard let app = findRunningApp(named: appName) else { + return ["type": "ax_set_value", "id": id ?? "", "ok": false, + "error": "App '\(appName)' not found or not running"] + } + let axApp = AXUIElementCreateApplication(app.processIdentifier) + let textRoles: Set = ["AXTextField", "AXTextArea", "AXSearchField", "AXComboBox"] + let lower = label.lowercased() + var setLabel = "" + let found = axWalk(axApp) { element, role, elemLabel in + guard textRoles.contains(role) else { return false } + let matchLabel = lower.isEmpty || elemLabel.lowercased().contains(lower) + guard matchLabel else { return false } + if AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, value as CFTypeRef) == .success { + setLabel = elemLabel + return true + } + return false + } + if found { + return ["type": "ax_set_value", "id": id ?? "", "ok": true, "error": NSNull(), + "field": setLabel] + } + return ["type": "ax_set_value", "id": id ?? "", "ok": false, + "error": "No text field matching '\(label)' found in '\(appName)'"] +} + +"#; diff --git a/crates/tinycomputer-accessibility/src/helper/swift_focus.rs b/crates/tinycomputer-accessibility/src/helper/swift_focus.rs new file mode 100644 index 00000000..ffadfc5d --- /dev/null +++ b/crates/tinycomputer-accessibility/src/helper/swift_focus.rs @@ -0,0 +1,223 @@ +//! Embedded Swift source fragment: imports, the thread-safe stdout writer, and +//! the accessibility focus-query logic (`queryFocusedElement` and its helpers). +#[cfg(target_os = "macos")] +pub(super) const SWIFT_HEADER_AND_FOCUS: &str = r##"import Cocoa +import Foundation +import ApplicationServices + +// MARK: - Thread-safe stdout writer + +let stdoutLock = NSLock() + +func writeResponse(_ dict: [String: Any]) { + guard let data = try? JSONSerialization.data(withJSONObject: dict), + let line = String(data: data, encoding: .utf8) else { return } + stdoutLock.lock() + print(line) + fflush(stdout) + stdoutLock.unlock() +} + +// MARK: - Accessibility Focus Query + +let textRoles: Set = ["AXTextArea", "AXTextField", "AXSearchField", "AXComboBox", "AXEditableText"] + +// Apps that need AXEnhancedUserInterface to expose focused text elements properly. +let chromiumAppPatterns = ["chrom", "electron", "code", "slack", "discord", "brave", "edge", "opera", "vivaldi", "arc"] + +func isChromiumApp(_ name: String) -> Bool { + let lower = name.lowercased() + return chromiumAppPatterns.contains(where: { lower.contains($0) }) +} + +func getAXStringAttr(_ element: AXUIElement, _ attr: String) -> String? { + var value: AnyObject? + let err = AXUIElementCopyAttributeValue(element, attr as CFString, &value) + guard err == .success, let str = value as? String, str != "missing value" else { return nil } + return str +} + +func getAXPosition(_ element: AXUIElement) -> (x: Int, y: Int)? { + var value: AnyObject? + let err = AXUIElementCopyAttributeValue(element, kAXPositionAttribute as String as CFString, &value) + guard err == .success else { return nil } + var point = CGPoint.zero + AXValueGetValue(value as! AXValue, .cgPoint, &point) + return (Int(point.x), Int(point.y)) +} + +func getAXSize(_ element: AXUIElement) -> (w: Int, h: Int)? { + var value: AnyObject? + let err = AXUIElementCopyAttributeValue(element, kAXSizeAttribute as String as CFString, &value) + guard err == .success else { return nil } + var size = CGSize.zero + AXValueGetValue(value as! AXValue, .cgSize, &size) + return (Int(size.width), Int(size.height)) +} + +func scanChildrenForText(_ parent: AXUIElement, depth: Int = 0, allowStaticFallback: Bool = false) -> (role: String, text: String, pos: (Int, Int)?, size: (Int, Int)?)? { + if depth > 5 { return nil } + var childrenRef: AnyObject? + let err = AXUIElementCopyAttributeValue(parent, kAXChildrenAttribute as String as CFString, &childrenRef) + guard err == .success, let children = childrenRef as? [AXUIElement] else { return nil } + + // First pass: look for text-role elements with content + for child in children.prefix(200) { + let role = getAXStringAttr(child, kAXRoleAttribute as String) ?? "" + if textRoles.contains(role) { + var text = getAXStringAttr(child, kAXValueAttribute as String) ?? "" + if text.isEmpty { + text = getAXStringAttr(child, kAXSelectedTextAttribute as String) ?? "" + } + if !text.isEmpty { + return (role, text, getAXPosition(child), getAXSize(child)) + } + } + } + + // Second pass: look for AXStaticText with prompt patterns (terminal support) + var staticFallback: (role: String, text: String, pos: (Int, Int)?, size: (Int, Int)?)? + for child in children.prefix(200) { + let role = getAXStringAttr(child, kAXRoleAttribute as String) ?? "" + if allowStaticFallback && role == "AXStaticText" { + let text = getAXStringAttr(child, kAXValueAttribute as String) ?? "" + if !text.isEmpty { + let isPrompt = text.contains("$ ") || text.contains("# ") || text.contains("> ") + let candidate = (role, text, getAXPosition(child), getAXSize(child)) + if isPrompt { + staticFallback = candidate + break + } + } + } + } + + // Recurse into children + for child in children.prefix(50) { + if let result = scanChildrenForText(child, depth: depth + 1, allowStaticFallback: allowStaticFallback) { + return result + } + } + return staticFallback +} + +func queryFocusedElement(id: String?) -> [String: Any] { + var result: [String: Any] = [ + "type": "focus", + "app_name": NSNull(), + "role": NSNull(), + "text": "", + "selected_text": NSNull(), + "x": NSNull(), "y": NSNull(), "w": NSNull(), "h": NSNull(), + "error": NSNull(), + "ax_trusted": AXIsProcessTrusted(), + ] + if let id = id { result["id"] = id } + + let systemWide = AXUIElementCreateSystemWide() + + // Get focused application + var appRef: AnyObject? + var appErr = AXUIElementCopyAttributeValue(systemWide, kAXFocusedApplicationAttribute as String as CFString, &appRef) + guard appErr == .success, let appElement = appRef else { + result["error"] = "ERROR:no_focused_application" + return result + } + + let appName = getAXStringAttr(appElement as! AXUIElement, kAXTitleAttribute as String) ?? "unknown" + result["app_name"] = appName + + // Enable AXEnhancedUserInterface for Chromium apps + if isChromiumApp(appName) { + AXUIElementSetAttributeValue(appElement as! AXUIElement, "AXEnhancedUserInterface" as CFString, true as CFBoolean) + } + + // Get focused element + var focusedRef: AnyObject? + let focusErr = AXUIElementCopyAttributeValue(appElement as! AXUIElement, kAXFocusedUIElementAttribute as String as CFString, &focusedRef) + + if focusErr == .success, let focused = focusedRef { + let focusedElement = focused as! AXUIElement + let role = getAXStringAttr(focusedElement, kAXRoleAttribute as String) ?? "unknown" + result["role"] = role + + var text = getAXStringAttr(focusedElement, kAXValueAttribute as String) ?? "" + let selectedText = getAXStringAttr(focusedElement, kAXSelectedTextAttribute as String) + result["selected_text"] = selectedText ?? NSNull() + + if text.isEmpty, let sel = selectedText, !sel.isEmpty { + text = sel + } + if text.isEmpty { + text = getAXStringAttr(focusedElement, kAXTitleAttribute as String) ?? "" + } + + if let pos = getAXPosition(focusedElement) { + result["x"] = pos.x + result["y"] = pos.y + } + if let size = getAXSize(focusedElement) { + result["w"] = size.w + result["h"] = size.h + } + + // If we got text from a text-role element, we're done + if !text.isEmpty && textRoles.contains(role) { + result["text"] = text + return result + } + + // If role is not a text role, still return text if it looks terminal-like + let terminalApps = ["terminal", "iterm2", "wezterm", "warp", "alacritty", "kitty", "ghostty", "hyper", "rio", "tabby", "wave", "contour", "foot"] + let normalizedAppName = appName.lowercased().filter { $0.isLetter || $0.isNumber } + let isTerminal = terminalApps.contains(normalizedAppName) + let isTerminalPrompt = role == "AXStaticText" && + (text.contains("$ ") || text.contains("# ") || text.contains("❯") || text.contains("➜") || text.contains("λ")) + if isTerminal && !text.isEmpty && (textRoles.contains(role) || isTerminalPrompt) { + result["text"] = text + return result + } + + // Text is empty or not from a text role — scan window children + if text.isEmpty || !textRoles.contains(role) { + // Try scanning focused window's children + var windowRef: AnyObject? + let winErr = AXUIElementCopyAttributeValue(appElement as! AXUIElement, kAXFocusedWindowAttribute as String as CFString, &windowRef) + if winErr == .success, let window = windowRef { + if let found = scanChildrenForText(window as! AXUIElement, allowStaticFallback: isTerminal) { + result["role"] = found.role + result["text"] = found.text + if let pos = found.pos { result["x"] = pos.0; result["y"] = pos.1 } + if let size = found.size { result["w"] = size.0; result["h"] = size.1 } + return result + } + } + + if text.isEmpty { + result["error"] = "ERROR:no_text_candidate_found" + } else { + result["error"] = "ERROR:no_text_candidate_found" + } + } else { + result["text"] = text + } + } else { + // No focused element found — try window scanning + var windowRef: AnyObject? + let winErr = AXUIElementCopyAttributeValue(appElement as! AXUIElement, kAXFocusedWindowAttribute as String as CFString, &windowRef) + if winErr == .success, let window = windowRef { + if let found = scanChildrenForText(window as! AXUIElement, allowStaticFallback: isTerminal) { + result["role"] = found.role + result["text"] = found.text + if let pos = found.pos { result["x"] = pos.0; result["y"] = pos.1 } + if let size = found.size { result["w"] = size.0; result["h"] = size.1 } + return result + } + } + result["error"] = "ERROR:-1728:no_focused_element" + } + + return result +} + +"##; diff --git a/crates/tinycomputer-accessibility/src/helper/swift_overlay.rs b/crates/tinycomputer-accessibility/src/helper/swift_overlay.rs new file mode 100644 index 00000000..c3624940 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/helper/swift_overlay.rs @@ -0,0 +1,215 @@ +//! Embedded Swift source fragment: the on-screen overlay controller and the +//! process's stdin-driven main dispatch loop. +#[cfg(target_os = "macos")] +pub(super) const SWIFT_OVERLAY_AND_MAIN: &str = r#"// MARK: - Overlay Controller + +final class OverlayController { + private var panel: NSPanel? + private var textField: NSTextField? + private var hintField: NSTextField? + private var hideWorkItem: DispatchWorkItem? + + func show(x: CGFloat, yTop: CGFloat, width: CGFloat, height: CGFloat, text: String, ttlMs: Int, tabHint: String) { + let showTabHint = !tabHint.isEmpty + // Detect current system appearance for contrast-appropriate colors. + let isDark: Bool = { + if #available(macOS 10.14, *) { + return NSApp.effectiveAppearance + .bestMatch(from: [.darkAqua, .aqua]) == .darkAqua + } + return false + }() + let bgColor = isDark + ? NSColor(white: 0.92, alpha: 0.82) // light badge on dark background + : NSColor(white: 0.10, alpha: 0.82) // dark badge on light background + let textColor = isDark + ? NSColor(white: 0.08, alpha: 0.95) + : NSColor(white: 1.0, alpha: 0.95) + + // Measure badge width from actual text metrics instead of char-count estimate. + let font = NSFont.systemFont(ofSize: 13) + let attrs: [NSAttributedString.Key: Any] = [.font: font] + let measured = (text as NSString).size(withAttributes: attrs) + let hintPad: CGFloat = showTabHint ? 80 : 16 + let panelWidth = min(480, max(140, ceil(measured.width) + hintPad)) + let panelHeight: CGFloat = 28 + + // Multi-monitor: find the screen containing the target or mouse cursor. + let screen: NSScreen? = { + let mainHeight = NSScreen.screens.first?.frame.height ?? 900 + if width > 0 && height > 0 { + let cocoaPoint = NSPoint(x: x + width / 2, y: mainHeight - (yTop + height / 2)) + if let s = NSScreen.screens.first(where: { $0.frame.contains(cocoaPoint) }) { + return s + } + } + let mouseLocation = NSEvent.mouseLocation + if let s = NSScreen.screens.first(where: { $0.frame.contains(mouseLocation) }) { + return s + } + return NSScreen.main ?? NSScreen.screens.first + }() + let screenFrame = screen?.frame ?? NSRect(x: 0, y: 0, width: 1440, height: 900) + let screenHeight = screenFrame.height + screenFrame.origin.y + + var originX: CGFloat + var originYCocoa: CGFloat + + if width > 0 && height > 0 { + originX = x + max(8, min(width - panelWidth - 8, 28)) + let originYTop = yTop + max(5, min(height - panelHeight - 4, 10)) + originYCocoa = max(6, screenHeight - originYTop - panelHeight) + } else { + let mouseLocation = NSEvent.mouseLocation + originX = mouseLocation.x + 8 + originYCocoa = mouseLocation.y - panelHeight - 8 + } + + // Clamp to screen bounds + originX = max(screenFrame.origin.x + 4, min(originX, screenFrame.origin.x + screenFrame.width - panelWidth - 4)) + originYCocoa = max(screenFrame.origin.y + 4, min(originYCocoa, screenFrame.origin.y + screenFrame.height - panelHeight - 4)) + + if panel == nil { + let p = NSPanel( + contentRect: NSRect(x: originX, y: originYCocoa, width: panelWidth, height: panelHeight), + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + p.level = .statusBar + p.hasShadow = false + p.isOpaque = false + p.backgroundColor = .clear + p.ignoresMouseEvents = true + p.collectionBehavior = [.canJoinAllSpaces, .transient] + + let content = NSView(frame: NSRect(x: 0, y: 0, width: panelWidth, height: panelHeight)) + content.wantsLayer = true + content.layer?.cornerRadius = 6 + content.layer?.backgroundColor = bgColor.cgColor + p.contentView = content + + let label = NSTextField(labelWithString: text) + label.frame = NSRect(x: 8, y: 5, width: panelWidth - (showTabHint ? 62 : 16), height: 18) + label.textColor = textColor + label.font = font + label.lineBreakMode = .byTruncatingTail + content.addSubview(label) + + let hint = NSTextField(labelWithString: tabHint.isEmpty ? "Tab ↵" : tabHint) + hint.frame = NSRect(x: panelWidth - 54, y: 5, width: 48, height: 18) + hint.textColor = NSColor(white: isDark ? 0.35 : 0.65, alpha: 1.0) + hint.font = NSFont.monospacedSystemFont(ofSize: 10, weight: .regular) + hint.alignment = .right + hint.isHidden = !showTabHint + content.addSubview(hint) + + panel = p + textField = label + hintField = hint + } + + // Re-apply colors on every show so runtime appearance changes are reflected. + panel?.contentView?.layer?.backgroundColor = bgColor.cgColor + textField?.textColor = textColor + hintField?.textColor = NSColor(white: isDark ? 0.35 : 0.65, alpha: 1.0) + hintField?.isHidden = !showTabHint + if showTabHint { + hintField?.stringValue = tabHint + } + + panel?.setFrame(NSRect(x: originX, y: originYCocoa, width: panelWidth, height: panelHeight), display: true) + panel?.contentView?.frame = NSRect(x: 0, y: 0, width: panelWidth, height: panelHeight) + textField?.frame = NSRect(x: 8, y: 5, width: panelWidth - (showTabHint ? 62 : 16), height: 18) + hintField?.frame = NSRect(x: panelWidth - 54, y: 5, width: 48, height: 18) + textField?.stringValue = text + panel?.orderFrontRegardless() + + hideWorkItem?.cancel() + let work = DispatchWorkItem { [weak self] in + self?.hide() + } + hideWorkItem = work + DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(max(120, ttlMs)), execute: work) + } + + func hide() { + panel?.orderOut(nil) + } +} + +// MARK: - Main Entry Point + +let app = NSApplication.shared +app.setActivationPolicy(.accessory) +let controller = OverlayController() + +DispatchQueue.global(qos: .userInitiated).async { + while let line = readLine() { + guard let data = line.data(using: .utf8), + let payload = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let kind = payload["type"] as? String else { + continue + } + let id = payload["id"] as? String + + switch kind { + case "focus": + let response = queryFocusedElement(id: id) + writeResponse(response) + + case "paste": + let text = (payload["text"] as? String) ?? "" + let response = pasteText(id: id, text: text) + writeResponse(response) + + case "ax_list": + let appName = (payload["app_name"] as? String) ?? "" + writeResponse(axListElements(appName: appName, id: id)) + + case "ax_press": + let appName = (payload["app_name"] as? String) ?? "" + let label = (payload["label"] as? String) ?? "" + writeResponse(axPress(appName: appName, label: label, id: id)) + + case "ax_set_value": + let appName = (payload["app_name"] as? String) ?? "" + let label = (payload["label"] as? String) ?? "" + let value = (payload["value"] as? String) ?? "" + writeResponse(axSetValue(appName: appName, label: label, value: value, id: id)) + + case "show": + let x = CGFloat((payload["x"] as? NSNumber)?.doubleValue ?? 0) + let y = CGFloat((payload["y"] as? NSNumber)?.doubleValue ?? 0) + let w = CGFloat((payload["w"] as? NSNumber)?.doubleValue ?? 0) + let h = CGFloat((payload["h"] as? NSNumber)?.doubleValue ?? 0) + let text = (payload["text"] as? String) ?? "" + let ttl = (payload["ttl_ms"] as? NSNumber)?.intValue ?? 900 + let tabHint: String = { + if let s = payload["tab_hint"] as? String { return s } + return "Tab ↵" + }() + DispatchQueue.main.async { + controller.show(x: x, yTop: y, width: w, height: h, text: text, ttlMs: ttl, tabHint: tabHint) + } + + case "hide": + DispatchQueue.main.async { + controller.hide() + } + + case "quit": + DispatchQueue.main.async { + controller.hide() + NSApplication.shared.terminate(nil) + } + return + + default: + break + } + } +} + +app.run() +"#; diff --git a/crates/tinycomputer-accessibility/src/helper/swift_paste.rs b/crates/tinycomputer-accessibility/src/helper/swift_paste.rs new file mode 100644 index 00000000..8ea06c41 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/helper/swift_paste.rs @@ -0,0 +1,44 @@ +//! Embedded Swift source fragment: the clipboard-based paste helper (`pasteText`). +#[cfg(target_os = "macos")] +pub(super) const SWIFT_PASTE: &str = r#"// MARK: - Paste Helper + +func pasteText(id: String?, text: String) -> [String: Any] { + var result: [String: Any] = ["type": "paste", "ok": true, "error": NSNull()] + if let id = id { result["id"] = id } + + let pb = NSPasteboard.general + let originalContents = pb.string(forType: .string) + + // Set clipboard to new text + pb.clearContents() + pb.setString(text, forType: .string) + + // Brief delay for clipboard to settle + usleep(10_000) // 10ms + + // Simulate Cmd+V via CGEvent + guard let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: 0x09, keyDown: true), + let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: 0x09, keyDown: false) else { + result["ok"] = false + result["error"] = "failed to create CGEvent" + return result + } + keyDown.flags = .maskCommand + keyUp.flags = .maskCommand + keyDown.post(tap: .cgSessionEventTap) + usleep(8_000) // 8ms between key down/up + keyUp.post(tap: .cgSessionEventTap) + + // Restore clipboard after delay + if let original = originalContents { + DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + .milliseconds(250)) { + let pb = NSPasteboard.general + pb.clearContents() + pb.setString(original, forType: .string) + } + } + + return result +} + +"#; diff --git a/crates/tinycomputer-accessibility/src/helper/swift_source.rs b/crates/tinycomputer-accessibility/src/helper/swift_source.rs new file mode 100644 index 00000000..a21fdb57 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/helper/swift_source.rs @@ -0,0 +1,19 @@ +//! Assembles the embedded Swift source for the unified helper process from +//! its per-responsibility fragments (focus query, paste, AX actions, overlay +//! and main loop). Splitting the fragments keeps each one reviewable; the +//! compiled Swift program is unaffected — this just concatenates them back +//! into one source string for `swiftc`. + +#[cfg(target_os = "macos")] +use super::swift_ax_actions::SWIFT_AX_ACTIONS; +#[cfg(target_os = "macos")] +use super::swift_focus::SWIFT_HEADER_AND_FOCUS; +#[cfg(target_os = "macos")] +use super::swift_overlay::SWIFT_OVERLAY_AND_MAIN; +#[cfg(target_os = "macos")] +use super::swift_paste::SWIFT_PASTE; + +#[cfg(target_os = "macos")] +pub(super) fn unified_swift_source() -> String { + format!("{SWIFT_HEADER_AND_FOCUS}{SWIFT_PASTE}{SWIFT_AX_ACTIONS}{SWIFT_OVERLAY_AND_MAIN}") +} diff --git a/crates/tinycomputer-accessibility/src/helper_tests.rs b/crates/tinycomputer-accessibility/src/helper_tests.rs new file mode 100644 index 00000000..85556b86 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/helper_tests.rs @@ -0,0 +1,7 @@ +//! Tests for the platform-independent entry points of the Swift helper module. + +#[cfg(not(target_os = "macos"))] +#[test] +fn precompile_entry_point_is_a_noop_off_macos() { + super::precompile_helper_background(); +} diff --git a/crates/tinycomputer-accessibility/src/lib.rs b/crates/tinycomputer-accessibility/src/lib.rs new file mode 100644 index 00000000..52f3fec3 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/lib.rs @@ -0,0 +1,53 @@ +//! Desktop accessibility middleware for hosts that need answers in-process. +//! +//! Centralises the macOS AX / `IOKit` FFI and the unified Swift helper process +//! (focus queries, paste, overlay in one persistent process), and exposes +//! focused-text inspection, system-permission detection (Accessibility, Input +//! Monitoring, Microphone), the Globe-key listener, "System Events" automation +//! denial tracking, terminal heuristics, and AX-string normalisation. A host +//! such as a voice pipeline calls these synchronously, so this is a plain +//! library with no bus and no async runtime; the `tinycomputer` module serves +//! the agent-facing desktop members separately. +//! +//! Behaviour outside macOS is limited to what the platform can answer: +//! focus queries return a typed unsupported-platform error, permission states are `Unsupported`, and +//! the microphone probe needs the `microphone-probe` feature. +//! +//! This crate reads no configuration and imports nothing from a host. + +mod automation_state; +mod error; +mod focus; +mod globe; +mod helper; +mod permissions; +mod terminal; +mod text_util; +mod types; + +pub use automation_state::{ + clear as clear_automation_denial, mark_system_events_denied, system_events_denied, +}; +pub use error::{Error, Result}; +pub use focus::{focused_text_context, focused_text_context_verbose, validate_focused_target}; +pub use globe::{ + GlobeHotkeyPollResult, GlobeHotkeyStatus, globe_listener_poll, globe_listener_start, + globe_listener_stop, +}; +pub use helper::precompile_helper_background; +#[cfg(target_os = "macos")] +pub use permissions::{ + detect_accessibility_permission, detect_input_monitoring_permission, open_macos_privacy_pane, + request_accessibility_access, +}; +pub use permissions::{ + detect_microphone_permission, detect_permissions, microphone_denied_message, permission_to_str, + request_microphone_access, +}; +pub use terminal::{ + extract_terminal_input_context, is_terminal_app, is_text_role, looks_like_terminal_buffer, +}; +pub use text_util::{normalize_ax_value, parse_ax_number, truncate_tail}; +pub use types::{ + ElementBounds, FocusedTextContext, PermissionKind, PermissionState, PermissionStatus, +}; diff --git a/crates/tinycomputer-accessibility/src/permissions.rs b/crates/tinycomputer-accessibility/src/permissions.rs new file mode 100644 index 00000000..7c3222c7 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/permissions.rs @@ -0,0 +1,308 @@ +//! Platform permission detection and requests for accessibility, input monitoring, and microphone access. +//! +//! The macOS permission checks call `ApplicationServices`, CoreFoundation and +//! `IOKit` directly, so this is the one module in the crate that allows `unsafe`. +#![allow(unsafe_code)] + +use super::types::{PermissionKind, PermissionState, PermissionStatus}; + +#[cfg(target_os = "macos")] +use std::ffi::c_void; + +#[cfg(target_os = "macos")] +type CFAllocatorRef = *const c_void; +#[cfg(target_os = "macos")] +type CFDictionaryRef = *const c_void; +#[cfg(target_os = "macos")] +type CFBooleanRef = *const c_void; +#[cfg(target_os = "macos")] +type CFStringRef = *const c_void; + +#[cfg(target_os = "macos")] +#[link(name = "ApplicationServices", kind = "framework")] +unsafe extern "C" { + fn AXIsProcessTrusted() -> bool; + fn AXIsProcessTrustedWithOptions(options: CFDictionaryRef) -> bool; + static kAXTrustedCheckOptionPrompt: CFStringRef; +} + +#[cfg(target_os = "macos")] +#[link(name = "CoreFoundation", kind = "framework")] +unsafe extern "C" { + static kCFAllocatorDefault: CFAllocatorRef; + static kCFBooleanTrue: CFBooleanRef; + fn CFDictionaryCreate( + allocator: CFAllocatorRef, + keys: *const *const c_void, + values: *const *const c_void, + num_values: isize, + key_callbacks: *const c_void, + value_callbacks: *const c_void, + ) -> CFDictionaryRef; + fn CFRelease(cf: *const c_void); +} + +#[cfg(target_os = "macos")] +#[link(name = "IOKit", kind = "framework")] +unsafe extern "C" { + fn IOHIDCheckAccess(request_type: i32) -> isize; +} + +#[cfg(target_os = "macos")] +const IOHID_REQUEST_TYPE_LISTEN_EVENT: i32 = 1; +#[cfg(target_os = "macos")] +const IOHID_ACCESS_GRANTED: isize = 0; +#[cfg(target_os = "macos")] +const IOHID_ACCESS_DENIED: isize = 1; + +#[must_use] +/// The wire name of a permission kind (`accessibility`, `input_monitoring`, `microphone`). +pub fn permission_to_str(permission: PermissionKind) -> &'static str { + match permission { + PermissionKind::Accessibility => "accessibility", + PermissionKind::InputMonitoring => "input_monitoring", + PermissionKind::Microphone => "microphone", + } +} + +#[cfg(target_os = "macos")] +/// Open a pane of System Settings > Privacy & Security, e.g. `Privacy_Accessibility`. +pub fn open_macos_privacy_pane(pane: &str) { + let url = format!("x-apple.systempreferences:com.apple.preference.security?{pane}"); + let _ = std::process::Command::new("open").arg(url).status(); +} + +#[cfg(target_os = "macos")] +/// Ask macOS to prompt for Accessibility access. +pub fn request_accessibility_access() { + // SAFETY: `kAXTrustedCheckOptionPrompt` and `kCFBooleanTrue` are immutable + // CoreFoundation globals; the dictionary is created with null callbacks + // (no retains), used once, and released here on the same thread. + unsafe { + let keys = [kAXTrustedCheckOptionPrompt]; + let values = [kCFBooleanTrue]; + let options = CFDictionaryCreate( + kCFAllocatorDefault, + keys.as_ptr(), + values.as_ptr(), + 1, + std::ptr::null(), + std::ptr::null(), + ); + let _ = AXIsProcessTrustedWithOptions(options); + if !options.is_null() { + CFRelease(options); + } + } +} + +#[cfg(target_os = "macos")] +#[must_use] +/// Whether this process is trusted for Accessibility. +pub fn detect_accessibility_permission() -> PermissionState { + // SAFETY: `AXIsProcessTrusted` takes no arguments and only reads TCC state. + unsafe { + if AXIsProcessTrusted() { + PermissionState::Granted + } else { + PermissionState::Denied + } + } +} + +#[cfg(target_os = "macos")] +#[must_use] +/// Whether this process may listen for input events (Input Monitoring). +pub fn detect_input_monitoring_permission() -> PermissionState { + // SAFETY: `IOHIDCheckAccess` takes a plain request-type integer and only + // reads TCC state. + let access = unsafe { IOHIDCheckAccess(IOHID_REQUEST_TYPE_LISTEN_EVENT) }; + match access { + IOHID_ACCESS_GRANTED => PermissionState::Granted, + IOHID_ACCESS_DENIED => PermissionState::Denied, + // `IOHID_ACCESS_UNKNOWN` and any future value. + _ => PermissionState::Unknown, + } +} + +// --------------------------------------------------------------------------- +// Microphone permission — cross-platform +// --------------------------------------------------------------------------- + +/// Detect whether the app has microphone permission. +/// +/// Uses CPAL to detect whether an input device is present. Device enumeration +/// does not prove recording authorization, so the result remains `Unknown` +/// until the host opens an input stream and observes whether capture is allowed. +/// +/// On **macOS** under hardened runtime, CPAL will fail to enumerate input +/// devices when the `com.apple.security.device.audio-input` entitlement is +/// missing or microphone permission is denied in System Settings. +/// +/// On **Windows**, `None` may indicate a privacy toggle denial or no hardware. +/// +/// **Linux** standard desktops don't enforce per-app permissions; Flatpak/Snap +/// sandboxes are detected separately. +#[cfg(all( + feature = "microphone-probe", + any(target_os = "macos", target_os = "windows") +))] +pub fn detect_microphone_permission() -> PermissionState { + use cpal::traits::HostTrait; + let host = cpal::default_host(); + match host.default_input_device() { + Some(device) => { + let name = + cpal::traits::DeviceTrait::name(&device).unwrap_or_else(|_| "".into()); + log::debug!( + "[permissions] input device detected; capture authorization is unverified — device: {name}" + ); + PermissionState::Unknown + } + None => { + log::debug!( + "[permissions] no default input device — possible permission denial or no mic connected" + ); + PermissionState::Unknown + } + } +} + +/// Detect microphone permission. See the `microphone-probe` feature; without it the answer is `Unknown` on desktop platforms. +#[cfg(all(feature = "microphone-probe", target_os = "linux"))] +#[must_use] +pub fn detect_microphone_permission() -> PermissionState { + // Standard Linux desktops (PulseAudio/PipeWire) don't enforce app-level mic permissions. + // Detect Flatpak sandbox — input device presence cannot confirm capture access. + let is_sandboxed = std::env::var("FLATPAK_ID").is_ok() + || std::path::Path::new("/run/flatpak").exists() + || std::env::var("SNAP").is_ok() + || std::env::var("SNAP_NAME").is_ok() + || std::env::var("SNAP_INSTANCE_NAME").is_ok(); + linux_microphone_permission(is_sandboxed, || { + use cpal::traits::HostTrait; + cpal::default_host().default_input_device().is_some() + }) +} + +#[cfg(all(feature = "microphone-probe", target_os = "linux"))] +fn linux_microphone_permission( + is_sandboxed: bool, + has_default_input_device: impl FnOnce() -> bool, +) -> PermissionState { + if !is_sandboxed { + PermissionState::Granted + } else if has_default_input_device() { + PermissionState::Unknown + } else { + log::debug!( + "[permissions] Linux (Flatpak): no default input device — possible sandbox restriction" + ); + PermissionState::Denied + } +} + +/// With the `microphone-probe` feature off, `cpal` is not compiled in, so there is no +/// audio-device API to probe and the microphone cannot be inspected. Report +/// `Unknown` on otherwise-supported desktop platforms rather than a misleading +/// `Granted`/`Denied`. +#[cfg(all( + not(feature = "microphone-probe"), + any(target_os = "macos", target_os = "windows", target_os = "linux") +))] +#[must_use] +/// Detect microphone permission. See the `microphone-probe` feature; without it the answer is `Unknown` on desktop platforms. +pub fn detect_microphone_permission() -> PermissionState { + log::debug!( + "[permissions] microphone probe unavailable (built without the `microphone-probe` feature)" + ); + PermissionState::Unknown +} + +#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] +/// Detect microphone permission. See the `microphone-probe` feature; without it the answer is `Unknown` on desktop platforms. +pub fn detect_microphone_permission() -> PermissionState { + PermissionState::Unsupported +} + +/// Open the operating system's microphone privacy settings where available. +/// +/// - **macOS**: Opens System Settings > Privacy & Security > Microphone. It does not +/// request authorization or trigger a system permission prompt. +/// - **Windows**: Opens the Privacy > Microphone settings page. +/// - **Linux**: No-op; sandbox guidance is included in error messages. +#[cfg(target_os = "macos")] +pub fn request_microphone_access() { + log::debug!("[permissions] requesting macOS microphone access via Privacy pane"); + open_macos_privacy_pane("Privacy_Microphone"); +} + +/// Send the user to where microphone access is granted, where the OS has one. +#[cfg(target_os = "windows")] +pub fn request_microphone_access() { + log::debug!("[permissions] opening Windows Privacy > Microphone settings"); + let _ = std::process::Command::new("cmd") + .args(["/C", "start", "ms-settings:privacy-microphone"]) + .status(); +} + +/// Send the user to where microphone access is granted, where the OS has one. +#[cfg(target_os = "linux")] +pub fn request_microphone_access() { + log::debug!("[permissions] Linux: no programmatic mic permission request available"); + // No-op — standard Linux desktops don't have an app-level permission gate. + // For Flatpak, the XDG Portal API (ashpd crate) could be used in the future. +} + +/// Send the user to where microphone access is granted, where the OS has one. +#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] +pub fn request_microphone_access() { + // Unsupported platform — no-op. +} + +#[cfg(test)] +#[path = "permissions_tests.rs"] +mod tests; + +/// Returns a platform-specific user-facing message when microphone permission is denied. +#[must_use] +pub fn microphone_denied_message() -> String { + #[cfg(target_os = "macos")] + { + "Microphone permission denied. Grant access in System Settings > Privacy & Security > Microphone, then restart the app.".to_string() + } + #[cfg(target_os = "windows")] + { + "Microphone access unavailable. Check Settings > Privacy & Security > Microphone and ensure the app is allowed. If no microphone is connected, plug one in.".to_string() + } + #[cfg(target_os = "linux")] + { + "No microphone device available. Check your audio settings and ensure a microphone is connected. If running in a Flatpak sandbox, grant microphone access via Flatseal or system settings.".to_string() + } + #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] + { + "Microphone access is not supported on this platform.".to_string() + } +} + +#[cfg(target_os = "macos")] +#[must_use] +/// Snapshot every permission this crate reports. +pub fn detect_permissions() -> PermissionStatus { + PermissionStatus { + accessibility: detect_accessibility_permission(), + input_monitoring: detect_input_monitoring_permission(), + microphone: detect_microphone_permission(), + } +} + +#[cfg(not(target_os = "macos"))] +#[must_use] +/// Snapshot every permission this crate reports. +pub fn detect_permissions() -> PermissionStatus { + PermissionStatus { + accessibility: PermissionState::Unsupported, + input_monitoring: PermissionState::Unsupported, + microphone: detect_microphone_permission(), + } +} diff --git a/crates/tinycomputer-accessibility/src/permissions_tests.rs b/crates/tinycomputer-accessibility/src/permissions_tests.rs new file mode 100644 index 00000000..0ed1248d --- /dev/null +++ b/crates/tinycomputer-accessibility/src/permissions_tests.rs @@ -0,0 +1,305 @@ +#![allow(clippy::unwrap_used, clippy::expect_used)] +//! Unit tests for `accessibility::permissions`. +//! +//! macOS-only FFI functions (`detect_accessibility_permission`, +//! `detect_input_monitoring_permission`, +//! `request_*`) call into Apple frameworks and cannot be exercised in a +//! cross-platform test binary without hardware. Tests here cover: +//! +//! - `permission_to_str` — pure logic, always available. +//! - `microphone_denied_message` — pure logic, always available. +//! - `detect_permissions` — non-macOS fallback (unsupported states). +//! - `detect_microphone_permission` — cross-platform probe guard. + +use super::*; + +// ── permission_to_str ───────────────────────────────────────────────────── + +#[test] +fn permission_to_str_accessibility() { + assert_eq!( + permission_to_str(PermissionKind::Accessibility), + "accessibility" + ); +} + +#[test] +fn permission_to_str_input_monitoring() { + assert_eq!( + permission_to_str(PermissionKind::InputMonitoring), + "input_monitoring" + ); +} + +#[test] +fn permission_to_str_microphone() { + assert_eq!(permission_to_str(PermissionKind::Microphone), "microphone"); +} + +#[test] +fn permission_to_str_is_snake_case_and_nonempty() { + for kind in [ + PermissionKind::Accessibility, + PermissionKind::InputMonitoring, + PermissionKind::Microphone, + ] { + let s = permission_to_str(kind); + assert!( + !s.is_empty(), + "permission_to_str should never return empty string" + ); + // Convention: snake_case, no spaces + assert!( + !s.contains(' '), + "permission string should not contain spaces: {s}" + ); + assert_eq!( + s, + s.to_ascii_lowercase(), + "permission string should be lowercase: {s}" + ); + } +} + +// ── microphone_denied_message ───────────────────────────────────────────── + +#[test] +fn microphone_denied_message_is_nonempty() { + let msg = microphone_denied_message(); + assert!(!msg.is_empty(), "denied message should not be empty"); +} + +#[test] +fn microphone_denied_message_is_human_readable() { + let msg = microphone_denied_message().to_ascii_lowercase(); + // All platform messages mention "microphone" as a cue to the user. + assert!( + msg.contains("microphone"), + "denied message should mention 'microphone': {msg}" + ); +} + +// ── detect_permissions (non-macOS path) ────────────────────────────────── + +/// On non-macOS platforms `detect_permissions` reports accessibility and +/// `input_monitoring` as `Unsupported`. Microphone gets a +/// real check via CPAL. +#[cfg(not(target_os = "macos"))] +#[test] +fn detect_permissions_non_macos_reports_unsupported_for_desktop_perms() { + let status = detect_permissions(); + assert_eq!( + status.accessibility, + PermissionState::Unsupported, + "accessibility should be Unsupported on non-macOS" + ); + assert_eq!( + status.input_monitoring, + PermissionState::Unsupported, + "input_monitoring should be Unsupported on non-macOS" + ); +} + +/// Microphone permission on non-macOS/non-Windows platforms should be +/// `Granted` (standard Linux desktop) or `Unknown`/`Denied` (Flatpak). It +/// should never be `Unsupported` unless the platform has a dedicated stub. +#[cfg(target_os = "linux")] +#[test] +fn detect_microphone_permission_linux_returns_valid_state() { + let state = detect_microphone_permission(); + assert!( + !matches!(state, PermissionState::Unsupported), + "Linux microphone state should not be Unsupported" + ); +} + +#[cfg(all(feature = "microphone-probe", target_os = "linux"))] +#[test] +fn linux_microphone_permission_handles_sandbox_probe_results() { + let mut probe_called = false; + assert_eq!( + linux_microphone_permission(false, || { + probe_called = true; + false + }), + PermissionState::Granted + ); + assert!(!probe_called); + assert_eq!( + linux_microphone_permission(true, || true), + PermissionState::Unknown + ); + assert_eq!( + linux_microphone_permission(true, || false), + PermissionState::Denied + ); +} + +// ── PermissionState serde round-trip ───────────────────────────────────── + +#[test] +fn permission_state_serde_round_trip() { + use crate::types::{PermissionState, PermissionStatus}; + + let status = PermissionStatus { + accessibility: PermissionState::Denied, + input_monitoring: PermissionState::Unknown, + microphone: PermissionState::Unsupported, + }; + let json = serde_json::to_string(&status).expect("serialize PermissionStatus"); + let back: PermissionStatus = serde_json::from_str(&json).expect("deserialize PermissionStatus"); + assert_eq!(back.accessibility, PermissionState::Denied); + assert_eq!(back.input_monitoring, PermissionState::Unknown); + assert_eq!(back.microphone, PermissionState::Unsupported); +} + +/// Partial permission state must survive as three independent fields on the +/// wire, spelled exactly as the renderer reads them (matrix 2.2.4). +/// +/// `permission_state_serde_round_trip` above proves Rust -> JSON -> Rust, which +/// passes even if the keys were `a`/`b`/`c` and the variants `V1`/`V2`: both +/// sides of that round trip use the same `derive`. Nothing pinned the *wire +/// spelling*, and the renderer does not go through `serde` — it indexes the +/// JSON by name. A `rename_all` change or a renamed field would leave every +/// lookup `undefined`, which renders as "not granted" indefinitely and looks +/// exactly like a permission the user never gave. +/// +/// The three states here are deliberately all different: a status that +/// collapsed to one verdict, or grew a fourth field, fails the shape check +/// rather than silently reporting the first field for all three. +#[test] +fn partial_permission_status_serializes_three_distinct_snake_case_fields() { + use crate::types::{PermissionState, PermissionStatus}; + + let status = PermissionStatus { + accessibility: PermissionState::Granted, + input_monitoring: PermissionState::Denied, + microphone: PermissionState::Unsupported, + }; + + let json = serde_json::to_value(&status).expect("serialize PermissionStatus"); + let object = json + .as_object() + .expect("PermissionStatus must serialize to a JSON object"); + + assert_eq!( + object.len(), + 3, + "PermissionStatus gained or lost a field; the renderer reads exactly \ + accessibility/input_monitoring/microphone: {json}" + ); + assert_eq!( + object.get("accessibility").and_then(|v| v.as_str()), + Some("granted"), + "accessibility must serialize as snake_case `granted`: {json}" + ); + assert_eq!( + object.get("input_monitoring").and_then(|v| v.as_str()), + Some("denied"), + "input_monitoring must keep its own value, not the accessibility one: {json}" + ); + assert_eq!( + object.get("microphone").and_then(|v| v.as_str()), + Some("unsupported"), + "microphone must keep its own value: {json}" + ); +} + +/// `Unsupported` is not `Granted`, and a caller cannot get away with treating +/// "not denied" as "granted" (matrix 2.2.4). +/// +/// On a non-macOS build `detect_permissions` reports accessibility and +/// `input_monitoring` as `Unsupported` while microphone is a real CPAL probe, so +/// the struct is genuinely mixed-provenance in production. Any consumer that +/// reduces it to one boolean is wrong on at least one field; this pins the +/// three-way distinction the reduction would erase. +#[test] +fn unsupported_is_distinguishable_from_granted_and_denied() { + use crate::types::PermissionState; + + let states = [ + PermissionState::Granted, + PermissionState::Denied, + PermissionState::Unknown, + PermissionState::Unsupported, + ]; + + let wire: Vec = states + .iter() + .map(|state| { + serde_json::to_value(state) + .expect("serialize PermissionState") + .as_str() + .expect("PermissionState serializes to a string") + .to_string() + }) + .collect(); + + assert_eq!( + wire, + vec!["granted", "denied", "unknown", "unsupported"], + "PermissionState wire spelling changed; the renderer compares these \ + strings literally" + ); + + let mut unique = wire.clone(); + unique.sort(); + unique.dedup(); + assert_eq!( + unique.len(), + states.len(), + "two PermissionState variants collapsed to the same wire value, so a \ + partial permission state cannot be told apart: {wire:?}" + ); +} + +// ── No stale denied cache across restart (automation_state) ─────────────── +// +// The `automation_state` module exposes a process-local atomic flag. The +// "no stale denied cache across restart" guarantee is enforced by the fact +// that the flag is `AtomicBool` initialized to `false` — each new process +// starts clean. The tests below verify the clean-start invariant at the +// module level and that `clear()` restores the initial state, which is the +// mechanism used by `autocomplete::start_if_enabled` on re-engagement. + +mod automation_state_stale_cache { + use crate::automation_state; + use crate::{clear_automation_denial, mark_system_events_denied, system_events_denied}; + + #[test] + fn fresh_state_is_not_denied() { + let _g = automation_state::test_lock(); + clear_automation_denial(); + assert!( + !system_events_denied(), + "after clear(), system_events_denied should be false (simulates process restart)" + ); + } + + #[test] + fn clear_resets_denial_flag() { + let _g = automation_state::test_lock(); + clear_automation_denial(); + mark_system_events_denied(); + assert!(system_events_denied(), "should be denied after mark"); + clear_automation_denial(); + assert!( + !system_events_denied(), + "clear() should erase the stale denied state" + ); + } + + #[test] + fn denied_flag_does_not_persist_through_clear() { + let _g = automation_state::test_lock(); + // Simulate: previous session left the flag set. + // clear() is called on re-engagement → no stale state carried over. + mark_system_events_denied(); + clear_automation_denial(); + // Re-query after clear — must be false, simulating a "fresh sidecar" read. + assert!( + !system_events_denied(), + "denial flag must not persist after clear() — no stale cache" + ); + } +} diff --git a/crates/tinycomputer-accessibility/src/terminal.rs b/crates/tinycomputer-accessibility/src/terminal.rs new file mode 100644 index 00000000..c5ab7106 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/terminal.rs @@ -0,0 +1,99 @@ +//! Terminal app detection and context extraction. + +/// Normalized terminal application names (lowercase, without punctuation). +/// Extend this list to support additional terminal emulators. +pub(crate) const TERMINAL_NAMES: &[&str] = &[ + "terminal", + "iterm2", + "wezterm", + "warp", + "alacritty", + "kitty", + "ghostty", + "hyper", + "rio", + "tabby", + "wave", + "contour", + "foot", +]; + +#[must_use] +/// Whether an accessibility role names an editable text element. +pub fn is_text_role(role: Option<&str>) -> bool { + matches!( + role.unwrap_or_default(), + "AXTextArea" | "AXTextField" | "AXSearchField" | "AXComboBox" | "AXEditableText" + ) +} + +#[must_use] +/// Whether an application name looks like a terminal emulator. +pub fn is_terminal_app(app_name: Option<&str>) -> bool { + let app: String = app_name + .unwrap_or_default() + .chars() + .filter(char::is_ascii_alphanumeric) + .flat_map(char::to_lowercase) + .collect(); + TERMINAL_NAMES.iter().any(|name| app == *name) +} + +#[must_use] +/// Whether captured text looks like a terminal scrollback buffer rather than prose. +pub fn looks_like_terminal_buffer(text: &str) -> bool { + let lower = text.to_ascii_lowercase(); + let line_count = text.lines().count(); + line_count >= 5 + && (lower.contains("$ ") + || lower.contains("# ") + || lower.contains("❯") + || lower.contains("[1] 0:") + || lower.contains("tmux") + || lower.contains("cargo run") + || lower.contains("git status")) +} + +fn is_terminal_noise_line(line: &str) -> bool { + let trimmed = line.trim(); + if trimmed.is_empty() { + return true; + } + trimmed.starts_with('•') + || trimmed.starts_with('└') + || trimmed.starts_with('─') + || trimmed.starts_with('│') + || (trimmed.starts_with('[') + && (trimmed.contains(" 0:") || trimmed.contains("[tmux]") || trimmed.contains("\"⠙"))) +} + +#[must_use] +/// Extract the text around the current prompt from a terminal buffer. +pub fn extract_terminal_input_context(text: &str) -> String { + let mut fallback = String::new(); + for raw_line in text.lines().rev().take(40) { + let line = raw_line.trim(); + if line.is_empty() { + continue; + } + if fallback.is_empty() && !is_terminal_noise_line(line) { + fallback = line.to_string(); + } + if is_terminal_noise_line(line) { + continue; + } + if line.contains("$ ") + || line.contains("# ") + || line.contains("❯") + || line.contains("➜") + || line.contains("λ") + { + return line.to_string(); + } + } + fallback +} + +#[cfg(test)] +#[path = "terminal_tests.rs"] +mod tests; diff --git a/crates/tinycomputer-accessibility/src/terminal_tests.rs b/crates/tinycomputer-accessibility/src/terminal_tests.rs new file mode 100644 index 00000000..66cbf668 --- /dev/null +++ b/crates/tinycomputer-accessibility/src/terminal_tests.rs @@ -0,0 +1,112 @@ +//! Tests for terminal detection and prompt extraction heuristics. + +use super::*; + +#[test] +fn is_text_role_accepts_known_roles() { + assert!(is_text_role(Some("AXTextArea"))); + assert!(is_text_role(Some("AXTextField"))); + assert!(is_text_role(Some("AXSearchField"))); + assert!(is_text_role(Some("AXComboBox"))); + assert!(is_text_role(Some("AXEditableText"))); +} + +#[test] +fn is_text_role_rejects_other_roles() { + assert!(!is_text_role(Some("AXButton"))); + assert!(!is_text_role(Some("AXImage"))); + assert!(!is_text_role(None)); + assert!(!is_text_role(Some(""))); +} + +#[test] +fn is_terminal_app_detects_known_terminals() { + assert!(is_terminal_app(Some("iTerm2"))); + assert!(is_terminal_app(Some("Terminal"))); + assert!(is_terminal_app(Some("WezTerm"))); + assert!(is_terminal_app(Some("Alacritty"))); + assert!(is_terminal_app(Some("kitty"))); + assert!(is_terminal_app(Some("Warp"))); + assert!(is_terminal_app(Some("Ghostty"))); +} + +#[test] +fn is_terminal_app_rejects_non_terminals() { + assert!(!is_terminal_app(Some("Safari"))); + assert!(!is_terminal_app(Some("Slack"))); + assert!(!is_terminal_app(None)); + assert!(!is_terminal_app(Some(""))); +} + +#[test] +fn is_terminal_app_requires_a_normalized_exact_name() { + assert!(is_terminal_app(Some("iTerm 2"))); + assert!(is_terminal_app(Some("foot"))); + assert!(!is_terminal_app(Some("Curio"))); + assert!(!is_terminal_app(Some("Footage"))); + assert!(!is_terminal_app(Some("Terminal Emulator Preview"))); +} + +#[test] +fn looks_like_terminal_buffer_detects_shell_prompts() { + let buffer = "line1\nline2\nline3\nline4\n$ cargo build\nCompiling...\n"; + assert!(looks_like_terminal_buffer(buffer)); +} + +#[test] +fn looks_like_terminal_buffer_rejects_short_text() { + assert!(!looks_like_terminal_buffer("hello")); + assert!(!looks_like_terminal_buffer("$ cmd")); +} + +#[test] +fn looks_like_terminal_buffer_detects_git_status() { + let buffer = "line1\nline2\nline3\nline4\ngit status\nOn branch main\n"; + assert!(looks_like_terminal_buffer(buffer)); +} + +#[test] +fn extract_terminal_input_context_finds_prompt_line() { + let text = "old output\n$ ls -la\ntotal 42\nfile1.txt\nfile2.txt\n"; + let ctx = extract_terminal_input_context(text); + assert!(ctx.contains("$ ls"), "expected prompt line, got: {ctx}"); +} + +#[test] +fn extract_terminal_input_context_skips_noise() { + let text = "actual content\n• bullet point\n└── tree branch\n│ pipe\n"; + let ctx = extract_terminal_input_context(text); + assert_eq!(ctx, "actual content"); +} + +#[test] +fn extract_terminal_input_context_empty_returns_empty() { + assert!(extract_terminal_input_context("").is_empty()); +} + +#[test] +fn extract_terminal_input_context_all_noise_returns_empty() { + let text = "\n\n\n"; + assert!(extract_terminal_input_context(text).is_empty()); +} + +#[test] +fn terminal_names_is_nonempty() { + assert!(!TERMINAL_NAMES.is_empty()); +} + +#[test] +fn is_terminal_noise_line_detects_noise() { + assert!(is_terminal_noise_line("")); + assert!(is_terminal_noise_line(" ")); + assert!(is_terminal_noise_line("• item")); + assert!(is_terminal_noise_line("└── branch")); + assert!(is_terminal_noise_line("─────")); + assert!(is_terminal_noise_line("│ pipe")); +} + +#[test] +fn is_terminal_noise_line_passes_normal_text() { + assert!(!is_terminal_noise_line("hello world")); + assert!(!is_terminal_noise_line("$ command")); +} diff --git a/crates/tinycomputer-accessibility/src/text_util.rs b/crates/tinycomputer-accessibility/src/text_util.rs new file mode 100644 index 00000000..2b72ee9f --- /dev/null +++ b/crates/tinycomputer-accessibility/src/text_util.rs @@ -0,0 +1,49 @@ +//! Shared text utilities for accessibility value parsing. + +#[must_use] +/// Keep the last `max_chars` characters of `text`. +pub fn truncate_tail(text: &str, max_chars: usize) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= max_chars { + return text.to_string(); + } + chars[chars.len() - max_chars..].iter().collect() +} + +#[must_use] +/// Trim an accessibility string value; the literal `missing value` becomes empty. +pub fn normalize_ax_value(raw: &str) -> String { + let v = raw.trim(); + if v.eq_ignore_ascii_case("missing value") { + String::new() + } else { + v.to_string() + } +} + +#[must_use] +/// Parse an accessibility number (comma or dot decimal) to a rounded `i32`; `None` if empty, non-finite, or out of range. +pub fn parse_ax_number(raw: &str) -> Option { + let trimmed = normalize_ax_value(raw); + if trimmed.is_empty() { + return None; + } + let cleaned = trimmed.replace(',', "."); + cleaned.parse::().ok().and_then(|v| { + if !v.is_finite() { + return None; + } + let rounded = v.round(); + if rounded < f64::from(i32::MIN) || rounded > f64::from(i32::MAX) { + return None; + } + // Range-checked against `i32` above, so the cast cannot truncate. + #[allow(clippy::cast_possible_truncation)] + let value = rounded as i32; + Some(value) + }) +} + +#[cfg(test)] +#[path = "text_util_tests.rs"] +mod tests; diff --git a/crates/tinycomputer-accessibility/src/text_util_tests.rs b/crates/tinycomputer-accessibility/src/text_util_tests.rs new file mode 100644 index 00000000..5043a93d --- /dev/null +++ b/crates/tinycomputer-accessibility/src/text_util_tests.rs @@ -0,0 +1,144 @@ +//! Tests for truncating and normalizing accessibility text and parsing AX numbers. + +use super::*; + +// --- truncate_tail --- + +#[test] +fn truncate_tail_shorter_than_max_returns_original() { + assert_eq!(truncate_tail("hello", 10), "hello"); +} + +#[test] +fn truncate_tail_exactly_max_returns_original() { + assert_eq!(truncate_tail("hello", 5), "hello"); +} + +#[test] +fn truncate_tail_longer_than_max_returns_tail() { + assert_eq!(truncate_tail("hello", 3), "llo"); +} + +#[test] +fn truncate_tail_empty_string() { + assert_eq!(truncate_tail("", 5), ""); +} + +#[test] +fn truncate_tail_zero_max_returns_empty() { + assert_eq!(truncate_tail("hello", 0), ""); +} + +#[test] +fn truncate_tail_multibyte_chars_counts_chars_not_bytes() { + // "héllo" is 5 chars; last 3 = "llo" + assert_eq!(truncate_tail("héllo", 3), "llo"); +} + +#[test] +fn truncate_tail_unicode_emoji_counts_codepoints() { + // "ab🎉cd" — 5 codepoints; last 3 = "🎉cd" + assert_eq!(truncate_tail("ab🎉cd", 3), "🎉cd"); +} + +// --- normalize_ax_value --- + +#[test] +fn normalize_ax_value_trims_whitespace() { + assert_eq!(normalize_ax_value(" hello "), "hello"); +} + +#[test] +fn normalize_ax_value_missing_value_lowercase_returns_empty() { + assert_eq!(normalize_ax_value("missing value"), ""); +} + +#[test] +fn normalize_ax_value_missing_value_uppercase_returns_empty() { + assert_eq!(normalize_ax_value("MISSING VALUE"), ""); +} + +#[test] +fn normalize_ax_value_mixed_case_missing_value_returns_empty() { + assert_eq!(normalize_ax_value("Missing Value"), ""); +} + +#[test] +fn normalize_ax_value_empty_string_returns_empty() { + assert_eq!(normalize_ax_value(""), ""); +} + +#[test] +fn normalize_ax_value_only_whitespace_returns_empty() { + assert_eq!(normalize_ax_value(" "), ""); +} + +#[test] +fn normalize_ax_value_regular_text_unchanged() { + assert_eq!(normalize_ax_value("some value"), "some value"); +} + +// --- parse_ax_number --- + +#[test] +fn parse_ax_number_integer_string() { + assert_eq!(parse_ax_number("42"), Some(42)); +} + +#[test] +fn parse_ax_number_negative_integer() { + assert_eq!(parse_ax_number("-7"), Some(-7)); +} + +#[test] +fn parse_ax_number_float_rounds_to_nearest() { + assert_eq!(parse_ax_number("42.4"), Some(42)); + assert_eq!(parse_ax_number("42.6"), Some(43)); +} + +#[test] +fn parse_ax_number_comma_treated_as_decimal_separator() { + // Locale-style: "1,5" → 1.5 → rounds to 2 + assert_eq!(parse_ax_number("1,5"), Some(2)); +} + +#[test] +fn parse_ax_number_missing_value_returns_none() { + assert_eq!(parse_ax_number("missing value"), None); +} + +#[test] +fn parse_ax_number_empty_returns_none() { + assert_eq!(parse_ax_number(""), None); +} + +#[test] +fn parse_ax_number_whitespace_only_returns_none() { + assert_eq!(parse_ax_number(" "), None); +} + +#[test] +fn parse_ax_number_non_numeric_returns_none() { + assert_eq!(parse_ax_number("abc"), None); +} + +#[test] +fn parse_ax_number_nan_returns_none() { + assert_eq!(parse_ax_number("NaN"), None); +} + +#[test] +fn parse_ax_number_infinity_returns_none() { + assert_eq!(parse_ax_number("inf"), None); + assert_eq!(parse_ax_number("infinity"), None); +} + +#[test] +fn parse_ax_number_zero() { + assert_eq!(parse_ax_number("0"), Some(0)); +} + +#[test] +fn parse_ax_number_trims_surrounding_whitespace() { + assert_eq!(parse_ax_number(" 10 "), Some(10)); +} diff --git a/crates/tinycomputer-accessibility/src/types.rs b/crates/tinycomputer-accessibility/src/types.rs new file mode 100644 index 00000000..75631cbf --- /dev/null +++ b/crates/tinycomputer-accessibility/src/types.rs @@ -0,0 +1,70 @@ +//! Shared platform types for accessibility, focus, and permissions. + +use serde::{Deserialize, Serialize}; + +/// Unified element bounds — used by autocomplete. +#[derive(Debug, Clone, Copy)] +pub struct ElementBounds { + /// Left edge. + pub x: i32, + /// Top edge. + pub y: i32, + /// Width. + pub width: i32, + /// Height. + pub height: i32, +} + +/// Context returned by an accessibility focus query. +#[derive(Debug, Clone)] +pub struct FocusedTextContext { + /// Frontmost application name. + pub app_name: Option, + /// Accessibility role of the focused element. + pub role: Option, + /// Text content of the element. + pub text: String, + /// Currently selected text, if any. + pub selected_text: Option, + /// Diagnostic from the helper when the query partly failed. + pub raw_error: Option, + /// Element bounds, if reported. + pub bounds: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +/// Whether a permission is granted. +pub enum PermissionState { + /// Permission granted. + Granted, + /// Permission denied. + Denied, + /// State cannot be determined. + Unknown, + /// Not applicable on this platform. + Unsupported, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +/// Every permission this crate reports, at one instant. +pub struct PermissionStatus { + /// Accessibility (AX) trust. + pub accessibility: PermissionState, + /// Input Monitoring (global key events). + pub input_monitoring: PermissionState, + /// Microphone access. + pub microphone: PermissionState, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +/// A permission this crate can detect or request. +pub enum PermissionKind { + /// Accessibility (AX) trust. + Accessibility, + /// Input Monitoring. + InputMonitoring, + /// Microphone access. + Microphone, +} diff --git a/docs/README.md b/docs/README.md index f08d79a0..ad9edcce 100644 --- a/docs/README.md +++ b/docs/README.md @@ -43,6 +43,7 @@ One friendly guide per crate and per top-level folder. Start at | [`tinycomputer-desktop`](crates/tinycomputer-desktop/README.md) | desktop apps through the accessibility tree | | [`tinycomputer-browser`](crates/tinycomputer-browser/README.md) | web pages through Chrome, and sight | | [`tinycomputer-cursor`](crates/tinycomputer-cursor/README.md) | the cursor you can watch | +| [`tinycomputer-accessibility`](crates/tinycomputer-accessibility/README.md) | in-process focus, permission and Globe-key answers for a host | | [`tinycomputer-skills`](crates/tinycomputer-skills/README.md) | the guide and schemas for agents that call tasks | | [`tinycomputer-examples`](crates/tinycomputer-examples/README.md) | examples, the lab, saved plans, the journal reader | diff --git a/docs/crates/tinycomputer-accessibility/README.md b/docs/crates/tinycomputer-accessibility/README.md new file mode 100644 index 00000000..ae7896c9 --- /dev/null +++ b/docs/crates/tinycomputer-accessibility/README.md @@ -0,0 +1,33 @@ +# tinycomputer-accessibility + +tinycomputer is a decision model (Jev) based harness for desktop and browser +automation. This crate is a small, separate piece for host applications: the +handful of operating-system answers a host wants **right now, in its own +process**, without going through the module bus. + +A voice-dictation host is the typical user. When the user presses a hotkey it +asks which application and text field is focused, checks that focus has not +moved before it pastes, asks whether it may listen for input events or use the +microphone, and listens for the macOS Globe (Fn) key. Each is a synchronous +call that returns immediately. + +## How it differs from the desktop surface + +| | `tinycomputer-accessibility` | `tinycomputer-desktop` / the module | +|---|---|---| +| Caller | a host, in-process, Rust API | any bus client, typed members | +| Job | OS facts: focus, permissions, Globe key | drive other apps: snapshot, click, type | +| Runtime | none; blocking calls | engine, refs, safety checks | + +## Platforms and features + +macOS does the real work (accessibility APIs and a small Swift helper compiled +on first use). Other platforms answer with typed unsupported errors or states +so a host can call the same API everywhere. Fallible operations use the crate's +`Error` and `Result` types. The `microphone-probe` feature uses `cpal` to +check for an input device, but device enumeration cannot confirm recording +authorization; macOS and Windows therefore report `Unknown` until a host opens +a capture stream. Leave the feature off if the host never records. + +See the [crate README](../../../crates/tinycomputer-accessibility/README.md) for +the module map. diff --git a/docs/project/README.md b/docs/project/README.md index f2950669..19de417b 100644 --- a/docs/project/README.md +++ b/docs/project/README.md @@ -67,6 +67,7 @@ README once its own documentation pass lands: | [`tinycomputer-bus`](../crates/tinycomputer-bus/README.md) | the wire contract: every type that crosses the bus, and the member names, with no runtime dependencies | | [`tinycomputer-core`](../crates/tinycomputer-core/README.md) | shared, engine-free domain logic: the `Surface` trait, keys, safety rules, records and facts | | [`tinycomputer-cursor`](../crates/tinycomputer-cursor/README.md) | the agent's on-screen cursor and the overlay window that draws it | +| [`tinycomputer-accessibility`](../crates/tinycomputer-accessibility/README.md) | in-process focus, permission and Globe-key answers for a host | | [`tinycomputer-desktop`](../crates/tinycomputer-desktop/README.md) | the `agent-desktop` adapter: one method per desktop member, conversion, and the permission preflight | | [`tinycomputer-browser`](../crates/tinycomputer-browser/README.md) | the `agent-browser` adapter: typed sessions over the linked engine | | [`tinycomputer-engine`](../crates/tinycomputer-engine/README.md) | the agent runtime: Jev decision loops, `RunGoal`, intent flows, and the task controller | diff --git a/vendor/agent-browser b/vendor/agent-browser index 9a749e47..cb1ac588 160000 --- a/vendor/agent-browser +++ b/vendor/agent-browser @@ -1 +1 @@ -Subproject commit 9a749e47aa872315db661cb00ff27e909bdce2e4 +Subproject commit cb1ac5885996325eacb31276fa95f5064ce0e12f diff --git a/vendor/tinybus b/vendor/tinybus index e5f1cd2d..df6f990c 160000 --- a/vendor/tinybus +++ b/vendor/tinybus @@ -1 +1 @@ -Subproject commit e5f1cd2d2a7b905ee82410a61219292ca8a85c1f +Subproject commit df6f990cec3b130db12d6cd0c8e24f11c30b28e6 diff --git a/vendor/tinyinference b/vendor/tinyinference index c29d5118..c144d609 160000 --- a/vendor/tinyinference +++ b/vendor/tinyinference @@ -1 +1 @@ -Subproject commit c29d5118eddfa12b83c266d7d5f07f6918dc3787 +Subproject commit c144d609b50cbe64c9af69ac3aa64518cce11c90