diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94e78e0..3385b16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,8 @@ jobs: sudo apt-get update -qq sudo apt-get install -y llvm python3 tools/check-alloc-symbols.py --arch ${{ matrix.arch }} + - name: Link libkore.so and check its exported ABI + run: python3 tools/check-program.py --arch ${{ matrix.arch }} miri: runs-on: ubuntu-latest timeout-minutes: 30 diff --git a/Cargo.lock b/Cargo.lock index df51275..5959e79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -802,6 +802,23 @@ dependencies = [ "kore-window", ] +[[package]] +name = "kore-c" +version = "0.1.0-alpha.1" +dependencies = [ + "kore-c-shim", + "kore-ffi", + "kore-ld", + "kore-std", +] + +[[package]] +name = "kore-c-shim" +version = "0.1.0" +dependencies = [ + "kore-rt", +] + [[package]] name = "kore-collections" version = "0.1.0" @@ -959,6 +976,8 @@ dependencies = [ "kore-dagfs", "kore-driver", "kore-rt", + "kore-svc-vdev", + "kore-vdev", "zeroize", ] @@ -1011,6 +1030,23 @@ dependencies = [ "kore-rt", ] +[[package]] +name = "kore-svc-vdev" +version = "0.0.0" +dependencies = [ + "kore-abi", + "kore-rt", + "kore-vdev", +] + +[[package]] +name = "kore-vdev" +version = "0.0.1" +dependencies = [ + "heapless", + "virtio-accel-transport", +] + [[package]] name = "kore-vol" version = "0.0.0" @@ -1314,9 +1350,9 @@ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustix" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ "bitflags", "errno", @@ -1661,6 +1697,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtio-accel-transport" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7747200a37db419a82c884730d5c31d1d80a6b4d14cc3de312064e4793dc289" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index b69f68b..ed8b435 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/MicroPerceptron/libkore" [workspace] resolver = "3" -members = ["tests/consumer", "crates/ffi", "crates/entry", "crates/wasi", "crates/lua", "crates/host"] +members = ["tests/consumer", "crates/ffi", "crates/c", "crates/entry", "crates/wasi", "crates/lua", "crates/host"] exclude = ["native"] [features] @@ -17,6 +17,7 @@ fs = ["dep:kore-dagfs", "kore-dagfs/runtime"] heap = ["kore-rt/heap"] crt0 = ["kore-rt/crt0"] driver = ["dep:kore-driver", "kore-driver/service"] +vdev = ["dep:kore-svc-vdev", "dep:kore-vdev"] # Byte-exact native export; source revision and hashes live in native/provenance.json. [dependencies] @@ -28,6 +29,10 @@ kore-app = { path = "native/crates/app" } kore-rt = { path = "native/crates/rt", default-features = false } kore-dagfs = { path = "native/crates/dagfs", optional = true } kore-driver = { path = "native/crates/driver", optional = true } +# A driver's side of a virtual device: the service's client without the +# service (no `host`), and the register block and memory views the two share. +kore-svc-vdev = { path = "native/services/vdev", default-features = false, optional = true } +kore-vdev = { path = "native/crates/vdev", optional = true } [package.metadata.kore] # Used by tools/local-kore.py to construct one coherent local source override, @@ -35,4 +40,4 @@ kore-driver = { path = "native/crates/driver", optional = true } # root: the offline volume procedure (and the filesystem service closure it # drives) that host programs such as vKore consume; it never enters a Kore # executable's graph. -owner-paths = { kore-start = "crates/start", kore-abi = "crates/abi", kore-app = "crates/app", kore-rt = "crates/rt", kore-dagfs = "crates/dagfs", kore-driver = "crates/driver", kore-late = "crates/late", kore-vol = "tools/kore-vol" } +owner-paths = { kore-start = "crates/start", kore-abi = "crates/abi", kore-app = "crates/app", kore-rt = "crates/rt", kore-c-shim = "crates/c-shim", kore-dagfs = "crates/dagfs", kore-driver = "crates/driver", kore-svc-vdev = "services/vdev", kore-vdev = "crates/vdev", kore-late = "crates/late", kore-ld = "crates/ld", kore-vol = "tools/kore-vol" } diff --git a/Makefile b/Makefile index ff12454..99d83ac 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ check: python3 tools/check-native.py fmt: - cargo fmt -p kore-std -p libkore-consumer -p kore-ffi -p kore-entry -p kore-wasi -p kore-lua -p kore-host --check + cargo fmt -p kore-std -p libkore-consumer -p kore-ffi -p kore-c -p kore-entry -p kore-wasi -p kore-lua -p kore-host --check $(ZIG) fmt --check build.zig build.zig.zon crates/ffi/tests/consumer.zig extern/zig/src/root.zig extern/zig/src/allocator.zig docs: @@ -30,7 +30,9 @@ owners: python3 tools/check-owners.py < target/owners.json # Actual Kore user targets, including its hardware-float x86 ABI. A stock -# x86_64-unknown-none check alone cannot certify this interface. `kore-lua` +# x86_64-unknown-none check alone cannot certify this interface. `kore-c` is +# linked rather than checked (`tools/check-program.py`): `libkore.so`'s +# exported ABI, soname and TLS model are properties of the linked image. `kore-lua` # is here because its engine exists only on these targets — its host build # carries the logic and none of the C — so a Kore-only regression in the # amalgamation would otherwise reach a keytest before it reached a @@ -46,6 +48,7 @@ cross: -Z build-std-features=compiler-builtins-mem; \ done python3 tools/check-alloc-symbols.py + python3 tools/check-program.py python3 tools/check-entry.py # kore-host drives real image files under POSIX record locks, which Miri does diff --git a/README.md b/README.md index caae6f0..4476d13 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,13 @@ machine's volume through the same exported owners a booted Kore runs. | --- | --- | | `kore-std` | Rust facade: filesystem, prelude, time, random, collections, process and optional driver APIs | | `kore-entry` | Executable-only adapter for a returning Rust `fn main()` on Kore | -| `kore-ffi` | C headers and Zig module over a Rust-founded host, including filesystem, time, random, allocation and worker APIs; no standalone C/Zig CRT | +| `kore-ffi` | C headers and Zig module over a Rust-founded host, including filesystem, time, random, allocation and worker APIs | +| `kore-c` | The runtime of `libkore.so`: `kore.h` as one shared object, and the host of an independent C program (`main` on a worker, its filesystem session, its end) | | `kore-lua` | Embeddable Lua 5.5 engine and bridge; interactive and opt-in restricted worker policies | | `kore-wasi` | WASI 0.2 component host composed by an embedder with its Wasmtime engine and native owners | | `kore-host` | Host-side (Linux/macOS) tooling: a Kore machine's DagFS volume, offline, through the filesystem row's own engine | -All six package manifests set `publish = false`; consume a reviewed Git +All seven package manifests set `publish = false`; consume a reviewed Git revision or a deliberately composed local checkout, not a registry version. The WASI host supports the `wasi:cli/command` world and native-backed I/O, filesystem, clocks and random imports; unsupported imports such as sockets diff --git a/crates/c/Cargo.toml b/crates/c/Cargo.toml new file mode 100644 index 0000000..ca3fa09 --- /dev/null +++ b/crates/c/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "kore-c" +version = "0.1.0-alpha.1" +edition = "2024" +publish = false +description = "The runtime of libkore.so: the C ABI and the host of an independent C program on Kore" +# No native library is linked: the key carries the headers this runtime +# implements to dependents' build scripts as `DEP_KORE_INCLUDE` (`build.rs`). +links = "kore" +build = "build.rs" + +# The runtime `libkore.so` is built from: a distribution's one-line `cdylib` +# crate names this one and links it under its own `[lib] name = "kore"`, so +# the image is `libkore.so` (Kore's is `user/runtime/libkore`). Link it into +# that shared object alone: it installs the process heap and the panic handler +# a C program's runtime needs, which a Rust executable brings itself. +[lib] +name = "kore_c" +path = "src/lib.rs" + +[dependencies] +kore-std = { package = "kore-std", path = "../..", features = ["heap", "fs", "vdev"] } +# The C ABI itself, with the standard C allocation and thread names a C +# program calls (`malloc`, `pthread_create`, ...) exported as real symbols. +kore-ffi = { path = "../ffi", features = ["c-stdlib"] } +# Kore's C library: the `printf` family and the standard streams, `` +# and ``'s parsers, `errno`. Its process half without its own +# `malloc` and `abort`: kore-ffi's allocator is this image's `malloc`, and the +# host ends a whole program. +kore-c-shim = { path = "../../native/crates/c-shim", default-features = false, features = ["runtime"] } +# The link window's placer: one cursor every `mmap` of the process takes its +# address from, as a Wasmtime engine's guest mappings do. +kore-ld = { path = "../../native/crates/ld", default-features = false } diff --git a/crates/c/build.rs b/crates/c/build.rs new file mode 100644 index 0000000..6928f56 --- /dev/null +++ b/crates/c/build.rs @@ -0,0 +1,29 @@ +//! Publish the C headers `libkore.so` implements to whoever links this crate +//! (`links = "kore"`): a dependent's build script reads `DEP_KORE_INCLUDE`, the +//! directory holding `kore.h` and `kore/`, from this exact source revision, so +//! the headers a distribution ships cannot drift from the image it builds. +use std::path::PathBuf; + +fn main() { + let manifest = + PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); + let include = manifest.join("../../extern/c/include"); + let include = include + .canonicalize() + .unwrap_or_else(|error| panic!("{}: {error}", include.display())); + for header in [ + "kore.h", + "kore/stdlib.h", + "kore/pthread.h", + "kore/program.h", + ] { + let path = include.join(header); + assert!( + path.is_file(), + "missing generated header {}", + path.display() + ); + println!("cargo:rerun-if-changed={}", path.display()); + } + println!("cargo:include={}", include.display()); +} diff --git a/crates/c/cbindgen.toml b/crates/c/cbindgen.toml new file mode 100644 index 0000000..c5f2d2d --- /dev/null +++ b/crates/c/cbindgen.toml @@ -0,0 +1,26 @@ +language = "C" +include_guard = "KORE_PROGRAM_H" +pragma_once = true +cpp_compat = true +usize_is_size_t = true +style = "both" +no_includes = true +includes = ["../kore.h"] +autogen_warning = "/* Generated by cbindgen 0.29.4 from kore-c. Do not edit. */" +header = "/* libkore.so: what an independent C program on Kore adds to kore.h. Link with -lkore. */" +[parse] +parse_deps = false +[export] +# The C library's half of the program lifecycle (its start and exit call +# these), not an application API: the C library declares them itself, as +# its declares `abort`. `KoreFsStep` is kore.h's typedef, which +# this header includes. +# `open`, `close`, `read`, `write`, `poll`, `mmap` and `munmap` are POSIX's, +# declared by , , and ; so is +# `struct pollfd`. +exclude = ["__kore_program_start", "__kore_program_exit", "__kore_program_arguments", "abort", "open", "close", "read", "write", "poll", "PollFd", "mmap", "munmap", "KoreFsStep"] +# No function takes the register block: a program maps it. +include = ["KoreVdevRegisters"] +[enum] +prefix_with_name = true +rename_variants = "ScreamingSnakeCase" diff --git a/crates/c/src/api.rs b/crates/c/src/api.rs new file mode 100644 index 0000000..a4d97b8 --- /dev/null +++ b/crates/c/src/api.rs @@ -0,0 +1,409 @@ +//! The C entry points `kore/program.h` declares: what an independent C program +//! on Kore adds to `kore.h` by linking `libkore.so`. +//! +//! Compiled on every target so the header has one source of declarations; +//! off Kore there is no program to host, and each entry refuses. + +use core::ffi::c_void; + +use kore_ffi::{KoreFsError, KoreStatus}; + +/// `kore.h`'s `KoreFsStep` as a C caller may pass it: possibly null. +pub type KoreFsStep = Option; + +/// The stack `main` runs on: a guarded strip of the task's stack window, +/// committed whole when the program starts. A frame deeper than this faults +/// on the guard page below it, as a stack overflow does. +/// +/// The size of the strip `tcc`, a large C program whose depth its input +/// decides, measured its need under Kore's runtime at and runs in; ISO C's +/// own guarantee of 127 nested blocks (C11 §5.2.4.1) needs a small part of +/// it. Larger strips come out of the 16 MiB window the program's SPMD gangs +/// share, and would be committed at every launch. +pub const KORE_MAIN_STACK_BYTES: usize = 1 << 20; + +/// Run `step` as this program's filesystem session, through the filesystem +/// face its launch carried, rooted where it was launched (its working +/// directory). The callbacks and their pointers obey `KoreFsStep`'s contract: +/// each callback runs on the program's host thread, as on any native +/// executor, while the caller waits, so a callback must not depend on the +/// caller's thread-local state. +/// +/// Returns `KORE_STATUS_OK` once the callback program ended and every view it +/// left open closed. `KORE_STATUS_QUARANTINED`: a reply was lost or +/// uncertain, and no session can run again in this program. +/// `KORE_STATUS_REFUSED`: a close was refused (`*error` names why, and the +/// view stays open), the launch carried no filesystem face +/// (`KORE_FS_ERROR_UNSUPPORTED`), or the caller is not the thread `main` runs +/// on, which alone waits for the host. `KORE_STATUS_CAPACITY`: the native +/// owner could not be formed. `KORE_STATUS_INVALID_ARGUMENT`: `step` is null. +/// `*error` is written whenever `error` is not null. +/// +/// # Safety +/// `step` obeys `KoreFsStep`'s pointer, serialization and no-unwind contract; +/// `user` remains valid, and the callbacks' alone, until this returns. `error` +/// is null or valid for one aligned write. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn kore_fs_run( + step: KoreFsStep, + user: *mut c_void, + error: *mut KoreFsError, +) -> KoreStatus { + let (status, cause) = match step { + None => (KoreStatus::InvalidArgument, KoreFsError::None), + // SAFETY: the caller's contract, forwarded whole. + #[cfg(target_os = "none")] + Some(step) => unsafe { crate::program::session(step, user) }, + // No program to host off Kore. + #[cfg(not(target_os = "none"))] + Some(_) => { + let _ = user; + (KoreStatus::Refused, KoreFsError::None) + } + }; + if !error.is_null() { + // SAFETY: the caller's contract: null or valid for one aligned write. + unsafe { error.write(cause) }; + } + status +} + +/// [`KoreFdInfo::kind`]: the memory supply the program's allocator grows +/// on. `mmap` answers a fresh, zeroed region of it. +pub const KORE_FD_MEMORY: u32 = 1; +/// [`KoreFdInfo::kind`]: a virtual device (``). `mmap` answers +/// its register block and memory to lend it, `read` its interrupt, and +/// `write` rings its doorbell. +pub const KORE_FD_DEVICE: u32 = 2; +/// [`KoreFdInfo::access`]: the capability may be read. +pub const KORE_FD_READ: u32 = 1 << 0; +/// [`KoreFdInfo::access`]: the capability may be written. +pub const KORE_FD_WRITE: u32 = 1 << 1; + +/// Bytes in a virtual device's register block: one page. +pub const KORE_VDEV_REGISTERS_BYTES: usize = 4096; +/// `mmap` offset on a virtual device's descriptor: the register block +/// ([`KoreVdevRegisters`]). +pub const KORE_VDEV_MAP_REGISTERS: i64 = 0; +/// `mmap` offset on a virtual device's descriptor: fresh, zeroed memory the +/// driver owns and lends the device. A device address is an offset into it. +/// The device takes a loan only in its reset state, so map it before the +/// first status write. +pub const KORE_VDEV_MAP_MEMORY: i64 = 4096; +/// [`KoreVdevRegisters::magic`]: `"kvdv"`, a Kore virtual device. +pub const KORE_VDEV_MAGIC: u32 = 0x7664_766b; +/// [`KoreVdevRegisters::version`]: this layout. +pub const KORE_VDEV_VERSION: u32 = 1; +/// [`KoreVdevRegisters::vendor_id`]: `"kore"`, a device Kore hosts. +pub const KORE_VDEV_VENDOR_KORE: u32 = 0x6572_6f6b; +/// [`KoreVdevRegisters::device_id`]: `"accl"`, the virtual accelerator, a +/// virtio-accel 1.0 device. Its name is `/kore/device/virtual/accel`. +pub const KORE_VDEV_ID_ACCEL: u32 = 0x6c63_6361; +/// Status: the driver has noticed the device (Virtio 1.3 section 2.1). +pub const KORE_VDEV_STATUS_ACKNOWLEDGE: u32 = 1; +/// Status: the driver knows how to drive it. +pub const KORE_VDEV_STATUS_DRIVER: u32 = 2; +/// Status: the driver is set up and the device may serve. +pub const KORE_VDEV_STATUS_DRIVER_OK: u32 = 4; +/// Status: feature negotiation is complete. +pub const KORE_VDEV_STATUS_FEATURES_OK: u32 = 8; +/// Status, written by the device: it cannot go on and must be reset. +pub const KORE_VDEV_STATUS_DEVICE_NEEDS_RESET: u32 = 64; +/// Status: the driver has given up on the device. +pub const KORE_VDEV_STATUS_FAILED: u32 = 128; +/// Feature bit 32, `VIRTIO_F_VERSION_1`: every device offers it and a +/// driver must accept it. Written out, because the header carries this +/// expression as it stands and `1 << 32` is not a 64-bit value in C. +pub const KORE_VDEV_F_VERSION_1: u64 = 0x1_0000_0000; +/// Bytes between the last register and the device configuration. +pub const KORE_VDEV_RESERVED_BYTES: usize = 168; +/// Bytes of the device configuration area, to the end of the block. +pub const KORE_VDEV_CONFIG_AREA_BYTES: usize = 3840; + +/// The register block of a Kore virtual device: what `mmap` at +/// `KORE_VDEV_MAP_REGISTERS` answers, declared `volatile` by a driver. +/// +/// Every word is little-endian and has one writer. A driver never writes a +/// device register, and what it writes to its own the device reads once, at +/// the step that uses it, when the driver rings the doorbell (a `write` of +/// one 32-bit word to the descriptor). The device answers by raising an +/// interrupt: it increments `interrupts`, and a `read` of one 32-bit word +/// from the descriptor returns once the count has moved. +/// +/// Bringing the device up is Virtio 1.3 section 3.1.1 over these registers, +/// with the memory lent first (`KORE_VDEV_MAP_MEMORY`). A device just opened +/// is in its reset state, both status words zero; one that is not is reset +/// first, by writing 0 to `driver_status` and ringing. Then write +/// `ACKNOWLEDGE | DRIVER` and ring; write the accepted features, add +/// `FEATURES_OK`, ring, and check `device_status` carries it; lay the queue +/// out in the lent memory, write `queue_num`, the three addresses and +/// `queue_ready`; add `DRIVER_OK` and ring. A new value of `driver_status` +/// is always answered with an interrupt, and a step the device refused is +/// one its `device_status` does not carry. The value the device last saw +/// announces nothing and is not answered, so a driver waits only for what +/// it asked. +#[repr(C)] +pub struct KoreVdevRegisters { + /// Device: `KORE_VDEV_MAGIC`. + pub magic: u32, + /// Device: `KORE_VDEV_VERSION`. + pub version: u32, + /// Device: which model this is (`KORE_VDEV_ID_ACCEL`). + pub device_id: u32, + /// Device: `KORE_VDEV_VENDOR_KORE`. + pub vendor_id: u32, + /// Device: the features it offers. + pub device_features: u64, + /// Driver: the features it accepts, a subset of the offer. + pub driver_features: u64, + /// Driver: the status it has reached (`KORE_VDEV_STATUS_*`). + pub driver_status: u32, + /// Device: the status it has accepted, and its own + /// `KORE_VDEV_STATUS_DEVICE_NEEDS_RESET`. + pub device_status: u32, + /// Device: the largest queue it takes. + pub queue_num_max: u32, + /// Driver: the queue's size, a power of two between the longest chain + /// the device takes (its configuration says how long) and + /// `queue_num_max`. + pub queue_num: u32, + /// Driver: nonzero once the queue's size and addresses are written. + pub queue_ready: u32, + /// Device: changes whenever the configuration area does. + pub config_generation: u32, + /// Driver: the descriptor table's device address. + pub queue_desc: u64, + /// Driver: the available ring's device address. + pub queue_driver: u64, + /// Driver: the used ring's device address. + pub queue_device: u64, + /// Device: bytes of device configuration in `config`. + pub config_bytes: u32, + /// Device: interrupts raised, wrapping. + pub interrupts: u32, + /// Unused. + pub reserved: [u8; KORE_VDEV_RESERVED_BYTES], + /// Device: the device configuration. + pub config: [u8; KORE_VDEV_CONFIG_AREA_BYTES], +} + +// The declaration is `kore-vdev`'s register block, offset for offset, and +// the constants are its constants: a drift fails this build. +const _: () = { + use core::mem::{offset_of, size_of}; + use kore_std::vdev::regs; + macro_rules! at { + ($($field:ident = $offset:expr),* $(,)?) => { + $(assert!(offset_of!(KoreVdevRegisters, $field) as u64 == $offset);)* + }; + } + at! { + magic = regs::MAGIC, + version = regs::VERSION, + device_id = regs::DEVICE_ID, + vendor_id = regs::VENDOR_ID, + device_features = regs::DEVICE_FEATURES, + driver_features = regs::DRIVER_FEATURES, + driver_status = regs::DRIVER_STATUS, + device_status = regs::DEVICE_STATUS, + queue_num_max = regs::QUEUE_NUM_MAX, + queue_num = regs::QUEUE_NUM, + queue_ready = regs::QUEUE_READY, + config_generation = regs::CONFIG_GENERATION, + queue_desc = regs::QUEUE_DESC, + queue_driver = regs::QUEUE_DRIVER, + queue_device = regs::QUEUE_DEVICE, + config_bytes = regs::CONFIG_BYTES, + interrupts = regs::INTERRUPTS, + config = regs::CONFIG, + } + assert!(size_of::() as u64 == regs::BYTES); + assert!(KORE_VDEV_REGISTERS_BYTES as u64 == regs::BYTES); + assert!(KORE_VDEV_MAP_REGISTERS == crate::device::MAP_REGISTERS); + assert!(KORE_VDEV_MAP_MEMORY == crate::device::MAP_MEMORY); + assert!(KORE_VDEV_MAGIC == regs::MAGIC_VALUE); + assert!(KORE_VDEV_VERSION == regs::VERSION_VALUE); + assert!(KORE_VDEV_VENDOR_KORE == regs::VENDOR_KORE); + assert!(KORE_VDEV_ID_ACCEL == crate::device::MODEL_ACCEL); + assert!(KORE_VDEV_STATUS_ACKNOWLEDGE == regs::STATUS_ACKNOWLEDGE); + assert!(KORE_VDEV_STATUS_DRIVER == regs::STATUS_DRIVER); + assert!(KORE_VDEV_STATUS_DRIVER_OK == regs::STATUS_DRIVER_OK); + assert!(KORE_VDEV_STATUS_FEATURES_OK == regs::STATUS_FEATURES_OK); + assert!(KORE_VDEV_STATUS_DEVICE_NEEDS_RESET == regs::STATUS_DEVICE_NEEDS_RESET); + assert!(KORE_VDEV_STATUS_FAILED == regs::STATUS_FAILED); + assert!(KORE_VDEV_F_VERSION_1 == regs::F_VERSION_1); +}; + +/// What a descriptor names, as the calling thread holds it +/// ([`kore_fd_describe`]). +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct KoreFdInfo { + /// What kind of capability it is: `KORE_FD_MEMORY` or `KORE_FD_DEVICE`. + pub kind: u32, + /// What the capability's own rights permit this thread, as + /// `KORE_FD_READ | KORE_FD_WRITE` bits: the rights it was founded with, + /// which the access `open` asked for can only narrow. + pub access: u32, + /// For `KORE_FD_MEMORY`, the supply's whole length in bytes: what the + /// allocator and every mapping share, not what is still free. For + /// `KORE_FD_DEVICE`, the register block's length. + pub length: u64, +} + +/// Describe the capability behind `fd`, a descriptor `open` answered +/// (``): `open` finds a capability this program was founded holding +/// by its name, and this reads what was found. Nothing is granted or changed. +/// +/// Capabilities are held per thread, so the answer is the calling thread's: +/// the same descriptor may describe a capability on the thread that opened +/// it and be refused on one founded without it. +/// +/// Returns `KORE_STATUS_OK` with `*out` written. +/// `KORE_STATUS_INVALID_ARGUMENT`: `out` is null. `KORE_STATUS_STALE_HANDLE`: +/// `fd` is not an open descriptor. `KORE_STATUS_REFUSED`: the calling thread +/// holds no such capability, or this is not a Kore program. A refusal leaves +/// `*out` unchanged. +/// +/// # Safety +/// `out` is null or valid for one aligned write. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn kore_fd_describe( + fd: core::ffi::c_int, + out: *mut KoreFdInfo, +) -> KoreStatus { + if out.is_null() { + return KoreStatus::InvalidArgument; + } + let Ok((row, _)) = crate::fd::TABLE.get(fd) else { + return KoreStatus::StaleHandle; + }; + let Some(info) = describe(&crate::fd::NAMES[row]) else { + return KoreStatus::Refused; + }; + // SAFETY: the caller's contract: valid for one aligned write. + unsafe { out.write(info) }; + KoreStatus::Ok +} + +/// What the calling thread holds at `name`. +#[cfg(target_os = "none")] +fn describe(name: &crate::fd::Name) -> Option { + use crate::fd::Kind; + let held = crate::fd::holding(name)?; + let (kind, length) = match name.kind { + Kind::Memory => ( + KORE_FD_MEMORY, + kore_std::runtime::cap::verbs::lease_geometry(held.slot)?.1, + ), + Kind::Device { .. } => (KORE_FD_DEVICE, kore_std::vdev::regs::BYTES), + }; + Some(KoreFdInfo { + kind, + access: if held.readable { KORE_FD_READ } else { 0 } + | if held.writable { KORE_FD_WRITE } else { 0 }, + length, + }) +} + +/// Off Kore no thread holds a capability. +#[cfg(not(target_os = "none"))] +fn describe(_name: &crate::fd::Name) -> Option { + None +} + +#[cfg(test)] +mod tests { + use super::*; + + unsafe extern "C" fn never( + _: *mut kore_ffi::KoreFsContext, + _: *const kore_ffi::KoreFsCompletion, + _: *mut c_void, + ) { + panic!("a session ran with no program to host"); + } + + /// The headers carry a constant's expression as Rust wrote it, and C + /// reads `1 << n` as an `int`: a shift that does not fit one is not the + /// value Rust meant. `KORE_VDEV_F_VERSION_1` was `(1 << 32)` once, which + /// a driver compiled as zero. + #[test] + fn no_header_constant_shifts_past_an_int() { + for (name, header) in [ + ( + "kore/program.h", + include_str!("../../../extern/c/include/kore/program.h"), + ), + ("kore.h", include_str!("../../../extern/c/include/kore.h")), + ] { + for line in header.lines().filter(|line| line.starts_with("#define ")) { + let mut rest = line; + while let Some(at) = rest.find("<<") { + rest = &rest[at + 2..]; + let digits: std::string::String = rest + .trim_start() + .chars() + .take_while(char::is_ascii_digit) + .collect(); + let shift: u32 = digits.parse().unwrap_or(u32::MAX); + assert!(shift < 31, "{name}: `{line}` shifts an int by {digits}"); + } + } + } + } + + #[test] + fn a_null_step_is_an_invalid_argument_and_the_error_is_written() { + let mut error = KoreFsError::Busy; + // SAFETY: a null step names no callback; `error` is valid. + let status = unsafe { kore_fs_run(None, core::ptr::null_mut(), &mut error) }; + assert_eq!(status, KoreStatus::InvalidArgument); + assert_eq!(error, KoreFsError::None); + } + + #[test] + fn with_no_program_to_host_a_session_is_refused_and_never_runs() { + let mut error = KoreFsError::Busy; + // SAFETY: `never` is never called; `error` is valid. + let status = unsafe { kore_fs_run(Some(never), core::ptr::null_mut(), &mut error) }; + assert_eq!(status, KoreStatus::Refused); + assert_eq!(error, KoreFsError::None); + // A null error pointer is permitted. + // SAFETY: as above. + let status = + unsafe { kore_fs_run(Some(never), core::ptr::null_mut(), core::ptr::null_mut()) }; + assert_eq!(status, KoreStatus::Refused); + } + + #[test] + fn describing_refuses_by_cause_and_leaves_the_answer_alone() { + let untouched = KoreFdInfo { + kind: 7, + access: 7, + length: 7, + }; + let mut info = untouched; + // SAFETY: null is permitted; `info` is valid. + unsafe { + assert_eq!( + kore_fd_describe(3, core::ptr::null_mut()), + KoreStatus::InvalidArgument + ); + // The streams' numbers and an unopened one are not descriptors. + for fd in [0, 1, 2, -1, 63, 1 << 20] { + assert_eq!(kore_fd_describe(fd, &mut info), KoreStatus::StaleHandle); + } + } + // An open descriptor, off Kore: no thread holds a capability. + let access = crate::fd::Access::of(crate::fd::O_RDWR).unwrap(); + let fd = crate::fd::TABLE.claim(0, access).unwrap(); + // SAFETY: `info` is valid. + assert_eq!( + unsafe { kore_fd_describe(fd, &mut info) }, + KoreStatus::Refused + ); + crate::fd::TABLE.release(fd).unwrap(); + assert_eq!(info, untouched); + } +} diff --git a/crates/c/src/args.rs b/crates/c/src/args.rs new file mode 100644 index 0000000..693bf2e --- /dev/null +++ b/crates/c/src/args.rs @@ -0,0 +1,247 @@ +//! **A C program's arguments**: `main`'s `argc` and `argv`. +//! +//! A launcher hands a program its command line on the process page: the name +//! it resolved the program by ([`ProcessIo::command`]) and the opaque tail +//! after it ([`ProcessIo::args`]). C wants a vector. The host reads the page +//! once, on thread 0 before the main thread exists, splits the tail the way +//! every kore program that wants words does ([`words`]: a POSIX shell's +//! quoting, without expansion), and keeps the vector for the life of the +//! program. The C library's start asks for it through +//! [`__kore_program_arguments`] and passes it to `main`. +//! +//! `argv[0]` is the command name, or the empty string when the launcher gave +//! none (C11 §5.1.2.2.1p2). A launch with no page at all has no arguments, +//! not even a name: `argc` is zero, which the same clause permits. The +//! strings are the program's to change, as that clause requires, and +//! `argv[argc]` is null. +//! +//! A tail that does not split, an unclosed quote or a lone trailing +//! backslash, is the launcher's line being malformed, not something `main` +//! could be handed: the host says so on `stderr` and ends the program before +//! it runs, as `wasm` and `tcc` answer the same line. +//! +//! [`ProcessIo::command`]: kore_std::abi::process::ProcessIo::command +//! [`ProcessIo::args`]: kore_std::abi::process::ProcessIo::args +//! [`words`]: kore_std::process::words + +extern crate alloc; + +use alloc::vec::Vec; + +use kore_std::process::words::{self, WordsError}; + +/// An argument vector's storage: every word NUL-terminated, back to back, +/// and where each begins. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Vector { + bytes: Vec, + starts: Vec, +} + +impl Vector { + /// `command`, then the words of `tail`. + /// + /// A NUL inside a word would end the C string early and hand `main` a + /// different argument than the line spelled, so such a line is no + /// vector either. + pub(crate) fn of(command: &[u8], tail: &[u8]) -> Result { + let mut vector = Self { + bytes: Vec::new(), + starts: Vec::new(), + }; + vector.push(command.iter().copied())?; + for word in words::words(tail) { + vector.push(word.map_err(Malformed::Words)?.bytes())?; + } + Ok(vector) + } + + fn push(&mut self, word: impl Iterator) -> Result<(), Malformed> { + self.starts.push(self.bytes.len()); + for byte in word { + if byte == 0 { + return Err(Malformed::Nul { + word: self.starts.len() - 1, + }); + } + self.bytes.push(byte); + } + self.bytes.push(0); + Ok(()) + } + + /// Words held: `argc`. + pub(crate) fn len(&self) -> usize { + self.starts.len() + } + + /// Word `index`, without its terminator. + #[cfg(test)] + fn word(&self, index: usize) -> &[u8] { + let start = self.starts[index]; + let end = self.bytes[start..] + .iter() + .position(|byte| *byte == 0) + .unwrap(); + &self.bytes[start..start + end] + } +} + +/// Why a command line is no argument vector. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Malformed { + /// The tail does not split into words. + Words(WordsError), + /// Word `word` (0 is the command name) holds a NUL byte. + Nul { word: usize }, +} + +impl core::fmt::Display for Malformed { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Words(error) => error.fmt(f), + Self::Nul { word } => write!(f, "argument {word} holds a NUL byte"), + } + } +} + +#[cfg(target_os = "none")] +pub(crate) use metal::capture; + +#[cfg(target_os = "none")] +mod metal { + use super::alloc::boxed::Box; + use super::alloc::vec::Vec; + use super::{Malformed, Vector}; + use core::ffi::{c_char, c_int}; + use core::sync::atomic::{AtomicI32, AtomicPtr, Ordering}; + use kore_std::runtime::entry::Rt; + + /// `argc`, and the vector `argv` points at: `argc` pointers and a null. + /// Published once by [`capture`] on thread 0 before the main thread is + /// founded, whose founding orders these stores before `main`'s loads. + static ARGC: AtomicI32 = AtomicI32::new(0); + static ARGV: AtomicPtr<*mut c_char> = AtomicPtr::new(core::ptr::null_mut()); + + /// Read the launcher's page and keep the program's argument vector. + /// Thread 0, once, before any other thread exists. A launch that carried + /// no page leaves the program without arguments. + pub(crate) fn capture(rt: &Rt) -> Result<(), Malformed> { + let Some(page) = kore_std::process::map(rt) else { + return Ok(()); + }; + // A page that does not validate is no command line: no arguments, + // as for a launch without one. + let (Ok(command), Ok(tail)) = (page.command(), page.args()) else { + return Ok(()); + }; + let Vector { bytes, starts } = Vector::of(command, tail)?; + // The program's for its whole life, and its to change: never freed. + let bytes: &'static mut [u8] = Box::leak(bytes.into_boxed_slice()); + let base = bytes.as_mut_ptr(); + let mut pointers: Vec<*mut c_char> = starts + .iter() + // SAFETY: each start is an offset inside `bytes`. + .map(|start| unsafe { base.add(*start) }.cast::()) + .collect(); + pointers.push(core::ptr::null_mut()); + let argc = c_int::try_from(starts.len()).unwrap_or(c_int::MAX); + let pointers: &'static mut [*mut c_char] = Box::leak(pointers.into_boxed_slice()); + ARGV.store(pointers.as_mut_ptr(), Ordering::Release); + ARGC.store(argc, Ordering::Release); + Ok(()) + } + + /// The program's argument vector: answers `argc` and stores `argv` + /// through `argv`, `argv[argc]` null. The C library's start calls it on + /// the main thread before `main`. A program whose launch carried no + /// command line gets zero and a vector holding only the null. + /// + /// # Safety + /// `argv` is null or names one writable `char **`. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn __kore_program_arguments(argv: *mut *mut *mut c_char) -> c_int { + /// The vector of a program without arguments: its null. + struct Empty(core::cell::UnsafeCell<[*mut c_char; 1]>); + // SAFETY: never written: C may not store through `argv[argc]`'s + // slot, and there is no other. + unsafe impl Sync for Empty {} + static EMPTY: Empty = Empty(core::cell::UnsafeCell::new([core::ptr::null_mut()])); + + let held = ARGV.load(Ordering::Acquire); + let (count, vector) = if held.is_null() { + (0, EMPTY.0.get().cast::<*mut c_char>()) + } else { + (ARGC.load(Ordering::Acquire), held) + }; + if !argv.is_null() { + // SAFETY: the caller's contract. + unsafe { argv.write(vector) }; + } + count + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn vector(command: &[u8], tail: &[u8]) -> Vec> { + let vector = Vector::of(command, tail).unwrap(); + (0..vector.len()) + .map(|index| vector.word(index).to_vec()) + .collect() + } + + #[test] + fn the_command_is_the_first_word_and_the_tail_follows_as_a_shell_splits_it() { + assert_eq!(vector(b"answer", b""), [b"answer".to_vec()]); + assert_eq!( + vector(b"copy", b" a.txt b.txt "), + [b"copy".to_vec(), b"a.txt".to_vec(), b"b.txt".to_vec()] + ); + assert_eq!( + vector(b"say", br#"-c 'two words' "a \"quote\"" back\ slash """#), + [ + b"say".to_vec(), + b"-c".to_vec(), + b"two words".to_vec(), + br#"a "quote""#.to_vec(), + b"back slash".to_vec(), + b"".to_vec(), + ] + ); + // A launcher that names nothing still gives `argv[0]`, empty. + assert_eq!(vector(b"", b"x"), [b"".to_vec(), b"x".to_vec()]); + } + + #[test] + fn every_word_is_its_own_terminated_string() { + let vector = Vector::of(b"p", b"ab c").unwrap(); + assert_eq!(vector.bytes, b"p\0ab\0c\0"); + assert_eq!(vector.starts, [0, 2, 5]); + assert_eq!(vector.len(), 3); + } + + #[test] + fn a_line_that_does_not_split_is_no_vector() { + assert_eq!( + Vector::of(b"p", b"'open"), + Err(Malformed::Words(WordsError::Unterminated { + quote: b'\'', + at: 0 + })) + ); + assert_eq!( + Vector::of(b"p", b"a \\"), + Err(Malformed::Words(WordsError::TrailingBackslash)) + ); + // A NUL would end the C string early: `a\0b` is not the word `a`. + assert_eq!(Vector::of(b"p", b"x a\0b"), Err(Malformed::Nul { word: 2 })); + assert_eq!(Vector::of(b"p\0q", b""), Err(Malformed::Nul { word: 0 })); + let mut text = alloc::string::String::new(); + core::fmt::Write::write_fmt(&mut text, format_args!("{}", Malformed::Nul { word: 2 })) + .unwrap(); + assert_eq!(text, "argument 2 holds a NUL byte"); + } +} diff --git a/crates/c/src/device.rs b/crates/c/src/device.rs new file mode 100644 index 0000000..4de9b3d --- /dev/null +++ b/crates/c/src/device.rs @@ -0,0 +1,883 @@ +//! A virtual device behind a descriptor. +//! +//! A program launched with Kore's virtual-device profile has a face with the +//! virtual-device service: a device with no hardware behind it, for a driver +//! to be written against (Kore's `res/VIRTUAL_DEVICE.md`). This module is +//! C's spelling for holding one, in the shape Linux's UIO gives a userspace +//! driver: +//! +//! - **`open`** a name of the device model. The service starts the model and +//! writes its register block. One descriptor at a time: the face is one +//! device's. +//! - **`mmap` at offset 0** answers the register block, one page the program +//! and the device both write. +//! - **`mmap` at offset one page** answers fresh, zeroed memory the driver +//! owns and lends the device: a device address is an offset into it. One +//! such region at a time, and it cannot be unmapped while the device is +//! open, because the device may be using it. +//! - **`read`** of one 32-bit word waits for an interrupt and answers the +//! interrupt count. Each interrupt is read once. +//! - **`write`** of one 32-bit word rings the doorbell. The word is not read. +//! - **`poll`** says whether an interrupt is waiting to be read, with a +//! timeout. +//! - **`close`** gives the device up. The service drops the model and its +//! view of the lent memory, which stays the program's until it is unmapped. +//! +//! # Whose capability does what +//! +//! Capabilities are per thread. Thread 0 was launched holding the face, so +//! everything that is a request to the service (open, lend, close) is posted +//! to it and waited for, as a mapping of the memory supply is. The interrupt +//! and the doorbell are the hot path and never cross a thread: this library +//! founds the program's threads holding their own share of the face +//! ([`shares`]), so `read` parks the calling thread in the kernel on the +//! interrupt count and `write` posts its notice directly. A thread founded +//! without a share is refused with `ENXIO`, whoever opened the descriptor. +//! +//! The request rules, the refusals and the state a descriptor's words keep +//! are plain data, so the host suite exercises the code the target runs; the +//! face is the metal half. + +use core::ffi::c_int; +use core::sync::atomic::{AtomicU32, Ordering}; + +use kore_std::vdev::client::process_face; +use kore_std::vdev::wire::VdevRefusal; + +use crate::fd::errno; +use crate::mmap::{MAP_FIXED, MAP_SHARED, PAGE, PROT_READ, PROT_WRITE, pages}; + +/// The face's instance in `role::SVC_RING`: the coordinate a device name +/// stands for. +pub(crate) const FACE: u16 = process_face::CONTROL_RING; +/// The face's kick in `role::SVC_SEND`: what a thread rings the doorbell on. +pub(crate) const KICK: u16 = process_face::REQUEST_KICK; + +/// The virtual accelerator's register-block id, `"accl"`. +pub(crate) const MODEL_ACCEL: u32 = u32::from_le_bytes(*b"accl"); + +/// `mmap` offsets on a device descriptor, UIO's convention: map `N` is at +/// offset `N` pages. +pub(crate) const MAP_REGISTERS: i64 = 0; +pub(crate) const MAP_MEMORY: i64 = PAGE as i64; + +/// What `read` answers and `write` takes: one 32-bit word. +pub(crate) const WORD: usize = core::mem::size_of::(); + +/// What an `mmap` on a device descriptor asks for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Map { + /// The register block. + Registers, + /// Memory to lend the device: `asked` whole pages' bytes. + Memory { asked: usize }, +} + +impl Map { + /// What `mmap`'s arguments ask of a device, or the `errno` that refuses + /// them. + /// + /// `EINVAL`: no length, an offset that is neither map, a register + /// mapping longer than the block, or anything but `MAP_SHARED` (the + /// device sees every store, so there is no private copy to make). + /// `ENOTSUP`: `MAP_FIXED`, and any protection but read or read-write. + /// `ENOMEM`: a length that does not round to pages. + pub(crate) fn of(len: usize, prot: c_int, flags: c_int, offset: i64) -> Result { + if len == 0 { + return Err(errno::EINVAL); + } + if flags & MAP_FIXED != 0 { + return Err(errno::ENOTSUP); + } + if flags != MAP_SHARED { + return Err(errno::EINVAL); + } + if prot != PROT_READ && prot != PROT_WRITE && prot != PROT_READ | PROT_WRITE { + return Err(errno::ENOTSUP); + } + match offset { + MAP_REGISTERS if len <= PAGE => Ok(Self::Registers), + MAP_MEMORY => pages(len) + .map(|asked| Self::Memory { asked }) + .ok_or(errno::ENOMEM), + _ => Err(errno::EINVAL), + } + } + + /// Whether the mapping needs the descriptor opened for writing: any + /// writable one, and lent memory always, since lending is an act and the + /// driver's queue lives there. + pub(crate) fn writes(self, prot: c_int) -> bool { + matches!(self, Self::Memory { .. }) || prot & PROT_WRITE != 0 + } +} + +/// The `errno` for a request the service refused. +pub(crate) fn refused(refusal: VdevRefusal) -> c_int { + match refusal { + // No model has that id. + VdevRefusal::NoDevice => errno::ENXIO, + // The device is not in a state for this: already open, or holding a + // queue in the memory it has. + VdevRefusal::Busy => errno::EBUSY, + // Every device the service hosts at once is taken: ask again when + // another driver closes. + VdevRefusal::Limit => errno::EAGAIN, + // What was lent is not memory a device can address. + VdevRefusal::Memory => errno::EINVAL, + // This face may not ask that. + VdevRefusal::Denied => errno::EACCES, + // The service did not answer, or answered out of turn. + VdevRefusal::Unavailable | VdevRefusal::Pending | VdevRefusal::Conflict => errno::EIO, + } +} + +/// The open device's words: which opening this is, and the interrupt count +/// last read. +/// +/// The generation is odd while a device is open. A blocked `read` belongs to +/// the opening it began under, and ends with `EBADF` when that opening does. +pub(crate) struct Opening { + generation: AtomicU32, + seen: AtomicU32, +} + +impl Opening { + pub(crate) const fn new() -> Self { + Self { + generation: AtomicU32::new(0), + seen: AtomicU32::new(0), + } + } + + /// The opening now, if a device is open. + pub(crate) fn current(&self) -> Option { + let generation = self.generation.load(Ordering::Acquire); + (generation & 1 != 0).then_some(generation) + } + + /// Whether `generation` is still the opening. + pub(crate) fn is(&self, generation: u32) -> bool { + self.generation.load(Ordering::Acquire) == generation + } + + /// A device opened with its interrupt count at `count`: interrupts + /// before now are not this opening's to read. `false` when one is open. + pub(crate) fn begin(&self, count: u32) -> bool { + let generation = self.generation.load(Ordering::Acquire); + if generation & 1 != 0 { + return false; + } + self.seen.store(count, Ordering::Release); + self.generation + .store(generation.wrapping_add(1), Ordering::Release); + true + } + + /// The device closed. `false` when none was open. + pub(crate) fn end(&self) -> bool { + let generation = self.generation.load(Ordering::Acquire); + if generation & 1 == 0 { + return false; + } + self.generation + .store(generation.wrapping_add(1), Ordering::Release); + true + } + + /// The interrupt count last read. + pub(crate) fn seen(&self) -> u32 { + self.seen.load(Ordering::Acquire) + } + + /// Take the interrupt that moved the count from `seen` to `count`. + /// `false` when another reader took it first. + pub(crate) fn take(&self, seen: u32, count: u32) -> bool { + self.seen + .compare_exchange(seen, count, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } +} + +#[cfg(target_os = "none")] +pub(crate) use metal::{ + Host, close, holds_kick, map, open, posted, read, readable, ring, shares, unmap, wait_readable, +}; + +/// Off Kore no thread holds a face. +#[cfg(not(target_os = "none"))] +pub(crate) fn holds_kick() -> bool { + false +} + +#[cfg(target_os = "none")] +mod metal { + extern crate alloc; + + use core::cell::UnsafeCell; + use core::ffi::c_int; + use core::ptr::NonNull; + use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; + use core::time::Duration; + + use kore_std::abi::cap::{ResourceKind, Rights}; + use kore_std::abi::task::role; + use kore_std::runtime::cap::{MappedLeaseView, RetireOutcome, StashedLease, verbs}; + use kore_std::runtime::entry::{self, Rt}; + use kore_std::runtime::futex::{self, WaitOutcome}; + use kore_std::runtime::pace; + use kore_std::runtime::thread::{self, WorkerCap}; + use kore_std::runtime::transport::TrapTransport; + use kore_std::vdev::client::{ + ProcessVdev, THREAD_SHARES, ThreadVdev, Waited, attach_process_face, attach_thread_face, + }; + use kore_std::vdev::mem::{Memory as _, Window}; + use kore_std::vdev::regs; + use kore_std::vdev::wire::VdevResponse; + + use super::{FACE, KICK, Map, Opening, PAGE, refused}; + use crate::fd::errno; + use crate::lock::Lock; + + /// The device this program has open, if it has one. + static OPENING: Opening = Opening::new(); + + /// The register block's address in this process, once thread 0 has + /// attached the face; zero before. The mapping is thread 0's and lasts + /// the program. + static REGISTERS: AtomicUsize = AtomicUsize::new(0); + + /// The lent memory's address while the program holds a mapping of it; + /// zero otherwise. + static MEMORY: AtomicUsize = AtomicUsize::new(0); + + /// The register block, to read the interrupt count through. + fn registers() -> Option { + let base = NonNull::new(REGISTERS.load(Ordering::Acquire) as *mut u8)?; + // SAFETY: thread 0 published the address of the face's second page, + // which it keeps mapped read-write for the life of the process. The + // device writes it concurrently, which is what a `Window` is for. + Some(unsafe { Window::new(base, regs::BYTES as usize) }) + } + + /// The calling thread's own part of the face. + fn face() -> Option { + attach_thread_face(&entry::thread_context()?) + } + + /// Whether the calling thread holds the face's kick with the right a + /// notice needs. + pub(crate) fn holds_kick() -> bool { + entry::thread_context() + .and_then(|rt| rt.bootstrap().handle(role::SVC_SEND, KICK)) + .and_then(|kick| verbs::capability_rights(kick, ResourceKind::ChanSend)) + .is_some_and(|rights| rights.contains(verbs::NOTICE_RIGHTS)) + } + + /// The endowment that founds a thread holding its share of the face: + /// what thread 0 gives the main thread, with `DERIVE` so the main thread + /// can found the thread controller holding it too. Empty for a launch + /// without a face. + pub(crate) fn shares() -> impl Iterator { + THREAD_SHARES.iter().filter_map(|share| { + thread::share_endowment( + share.role, + share.index, + share.kind, + share.rights | Rights::DERIVE, + ) + }) + } + + // The table's row names the same coordinate the shares do. + const _: () = assert!(THREAD_SHARES[0].index == FACE && THREAD_SHARES[1].index == KICK); + + // ── The threads' half ─────────────────────────────────────────────── + + /// `read`: wait for an interrupt of this opening and take it. + /// + /// `EBADF`: no device is open, or it closed while this thread waited. + /// `ENXIO`: the calling thread holds no share of the face. `EIO`: the + /// thread cannot wait on the device (its share is gone, or the kernel + /// will not park it). + pub(crate) fn read() -> Result { + let generation = OPENING.current().ok_or(errno::EBADF)?; + let registers = registers().ok_or(errno::EBADF)?; + let face = face().ok_or(errno::ENXIO)?; + loop { + let seen = OPENING.seen(); + match face.wait(registers, seen, None, || OPENING.is(generation)) { + // Another reader may have taken this interrupt: then wait + // for the next. + Waited::Interrupt(count) => { + if OPENING.take(seen, count) { + return Ok(count); + } + } + Waited::Stopped => return Err(errno::EBADF), + Waited::Deadline | Waited::Gone => return Err(errno::EIO), + } + } + } + + /// `write`: ring the doorbell. A notice the service's doorbell had no + /// room for is not an error: it has a wake standing and looks anyway. + /// + /// `EBADF`: no device is open. `ENXIO`: the calling thread holds no + /// share of the face. + pub(crate) fn ring() -> Result<(), c_int> { + OPENING.current().ok_or(errno::EBADF)?; + let _ = face().ok_or(errno::ENXIO)?.doorbell(); + Ok(()) + } + + /// Whether an interrupt is waiting to be read. `None` when no device is + /// open. + pub(crate) fn readable() -> Option { + OPENING.current()?; + let count = registers()?.load_u32(regs::INTERRUPTS).ok()?; + Some(count != OPENING.seen()) + } + + /// `poll`'s wait: until an interrupt is waiting to be read or + /// `deadline_ns` passes. `Ok(true)`: one is waiting. `Ok(false)`: the + /// deadline passed. `Err`: as [`read`], and the caller reports it on the + /// descriptor. + pub(crate) fn wait_readable(deadline_ns: Option) -> Result { + let generation = OPENING.current().ok_or(errno::EBADF)?; + let registers = registers().ok_or(errno::EBADF)?; + let face = face().ok_or(errno::ENXIO)?; + match face.wait(registers, OPENING.seen(), deadline_ns, || { + OPENING.is(generation) + }) { + Waited::Interrupt(_) => Ok(true), + Waited::Deadline => Ok(false), + Waited::Stopped => Err(errno::EBADF), + Waited::Gone => Err(errno::EIO), + } + } + + /// `open`: have the service start the model with this id. + /// + /// `EBUSY`: this program has a device open. `ENXIO`: the launch carried + /// no face, the program has no host to ask, or no model has that id. + /// `ENOTSUP`: asked from thread 0 itself, which cannot wait for its own + /// answer. Otherwise as the service refuses ([`refused`]). + pub(crate) fn open(model: u32) -> Result<(), c_int> { + ask(Op::Open { model }).map(|_| ()) + } + + /// `close`: give the device up. Nothing is reported: a device that could + /// not be closed ends with the program. + pub(crate) fn close() { + let _ = ask(Op::Close); + } + + /// `mmap` on the open device. + /// + /// `EBADF`: no device is open. `EBUSY`: the program already holds a + /// mapping of device memory. `ENOMEM`: the supply or the link window has + /// no room for the region, or the kernel refused the map. Otherwise as + /// the service refuses the loan ([`refused`]). + pub(crate) fn map(map: Map) -> Result<*mut u8, c_int> { + OPENING.current().ok_or(errno::EBADF)?; + match map { + Map::Registers => NonNull::new(REGISTERS.load(Ordering::Acquire) as *mut u8) + .map(NonNull::as_ptr) + .ok_or(errno::EBADF), + Map::Memory { asked } => ask(Op::Lend { asked }).map(|base| base as *mut u8), + } + } + + /// `munmap` of a device mapping. `None` when `base` is not one, and the + /// memory supply's rules apply. + /// + /// The register block is the process's for its life: unmapping it is + /// accepted and changes nothing. `EBUSY`: the lent memory, while the + /// device is open. `EINVAL`: `len` is not the mapping's length. + /// `EAGAIN`: the kernel did not confirm the unmap. + pub(crate) fn unmap(base: usize, len: usize) -> Option> { + if base == 0 { + return None; + } + if base == REGISTERS.load(Ordering::Acquire) { + return Some(if len == 0 || len > PAGE { + Err(errno::EINVAL) + } else { + Ok(()) + }); + } + if base != MEMORY.load(Ordering::Acquire) { + return None; + } + Some(ask(Op::Release { base, len }).map(|_| ())) + } + + // ── Reaching thread 0 ─────────────────────────────────────────────── + + /// One request of thread 0. + #[derive(Clone, Copy)] + enum Op { + Open { model: u32 }, + Close, + Lend { asked: usize }, + Release { base: usize, len: usize }, + } + + // The cell's states. A caller owns it at IDLE (under [`CALLERS`]) and at + // SERVED, thread 0 at POSTED. + const IDLE: u32 = 0; + const POSTED: u32 = 1; + const SERVED: u32 = 2; + + /// One request at a time: the caller holding [`CALLERS`] writes `op` and + /// posts, thread 0 writes `answer` and serves. + struct Call { + state: AtomicU32, + op: UnsafeCell, + answer: UnsafeCell>, + } + // SAFETY: `op` and `answer` are touched only by the side `state` hands + // the cell to. + unsafe impl Sync for Call {} + static CALL: Call = Call { + state: AtomicU32::new(IDLE), + op: UnsafeCell::new(Op::Close), + answer: UnsafeCell::new(Ok(0)), + }; + + /// Callers take turns at the cell. + static CALLERS: Lock = Lock::new(); + + /// How long a caller waits before it re-reads the cell: thread 0 kicks + /// the main thread when it has answered, and holds no send into a + /// pthread's doorbell, which rides this slice (the mapping cell's rule). + const SLICE: Duration = Duration::from_nanos(pace::CEDE_NS); + + /// Whether a request waits for thread 0. + pub(crate) fn posted() -> bool { + CALL.state.load(Ordering::Acquire) == POSTED + } + + /// Have thread 0 carry `op` out, and wait for its answer. + fn ask(op: Op) -> Result { + if crate::program::on_host() { + // Thread 0 answers these between its own turns: it cannot wait + // here for an answer only it can give. + return Err(errno::ENOTSUP); + } + if !crate::program::hosted() { + return Err(errno::ENXIO); + } + CALLERS.acquire(); + // SAFETY: IDLE under the lock hands the cell to this caller. + unsafe { *CALL.op.get() = op }; + CALL.state.store(POSTED, Ordering::SeqCst); + // Refused only when no post stood, and then thread 0 re-reads the + // cell before it parks. + let _ = thread::kick_parent(); + loop { + match CALL.state.load(Ordering::Acquire) { + SERVED => break, + state => { + if futex::wait(&CALL.state, state, Some(SLICE)) == WaitOutcome::Unsupported { + // No doorbell to park on: the deadline park any + // thread can. + let _ = pace::cede(); + } + } + } + } + // SAFETY: SERVED hands the cell back to this caller. + let answer = unsafe { *CALL.answer.get() }; + CALL.state.store(IDLE, Ordering::Release); + CALLERS.release(); + answer + } + + // ── Thread 0's half ───────────────────────────────────────────────── + + /// The memory lent to the device, or kept from the last loan. + enum Lent { + /// Nothing carved. + Nothing, + /// Unmapped. Its lease and address serve the next loan of its + /// length: the supply's cursors and the window's only advance. + Spare { + lease: StashedLease, + base: usize, + len: usize, + }, + /// Mapped in this process. `lent` while the device has it. + Mapped { + lease: StashedLease, + view: MappedLeaseView, + base: usize, + len: usize, + lent: bool, + }, + } + + /// Thread 0's side of the program's device: the face, attached on first + /// use, and the memory lent through it. + pub(crate) struct Host<'r, 't> { + rt: &'r Rt, + transport: &'t TrapTransport, + face: Option>, + /// The launch carried no face: every later request is refused the + /// same way without asking again. + absent: bool, + memory: Lent, + } + + impl<'r, 't> Host<'r, 't> { + pub(crate) fn new(rt: &'r Rt, transport: &'t TrapTransport) -> Self { + Self { + rt, + transport, + face: None, + absent: false, + memory: Lent::Nothing, + } + } + + /// Answer the posted request, if there is one. `true` when one was + /// answered. Dropped mid-request when the program ends first; the + /// caller it leaves waiting is stopped with the rest of the program. + pub(crate) async fn serve(&mut self) -> bool { + if !posted() { + return false; + } + // SAFETY: POSTED hands the cell to thread 0 until it stores + // SERVED. + let op = unsafe { *CALL.op.get() }; + let answer = self.perform(op).await; + // SAFETY: as above. + unsafe { *CALL.answer.get() = answer }; + // SeqCst, as the futex contract asks of the store before a wake. + CALL.state.store(SERVED, Ordering::SeqCst); + futex::wake(&CALL.state); + true + } + + fn face(&mut self) -> Result<&mut ProcessVdev<'t>, c_int> { + if self.absent { + return Err(errno::ENXIO); + } + if self.face.is_none() { + match attach_process_face(self.rt, self.transport) { + Some(face) => { + REGISTERS + .store(face.registers().base().as_ptr() as usize, Ordering::Release); + self.face = Some(face); + } + None => { + self.absent = true; + return Err(errno::ENXIO); + } + } + } + self.face.as_mut().ok_or(errno::ENXIO) + } + + async fn perform(&mut self, op: Op) -> Result { + match op { + Op::Open { model } => self.open(model).await, + Op::Close => self.close().await, + Op::Lend { asked } => self.lend(asked).await, + Op::Release { base, len } => self.release(base, len), + } + } + + async fn open(&mut self, model: u32) -> Result { + if OPENING.current().is_some() { + return Err(errno::EBUSY); + } + let face = self.face()?; + match face.open(model).await { + VdevResponse::Accepted => { + // Interrupts from before this opening are not its to + // read. + OPENING.begin(face.interrupts()); + Ok(0) + } + VdevResponse::Refused(refusal) => Err(refused(refusal)), + _ => Err(errno::EIO), + } + } + + async fn close(&mut self) -> Result { + if !OPENING.end() { + return Ok(0); + } + let face = self.face()?; + // The service drops the model and its view of what was lent. + // Only its word for that frees the memory to be unmapped: a + // close it did not answer leaves the loan standing, and the + // face ends with the program. + let closed = matches!(face.close().await, VdevResponse::Accepted); + if closed && let Lent::Mapped { lent, .. } = &mut self.memory { + *lent = false; + } + Ok(0) + } + + async fn lend(&mut self, asked: usize) -> Result { + if OPENING.current().is_none() { + return Err(errno::EBADF); + } + // A mapping the program was never given (a loan the service + // refused, whose unmap the kernel did not confirm) is retired + // before anything else is tried. + if MEMORY.load(Ordering::Acquire) == 0 { + self.retire_unpublished(); + } + let (lease, base, len) = match core::mem::replace(&mut self.memory, Lent::Nothing) { + Lent::Spare { lease, base, len } if len == asked => (lease, base, len), + mapped @ Lent::Mapped { .. } => { + self.memory = mapped; + return Err(errno::EBUSY); + } + // A spare of another length is given up for good, and + // nothing carved is nothing to reuse. + Lent::Spare { .. } | Lent::Nothing => { + let lease = kore_std::runtime::heap::reserve_from_top(asked as u64) + .ok_or(errno::ENOMEM)?; + let len = lease + .slot() + .and_then(verbs::lease_geometry) + .and_then(|(_, len)| usize::try_from(len).ok()) + .ok_or(errno::ENOMEM)?; + let base = crate::mmap::place(len).ok_or(errno::ENOMEM)?; + (lease, base, len) + } + }; + let Some(view) = crate::mmap::map_zeroed(&lease, base, len, true) else { + self.memory = Lent::Spare { lease, base, len }; + return Err(errno::ENOMEM); + }; + let face = match self.face() { + Ok(face) => face, + Err(code) => { + self.memory = Lent::Mapped { + lease, + view, + base, + len, + lent: false, + }; + self.retire_unpublished(); + return Err(code); + } + }; + let answer = face.lend(&lease, len as u64).await; + let lent = matches!( + answer, + VdevResponse::Lent { pages } if pages as usize == len / PAGE + ); + self.memory = Lent::Mapped { + lease, + view, + base, + len, + lent, + }; + if lent { + MEMORY.store(base, Ordering::Release); + return Ok(base); + } + self.retire_unpublished(); + Err(match answer { + VdevResponse::Refused(refusal) => refused(refusal), + _ => errno::EIO, + }) + } + + /// Unmap a mapping the program does not hold and the device does not + /// have. One the kernel does not confirm stays as it is, to be tried + /// again. + fn retire_unpublished(&mut self) { + self.memory = match core::mem::replace(&mut self.memory, Lent::Nothing) { + Lent::Mapped { + lease, + mut view, + base, + len, + lent: false, + } => match view.try_retire_outcome() { + RetireOutcome::Retired | RetireOutcome::AlreadyDead => { + Lent::Spare { lease, base, len } + } + RetireOutcome::Retryable => Lent::Mapped { + lease, + view, + base, + len, + lent: false, + }, + }, + other => other, + }; + } + + fn release(&mut self, at: usize, asked: usize) -> Result { + match &self.memory { + Lent::Mapped { base, .. } if *base != at => return Err(errno::EINVAL), + Lent::Mapped { lent: true, .. } => return Err(errno::EBUSY), + Lent::Mapped { len, .. } => { + if crate::mmap::pages(asked) != Some(*len) { + return Err(errno::EINVAL); + } + } + Lent::Spare { .. } | Lent::Nothing => return Err(errno::EINVAL), + } + // The program's mapping ends here whether or not the kernel + // confirms it at once: the address is no longer its to use. + MEMORY.store(0, Ordering::Release); + self.retire_unpublished(); + match self.memory { + Lent::Mapped { base, .. } => { + // Still mapped: the program may ask again. + MEMORY.store(base, Ordering::Release); + Err(errno::EAGAIN) + } + _ => Ok(0), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const RW: c_int = PROT_READ | PROT_WRITE; + const MAP_PRIVATE: c_int = crate::mmap::MAP_PRIVATE; + + #[test] + fn offset_zero_is_the_register_block_and_one_page_is_memory() { + assert_eq!(Map::of(PAGE, RW, MAP_SHARED, 0), Ok(Map::Registers)); + // A shorter length is still the one page. + assert_eq!(Map::of(0x58, PROT_READ, MAP_SHARED, 0), Ok(Map::Registers)); + assert_eq!( + Map::of(1 << 20, RW, MAP_SHARED, MAP_MEMORY), + Ok(Map::Memory { asked: 1 << 20 }) + ); + // Lent memory rounds up to whole pages, as any mapping does. + assert_eq!( + Map::of(PAGE + 1, RW, MAP_SHARED, MAP_MEMORY), + Ok(Map::Memory { asked: 2 * PAGE }) + ); + } + + #[test] + fn a_device_mapping_is_refused_by_cause() { + assert_eq!(Map::of(0, RW, MAP_SHARED, 0), Err(errno::EINVAL)); + // The block is one page. + assert_eq!(Map::of(PAGE + 1, RW, MAP_SHARED, 0), Err(errno::EINVAL)); + // No third map, and no offset inside one. + for offset in [-1, 1, 2 * PAGE as i64, MAP_MEMORY + 1] { + assert_eq!( + Map::of(PAGE, RW, MAP_SHARED, offset), + Err(errno::EINVAL), + "{offset}" + ); + } + // The device sees every store: there is no private copy. + assert_eq!(Map::of(PAGE, RW, MAP_PRIVATE, 0), Err(errno::EINVAL)); + assert_eq!( + Map::of(PAGE, RW, MAP_SHARED | MAP_PRIVATE, 0), + Err(errno::EINVAL) + ); + assert_eq!( + Map::of(PAGE, RW, MAP_SHARED | MAP_FIXED, 0), + Err(errno::ENOTSUP) + ); + // PROT_NONE and PROT_EXEC. + for prot in [0, 4, RW | 4] { + assert_eq!( + Map::of(PAGE, prot, MAP_SHARED, 0), + Err(errno::ENOTSUP), + "{prot}" + ); + } + assert_eq!( + Map::of(usize::MAX, RW, MAP_SHARED, MAP_MEMORY), + Err(errno::ENOMEM) + ); + } + + #[test] + fn lent_memory_and_a_writable_block_need_a_writable_descriptor() { + assert!(!Map::Registers.writes(PROT_READ)); + assert!(Map::Registers.writes(RW)); + assert!(Map::Registers.writes(PROT_WRITE)); + // Lending is an act, whatever protection the mapping asks. + assert!(Map::Memory { asked: PAGE }.writes(PROT_READ)); + assert!(Map::Memory { asked: PAGE }.writes(RW)); + } + + #[test] + fn every_refusal_has_an_errno() { + assert_eq!(refused(VdevRefusal::NoDevice), errno::ENXIO); + assert_eq!(refused(VdevRefusal::Busy), errno::EBUSY); + assert_eq!(refused(VdevRefusal::Limit), errno::EAGAIN); + assert_eq!(refused(VdevRefusal::Memory), errno::EINVAL); + assert_eq!(refused(VdevRefusal::Denied), errno::EACCES); + for refusal in [ + VdevRefusal::Unavailable, + VdevRefusal::Pending, + VdevRefusal::Conflict, + ] { + assert_eq!(refused(refusal), errno::EIO, "{refusal:?}"); + } + } + + #[test] + fn one_device_is_open_at_a_time_and_each_opening_is_its_own() { + let opening = Opening::new(); + assert_eq!(opening.current(), None); + assert!(!opening.end(), "nothing was open"); + assert!(opening.begin(7)); + let first = opening.current().expect("open"); + assert!(opening.is(first)); + assert_eq!( + opening.seen(), + 7, + "interrupts before the opening are not its" + ); + assert!(!opening.begin(9), "a second device"); + assert_eq!(opening.seen(), 7, "a refused opening changes nothing"); + assert!(opening.end()); + assert_eq!(opening.current(), None); + assert!(!opening.is(first), "a reader of the first opening ends"); + assert!(opening.begin(0)); + let second = opening.current().expect("open again"); + assert_ne!(first, second); + assert!(!opening.is(first)); + } + + #[test] + fn an_interrupt_is_read_once() { + let opening = Opening::new(); + assert!(opening.begin(0)); + // Two readers saw the count at zero and woke on the same interrupt. + assert!(opening.take(0, 1)); + assert!(!opening.take(0, 1), "the second waits for the next"); + assert_eq!(opening.seen(), 1); + // The count wraps. + assert!(opening.take(1, u32::MAX)); + assert!(opening.take(u32::MAX, 0)); + assert_eq!(opening.seen(), 0); + } + + #[test] + fn the_names_coordinates_are_the_faces() { + assert_eq!(MODEL_ACCEL, 0x6c63_6361); + assert_eq!(MAP_MEMORY, 0x1000); + assert_eq!(WORD, 4); + let shares = kore_std::vdev::client::THREAD_SHARES; + assert_eq!(shares[0].index, FACE); + assert_eq!(shares[1].index, KICK); + } +} diff --git a/crates/c/src/exit.rs b/crates/c/src/exit.rs new file mode 100644 index 0000000..027788b --- /dev/null +++ b/crates/c/src/exit.rs @@ -0,0 +1,114 @@ +//! The program's end request: one word any of the program's threads posts and +//! the host reads. The first request wins; later ones only end their thread. + +use core::ffi::c_int; +use core::sync::atomic::{AtomicU64, Ordering}; + +/// One end request. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Exit { + /// The C status: the wire's `i32`, committed as given. + pub(crate) status: c_int, + /// `exit` rather than `_Exit` or a panic: finalizers may run. + pub(crate) finalize: bool, + /// The main thread asked, and is ending itself; the host waits for it + /// before the barrier, so a program that ran on it alone stops cleanly. + pub(crate) from_main: bool, +} + +const POSTED: u64 = 1 << 63; +const FINALIZE: u64 = 1 << 33; +const FROM_MAIN: u64 = 1 << 32; + +impl Exit { + fn word(self) -> u64 { + POSTED + | (u64::from(self.finalize) * FINALIZE) + | (u64::from(self.from_main) * FROM_MAIN) + | u64::from(self.status as u32) + } + + fn from_word(word: u64) -> Option { + (word & POSTED != 0).then_some(Self { + status: word as u32 as c_int, + finalize: word & FINALIZE != 0, + from_main: word & FROM_MAIN != 0, + }) + } +} + +/// Zero until the first request, then that request, for the program's life. +pub(crate) struct ExitWord(AtomicU64); + +impl ExitWord { + pub(crate) const fn new() -> Self { + Self(AtomicU64::new(0)) + } + + /// The request, once one was posted. + pub(crate) fn posted(&self) -> Option { + Exit::from_word(self.0.load(Ordering::Acquire)) + } + + /// Ask for `exit`. `false` when another request was first; the word keeps + /// that one. + pub(crate) fn post(&self, exit: Exit) -> bool { + self.0 + .compare_exchange(0, exit.word(), Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_request_round_trips_every_status_and_flag() { + for status in [0, 1, 42, -1, c_int::MIN, c_int::MAX] { + for finalize in [false, true] { + for from_main in [false, true] { + let exit = Exit { + status, + finalize, + from_main, + }; + let word = ExitWord::new(); + assert_eq!(word.posted(), None); + assert!(word.post(exit)); + assert_eq!(word.posted(), Some(exit)); + } + } + } + } + + #[test] + fn the_first_request_wins() { + let word = ExitWord::new(); + let first = Exit { + status: 3, + finalize: true, + from_main: false, + }; + let second = Exit { + status: 7, + finalize: false, + from_main: true, + }; + assert!(word.post(first)); + assert!(!word.post(second)); + assert_eq!(word.posted(), Some(first)); + } + + #[test] + fn a_status_of_zero_is_still_a_request() { + let word = ExitWord::new(); + let exit = Exit { + status: 0, + finalize: false, + from_main: false, + }; + assert!(word.post(exit)); + assert_eq!(word.posted(), Some(exit)); + } +} diff --git a/crates/c/src/fd.rs b/crates/c/src/fd.rs new file mode 100644 index 0000000..94cca39 --- /dev/null +++ b/crates/c/src/fd.rs @@ -0,0 +1,455 @@ +//! Descriptors: C's spelling for "a capability I hold". +//! +//! A program's authority arrives as capabilities at fixed places: its boot +//! block names each by a role and an index, and the capability itself sits in +//! the thread's arena. `open` is the readable way to say which one is meant. +//! It grants nothing and reaches no filesystem: +//! +//! - **A name is a row of one static table** ([`NAMES`]): a path-shaped +//! string, the bootstrap coordinate it stands for, and what kind of +//! capability lives there. A path that is no row is `ENOENT`. +//! - **A descriptor stores the row, never the capability.** Capabilities are +//! held per thread on Kore, and a thread is a task: `main` runs on one, a +//! `pthread` on another, and each has its own boot block. So every use of a +//! descriptor resolves the row's coordinate in the *calling* thread's boot +//! block. Descriptor numbers are the process's, as C expects, and no +//! authority crosses a thread through one: a thread that was not founded +//! holding the capability gets `ENXIO`, whoever opened the descriptor. +//! - **`close` forgets the number.** The capability was the program's before +//! `open` and is after `close`; nothing is revoked. +//! +//! Descriptors 0, 1 and 2 are the standard streams' numbers and are never +//! handed out; the streams themselves are ``'s. +//! +//! The table is plain data over atomics, so the host suite exercises the code +//! the target runs. Finding a row's capability in the boot block is the metal +//! half ([`holding`]). + +use core::ffi::c_int; +use core::sync::atomic::{AtomicU32, Ordering}; + +use kore_std::abi::task::role; + +/// `errno` values this module and [`crate::mmap`] report: musl's numbers, +/// which kore-c-shim's `errno` and the hosted `` share. +pub(crate) mod errno { + use core::ffi::c_int; + pub(crate) const ENOENT: c_int = 2; + pub(crate) const EIO: c_int = 5; + pub(crate) const ENXIO: c_int = 6; + pub(crate) const EBADF: c_int = 9; + pub(crate) const EAGAIN: c_int = 11; + pub(crate) const ENOMEM: c_int = 12; + pub(crate) const EACCES: c_int = 13; + pub(crate) const EFAULT: c_int = 14; + pub(crate) const EBUSY: c_int = 16; + pub(crate) const EINVAL: c_int = 22; + pub(crate) const EMFILE: c_int = 24; + pub(crate) const ENOTSUP: c_int = 95; +} + +/// What kind of capability a name stands for: which operations its +/// descriptor has. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kind { + /// The memory supply the program's allocator grows on. `mmap` takes a + /// region off it, through the runtime's own reservation so the two never + /// meet ([`crate::mmap`]). + Memory, + /// A virtual device of the launch's virtual-device face, by its + /// register-block id: registers and lent memory through `mmap`, the + /// interrupt through `read`, the doorbell through `write` + /// ([`crate::device`]). + Device { + /// The device model's id, the register block's `DEVICE_ID`. + model: u32, + }, +} + +/// One row: a name and the bootstrap coordinate it stands for. +#[derive(Debug)] +pub(crate) struct Name { + /// The path `open` matches, byte for byte. + pub(crate) path: &'static [u8], + /// The boot block role of the capability. + pub(crate) role: u16, + /// The instance within that role. + pub(crate) index: u16, + /// What lives there. + pub(crate) kind: Kind, +} + +/// **The names.** Each is a coordinate every thread that holds the +/// capability finds it under, so a row is true on whichever thread asks. +/// +/// `/kore/memory` is `role::HEAP@0`: the launch's memory supply, which +/// kore-rt endows every thread that may allocate with a share of +/// (`WorkerCap::heap`). +/// +/// `/kore/device/virtual/accel` is the virtual accelerator on the launch's +/// virtual-device face. Its row's coordinate is the face's memory, whose +/// second page is the register block: the share of it this library founds +/// the program's threads holding is what a thread parks on the interrupt +/// through. The face's kick, beside it, is what a thread rings the doorbell +/// through ([`crate::device::KICK`]). +pub(crate) static NAMES: [Name; 2] = [ + Name { + path: b"/kore/memory", + role: role::HEAP, + index: 0, + kind: Kind::Memory, + }, + Name { + path: b"/kore/device/virtual/accel", + role: role::SVC_RING, + index: crate::device::FACE, + kind: Kind::Device { + model: crate::device::MODEL_ACCEL, + }, + }, +]; + +/// The row `path` names. +pub(crate) fn resolve(path: &[u8]) -> Option { + NAMES.iter().position(|name| name.path == path) +} + +/// What an `open` asked to do with the capability. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Access { + pub(crate) read: bool, + pub(crate) write: bool, +} + +/// ``'s access modes, musl's values. They are the only flags `open` +/// takes: every other flag describes a file this namespace does not have. +pub(crate) const O_RDONLY: c_int = 0; +pub(crate) const O_WRONLY: c_int = 1; +pub(crate) const O_RDWR: c_int = 2; + +impl Access { + /// The access `flags` ask for, or `EINVAL` for any flag but an access mode. + pub(crate) fn of(flags: c_int) -> Result { + match flags { + O_RDONLY => Ok(Self { + read: true, + write: false, + }), + O_WRONLY => Ok(Self { + read: false, + write: true, + }), + O_RDWR => Ok(Self { + read: true, + write: true, + }), + _ => Err(errno::EINVAL), + } + } + + /// Whether a capability that can be read (`ACCESS`) and, if `writable`, + /// written (`MODIFY`) permits this access. + pub(crate) fn within(self, readable: bool, writable: bool) -> bool { + (!self.read || readable) && (!self.write || writable) + } + + const fn bits(self) -> u32 { + self.read as u32 | (self.write as u32) << 1 + } + + const fn from_bits(bits: u32) -> Self { + Self { + read: bits & 1 != 0, + write: bits & 2 != 0, + } + } +} + +/// The first number `open` hands out: 0, 1 and 2 are the standard streams'. +pub(crate) const FIRST: c_int = 3; + +/// How many descriptors a process may hold open. A descriptor is one word +/// naming a row, and the rows are few, so this bounds a program that forgets +/// to `close` rather than one that needs many: sixty-four entries are one +/// 256-byte table, scanned whole only by an `open` that finds it full. +pub(crate) const CAPACITY: usize = 64; + +/// A descriptor table: one word per descriptor, zero when free. +/// +/// A live word is [`LIVE`], the access in two bits, and the row's index. +/// Claiming and releasing are one compare-and-swap each, so threads opening +/// and closing at once never share a lock, and a descriptor is never handed +/// to two of them. +pub(crate) struct Table { + words: [AtomicU32; N], +} + +const LIVE: u32 = 1 << 31; +const ACCESS_SHIFT: u32 = 16; +const ROW_MASK: u32 = (1 << ACCESS_SHIFT) - 1; + +// Every row's index fits its field. +const _: () = assert!(NAMES.len() <= ROW_MASK as usize + 1); + +impl Table { + pub(crate) const fn new() -> Self { + Self { + words: [const { AtomicU32::new(0) }; N], + } + } + + /// Open a descriptor for `row` with `access`: the lowest free number, as + /// POSIX has it, or `EMFILE` when every one is taken. + pub(crate) fn claim(&self, row: usize, access: Access) -> Result { + let word = LIVE | access.bits() << ACCESS_SHIFT | row as u32; + for (at, slot) in self.words.iter().enumerate() { + if slot + .compare_exchange(0, word, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + return Ok(FIRST + at as c_int); + } + } + Err(errno::EMFILE) + } + + /// The row and access behind `fd`, or `EBADF`. + pub(crate) fn get(&self, fd: c_int) -> Result<(usize, Access), c_int> { + let word = self.slot(fd)?.load(Ordering::Acquire); + if word & LIVE == 0 { + return Err(errno::EBADF); + } + Ok(( + (word & ROW_MASK) as usize, + Access::from_bits(word >> ACCESS_SHIFT), + )) + } + + /// Close `fd`, or `EBADF` when it was not open. + pub(crate) fn release(&self, fd: c_int) -> Result<(), c_int> { + match self.slot(fd)?.swap(0, Ordering::AcqRel) { + 0 => Err(errno::EBADF), + _ => Ok(()), + } + } + + fn slot(&self, fd: c_int) -> Result<&AtomicU32, c_int> { + fd.checked_sub(FIRST) + .and_then(|at| usize::try_from(at).ok()) + .and_then(|at| self.words.get(at)) + .ok_or(errno::EBADF) + } +} + +/// The process's descriptors. +pub(crate) static TABLE: Table = Table::new(); + +/// What the calling thread holds at a row's coordinate. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Holding { + /// The capability's arena slot, a name this thread alone may use. + pub(crate) slot: kore_std::abi::session::SlotRef, + /// It carries `ACCESS`. + pub(crate) readable: bool, + /// It carries `MODIFY`. + pub(crate) writable: bool, +} + +/// **The capability `name` stands for, as the calling thread holds it**: +/// its slot from this thread's boot block, and the rights its record +/// carries. `None` when the thread was not founded holding one of that kind +/// there. The record is the kernel's, read through the arena's read-only +/// alias, and every later effect binds the slot again in the kernel, so this +/// is a description and never a grant. +#[cfg(target_os = "none")] +pub(crate) fn holding(name: &Name) -> Option { + use kore_std::abi::cap::{ResourceKind, Rights}; + use kore_std::runtime::cap::verbs; + let slot = kore_std::runtime::entry::thread_context()? + .bootstrap() + .handle(name.role, name.index)?; + let rights = verbs::capability_rights(slot, ResourceKind::MemLease)?; + let writable = match name.kind { + Kind::Memory => rights.contains(Rights::MODIFY), + // What a thread writes of a device is its doorbell: a notice on the + // face's kick, which is a capability beside the face's memory. + Kind::Device { .. } => crate::device::holds_kick(), + }; + Some(Holding { + slot, + readable: rights.contains(Rights::ACCESS), + writable, + }) +} + +/// `open`: a descriptor for the capability `path` names, with `flags`' +/// access. +/// +/// `ENOENT`: no such name. `EINVAL`: a flag other than an access mode. +/// `ENXIO`: the calling thread holds no such capability. `EACCES`: it holds +/// one whose rights do not cover the access. `EMFILE`: no descriptor is free. +#[cfg(target_os = "none")] +pub(crate) fn open(path: &[u8], flags: c_int) -> Result { + let row = resolve(path).ok_or(errno::ENOENT)?; + let access = Access::of(flags)?; + let held = holding(&NAMES[row]).ok_or(errno::ENXIO)?; + if !access.within(held.readable, held.writable) { + return Err(errno::EACCES); + } + match NAMES[row].kind { + Kind::Memory => TABLE.claim(row, access), + // A device is opened as well as named: the service starts the model + // and writes its register block. + Kind::Device { model } => { + crate::device::open(model)?; + TABLE + .claim(row, access) + .inspect_err(|_| crate::device::close()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_path_is_a_row_or_nothing() { + assert_eq!(resolve(b"/kore/memory"), Some(0)); + for path in [ + &b""[..], + b"/", + b"/kore", + b"/kore/memory/", + b"/kore/memory\0", + b"kore/memory", + b"/KORE/MEMORY", + b"/dev/zero", + ] { + assert_eq!(resolve(path), None, "{:?}", core::str::from_utf8(path)); + } + assert_eq!((NAMES[0].role, NAMES[0].index), (role::HEAP, 0)); + assert_eq!(resolve(b"/kore/device/virtual/accel"), Some(1)); + assert_eq!(resolve(b"/kore/device/virtual"), None); + assert_eq!(resolve(b"/kore/device/virtual/accl"), None); + assert_eq!(NAMES[1].role, role::SVC_RING); + assert_eq!( + NAMES[1].kind, + Kind::Device { + model: u32::from_le_bytes(*b"accl") + } + ); + } + + #[test] + fn only_an_access_mode_is_a_flag() { + assert_eq!( + Access::of(O_RDONLY), + Ok(Access { + read: true, + write: false + }) + ); + assert_eq!( + Access::of(O_WRONLY), + Ok(Access { + read: false, + write: true + }) + ); + assert_eq!( + Access::of(O_RDWR), + Ok(Access { + read: true, + write: true + }) + ); + // The access modes are not bits: 3 is no mode, and neither is a mode + // with O_CREAT (0o100), O_TRUNC (0o1000) or O_CLOEXEC (0o2000000). + for flags in [3, O_RDWR | 0o100, O_WRONLY | 0o1000, 0o2000000, -1] { + assert_eq!(Access::of(flags), Err(errno::EINVAL), "{flags:#o}"); + } + } + + #[test] + fn access_is_held_to_the_capabilitys_rights() { + let read = Access::of(O_RDONLY).unwrap(); + let write = Access::of(O_WRONLY).unwrap(); + let both = Access::of(O_RDWR).unwrap(); + // (readable, writable) of the capability. + assert!(read.within(true, false) && read.within(true, true)); + assert!(!read.within(false, true)); + assert!(write.within(false, true) && !write.within(true, false)); + assert!(both.within(true, true)); + assert!(!both.within(true, false) && !both.within(false, true)); + } + + #[test] + fn the_lowest_free_number_is_handed_out_from_three() { + let table = Table::<4>::new(); + let both = Access::of(O_RDWR).unwrap(); + let read = Access::of(O_RDONLY).unwrap(); + assert_eq!(table.claim(0, both), Ok(3)); + assert_eq!(table.claim(0, read), Ok(4)); + assert_eq!(table.claim(0, both), Ok(5)); + assert_eq!(table.get(4), Ok((0, read))); + assert_eq!(table.release(4), Ok(())); + assert_eq!(table.get(4), Err(errno::EBADF)); + // The hole is filled before a higher number is used. + assert_eq!(table.claim(0, both), Ok(4)); + assert_eq!(table.claim(0, both), Ok(6)); + assert_eq!(table.claim(0, both), Err(errno::EMFILE)); + assert_eq!(table.get(6), Ok((0, both))); + } + + #[test] + fn a_number_that_is_not_open_is_ebadf() { + let table = Table::<4>::new(); + // The streams' numbers, a negative, one past the table, and a free one. + for fd in [0, 1, 2, -1, c_int::MIN, 7, c_int::MAX, 3] { + assert_eq!(table.get(fd), Err(errno::EBADF), "{fd}"); + assert_eq!(table.release(fd), Err(errno::EBADF), "{fd}"); + } + let fd = table.claim(0, Access::of(O_RDONLY).unwrap()).unwrap(); + assert_eq!(table.release(fd), Ok(())); + // A second close is of a descriptor that is no longer open. + assert_eq!(table.release(fd), Err(errno::EBADF)); + } + + #[test] + fn the_row_and_access_survive_the_word() { + let table = Table::<2>::new(); + let row = ROW_MASK as usize; + for flags in [O_RDONLY, O_WRONLY, O_RDWR] { + let access = Access::of(flags).unwrap(); + let fd = table.claim(row, access).unwrap(); + assert_eq!(table.get(fd), Ok((row, access))); + table.release(fd).unwrap(); + } + } + + #[test] + fn threads_never_share_a_descriptor() { + use std::collections::HashSet; + use std::sync::Arc; + let table = Arc::new(Table::<64>::new()); + let access = Access::of(O_RDWR).unwrap(); + let claims: Vec<_> = (0..8) + .map(|_| { + let table = Arc::clone(&table); + std::thread::spawn(move || { + (0..8) + .map(|_| table.claim(0, access).unwrap()) + .collect::>() + }) + }) + .collect(); + let all: Vec = claims + .into_iter() + .flat_map(|thread| thread.join().unwrap()) + .collect(); + assert_eq!(all.len(), 64); + assert_eq!(all.iter().copied().collect::>().len(), 64); + assert_eq!(table.claim(0, access), Err(errno::EMFILE)); + } +} diff --git a/crates/c/src/lib.rs b/crates/c/src/lib.rs new file mode 100644 index 0000000..aa84b4d --- /dev/null +++ b/crates/c/src/lib.rs @@ -0,0 +1,86 @@ +//! The runtime of `libkore.so`: libkore's C ABI as one shared object, and the +//! host of an independent C program on Kore. +//! +//! A C program a compiler wrote on Kore (`tcc` at the shell) links +//! `libkore.so` with `-lkore` and includes `kore.h`. From one image the whole +//! system shares, it then has: +//! +//! - `kore.h`'s functions (`kore-ffi`), and the standard C names that +//! `kore/stdlib.h` and `kore/pthread.h` alias, as real symbols (`malloc` +//! to `free`, `pthread_create` to `pthread_exit`), over the process heap this +//! library installs; +//! - a host: the program's `main` runs on a main thread with a C stack, while +//! thread 0 serves its filesystem session ([`api::kore_fs_run`], declared +//! in `kore/program.h`) and ends the process when any thread exits. +//! +//! # The shared object +//! +//! A distribution builds the image from a `cdylib` crate whose `[lib]` is +//! named `kore` and whose source names this crate (`use kore_c as _;`); Kore's +//! is `user/runtime/libkore`. Its link carries the contract consumers rely on, +//! which `tools/check-program.py` checks for every ISA: +//! +//! - `--soname=libkore.so`, the name consumers record as `DT_NEEDED`; +//! - `--no-undefined`: the library binds nothing from the program it links into; +//! - a shared object with no entry point, so it registers as a library; +//! - static initial-exec TLS only, which the Kore user targets select; +//! - exactly the declared ABI exported (`kore.h`, `kore/program.h`, the C names +//! `kore/stdlib.h` and `kore/pthread.h` alias, and the C library's two +//! lifecycle entry points). +//! +//! It is a C program's runtime, never a Rust program's: a Rust executable links +//! `kore-std` with its own entry, and two runtimes in one process would grow one +//! heap supply behind two cursors. +#![cfg_attr(target_os = "none", no_std)] +#![cfg_attr(target_os = "none", feature(thread_local, c_variadic))] +#![deny(unsafe_op_in_unsafe_fn)] + +// The C ABI's exports are this image's reason to exist; naming the crate keeps +// them in the link. +use kore_ffi as _; +// Kore's C library, exported whole from this image. +#[cfg(target_os = "none")] +use kore_c_shim as _; + +pub mod api; +// Off Kore only the host suite reads the vector's storage: the page it is +// read from and the hook that answers it are the Kore program's alone. +#[cfg(any(test, target_os = "none"))] +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +mod args; +// Off Kore only the descriptor table's rows and the host suite read this: +// the face is the Kore program's. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +mod device; +#[cfg(any(test, target_os = "none"))] +mod exit; +// Off Kore only `kore_fd_describe` and the host suite read these two: the +// names, the verbs and most refusals are the Kore program's alone. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +mod fd; +#[cfg(target_os = "none")] +mod lock; +// Off Kore only the host suite and the device's mapping rules read this. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +mod mmap; +#[cfg(target_os = "none")] +mod posix; +#[cfg(target_os = "none")] +mod program; +#[cfg(any(test, target_os = "none"))] +mod stdio; + +/// The process heap over the founder's `role::HEAP` supply, grown on demand. +/// The program's allocations (`malloc`, `kore_alloc_*`) and this library's +/// own draw from it; with no supply every allocation answers null. +#[cfg(target_os = "none")] +#[global_allocator] +static PROCESS_HEAP: kore_std::runtime::heap::Heap = + kore_std::runtime::heap::Heap::new_in(kore_std::runtime::heap::HeapSupplySource); + +/// A panic in this library ends the program, recorded; never a silent park. +#[cfg(target_os = "none")] +#[panic_handler] +fn panic(info: &core::panic::PanicInfo<'_>) -> ! { + program::panicked(info) +} diff --git a/crates/c/src/lock.rs b/crates/c/src/lock.rs new file mode 100644 index 0000000..b54a2f1 --- /dev/null +++ b/crates/c/src/lock.rs @@ -0,0 +1,54 @@ +//! The futex mutex this runtime's threads take turns with: the streams' +//! lock, and the callers of thread 0's memory cell. +//! +//! The classic three states. A waiter's park is bounded by the runtime's +//! slice, because a holder's wake may not reach it: a thread holds a send +//! into a peer's doorbell only where the thread controller relays one. + +use core::sync::atomic::{AtomicU32, Ordering}; +use core::time::Duration; + +use kore_std::runtime::futex::{self, WaitOutcome}; +use kore_std::runtime::pace; + +const FREE: u32 = 0; +const HELD: u32 = 1; +const CONTENDED: u32 = 2; + +/// How long a waiter parks before it tries again: the runtime's slice for a +/// waiter its waker cannot reach (the thread controller's `UNREACHED_SLICE`). +const SLICE: Duration = Duration::from_nanos(pace::CEDE_NS); + +/// A mutex over one futex word. Not reentrant: a holder that needs to be +/// counts its own depth ([`crate::stdio`]). +pub(crate) struct Lock(AtomicU32); + +impl Lock { + pub(crate) const fn new() -> Self { + Self(AtomicU32::new(FREE)) + } + + pub(crate) fn acquire(&self) { + if self + .0 + .compare_exchange(FREE, HELD, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + return; + } + while self.0.swap(CONTENDED, Ordering::Acquire) != FREE { + // Bounded: the holder's wake may not reach this thread. + if futex::wait(&self.0, CONTENDED, Some(SLICE)) == WaitOutcome::Unsupported { + // No doorbell to park on: the deadline park any thread can. + let _ = pace::cede(); + } + } + } + + pub(crate) fn release(&self) { + // SeqCst, as the futex contract asks of the store before a wake. + if self.0.swap(FREE, Ordering::SeqCst) == CONTENDED { + futex::wake(&self.0); + } + } +} diff --git a/crates/c/src/mmap.rs b/crates/c/src/mmap.rs new file mode 100644 index 0000000..9f40a21 --- /dev/null +++ b/crates/c/src/mmap.rs @@ -0,0 +1,690 @@ +//! `mmap` and `munmap` over a memory capability. +//! +//! `mmap(NULL, len, prot, flags, fd, 0)` on a descriptor for the program's +//! memory supply (`/kore/memory`, [`crate::fd`]) answers `len` bytes of fresh, +//! zeroed memory, as mapping `/dev/zero` does on a POSIX system. Underneath +//! it is three of kore-rt's verbs, the sequence a Wasmtime engine's guest +//! memory already takes (`kore-wasm-platform`): +//! +//! 1. **Carve.** `heap::reserve_from_top` takes a lease off the top of the +//! supply the allocator grows from the bottom of. The two share cursors, so +//! a region and `malloc`'s extents never meet, which a raw carve at an +//! offset of the same capability could not promise. +//! 2. **Place.** The address comes from the free part of the task's link +//! window (`Rt::link_free_span`), from one process-wide cursor. The caller +//! never picks it: a hint is ignored and `MAP_FIXED` is refused. +//! 3. **Map.** The whole lease is mapped there. +//! +//! Thread 0 does all three, for whichever thread asked. Every thread that +//! may allocate is founded holding a share of the supply, and `open` and +//! `mmap` hold the caller's access to that share's rights. But a share +//! carries no `REVOKE` (the founder keeps it, to tear the share down), and +//! a region that can be unmapped again is a carve its holder may revoke. The +//! supply's root is thread 0's, so the caller posts its request there and +//! waits, as a filesystem session does. A mapping is therefore the +//! process's: any thread may unmap it. +//! +//! # What `munmap` returns +//! +//! The supply's cursors and the window's cursor only advance: bytes carved +//! and addresses placed are spent for the life of the program. So `munmap` +//! removes the mapping, and a stale access faults, but it *parks* the region: +//! its lease and its address stay with the program, and the next `mmap` of the +//! same length takes a parked region, zeroed again, before anything new is +//! carved. A program that maps and unmaps one size in a loop therefore uses +//! one region. One that maps ever-different sizes spends supply each time, +//! and ends with `ENOMEM`: the table holds [`REGIONS`] regions, and when all +//! are taken a parked one of another length is given up for good. +//! +//! The request rules and the region table are plain data, so the host suite +//! exercises the code the target runs; the verbs are the metal half. + +use core::ffi::c_int; + +use crate::fd::errno; + +/// The mapping granule: kore's user targets pin a 4 KiB page +/// (`max-page-size=4096`), which is also the granule a supply is carved in +/// (`kore_rt::mem_source::ALIGN`), so a region is exactly its pages. +pub(crate) const PAGE: usize = 4096; + +/// ``'s protections, musl's values. +pub(crate) const PROT_READ: c_int = 1; +pub(crate) const PROT_WRITE: c_int = 2; +/// ``'s flags, musl's values. +pub(crate) const MAP_SHARED: c_int = 0x01; +pub(crate) const MAP_PRIVATE: c_int = 0x02; +pub(crate) const MAP_FIXED: c_int = 0x10; + +/// A request `mmap` can serve. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Request { + /// Whether the mapping is writable. + pub(crate) write: bool, + /// The length asked for, in whole pages' bytes. + pub(crate) asked: usize, +} + +/// The whole pages `len` bytes take, or `None` when that overflows. +pub(crate) fn pages(len: usize) -> Option { + len.checked_add(PAGE - 1).map(|bytes| bytes & !(PAGE - 1)) +} + +impl Request { + /// What `mmap`'s arguments ask for, or the `errno` that refuses them. + /// + /// `EINVAL`: no length, an offset (a supply has no positions a caller can + /// name), neither or both of `MAP_SHARED` and `MAP_PRIVATE`, or a flag + /// with no meaning here. `ENOTSUP`: `MAP_FIXED` (the kernel's side picks + /// every address), and any protection but read or read-write: memory a + /// program wrote is never executable, and a mapping nothing can touch is + /// not one this API reserves. `ENOMEM`: a length that does not round to + /// pages. + /// + /// `MAP_SHARED` and `MAP_PRIVATE` are both accepted and mean the same: + /// the region is fresh and this program's alone, so there is no other + /// mapper for a store to reach or to be kept from. `PROT_WRITE` alone is + /// read-write, which POSIX permits and every Kore ISA's page tables imply. + pub(crate) fn of(len: usize, prot: c_int, flags: c_int, offset: i64) -> Result { + if len == 0 || offset != 0 { + return Err(errno::EINVAL); + } + if flags & MAP_FIXED != 0 { + return Err(errno::ENOTSUP); + } + if flags != MAP_SHARED && flags != MAP_PRIVATE { + return Err(errno::EINVAL); + } + let write = match prot { + PROT_READ => false, + p if p == PROT_WRITE || p == PROT_READ | PROT_WRITE => true, + _ => return Err(errno::ENOTSUP), + }; + let asked = pages(len).ok_or(errno::ENOMEM)?; + Ok(Self { write, asked }) + } +} + +/// How many regions a program may hold, mapped and parked together. A region +/// is a lease and a view in thread 0's arena and a row here; sixteen is the +/// table a Wasmtime engine keeps for the same purpose +/// (`kore_wasm_platform::MAX_MAPPINGS`), small enough to scan, and a program +/// that needs more distinct regions than that at once is better served by +/// one larger mapping it divides itself. +pub(crate) const REGIONS: usize = 16; + +/// One region carved for the program: mapped, or parked for the next `mmap` +/// of its length. `L` is its lease and `V` its view, capabilities dropped +/// with the row. +pub(crate) enum Region { + Vacant, + /// Mapped at `base` for a request of `asked` bytes. + Mapped { + lease: L, + view: V, + base: usize, + asked: usize, + }, + /// Unmapped, keeping its lease and the address it had. + Parked { + lease: L, + base: usize, + /// The lease's own length, which a map must name exactly. + len: usize, + asked: usize, + }, +} + +/// A program's regions. +pub(crate) struct Regions { + rows: [Region; N], +} + +impl Regions { + pub(crate) const fn new() -> Self { + Self { + rows: [const { Region::Vacant }; N], + } + } + + /// A row nothing occupies. + pub(crate) fn vacant(&self) -> Option { + self.rows + .iter() + .position(|row| matches!(row, Region::Vacant)) + } + + /// A parked region that served a request of exactly `asked` bytes. + pub(crate) fn parked(&self, asked: usize) -> Option { + self.rows + .iter() + .position(|row| matches!(row, Region::Parked { asked: was, .. } if *was == asked)) + } + + /// The region mapped at exactly `base`. + pub(crate) fn mapped(&self, base: usize) -> Option { + self.rows + .iter() + .position(|row| matches!(row, Region::Mapped { base: at, .. } if *at == base)) + } + + /// Any parked region: what a full table gives up for a new one. + pub(crate) fn any_parked(&self) -> Option { + self.rows + .iter() + .position(|row| matches!(row, Region::Parked { .. })) + } + + /// Take row `at`, leaving it vacant. + pub(crate) fn take(&mut self, at: usize) -> Region { + core::mem::replace(&mut self.rows[at], Region::Vacant) + } + + /// Fill row `at`. + pub(crate) fn put(&mut self, at: usize, region: Region) { + self.rows[at] = region; + } +} + +#[cfg(target_os = "none")] +pub(crate) use metal::{arm, map, map_zeroed, place, posted, serve, unmap}; + +#[cfg(target_os = "none")] +mod metal { + use core::cell::UnsafeCell; + use core::ffi::c_int; + use core::sync::atomic::{AtomicU32, Ordering}; + use core::time::Duration; + + use kore_ld::place::SharedPlacer; + use kore_std::runtime::cap::{MappedLeaseView, RetireOutcome, StashedLease, verbs}; + use kore_std::runtime::entry::Rt; + use kore_std::runtime::futex::{self, WaitOutcome}; + use kore_std::runtime::{pace, thread}; + + use super::{PAGE, REGIONS, Region, Regions, Request}; + use crate::fd::{Holding, errno}; + use crate::lock::Lock; + + /// Where regions land: the free part of the task's link window, armed + /// once by the host before any other thread exists. Unarmed (a program + /// with no host, or no link window), every `mmap` is `ENOMEM`. + static WINDOW: SharedPlacer = SharedPlacer::unarmed(); + + /// The process's regions, which thread 0 alone touches: the leases and + /// views in it are capabilities of its arena. + struct HostOnly(UnsafeCell); + // SAFETY: only thread 0 forms a reference ([`perform`]'s contract). + unsafe impl Sync for HostOnly {} + static REGIONS_HELD: HostOnly> = + HostOnly(UnsafeCell::new(Regions::new())); + + /// Publish the window regions are placed in: `rt`'s free link span + /// (thread 0's, which is the process's). A task without one leaves the + /// placer unarmed. + pub(crate) fn arm(rt: &Rt) { + if let Ok(span) = rt.link_free_span() { + let _ = WINDOW.arm(span.free, span.limit - span.free); + } + } + + /// An address for `len` bytes in the window regions land in, which is + /// spent for the life of the program. `None` when the window is unarmed + /// or has no room. + pub(crate) fn place(len: usize) -> Option { + WINDOW + .choose(len as u64, PAGE as u64) + .ok() + .and_then(|base| usize::try_from(base).ok()) + } + + /// Map `lease` whole at `base`, zeroed, with the request's access. + /// + /// The lease is mapped writable to be zeroed whatever was asked: a parked + /// region holds what its last user left, and a fresh one is zeroed too + /// rather than trusted to be. A read-only request is then mapped again + /// without write. + pub(crate) fn map_zeroed( + lease: &StashedLease, + base: usize, + len: usize, + write: bool, + ) -> Option { + let slot = lease.slot()?; + let mut view = verbs::map_owned_lease_at_rw(slot, base as u64, len as u64)?; + // SAFETY: `view` maps `len` writable bytes at its pointer, which + // nothing else has been handed yet. + unsafe { view.as_mut_ptr().write_bytes(0, len) }; + if write { + return Some(view); + } + match view.try_retire_outcome() { + RetireOutcome::Retired | RetireOutcome::AlreadyDead => {} + // Still mapped writable: refusing here would strand the view, so + // the caller gets nothing and the row keeps the lease. + RetireOutcome::Retryable => return None, + } + verbs::map_owned_lease_at(slot, base as u64, len as u64) + } + + /// One request of thread 0. + #[derive(Clone, Copy)] + enum Op { + Map(Request), + Unmap { base: usize, len: usize }, + } + + /// Carry `op` out. Thread 0 only: the supply's root, with the `REVOKE` + /// an unmappable carve needs, is a capability of its arena. + fn perform(op: Op) -> Result { + // SAFETY: thread 0 is the one caller, and it is not re-entered: no + // verb below calls back into this module. + let regions = unsafe { &mut *REGIONS_HELD.0.get() }; + match op { + Op::Map(request) => map_here(regions, request), + Op::Unmap { base, len } => unmap_here(regions, base, len).map(|()| 0), + } + } + + fn map_here( + regions: &mut Regions, + request: Request, + ) -> Result { + let (row, lease, base, len) = match regions.parked(request.asked) { + Some(row) => match regions.take(row) { + Region::Parked { + lease, base, len, .. + } => (row, lease, base, len), + // `parked` answered this row. + _ => return Err(errno::ENOMEM), + }, + None => { + // The row first: a carve with nowhere to be recorded would + // spend supply for nothing. A full table gives up a parked + // region of another length, whose lease ends with its row. + let row = match regions.vacant() { + Some(row) => row, + None => { + let row = regions.any_parked().ok_or(errno::ENOMEM)?; + drop(regions.take(row)); + row + } + }; + let lease = kore_std::runtime::heap::reserve_from_top(request.asked as u64) + .ok_or(errno::ENOMEM)?; + let len = lease + .slot() + .and_then(verbs::lease_geometry) + .and_then(|(_, len)| usize::try_from(len).ok()) + .ok_or(errno::ENOMEM)?; + let base = place(len).ok_or(errno::ENOMEM)?; + (row, lease, base, len) + } + }; + match map_zeroed(&lease, base, len, request.write) { + Some(view) => { + regions.put( + row, + Region::Mapped { + lease, + view, + base, + asked: request.asked, + }, + ); + Ok(base) + } + None => { + regions.put( + row, + Region::Parked { + lease, + base, + len, + asked: request.asked, + }, + ); + Err(errno::ENOMEM) + } + } + } + + fn unmap_here( + regions: &mut Regions, + base: usize, + len: usize, + ) -> Result<(), c_int> { + let asked = super::pages(len).filter(|&bytes| bytes != 0); + let row = regions.mapped(base).ok_or(errno::EINVAL)?; + let Region::Mapped { + lease, + mut view, + base, + asked: was, + } = regions.take(row) + else { + // `mapped` answered this row. + return Err(errno::EINVAL); + }; + if asked != Some(was) { + regions.put( + row, + Region::Mapped { + lease, + view, + base, + asked: was, + }, + ); + return Err(errno::EINVAL); + } + let len = view.len(); + match view.try_retire_outcome() { + RetireOutcome::Retired | RetireOutcome::AlreadyDead => { + regions.put( + row, + Region::Parked { + lease, + base, + len, + asked: was, + }, + ); + Ok(()) + } + RetireOutcome::Retryable => { + regions.put( + row, + Region::Mapped { + lease, + view, + base, + asked: was, + }, + ); + Err(errno::EAGAIN) + } + } + } + + // ── Reaching thread 0 ─────────────────────────────────────────────── + + // The cell's states. A caller owns it at IDLE (under [`CALLERS`]) and at + // SERVED, thread 0 at POSTED. + const IDLE: u32 = 0; + const POSTED: u32 = 1; + const SERVED: u32 = 2; + + /// One request at a time: the caller holding [`CALLERS`] writes `op` and + /// posts, thread 0 writes `answer` and serves. + struct Call { + state: AtomicU32, + op: UnsafeCell, + answer: UnsafeCell>, + } + // SAFETY: `op` and `answer` are touched only by the side `state` hands + // the cell to. + unsafe impl Sync for Call {} + static CALL: Call = Call { + state: AtomicU32::new(IDLE), + op: UnsafeCell::new(Op::Unmap { base: 0, len: 0 }), + answer: UnsafeCell::new(Ok(0)), + }; + + /// Callers take turns at the cell. + static CALLERS: Lock = Lock::new(); + + /// How long a caller waits before it re-reads the cell: thread 0 kicks + /// the main thread when it has answered, and holds no send into a + /// pthread's doorbell, which rides this slice (the streams' rule). + const SLICE: Duration = Duration::from_nanos(pace::CEDE_NS); + + /// Whether a request waits for thread 0. + pub(crate) fn posted() -> bool { + CALL.state.load(Ordering::Acquire) == POSTED + } + + /// Thread 0: answer the posted request, if there is one. `true` when one + /// was answered, and its caller should be woken. + pub(crate) fn serve() -> bool { + if !posted() { + return false; + } + // SAFETY: POSTED hands the cell to thread 0 until it stores SERVED. + let answer = perform(unsafe { *CALL.op.get() }); + // SAFETY: as above. + unsafe { *CALL.answer.get() = answer }; + // SeqCst, as the futex contract asks of the store before a wake. + CALL.state.store(SERVED, Ordering::SeqCst); + futex::wake(&CALL.state); + true + } + + /// Have thread 0 carry `op` out, and wait for its answer. + /// + /// Thread 0 itself (a filesystem callback, a program with no host) does + /// it in place. Any other thread of a program without a host has no one + /// to ask: `ENOMEM`. + fn ask(op: Op) -> Result { + if crate::program::on_host() { + return perform(op); + } + if !crate::program::hosted() { + return Err(errno::ENOMEM); + } + CALLERS.acquire(); + // SAFETY: IDLE under the lock hands the cell to this caller. + unsafe { *CALL.op.get() = op }; + CALL.state.store(POSTED, Ordering::SeqCst); + // Refused only when no post stood, and then thread 0 re-reads the + // cell before it parks. + let _ = thread::kick_parent(); + loop { + match CALL.state.load(Ordering::Acquire) { + SERVED => break, + state => { + if futex::wait(&CALL.state, state, Some(SLICE)) == WaitOutcome::Unsupported { + // No doorbell to park on: the deadline park any + // thread can. + let _ = pace::cede(); + } + } + } + } + // SAFETY: SERVED hands the cell back to this caller. + let answer = unsafe { *CALL.answer.get() }; + CALL.state.store(IDLE, Ordering::Release); + CALLERS.release(); + answer + } + + /// `mmap` on a memory supply the calling thread holds as `held`. + /// + /// The caller's own share is what `open` and this call hold the access + /// to. The region is carved by thread 0, which holds the supply's root: + /// a share a thread is founded with carries no `REVOKE`, and a carve + /// that can be unmapped again needs it. + /// + /// `EACCES`: a writable mapping of a capability without `MODIFY`. + /// `ENOMEM`: every row of the table is a live mapping, the supply or the + /// window has no room, the kernel refused the map, or the program has no + /// host to carve for this thread. A region whose map is refused is + /// parked, so its supply bytes and address serve the next request of its + /// length. + pub(crate) fn map(held: Holding, request: Request) -> Result<*mut u8, c_int> { + if request.write && !held.writable { + return Err(errno::EACCES); + } + ask(Op::Map(request)).map(|base| base as *mut u8) + } + + /// `munmap`: remove the mapping at `base`, which `len` must name whole, + /// and park its region. + /// + /// `EINVAL`: the program has no mapping at exactly `base`, or `len` is + /// not that mapping's length. `EAGAIN`: the kernel did not confirm the + /// unmap; the mapping stands and the call may be made again. + pub(crate) fn unmap(base: usize, len: usize) -> Result<(), c_int> { + ask(Op::Unmap { base, len }).map(|_| ()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const RW: c_int = PROT_READ | PROT_WRITE; + + #[test] + fn a_length_is_whole_pages() { + assert_eq!(pages(1), Some(PAGE)); + assert_eq!(pages(PAGE), Some(PAGE)); + assert_eq!(pages(PAGE + 1), Some(2 * PAGE)); + assert_eq!(pages(usize::MAX), None); + assert_eq!(pages(usize::MAX - PAGE + 2), None); + assert_eq!(pages(usize::MAX - PAGE + 1), Some(usize::MAX - PAGE + 1)); + } + + #[test] + fn a_readable_region_of_the_supply_is_served() { + assert_eq!( + Request::of(1 << 20, RW, MAP_PRIVATE, 0), + Ok(Request { + write: true, + asked: 1 << 20 + }) + ); + assert_eq!( + Request::of(1, PROT_READ, MAP_SHARED, 0), + Ok(Request { + write: false, + asked: PAGE + }) + ); + // Write alone is read-write. + assert_eq!( + Request::of(PAGE, PROT_WRITE, MAP_SHARED, 0), + Ok(Request { + write: true, + asked: PAGE + }) + ); + } + + #[test] + fn what_has_no_meaning_here_is_refused_by_name() { + // No length, and an offset into a supply. + assert_eq!(Request::of(0, RW, MAP_PRIVATE, 0), Err(errno::EINVAL)); + assert_eq!( + Request::of(PAGE, RW, MAP_PRIVATE, PAGE as i64), + Err(errno::EINVAL) + ); + assert_eq!(Request::of(PAGE, RW, MAP_PRIVATE, -1), Err(errno::EINVAL)); + // Neither, both, or a stranger: MAP_ANONYMOUS is 0x20. + assert_eq!(Request::of(PAGE, RW, 0, 0), Err(errno::EINVAL)); + assert_eq!( + Request::of(PAGE, RW, MAP_SHARED | MAP_PRIVATE, 0), + Err(errno::EINVAL) + ); + assert_eq!( + Request::of(PAGE, RW, MAP_PRIVATE | 0x20, 0), + Err(errno::EINVAL) + ); + // The caller never picks the address. + assert_eq!( + Request::of(PAGE, RW, MAP_PRIVATE | MAP_FIXED, 0), + Err(errno::ENOTSUP) + ); + // No access, executable (4), and bits past the three. + for prot in [0, 4, PROT_READ | 4, RW | 4, 8, -1] { + assert_eq!( + Request::of(PAGE, prot, MAP_PRIVATE, 0), + Err(errno::ENOTSUP), + "{prot}" + ); + } + assert_eq!( + Request::of(usize::MAX, RW, MAP_PRIVATE, 0), + Err(errno::ENOMEM) + ); + } + + type Rows = Regions; + + fn mapped(lease: u32, base: usize, asked: usize) -> Region { + Region::Mapped { + lease, + view: lease, + base, + asked, + } + } + + fn park(rows: &mut Rows, base: usize) { + let row = rows.mapped(base).unwrap(); + let Region::Mapped { + lease, base, asked, .. + } = rows.take(row) + else { + unreachable!() + }; + rows.put( + row, + Region::Parked { + lease, + base, + len: asked, + asked, + }, + ); + } + + #[test] + fn an_unmapped_region_serves_the_next_request_of_its_length() { + let mut rows = Rows::new(); + assert_eq!(rows.vacant(), Some(0)); + rows.put(0, mapped(7, 0x1000, PAGE)); + rows.put(1, mapped(8, 0x9000, 2 * PAGE)); + assert_eq!(rows.vacant(), Some(2)); + assert_eq!((rows.parked(PAGE), rows.any_parked()), (None, None)); + park(&mut rows, 0x1000); + // Parked, it is no longer a mapping, and only its own length takes it. + assert_eq!(rows.mapped(0x1000), None); + assert_eq!(rows.parked(2 * PAGE), None); + assert_eq!(rows.parked(PAGE), Some(0)); + let Region::Parked { lease, base, .. } = rows.take(0) else { + panic!("the parked row") + }; + // The same lease at the same address. + assert_eq!((lease, base), (7, 0x1000)); + assert_eq!(rows.vacant(), Some(0)); + } + + #[test] + fn a_mapping_is_found_by_its_base_alone() { + let mut rows = Rows::new(); + rows.put(1, mapped(1, 0x4000, 4 * PAGE)); + assert_eq!(rows.mapped(0x4000), Some(1)); + // An address inside it is not its base. + assert_eq!(rows.mapped(0x5000), None); + assert_eq!(rows.mapped(0), None); + } + + #[test] + fn a_full_table_has_no_vacant_row() { + let mut rows = Rows::new(); + for at in 0..3 { + rows.put(at, mapped(at as u32, 0x1000 * (at + 1), PAGE)); + } + assert_eq!(rows.vacant(), None); + park(&mut rows, 0x2000); + // A parked row is still occupied: it holds a lease. It is what a + // request of another length takes the place of. + assert_eq!(rows.vacant(), None); + assert_eq!(rows.parked(PAGE), Some(1)); + assert_eq!(rows.parked(2 * PAGE), None); + assert_eq!(rows.any_parked(), Some(1)); + assert!(matches!(rows.take(1), Region::Parked { lease: 1, .. })); + assert_eq!((rows.vacant(), rows.any_parked()), (Some(1), None)); + } +} diff --git a/crates/c/src/posix.rs b/crates/c/src/posix.rs new file mode 100644 index 0000000..113018d --- /dev/null +++ b/crates/c/src/posix.rs @@ -0,0 +1,368 @@ +//! The POSIX names over [`crate::fd`], [`crate::mmap`] and [`crate::device`]: +//! `open`, `close`, `read`, `write`, `poll`, `mmap` and `munmap`, as +//! ``, ``, `` and `` declare them. Each +//! sets `errno` and answers `-1` (`MAP_FAILED` for `mmap`) when it refuses. +//! +//! Exported on Kore alone, like the rest of the C library: on a host these +//! names are the host libc's. + +use core::ffi::{CStr, c_char, c_int, c_short, c_ulong, c_void}; + +use kore_c_shim::errno::set as set_errno; + +use crate::device; +use crate::fd::{self, Kind, NAMES, TABLE, errno}; +use crate::mmap::{self, Request}; + +/// `mmap`'s refusal: `MAP_FAILED`, `(void *)-1`. +const MAP_FAILED: *mut c_void = usize::MAX as *mut c_void; + +fn refused(code: c_int) -> c_int { + set_errno(code); + -1 +} + +/// `open(path, flags)`: a descriptor for the capability `path` names +/// ([`fd::open`]). The mode a variadic `open` may carry belongs to `O_CREAT`, +/// which no name here takes, so it is never read. +/// +/// # Safety +/// `path` is null or a NUL-terminated string. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn open(path: *const c_char, flags: c_int, _mode: ...) -> c_int { + if path.is_null() { + return refused(errno::EFAULT); + } + // SAFETY: the caller's contract: a NUL-terminated string. + let path = unsafe { CStr::from_ptr(path) }.to_bytes(); + fd::open(path, flags).unwrap_or_else(refused) +} + +/// `close(fd)`: forget the descriptor. The capability it named stays the +/// program's, and mappings made through it stand. A device's descriptor +/// gives the device up as well ([`device`]). +#[unsafe(no_mangle)] +pub extern "C" fn close(fd: c_int) -> c_int { + let device = matches!( + TABLE.get(fd), + Ok((row, _)) if matches!(NAMES[row].kind, Kind::Device { .. }) + ); + match TABLE.release(fd) { + Ok(()) => { + if device { + device::close(); + } + 0 + } + Err(code) => refused(code), + } +} + +/// The numbers of `stdout` and `stderr`, which no `open` hands out. +const STDOUT: c_int = 1; +const STDERR: c_int = 2; + +/// `read(fd, buf, count)`: on a device's descriptor, wait for an interrupt +/// and answer the device's interrupt count, one 32-bit word, as a UIO device +/// does ([`device::read`]). +/// +/// `EBADF`: `fd` is not open for reading. `EINVAL`: `count` is not four, or +/// the descriptor names something that is not read (the memory supply). +/// `EFAULT`: `buf` is null. Otherwise as [`device::read`] refuses. +/// +/// # Safety +/// `buf` is null or valid for `count` bytes of writes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read(fd: c_int, buf: *mut c_void, count: usize) -> isize { + let answer = TABLE.get(fd).and_then(|(row, access)| { + if !access.read { + return Err(errno::EBADF); + } + match NAMES[row].kind { + Kind::Memory => Err(errno::EINVAL), + Kind::Device { .. } => { + if count != device::WORD { + return Err(errno::EINVAL); + } + if buf.is_null() { + return Err(errno::EFAULT); + } + device::read() + } + } + }); + match answer { + Ok(interrupts) => { + // SAFETY: the caller's contract: four writable bytes, of any + // alignment. + unsafe { buf.cast::().write_unaligned(interrupts) }; + device::WORD as isize + } + Err(code) => refused(code) as isize, + } +} + +/// `write(fd, buf, count)`: on descriptor 1 or 2, `count` bytes to `stdout` +/// or `stderr`, in order with what `` writes; on a device's +/// descriptor, one 32-bit word rings the doorbell ([`device::ring`]). The +/// word is not read. +/// +/// `EBADF`: `fd` is not open for writing, or the launch carried no stream. +/// `EINVAL`: a device was written anything but four bytes, or the descriptor +/// names something that is not written (the memory supply). `EFAULT`: `buf` +/// is null and `count` is not zero. Otherwise as [`device::ring`] refuses. +/// +/// # Safety +/// `buf` is null or valid for `count` bytes of reads. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn write(fd: c_int, buf: *const c_void, count: usize) -> isize { + if fd == STDOUT || fd == STDERR { + if count == 0 { + return 0; + } + if buf.is_null() { + return refused(errno::EFAULT) as isize; + } + // POSIX leaves a count past `SSIZE_MAX` to the implementation: this + // one writes what the return value can report. + let count = count.min(isize::MAX as usize); + // SAFETY: the caller's contract: `count` readable bytes. + let bytes = unsafe { core::slice::from_raw_parts(buf.cast::(), count) }; + return if crate::stdio::write_standard(fd == STDERR, bytes) { + count as isize + } else { + refused(errno::EBADF) as isize + }; + } + let answer = TABLE.get(fd).and_then(|(row, access)| { + if !access.write { + return Err(errno::EBADF); + } + match NAMES[row].kind { + Kind::Memory => Err(errno::EINVAL), + Kind::Device { .. } => { + if count != device::WORD { + return Err(errno::EINVAL); + } + if buf.is_null() { + return Err(errno::EFAULT); + } + device::ring() + } + } + }); + match answer { + Ok(()) => device::WORD as isize, + Err(code) => refused(code) as isize, + } +} + +/// ``'s `struct pollfd`, musl's layout. +#[repr(C)] +pub struct PollFd { + fd: c_int, + events: c_short, + revents: c_short, +} + +/// ``'s events, musl's values. +const POLLIN: c_short = 0x001; +const POLLOUT: c_short = 0x004; +const POLLERR: c_short = 0x008; +const POLLNVAL: c_short = 0x020; + +/// What each entry can report now, and whether one of them asks for a +/// device's interrupt and does not have it: the one thing a `poll` here can +/// wait for. +/// +/// # Safety +/// `fds` is valid for `nfds` entries. +unsafe fn poll_pass(fds: *mut PollFd, nfds: usize) -> (c_int, bool) { + let mut ready = 0; + let mut waits = false; + for at in 0..nfds { + // SAFETY: the caller's contract; one entry is borrowed at a time. + let entry = unsafe { &mut *fds.add(at) }; + entry.revents = 0; + if entry.fd < 0 { + // POSIX: a negative descriptor is skipped. + continue; + } + entry.revents = match TABLE.get(entry.fd) { + // A standard stream takes a write whenever it is asked. + Err(_) if entry.fd == STDOUT || entry.fd == STDERR => entry.events & POLLOUT, + Err(_) => POLLNVAL, + Ok((row, access)) => match NAMES[row].kind { + // Mapping a supply never waits. + Kind::Memory => entry.events & (POLLIN | POLLOUT), + Kind::Device { .. } => { + // The doorbell never waits. + let mut events = if access.write { + entry.events & POLLOUT + } else { + 0 + }; + if access.read && entry.events & POLLIN != 0 { + match device::readable() { + Some(true) => events |= POLLIN, + Some(false) => waits = true, + None => events |= POLLNVAL, + } + } + events + } + }, + }; + if entry.revents != 0 { + ready += 1; + } + } + (ready, waits) +} + +/// `poll(fds, nfds, timeout)`: which descriptors are ready, waiting up to +/// `timeout` milliseconds (forever when negative) for a device's interrupt. +/// +/// `POLLIN` on a device's descriptor is an interrupt waiting to be `read`; +/// `POLLOUT` is always ready, there and on the standard output streams; a +/// memory supply's descriptor is always ready; a number that is not open +/// reports `POLLNVAL`. A device that closes during the wait reports +/// `POLLNVAL`, and one this thread cannot wait on `POLLERR`. +/// +/// `EFAULT`: `fds` is null and `nfds` is not zero. `EINVAL`: more entries +/// than there are descriptors, or a wait with no timeout that nothing in the +/// set can end. A wait with a timeout and nothing to wait for sleeps it out +/// and answers zero. +/// +/// # Safety +/// `fds` is null or valid for `nfds` entries of reads and writes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn poll(fds: *mut PollFd, nfds: c_ulong, timeout: c_int) -> c_int { + let Some(nfds) = usize::try_from(nfds) + .ok() + .filter(|&nfds| nfds <= fd::CAPACITY + fd::FIRST as usize) + else { + return refused(errno::EINVAL); + }; + if nfds != 0 && fds.is_null() { + return refused(errno::EFAULT); + } + // SAFETY: the caller's contract. + let (ready, waits) = unsafe { poll_pass(fds, nfds) }; + if ready != 0 || timeout == 0 { + return ready; + } + let deadline = match u64::try_from(timeout) { + Ok(milliseconds) => { + // The calibrated counter, or the kernel's own stamp on a target + // without one: a timeout is honoured wherever a thread can park. + let Some(now) = kore_std::runtime::clock::deadline_base_nanos() else { + return refused(errno::EINVAL); + }; + Some(now.saturating_add(milliseconds.saturating_mul(1_000_000))) + } + Err(_) => None, + }; + if !waits { + return match deadline { + // Nothing here can become ready: the timeout is the whole wait. + Some(at) => { + // A deadline park of this thread; a thread that cannot park + // answers at once, which the caller's own loop absorbs. + let _ = kore_std::runtime::cap::verbs::sleep_until(at); + 0 + } + None => refused(errno::EINVAL), + }; + } + match device::wait_readable(deadline) { + // An interrupt, or the timeout: either way the entries say which. + // SAFETY: the caller's contract. + Ok(_) => unsafe { poll_pass(fds, nfds) }.0, + Err(code) => { + // The device closed, or cannot be waited on: say so on its + // entries, which is what ended the wait. + let event = if code == errno::EBADF { + POLLNVAL + } else { + POLLERR + }; + let mut ready = 0; + for at in 0..nfds { + // SAFETY: the caller's contract. + let entry = unsafe { &mut *fds.add(at) }; + let device = matches!( + TABLE.get(entry.fd), + Ok((row, _)) if matches!(NAMES[row].kind, Kind::Device { .. }) + ); + if device && entry.events & POLLIN != 0 { + entry.revents |= event; + } + if entry.revents != 0 { + ready += 1; + } + } + ready + } + } +} + +/// `mmap(addr, len, prot, flags, fd, offset)`: of the memory supply +/// ([`mmap`]'s module docs), or of a device, whose offset says which of its +/// maps is meant ([`device::Map`]). `addr` is a hint POSIX lets an +/// implementation ignore, and it is. +/// +/// `EBADF`: `fd` is not open. `EACCES`: it was not opened for reading, or a +/// writable mapping was asked of one not opened for writing. `ENXIO`: the +/// calling thread does not hold the capability. Otherwise as +/// [`Request::of`] and [`mmap::map`] refuse. +#[unsafe(no_mangle)] +pub extern "C" fn mmap( + _addr: *mut c_void, + len: usize, + prot: c_int, + flags: c_int, + fd: c_int, + offset: i64, +) -> *mut c_void { + let mapped = TABLE.get(fd).and_then(|(row, access)| { + let name = &NAMES[row]; + match name.kind { + Kind::Memory => { + let request = Request::of(len, prot, flags, offset)?; + if !access.read || (request.write && !access.write) { + return Err(errno::EACCES); + } + let held = fd::holding(name).ok_or(errno::ENXIO)?; + mmap::map(held, request) + } + Kind::Device { .. } => { + let map = device::Map::of(len, prot, flags, offset)?; + if !access.read || (map.writes(prot) && !access.write) { + return Err(errno::EACCES); + } + // The calling thread's own share, as for any descriptor. + fd::holding(name).ok_or(errno::ENXIO)?; + device::map(map) + } + } + }); + match mapped { + Ok(base) => base.cast(), + Err(code) => { + set_errno(code); + MAP_FAILED + } + } +} + +/// `munmap(addr, len)`: unmap the program's mapping at `addr`, whole +/// ([`mmap::unmap`], or [`device::unmap`] for a device's). +#[unsafe(no_mangle)] +pub extern "C" fn munmap(addr: *mut c_void, len: usize) -> c_int { + let unmapped = device::unmap(addr.addr(), len).unwrap_or_else(|| mmap::unmap(addr.addr(), len)); + match unmapped { + Ok(()) => 0, + Err(code) => refused(code), + } +} diff --git a/crates/c/src/program.rs b/crates/c/src/program.rs new file mode 100644 index 0000000..96a8055 --- /dev/null +++ b/crates/c/src/program.rs @@ -0,0 +1,705 @@ +//! The host of an independent C program: `libkore.so`'s half of the program +//! lifecycle Kore's C library starts and ends. +//! +//! A C program on Kore links its C library's start files whatever else it +//! links. Their `_start` hands the program to the C library, which, when the +//! program also links this library, hands it on to [`__kore_program_start`]. +//! From there: +//! +//! - **The program runs on its main thread**, a worker with a C stack of +//! [`KORE_MAIN_STACK_BYTES`](crate::api::KORE_MAIN_STACK_BYTES), not on +//! thread 0. Thread 0's stack is the +//! founding's, sized for a runtime, and thread 0 is the process owner: the +//! one thread that may stop the others. +//! - **Thread 0 is the host.** It keeps the task's executor and its service +//! faces, and serves what the program's threads ask of it through one +//! request cell and one word, the shape `kore-lua`'s VM bridge and `tcc` +//! share: the program's filesystem session ([`kore_fs_run`](crate::api::kore_fs_run)) +//! and its end. +//! - **Any thread can end the program.** `exit` and `_Exit` reach +//! [`__kore_program_exit`] once the C library has run its `atexit` +//! handlers; the first request wins. The host stops every other thread, +//! runs the finalizers when none was still running, and ends the process +//! with the C status through the C library's own exit. +//! +//! ```text +//! thread 0 (the host) main thread (C) +//! ─────────────────── ─────────────── +//! found the main thread ────────────────────▶ program(): main(), then exit +//! park on the wake channel … kore_fs_run(step, user) +//! ◀──────── SESSION_POSTED, kick_parent ───── futex::wait on the word +//! run the session (its C callbacks run here) +//! SESSION_SERVED, kick ─────────────────────▶ returns the session's status +//! … exit(status): atexit handlers +//! ◀──────── the exit word, kick_parent ────── the thread ends +//! join it, raise the barrier, finalize +//! end(status): the C library's exit +//! ``` +//! +//! Every park is a doorbell park; nothing here spins. The ordering is +//! `kore-lua`'s: a poster stores, then kicks; the host stands its post, +//! re-reads both words, then parks, so a kick that found no post standing +//! announced a store the re-read sees. Every thread of the program can kick +//! the host: the main thread holds the wake channel's send, and each thread it +//! starts is founded holding one too (`thread_controller`). +//! +//! A launch that cannot carry the host (no channel space or heap, a stack +//! window that cannot place the main thread) runs the program on thread 0, as +//! the C library alone would: allocation, threads, time and random work, a +//! filesystem session is refused, and only thread 0's exit ends the program. + +extern crate alloc; + +use core::cell::{Cell, UnsafeCell}; +use core::ffi::{c_int, c_void}; +use core::future::poll_fn; +use core::pin::pin; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use core::task::Poll; + +use kore_ffi::{KoreFsError, KoreFsStep, KoreStatus, Session}; + +use crate::exit::{Exit, ExitWord}; +use kore_std::ExitCode; +use kore_std::abi::session::Status; +use kore_std::fs::Fs; +use kore_std::runtime::chan::{NoticeRx, RecvError}; +use kore_std::runtime::entry::{self, Rt}; +use kore_std::runtime::executor::{self, Executor}; +use kore_std::runtime::futex::{self, WaitOutcome}; +use kore_std::runtime::thread::{self, DoorbellWorker, WorkerStack}; +use kore_std::runtime::transport::TrapTransport; +use kore_std::runtime::{crash, startup}; + +/// The C library's program: runs `main`, then `exit`s with its value. +pub type Program = unsafe extern "C" fn() -> !; +/// The C library's exit: commits `status` on the calling thread's own handle. +pub type End = unsafe extern "C" fn(c_int) -> !; + +/// The program, as [`__kore_program_start`] received it, for the main thread. +static PROGRAM: AtomicUsize = AtomicUsize::new(0); +/// The C library's exit, as [`__kore_program_start`] received it, for an +/// [`abort`] on thread 0. +static END: AtomicUsize = AtomicUsize::new(0); +/// Set by the first [`__kore_program_start`]; a second never founds a host. +static STARTED: AtomicBool = AtomicBool::new(false); +/// The host serves: the main thread exists and thread 0 parks for it. Set +/// before the main thread is founded, since its first call may need it. +static HOSTED: AtomicBool = AtomicBool::new(false); + +/// This thread's part in the program. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Role { + /// A thread the program started, or thread 0 before the program runs. + Other, + /// Thread 0, once it holds the program: the host, or the program itself + /// when the launch could not carry a host. + Host, + /// The thread `main` runs on. + Main, +} + +#[thread_local] +static ROLE: Cell = Cell::new(Role::Other); + +/// Whether the calling thread is the host: thread 0 holding the program. +pub(crate) fn on_host() -> bool { + ROLE.get() == Role::Host +} + +/// Whether a host serves: thread 0 answers what other threads post. +pub(crate) fn hosted() -> bool { + HOSTED.load(Ordering::Acquire) +} + +/// Whether the calling thread is the main thread, the one the host can kick. +pub(crate) fn on_main() -> bool { + ROLE.get() == Role::Main +} + +// ── The end ───────────────────────────────────────────────────────────── + +/// The program's end, once asked for (see [`ExitWord`]). +static EXIT: ExitWord = ExitWord::new(); + +// ── The session cell ──────────────────────────────────────────────────── + +// The word's states. The main thread owns the cell at IDLE and SERVED, the +// host at POSTED; every hand-over is a Release store matched by an Acquire. +const IDLE: u32 = 0; +const POSTED: u32 = 1; +const SERVED: u32 = 2; + +struct Request { + state: AtomicU32, + payload: UnsafeCell, +} + +struct Payload { + // main thread → host: the session's callback program. + step: Option, + user: *mut c_void, + // host → main thread: how the session ended. + status: KoreStatus, + error: KoreFsError, +} + +// SAFETY: `payload` is touched only by the side the word hands it to (above). +unsafe impl Sync for Request {} + +static SESSION: Request = Request { + state: AtomicU32::new(IDLE), + payload: UnsafeCell::new(Payload { + step: None, + user: core::ptr::null_mut(), + status: KoreStatus::Ok, + error: KoreFsError::None, + }), +}; + +impl Request { + /// # Safety + /// The caller observed (Acquire) a state that hands it the cell, and holds + /// no other reference across the store that hands it back. + #[allow(clippy::mut_from_ref)] + unsafe fn payload(&self) -> &mut Payload { + unsafe { &mut *self.payload.get() } + } +} + +/// Anything the host must answer: an end, a session, a mapping, or a +/// request of the program's device. +fn pending() -> bool { + EXIT.posted().is_some() + || SESSION.state.load(Ordering::Acquire) == POSTED + || crate::mmap::posted() + || crate::device::posted() +} + +// ── Entry points the C library calls ──────────────────────────────────── + +/// Run the program. The C library's start hands it here when the program +/// links libkore: `program` runs `main` and then `exit`; `end` is the C +/// library's exit, which commits a status on the calling thread's own handle. +/// Never returns. +/// +/// # Safety +/// Called once, by the C library's start on the founded thread and before the +/// program has run. Neither `program` nor `end` returns. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn __kore_program_start(program: Option, end: Option) -> ! { + let (Some(program), Some(end)) = (program, end) else { + // A C library that hands over nothing to run has broken its half of + // this contract; say so rather than run on. + thread::exit(ExitCode::INTERNAL) + }; + if STARTED.swap(true, Ordering::AcqRel) { + // A second start founds no second host. + unsafe { program() } + } + ROLE.set(Role::Host); + PROGRAM.store(program as usize, Ordering::Release); + END.store(end as usize, Ordering::Release); + match Founded::found() { + Some(host) => host.serve(end), + None => unsafe { program() }, + } +} + +/// End the program with `status`. The C library's `exit` calls this after its +/// `atexit` handlers with `finalize` nonzero, `_Exit` with zero, on whatever +/// thread called them. +/// +/// Returns only when the caller must commit the exit itself: it is thread 0, +/// every other thread is stopped, and, for `exit`, the finalizers have run if +/// none was still running. On any other thread the request goes to the host +/// and the calling thread ends. +#[unsafe(no_mangle)] +pub extern "C" fn __kore_program_exit(status: c_int, finalize: c_int) { + let finalize = finalize != 0; + match ROLE.get() { + // Thread 0 itself: the program when there is no host, or a callback + // or finalizer the host is running. It cannot wait for its own loop, + // so it relays what the launcher has room for now; the barrier is + // this thread's. + Role::Host => { + crate::stdio::flush_now(); + startup::barrier(finalize) + } + role if HOSTED.load(Ordering::Acquire) => { + let _ = EXIT.post(Exit { + status, + finalize, + from_main: role == Role::Main, + }); + // Refused only when no post stood, and then the host re-reads the + // word before it parks: the store above reaches it either way. + let _ = thread::kick_parent(); + thread::exit(ExitCode::OK) + } + // No host: only thread 0 owns the process. Anywhere else the barrier + // refuses and ends this thread alone. + _ => startup::barrier(finalize), + } +} + +/// `abort` (C11 7.22.4.1): the program's abnormal end. It ends with the +/// aborted status, 134 — what a shell reports for a program `SIGABRT` ended — +/// and, like `_Exit`, runs no `atexit` handler or finalizer. No stream +/// buffers, so what the program wrote before has already left it: the host's +/// loop relays all of it before the end when another thread aborts, and a +/// callback's `abort` on the host itself relays what the launcher has room +/// for at once (`stdio::flush_now`). +/// +/// The C library's `_Exit` lives in the program, so this takes `_Exit`'s +/// path itself: any other thread hands the status to the host and ends, and +/// thread 0, the barrier raised, commits the end the C library handed over. +#[unsafe(no_mangle)] +pub extern "C" fn abort() -> ! { + let status = ExitCode::ABORTED.get(); + __kore_program_exit(status, 0); + match END.load(Ordering::Acquire) { + // A program the C library never started: thread 0 ends the task. + 0 => thread::exit(ExitCode::ABORTED), + end => { + // SAFETY: `__kore_program_start` stored an `End`, which never + // returns; every other thread is stopped. + let end: End = unsafe { core::mem::transmute(end) }; + unsafe { end(status) } + } + } +} + +/// The main thread's half of [`kore_fs_run`](crate::api::kore_fs_run): post +/// the session to the host and wait for its answer. +/// +/// # Safety +/// `kore_fs_run`'s contract for `step` and `user`. +pub(crate) unsafe fn session(step: KoreFsStep, user: *mut c_void) -> (KoreStatus, KoreFsError) { + if ROLE.get() != Role::Main || !HOSTED.load(Ordering::Acquire) { + return (KoreStatus::Refused, KoreFsError::None); + } + // The main thread is the only poster, and it waits out each request: the + // cell is idle whenever control is here. + { + // SAFETY: IDLE hands the cell to the main thread. + let payload = unsafe { SESSION.payload() }; + payload.step = Some(step); + payload.user = user; + } + SESSION.state.store(POSTED, Ordering::Release); + // Refused only when no post stood, and then the host re-reads the word + // before it parks. + let _ = thread::kick_parent(); + loop { + match SESSION.state.load(Ordering::Acquire) { + SERVED => break, + state => { + if futex::wait(&SESSION.state, state, None) == WaitOutcome::Unsupported { + // The main thread is founded with its own doorbell, so it + // can always park; if it cannot, spinning is the one + // thing left, and this host will not. End the program. + fatal() + } + } + } + } + let answer = { + // SAFETY: SERVED hands the cell back to the main thread. + let payload = unsafe { SESSION.payload() }; + payload.step = None; + payload.user = core::ptr::null_mut(); + (payload.status, payload.error) + }; + SESSION.state.store(IDLE, Ordering::Release); + answer +} + +/// This library's panic: the record first, then the end. A panic on thread 0 +/// stops the program there; anywhere else it asks the host to, so the process +/// ends with the panic rather than running on without the panicking thread. +pub(crate) fn panicked(info: &core::panic::PanicInfo<'_>) -> ! { + crash::report(info); + match ROLE.get() { + Role::Host => { + crate::stdio::flush_now(); + startup::barrier(false) + } + role if HOSTED.load(Ordering::Acquire) => { + let _ = EXIT.post(Exit { + status: ExitCode::PANIC.get(), + finalize: false, + from_main: role == Role::Main, + }); + let _ = thread::kick_parent(); + } + _ => {} + } + thread::exit(ExitCode::PANIC) +} + +/// A host invariant broke on a program thread: end the program with +/// `INTERNAL`, never by spinning. +fn fatal() -> ! { + let _ = EXIT.post(Exit { + status: ExitCode::INTERNAL.get(), + finalize: false, + from_main: ROLE.get() == Role::Main, + }); + let _ = thread::kick_parent(); + thread::exit(ExitCode::INTERNAL) +} + +// ── The host ──────────────────────────────────────────────────────────── + +/// The host's wake channel broke or a park was refused: it cannot hear the +/// program, so it ends it. +struct Fatal; + +/// Thread 0 with its program founded: the runtime context, the executor's +/// doorbell, the launcher's stream when the launch carried one, and the main +/// thread. +struct Founded { + rt: Rt, + doorbell: executor::Doorbell, + output: Option<&'static crate::stdio::Endpoint>, + worker: DoorbellWorker, +} + +impl Founded { + /// Found the main thread. `None` when the launch cannot carry the host. + fn found() -> Option { + let rt = entry::thread_context()?; + let doorbell = rt.doorbell()?; + // The C streams reach the launcher through this thread: map its + // stream and install the sink before any thread can write. + let output = crate::stdio::install(&rt); + // `main`'s arguments, read from the launcher's page while this is + // still the only thread. A line that is no argument vector cannot + // be handed to `main`: say why on `stderr` and end here, before the + // program runs. + if let Err(malformed) = crate::args::capture(&rt) { + crate::stdio::report(alloc::format!("kore: {malformed}\n").as_bytes()); + crate::stdio::flush_now(); + // SAFETY: the C library's exit, which `__kore_program_start` + // stored before it called this. + let end: End = unsafe { core::mem::transmute(END.load(Ordering::Acquire)) }; + unsafe { end(ExitCode::USAGE.get()) } + } + // Where `mmap` places: this task's free link span, published before + // any thread that could map exists. + crate::mmap::arm(&rt); + HOSTED.store(true, Ordering::Release); + // A mint grant of its own: the first `pthread_create` on the main + // thread starts kore-rt's thread controller there, which mints the + // doorbells its threads park on (`thread_controller`). Without one the + // controller still runs, taking the deadline park. + // + // And its share of the launch's virtual-device face, when it carried + // one: the interrupt and the doorbell are a thread's own + // capabilities, so each thread of the program is founded holding + // them. This thread gives the main thread its share here; the + // controller the main thread starts hands it on to every thread it + // founds. + let mut endowment: alloc::vec::Vec = + thread::channel_space_endowment().into_iter().collect(); + endowment.extend(crate::device::shares()); + for share in kore_std::vdev::client::THREAD_SHARES { + let _ = kore_ffi::hand_on_to_threads(share); + } + match thread::spawn_with_doorbell_on( + main_thread, + &endowment, + WorkerStack::Bytes(crate::api::KORE_MAIN_STACK_BYTES), + ) { + Some(worker) => Some(Self { + rt, + doorbell, + output, + worker, + }), + None => { + HOSTED.store(false, Ordering::Release); + // No host will drain the stream: writes report `EOF`, as + // they do for a launch that carried none. + if output.is_some() { + crate::stdio::abandon(); + } + None + } + } + } + + /// Serve the program until it ends, then end the process. + fn serve(self, end: End) -> ! { + let Self { + rt, + doorbell, + output, + worker, + } = self; + let transport = TrapTransport::new(); + let exit = { + let mut executor = Executor::new(entry::transport(), doorbell); + executor.run(async { + let exit = host(&rt, &transport, &worker, output).await; + // Whatever ended the program, what it wrote reaches the + // launcher, and then the stream's close, before the process + // ends: the launcher shows all of it before the status. + if let Some(output) = output { + crate::stdio::finish(output, &|| { + let _ = worker.kick(); + }) + .await; + } + exit + }) + }; + let exit = exit.unwrap_or(Exit { + status: ExitCode::INTERNAL.get(), + finalize: false, + from_main: false, + }); + if exit.from_main { + // The main thread asked and is ending itself: wait for it, so a + // program that ran on it alone is stopped cleanly. + let _ = worker.join(); + } else { + // Still running: the barrier stops it, and its channels end with + // the process. Its join ticket must not try to reclaim it first. + core::mem::forget(worker); + } + startup::barrier(exit.finalize); + // SAFETY: the C library's exit, as `__kore_program_start` received it. + unsafe { end(exit.status) } + } +} + +/// The main thread's entry: the program, as the C library handed it over. +fn main_thread() { + ROLE.set(Role::Main); + // SAFETY: `__kore_program_start` stored a `Program` before founding this + // thread, and the founding's session ordered that store before this load. + let program: Program = unsafe { core::mem::transmute(PROGRAM.load(Ordering::Acquire)) }; + unsafe { program() } +} + +/// The host's loop: answer each session the main thread posts, until any +/// thread asks for the end. +async fn host( + rt: &Rt, + transport: &TrapTransport, + worker: &DoorbellWorker, + output: Option<&'static crate::stdio::Endpoint>, +) -> Result { + let mut wake = NoticeRx::new(transport, worker.wake_recv()); + let mut filesystem = Filesystem::new(rt, transport); + let mut device = crate::device::Host::new(rt, transport); + // The one writer the host can kick, woken when its room is made. + let wake_main = || { + let _ = worker.kick(); + }; + loop { + if let Some(exit) = EXIT.posted() { + return Ok(exit); + } + if let Some(output) = output { + // One bounded pass: the end and the next session are served + // between passes, however fast the program writes. + crate::stdio::drain(output, &wake_main).await; + if let Some(exit) = EXIT.posted() { + return Ok(exit); + } + } + // A thread's `mmap` or `munmap`: thread 0 holds the supply's root. + if crate::mmap::serve() { + wake_main(); + continue; + } + // A thread's request of the program's device: this thread holds + // the face. The service's answer is waited for like a session's, so + // an end requested meanwhile still ends the program. + if crate::device::posted() { + let request = pin!(device.serve()); + let ended = pin!(until_exit(&mut wake, output, &wake_main)); + if race(request, ended).await.is_some() { + wake_main(); + } + continue; + } + if SESSION.state.load(Ordering::Acquire) == POSTED { + // SAFETY: POSTED hands the cell to the host until it stores SERVED. + let (step, user) = { + let payload = unsafe { SESSION.payload() }; + (payload.step, payload.user) + }; + let Some(step) = step else { + return Err(Fatal); + }; + // An end requested mid-session ends the program without waiting + // for the session: the barrier stops the waiting main thread, and + // the abandoned session's owner is quarantined, as cancellation + // leaves it. + let session = pin!(filesystem.run(step, user)); + let ended = pin!(until_exit(&mut wake, output, &wake_main)); + let outcome = race(session, ended).await; + let Some((status, error)) = outcome else { + continue; + }; + { + // SAFETY: still POSTED; the host owns the cell. + let payload = unsafe { SESSION.payload() }; + payload.status = status; + payload.error = error; + } + SESSION.state.store(SERVED, Ordering::Release); + let _ = worker.kick(); + continue; + } + // Armed before the recheck `park` makes, so a writer that published + // meanwhile either kicks or is seen. + crate::stdio::arm(); + park(&mut wake, || pending() || crate::stdio::pending()).await?; + } +} + +/// Poll `first` and then `second` until one completes: `Some` with the first's +/// output, or `None` when the second finished first. +async fn race( + mut first: core::pin::Pin<&mut A>, + mut second: core::pin::Pin<&mut B>, +) -> Option { + poll_fn(|cx| { + if let Poll::Ready(output) = first.as_mut().poll(cx) { + return Poll::Ready(Some(output)); + } + if second.as_mut().poll(cx).is_ready() { + return Poll::Ready(None); + } + Poll::Pending + }) + .await +} + +/// How soon a receive refused for want of a standing post looks again: the +/// tree's figure for a rotating notice receiver (the WASI host's handoff). +const WOULD_BLOCK_RETRY_NS: u64 = 100_000; + +/// Park until a kick, a spurious wake, or `ready` holds, having stood the +/// wake post before re-reading what `ready` reads. +async fn park( + wake: &mut NoticeRx<'_, TrapTransport>, + ready: impl Fn() -> bool, +) -> Result<(), Fatal> { + loop { + match wake.try_recv() { + Ok(Some(_)) => return Ok(()), + Ok(None) => { + if ready() { + return Ok(()); + } + let Some(post) = wake.standing_post() else { + return Err(Fatal); + }; + let _ = executor::next_wake_on(post).await; + } + Err(RecvError::Refused(Status::WouldBlock)) => { + // No post stands to wake on: look again shortly, as the + // tree's other receivers do, rather than wait for a wake + // nothing armed. + let _ = executor::next_wake_for(WOULD_BLOCK_RETRY_NS).await; + } + Err(_) => return Err(Fatal), + } + } +} + +/// Resolve once an end is posted, parking on the wake channel meanwhile and +/// draining what the program writes: the session being served is not a +/// reason to wake, only an end or output is. On a broken channel it never +/// resolves, since the host can no longer hear an end; the session it races +/// then runs to its own completion. Dropped when the session ends first, +/// which the drain survives (`stdio::drain`). Each pass yields, so the +/// session it races is polled however fast the program writes. +async fn until_exit( + wake: &mut NoticeRx<'_, TrapTransport>, + output: Option<&'static crate::stdio::Endpoint>, + wake_main: &dyn Fn(), +) { + let ended = || EXIT.posted().is_some(); + while !ended() { + if let Some(output) = output { + crate::stdio::drain(output, wake_main).await; + executor::yield_now().await; + } + // Another thread's mapping does not wait for the session to end, + // and the session is polled between two of them. + if crate::mmap::serve() { + wake_main(); + executor::yield_now().await; + continue; + } + crate::stdio::arm(); + if park(wake, || { + ended() || crate::stdio::pending() || crate::mmap::posted() + }) + .await + .is_err() + { + core::future::pending::<()>().await; + } + } +} + +/// The program's filesystem: attached on first use, one native owner for the +/// program's life. +struct Filesystem<'r, 't> { + rt: &'r Rt, + transport: &'t TrapTransport, + session: Option>, + /// The launch carried no face, or the owner could not be formed: every + /// later session is refused the same way without asking again. + refused: Option<(KoreStatus, KoreFsError)>, +} + +impl<'r, 't> Filesystem<'r, 't> { + fn new(rt: &'r Rt, transport: &'t TrapTransport) -> Self { + Self { + rt, + transport, + session: None, + refused: None, + } + } + + async fn run(&mut self, step: KoreFsStep, user: *mut c_void) -> (KoreStatus, KoreFsError) { + if let Some(refused) = self.refused { + return refused; + } + if self.session.is_none() { + let Some(fs) = Fs::attach(self.rt, self.transport).await else { + let refused = (KoreStatus::Refused, KoreFsError::Unsupported); + self.refused = Some(refused); + return refused; + }; + match Session::new(fs) { + Ok(session) => self.session = Some(session), + Err(_) => { + let refused = (KoreStatus::Capacity, KoreFsError::None); + self.refused = Some(refused); + return refused; + } + } + } + let Some(session) = self.session.as_mut() else { + return (KoreStatus::Refused, KoreFsError::None); + }; + // SAFETY: `kore_fs_run`'s caller took on `Session::run`'s contract for + // `step` and `user`, and waits until this future completes or is + // dropped with the program's end. + match unsafe { session.run(step, user) }.await { + Ok(()) => (KoreStatus::Ok, KoreFsError::None), + Err(error) => error.foreign(), + } + } +} diff --git a/crates/c/src/stdio.rs b/crates/c/src/stdio.rs new file mode 100644 index 0000000..44964ab --- /dev/null +++ b/crates/c/src/stdio.rs @@ -0,0 +1,976 @@ +//! A C program's `stdout` and `stderr`. +//! +//! The launcher's stream (the launch's `PROFILE_INTERACTIVE` window) is +//! mapped on thread 0, the host, and is bound to it: only thread 0 may publish +//! into it. Every thread writes the C streams, so their bytes reach the +//! stream through the host, by one in-process ring: +//! +//! - **A writer** holds the streams' lock for its whole C call: kore-c-shim +//! takes it around each stream call, as POSIX's `flockfile` does, so one +//! call's output is never split by another thread's. Under the lock the +//! writer is the ring's one producer. It waits for room for each piece, +//! copies it in, and kicks the host only if the host armed itself before +//! parking, so a loop of `printf` costs no trap per call. The host can +//! kick the main thread but holds no send into a pthread's doorbell, so +//! every wait for room is bounded by the runtime's slice: a writer the +//! host cannot reach re-reads the ring each slice, as the thread +//! controller's unreached clients do. +//! - **The host** drains the ring a pass at a time — what was queued when +//! the pass began — into the stream's output records, as many as the +//! launcher has room for, and waits for their capacity when they are full: +//! a slow terminal holds the program back rather than losing its output. +//! Between passes it serves the program's end and its filesystem sessions, +//! so a program that never stops printing still ends when it asks to. +//! - **The host's own writes** (a filesystem step's callbacks run on it) +//! cannot wait for the host. They are kept in order beside the ring, each +//! stamped with the ring position it follows, and drained in that order. +//! The host never takes the streams' lock, which a writer waiting on the +//! host may hold. +//! - **The end** drains what was queued and closes the stream before the +//! process ends, so the launcher shows the program's output before its exit +//! status. An end committed on the host itself (a callback's `exit` or +//! `abort`, a panic there) cannot wait, and relays at once what the +//! launcher has room for. A write after the end, or once the stream is +//! gone, reports `EOF`. +//! +//! The ring and its record codec are plain data over atomics ([`Ring`]), so +//! the host suite exercises the code the target runs; the endpoint, the lock's +//! parks and the kicks are the metal half (`metal`). + +use core::cell::UnsafeCell; +#[cfg(target_os = "none")] +use core::sync::atomic::AtomicBool; +use core::sync::atomic::{AtomicU32, Ordering}; + +/// Which C stream a record carries. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Stream { + Out, + Err, +} + +impl Stream { + const fn tag(self) -> u8 { + match self { + Self::Out => 1, + Self::Err => 2, + } + } + + const fn from_tag(tag: u8) -> Option { + match tag { + 1 => Some(Self::Out), + 2 => Some(Self::Err), + _ => None, + } + } +} + +/// Bytes of one record's header: its stream's tag and its length. +const HEADER: usize = 2; +/// The most payload one record carries: its length is one byte. +pub(crate) const RECORD_PAYLOAD: usize = u8::MAX as usize; + +/// The ring's bytes ([`RING`]). A power of two, so a position is an index by +/// a mask, and far below `u32`'s range, so the free-running positions' +/// difference never wraps past a lap. +const RING_BYTES: usize = 1 << 14; + +/// The ring bytes `len` payload bytes take, in whole records. +pub(crate) const fn framed(len: usize) -> usize { + len + HEADER * len.div_ceil(RECORD_PAYLOAD) +} + +/// The largest piece the ring keeps whole: half of it, so a writer waiting +/// for that much room never waits for more room than the host can make +/// while another piece is queued ahead of it. A longer write goes in pieces +/// of this size, under the one lock its call holds. +pub(crate) const WHOLE_WRITE: usize = + RING_BYTES / 2 - HEADER * (RING_BYTES / 2).div_ceil(RECORD_PAYLOAD); + +const _: () = assert!(RING_BYTES.is_power_of_two()); +const _: () = assert!(framed(WHOLE_WRITE) <= RING_BYTES / 2); + +/// A cursor on a cache line of its own: the writer's `tail` and the host's +/// `head` move on different cores, and must not share a line. +#[repr(align(64))] +struct Cursor(AtomicU32); + +/// One ring of framed records over free-running `u32` positions: `tail` is +/// written only by the lock holder, `head` only by the host. +pub(crate) struct Ring { + bytes: UnsafeCell<[u8; N]>, + /// The first byte the host has not consumed. + head: Cursor, + /// One past the last byte a writer published. + tail: Cursor, +} + +// SAFETY: the bytes between `head` and `tail` belong to the host, the rest to +// the lock holder; each side publishes its cursor after its last access and +// reads the other's with Acquire before its first. +unsafe impl Sync for Ring {} + +impl Ring { + pub(crate) const fn new() -> Self { + Self { + bytes: UnsafeCell::new([0; N]), + head: Cursor(AtomicU32::new(0)), + tail: Cursor(AtomicU32::new(0)), + } + } + + /// The host's cursor, for a writer to wait on. + pub(crate) fn head(&self) -> &AtomicU32 { + &self.head.0 + } + + /// Bytes published and not consumed, at most: `head` is read first, and + /// `tail` never falls behind any `head` read before it. + pub(crate) fn queued(&self) -> usize { + let head = self.head.0.load(Ordering::Acquire); + self.tail.0.load(Ordering::Acquire).wrapping_sub(head) as usize + } + + /// Free bytes: exact under the lock, where `tail` is this thread's; + /// conservative outside it, where the host and another writer may both + /// have moved between the two reads. + pub(crate) fn room(&self) -> usize { + N.saturating_sub(self.queued()) + } + + /// Room for `bytes` more, framed. + pub(crate) fn fits(&self, bytes: usize) -> bool { + self.room() >= framed(bytes) + } + + /// The position the next record will be published at. + pub(crate) fn tail(&self) -> u32 { + self.tail.0.load(Ordering::Acquire) + } + + /// Publish `payload` as records of `stream`, whole. + /// + /// # Safety + /// The caller is the ring's one producer (it holds the streams' lock) and + /// saw [`fits`](Self::fits) for `payload` under that lock. + pub(crate) unsafe fn publish(&self, stream: Stream, payload: &[u8]) { + let mut at = self.tail.0.load(Ordering::Relaxed); + for record in payload.chunks(RECORD_PAYLOAD) { + // SAFETY: the caller's contract: `[tail, tail + framed)` is free, + // so the host reads none of these bytes until the store below. + unsafe { + self.put(at, stream.tag()); + self.put(at.wrapping_add(1), record.len() as u8); + for (i, byte) in record.iter().enumerate() { + self.put(at.wrapping_add((HEADER + i) as u32), *byte); + } + } + at = at.wrapping_add((HEADER + record.len()) as u32); + } + // SeqCst, not Release: the host's park recheck reads `tail` after + // arming, and the writer reads the armed flag after this store — the + // Dekker pair that makes a kick unnecessary unless the host armed. + self.tail.0.store(at, Ordering::SeqCst); + } + + /// The record at the host's cursor, if any: its stream and its payload, + /// copied into `out`. The host is the one consumer. + pub(crate) fn peek(&self, out: &mut [u8; RECORD_PAYLOAD]) -> Option<(Stream, usize)> { + let head = self.head.0.load(Ordering::Relaxed); + if self.tail.0.load(Ordering::Acquire) == head { + return None; + } + // SAFETY: `[head, tail)` holds whole records the Acquire above made + // visible, and only this consumer advances `head`. + unsafe { + let stream = Stream::from_tag(self.get(head))?; + let len = self.get(head.wrapping_add(1)) as usize; + for (i, byte) in out.iter_mut().take(len).enumerate() { + *byte = self.get(head.wrapping_add((HEADER + i) as u32)); + } + Some((stream, len)) + } + } + + /// Consume the record [`peek`](Self::peek) returned, `len` payload bytes. + pub(crate) fn consume(&self, len: usize) { + let head = self.head.0.load(Ordering::Relaxed); + self.head + .0 + .store(head.wrapping_add((HEADER + len) as u32), Ordering::Release); + } + + /// Drop every record published so far, as the one consumer. + pub(crate) fn discard(&self) { + let tail = self.tail.0.load(Ordering::Acquire); + self.head.0.store(tail, Ordering::Release); + } + + unsafe fn put(&self, at: u32, byte: u8) { + // SAFETY: the caller owns byte `at` (see `publish`). + unsafe { (*self.bytes.get())[at as usize & (N - 1)] = byte } + } + + unsafe fn get(&self, at: u32) -> u8 { + // SAFETY: the caller owns byte `at` (see `peek`). + unsafe { (*self.bytes.get())[at as usize & (N - 1)] } + } +} + +/// The program's one ring. +#[cfg(target_os = "none")] +pub(crate) static RING: Ring = Ring::new(); + +/// The host is parked, or about to, and a writer must kick it (see +/// [`Ring::publish`]). +#[cfg(target_os = "none")] +static ARMED: AtomicBool = AtomicBool::new(false); + +/// The room the main thread waits for, framed, or 0: the one writer the host +/// can kick, kicked once that much is free rather than at every record. +#[cfg(target_os = "none")] +static MAIN_NEEDS: AtomicU32 = AtomicU32::new(0); + +/// The launcher's stream is gone, or there is no host to drain the ring: +/// writes fail rather than wait for a drain that will not come. +#[cfg(target_os = "none")] +static LOST: AtomicBool = AtomicBool::new(false); + +/// The program has ended: nothing written from here is relayed. +#[cfg(target_os = "none")] +static FINISHED: AtomicBool = AtomicBool::new(false); + +/// Writes report `EOF` from here on. +#[cfg(target_os = "none")] +fn closed() -> bool { + LOST.load(Ordering::Acquire) || FINISHED.load(Ordering::Acquire) +} + +/// Whether the host has anything to drain. +#[cfg(target_os = "none")] +pub(crate) fn pending() -> bool { + !closed() && (RING.queued() != 0 || metal::host_pending()) +} + +/// A write the host made while it could not drain: its bytes, and the ring +/// position it follows. +#[cfg(target_os = "none")] +pub(crate) struct Held { + pub(crate) after: u32, + pub(crate) stream: Stream, + pub(crate) bytes: alloc::vec::Vec, +} + +/// Whether a held write is the next item to drain: it belongs at ring +/// position `held_after`, so it goes once the host's cursor reaches that +/// point, after every record published before it and before every one +/// published after. A position the cursor has already passed (a write made +/// during a call whose first records were already relayed) goes at once. +/// Positions are free-running, and a stamp never strays half the `u32` +/// range from the cursor. +pub(crate) fn held_is_next(held_after: u32, head: u32) -> bool { + head.wrapping_sub(held_after) as i32 >= 0 +} + +extern crate alloc; + +#[cfg(target_os = "none")] +pub(crate) use metal::{ + Endpoint, abandon, arm, drain, finish, flush_now, install, report, write_standard, +}; + +#[cfg(target_os = "none")] +mod metal { + use super::*; + use alloc::collections::VecDeque; + use alloc::vec::Vec; + use core::cell::{Cell, RefCell}; + use core::mem::MaybeUninit; + use core::sync::atomic::AtomicU64; + use core::time::Duration; + use kore_c_shim::stdio::{Standard, StandardSink, install_standard_sink}; + use kore_std::abi::process::{PROCESS_OUTPUT_CHUNK_BYTES, ProcessOutputEvent}; + use kore_std::process::{ + MappedProcessStdioChild, ProcessStdioParkStep, ProcessStdioWaitFor, ProcessStdioWake, + }; + use kore_std::runtime::futex::{self, WaitOutcome}; + use kore_std::runtime::{executor, pace, thread}; + + /// How long a writer waits before it re-reads what it waits for: the + /// runtime's slice for a waiter its waker cannot reach (the thread + /// controller's `UNREACHED_SLICE`). The host kicks the main thread + /// sooner; a pthread rides the slice. + const SLICE: Duration = Duration::from_nanos(pace::CEDE_NS); + + /// The host's half: the launcher's stream and the writes the host held. + /// Thread 0 alone touches it (the stream is bound to that thread), and + /// never across an await. + pub(crate) struct Endpoint { + stdio: RefCell, + held: RefCell>, + /// Bytes of the next item (the front held write, or the ring's head + /// record) already published. A drain can be dropped between two + /// records it publishes (it races a filesystem session), so it + /// consumes an item only once all of it is out, and resumes past + /// what it already sent rather than send it twice. + sent: Cell, + } + + /// Thread 0's endpoint, in static storage: the stream is mapped before + /// the main thread is founded, and a launch without a heap still has + /// one. + struct EndpointCell(UnsafeCell>); + // SAFETY: written once by the host before any other thread exists, and + // read only on thread 0 (`crate::program::on_host`) once `MAPPED` is set. + unsafe impl Sync for EndpointCell {} + static ENDPOINT: EndpointCell = EndpointCell(UnsafeCell::new(MaybeUninit::uninit())); + static MAPPED: AtomicBool = AtomicBool::new(false); + + fn host() -> Option<&'static Endpoint> { + if !crate::program::on_host() || !MAPPED.load(Ordering::Acquire) { + return None; + } + // SAFETY: on thread 0, after `install` wrote it (see `EndpointCell`). + Some(unsafe { (*ENDPOINT.0.get()).assume_init_ref() }) + } + + /// A line on `stderr` from the host itself: what it must say when the + /// program cannot run. Kept in order and relayed as any write made on + /// the host is. Nothing is said when the launch carried no stream. + pub(crate) fn report(line: &[u8]) { + let _ = sink(Standard::Error, line); + } + + /// `write` on a standard stream's descriptor: `bytes` to `stdout`, or to + /// `stderr` when `error`, in order with what `` writes. `false` + /// when the launch carried no stream or it has closed. + pub(crate) fn write_standard(error: bool, bytes: &[u8]) -> bool { + sink( + if error { + Standard::Error + } else { + Standard::Output + }, + bytes, + ) + } + + pub(crate) fn host_pending() -> bool { + host().is_some_and(|endpoint| !endpoint.held.borrow().is_empty()) + } + + /// The C streams' sink and their lock. + static SINK: StandardSink = StandardSink { + write: sink, + lock: lock_streams, + unlock: unlock_streams, + }; + + /// Map the launcher's stream and send the C streams to it. `None` when + /// the launch carried no stream: `stdout` then reports `EOF`, as it does + /// for every program that has none. + pub(crate) fn install(rt: &kore_std::runtime::entry::Rt) -> Option<&'static Endpoint> { + let stdio = kore_std::process::map_stdio(rt)?; + // SAFETY: thread 0, before any other thread exists, and only once + // (`__kore_program_start` founds one host). + let endpoint = unsafe { + (*ENDPOINT.0.get()).write(Endpoint { + stdio: RefCell::new(stdio), + held: RefCell::new(VecDeque::new()), + sent: Cell::new(0), + }) + }; + MAPPED.store(true, Ordering::Release); + install_standard_sink(&SINK); + Some(endpoint) + } + + /// The host could not be founded after the stream was mapped: no one + /// will drain it, so every write reports `EOF`. + pub(crate) fn abandon() { + LOST.store(true, Ordering::SeqCst); + } + + // ── The streams' lock ─────────────────────────────────────────────── + + /// The streams' lock, reentrant per thread ([`DEPTH`]): a stream call + /// holds it across all its writes, and the sink each write reaches takes + /// it again. + static LOCK: crate::lock::Lock = crate::lock::Lock::new(); + + /// How many times this thread holds [`LOCK`]. + #[thread_local] + static DEPTH: Cell = Cell::new(0); + + /// The call holding [`LOCK`], if any: [`IN_CALL`] beside the ring's + /// `tail` when it began. The host's own writes made meanwhile are + /// stamped there, so they go before the call rather than between its + /// pieces. + static CALL: AtomicU64 = AtomicU64::new(0); + const IN_CALL: u64 = 1 << u32::BITS; + + /// Take the streams' lock. The host never does: a writer that holds it + /// may be waiting for the host to drain. + fn lock_streams() { + if crate::program::on_host() { + return; + } + let depth = DEPTH.get(); + if depth == 0 { + LOCK.acquire(); + // The holder is the ring's one producer: `tail` is exact here. + CALL.store(IN_CALL | u64::from(RING.tail()), Ordering::SeqCst); + } + DEPTH.set(depth + 1); + } + + fn unlock_streams() { + if crate::program::on_host() { + return; + } + match DEPTH.get() { + // Released more often than taken: the caller's mistake, and not + // a reason to release another thread's hold. + 0 => {} + 1 => { + DEPTH.set(0); + CALL.store(0, Ordering::SeqCst); + LOCK.release(); + } + depth => DEPTH.set(depth - 1), + } + } + + // ── Writers ───────────────────────────────────────────────────────── + + fn sink(standard: Standard, bytes: &[u8]) -> bool { + let stream = match standard { + Standard::Output => Stream::Out, + Standard::Error => Stream::Err, + }; + if bytes.is_empty() { + return true; + } + if closed() { + return false; + } + if let Some(endpoint) = host() { + return hold(endpoint, stream, bytes); + } + // The call's lock, again: this thread is the ring's one producer for + // every piece. + lock_streams(); + let written = bytes.chunks(WHOLE_WRITE).all(|piece| write(stream, piece)); + unlock_streams(); + written + } + + /// One piece, under the streams' lock: wait for its room, then copy it + /// in. + fn write(stream: Stream, piece: &[u8]) -> bool { + loop { + if closed() { + return false; + } + // Read before the room check: a wait on this value returns at + // once if the host has moved since. + let head = RING.head().load(Ordering::SeqCst); + if RING.fits(piece.len()) { + // SAFETY: the lock makes this thread the one producer, and + // the room was checked under it. + unsafe { RING.publish(stream, piece) }; + if ARMED.swap(false, Ordering::SeqCst) { + let _ = thread::kick_parent(); + } + return true; + } + // The ring is full of records the host has not drained. A host + // parked on its wake (armed) is kicked; one that is not is + // draining already, or waiting for the launcher's room, and + // moves its cursor when it has drained. + let main = crate::program::on_main(); + if main { + MAIN_NEEDS.store(framed(piece.len()) as u32, Ordering::SeqCst); + } + if ARMED.swap(false, Ordering::SeqCst) { + let _ = thread::kick_parent(); + } + let outcome = futex::wait(RING.head(), head, Some(SLICE)); + if main { + MAIN_NEEDS.store(0, Ordering::SeqCst); + } + if outcome == WaitOutcome::Unsupported { + let _ = pace::cede(); + } + } + } + + /// The host's own write: kept, stamped, for its next drain. Stamped at + /// the start of a call in progress, so it does not split that call. + fn hold(endpoint: &Endpoint, stream: Stream, bytes: &[u8]) -> bool { + let mut copy = Vec::new(); + if copy.try_reserve_exact(bytes.len()).is_err() { + return false; + } + copy.extend_from_slice(bytes); + let mut held = endpoint.held.borrow_mut(); + if held.try_reserve(1).is_err() { + return false; + } + let call = CALL.load(Ordering::SeqCst); + let after = if call & IN_CALL != 0 { + call as u32 + } else { + RING.tail() + }; + held.push_back(Held { + after, + stream, + bytes: copy, + }); + true + } + + // ── The host ──────────────────────────────────────────────────────── + + /// How one item went out. + enum Pumped { + /// All of it. + Done, + /// The launcher has no room for the next record: wait, or stop. + Full, + /// The launcher's stream is gone. + Lost, + } + + /// Publish what is left of one item, `len` bytes that `fill` copies out + /// from an offset, as stream records while the launcher has room, past + /// what [`Endpoint::sent`] says is out. Never waits. + fn pump( + endpoint: &Endpoint, + stream: Stream, + len: usize, + fill: &dyn Fn(usize, &mut [u8]) -> usize, + ) -> Pumped { + let mut stdio = endpoint.stdio.borrow_mut(); + while endpoint.sent.get() < len { + if stdio.output_capacity_left() == 0 { + return Pumped::Full; + } + let from = endpoint.sent.get(); + let mut chunk = [0u8; PROCESS_OUTPUT_CHUNK_BYTES]; + let want = (len - from).min(PROCESS_OUTPUT_CHUNK_BYTES); + let filled = fill(from, &mut chunk[..want]); + if filled == 0 { + // The item vanished under the drain: nothing can resume it. + return Pumped::Lost; + } + let event = match stream { + Stream::Out => ProcessOutputEvent::stdout(&chunk[..filled]), + Stream::Err => ProcessOutputEvent::stderr(&chunk[..filled]), + }; + let Ok(event) = event else { + return Pumped::Lost; + }; + match stdio.send_output_now(event) { + Ok(ProcessStdioWake::Unreached(_)) | Err(_) => return Pumped::Lost, + Ok(_) => endpoint.sent.set(from + filled), + } + } + endpoint.sent.set(0); + Pumped::Done + } + + /// [`pump`] one item whole, waiting for the launcher's room between its + /// records. `false` once the stream is gone. Dropped at a wait, it has + /// published whole records only, and `sent` says how far. + async fn send( + endpoint: &Endpoint, + stream: Stream, + len: usize, + fill: &dyn Fn(usize, &mut [u8]) -> usize, + ) -> bool { + loop { + match pump(endpoint, stream, len, fill) { + Pumped::Done => return true, + Pumped::Lost => return false, + Pumped::Full => { + if !room(endpoint).await { + return false; + } + } + } + } + } + + /// Copy the front held write's bytes from `from` into `out`. + fn held_bytes(endpoint: &Endpoint, from: usize, out: &mut [u8]) -> usize { + let held = endpoint.held.borrow(); + let Some(front) = held.front() else { + return 0; + }; + let Some(bytes) = front.bytes.get(from..from + out.len()) else { + return 0; + }; + out.copy_from_slice(bytes); + out.len() + } + + /// The next item due, if any: the front held write when the host's + /// cursor reached its stamp, else the ring's head record if it lies + /// before `end`. + enum Next { + Held(Stream, usize), + Record(Stream, usize), + None, + } + + fn next(endpoint: &Endpoint, end: u32, record: &mut [u8; RECORD_PAYLOAD]) -> Next { + let head = RING.head().load(Ordering::Acquire); + if let Some(front) = endpoint.held.borrow().front() + && held_is_next(front.after, head) + { + return Next::Held(front.stream, front.bytes.len()); + } + if head == end { + return Next::None; + } + match RING.peek(record) { + Some((stream, len)) => Next::Record(stream, len), + None => Next::None, + } + } + + /// A record went out: consume it, and wake the main thread once the + /// room it waits for is free. Fenced, so the cursor's store is ordered + /// before the read of what main needs, the other half of the writer's + /// pair. + fn consumed(len: usize, wake_main: &dyn Fn()) { + RING.consume(len); + core::sync::atomic::fence(Ordering::SeqCst); + let needs = MAIN_NEEDS.load(Ordering::SeqCst) as usize; + if needs != 0 && RING.room() >= needs { + wake_main(); + } + } + + /// The launcher's stream is gone: drop everything queued, and let every + /// writer see it. + fn lose(endpoint: &Endpoint, wake_main: &dyn Fn()) { + LOST.store(true, Ordering::SeqCst); + endpoint.held.borrow_mut().clear(); + endpoint.sent.set(0); + RING.discard(); + core::sync::atomic::fence(Ordering::SeqCst); + if MAIN_NEEDS.load(Ordering::SeqCst) != 0 { + wake_main(); + } + } + + /// How long a rotating notice receiver's handoff is given before the park + /// is prepared again: the WASI host's figure. + const HANDOFF_NS: u64 = 100_000; + + /// Wait until the launcher has room for one more record. `false` once its + /// stream is gone. The endpoint is borrowed to prepare and settle each + /// park, never across the wait (the WASI host's shape). + async fn room(endpoint: &Endpoint) -> bool { + loop { + let step = { + let mut stdio = endpoint.stdio.borrow_mut(); + // The drain snapshot comes before the capacity check, so room + // the launcher makes between them is seen by the park's + // recheck. + let drain = stdio.begin_drain(); + if stdio.output_capacity_left() > 0 { + return true; + } + match stdio.prepare_park(&drain, ProcessStdioWaitFor::Capacity) { + Ok(step) => step, + Err(_) => return false, + } + }; + match step { + ProcessStdioParkStep::Changed => {} + ProcessStdioParkStep::Handoff => { + let _ = executor::next_wake_for(HANDOFF_NS).await; + } + ProcessStdioParkStep::Park(ticket) => { + let reason = executor::next_wake_on(ticket.post()).await; + if endpoint + .stdio + .borrow_mut() + .settle_park(ticket, reason) + .is_err() + { + return false; + } + } + } + } + } + + /// One pass: what the ring held when it began, and the host's writes + /// held by then, in order. Bounded on both, so the host serves the + /// program between passes: a filesystem step's callbacks hold writes + /// while a pass waits for the launcher's room, and those wait for the + /// next pass. Cancellation-safe at every wait: an item leaves its queue + /// only once all of it is published. + pub(crate) async fn drain(endpoint: &Endpoint, wake_main: &dyn Fn()) { + let end = RING.tail(); + let mut held_left = endpoint.held.borrow().len(); + let mut record = [0u8; RECORD_PAYLOAD]; + loop { + if LOST.load(Ordering::Acquire) { + lose(endpoint, wake_main); + return; + } + let sent = match next(endpoint, end, &mut record) { + // A write held after the pass began: the next pass's, and so + // is everything after it. + Next::Held(..) if held_left == 0 => return, + Next::Held(stream, len) => { + let fill = |from: usize, out: &mut [u8]| held_bytes(endpoint, from, out); + let sent = send(endpoint, stream, len, &fill).await; + if sent { + endpoint.held.borrow_mut().pop_front(); + held_left -= 1; + } + sent + } + Next::Record(stream, len) => { + let bytes = record; + let fill = |from: usize, out: &mut [u8]| { + out.copy_from_slice(&bytes[from..from + out.len()]); + out.len() + }; + let sent = send(endpoint, stream, len, &fill).await; + if sent { + consumed(len, wake_main); + } + sent + } + Next::None => return, + }; + if !sent { + lose(endpoint, wake_main); + return; + } + } + } + + /// The host arms itself before it rechecks what it would park for: a + /// writer that published before this reads it and kicks, and one that + /// publishes after is seen by the recheck (the Dekker pair with + /// [`Ring::publish`]'s `SeqCst` store and the writer's swap). + pub(crate) fn arm() { + ARMED.store(true, Ordering::SeqCst); + core::sync::atomic::fence(Ordering::SeqCst); + } + + /// The program's end on the host's loop: drain what was queued and close + /// the stream. A write from here on reports `EOF`. + pub(crate) async fn finish(endpoint: &Endpoint, wake_main: &dyn Fn()) { + FINISHED.store(true, Ordering::SeqCst); + drain(endpoint, wake_main).await; + if LOST.load(Ordering::Acquire) || !room(endpoint).await { + return; + } + let _ = endpoint + .stdio + .borrow_mut() + .send_output_now(ProcessOutputEvent::closed()); + } + + /// The program's end on the host itself, which cannot wait (a callback's + /// `exit` or `abort`, a panic there): relay at once what the launcher has + /// room for, and close the stream if all of it went. A write from here on + /// reports `EOF`. + pub(crate) fn flush_now() { + let Some(endpoint) = host() else { + return; + }; + if closed() { + return; + } + FINISHED.store(true, Ordering::SeqCst); + // A borrow still held (a panic inside the drain itself) leaves the + // stream as it is rather than panic again. + if endpoint.stdio.try_borrow_mut().is_err() || endpoint.held.try_borrow_mut().is_err() { + return; + } + let end = RING.tail(); + let mut record = [0u8; RECORD_PAYLOAD]; + loop { + let pumped = match next(endpoint, end, &mut record) { + Next::Held(stream, len) => { + let fill = |from: usize, out: &mut [u8]| held_bytes(endpoint, from, out); + let pumped = pump(endpoint, stream, len, &fill); + if matches!(pumped, Pumped::Done) { + endpoint.held.borrow_mut().pop_front(); + } + pumped + } + Next::Record(stream, len) => { + let bytes = record; + let fill = |from: usize, out: &mut [u8]| { + out.copy_from_slice(&bytes[from..from + out.len()]); + out.len() + }; + let pumped = pump(endpoint, stream, len, &fill); + if matches!(pumped, Pumped::Done) { + RING.consume(len); + } + pumped + } + Next::None => break, + }; + if !matches!(pumped, Pumped::Done) { + return; + } + } + let mut stdio = endpoint.stdio.borrow_mut(); + if stdio.output_capacity_left() > 0 { + let _ = stdio.send_output_now(ProcessOutputEvent::closed()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn drain(ring: &Ring) -> std::vec::Vec<(Stream, std::vec::Vec)> { + let mut out = std::vec::Vec::new(); + let mut record = [0u8; RECORD_PAYLOAD]; + while let Some((stream, len)) = ring.peek(&mut record) { + out.push((stream, record[..len].to_vec())); + ring.consume(len); + } + out + } + + /// A write longer than a record is framed into records of at most a + /// record's payload, in order, and the ring accounts every header. + #[test] + fn a_write_is_framed_into_records_and_read_back_in_order() { + let ring = Ring::<1024>::new(); + let bytes: std::vec::Vec = (0..600u32).map(|i| i as u8).collect(); + assert!(ring.fits(bytes.len())); + unsafe { ring.publish(Stream::Out, &bytes) }; + assert_eq!(ring.queued(), framed(600)); + assert_eq!(framed(600), 600 + 3 * HEADER); + unsafe { ring.publish(Stream::Err, b"oops") }; + let records = drain(&ring); + let joined: std::vec::Vec = records + .iter() + .filter(|(stream, _)| *stream == Stream::Out) + .flat_map(|(_, bytes)| bytes.iter().copied()) + .collect(); + assert_eq!(joined, bytes); + assert_eq!(records.len(), 4); + assert_eq!(records[3], (Stream::Err, b"oops".to_vec())); + assert!( + records + .iter() + .all(|(_, bytes)| bytes.len() <= RECORD_PAYLOAD) + ); + assert_eq!(ring.queued(), 0); + } + + /// The positions run free across many laps and past `u32::MAX`; every + /// record still reads back whole, and room is exactly what is unconsumed. + #[test] + fn the_ring_wraps_its_bytes_and_its_positions() { + let ring = Ring::<64>::new(); + ring.head.0.store(u32::MAX - 20, Ordering::Relaxed); + ring.tail.0.store(u32::MAX - 20, Ordering::Relaxed); + for round in 0..200u32 { + let payload: std::vec::Vec = + (0..(round % 40) + 1).map(|i| (i + round) as u8).collect(); + assert!(ring.fits(payload.len()), "round {round}"); + unsafe { ring.publish(Stream::Out, &payload) }; + assert!(!ring.fits(64), "a full lap never fits"); + let records = drain(&ring); + let joined: std::vec::Vec = records.into_iter().flat_map(|(_, b)| b).collect(); + assert_eq!(joined, payload, "round {round}"); + } + } + + /// Room is the framed size: a write fits only while its records and + /// their headers do. + #[test] + fn room_counts_headers() { + let ring = Ring::<16>::new(); + assert!(ring.fits(14)); + assert!(!ring.fits(15)); + unsafe { ring.publish(Stream::Out, &[7; 10]) }; + assert!(ring.fits(2)); + assert!(!ring.fits(3)); + } + + /// A cursor read out of order — the host's moved past a `tail` read + /// before it — never reads as room the ring does not have. + #[test] + fn a_stale_read_never_finds_room_that_is_not_there() { + let ring = Ring::<16>::new(); + // A reader that saw `tail` = 0, then a `head` the host has advanced + // to 10, would compute a wrapped, enormous backlog. + ring.tail.0.store(0, Ordering::Relaxed); + ring.head.0.store(10, Ordering::Relaxed); + assert!(!ring.fits(1)); + // The real order, head first, is exact. + ring.tail.0.store(14, Ordering::Relaxed); + assert_eq!(ring.queued(), 4); + assert!(ring.fits(10)); + } + + /// Discarding consumes every published record at once. + #[test] + fn discard_drops_everything_published() { + let ring = Ring::<64>::new(); + unsafe { ring.publish(Stream::Out, b"lost") }; + unsafe { ring.publish(Stream::Err, b"too") }; + ring.discard(); + assert_eq!(ring.queued(), 0); + assert!(drain(&ring).is_empty()); + } + + /// The largest piece kept whole leaves room for another queued ahead of + /// it, whatever was framed in between. + #[test] + fn a_whole_write_fits_half_the_ring() { + assert!(framed(WHOLE_WRITE) <= RING_BYTES / 2); + assert!(framed(WHOLE_WRITE + RECORD_PAYLOAD) > RING_BYTES / 2); + } + + /// The host's held writes go after the ring records that preceded them + /// and before the ones that followed. + #[test] + fn a_held_write_drains_after_what_preceded_it() { + let ring = Ring::<256>::new(); + unsafe { ring.publish(Stream::Out, b"a") }; + let stamp = ring.tail(); + unsafe { ring.publish(Stream::Out, b"b") }; + let mut record = [0u8; RECORD_PAYLOAD]; + assert!(!held_is_next(stamp, ring.head().load(Ordering::Relaxed))); + let (_, len) = ring.peek(&mut record).unwrap(); + assert_eq!(&record[..len], b"a"); + ring.consume(len); + assert!(held_is_next(stamp, ring.head().load(Ordering::Relaxed))); + } + + /// A write stamped where a call began goes at once if the cursor already + /// passed that point, and never if the point is still ahead, across the + /// positions' wrap. + #[test] + fn a_held_write_behind_the_cursor_goes_at_once() { + for base in [0u32, u32::MAX - 3] { + let head = base.wrapping_add(10); + assert!(held_is_next(base, head), "passed at {base}"); + assert!(held_is_next(head, head), "reached at {base}"); + assert!(!held_is_next(head.wrapping_add(1), head), "ahead at {base}"); + } + } +} diff --git a/crates/c/tests/program.c b/crates/c/tests/program.c new file mode 100644 index 0000000..fda382c --- /dev/null +++ b/crates/c/tests/program.c @@ -0,0 +1,44 @@ +/* kore/program.h as an independent C program uses it: a session whose + * callbacks are kore.h's own KoreFsStep, run through kore_fs_run, and a + * descriptor described through kore_fd_describe. Compiled + * freestanding for every Kore ISA by tools/check-bindings.py. */ +#include + +struct reader { + int done; +}; + +static void open_root(struct KoreFsContext *context, const struct KoreFsCompletion *completion, + void *user) +{ + struct reader *reader = user; + if (completion->operation == KORE_OP_START) { + (void)kore_fs_open(context, (const uint8_t *)"", 0, KORE_FS_READ); + return; + } + reader->done = 1; +} + +/* The entry's exact type, so a drift in either header fails the build. */ +static KoreStatus (*const run)(KoreFsStep, void *, KoreFsError *) = kore_fs_run; + +int program_session(void) +{ + struct reader reader = {0}; + KoreFsError error = KORE_FS_ERROR_NONE; + KoreStatus status = run(open_root, &reader, &error); + return status == KORE_STATUS_OK && error == KORE_FS_ERROR_NONE && reader.done; +} + +/* A descriptor's description: the exact entry type, its fields and the + * constants its fields take. */ +static KoreStatus (*const describe)(int, KoreFdInfo *) = kore_fd_describe; + +int program_memory(int fd) +{ + KoreFdInfo info = {0}; + if (describe(fd, &info) != KORE_STATUS_OK) + return 0; + return info.kind == KORE_FD_MEMORY && (info.access & (KORE_FD_READ | KORE_FD_WRITE)) != 0 && + info.length != 0; +} diff --git a/crates/ffi/src/futex.rs b/crates/ffi/src/futex.rs index c76621d..6e04454 100644 --- a/crates/ffi/src/futex.rs +++ b/crates/ffi/src/futex.rs @@ -91,7 +91,8 @@ pub unsafe extern "C" fn kore_futex_wait( Ok(time) => time.as_nanos(), Err(_) => return KoreFutexOutcome::Unsupported, }; - let slice = deadline.map_or(1_000_000, |end| end.saturating_sub(now).min(1_000_000)); + let slice = crate::thread_local::CANCEL_SLICE_NS; + let slice = deadline.map_or(slice, |end| end.saturating_sub(now).min(slice)); let outcome = futex::wait(word, expected, Some(Duration::from_nanos(slice))); unsafe { crate::kore_thread_testcancel() }; if outcome != futex::WaitOutcome::TimedOut { diff --git a/crates/ffi/src/session.rs b/crates/ffi/src/session.rs index 6176f38..658b83f 100644 --- a/crates/ffi/src/session.rs +++ b/crates/ffi/src/session.rs @@ -25,6 +25,21 @@ pub enum RunError { Cleanup(Refusal), } +impl RunError { + /// What a foreign caller reads for a failed run, in the submission-status + /// vocabulary: `Quarantined` for an owner that may never run again, and + /// `Refused` with the refused close's error, whose view the owner kept. + pub fn foreign(&self) -> (KoreStatus, KoreFsError) { + match self { + Self::Quarantined => (KoreStatus::Quarantined, KoreFsError::None), + Self::Cleanup(Refusal::Fs(error)) => (KoreStatus::Refused, classify(*error)), + // A close answers in the filesystem's vocabulary or quarantines; + // no other refusal reaches cleanup. + Self::Cleanup(_) => (KoreStatus::Refused, KoreFsError::Unknown), + } + } +} + impl<'w, 't> Session<'w, 't> { /// Consume the real native face. Identifier exhaustion returns it unchanged. /// Close any previously opened external views before handing off this face: diff --git a/crates/ffi/src/thread.rs b/crates/ffi/src/thread.rs index a5a550c..14e7d1e 100644 --- a/crates/ffi/src/thread.rs +++ b/crates/ffi/src/thread.rs @@ -31,6 +31,15 @@ pub const KORE_THREAD_ECANCELED: i32 = 125; pub const KORE_THREAD_ENOTSUP: i32 = 95; static THREADS: Controller = Controller::new(); + +/// Have every thread this process starts through `pthread_create` hold +/// `share`: a capability its founder holds at a boot-block coordinate, which +/// each founding re-derives (`Controller::hand_on`). For a program's host to +/// call before the first thread is created; `false` once the controller has +/// started, or when it can hand on no more. +pub fn hand_on_to_threads(share: kore::runtime::thread_controller::Handed) -> bool { + THREADS.hand_on(share) +} struct Ticket { id: Id, control: Arc, @@ -98,7 +107,11 @@ fn retire(argument: usize) { // SAFETY: called exactly once by native owner after successful reclamation. unsafe { let context = NonNull::new_unchecked(argument as *mut StartContext); - context.as_ref().control.retired.store(1, Ordering::Release); + let control = &context.as_ref().control; + // Store, then wake (the futex's order): a joiner parked on the word + // re-reads it once it is registered, so neither can miss the other. + control.retired.store(1, Ordering::SeqCst); + kore::futex::wake_all(&control.retired); release(context); } } @@ -196,16 +209,24 @@ pub unsafe extern "C" fn kore_thread_join( return KORE_THREAD_EDEADLK; } // Wait without claiming the native join record: cancellation must leave the - // target joinable. Only the controller can retire native capabilities. + // target joinable. Only the controller can retire native capabilities, and + // it wakes `retired` when it does. loop { unsafe { crate::kore_thread_testcancel() }; let control = unsafe { &ticket.as_ref().control }; - if control.retired.load(Ordering::Acquire) != 0 { + if control.retired.load(Ordering::SeqCst) != 0 { break; } - // Independent native workers need not own a doorbell. Use the - // controller's deadline park, which requires no peer wake capability. - if !kore::runtime::pace::cede() { + // A cancellation request is a flag, not a wake: a joiner that can be + // cancelled parks one slice at a time, as a cancellable futex wait + // does. Any other joiner parks until the controller retires the target. + let slice = crate::thread_local::cancellable() + .then(|| core::time::Duration::from_nanos(crate::thread_local::CANCEL_SLICE_NS)); + if kore::futex::wait(&control.retired, 0, slice) == kore::futex::WaitOutcome::Unsupported + // A native worker with no doorbell takes the deadline park, which + // needs no wake capability at all. + && !kore::runtime::pace::cede() + { return KORE_THREAD_EAGAIN; } } diff --git a/crates/ffi/src/thread_local.rs b/crates/ffi/src/thread_local.rs index d160bc3..64bd577 100644 --- a/crates/ffi/src/thread_local.rs +++ b/crates/ffi/src/thread_local.rs @@ -15,6 +15,12 @@ pub const KORE_THREAD_CANCEL_ENABLE: i32 = 0; pub const KORE_THREAD_CANCEL_DISABLE: i32 = 1; pub const KORE_THREAD_CANCEL_DEFERRED: i32 = 0; pub const KORE_THREAD_CANCEL_ASYNCHRONOUS: i32 = 1; +/// How long one park lasts at a cancellation point that waits: a request is +/// a flag no wake carries, so a cancellable wait re-checks it this often. +/// One millisecond is the cadence the native runtime parks an unreachable +/// waiter at (`kore-rt`'s `pace::CEDE_NS`): a cancellation is acted on within +/// one, and an idle cancellable waiter wakes no more often than that. +pub(crate) const CANCEL_SLICE_NS: u64 = 1_000_000; pub type KoreThreadKey = u64; pub type KoreThreadDestructor = Option; @@ -487,7 +493,7 @@ pub unsafe extern "C" fn pthread_setcanceltype(kind: i32, old: *mut i32) -> i32 /// Same contract as kore_thread_testcancel. #[cfg(all(feature = "c-stdlib", target_os = "none"))] #[unsafe(no_mangle)] -pub unsafe extern "C" fn pthread_testcancel() -> () { +pub unsafe extern "C" fn pthread_testcancel() { unsafe { crate::kore_thread_testcancel() } } diff --git a/docs/languages.md b/docs/languages.md index 07bd6ae..59d6ded 100644 --- a/docs/languages.md +++ b/docs/languages.md @@ -20,9 +20,9 @@ A Rust host attaches `Fs` using its real founding face, then moves it into `kore_ffi::Session`, after closing any external views opened before handoff. The host awaits `Session::run(step, user)` on the existing Kore executor. No second executor, global allocator or global filesystem root is -installed. The C/Zig callback is linked into that same application's image. -This is not yet an independent C `main`/CRT, registry-published static SDK, or a -POSIX syscall veneer. +installed. The C/Zig callback is linked into that same application's image. A C +program with no Rust host gets one from `libkore.so` ([below](#an-independent-c-program-libkoreso)). +Neither is a registry-published static SDK or a POSIX syscall veneer. Each callback receives an opaque context and the preceding completion. Initially that completion is `KORE_OP_START`. Queue one operation and return. Submission @@ -64,6 +64,236 @@ if requests (such as zero-byte reads) complete immediately. C/Zig callbacks must return promptly: the adapter does not preempt arbitrary foreign code inside a callback. +## An independent C program: `libkore.so` + +A C program a compiler writes on Kore (`tcc` at Kore's shell) has no Rust +host. `kore-c` is its runtime. A distribution builds `libkore.so` from a +one-line `cdylib` named `kore` over it, linked with `--soname=libkore.so +--no-undefined` (Kore's `user/runtime/libkore`). It ships the library beside +the headers, which `kore-c` publishes to that crate's build script as +`DEP_KORE_INCLUDE`, so they come from the same revision. A program links it with +`-lkore` — Kore's TCC links it into every program, its `-lc` reading a script +that joins `libkore.so` to the program's own `libc.a` — and gets: + +- every `kore.h` function, and the C names `kore/stdlib.h` and + `kore/pthread.h` alias as real symbols (`c-stdlib`), over the process heap + `kore-c` installs on the launch's `HEAP` supply; +- the rest of the C library the program's headers declare (Kore's hosted + ``, ``, `` and ``, beside TCC): + `kore-c-shim`'s `printf` family, every conversion C has but the wide ones and + every float digit exact, `stdout` and `stderr`, the `strto*` parsers, + `qsort`, `strerror`, the string routines and a per-thread `errno`, and + `kore-c`'s own `abort`; +- descriptors over the capabilities it holds (``, ``, + ``: `open`, `close`, `mmap`, `munmap`), and + `kore_fd_describe` (`kore/program.h`), described below; +- a host on thread 0, entered from the C library's start; +- `kore_fs_run(step, user, &error)` (`kore/program.h`), which runs a + `KoreFsStep` program as the program's filesystem session. + +The host is the shape `kore-lua`'s VM bridge has. `main` runs on a main +thread, a `DoorbellWorker` with a 1 MiB guarded stack +(`KORE_MAIN_STACK_BYTES`). Thread 0 is the process owner, and keeps the +executor and the launch's service faces. The two share one request cell and +one word. Every park is a doorbell park; nothing spins. + +- **Filesystem.** `kore_fs_run` from the main thread posts the session and + waits. The host attaches the launch's filesystem face on first use, rooted + at the directory the program was launched in, and runs `kore-ffi`'s + `Session` on its executor. The callbacks run there, as on any native + executor, while the caller waits: they must not depend on its + thread-local state, nor wait for another program thread — while one runs + the host serves nothing else, the streams included. A thread that cannot park (any but the main thread) + is refused, as is a launch with no face. A quarantined session can never + run again. +- **The streams.** `stdout` and `stderr` reach the launch's interactive + stream (`PROFILE_INTERACTIVE`), the terminal the program was run from, + which only thread 0 may publish into. Any thread's write goes whole into + one in-process ring under a futex lock, which a stream call holds across + all its writes (POSIX's `flockfile`): the holder is the ring's one + producer, so one call's output is never split by another thread's, and it + kicks the host only if the host armed itself before parking. Every wait + for room is bounded by the runtime's slice: the host kicks the main thread + when its room is free, and a pthread, which the host cannot reach, + re-reads the ring each slice. The host drains the + ring into the stream's records as the launcher has room, parks on the + stream's capacity when it has none — a slow terminal holds the program + back rather than losing its output — and wakes the writers waiting for + space, a pass at a time, serving the program's end and its sessions + between passes. The host's own writes (a filesystem step's callbacks) are + kept in order beside the ring, before any call another thread has in + progress. A launch without the stream has no sink, and a + write to either stream returns `EOF`. +- **The end.** The C library's `exit` and `_Exit` call + `__kore_program_exit(status, finalize)` after their `atexit` handlers, on + whatever thread called them. The first request wins and reaches the host, + since threads the program starts inherit a send into its wake channel. + - The host waits for a main thread that asked to end itself. + - It then raises kore-rt's process barrier, stopping every other thread + and running the finalizers only if none was still running. Once a + program has started a thread, `kore-ffi`'s thread controller is one of + them, so such a program's finalizers do not run; `atexit` handlers + always do. + - It commits the C status, the wire's `i32`, through the C library's own + exit. + - An end requested mid-session does not wait for the session. + - A panic in the library ends the program the same way, its record + published first. + - `abort` takes `_Exit`'s path with the aborted status, 134, what a shell + reports for a program `SIGABRT` ended: no `atexit` handler runs. + - An end the host's loop serves drains the ring and closes the stream + before the status, so the launcher shows all of the program's output + first. An end committed on the host itself (a callback's `exit` or + `abort`, a panic there) cannot wait: it relays what the launcher has room + for, and closes the stream if that was all of it. +- **Without a host.** A launch that cannot carry one (no channel space or + heap) runs the program on thread 0, as its C library alone would. Then the + filesystem session is refused, and only thread 0's exit ends the program. + +### Descriptors: C's spelling for a held capability + +A program's authority is the capabilities it was founded holding. Each sits +at a fixed place: the boot block names it by a role and an index, and the +capability is a slot of the thread's arena. `open` is the readable way to say +which one is meant, and nothing more: + +```c +int fd = open("/kore/memory", O_RDWR); +char *region = mmap(NULL, 1 << 20, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); +... +munmap(region, 1 << 20); +close(fd); +``` + +- **The path is not a file.** `open` looks the string up in one static table + (`crates/c/src/fd.rs`) whose rows are a name and the bootstrap coordinate it + stands for. No filesystem is asked, and no name outside the table exists: + `ENOENT`. The first row is `/kore/memory`, `role::HEAP@0`, the memory supply + the allocator grows on. +- **It grants nothing.** `open` succeeds only for a capability the calling + thread holds (`ENXIO` otherwise), with access its rights cover (`EACCES`). + The only flags are the access modes. `close` forgets the number; the + capability stays the program's, and mappings made through it stand. +- **A descriptor stores the coordinate, never the capability.** Capabilities + are per thread on Kore, and `main` and every `pthread` have their own boot + block. Each use resolves the coordinate in the *calling* thread's. Numbers + are the process's, as C expects, from 3 (0 to 2 are the streams'), and no + authority crosses a thread through one. +- **`kore_fd_describe(fd, &info)`** reads what the calling thread holds + there: its kind (`KORE_FD_MEMORY` or `KORE_FD_DEVICE`), what its rights + permit, and for a memory supply its whole length. + +`mmap` on the memory supply answers fresh, zeroed memory, as mapping +`/dev/zero` does elsewhere. It is three of kore-rt's verbs, the sequence a +Wasmtime engine's guest memory takes: a lease carved from the supply's top, +clear of the allocator, which grows from its bottom; an address from the +free part of the task's link window (`Rt::link_free_span`); the whole lease +mapped there. The rules follow from that: + +- **The caller never picks the address.** A hint is ignored, and `MAP_FIXED` + is `ENOTSUP`. +- **Read, or read and write.** `PROT_EXEC` is `ENOTSUP`: memory a program + wrote is never executable. `MAP_SHARED` and `MAP_PRIVATE` mean the same, + since the region is this program's alone. There is no offset, and no + `MAP_ANONYMOUS`: memory comes from a capability, so a descriptor names it. +- **Thread 0 maps, for whichever thread asked.** Every thread that may + allocate is founded holding a share of the supply, and `open` and `mmap` + hold the caller's access to that share's rights. A share carries no + `REVOKE`, though, and a region that can be unmapped again is a carve its + holder may revoke, so the caller posts its request to thread 0, which + holds the supply's root, and waits as a filesystem session does. A mapping + is therefore the process's: any thread may `munmap` it, whole, by its base + and its length. +- **`munmap` parks the region.** The supply's cursors and the window's only + advance, so carved bytes and placed addresses never return to them. The + mapping is removed and a stale access faults, but the program keeps the + lease and its address, and the next `mmap` of the same length takes it, + zeroed again, before anything new is carved. One size mapped and unmapped + in a loop uses one region. Ever-different sizes spend supply each time and + end with `ENOMEM`. A program holds sixteen regions, mapped and parked + together. + +### A virtual device + +A program launched with Kore's virtual-device profile has a face with the +virtual-device service: a device with no hardware behind it, for a driver to +be written against (Kore's `res/VIRTUAL_DEVICE.md`). Its name is a second row +of the table, and its descriptor has the shape Linux's UIO gives a userspace +driver (`crates/c/src/device.rs`): + +```c +int fd = open("/kore/device/virtual/accel", O_RDWR); +volatile KoreVdevRegisters *regs = + mmap(NULL, KORE_VDEV_REGISTERS_BYTES, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, KORE_VDEV_MAP_REGISTERS); +uint8_t *lent = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, KORE_VDEV_MAP_MEMORY); +... +uint32_t word = 1; +write(fd, &word, sizeof word); /* ring the doorbell */ +read(fd, &word, sizeof word); /* wait for the interrupt */ +``` + +- **`open` starts the device.** The service starts the model the name + stands for and writes its register block. A program has one device open at + a time (`EBUSY`), and `close` gives it up. +- **Offset 0 is the register block**, one page the program and the device + both write. `KoreVdevRegisters` (``) is its layout, held to + `kore-vdev`'s register constants by this crate's build. +- **Offset one page is memory to lend.** It is fresh and zeroed, the + program's own, and a device address is an offset into it. The device takes + a loan only in its reset state, so a driver maps it before its first status + write. One region at a time, and it cannot be unmapped while the device is + open (`EBUSY`), since the device may be using it. +- **`read` waits for an interrupt** and answers the device's interrupt count, + one 32-bit word. Each interrupt is read once. **`write`** of one word rings + the doorbell. **`poll`** waits for the interrupt with a timeout. +- **The interrupt and the doorbell never cross a thread.** They are + capabilities, and capabilities are per thread, so the host founds the main + thread holding its own share of the face and the thread controller hands + the share on to every `pthread`. `read` parks the calling thread in the + kernel on the interrupt count, and `write` posts its notice directly. + `open`, the loan and `close` are requests to the service, which thread 0's + face carries: they are posted to it and waited for, as a mapping of the + memory supply is. + +`write` on descriptors 1 and 2 writes to `stdout` and `stderr`, in order +with what `` writes. + +**`main` gets its arguments.** The launcher's page carries the name it +resolved the program by and the rest of the command line. The host reads it +once, on thread 0 before the main thread exists, and splits the rest as every +Kore program that wants words does: a POSIX shell's quoting, without +expansion. `argv[0]` is the name, or the empty string when the launcher gave +none, `argv[argc]` is null, and the strings are the program's to change. A +line that does not split (an unclosed quote, a lone trailing backslash) is +not handed to `main`: the host says why on `stderr` and the program ends with +status 5 before it runs. `envp` is the empty vector, since a Kore task has no +environment. A launch that carried no page runs with `argc` zero. + +The C library's half is three weak references. Its start calls +`__kore_program_start(program, end)` when the program links libkore, asks +`__kore_program_arguments(&argv)` for the vector before it calls `main`, and +its exit calls `__kore_program_exit`. A program without libkore is unchanged. On +Kore those are the start files Kore ships with TCC. `tools/check-program.py` +(`make cross`, and CI per ISA) links the shared object as a distribution does +and checks it: + +- a library with no entry point; +- its soname, nothing needed or undefined; +- static initial-exec TLS only; +- exactly the declared ABI exported: `kore.h`, `kore/program.h`, the + `c-stdlib` names, the two lifecycle entry points, and the C library names + Kore's hosted headers declare (`C_LIBRARY`). + +Kore's `make keytest-tcc-x86` compiles a C program against these headers at +Kore's shell, links `libkore.so`, and runs it through every surface above. It +also compiles Kore's `hello.c`, written against the standard headers alone, +reads every line it prints off the terminal, and ends a program by `abort` +(Kore `res/TCC.md` §10). That is boot evidence; this repository's checks are +compiler and link evidence. Task preparation (`kore_task_*`) still needs a +Rust host to admit its batch: the independent host does not admit one. + ## Failure and lifetime Filesystem and provider errors have explicit generated foreign enums. Monotonic diff --git a/docs/provenance.md b/docs/provenance.md index 6554b1d..737162c 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -318,3 +318,36 @@ three ISAs. The Zig namespace and the `stdlib.h`/`pthread.h` aliases derive from that header's function list alone, so they are current as well. Not run locally: `zig fmt`, `test-foreign`, the rest of `bindings-check` and `kore-lua`'s Kore-target build; CI's host job runs them. + +## A C program's runtime, over Kore #1494's kore-rt + +The export records Kore `0018956a3aca19dc6acea8d28fb8ec9fadefcb09`, a commit +on MicroPerceptron/kore#1494's branch, which brings TinyCC to Kore's shell. +Eleven of the 26 commits since `7074bda2` reach the closure, which keeps its +26 packages and 382 files: + +- `kore-rt` carries what `kore-c`'s host needs: + - `startup::barrier`, the process barrier without the exit; + - managed threads that inherit their founder's wake send; + - `crash::report`, a panic's report without the thread's end; + - also a per-thread C `errno` word, guarded worker stacks sized in bytes, + and the verbs and bundle call a shell's private install takes. +- `kore-rt`'s thread controller, which `kore-ffi`'s pthreads run on, parks + on the futex and never polls (Kore `0018956a`). Its threads get their own + doorbells and the controller relays the wakes they cannot deliver to each + other. A thread's end reaches it through the exit hook `thread::exit` + runs, and its reap parks on a join arm. Here `kore-ffi`'s join parks on + its retirement word, which the controller wakes. `kore-c`'s main thread + holds a channel-space grant to mint the controller's doorbells from. +- `kore-abi` carries the install request and its replies, and the claimant + band sized by driver rows (`MAX_CLAIMANT_FACES`), which the device service + uses. +- The services' `boot.toml` digests were resealed three times. + +No exported manifest changed, so `native/Cargo.lock` did not. + +`kore-c` is new: the runtime of `libkore.so`, and the host of an independent +C program (`docs/languages.md`). It is the first libkore package a C program +on Kore links without a Rust host. Its shared object is a distribution's +one-line `cdylib` over it. `tools/check-program.py` links that object for the +three Kore ISAs and checks its shape and its exact exported ABI. diff --git a/extern/c/include/kore/program.h b/extern/c/include/kore/program.h new file mode 100644 index 0000000..9eda7e8 --- /dev/null +++ b/extern/c/include/kore/program.h @@ -0,0 +1,317 @@ +/* libkore.so: what an independent C program on Kore adds to kore.h. Link with -lkore. */ + +#ifndef KORE_PROGRAM_H +#define KORE_PROGRAM_H + +#pragma once + +/* Generated by cbindgen 0.29.4 from kore-c. Do not edit. */ + +#include "../kore.h" + +/** + * The stack `main` runs on: a guarded strip of the task's stack window, + * committed whole when the program starts. A frame deeper than this faults + * on the guard page below it, as a stack overflow does. + * + * The size of the strip `tcc`, a large C program whose depth its input + * decides, measured its need under Kore's runtime at and runs in; ISO C's + * own guarantee of 127 nested blocks (C11 §5.2.4.1) needs a small part of + * it. Larger strips come out of the 16 MiB window the program's SPMD gangs + * share, and would be committed at every launch. + */ +#define KORE_MAIN_STACK_BYTES (1 << 20) + +/** + * [`KoreFdInfo::kind`]: the memory supply the program's allocator grows + * on. `mmap` answers a fresh, zeroed region of it. + */ +#define KORE_FD_MEMORY 1 + +/** + * [`KoreFdInfo::kind`]: a virtual device (``). `mmap` answers + * its register block and memory to lend it, `read` its interrupt, and + * `write` rings its doorbell. + */ +#define KORE_FD_DEVICE 2 + +/** + * [`KoreFdInfo::access`]: the capability may be read. + */ +#define KORE_FD_READ (1 << 0) + +/** + * [`KoreFdInfo::access`]: the capability may be written. + */ +#define KORE_FD_WRITE (1 << 1) + +/** + * Bytes in a virtual device's register block: one page. + */ +#define KORE_VDEV_REGISTERS_BYTES 4096 + +/** + * `mmap` offset on a virtual device's descriptor: the register block + * ([`KoreVdevRegisters`]). + */ +#define KORE_VDEV_MAP_REGISTERS 0 + +/** + * `mmap` offset on a virtual device's descriptor: fresh, zeroed memory the + * driver owns and lends the device. A device address is an offset into it. + * The device takes a loan only in its reset state, so map it before the + * first status write. + */ +#define KORE_VDEV_MAP_MEMORY 4096 + +/** + * [`KoreVdevRegisters::magic`]: `"kvdv"`, a Kore virtual device. + */ +#define KORE_VDEV_MAGIC 1986295403 + +/** + * [`KoreVdevRegisters::version`]: this layout. + */ +#define KORE_VDEV_VERSION 1 + +/** + * [`KoreVdevRegisters::vendor_id`]: `"kore"`, a device Kore hosts. + */ +#define KORE_VDEV_VENDOR_KORE 1701998443 + +/** + * [`KoreVdevRegisters::device_id`]: `"accl"`, the virtual accelerator, a + * virtio-accel 1.0 device. Its name is `/kore/device/virtual/accel`. + */ +#define KORE_VDEV_ID_ACCEL 1818452833 + +/** + * Status: the driver has noticed the device (Virtio 1.3 section 2.1). + */ +#define KORE_VDEV_STATUS_ACKNOWLEDGE 1 + +/** + * Status: the driver knows how to drive it. + */ +#define KORE_VDEV_STATUS_DRIVER 2 + +/** + * Status: the driver is set up and the device may serve. + */ +#define KORE_VDEV_STATUS_DRIVER_OK 4 + +/** + * Status: feature negotiation is complete. + */ +#define KORE_VDEV_STATUS_FEATURES_OK 8 + +/** + * Status, written by the device: it cannot go on and must be reset. + */ +#define KORE_VDEV_STATUS_DEVICE_NEEDS_RESET 64 + +/** + * Status: the driver has given up on the device. + */ +#define KORE_VDEV_STATUS_FAILED 128 + +/** + * Feature bit 32, `VIRTIO_F_VERSION_1`: every device offers it and a + * driver must accept it. Written out, because the header carries this + * expression as it stands and `1 << 32` is not a 64-bit value in C. + */ +#define KORE_VDEV_F_VERSION_1 4294967296 + +/** + * Bytes between the last register and the device configuration. + */ +#define KORE_VDEV_RESERVED_BYTES 168 + +/** + * Bytes of the device configuration area, to the end of the block. + */ +#define KORE_VDEV_CONFIG_AREA_BYTES 3840 + +/** + * What a descriptor names, as the calling thread holds it + * ([`kore_fd_describe`]). + */ +typedef struct KoreFdInfo { + /** + * What kind of capability it is: `KORE_FD_MEMORY` or `KORE_FD_DEVICE`. + */ + uint32_t kind; + /** + * What the capability's own rights permit this thread, as + * `KORE_FD_READ | KORE_FD_WRITE` bits: the rights it was founded with, + * which the access `open` asked for can only narrow. + */ + uint32_t access; + /** + * For `KORE_FD_MEMORY`, the supply's whole length in bytes: what the + * allocator and every mapping share, not what is still free. For + * `KORE_FD_DEVICE`, the register block's length. + */ + uint64_t length; +} KoreFdInfo; + +/** + * The register block of a Kore virtual device: what `mmap` at + * `KORE_VDEV_MAP_REGISTERS` answers, declared `volatile` by a driver. + * + * Every word is little-endian and has one writer. A driver never writes a + * device register, and what it writes to its own the device reads once, at + * the step that uses it, when the driver rings the doorbell (a `write` of + * one 32-bit word to the descriptor). The device answers by raising an + * interrupt: it increments `interrupts`, and a `read` of one 32-bit word + * from the descriptor returns once the count has moved. + * + * Bringing the device up is Virtio 1.3 section 3.1.1 over these registers, + * with the memory lent first (`KORE_VDEV_MAP_MEMORY`). A device just opened + * is in its reset state, both status words zero; one that is not is reset + * first, by writing 0 to `driver_status` and ringing. Then write + * `ACKNOWLEDGE | DRIVER` and ring; write the accepted features, add + * `FEATURES_OK`, ring, and check `device_status` carries it; lay the queue + * out in the lent memory, write `queue_num`, the three addresses and + * `queue_ready`; add `DRIVER_OK` and ring. A new value of `driver_status` + * is always answered with an interrupt, and a step the device refused is + * one its `device_status` does not carry. The value the device last saw + * announces nothing and is not answered, so a driver waits only for what + * it asked. + */ +typedef struct KoreVdevRegisters { + /** + * Device: `KORE_VDEV_MAGIC`. + */ + uint32_t magic; + /** + * Device: `KORE_VDEV_VERSION`. + */ + uint32_t version; + /** + * Device: which model this is (`KORE_VDEV_ID_ACCEL`). + */ + uint32_t device_id; + /** + * Device: `KORE_VDEV_VENDOR_KORE`. + */ + uint32_t vendor_id; + /** + * Device: the features it offers. + */ + uint64_t device_features; + /** + * Driver: the features it accepts, a subset of the offer. + */ + uint64_t driver_features; + /** + * Driver: the status it has reached (`KORE_VDEV_STATUS_*`). + */ + uint32_t driver_status; + /** + * Device: the status it has accepted, and its own + * `KORE_VDEV_STATUS_DEVICE_NEEDS_RESET`. + */ + uint32_t device_status; + /** + * Device: the largest queue it takes. + */ + uint32_t queue_num_max; + /** + * Driver: the queue's size, a power of two between the longest chain + * the device takes (its configuration says how long) and + * `queue_num_max`. + */ + uint32_t queue_num; + /** + * Driver: nonzero once the queue's size and addresses are written. + */ + uint32_t queue_ready; + /** + * Device: changes whenever the configuration area does. + */ + uint32_t config_generation; + /** + * Driver: the descriptor table's device address. + */ + uint64_t queue_desc; + /** + * Driver: the available ring's device address. + */ + uint64_t queue_driver; + /** + * Driver: the used ring's device address. + */ + uint64_t queue_device; + /** + * Device: bytes of device configuration in `config`. + */ + uint32_t config_bytes; + /** + * Device: interrupts raised, wrapping. + */ + uint32_t interrupts; + /** + * Unused. + */ + uint8_t reserved[KORE_VDEV_RESERVED_BYTES]; + /** + * Device: the device configuration. + */ + uint8_t config[KORE_VDEV_CONFIG_AREA_BYTES]; +} KoreVdevRegisters; + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +/** + * Run `step` as this program's filesystem session, through the filesystem + * face its launch carried, rooted where it was launched (its working + * directory). The callbacks and their pointers obey `KoreFsStep`'s contract: + * each callback runs on the program's host thread, as on any native + * executor, while the caller waits, so a callback must not depend on the + * caller's thread-local state. + * + * Returns `KORE_STATUS_OK` once the callback program ended and every view it + * left open closed. `KORE_STATUS_QUARANTINED`: a reply was lost or + * uncertain, and no session can run again in this program. + * `KORE_STATUS_REFUSED`: a close was refused (`*error` names why, and the + * view stays open), the launch carried no filesystem face + * (`KORE_FS_ERROR_UNSUPPORTED`), or the caller is not the thread `main` runs + * on, which alone waits for the host. `KORE_STATUS_CAPACITY`: the native + * owner could not be formed. `KORE_STATUS_INVALID_ARGUMENT`: `step` is null. + * `*error` is written whenever `error` is not null. + * + * # Safety + * `step` obeys `KoreFsStep`'s pointer, serialization and no-unwind contract; + * `user` remains valid, and the callbacks' alone, until this returns. `error` + * is null or valid for one aligned write. + */ +KoreStatus kore_fs_run(KoreFsStep step, void *user, KoreFsError *error); + +/** + * Describe the capability behind `fd`, a descriptor `open` answered + * (``): `open` finds a capability this program was founded holding + * by its name, and this reads what was found. Nothing is granted or changed. + * + * Capabilities are held per thread, so the answer is the calling thread's: + * the same descriptor may describe a capability on the thread that opened + * it and be refused on one founded without it. + * + * Returns `KORE_STATUS_OK` with `*out` written. + * `KORE_STATUS_INVALID_ARGUMENT`: `out` is null. `KORE_STATUS_STALE_HANDLE`: + * `fd` is not an open descriptor. `KORE_STATUS_REFUSED`: the calling thread + * holds no such capability, or this is not a Kore program. A refusal leaves + * `*out` unchanged. + * + * # Safety + * `out` is null or valid for one aligned write. + */ +KoreStatus kore_fd_describe(int fd, struct KoreFdInfo *out); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif /* KORE_PROGRAM_H */ diff --git a/native/Cargo.lock b/native/Cargo.lock index 8eef346..fbe8c84 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -247,6 +247,14 @@ dependencies = [ "kore-window", ] +[[package]] +name = "kore-c-shim" +version = "0.1.0" +dependencies = [ + "kore-rt", + "rustc_apfloat", +] + [[package]] name = "kore-collections" version = "0.1.0" @@ -373,6 +381,15 @@ dependencies = [ "kore-rt", ] +[[package]] +name = "kore-svc-accel-spmd" +version = "0.0.1" +dependencies = [ + "kore-driver", + "kore-rt", + "virtio-accel-core", +] + [[package]] name = "kore-svc-block" version = "0.0.1" @@ -477,10 +494,48 @@ dependencies = [ "lz4_flex", ] +[[package]] +name = "kore-svc-vdev" +version = "0.0.0" +dependencies = [ + "kore-abi", + "kore-rt", + "kore-vdev", + "kore-vdev-accel", + "virtio-accel-guest", + "virtio-accel-proto", + "virtio-accel-transport", +] + [[package]] name = "kore-uart" version = "0.0.0" +[[package]] +name = "kore-vdev" +version = "0.0.1" +dependencies = [ + "heapless", + "virtio-accel-split-queue", + "virtio-accel-transport", +] + +[[package]] +name = "kore-vdev-accel" +version = "0.0.1" +dependencies = [ + "kore-driver", + "kore-rt", + "kore-svc-accel-spmd", + "kore-vdev", + "virtio-accel-core", + "virtio-accel-device", + "virtio-accel-guest", + "virtio-accel-proto", + "virtio-accel-transport", + "zerocopy", +] + [[package]] name = "kore-vol" version = "0.0.0" @@ -583,6 +638,16 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + [[package]] name = "rustix" version = "1.1.6" @@ -622,6 +687,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "smallvec" +version = "1.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149309498a5f7334b12ddae0cad8aba81ce785fe91755068867e017cf98fee46" + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -740,6 +811,49 @@ dependencies = [ "virtio-accel-transport", ] +[[package]] +name = "virtio-accel-device" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad6a272d3b25527b3a6ea2de02088fead249efb8bbd8d2a2590f7151eb0ea463" +dependencies = [ + "virtio-accel-core", + "virtio-accel-proto", + "virtio-accel-transport", + "zerocopy", +] + +[[package]] +name = "virtio-accel-guest" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520c262718f4214d42333ed985f3f86e9589d8f3509a1afe69f4606cd120f4da" +dependencies = [ + "bitflags", + "virtio-accel-proto", + "virtio-accel-transport", + "zerocopy", +] + +[[package]] +name = "virtio-accel-proto" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cda1ee8cd8ebd8b100ceae0ccd9de228036d3c1dc50483f37c5f5b9c63c0f0f" +dependencies = [ + "bitflags", + "zerocopy", +] + +[[package]] +name = "virtio-accel-split-queue" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf3ad9ce01297c6bf1f01ffaf9cc4e11bff4d639661af6b5c1136d2819d6291" +dependencies = [ + "virtio-accel-transport", +] + [[package]] name = "virtio-accel-transport" version = "0.4.1" @@ -778,18 +892,18 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "zerocopy" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" dependencies = [ "proc-macro2", "quote", diff --git a/native/Cargo.toml b/native/Cargo.toml index 20a77a3..8487af6 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -3,6 +3,7 @@ resolver = "3" members = [ "crates/abi", "crates/app", + "crates/c-shim", "crates/collections", "crates/dagfs", "crates/driver", @@ -15,8 +16,11 @@ members = [ "crates/scrub", "crates/start", "crates/uart", + "crates/vdev", + "crates/vdev-accel", "crates/window", "drivers/nvme", + "services/accel-spmd", "services/block", "services/dagfs", "services/device", @@ -26,5 +30,6 @@ members = [ "services/mm", "services/output", "services/swap", + "services/vdev", "tools/kore-vol", ] diff --git a/native/crates/abi/src/command.rs b/native/crates/abi/src/command.rs index d62b18c..f34eca2 100644 --- a/native/crates/abi/src/command.rs +++ b/native/crates/abi/src/command.rs @@ -7,14 +7,22 @@ use crate::image::{ImageName, NAME_BYTES}; -/// Snapshot wire version. -pub const COMMAND_SNAPSHOT_ABI: u8 = 1; +/// Snapshot wire version. 2: an entry may carry no usage. A launcher +/// carries the usage of the rows its command line names and only the name +/// and summary of the rest ([`CommandSnapshotBuilder::push_listed`]), so the +/// record grows with the catalogue's names and summaries rather than with +/// every row's usage, and has room for the programs a task installs. A `1` +/// reader refuses an empty usage. +pub const COMMAND_SNAPSHOT_ABI: u8 = 2; /// The largest task-local command set accepted by the decoder. Raised /// from 32 when the desktop's `editor` and `power-ui` rows joined the two /// native filesystem verbs, and from 40 when Files and the documents editor -/// became two rows; the byte budget (`PROCESS_CONTEXT_BYTES`) is the other -/// bound, proven by exec's host test over the real catalogue. -pub const MAX_COMMAND_SNAPSHOT_ENTRIES: usize = 48; +/// became two rows; and from 48 for the programs a task installs, which +/// its launcher lists beside the catalogue (47 rows and a shell's share of +/// the installed pool no longer fit 48). The byte budget +/// (`PROCESS_CONTEXT_BYTES`) is the other bound; exec's host test proves +/// both over the real catalogue with that share installed. +pub const MAX_COMMAND_SNAPSHOT_ENTRIES: usize = 64; /// The launcher holds a power-control face and therefore handles these two /// privileged controls in-process. pub const COMMAND_SNAPSHOT_SYSTEM_POWER: u8 = 1 << 0; @@ -38,6 +46,8 @@ pub enum CommandSnapshotError { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct CommandDescription<'a> { pub local_name: &'a str, + /// The command's usage, or empty when the launcher carried none for + /// this row. pub usage: &'a str, pub summary: &'a str, } @@ -67,13 +77,35 @@ impl<'a> CommandSnapshotBuilder<'a> { }) } - /// Append one local namespace row. The executable capability paired with - /// this metadata deliberately is not accepted by this API. + /// Append one local namespace row with its usage. The executable + /// capability paired with this metadata deliberately is not accepted by + /// this API. pub fn push( &mut self, local_name: &ImageName, usage: &[u8], summary: &[u8], + ) -> Result<(), CommandSnapshotError> { + validate_text(usage, false)?; + self.append(local_name, usage, summary) + } + + /// Append one local namespace row without its usage: a row the reader + /// lists but was not asked to describe. + pub fn push_listed( + &mut self, + local_name: &ImageName, + summary: &[u8], + ) -> Result<(), CommandSnapshotError> { + self.append(local_name, &[], summary) + } + + /// `usage` is valid text, or empty for a row that carries none. + fn append( + &mut self, + local_name: &ImageName, + usage: &[u8], + summary: &[u8], ) -> Result<(), CommandSnapshotError> { if usize::from(self.count) >= MAX_COMMAND_SNAPSHOT_ENTRIES { return Err(CommandSnapshotError::TooManyEntries); @@ -82,7 +114,6 @@ impl<'a> CommandSnapshotBuilder<'a> { .as_str() .ok_or(CommandSnapshotError::InvalidText)?; validate_text(name.as_bytes(), true)?; - validate_text(usage, false)?; validate_text(summary, false)?; if name.len() > u8::MAX as usize || usage.len() > u8::MAX as usize @@ -260,7 +291,10 @@ fn decode_entry( cursor += usage_len; let summary = &bytes[cursor..cursor + summary_len]; validate_text(name, true)?; - validate_text(usage, false)?; + // A row the launcher only listed carries no usage. + if !usage.is_empty() { + validate_text(usage, false)?; + } validate_text(summary, false)?; Ok(( CommandDescription { @@ -318,6 +352,47 @@ mod tests { assert!(snapshot.find(b"touch").is_none()); } + #[test] + fn a_listed_row_carries_its_name_and_summary_and_no_usage() { + let mut bytes = [0u8; 256]; + let mut builder = CommandSnapshotBuilder::new(&mut bytes, 0).unwrap(); + builder + .push_listed(&image("hello"), b"dynamic-linker witness") + .unwrap(); + builder + .push(&image("ls"), b"ls [-l] [path]", b"list directory entries") + .unwrap(); + let listed_only = 4 + "hello".len() + "dynamic-linker witness".len(); + let len = builder.finish(); + let snapshot = CommandSnapshot::parse(&bytes[..len]).unwrap(); + assert_eq!( + snapshot.find(b"hello"), + Some(CommandDescription { + local_name: "hello", + usage: "", + summary: "dynamic-linker witness", + }) + ); + assert_eq!(snapshot.find(b"ls").unwrap().usage, "ls [-l] [path]"); + // The usage a listed row leaves out is exactly what it saves. + assert_eq!( + len, + 8 + listed_only + 4 + "ls".len() + "ls [-l] [path]".len() + 22 + ); + // A row that is described carries a usage: `push` takes no empty one, + // and a summary is never optional. + let mut bytes = [0u8; 64]; + let mut builder = CommandSnapshotBuilder::new(&mut bytes, 0).unwrap(); + assert_eq!( + builder.push(&image("one"), b"", b"first"), + Err(CommandSnapshotError::InvalidText) + ); + assert_eq!( + builder.push_listed(&image("one"), b""), + Err(CommandSnapshotError::InvalidText) + ); + } + #[test] fn duplicate_and_malformed_rows_fail_closed() { let mut bytes = [0u8; 128]; diff --git a/native/crates/abi/src/device.rs b/native/crates/abi/src/device.rs index 60b8e89..f739f5e 100644 --- a/native/crates/abi/src/device.rs +++ b/native/crates/abi/src/device.rs @@ -20,9 +20,13 @@ //! route) — and teardown is an ordinary `Revoke` of the leaf. mod census; +mod class; mod i2c_admission; +mod state; pub use census::*; +pub use class::*; pub use i2c_admission::*; +pub use state::*; /// An I/O virtual address — a device-domain address translated by the /// IOMMU. **Never** a physical address: userland holds only this, so it can @@ -89,6 +93,26 @@ pub enum DeviceSpecies { /// its banks. pub const MAX_ROOT_COMPLEXES: usize = 8; +/// Claimant faces one device broker can mint: one per driver row, the +/// founder minting face `n` for driver row `n` (`MintFace`, +/// `res/DEVICE_SERVICE.md` §10). Five today: input, block, accel, net and +/// Xe. +/// +/// The bound is shared, not chosen. The kernel founder seats driver row +/// `n`'s one executor bell at a fixed slot of its common pin island and +/// asserts the island holds this many. Exec's manifest gate refuses a +/// driver row past it. The broker sizes its face table, its claimant band +/// ([`CLAIMANT_FACE_SUPPLY_BYTES`](crate::task::CLAIMANT_FACE_SUPPLY_BYTES)) +/// and its landing band (two base-arena slots per face, below the +/// startup-link band, which it asserts) from it. Raising it costs one +/// ring stride per broker, two broker slots and one founder bell slot per +/// face; the founder's pins are the first of those to run out. +/// +/// Deliberately not the image service's face count: that one grows with +/// the executable catalogue, and a broker band sized by it would grow with +/// every program the system can run. +pub const MAX_CLAIMANT_FACES: usize = 5; + impl DeviceSpecies { /// Decode the wire byte; `None` for an unknown species (fail-closed — /// a broker that cannot name the species must not derive). diff --git a/native/crates/abi/src/device/class.rs b/native/crates/abi/src/device/class.rs new file mode 100644 index 0000000..e43aa63 --- /dev/null +++ b/native/crates/abi/src/device/class.rs @@ -0,0 +1,153 @@ +//! What a PCI function's **class code** says it is to a driver. +//! +//! A function is offered to drivers on positive evidence only. The class +//! code is the one identification every vendor's function carries, so it is +//! the evidence asked first, and it is asked in one direction: a class named +//! here as a peripheral admits a function; no class, and no absence of one, +//! proves a function safe to hand out. Everything this table does not name +//! is [`Platform`](PciClassRole::Platform), the functions a machine keeps +//! for itself: its management engine, flash and SMBus controllers, power +//! and thermal telemetry, shared memory, and whatever a vendor adds next. +//! Those differ on every platform and their classes are mostly the +//! catch-all "other" subclasses, so they are not enumerated: they are what +//! is left. A new or unknown function is therefore kept, never offered. +//! +//! The kernel decides ([`DeviceRecord::DRIVER_ENDPOINT`]); a reader of a +//! census row uses the same table to say *why* a function is withheld. +//! Codes are the PCI-SIG's (PCI Code and ID Assignment Specification, +//! "Class Codes"). +//! +//! [`DeviceRecord::DRIVER_ENDPOINT`]: super::DeviceRecord::DRIVER_ENDPOINT + +/// What a class code alone admits a function as. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PciClassRole { + /// Bus fabric: a bridge of any kind (`06`), or an IOMMU (`08/06`). It + /// carries other functions' traffic and no driver takes it. + Fabric, + /// A display controller (`03`), or the legacy VGA-compatible spelling + /// (`00/01`). A peripheral, but one firmware may still be scanning out + /// of: the kernel also requires proof of its boot ownership. + Display, + /// A device with a standard programming model that a driver is written + /// against, and that a machine keeps running without. + Peripheral, + /// Anything else: a function of the platform itself. Not offered. + Platform, +} + +impl PciClassRole { + /// The role `class`/`subclass`/`progif` admit. + pub const fn of(class: u8, subclass: u8, progif: u8) -> Self { + match (class, subclass) { + (0x06, _) | (0x08, 0x06) => Self::Fabric, + (0x03, _) | (0x00, 0x01) => Self::Display, + // Mass storage, network, multimedia (audio, video, telephony), + // input, wireless and processing accelerators: every subclass + // is a device someone attaches or a driver brings up, including + // the vendor-specific "other" ones (a Wi-Fi radio is `02/80`). + (0x01 | 0x02 | 0x04 | 0x09 | 0x0d | 0x12, _) => Self::Peripheral, + // A UART with a named register model: 16450 through 16950. + // Programming interface 0 ("generic XT") names none, and is + // what a platform stamps on functions that are not UARTs at + // all (a sensor hub, a management engine's serial redirection). + (0x07, 0x00) if matches!(progif, 0x01..=0x06) => Self::Peripheral, + // An SD host controller: the one base-system peripheral that is + // a slot for removable media rather than part of the machine. + (0x08, 0x05) => Self::Peripheral, + // Host controllers of the external peripheral buses: IEEE 1394, + // USB (every interface, USB4 included), Fibre Channel and + // InfiniBand. SMBus (`0c/05`), IPMI (`0c/08`), I3C (`0c/0a`) + // and "other" (`0c/80`: a platform's I2C, SPI and flash + // controllers) are the machine's own buses. + (0x0c, 0x00 | 0x03 | 0x04 | 0x07) => Self::Peripheral, + _ => Self::Platform, + } + } +} + +#[cfg(test)] +mod tests { + use super::PciClassRole::{self, Display, Fabric, Peripheral, Platform}; + + #[test] + fn a_class_admits_a_peripheral_and_everything_unnamed_is_the_platforms() { + for (class, subclass, progif, role, what) in [ + (0x06, 0x00, 0x00, Fabric, "host bridge"), + (0x06, 0x01, 0x00, Fabric, "ISA/LPC bridge"), + (0x06, 0x04, 0x00, Fabric, "PCI-to-PCI bridge"), + (0x08, 0x06, 0x00, Fabric, "IOMMU"), + (0x03, 0x00, 0x00, Display, "VGA controller"), + (0x03, 0x80, 0x00, Display, "display, other"), + (0x00, 0x01, 0x00, Display, "legacy VGA-compatible"), + (0x01, 0x08, 0x02, Peripheral, "NVMe"), + (0x01, 0x06, 0x01, Peripheral, "AHCI"), + (0x02, 0x00, 0x00, Peripheral, "Ethernet"), + (0x02, 0x80, 0x00, Peripheral, "Wi-Fi radio"), + (0x04, 0x03, 0x00, Peripheral, "HD Audio"), + (0x09, 0x00, 0x00, Peripheral, "keyboard"), + (0x0d, 0x11, 0x00, Peripheral, "Bluetooth"), + (0x12, 0x00, 0x00, Peripheral, "neural accelerator"), + (0x07, 0x00, 0x02, Peripheral, "16550 UART"), + (0x08, 0x05, 0x01, Peripheral, "SD host controller"), + (0x0c, 0x03, 0x30, Peripheral, "xHCI"), + (0x0c, 0x03, 0x40, Peripheral, "USB4 host interface"), + // What a machine keeps. The identities are the ones one Lunar + // Lake laptop and one AMD laptop enumerate. + (0x05, 0x00, 0x00, Platform, "PMC shared SRAM"), + (0x07, 0x80, 0x00, Platform, "management engine HECI"), + ( + 0x07, + 0x00, + 0x00, + Platform, + "sensor hub stamped generic serial", + ), + ( + 0x0c, + 0x80, + 0x00, + Platform, + "SPI flash or LPSS I2C controller", + ), + (0x0c, 0x05, 0x00, Platform, "SMBus"), + (0x0c, 0x0a, 0x00, Platform, "I3C"), + (0x11, 0x80, 0x00, Platform, "thermal and power tuning"), + (0x10, 0x80, 0x00, Platform, "platform security processor"), + (0x13, 0x00, 0x00, Platform, "instrumentation"), + (0x08, 0x80, 0x00, Platform, "system peripheral, other"), + (0x00, 0x00, 0x00, Platform, "unclassified"), + (0x00, 0xff, 0x00, Platform, "unclassified, vendor"), + (0xff, 0x00, 0x00, Platform, "unassigned"), + ] { + assert_eq!( + PciClassRole::of(class, subclass, progif), + role, + "{what} ({class:02x}/{subclass:02x}.{progif:02x})" + ); + } + } + + /// The table only ever admits by naming: no class outside the ones it + /// names is a peripheral, whatever its subclass or interface. + #[test] + fn no_unnamed_class_is_ever_a_peripheral() { + for class in 0..=u8::MAX { + if matches!( + class, + 0x01 | 0x02 | 0x04 | 0x07 | 0x08 | 0x09 | 0x0c | 0x0d | 0x12 + ) { + continue; + } + for subclass in 0..=u8::MAX { + for progif in [0x00, 0x01, 0x02, 0x30, 0x80, 0xff] { + assert_ne!( + PciClassRole::of(class, subclass, progif), + Peripheral, + "{class:02x}/{subclass:02x}.{progif:02x}" + ); + } + } + } + } +} diff --git a/native/crates/abi/src/device/state.rs b/native/crates/abi/src/device/state.rs new file mode 100644 index 0000000..dacf681 --- /dev/null +++ b/native/crates/abi/src/device/state.rs @@ -0,0 +1,923 @@ +//! The read-only **live state** of the PCI functions the census +//! ([`super::census`]) lists, and the shape of one function's BARs. +//! +//! The census is boot-frozen identity. This is what changes after boot, and +//! what a driver writer asks before touching a device: whether a driver +//! claimed the function, which founded row holds that claim, which BARs exist, +//! which of them a driver may map and which are mapped now, and how the +//! function can interrupt. A BAR is its index, its size and the length a +//! driver may map; **no address crosses** (res/DEVICE_SERVICE.md §1.1), so +//! nothing here says where a BAR lives. +//! +//! Descriptions only, as a census row is: a state grants no inventory, +//! claim, mapping or route, and reading one changes nothing. It is a +//! snapshot. A claim can land or a route can be released the moment after +//! the kernel answers, so no consumer may treat it as permission. +//! +//! The path is the census's: the kernel answers one +//! [`SystemMetric::DeviceState`](crate::session::SystemMetric::DeviceState) +//! or [`SystemMetric::DeviceBar`](crate::session::SystemMetric::DeviceBar) +//! observation on the `SystemObserve` capability the monitor holds; the +//! monitor samples them on each request (never cached, since they are live) +//! and vends [`DeviceStatePage`]s and one [`DeviceBarTable`]; exec relays +//! them to a launcher; the launcher stamps a [`DeviceInventory`] into the +//! process-context prefix of an executable whose bootstrap row declares it. + +use zerocopy::byteorder::{LittleEndian, U16, U64}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; + +use super::census::{DeviceCensus, DeviceCensusPage, MAX_DEVICE_CENSUS_ROWS}; + +/// BARs a PCI function has (PCI Local Bus Specification 3.0 §6.2.5: six +/// Base Address Registers in a type 0 header). +pub const DEVICE_BARS: usize = 6; +/// Every BAR's bit, for the three per-BAR bitmaps of a [`DeviceState`]. +pub const DEVICE_BAR_MASK: u8 = (1 << DEVICE_BARS) - 1; +/// `DeviceState::driver` when no running founded row holds the function. +pub const DEVICE_ROW_NONE: u8 = 0xFF; +/// States per vended page: 8 header bytes + 128 = 136 bytes, the census +/// page's size, so a page rides the monitor and exec reply wires as one. +pub const DEVICE_STATE_PAGE_ROWS: usize = 16; +/// Pages the states of a full census span. +pub const DEVICE_STATE_PAGES: usize = MAX_DEVICE_CENSUS_ROWS.div_ceil(DEVICE_STATE_PAGE_ROWS); +/// Layout version of [`DeviceInventory`]. A zeroed record reads 0 and is +/// *unavailable*. +pub const DEVICE_INVENTORY_ABI: u16 = 1; +/// [`SystemMetric::DeviceBar`](crate::session::SystemMetric::DeviceBar)'s +/// operand is `index | bar << DEVICE_BAR_OPERAND_SHIFT`: the catalog index +/// in the low 13 bits (every index a census can hold fits, asserted below), +/// the BAR number in the top three. +pub const DEVICE_BAR_OPERAND_SHIFT: u32 = 13; + +/// The operand naming BAR `bar` of the function at catalog `index`; `None` +/// when either is out of the operand's range. +pub const fn device_bar_operand(index: u16, bar: u8) -> Option { + if index >> DEVICE_BAR_OPERAND_SHIFT != 0 || bar as usize >= DEVICE_BARS { + return None; + } + Some(index | (bar as u16) << DEVICE_BAR_OPERAND_SHIFT) +} + +/// The `(index, bar)` a [`device_bar_operand`] names; `None` for a BAR +/// number no function has. +pub const fn device_bar_of(operand: u16) -> Option<(u16, u8)> { + let bar = (operand >> DEVICE_BAR_OPERAND_SHIFT) as u8; + if bar as usize >= DEVICE_BARS { + return None; + } + Some((operand & ((1 << DEVICE_BAR_OPERAND_SHIFT) - 1), bar)) +} + +/// One function's live state. 8 bytes, frozen: one observation's value +/// word, byte for byte. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceState { + /// [`Self::PRESENT`] and the bits below it. + pub flags: u8, + /// The founded row (the founding record's index) whose task holds this + /// function's claim, or [`DEVICE_ROW_NONE`]. The kernel records whom a + /// claim's grant is sealed for when the claim commits, and follows that + /// exact grant if it is carried on, so this names the claimant whether + /// or not it does DMA. A claim whose task is not a running founded row + /// (it ended, or it is a program) reads none: the claim still stands. + pub driver: u8, + /// Bit `n`: BAR `n` exists as memory the kernel resolved. An I/O BAR is + /// not one: no driver may reach port space through a BAR. + pub bars_present: u8, + /// Bit `n`: a driver may map a prefix of BAR `n`. + pub bars_mappable: u8, + /// Bit `n`: BAR `n` is mapped by a driver now. + pub bars_mapped: u8, + /// Vectors the function's MSI capability can deliver (1, 2, 4, 8, 16 or + /// 32); 0 when it has none. + pub msi_vectors: u8, + /// Entries in the function's MSI-X table; 0 when it has none. + pub msix_entries: U16, +} + +impl DeviceState { + /// A populated state; clear marks the zero state. + pub const PRESENT: u8 = 1 << 0; + /// A driver holds the function: its claim is committed. + pub const CLAIMED: u8 = 1 << 1; + /// An interrupt route is established on the function now. + pub const ROUTED: u8 = 1 << 2; + /// A driver can establish a remapped MSI-family route (MSI or MSI-X), + /// the only kind userland may hold. + pub const ROUTE_MSI: u8 = 1 << 3; + /// A driver can establish a remapped MSI-X route. + pub const ROUTE_MSIX: u8 = 1 << 4; + /// The function has a legacy interrupt pin; [`Self::pin`] says which. + pub const HAS_PIN: u8 = 1 << 5; + /// Bits 6 and 7: the legacy pin, 0 for INTA through 3 for INTD. + pub const PIN_SHIFT: u32 = 6; + + pub const EMPTY: Self = Self { + flags: 0, + driver: 0, + bars_present: 0, + bars_mappable: 0, + bars_mapped: 0, + msi_vectors: 0, + msix_entries: U16::ZERO, + }; + + pub const fn is_present(&self) -> bool { + self.flags & Self::PRESENT != 0 + } + + pub const fn claimed(&self) -> bool { + self.flags & Self::CLAIMED != 0 + } + + /// The legacy interrupt pin, 0 for INTA through 3 for INTD. + pub const fn pin(&self) -> Option { + if self.flags & Self::HAS_PIN == 0 { + return None; + } + Some(self.flags >> Self::PIN_SHIFT) + } + + /// The founded row driving the function's DMA. + pub const fn driver(&self) -> Option { + if self.driver == DEVICE_ROW_NONE { + None + } else { + Some(self.driver) + } + } + + pub const fn validate(&self) -> bool { + if !self.is_present() { + return self.flags == 0 + && self.driver == 0 + && self.bars_present == 0 + && self.bars_mappable == 0 + && self.bars_mapped == 0 + && self.msi_vectors == 0 + && self.msix_entries.get() == 0; + } + // A pin without its flag would be a second spelling of "no pin". + (self.flags & Self::HAS_PIN != 0 || self.flags >> Self::PIN_SHIFT == 0) + && self.bars_present & !DEVICE_BAR_MASK == 0 + // Only a BAR that exists can be mapped, and only one a driver + // may map is. + && self.bars_mappable & !self.bars_present == 0 + && self.bars_mapped & !self.bars_mappable == 0 + && (self.msi_vectors == 0 + || (self.msi_vectors.is_power_of_two() && self.msi_vectors <= 32)) + && self.msix_entries.get() <= super::DeviceRecord::MAX_MSIX_TABLE_ENTRIES + } + + /// The state as one `SystemObserve` report, `(value, aux)`, with the + /// function's catalog `index` echoed in `aux[63:48]` so a reader can + /// prove the report answers the function it asked for. + pub const fn to_observation(&self, index: u16) -> (u64, u64) { + let value = u64::from_le_bytes([ + self.flags, + self.driver, + self.bars_present, + self.bars_mappable, + self.bars_mapped, + self.msi_vectors, + self.msix_entries.get().to_le_bytes()[0], + self.msix_entries.get().to_le_bytes()[1], + ]); + (value, (index as u64) << 48) + } + + /// Inverse of [`to_observation`](Self::to_observation); `None` when the + /// echoed index is not `index`, the rest of `aux` is not zero, or the + /// state is not a valid populated one. + pub const fn from_observation(index: u16, value: u64, aux: u64) -> Option { + if aux != (index as u64) << 48 { + return None; + } + let bytes = value.to_le_bytes(); + let state = Self { + flags: bytes[0], + driver: bytes[1], + bars_present: bytes[2], + bars_mappable: bytes[3], + bars_mapped: bytes[4], + msi_vectors: bytes[5], + msix_entries: U16::new(u16::from_le_bytes([bytes[6], bytes[7]])), + }; + if state.is_present() && state.validate() { + Some(state) + } else { + None + } + } +} + +/// One vended page of states, row for row beside the census's rows. +/// 136 bytes, frozen. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceStatePage { + /// Page ordinal, `0..DEVICE_STATE_PAGES`. + pub page: u8, + /// Populated states in this page, `0..=DEVICE_STATE_PAGE_ROWS`. + pub rows_valid: u8, + /// States the whole table holds: the census's `count`. + pub count: U16, + pub reserved: [u8; 4], + pub states: [DeviceState; DEVICE_STATE_PAGE_ROWS], +} + +impl DeviceStatePage { + pub const EMPTY: Self = Self { + page: 0, + rows_valid: 0, + count: U16::ZERO, + reserved: [0; 4], + states: [DeviceState::EMPTY; DEVICE_STATE_PAGE_ROWS], + }; + + /// Page `page` of a table of `count` states, empty: the assembler's + /// start. `None` past the table's last page, though a table of no states + /// still has page 0. + pub const fn new(page: u8, count: u16) -> Option { + let count = if count as usize > MAX_DEVICE_CENSUS_ROWS { + MAX_DEVICE_CENSUS_ROWS + } else { + count as usize + }; + let pages = count.div_ceil(DEVICE_STATE_PAGE_ROWS); + let last = if pages == 0 { 0 } else { pages - 1 }; + if page as usize > last { + return None; + } + Some(Self { + page, + rows_valid: 0, + count: U16::new(count as u16), + reserved: [0; 4], + states: [DeviceState::EMPTY; DEVICE_STATE_PAGE_ROWS], + }) + } + + /// The catalog index of this page's first state. + pub const fn first(&self) -> usize { + self.page as usize * DEVICE_STATE_PAGE_ROWS + } + + /// States this page holds once complete. + pub const fn expected(&self) -> usize { + let left = (self.count.get() as usize).saturating_sub(self.first()); + if left > DEVICE_STATE_PAGE_ROWS { + DEVICE_STATE_PAGE_ROWS + } else { + left + } + } + + /// Append the next state, from one kernel observation of the function at + /// the catalog index this page reaches next; `false` when the page is + /// full or the report does not decode. + pub fn push(&mut self, value: u64, aux: u64) -> bool { + let at = self.rows_valid as usize; + if at >= self.expected() { + return false; + } + let Some(state) = DeviceState::from_observation((self.first() + at) as u16, value, aux) + else { + return false; + }; + self.states[at] = state; + self.rows_valid += 1; + true + } + + pub fn validate(&self) -> bool { + let count = self.count.get() as usize; + if self.page as usize >= DEVICE_STATE_PAGES + || count > MAX_DEVICE_CENSUS_ROWS + || self.reserved != [0; 4] + || self.rows_valid as usize != self.expected() + { + return false; + } + let valid = self.rows_valid as usize; + self.states + .iter() + .enumerate() + .all(|(i, state)| state.validate() && state.is_present() == (i < valid)) + } +} + +/// One BAR's shape. 16 bytes, frozen. Lengths only: where the BAR lives is +/// not here and never will be. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceBar { + /// The BAR's whole size in bytes; 0 for a BAR the function has not. + pub size: U64, + /// Bytes from the BAR's start a driver may map: the whole BAR, or the + /// part before the page its MSI-X table or pending bits begin in, which + /// stay the kernel's. 0 when a driver may map none of it. + pub mappable: U64, +} + +impl DeviceBar { + pub const EMPTY: Self = Self { + size: U64::ZERO, + mappable: U64::ZERO, + }; + + /// Bits of `aux` the mappable length takes; the BAR number is echoed + /// above them. + const AUX_LENGTH_BITS: u32 = 56; + + pub const fn is_present(&self) -> bool { + self.size.get() != 0 + } + + pub const fn validate(&self) -> bool { + self.mappable.get() <= self.size.get() + } + + /// The BAR as one `SystemObserve` report, `(value, aux)`: its size, and + /// its mappable length with the BAR number echoed in `aux[58:56]`. + /// `None` when a length does not leave the echo room, which no BAR of a + /// machine with a 56-bit physical address space or less can reach. + pub const fn to_observation(&self, bar: u8) -> Option<(u64, u64)> { + if self.mappable.get() >> Self::AUX_LENGTH_BITS != 0 || bar as usize >= DEVICE_BARS { + return None; + } + Some(( + self.size.get(), + self.mappable.get() | (bar as u64) << Self::AUX_LENGTH_BITS, + )) + } + + /// Inverse of [`to_observation`](Self::to_observation); `None` when the + /// echoed BAR is not `bar` or more can be mapped than the BAR holds. + pub const fn from_observation(bar: u8, value: u64, aux: u64) -> Option { + if aux >> Self::AUX_LENGTH_BITS != bar as u64 { + return None; + } + let shape = Self { + size: U64::new(value), + mappable: U64::new(aux & ((1 << Self::AUX_LENGTH_BITS) - 1)), + }; + if shape.validate() { Some(shape) } else { None } + } +} + +/// The BARs of one function. 104 bytes, frozen. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceBarTable { + /// The function's catalog index: its row in the census. + pub index: U16, + /// 1 for a populated table; 0 marks the zero table. + pub present: u8, + pub reserved: [u8; 5], + pub bars: [DeviceBar; DEVICE_BARS], +} + +impl DeviceBarTable { + pub const EMPTY: Self = Self { + index: U16::ZERO, + present: 0, + reserved: [0; 5], + bars: [DeviceBar::EMPTY; DEVICE_BARS], + }; + + /// The table of the function at catalog `index`, every BAR absent: the + /// assembler's start. + pub const fn new(index: u16) -> Self { + Self { + index: U16::new(index), + present: 1, + reserved: [0; 5], + bars: [DeviceBar::EMPTY; DEVICE_BARS], + } + } + + pub const fn is_present(&self) -> bool { + self.present == 1 + } + + /// Set BAR `bar` from one kernel observation; `false` when the report + /// does not decode. + pub fn set_bar(&mut self, bar: u8, value: u64, aux: u64) -> bool { + let (Some(slot), Some(shape)) = ( + self.bars.get_mut(bar as usize), + DeviceBar::from_observation(bar, value, aux), + ) else { + return false; + }; + *slot = shape; + true + } + + pub fn validate(&self) -> bool { + match self.present { + 0 => *self == Self::EMPTY, + 1 => self.reserved == [0; 5] && self.bars.iter().all(DeviceBar::validate), + _ => false, + } + } +} + +/// What a device tool is launched with: the census, each of its rows' live +/// state, and the BARs of the one function the command line named, if it +/// named one the census holds. 3008 bytes, inside the process-context +/// prefix (asserted below). +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceInventory { + /// [`DEVICE_INVENTORY_ABI`]; 0 = unavailable. + pub abi: U16, + pub reserved: [u8; 6], + pub census: DeviceCensus, + /// `states[i]` is the live state of `census.rows[i]`, for every row the + /// census holds; the zero state past them. + pub states: [DeviceState; MAX_DEVICE_CENSUS_ROWS], + /// The named function's BARs, or the zero table. + pub bars: DeviceBarTable, +} + +impl DeviceInventory { + pub const EMPTY: Self = Self { + abi: U16::ZERO, + reserved: [0; 6], + census: DeviceCensus::EMPTY, + states: [DeviceState::EMPTY; MAX_DEVICE_CENSUS_ROWS], + bars: DeviceBarTable::EMPTY, + }; + + /// An available inventory over `census`, with no state and no BARs yet. + pub const fn new(census: DeviceCensus) -> Self { + Self { + abi: U16::new(DEVICE_INVENTORY_ABI), + reserved: [0; 6], + census, + states: [DeviceState::EMPTY; MAX_DEVICE_CENSUS_ROWS], + bars: DeviceBarTable::EMPTY, + } + } + + pub const fn available(&self) -> bool { + self.abi.get() == DEVICE_INVENTORY_ABI + } + + /// Fold one vended page of states in; `false`, changing nothing, when + /// the page does not validate or is not a page of this census's table. + pub fn insert_states(&mut self, page: &DeviceStatePage) -> bool { + if !self.available() || !page.validate() || page.count != self.census.count { + return false; + } + let first = page.first(); + let valid = page.rows_valid as usize; + self.states[first..first + valid].copy_from_slice(&page.states[..valid]); + true + } + + /// Whether every census row has its state. + pub fn complete(&self) -> bool { + let count = self.census.rows().len(); + self.available() && self.states[..count].iter().all(DeviceState::is_present) + } + + pub fn validate(&self) -> bool { + if self.abi.get() == 0 { + return *self == Self::EMPTY; + } + let count = self.census.rows().len(); + self.abi.get() == DEVICE_INVENTORY_ABI + && self.reserved == [0; 6] + && self.census.available() + && self.census.validate() + && self + .states + .iter() + .enumerate() + .all(|(i, state)| state.validate() && (i < count || !state.is_present())) + && self.bars.validate() + && (!self.bars.is_present() || (self.bars.index.get() as usize) < count) + } +} + +/// Bytes of the device payload a monitor or exec reply carries. +pub const DEVICE_PAGE_BYTES: usize = 136; + +/// The device payload slot of the monitor's and exec's reply wires: one +/// census page, one page of states, or one function's BARs, as the reply's +/// tag says, and zero on every other reply. Bytes, so the three records +/// share the slot without one posing as another: each accessor decodes and +/// validates its own. +#[repr(transparent)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DevicePage(pub [u8; DEVICE_PAGE_BYTES]); + +impl DevicePage { + pub const EMPTY: Self = Self([0; DEVICE_PAGE_BYTES]); + + pub fn of_census(page: &DeviceCensusPage) -> Self { + let mut out = Self::EMPTY; + out.0.copy_from_slice(page.as_bytes()); + out + } + + /// The census page this slot carries, validated. + pub fn census(&self) -> Option { + DeviceCensusPage::read_from_bytes(&self.0) + .ok() + .filter(DeviceCensusPage::validate) + } + + pub fn of_states(page: &DeviceStatePage) -> Self { + let mut out = Self::EMPTY; + out.0.copy_from_slice(page.as_bytes()); + out + } + + /// The page of states this slot carries, validated. + pub fn states(&self) -> Option { + DeviceStatePage::read_from_bytes(&self.0) + .ok() + .filter(DeviceStatePage::validate) + } + + pub fn of_bars(table: &DeviceBarTable) -> Self { + let mut out = Self::EMPTY; + out.0[..core::mem::size_of::()].copy_from_slice(table.as_bytes()); + out + } + + /// The BAR table this slot carries: a populated, valid one, with the + /// slot's remaining bytes zero. + pub fn bars(&self) -> Option { + let (table, rest) = DeviceBarTable::read_from_prefix(&self.0).ok()?; + (table.is_present() && table.validate() && rest.iter().all(|byte| *byte == 0)) + .then_some(table) + } +} + +/// A PCI function's address as a person writes it: `ssss:bb:dd.f`, or +/// `bb:dd.f` in segment 0, every field hexadecimal but the function, which +/// is one digit `0` to `7`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PciAddress { + pub segment: u16, + /// `bus << 8 | device << 3 | function`, a census row's `locator`. + pub locator: u16, +} + +impl PciAddress { + /// Bytes of the longest spelling, `ssss:bb:dd.f`. + pub const MAX_TEXT: usize = 12; + + /// Parse `text`, the whole of it. `None` unless it is exactly an + /// address: a launcher looks for one among a command line's words, so a + /// word that merely resembles one must not name a device. + pub fn parse(text: &[u8]) -> Option { + let (rest, function) = text.split_last_chunk::<2>()?; + let [b'.', function @ b'0'..=b'7'] = *function else { + return None; + }; + let mut fields = rest.rsplitn(3, |byte| *byte == b':'); + let device = hex_field(fields.next()?, 2)?; + let bus = hex_field(fields.next()?, 2)?; + let segment = match fields.next() { + Some(field) => hex_field(field, 4)?, + None => 0, + }; + if device > 0x1F { + return None; + } + Some(Self { + segment, + locator: bus << 8 | device << 3 | (function - b'0') as u16, + }) + } + + /// The census row this address names. + pub fn row_in(&self, census: &DeviceCensus) -> Option { + census + .rows() + .iter() + .position(|row| row.segment.get() == self.segment && row.locator.get() == self.locator) + .map(|index| index as u16) + } +} + +/// `field` as exactly `digits` hexadecimal digits. +fn hex_field(field: &[u8], digits: usize) -> Option { + if field.len() != digits { + return None; + } + field.iter().try_fold(0u16, |value, byte| { + let digit = (*byte as char).to_digit(16)?; + Some(value << 4 | digit as u16) + }) +} + +const _: () = assert!(core::mem::size_of::() == 8); +const _: () = assert!(core::mem::size_of::() == 8 + 8 * DEVICE_STATE_PAGE_ROWS); +const _: () = assert!( + core::mem::size_of::() == core::mem::size_of::() +); +const _: () = assert!(core::mem::size_of::() == DEVICE_PAGE_BYTES); +const _: () = assert!(core::mem::size_of::() == DEVICE_PAGE_BYTES); +const _: () = assert!(core::mem::size_of::() <= DEVICE_PAGE_BYTES); +const _: () = assert!(core::mem::size_of::() == 16); +const _: () = assert!(core::mem::size_of::() == 8 + 16 * DEVICE_BARS); +const _: () = assert!(core::mem::size_of::() == 3008); +const _: () = + assert!(core::mem::size_of::() <= crate::process::PROCESS_CONTEXT_BYTES); +const _: () = assert!(DEVICE_STATE_PAGES <= u8::MAX as usize); +// Every index a census row can carry fits the BAR operand beside the BAR. +const _: () = assert!(MAX_DEVICE_CENSUS_ROWS <= 1 << DEVICE_BAR_OPERAND_SHIFT); +const _: () = assert!(DEVICE_BARS <= 1 << (u16::BITS - DEVICE_BAR_OPERAND_SHIFT)); + +#[cfg(test)] +mod tests { + use super::*; + use crate::device::DeviceCensusRow; + + fn state(flags: u8) -> DeviceState { + DeviceState { + flags: DeviceState::PRESENT | flags, + driver: DEVICE_ROW_NONE, + bars_present: 0b01_0001, + bars_mappable: 0b00_0001, + bars_mapped: 0, + msi_vectors: 1, + msix_entries: U16::new(0), + } + } + + fn census(count: u16) -> DeviceCensus { + let mut census = DeviceCensus::new(count, 1); + for index in 0..count { + census.rows[index as usize] = DeviceCensusRow { + locator: U16::new(index << 3), + present: 1, + ..DeviceCensusRow::EMPTY + }; + } + census + } + + #[test] + fn a_state_round_trips_one_observation() { + let mut state = state(DeviceState::CLAIMED | DeviceState::ROUTE_MSI | DeviceState::HAS_PIN); + state.flags |= 2 << DeviceState::PIN_SHIFT; + state.driver = 7; + state.msix_entries = U16::new(0); + let (value, aux) = state.to_observation(41); + assert_eq!(DeviceState::from_observation(41, value, aux), Some(state)); + assert_eq!(state.pin(), Some(2)); + assert_eq!(state.driver(), Some(7)); + assert!(state.claimed()); + // Another function's report, or one with stray auxiliary bits, is + // not this function's state. + assert_eq!(DeviceState::from_observation(40, value, aux), None); + assert_eq!(DeviceState::from_observation(41, value, aux | 1), None); + } + + #[test] + fn an_impossible_state_is_refused() { + let ok = state(0); + assert!(ok.validate()); + // The zero state is the empty row, never a kernel answer. + assert!(DeviceState::EMPTY.validate()); + assert_eq!(DeviceState::from_observation(0, 0, 0), None); + for broken in [ + DeviceState { + bars_present: 0x40, + ..ok + }, + DeviceState { + bars_mappable: 0b10, + ..ok + }, + DeviceState { + bars_mapped: 0b01_0000, + ..ok + }, + DeviceState { + msi_vectors: 3, + ..ok + }, + DeviceState { + msi_vectors: 64, + ..ok + }, + DeviceState { + flags: ok.flags | 1 << DeviceState::PIN_SHIFT, + ..ok + }, + DeviceState { + msix_entries: U16::new(crate::device::DeviceRecord::MAX_MSIX_TABLE_ENTRIES + 1), + ..ok + }, + ] { + assert!(!broken.validate(), "{broken:?}"); + } + } + + #[test] + fn a_page_takes_exactly_its_rows_in_order() { + // 20 states: page 0 holds 16, page 1 the last 4, and there is no + // page 2. + let mut first = DeviceStatePage::new(0, 20).unwrap(); + for index in 0..16u16 { + let (value, aux) = state(0).to_observation(index); + assert!(first.push(value, aux)); + } + let (value, aux) = state(0).to_observation(16); + assert!(!first.push(value, aux), "a full page takes no more"); + assert!(first.validate()); + + let mut second = DeviceStatePage::new(1, 20).unwrap(); + // The next state is index 16's; index 17's report is out of order. + let (value, aux) = state(0).to_observation(17); + assert!(!second.push(value, aux)); + assert!(!second.validate(), "a short page is not a page"); + for index in 16..20u16 { + let (value, aux) = state(0).to_observation(index); + assert!(second.push(value, aux)); + } + assert!(second.validate()); + assert!(DeviceStatePage::new(2, 20).is_none()); + // A machine with no function still has an empty page 0. + assert!(DeviceStatePage::new(0, 0).unwrap().validate()); + assert!(DeviceStatePage::new(1, 0).is_none()); + } + + #[test] + fn a_bar_round_trips_and_never_maps_more_than_it_holds() { + let bar = DeviceBar { + size: U64::new(0x10_0000), + mappable: U64::new(0x8000), + }; + let (value, aux) = bar.to_observation(4).unwrap(); + assert_eq!(DeviceBar::from_observation(4, value, aux), Some(bar)); + assert_eq!(DeviceBar::from_observation(3, value, aux), None); + // An absent BAR is the zero shape, and still answers for its number. + let (value, aux) = DeviceBar::EMPTY.to_observation(5).unwrap(); + assert_eq!( + DeviceBar::from_observation(5, value, aux), + Some(DeviceBar::EMPTY) + ); + assert_eq!(DeviceBar::from_observation(0, 0x1000, 0x2000), None); + assert!(bar.to_observation(6).is_none()); + let wide = DeviceBar { + size: U64::new(u64::MAX), + mappable: U64::new(1 << 56), + }; + assert!(wide.to_observation(0).is_none()); + } + + #[test] + fn a_bar_operand_names_a_function_and_a_bar() { + let operand = device_bar_operand(119, 5).unwrap(); + assert_eq!(device_bar_of(operand), Some((119, 5))); + assert_eq!(device_bar_operand(0, 0), Some(0)); + assert!(device_bar_operand(1 << DEVICE_BAR_OPERAND_SHIFT, 0).is_none()); + assert!(device_bar_operand(0, 6).is_none()); + assert!(device_bar_of(7 << DEVICE_BAR_OPERAND_SHIFT).is_none()); + } + + #[test] + fn an_inventory_holds_a_state_per_census_row() { + let mut inventory = DeviceInventory::new(census(20)); + assert!(inventory.validate()); + assert!(!inventory.complete()); + for page in 0..2u8 { + let mut states = DeviceStatePage::new(page, 20).unwrap(); + for index in states.first()..states.first() + states.expected() { + let (value, aux) = state(0).to_observation(index as u16); + assert!(states.push(value, aux)); + } + assert!(inventory.insert_states(&states)); + } + assert!(inventory.complete()); + assert!(inventory.validate()); + // A page of another census's table is refused whole. + let other = DeviceStatePage::new(0, 21).unwrap(); + assert!(!inventory.insert_states(&other)); + + let mut bars = DeviceBarTable::new(19); + let shape = DeviceBar { + size: U64::new(0x4000), + mappable: U64::new(0x4000), + }; + let (value, aux) = shape.to_observation(0).unwrap(); + assert!(bars.set_bar(0, value, aux)); + assert!(!bars.set_bar(1, value, aux), "the echo names BAR 0"); + inventory.bars = bars; + assert!(inventory.validate()); + // BARs of a function the census does not hold are not this + // inventory's. + inventory.bars.index = U16::new(20); + assert!(!inventory.validate()); + assert!(DeviceInventory::EMPTY.validate()); + assert!(!DeviceInventory::EMPTY.available()); + } + + #[test] + fn a_page_slot_decodes_only_the_record_it_carries() { + let mut states = DeviceStatePage::new(0, 2).unwrap(); + for index in 0..2u16 { + let (value, aux) = state(0).to_observation(index); + assert!(states.push(value, aux)); + } + let slot = DevicePage::of_states(&states); + assert_eq!(slot.states(), Some(states)); + + let mut table = DeviceBarTable::new(3); + let shape = DeviceBar { + size: U64::new(0x1000), + mappable: U64::new(0x1000), + }; + let (value, aux) = shape.to_observation(2).unwrap(); + assert!(table.set_bar(2, value, aux)); + let slot = DevicePage::of_bars(&table); + assert_eq!(slot.bars(), Some(table)); + // Bytes past the table are not part of it. + let mut stray = slot; + stray.0[DEVICE_PAGE_BYTES - 1] = 1; + assert_eq!(stray.bars(), None); + + let page = census(3).page(0).unwrap(); + assert_eq!(DevicePage::of_census(&page).census(), Some(page)); + // The zero slot is no record at all. + assert_eq!(DevicePage::EMPTY.bars(), None); + assert_eq!( + DevicePage::EMPTY.census().map(|page| page.rows_valid), + Some(0) + ); + } + + #[test] + fn an_address_is_exactly_an_address() { + let address = PciAddress::parse(b"0000:00:1f.3").unwrap(); + assert_eq!((address.segment, address.locator), (0, 0x1F << 3 | 3)); + assert_eq!(PciAddress::parse(b"00:1f.3"), Some(address)); + assert_eq!( + PciAddress::parse(b"0001:ab:02.0"), + Some(PciAddress { + segment: 1, + locator: 0xAB << 8 | 2 << 3 + }) + ); + for not in [ + &b""[..], + b"1f.3", + b"00:1f", + b"00:1f.8", + b"00:20.0", + b"0:1f.3", + b"000:00:1f.3", + b"00:1f.3x", + b"x0:1f.3", + b"0000:00:1f:3", + b"00000:00:1f.3", + b"8086:a828", + ] { + assert_eq!( + PciAddress::parse(not), + None, + "{:?}", + core::str::from_utf8(not) + ); + } + let census = census(4); + assert_eq!( + PciAddress { + segment: 0, + locator: 3 << 3 + } + .row_in(&census), + Some(3) + ); + assert_eq!( + PciAddress { + segment: 1, + locator: 3 << 3 + } + .row_in(&census), + None + ); + } +} diff --git a/native/crates/abi/src/image.rs b/native/crates/abi/src/image.rs index 38cbe11..12d98a2 100644 --- a/native/crates/abi/src/image.rs +++ b/native/crates/abi/src/image.rs @@ -180,6 +180,18 @@ pub const PROFILE_MODEL_DATA: u32 = 1 << 13; /// facilities every window preset is built on. pub const PROFILE_WINDOW: u32 = 1 << 14; +/// Launch profile: the child receives a **virtual-device face** — its own +/// request ring and register page with the virtual-device service, a send +/// onto that service's doorbell, and a return leg to lend it memory on +/// (`res/VIRTUAL_DEVICE.md`). +/// +/// The face is what lets a program be a driver of a device with no hardware +/// behind it. It is derived per launch and dies with the child, and it +/// carries no authority over the machine: the service it reaches holds +/// none. A launch that asks for it and cannot be given it still launches; +/// the program then finds no device to open. +pub const PROFILE_VIRTUAL_DEVICE: u32 = 1 << 15; + pub const PROFILE_KNOWN: u32 = PROFILE_TELEMETRY | PROFILE_FILESYSTEM | PROFILE_FS_VIEW_BINDER @@ -194,7 +206,8 @@ pub const PROFILE_KNOWN: u32 = PROFILE_TELEMETRY | PROFILE_ENGINE | PROFILE_GPU | PROFILE_MODEL_DATA - | PROFILE_WINDOW; + | PROFILE_WINDOW + | PROFILE_VIRTUAL_DEVICE; /// Validate the profile vocabulary and its dependencies: both filesystem /// auxiliary endpoint kinds are meaningful only beside an actual FS face, an diff --git a/native/crates/abi/src/pkg.rs b/native/crates/abi/src/pkg.rs index bac3b8f..ac109a8 100644 --- a/native/crates/abi/src/pkg.rs +++ b/native/crates/abi/src/pkg.rs @@ -27,13 +27,27 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; /// Layout version. A zeroed record reads `abi == 0` and is *not a /// manifest*; a reader refuses rather than guessing. -pub const PACKAGE_ABI: u16 = 1; -/// Members one manifest may declare. The bootstrap archive today carries -/// 33 executables and 3 libraries (`BOOTSTRAP_EXECUTABLES` / -/// `BOOTSTRAP_LIBRARIES` in exec), and the system package `kore` must list -/// every one of them; 48 leaves a third free before the record grows a -/// second page. `size_of::()` is pinned below. -pub const MAX_PACKAGE_MEMBERS: usize = 48; +/// +/// 2: [`MAX_PACKAGE_MEMBERS`] grew from 48 to 96, so the record grew with +/// it. Version 1 manifests were only ever written — exec rewrites the +/// system package's in place at every boot, and no installer reads one +/// yet — so nothing has to read both shapes; the bump is what lets a future +/// reader tell them apart rather than misread one as the other. +pub const PACKAGE_ABI: u16 = 2; +/// Members one manifest may declare: every bootstrap executable and library +/// (`BOOTSTRAP_EXECUTABLES` / `BOOTSTRAP_LIBRARIES` in exec), which the +/// system package `kore` must list in full. +/// +/// **Where the number comes from.** The record is read whole by one +/// `ReadVersion` transfer (`MAX_TRANSFER`, 4096 bytes): 48 bytes of header +/// and name, then 40 a member, so one page holds 100. 96 is the most of +/// that in whole 32-member steps. 48 was the first ceiling, reached exactly +/// when the catalogue held 45 executables and 3 libraries; the C compiler +/// was the 49th member. **What breaks if it moves.** Larger needs a second +/// transfer to read the record (a reader change, and another +/// [`PACKAGE_ABI`]); smaller fails exec's build, which asserts the +/// catalogue fits. `size_of::()` is pinned below. +pub const MAX_PACKAGE_MEMBERS: usize = 96; /// A member or package name: the store's own name bound /// ([`FS_NAME_BYTES`](crate::svc::FS_NAME_BYTES)), zero-padded, so a /// manifest name is always a legal DagFS name and never needs clamping. @@ -255,12 +269,13 @@ impl PackageManifest { } const _: () = assert!(core::mem::size_of::() == 8 + PACKAGE_NAME_BYTES); -/// 16-byte header + 32-byte name + 48 × 40-byte members = 1968 bytes: one +/// 16-byte header + 32-byte name + 96 × 40-byte members = 3888 bytes: one /// `ReadVersion` transfer (`MAX_TRANSFER` = 4096) reads a whole manifest. const _: () = assert!( core::mem::size_of::() == 16 + PACKAGE_NAME_BYTES + MAX_PACKAGE_MEMBERS * (8 + PACKAGE_NAME_BYTES) ); +const _: () = assert!(core::mem::size_of::() <= 4096); #[cfg(test)] mod tests { @@ -281,7 +296,7 @@ mod tests { fn a_manifest_round_trips_through_its_bytes_and_validates() { let manifest = system_package(); let bytes = manifest.as_bytes(); - assert_eq!(bytes.len(), 1968); + assert_eq!(bytes.len(), 3888); let read = PackageManifest::read_from_bytes(bytes).unwrap(); assert_eq!(read, manifest); assert_eq!(read.validate(), Ok(())); @@ -297,7 +312,7 @@ mod tests { #[test] fn a_zeroed_record_is_not_a_manifest() { - let zero = PackageManifest::read_from_bytes(&[0u8; 1968]).unwrap(); + let zero = PackageManifest::read_from_bytes(&[0u8; 3888]).unwrap(); assert_eq!(zero.validate(), Err(ManifestError::Abi(0))); } @@ -333,7 +348,7 @@ mod tests { } #[test] - fn push_refuses_duplicates_and_the_forty_ninth_member() { + fn push_refuses_duplicates_and_a_member_past_the_ceiling() { let mut manifest = system_package(); assert!( manifest diff --git a/native/crates/abi/src/process.rs b/native/crates/abi/src/process.rs index 48b0841..b1e72eb 100644 --- a/native/crates/abi/src/process.rs +++ b/native/crates/abi/src/process.rs @@ -565,8 +565,11 @@ pub const PROCESS_OUTPUT_BYTES: usize = PROCESS_IO_BYTES - HEADER_BYTES - PROCES /// would take that word for zero-or-corrupt. Each change shifts an offset or /// a meaning and therefore burns a version. 13: the system snapshot's /// scheduler-timer arm census, paid from the argument buffer together with -/// the output bytes its status row needs, so no report lost a row. -pub const PROCESS_IO_ABI: u16 = 13; +/// the output bytes its status row needs, so no report lost a row. 14: the +/// second reserved word gains a meaning, the length of the command name the +/// launcher resolved the program by, which rides the argument buffer after +/// the tail ([`ProcessIo::command`]): a `13` reader requires that word zero. +pub const PROCESS_IO_ABI: u16 = 14; /// One launch-time snapshot of exec's bounded process table. /// @@ -717,9 +720,12 @@ pub struct ProcessIo { /// holds the frame service's diagnostic face. net: crate::monitor::NetSnapshot, // `reserved[0]` was intentionally available for an additive ABI field. - // It now carries the immutable launcher-context live length; the remaining - // words stay reserved and must be zero. + // It now carries the immutable launcher-context live length, and + // `reserved[1]` the command name's length; the remaining words stay + // reserved and must be zero. reserved: [U32; 4], + /// The argument tail, `arg_len` bytes, then the command name, + /// `reserved[1]` bytes; zero after them. args: [u8; PROCESS_ARG_BYTES], /// Alternating payload: launcher context before START, child output after /// execution. The task lifecycle is the exclusive-ownership boundary. @@ -776,8 +782,10 @@ impl ProcessIo { && self.flags.get() == 0 && self.telemetry.validate() && self.filesystem.validate() - && self.reserved[1..].iter().all(|v| v.get() == 0) + && self.reserved[2..].iter().all(|v| v.get() == 0) && usize::from(self.arg_len.get()) <= PROCESS_ARG_BYTES + && usize::try_from(self.reserved[1].get()) + .is_ok_and(|name| name <= PROCESS_ARG_BYTES - usize::from(self.arg_len.get())) && usize::try_from(self.reserved[0].get()).is_ok_and(|len| len <= PROCESS_CONTEXT_BYTES) && usize::from(self.output_len.get()) <= PROCESS_OUTPUT_BYTES && (self.reserved[0].get() == 0 || self.output_len.get() == 0) @@ -791,6 +799,36 @@ impl ProcessIo { Ok(&self.args[..usize::from(self.arg_len.get())]) } + /// The name the launcher resolved this program by: the first word of the + /// command line it was run from, which the tail in [`Self::args`] does + /// not hold. Empty when the launcher gave none. A label: it selects + /// nothing and proves nothing about the image. + pub fn command(&self) -> Result<&[u8], ProcessIoError> { + if !self.validate() { + return Err(ProcessIoError::Invalid); + } + let tail = usize::from(self.arg_len.get()); + Ok(&self.args[tail..tail + self.reserved[1].get() as usize]) + } + + /// Publish the command name before the child starts. It shares the + /// argument buffer with the tail, so a name the buffer has no room for + /// after the tail is refused whole and the page is unchanged. + pub fn write_command(&mut self, name: &[u8]) -> Result<(), ProcessIoError> { + if !self.validate() { + return Err(ProcessIoError::Invalid); + } + let tail = usize::from(self.arg_len.get()); + let end = tail + .checked_add(name.len()) + .filter(|end| *end <= PROCESS_ARG_BYTES) + .ok_or(ProcessIoError::ArgumentsTooLong)?; + self.args[tail..].fill(0); + self.args[tail..end].copy_from_slice(name); + self.reserved[1].set(name.len() as u32); + Ok(()) + } + /// Immutable launcher context for this invocation. It is data, never an /// authority selector; a program must still hold every capability needed /// for the operation it describes. @@ -1248,6 +1286,55 @@ mod tests { assert_eq!(page.output(), Err(ProcessIoError::Invalid)); } + #[test] + fn the_command_name_rides_after_the_tail_and_leaves_it_alone() { + let mut page = ProcessIo::new_zeroed(); + page.initialize_with_context( + b"-v 'two words'", + b"context", + crate::monitor::TelemetrySnapshot::EMPTY, + ) + .unwrap(); + // A launcher that names nothing leaves the name empty. + assert_eq!(page.command().unwrap(), b""); + page.write_command(b"answer").unwrap(); + assert_eq!(page.command().unwrap(), b"answer"); + assert_eq!(page.args().unwrap(), b"-v 'two words'"); + assert_eq!(page.context().unwrap(), b"context"); + // A second name replaces the first whole, a shorter one included. + page.write_command(b"a").unwrap(); + assert_eq!(page.command().unwrap(), b"a"); + // The buffer is the tail's and the name's together: a name that + // does not fit after the tail is refused and nothing changes. + let long = [b'n'; PROCESS_ARG_BYTES]; + assert_eq!( + page.write_command(&long), + Err(ProcessIoError::ArgumentsTooLong) + ); + assert_eq!(page.command().unwrap(), b"a"); + assert_eq!(page.args().unwrap(), b"-v 'two words'"); + let fits = &long[..PROCESS_ARG_BYTES - page.args().unwrap().len()]; + page.write_command(fits).unwrap(); + assert_eq!(page.command().unwrap(), fits); + // The next launch starts with no name. + page.initialize(b"", crate::monitor::TelemetrySnapshot::EMPTY) + .unwrap(); + assert_eq!(page.command().unwrap(), b""); + } + + #[test] + fn a_command_name_past_the_buffer_fails_closed() { + let mut page = ProcessIo::new_zeroed(); + page.initialize(b"tail", crate::monitor::TelemetrySnapshot::EMPTY) + .unwrap(); + page.reserved[1].set((PROCESS_ARG_BYTES - 3) as u32); + assert_eq!(page.command(), Err(ProcessIoError::Invalid)); + assert_eq!(page.args(), Err(ProcessIoError::Invalid)); + page.reserved[1].set(0); + page.reserved[2].set(1); + assert_eq!(page.args(), Err(ProcessIoError::Invalid)); + } + #[test] fn malformed_context_length_fails_closed() { let mut page = ProcessIo::new_zeroed(); diff --git a/native/crates/abi/src/session.rs b/native/crates/abi/src/session.rs index ac3aa27..c010a5a 100644 --- a/native/crates/abi/src/session.rs +++ b/native/crates/abi/src/session.rs @@ -2588,6 +2588,28 @@ pub const REVOKE_KIND_EXEC_RETURN: u8 = 7; /// window survives, the latch re-anchors on it. pub const REVOKE_KIND_UNSHARE_IOVA_LIVE: u8 = 8; +/// The [`Invocation::kind`] selector for the consuming **writable lease → +/// frozen lease** transition through [`OpCode::Revoke`]: the exclusive +/// intake of a registrar that must validate bytes it did not write +/// (`res/IMAGE.md` §6b). +/// +/// The subject is a held or same-session `MemLease` carrying the complete +/// lifecycle grant `ACCESS|MODIFY|DERIVE|REVOKE`. Commit consumes that +/// capability and its complete descendant lineage — every view, carve and +/// route derived from it dies, wherever it travelled — removes every +/// CPU/DMA/guest alias, and writes a fresh `MemLease` over the same window +/// into `result`, carrying the same grant. From then on no lessor of the +/// backing may write the window, the successor and its carves included: the +/// family's claim table holds it immutable for as long as the successor +/// lives. A carve whose parent or sibling still holds a writable view of the +/// window is refused before anything commits, and so is an executable or +/// swap-sealed lease. +/// +/// The successor keeps `MODIFY` because a registrar carves the window's text +/// and finalizes it, which a frozen window admits ([`REVOKE_KIND_EXEC`] +/// leaves it immutable too); `MODIFY` never again reaches a write. +pub const REVOKE_KIND_FREEZE: u8 = 9; + /// Argument payload for a **page delegation** derive (`Derive`, /// [`kind`](Invocation::kind)` = Region` on a held/staged `Region` /// subject; res/IMAGE.md §9a): re-home the subject region @@ -3123,6 +3145,21 @@ pub enum SystemMetric { IommuUnit = 35, /// Indexed capabilities and scope counts, Unit::features() observation. IommuUnitFeatures = 36, + /// One PCI function's live state; operand = its catalog index, as for + /// [`DeviceRow`](Self::DeviceRow). The pair is + /// [`DeviceState::to_observation`](crate::device::DeviceState::to_observation): + /// whether a driver claimed it, the founded row driving its DMA, which + /// BARs exist, may be mapped and are mapped, and how it can interrupt. + /// A snapshot, never permission; no address. + DeviceState = 37, + /// One BAR's shape; operand = + /// [`device_bar_operand`](crate::device::device_bar_operand) of the + /// function's catalog index and the BAR number. The pair is + /// [`DeviceBar::to_observation`](crate::device::DeviceBar::to_observation): + /// the BAR's size and the length a driver may map. Lengths only: where + /// the BAR lives never crosses. A BAR the function has not answers the + /// zero shape. + DeviceBar = 38, } /// Records the kernel retains for [`SystemMetric::IommuFaultRecord`]. @@ -3494,6 +3531,8 @@ impl SystemObserveReq { matches!( metric, SystemMetric::DeviceRow + | SystemMetric::DeviceState + | SystemMetric::DeviceBar | SystemMetric::ProcessorCapture | SystemMetric::ProcessorIdentity | SystemMetric::ProcessorTopology diff --git a/native/crates/abi/src/svc.rs b/native/crates/abi/src/svc.rs index 703543e..0c06b51 100644 --- a/native/crates/abi/src/svc.rs +++ b/native/crates/abi/src/svc.rs @@ -205,6 +205,13 @@ impl MonitorReqWire { /// records. The monitor assembles the census from its own /// `SystemObserve` capability; the request carries no authority. pub const OP_DEVICE_CENSUS: u8 = 2; + /// One page of the census functions' live state + /// (`crate::device::state`): `reserved[0]` is the page ordinal. Sampled + /// when asked, never cached; the request carries no authority. + pub const OP_DEVICE_STATES: u8 = 5; + /// One function's BAR shapes: `reserved[0..2]` is its catalog index, + /// little-endian. Lengths only. + pub const OP_DEVICE_BARS: u8 = 6; } /// Monitor reply: a complete bounded snapshot or a closed refusal. The @@ -222,9 +229,11 @@ pub struct MonitorRepWire { /// Tagged observation payload: ProcessorPage for processor replies, or /// iommu_status::Snapshot encoded by its wire-page methods for IOMMU replies. pub processor: crate::processor_status::ProcessorPage, - /// [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS) only; the zero page - /// on every other tag (the codec refuses a stray one). - pub census: crate::device::DeviceCensusPage, + /// The device record of [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS), + /// [`TAG_DEVICE_STATES`](Self::TAG_DEVICE_STATES) or + /// [`TAG_DEVICE_BARS`](Self::TAG_DEVICE_BARS); the zero slot on every + /// other tag (the codec refuses a stray one). + pub device: crate::device::DevicePage, } impl MonitorRepWire { @@ -233,6 +242,8 @@ impl MonitorRepWire { pub const TAG_SNAPSHOT: u8 = 1; pub const TAG_REFUSED: u8 = 2; pub const TAG_DEVICE_CENSUS: u8 = 3; + pub const TAG_DEVICE_STATES: u8 = 6; + pub const TAG_DEVICE_BARS: u8 = 7; pub const REFUSE_DENIED: u8 = 1; } @@ -1502,6 +1513,40 @@ impl TaskReqWire { /// `slot` names the face; no plan, no target, no grants. Seal-exempt, /// via==0 only: provisioning is the administrator's, not a task's. pub const OP_PROVISION_HOME: u8 = 25; + /// Install one file from the caller's own tree as a **private** + /// executable generation (`res/EXECUTABLE_CAPABILITY_MIGRATION.md` §7.4): + /// exec reads the file through the caller's conveyed view into the + /// caller's own memory supply, freezes that supply, registers it, and + /// grants an executable capability for it. + /// + /// The request is a client face's atomic context bundle without an + /// executable member: the supply on the ProcessIo lane, the view on the + /// filesystem-view lane, and on the first auxiliary lane the receive end + /// of a channel whose root the caller keeps — the generation's lifetime: + /// revoking that root, or the caller's death, retires it. The proof lane + /// and the other auxiliary lanes are vacant. `slot` carries the supply's + /// capability tag, so exec selects exactly this request's delivery and + /// never a stale one (tags are never reissued). The path rides the + /// supply itself ([`InstallPathHeader`]) and is overwritten by the + /// file's bytes, so the record keeps its size. Every other field is + /// zero. A file longer than the supply is answered + /// [`TAG_INSTALL_SUPPLY_SHORT`](TaskRepWire::TAG_INSTALL_SUPPLY_SHORT) + /// with the size it needs. + /// + /// Reading suffices, by the filesystem's own rule: `BIND_EXECUTABLE` + /// governs the shared executable namespace, not private code a task + /// makes executable from its own lease. The generation binds no global + /// name: only the capability's holder reaches it. + pub const OP_INSTALL_FILE: u8 = 26; + /// One page of the census functions' live state + /// (`crate::device::state`), relayed from the monitor as + /// [`OP_DEVICE_CENSUS`](Self::OP_DEVICE_CENSUS)'s pages are, but never + /// cached: `slot` is the page ordinal. Bind-inert data, no grants, + /// answered on any launcher pair. + pub const OP_DEVICE_STATES: u8 = 27; + /// One function's BAR shapes, relayed from the monitor: `slot` is the + /// function's catalog index. Lengths only; no grants. + pub const OP_DEVICE_BARS: u8 = 28; /// Every retired `TaskReqWire` discriminant, in one definition the /// decoder cites (the [`derive_kind_known`](crate::session::derive_kind_known) @@ -1568,6 +1613,66 @@ impl TaskReqWire { pub const INSTALL_FROM_TREE: u8 = 1 << 1; } +/// The path an [`OP_INSTALL_FILE`](TaskReqWire::OP_INSTALL_FILE) request +/// installs, which the caller writes at offset 0 of the supply it conveys: +/// this header, then [`len`](Self::len) UTF-8 bytes. The path is relative +/// to the conveyed view and `/`-separated. Exec copies it out before the +/// first file byte lands and the file then overwrites it, so neither the +/// record nor the frozen image carries it. +/// +/// A path rides the supply rather than the record because a legal path is +/// longer than every free field: the record's `target` holds 64 bytes, the +/// deepest legal path [`FS_SEARCH_PATH_BYTES`], and the pair already uses +/// all but 128 bytes of its ring (`kore_svc_exec::wire`). +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct InstallPathHeader { + /// Live path bytes after the header (`1..=MAX_PATH`). + pub len: U16, + /// Reserved; must be zero. + pub reserved: [u8; 6], +} + +impl InstallPathHeader { + /// Wire size of the header (repr(C), 2 + 6). + pub const BYTES: usize = 8; + /// The longest path a request carries: the deepest legal filesystem + /// path, the bound a search hit spells. + pub const MAX_PATH: usize = FS_SEARCH_PATH_BYTES; + /// Bytes a request's path occupies at the head of its supply. + pub const MAX_BYTES: usize = Self::BYTES + Self::MAX_PATH; + + /// The header for `path`, or `None` for an empty path or one past + /// [`MAX_PATH`](Self::MAX_PATH). + pub fn for_path(path: &[u8]) -> Option { + if path.is_empty() || path.len() > Self::MAX_PATH { + return None; + } + Some(Self { + len: U16::new(path.len() as u16), + reserved: [0; 6], + }) + } + + /// The path a supply's head carries: a well-formed header followed by + /// that many UTF-8 bytes, or `None`. Component rules (no empty, `.` or + /// `..` component; the filesystem's name bound) are the reader's to + /// apply, against the filesystem it walks. + pub fn path(head: &[u8]) -> Option<&str> { + let (header, rest) = Self::read_from_prefix(head).ok()?; + let len = usize::from(header.len.get()); + if header.reserved != [0; 6] || len == 0 || len > Self::MAX_PATH { + return None; + } + core::str::from_utf8(rest.get(..len)?).ok() + } +} + +const _: () = assert!(core::mem::size_of::() == InstallPathHeader::BYTES); +const _: () = assert!(InstallPathHeader::MAX_PATH <= u16::MAX as usize); + /// The exec reply wire form (`TaskResponse`'s crossing shape). An owned /// `Spawned` reply carries exactly one task-handle grant. `id` remains /// correlation data only (hash-derived, unordered). A `Spawned` reply uses @@ -1615,9 +1720,11 @@ pub struct TaskRepWire { /// Tagged observation payload: ProcessorPage for processor replies, or /// iommu_status::Snapshot encoded by its wire-page methods for IOMMU replies. pub processor: crate::processor_status::ProcessorPage, - /// [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS)'s page; the zero - /// page on every other tag. - pub census: crate::device::DeviceCensusPage, + /// The device record of [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS), + /// [`TAG_DEVICE_STATES`](Self::TAG_DEVICE_STATES) or + /// [`TAG_DEVICE_BARS`](Self::TAG_DEVICE_BARS); the zero slot on every + /// other tag. + pub device: crate::device::DevicePage, /// Only `REFUSE_LEDGER_FULL` with `index == 1` may carry this bounded /// failure snapshot. Every other reply requires the all-zero form. pub cleanup: CleanupRowDiagnosticWire, @@ -1750,6 +1857,24 @@ impl TaskRepWire { /// unprovisioned home is a posture (no store, or a stage that failed and /// is retried next boot), never a refusal. pub const TAG_HOME_PROVISIONED: u8 = 25; + /// Reply to [`TaskReqWire::OP_INSTALL_FILE`]: the private generation was + /// registered, and the reply grants its executable endpoint. `index` + /// carries the segment count and `status` the launch profile word, as + /// [`TAG_TREE_ACQUIRED`](Self::TAG_TREE_ACQUIRED) carries them. + pub const TAG_FILE_INSTALLED: u8 = 26; + /// Reply to [`TaskReqWire::OP_INSTALL_FILE`]: the file is longer than + /// the conveyed supply. `id` is the page-rounded size the file needs, + /// which a caller may retry with; nothing was registered, and the supply + /// was retired, so its memory is the caller's again. + pub const TAG_INSTALL_SUPPLY_SHORT: u8 = 27; + /// Reply to [`TaskReqWire::OP_DEVICE_STATES`]: `device` is the page. + pub const TAG_DEVICE_STATES: u8 = 28; + /// Reply to [`TaskReqWire::OP_DEVICE_BARS`]: `device` is the table. + pub const TAG_DEVICE_BARS: u8 = 29; + /// The unit [`TAG_INSTALL_SUPPLY_SHORT`](Self::TAG_INSTALL_SUPPLY_SHORT) + /// counts in: mm grants a base-class supply in whole 4 KiB pages, so a + /// size the caller can ask for is a multiple of it. + pub const INSTALL_SUPPLY_GRANULE: u64 = 4096; /// `TAG_HOME_PROVISIONED.index`: the home is recorded at the current /// skeleton version. pub const HOME_PROVISIONED: u8 = 1 << 0; @@ -1854,6 +1979,12 @@ impl TaskRepWire { pub const REFUSE_PEER_ADMISSION_WEDGED: u8 = 22; pub const REFUSE_MODEL_UNAVAILABLE: u8 = 23; pub const REFUSE_MODEL_MEMORY_UNAVAILABLE: u8 = 24; + /// `TaskRefusal::FileUnavailable` — an install's path named no file the + /// presented view could read whole: no such entry, a directory, or a + /// read the filesystem refused. Distinct from + /// [`REFUSE_IMAGE_UNAVAILABLE`](Self::REFUSE_IMAGE_UNAVAILABLE), which + /// says the bytes were read and are not a dynamic executable. + pub const REFUSE_FILE_UNAVAILABLE: u8 = 25; } // Live task discriminants stay disjoint from the frozen retired band on @@ -3834,6 +3965,16 @@ impl InReqWire { /// `InputStreamRequest::Reprint` — echo the line being edited, cursor /// in place, then apply every key queued since `Hold`. pub const OP_REPRINT: u8 = 6; + /// `InputStreamRequest::Insert` — type text into the **held** line at + /// its cursor, as if typed, without echo (the `Reprint` shows it): + /// `reserved0` = the byte count (1 to [`INSERT_BYTES`](Self::INSERT_BYTES)), + /// `reserved2..reserved4` = the bytes, little-endian, the words past the + /// count zero. Printable ASCII only: the line takes bytes up to the first + /// it cannot hold — any other byte, or one past a full line — so no + /// insert can ever complete a line. + pub const OP_INSERT: u8 = 7; + /// The most one `Insert` carries: its three data words. + pub const INSERT_BYTES: usize = 24; } /// The stream-input-service reply wire form. The legacy stream counters are @@ -3926,6 +4067,9 @@ impl InRepWire { /// `InputStreamResponse::Reprinted` — the line is echoed again and the /// held keys applied. pub const TAG_REPRINTED: u8 = 7; + /// `InputStreamResponse::Inserted` — `produced` = how many of the + /// insert's bytes the line took (zero to its count). + pub const TAG_INSERTED: u8 = 8; /// `InputStreamRefusal::Busy` — a queue is already attached (the v1 /// single-peer bound, `res/USERLAND.md` §5.9). @@ -3935,6 +4079,9 @@ impl InRepWire { /// `InputStreamRefusal::BadQueue` — the delivered slot was absent or /// not a well-formed queue window. pub const REFUSE_BAD_QUEUE: u8 = 3; + /// `InputStreamRefusal::NotHeld` — an insert into a line that is not + /// held, or not being edited (a completed line still in flight). + pub const REFUSE_NOT_HELD: u8 = 4; } /// One data-plane chunk — the `ring::spsc` record the backend @@ -3959,10 +4106,128 @@ impl InChunk { pub const CHUNK_DATA: usize = 24; } +// ─── vdev — the virtual-device service (kore-svc-vdev) ────────────── +// +// Kept last, after every other service's records: a record added in the +// middle of this file moves the lines after it, and with them the panic +// locations compiled into every image that uses those lines. + +/// The virtual-device request wire form (`VdevRequest`'s crossing shape). +/// +/// The 8-byte shape of its siblings. Discriminants are **1-based**, so a +/// zeroed record is invalid and never an administration request. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct VdevReqWire { + pub op: u8, + pub reserved: [u8; 7], +} + +impl VdevReqWire { + /// Administration pair: one component of a driver's face; the part, then + /// the announced return landing as a little-endian `u32`. + pub const OP_ADMIT_PEER: u8 = 1; + pub const OP_TAKE_ADMISSION: u8 = 2; + /// Administration pair: the peer's arrival as a little-endian `u16`. + pub const OP_RETIRE_PEER: u8 = 3; + pub const OP_CONFIRM_RETIRED: u8 = 4; + /// A driver ends its own face. + pub const OP_CLOSE_PEER: u8 = 5; + /// A driver asks for a device model by its register-block id, a + /// little-endian `u32`. + pub const OP_OPEN: u8 = 6; + /// A driver lends the memory device addresses are offsets into: the + /// announced return landing as a little-endian `u32`. + pub const OP_LEND: u8 = 7; + /// A driver gives its device up, and takes back the memory it lent. + pub const OP_CLOSE: u8 = 8; +} + +/// The virtual-device reply wire form. The service mints no authority, so +/// `grants` is pinned zero and a non-zero value fails the codec closed. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct VdevRepWire { + pub tag: u8, + pub refusal: u8, + /// Always zero. + pub grants: u8, + pub reserved: [u8; 5], +} + +impl VdevRepWire { + pub const TAG_ACCEPTED: u8 = 1; + pub const TAG_REFUSED: u8 = 2; + /// The admitted peer's arrival as a little-endian `u16`. + pub const TAG_PEER_ADMITTED: u8 = 3; + /// The memory the device now addresses, in 4 KiB pages, as a + /// little-endian `u32`. + pub const TAG_LENT: u8 = 4; + + /// The request is not this face's to make. + pub const REFUSE_DENIED: u8 = 1; + /// The admission result is not published yet. + pub const REFUSE_PENDING: u8 = 2; + /// An earlier admission must be settled first. + pub const REFUSE_CONFLICT: u8 = 3; + /// The service cannot do this now: no mapping geometry, or a model that + /// would not start. + pub const REFUSE_UNAVAILABLE: u8 = 4; + /// No device model has that id. + pub const REFUSE_NO_DEVICE: u8 = 5; + /// The device is not in a state for this: already open, not open, or + /// holding a queue in the memory it has. + pub const REFUSE_BUSY: u8 = 6; + /// Every device the service hosts at once is taken. + pub const REFUSE_LIMIT: u8 = 7; + /// What was lent is not memory the device can address. + pub const REFUSE_MEMORY: u8 = 8; +} + #[cfg(test)] mod tests { use super::*; + /// An install path round-trips through a supply's head, and every + /// malformed head refuses: empty, past the bound, a reserved byte set, + /// a length past the bytes present, and bytes that are not UTF-8. + #[test] + fn install_paths_ride_a_supply_head() { + let mut head = [0u8; InstallPathHeader::MAX_BYTES + 1]; + let path = b"build/hello"; + let header = InstallPathHeader::for_path(path).unwrap(); + head[..InstallPathHeader::BYTES].copy_from_slice(header.as_bytes()); + head[InstallPathHeader::BYTES..][..path.len()].copy_from_slice(path); + assert_eq!(InstallPathHeader::path(&head), Some("build/hello")); + + assert_eq!(InstallPathHeader::for_path(b""), None); + let longest = [b'a'; InstallPathHeader::MAX_PATH]; + assert!(InstallPathHeader::for_path(&longest).is_some()); + let too_long = [b'a'; InstallPathHeader::MAX_PATH + 1]; + assert_eq!(InstallPathHeader::for_path(&too_long), None); + + let mut reserved = head; + reserved[7] = 1; + assert_eq!(InstallPathHeader::path(&reserved), None); + let mut empty = head; + empty[..2].copy_from_slice(&0u16.to_le_bytes()); + assert_eq!(InstallPathHeader::path(&empty), None); + let mut past = head; + past[..2].copy_from_slice(&((InstallPathHeader::MAX_PATH + 1) as u16).to_le_bytes()); + assert_eq!(InstallPathHeader::path(&past), None); + assert_eq!( + InstallPathHeader::path(&head[..InstallPathHeader::BYTES + path.len() - 1]), + None + ); + let mut invalid = head; + invalid[InstallPathHeader::BYTES] = 0xff; + assert_eq!(InstallPathHeader::path(&invalid), None); + } + /// Every wire form reinterprets in place from arbitrary bytes and /// round-trips through its own bytes — the shared-ring contract /// (any bit pattern is *readable*; the codec's job is refusal). @@ -4335,6 +4600,32 @@ impl SurReqWire { /// compositor snapshots the name and ends the pick. No subject /// ([`SurRepWire::TAG_PICK_ANSWERED`]). pub const OP_PICKED: u8 = 13; + /// `SurfaceRequest::CopyChunk` — one piece of the text this window puts + /// on its seat's clipboard (`res/CLIPBOARD.md`), written at the head of + /// its pair lease's request plane (`kore_window::menu`'s page): `a` = + /// the piece's offset in the text `| len << 32` (1 to the plane's size), + /// `b` = the whole text's byte length (1 to + /// `kore_window::clipboard::CLIPBOARD_BYTES`). The piece at offset zero + /// begins a copy, and only the seat's focused window may begin one, on + /// a gesture of the user's it has not spent; later pieces continue it + /// in order, and the piece that reaches the length ends it. The + /// compositor copies each piece out before replying, so the plane may be + /// reused once answered. No subject ([`SurRepWire::TAG_COPIED`]). + pub const OP_COPY_CHUNK: u8 = 14; + /// `SurfaceRequest::Paste` — read the seat's clipboard into this + /// window's request plane: `a` = the offset to read from. Offset zero + /// begins a read, and only the seat's focused window may begin one, on + /// a gesture of the user's it has not spent; later offsets continue it + /// until it reaches the end or the clipboard changes. No subject + /// ([`SurRepWire::TAG_CLIPBOARD`], [`SurRepWire::TAG_CLIPBOARD_EMPTY`]). + pub const OP_PASTE: u8 = 15; + /// `SurfaceRequest::PasteToShell` — the seat's terminal window asks for + /// the clipboard to be typed into its shell's line: queued as an inert + /// `ShellAction::Paste` for its controller, which reads the text and + /// edits the line itself. Only the seat's focused terminal window may + /// ask, on a gesture it has not spent. No words, no subject + /// ([`SurRepWire::TAG_PASTE_QUEUED`], [`SurRepWire::TAG_CLIPBOARD_EMPTY`]). + pub const OP_PASTE_TO_SHELL: u8 = 16; /// [`OP_SET_CURSOR`](Self::OP_SET_CURSOR) `a`: the standard arrow. pub const CURSOR_DEFAULT: u64 = 0; @@ -4456,6 +4747,20 @@ impl SurRepWire { /// `SurfaceResponse::PickAnswered` — `a` echoes the token answered; the /// pick has ended and the plane may be reused. pub const TAG_PICK_ANSWERED: u8 = 14; + /// `SurfaceResponse::Copied` — `a` = the text copied so far (the + /// piece's offset plus its length, echoed); when it is the whole + /// length, the text is the seat's clipboard. + pub const TAG_COPIED: u8 = 15; + /// `SurfaceResponse::Clipboard` — `a` = the offset read from (echoed) + /// `| len << 32`, the bytes now at the head of the request plane (1 to + /// the plane's size), `b` = the clipboard text's whole byte length. + pub const TAG_CLIPBOARD: u8 = 16; + /// `SurfaceResponse::ClipboardEmpty` — the seat's clipboard holds no + /// text. No words. + pub const TAG_CLIPBOARD_EMPTY: u8 = 17; + /// `SurfaceResponse::PasteQueued` — `a` = the byte length of the text + /// the controller was asked to type. + pub const TAG_PASTE_QUEUED: u8 = 18; /// A v1 vocabulary gate: an unknown op, a non-XRGB8888 format, a /// non-CPU reach, a non-LINEAR modifier, a non-empty flag set. @@ -4472,6 +4777,10 @@ impl SurRepWire { /// A capability transfer or teardown did not settle. A retiring surface /// keeps its backing reserved until revocation succeeds; retry is safe. pub const REFUSE_BUSY: u8 = 6; + /// The user did not ask: a clipboard verb from a window that is not its + /// seat's focused window, or with no unspent gesture of the user's + /// behind it. + pub const REFUSE_DENIED: u8 = 7; } /// One damage rectangle of a [`CommitWire`] record — pixel-addressed, @@ -4815,7 +5124,17 @@ impl FsReqWire { /// route no longer validates. The view must belong to the caller's task. /// `window_offset` is return-post correlation. pub const OP_MATCH_VIEW: u8 = 65; - pub const OP_MAX: u8 = Self::OP_MATCH_VIEW; + /// Arm watch slot `kind` on the directory `handle`, tagged `length` + /// (non-zero, the client's own name for this arming): from now on the + /// pair's watch plane — the event channel after the rings in this pair's + /// own window — counts every change to what a listing of that directory + /// shows. Re-arming a slot replaces its watch. Observation, not + /// authority: the handle's route is revalidated before every count, and + /// a route that stops proving ends the watch. Answered `Done`. + pub const OP_WATCH: u8 = 66; + /// Disarm watch slot `kind`. Answered `Done`, armed or not. + pub const OP_UNWATCH: u8 = 67; + pub const OP_MAX: u8 = Self::OP_UNWATCH; /// [`OP_STOR_MAINTAIN`](Self::OP_STOR_MAINTAIN): report only. pub const MAINTAIN_STATUS: u8 = 0; @@ -4978,6 +5297,59 @@ impl FsDirEntryWire { pub const BYTES: usize = 48; } +/// Watch slots one filesystem pair may arm ([`FsReqWire::OP_WATCH`]). +/// +/// Sized by the client that follows the most folders at once: the Files +/// explorer watches the folder it shows and the sidebar's live folders (its +/// root and the expanded folders whose lines are on screen), fifteen of them +/// at most. Every armed watch holds a handle, so this stays under half the +/// service's per-session handle bound (32, `kore_dagfs::bounds`) — a client +/// watching every slot keeps the handles a listing needs. The plane's +/// `ended` word is one bit per slot, so 32 is the layout's own ceiling. +pub const FS_WATCH_SLOTS: usize = 16; +const _: () = assert!(FS_WATCH_SLOTS <= u32::BITS as usize); + +/// The level state of a filesystem pair's **watch plane**: the event channel +/// (`res/EVENT.md`) the service formats after the request/reply rings in the +/// pair's own window and republishes whole whenever a watched directory +/// changes. Slot `k` belongs to the watch [`FsReqWire::OP_WATCH`] armed with +/// slot `k`. 136 bytes. +/// +/// Counts, not contents: a change is "what a listing of this directory, or +/// of one of its child folders, shows may differ" — an entry made, removed +/// or renamed in it, a child's length, revision or folder count moved, or an +/// entry made or removed in a child folder (its line's count and whether it +/// holds folders) — and the client lists again through its own handle, whose +/// route the service revalidates. Nothing here names an entry, so the plane +/// carries no observation the handle could not make itself. +#[repr(C)] +#[derive( + Clone, Copy, Debug, Default, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, +)] +pub struct FsWatchState { + /// Per slot: the tag the slot was armed with; zero while disarmed. A + /// client reads a slot's counts only under the tag it armed, so a count + /// left by the slot's previous watch is never mistaken for its own. + pub tag: [u32; FS_WATCH_SLOTS], + /// Per slot: changes seen since the slot was armed, wrapping. Coalesced: + /// several changes between two service passes advance it once. + pub changes: [u32; FS_WATCH_SLOTS], + /// Bit `k`: slot `k`'s watch ended — its handle closed, its route stopped + /// proving, or its directory was reclaimed. Cleared by re-arming. + pub ended: u32, + /// Reserved; zero. + pub reserved: u32, +} + +/// The watch plane's record ring: one slot of a reserved edge kind, never +/// pushed. A later per-entry change record lands in it without moving the +/// plane's state or its offset. +pub type FsWatchRecord = u64; +/// Records the watch plane's ring holds. +pub const FS_WATCH_RECORDS: u32 = 1; + +const _: () = assert!(core::mem::size_of::() == 136); + /// Maximum bytes in one view-relative search path. /// /// The filesystem admits thirty-two 32-byte components diff --git a/native/crates/abi/src/task.rs b/native/crates/abi/src/task.rs index baed7a9..e4f934d 100644 --- a/native/crates/abi/src/task.rs +++ b/native/crates/abi/src/task.rs @@ -280,9 +280,12 @@ pub mod role { /// the face table's slot arithmetic must never interleave with a /// heap tier's. /// - /// Geometry contract: [`FACE_SUPPLY_BYTES`](super::FACE_SUPPLY_BYTES) - /// = [`FACE_RING_STRIDE`](super::FACE_RING_STRIDE) rings — the - /// founder mints exactly that, the face table carves exactly that. + /// Geometry contract, one per carrying row: the image service's + /// [`FACE_SUPPLY_BYTES`](super::FACE_SUPPLY_BYTES) and a broker's + /// [`CLAIMANT_FACE_SUPPLY_BYTES`](super::CLAIMANT_FACE_SUPPLY_BYTES), + /// each a whole number of + /// [`FACE_RING_STRIDE`](super::FACE_RING_STRIDE) rings — the founder + /// mints exactly that, the face table carves exactly that. pub const FACE_SUPPLY: u16 = 29; /// Read-only machine telemetry authority, endowed only to the monitor /// service. The capability carries `ACCESS | DERIVE`; observation is a @@ -462,35 +465,62 @@ pub const INTERACTIVE_CHILD_LOADER_FACES: usize = DESKTOP_BOOTSTRAP_WINDOW_SLOTS + 2 * DESKTOP_CONTROLLER_CAPACITY; /// Live immutable image-generation peers the boot catalogue can hold at once: -/// the 45-row bootstrap catalogue — the desktop's `files`, its documents +/// the 48-row bootstrap catalogue — the desktop's `files`, its documents /// `editor`, `photos`, `settings` and `power-ui`, the screenshot writer `capture`, beside /// the native filesystem verbs `append`, `sync` and `pack`, the two /// WebAssembly rows, `wasm-witness` and `wasm`, `jsapp`, the JS app host, -/// `inference` / `model-data-witness`, and `shell`, the guest shell `term` -/// opens — plus the founded shell, the graphical terminal, and three -/// extraction parsers. +/// `inference` / `model-data-witness`, `shell`, the guest shell `term` +/// opens, `tcc`, the C compiler, `ioctl`, the device tool, and `vdev-test`, +/// the virtual accelerator's driver — plus the +/// founded shell, the graphical terminal, and three extraction parsers. /// Exec asserts this covers its catalogue (`kore-svc-exec`), the one place /// that sees both. Named /// separately from the interactive reservation below because the two grow for /// unrelated reasons: this one when a row joins the catalogue, that one when /// the desktop admits another window or controller. -pub const BOOTSTRAP_IMAGE_PEERS: usize = 50; +pub const BOOTSTRAP_IMAGE_PEERS: usize = 53; -/// Bootstrap face-supply sizing. Image and runtime grow their face tables and -/// rings on demand; this reserves enough initial memory for the boot catalogue -/// and for a full desktop that can still run one command per controller. -pub const MAX_MINTED_FACE_PAIRS: usize = BOOTSTRAP_IMAGE_PEERS + INTERACTIVE_CHILD_LOADER_FACES; +/// Private image generations installed at runtime that can live at once, +/// across every session: exec's `InstallFile` rows, each a file a task +/// compiled or copied into its own tree and made executable from its own +/// memory supply. +/// +/// Each one costs a ring stride of face supply here (8 × 16 KiB, 128 KiB in +/// all) and one fixed landing in exec's convey table, which takes it from +/// exec's low convey band: eight leave that band thirty slots, above the +/// twelve the founder claims in it (exec asserts the band still holds +/// them). Exec lets one client face hold at most half of these rows, so no +/// session can exhaust the rest. Raising it costs exactly those two things; +/// lowering it below two would let one session's install starve another's. +pub const INSTALLED_IMAGE_PEERS: usize = 8; -/// Total byte length of a [`role::FACE_SUPPLY`] endowment — exactly -/// [`MAX_MINTED_FACE_PAIRS`] ring strides. +/// Bootstrap face-supply sizing. Image and runtime grow their face tables and +/// rings on demand; this reserves enough initial memory for the boot +/// catalogue, the installed generations, and a full desktop that can still +/// run one command per controller. +pub const MAX_MINTED_FACE_PAIRS: usize = + BOOTSTRAP_IMAGE_PEERS + INSTALLED_IMAGE_PEERS + INTERACTIVE_CHILD_LOADER_FACES; + +/// Total byte length of the image service's [`role::FACE_SUPPLY`] +/// endowment — exactly [`MAX_MINTED_FACE_PAIRS`] ring strides. /// The **cross-boundary geometry contract**: the kernel founder mints a -/// lease (or, for the device broker, a full-span sub of the shelf-held -/// claimant band) of exactly this length; the carrying service's face -/// table carves its slots against the same number (its claim checks the -/// span). One constant, cited by both sides, so a drift refuses at -/// bind instead of carving a phantom ring. +/// lease of exactly this length; the image face table carves its slots +/// against the same number (its claim checks the span). One constant, +/// cited by both sides, so a drift refuses at bind instead of carving a +/// phantom ring. The device broker carries the same role under its own +/// contract, [`CLAIMANT_FACE_SUPPLY_BYTES`]. pub const FACE_SUPPLY_BYTES: u64 = MAX_MINTED_FACE_PAIRS as u64 * FACE_RING_STRIDE; +/// Total byte length of a device broker's [`role::FACE_SUPPLY`] +/// endowment, its claimant band — exactly +/// [`MAX_CLAIMANT_FACES`](crate::device::MAX_CLAIMANT_FACES) ring +/// strides, one per driver row. The same contract as +/// [`FACE_SUPPLY_BYTES`]: the kernel founder mints each broker's band at +/// this length and carves the broker's full-span sub off it, and the +/// broker refuses readiness on any other span. +pub const CLAIMANT_FACE_SUPPLY_BYTES: u64 = + crate::device::MAX_CLAIMANT_FACES as u64 * FACE_RING_STRIDE; + /// **Geometry roles** — the window kinds a task learns its own VAs for from the /// [`GeomBlock`] (`res/USERLAND.md` §4; the fixed-VA elimination rung). Unlike /// [`role`] handle entries (which name an arena *slot*), a geometry entry names diff --git a/native/crates/c-shim/Cargo.toml b/native/crates/c-shim/Cargo.toml new file mode 100644 index 0000000..1dde6da --- /dev/null +++ b/native/crates/c-shim/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "kore-c-shim" +version = "0.1.0" +edition = "2024" +publish = false +description = "The C symbols a vendored C library imports on a kore user target" + +[features] +default = ["runtime", "malloc", "abort", "wide-classes"] +# The process half: the `` writes over the standard sink the process +# installs, `errno` and the parsers that set it. An executable takes it; a +# shared library leaves it out and imports those symbols from the executable +# it links into, so one process never holds two heaps. +runtime = ["dep:kore-rt"] +# `malloc`, `calloc`, `realloc` and `free` over the task's global heap. A C +# runtime that exports its own allocator under those names (libkore.so) +# leaves this out, so one process never holds two `malloc`s whose blocks +# the other's `free` would misread. +malloc = ["runtime"] +# `abort`, ending the calling task with the aborted status. A C runtime that +# ends a whole program its own way (libkore.so's host) leaves this out and +# exports its own. +abort = ["runtime"] +# The wide character classes (`iswalnum` … `iswxdigit`, `towlower`, +# `towupper`) a vendored C library calls: tree-sitter's scanners. Pure, so a +# shared library may carry its own. A C runtime whose headers declare no +# `` (libkore.so) leaves them out, so it exports exactly the ABI +# its headers declare. +wide-classes = [] +# `strtold` and `ldexpl` as byte-level helpers a consumer's C layer wraps +# (`long_double`): C's `long double` has no Rust spelling, so the C side owns +# the type and this crate the arithmetic. Off unless a consumer compiles C +# that converts `long double` (TCC does: its float literals). +long-double = ["runtime", "dep:rustc_apfloat"] + +[dependencies] +kore-rt = { path = "../rt", optional = true } +# LLVM's APFloat (the code clang folds float literals with): correctly +# rounded decimal conversion and scaling for x87 double-extended and IEEE +# quad. `no_std`, `forbid(unsafe_code)`; it allocates only to convert decimal. +rustc_apfloat = { version = "=0.2.3", optional = true } + +[dev-dependencies] +rustc_apfloat = "=0.2.3" diff --git a/native/crates/c-shim/src/alloc_symbols.rs b/native/crates/c-shim/src/alloc_symbols.rs new file mode 100644 index 0000000..6e97947 --- /dev/null +++ b/native/crates/c-shim/src/alloc_symbols.rs @@ -0,0 +1,107 @@ +//! `malloc`, `calloc`, `realloc` and `free` over the task's global heap. +//! +//! C frees without a size and Rust deallocates with one, so every block +//! carries its payload size in a header just below the pointer C sees. The +//! header is one `max_align_t` wide, so the payload keeps C's alignment +//! guarantee. Overflow and exhaustion return null, as C expects; a size of +//! zero returns null too, which `free` and `realloc` accept. + +extern crate alloc; + +use core::alloc::Layout; +use core::ffi::c_void; +use core::ptr; + +/// C's `max_align_t` on every kore target: the payload alignment `malloc` +/// promises, and the header's width. +const ALIGN: usize = 16; + +fn layout(payload: usize) -> Option { + Layout::from_size_align(payload.checked_add(ALIGN)?, ALIGN).ok() +} + +/// The payload size recorded below `payload`. +/// +/// # Safety +/// `payload` came from this module and is live. +unsafe fn payload_size(payload: *mut c_void) -> usize { + // SAFETY: the caller's contract; the header sits `ALIGN` bytes below. + unsafe { payload.cast::().sub(ALIGN).cast::().read() } +} + +/// Write `size` into the block's header and return its payload. +/// +/// # Safety +/// `block` is a live allocation of `layout(size)`. +unsafe fn finish(block: *mut u8, size: usize) -> *mut c_void { + if block.is_null() { + return ptr::null_mut(); + } + // SAFETY: the block is at least `ALIGN` bytes and `ALIGN`-aligned. + unsafe { + block.cast::().write(size); + block.add(ALIGN).cast() + } +} + +#[unsafe(no_mangle)] +extern "C" fn malloc(size: usize) -> *mut c_void { + if size == 0 { + return ptr::null_mut(); + } + let Some(layout) = layout(size) else { + return ptr::null_mut(); + }; + // SAFETY: the layout is non-zero; the header is written before return. + unsafe { finish(alloc::alloc::alloc(layout), size) } +} + +#[unsafe(no_mangle)] +extern "C" fn calloc(count: usize, size: usize) -> *mut c_void { + let Some(total) = count.checked_mul(size).filter(|&total| total != 0) else { + return ptr::null_mut(); + }; + let Some(layout) = layout(total) else { + return ptr::null_mut(); + }; + // SAFETY: as `malloc`; the zeroed allocation covers the payload. + unsafe { finish(alloc::alloc::alloc_zeroed(layout), total) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn realloc(payload: *mut c_void, size: usize) -> *mut c_void { + if payload.is_null() { + return malloc(size); + } + if size == 0 { + // SAFETY: the caller's pointer came from this module. + unsafe { free(payload) }; + return ptr::null_mut(); + } + // SAFETY: a non-null pointer C passes back came from this module. + let old = unsafe { payload_size(payload) }; + let (Some(old_layout), Some(_)) = (layout(old), layout(size)) else { + return ptr::null_mut(); + }; + // SAFETY: the block and its layout are the live allocation; the new size + // fits a layout (checked above). + unsafe { + let block = payload.cast::().sub(ALIGN); + finish(alloc::alloc::realloc(block, old_layout, size + ALIGN), size) + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn free(payload: *mut c_void) { + if payload.is_null() { + return; + } + // SAFETY: a non-null pointer C passes back came from this module, so + // its header names the layout it was allocated with. + unsafe { + let size = payload_size(payload); + if let Some(layout) = layout(size) { + alloc::alloc::dealloc(payload.cast::().sub(ALIGN), layout); + } + } +} diff --git a/native/crates/c-shim/src/errno.rs b/native/crates/c-shim/src/errno.rs new file mode 100644 index 0000000..7a5fbf8 --- /dev/null +++ b/native/crates/c-shim/src/errno.rs @@ -0,0 +1,96 @@ +//! `errno`: one per thread, as C requires, and the codes this crate and its +//! consumers set. +//! +//! The word is the calling thread's runtime-block slot +//! ([`kore_rt::thread::c_errno`]), not an ELF thread-local: a TLS segment in +//! every image that carries C would buy each of them a TCB migration, a +//! template every spawn must place, and `kore-late`'s refusal of a host with +//! TLS — all for one `int`. +//! +//! The numbers are musl's `` — the headers every kore C build +//! compiles against (`zig cc -target -linux-musl`), which use the +//! same values on x86_64, aarch64 and riscv64. A consumer's C code compares +//! `errno` against those headers' macros, so these must agree with them. + +use core::ffi::{CStr, c_int}; + +pub const EPERM: c_int = 1; +pub const ENOENT: c_int = 2; +pub const EIO: c_int = 5; +pub const ENXIO: c_int = 6; +pub const EBADF: c_int = 9; +pub const EAGAIN: c_int = 11; +pub const ENOMEM: c_int = 12; +pub const EACCES: c_int = 13; +pub const EFAULT: c_int = 14; +pub const EBUSY: c_int = 16; +pub const EEXIST: c_int = 17; +pub const ENOTDIR: c_int = 20; +pub const EISDIR: c_int = 21; +pub const EINVAL: c_int = 22; +pub const ENFILE: c_int = 23; +pub const EMFILE: c_int = 24; +pub const EFBIG: c_int = 27; +pub const ENOSPC: c_int = 28; +pub const ESPIPE: c_int = 29; +pub const EROFS: c_int = 30; +pub const ERANGE: c_int = 34; +pub const ENAMETOOLONG: c_int = 36; +pub const ENOSYS: c_int = 38; +pub const ENOTEMPTY: c_int = 39; +pub const ELOOP: c_int = 40; +pub const EOVERFLOW: c_int = 75; +pub const ENOTSUP: c_int = 95; + +/// Set this thread's `errno`. +pub fn set(code: c_int) { + // SAFETY: the calling thread's own word, live for its life and touched + // by no other thread (`kore_rt::thread::c_errno`'s contract). + unsafe { kore_rt::thread::c_errno().write(code) }; +} + +/// This thread's `errno`. +pub fn get() -> c_int { + // SAFETY: as `set`. + unsafe { kore_rt::thread::c_errno().read() } +} + +#[unsafe(no_mangle)] +extern "C" fn __errno_location() -> *mut c_int { + kore_rt::thread::c_errno() +} + +/// The message `strerror` answers for `code`. +pub fn describe(code: c_int) -> &'static CStr { + match code { + 0 => c"No error information", + EPERM => c"Operation not permitted", + ENOENT => c"No such file or directory", + EIO => c"I/O error", + ENXIO => c"No such device or address", + EBADF => c"Bad file descriptor", + EAGAIN => c"Resource temporarily unavailable", + ENOMEM => c"Out of memory", + EACCES => c"Permission denied", + EFAULT => c"Bad address", + EBUSY => c"Resource busy", + EEXIST => c"File exists", + ENOTDIR => c"Not a directory", + EISDIR => c"Is a directory", + EINVAL => c"Invalid argument", + ENFILE => c"Too many open files in system", + EMFILE => c"No file descriptors available", + EFBIG => c"File too large", + ENOSPC => c"No space left on device", + ESPIPE => c"Invalid seek", + EROFS => c"Read-only file system", + ERANGE => c"Result not representable", + ENAMETOOLONG => c"Filename too long", + ENOSYS => c"Function not implemented", + ENOTEMPTY => c"Directory not empty", + ELOOP => c"Symbolic link loop", + EOVERFLOW => c"Value too large for data type", + ENOTSUP => c"Not supported", + _ => c"Unknown error", + } +} diff --git a/native/crates/c-shim/src/format.rs b/native/crates/c-shim/src/format.rs new file mode 100644 index 0000000..0702d76 --- /dev/null +++ b/native/crates/c-shim/src/format.rs @@ -0,0 +1,724 @@ +//! The `printf` conversion engine behind every formatted write in this crate. +//! +//! One engine serves `snprintf` into a caller's buffer and `fprintf` into a +//! stream, so the two cannot disagree about what a conversion means. It is +//! generic over where arguments come from ([`Args`]: a C `va_list` on the +//! target, a scripted list in host tests) and where bytes go ([`Sink`]), and +//! it is pure: the host suite drives exactly the code the target runs. +//! +//! ## What it formats +//! +//! ISO C's integer, character, string and pointer conversions in full — +//! `d i o u x X c s p %`, the `- + space # 0` flags, field width and +//! precision (both also as `*`), and the `hh h l ll j z t` length modifiers. +//! A precision bounds a `%s` read, so `%.*s` over a counted, unterminated +//! string never reads past its count. `%p` is spelled `0x` and the address +//! in lowercase hexadecimal (the spelling is implementation-defined; this is +//! the one this crate has always used). +//! +//! The float conversions `e E f F g G a A` too, of a `double` and (with `L`) +//! of a `long double` in either of kore's formats, every flag, width and +//! precision honoured and every digit exact ([`float`]): the C programs +//! libkore.so runs print through this engine. +//! +//! ## What it deliberately does not +//! +//! - **`%n`** stores nothing: its pointer is consumed and ignored. A format +//! that writes through an argument is the classic format-string attack, +//! and no consumer needs one. +//! - **Wide characters**: an ASCII `%lc` is written as its byte and anything +//! else as `?`; `%ls` consumes its pointer and is spelled back. +//! +//! An unknown conversion is spelled back and consumes nothing — its type +//! cannot be known, and guessing would misalign every argument after it. + +use core::ffi::c_int; +use core::marker::PhantomData; + +mod float; + +pub use float::Float; + +/// The C type a length modifier names. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Length { + /// `hh`: `char`, promoted to `int` in a variadic call. + Char, + /// `h`: `short`, promoted to `int`. + Short, + /// No modifier: `int`. + Int, + /// `l`: `long`. + Long, + /// `ll` (and BSD's `q`): `long long`. + LongLong, + /// `j`: `intmax_t`. + Max, + /// `z`: `size_t`. + Size, + /// `t`: `ptrdiff_t`. + Diff, + /// `L`: `long double` beside a float conversion; `long long` beside an + /// integer one, as glibc reads it. + LongDouble, +} + +/// Where each conversion's argument comes from. +/// +/// Every method consumes exactly one argument of the C type it names, as a +/// variadic callee must. +pub trait Args { + /// A signed integer of `length`'s type, sign-extended. + fn signed(&mut self, length: Length) -> i64; + /// An unsigned integer of `length`'s type, zero-extended. + fn unsigned(&mut self, length: Length) -> u64; + /// An `int`: a `%c` argument, or a `*` width or precision. + fn int(&mut self) -> c_int; + /// A pointer, as an address: `%p`, `%n`, `%ls`, and `%s`'s string. + fn pointer(&mut self) -> usize; + /// A `double` (a `float` arrives promoted to one). + fn double(&mut self) -> f64; + /// A `long double`, read in the C ABI's own format. Rust has no type for + /// it, so the source decodes it. + fn long_double(&mut self) -> Float; +} + +/// Where formatted bytes go. The engine counts them; a sink only stores. +pub trait Sink { + /// Accept the next run of output bytes. + fn put(&mut self, bytes: &[u8]); +} + +/// Format `format` into `sink`, drawing arguments from `args`, and return +/// how many bytes the complete output has — the count `snprintf` returns +/// even when its buffer is shorter. +/// +/// # Safety +/// `format` is NUL-terminated, and its conversions match the arguments +/// `args` yields. +pub unsafe fn format(sink: &mut impl Sink, format: *const u8, args: &mut impl Args) -> usize { + let format = Format { + base: format, + _bytes: PhantomData, + }; + let mut out = Out { sink, count: 0 }; + let mut at = 0usize; + loop { + // The literal run up to the next conversion, written whole. + let start = at; + // SAFETY (every `format.at` / `format.slice` below): reads stop at + // the terminator, and `at` only advances past bytes already read as + // non-NUL. + while !matches!(unsafe { format.at(at) }, 0 | b'%') { + at += 1; + } + out.put(unsafe { format.slice(start, at) }); + if unsafe { format.at(at) } == 0 { + return out.count; + } + let spec_start = at; + at += 1; + + let mut spec = Spec::default(); + loop { + match unsafe { format.at(at) } { + b'-' => spec.left = true, + b'+' => spec.plus = true, + b' ' => spec.space = true, + b'#' => spec.alternate = true, + b'0' => spec.zero = true, + _ => break, + } + at += 1; + } + if unsafe { format.at(at) } == b'*' { + at += 1; + let width = args.int(); + // A negative `*` width is a `-` flag and a positive width. + spec.left |= width < 0; + spec.width = (width.unsigned_abs() as usize).min(MAX_FIELD); + } else { + spec.width = unsafe { format.decimal(&mut at) }; + } + if unsafe { format.at(at) } == b'.' { + at += 1; + if unsafe { format.at(at) } == b'*' { + at += 1; + // A negative precision is taken as if it were omitted. + spec.precision = usize::try_from(args.int()).ok(); + } else { + spec.precision = Some(unsafe { format.decimal(&mut at) }); + } + } + let (length, skip) = match unsafe { [format.at(at), format.at(at + 1)] } { + [b'h', b'h'] => (Length::Char, 2), + [b'h', _] => (Length::Short, 1), + [b'l', b'l'] => (Length::LongLong, 2), + [b'l', _] => (Length::Long, 1), + [b'q', _] => (Length::LongLong, 1), + [b'j', _] => (Length::Max, 1), + [b'z', _] => (Length::Size, 1), + [b't', _] => (Length::Diff, 1), + [b'L', _] => (Length::LongDouble, 1), + _ => (Length::Int, 0), + }; + at += skip; + let conversion = unsafe { format.at(at) }; + if conversion == 0 { + // A specification cut off by the terminator: spell it back. + out.put(unsafe { format.slice(spec_start, at) }); + return out.count; + } + at += 1; + // The specification's own bytes, for the conversions spelled back. + let spelled = unsafe { format.slice(spec_start, at) }; + + match conversion { + b'%' => out.put(b"%"), + b'd' | b'i' => { + let value = args.signed(length); + integer( + &mut out, + spec, + value.unsigned_abs(), + value < 0, + Radix::Signed, + ); + } + b'u' => integer(&mut out, spec, args.unsigned(length), false, Radix::Decimal), + b'o' => integer(&mut out, spec, args.unsigned(length), false, Radix::Octal), + b'x' => integer(&mut out, spec, args.unsigned(length), false, Radix::Hex), + b'X' => integer( + &mut out, + spec, + args.unsigned(length), + false, + Radix::HexUpper, + ), + b'c' => { + let value = args.int(); + let byte = if length == Length::Long { + // `wint_t`: only an ASCII code point has a byte here. + u8::try_from(value) + .ok() + .filter(u8::is_ascii) + .unwrap_or(b'?') + } else { + value as u8 + }; + text(&mut out, spec, &[byte]); + } + b's' if length == Length::Long => { + args.pointer(); + out.put(spelled); + } + b's' => { + let address = args.pointer() as *const u8; + if address.is_null() { + text(&mut out, spec, b"(null)"); + } else { + // SAFETY: C promises a string readable to its NUL, or to + // the precision when one bounds the read. + let bytes = unsafe { c_string(address, spec.precision) }; + text(&mut out, spec, bytes); + } + } + b'p' => { + let mut digits = [0u8; 22]; + let digits = render(args.pointer() as u64, Radix::Hex, &mut digits); + let mut buffer = [0u8; 18]; + buffer[..2].copy_from_slice(b"0x"); + buffer[2..2 + digits.len()].copy_from_slice(digits); + text(&mut out, spec, &buffer[..2 + digits.len()]); + } + b'n' => { + args.pointer(); + } + b'e' | b'E' | b'f' | b'F' | b'g' | b'G' | b'a' | b'A' => { + if length == Length::LongDouble { + let value = args.long_double(); + float::convert::<{ float::LONG_DOUBLE_LIMBS }, _>( + &mut out, spec, conversion, value, + ); + } else { + let value = Float::from_f64(args.double()); + float::convert::<{ float::DOUBLE_LIMBS }, _>(&mut out, spec, conversion, value); + } + } + _ => out.put(spelled), + } + } +} + +/// A NUL-terminated C format string. +struct Format<'f> { + base: *const u8, + _bytes: PhantomData<&'f [u8]>, +} + +impl<'f> Format<'f> { + /// # Safety + /// `at` lies at or before the terminator. + unsafe fn at(&self, at: usize) -> u8 { + // SAFETY: the caller's contract. + unsafe { *self.base.add(at) } + } + + /// The bytes in `start..end`. + /// + /// # Safety + /// The range was read and lies before the terminator. + unsafe fn slice(&self, start: usize, end: usize) -> &'f [u8] { + // SAFETY: the caller's contract. + unsafe { core::slice::from_raw_parts(self.base.add(start), end - start) } + } + + /// A run of decimal digits as a field width or precision, saturating at + /// [`MAX_FIELD`]. + /// + /// # Safety + /// `at` lies at or before the terminator. + unsafe fn decimal(&self, at: &mut usize) -> usize { + let mut value = 0usize; + // SAFETY: the caller's contract; a digit is never the terminator. + while let digit @ b'0'..=b'9' = unsafe { self.at(*at) } { + value = value + .saturating_mul(10) + .saturating_add(usize::from(digit - b'0')) + .min(MAX_FIELD); + *at += 1; + } + value + } +} + +/// A `*` or decimal width or precision is an `int`; this is the largest. +const MAX_FIELD: usize = c_int::MAX as usize; + +/// Output padding comes from these, a run at a time. +const SPACES: [u8; 32] = [b' '; 32]; +const ZEROS: [u8; 32] = [b'0'; 32]; + +/// The engine's counting front: every byte a sink sees passes through here. +struct Out<'s, S: Sink> { + sink: &'s mut S, + count: usize, +} + +impl Out<'_, S> { + fn put(&mut self, bytes: &[u8]) { + if !bytes.is_empty() { + self.sink.put(bytes); + self.count = self.count.saturating_add(bytes.len()); + } + } + + fn pad(&mut self, fill: &[u8; 32], mut count: usize) { + while count > 0 { + let run = count.min(fill.len()); + self.put(&fill[..run]); + count -= run; + } + } +} + +/// One parsed conversion specification. +#[derive(Clone, Copy, Debug, Default)] +struct Spec { + left: bool, + plus: bool, + space: bool, + alternate: bool, + zero: bool, + width: usize, + precision: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Radix { + Octal, + /// `%d`/`%i`: decimal, and the only radix the `+` and space flags mark. + Signed, + /// `%u`. + Decimal, + Hex, + HexUpper, +} + +/// The bytes of the C string at `address`, stopping at its NUL or, when +/// `limit` is given, after `limit` bytes — whichever comes first. +/// +/// # Safety +/// `address` is readable up to its NUL or `limit`. +unsafe fn c_string<'s>(address: *const u8, limit: Option) -> &'s [u8] { + let limit = limit.unwrap_or(usize::MAX); + let mut len = 0usize; + // SAFETY: the caller's contract bounds every read. + while len < limit && unsafe { *address.add(len) } != 0 { + len += 1; + } + unsafe { core::slice::from_raw_parts(address, len) } +} + +/// `value`'s digits, written into the tail of `buffer` (22 bytes hold a u64 +/// in octal). +fn render(mut value: u64, radix: Radix, buffer: &mut [u8; 22]) -> &[u8] { + let (base, digits): (u64, &[u8; 16]) = match radix { + Radix::Octal => (8, b"0123456789abcdef"), + Radix::Signed | Radix::Decimal => (10, b"0123456789abcdef"), + Radix::Hex => (16, b"0123456789abcdef"), + Radix::HexUpper => (16, b"0123456789ABCDEF"), + }; + let mut at = buffer.len(); + loop { + at -= 1; + buffer[at] = digits[(value % base) as usize]; + value /= base; + if value == 0 { + break; + } + } + &buffer[at..] +} + +/// An integer conversion: sign or prefix, the precision's leading zeros, +/// digits, and the field's padding, placed as ISO C 7.21.6.1 places them. +fn integer(out: &mut Out<'_, impl Sink>, spec: Spec, magnitude: u64, negative: bool, radix: Radix) { + let mut buffer = [0u8; 22]; + let mut digits = render(magnitude, radix, &mut buffer); + // A zero precision prints no digits for a zero value. + if magnitude == 0 && spec.precision == Some(0) { + digits = &[]; + } + let prefix: &[u8] = match radix { + _ if negative => b"-", + Radix::Signed if spec.plus => b"+", + Radix::Signed if spec.space => b" ", + Radix::Hex if spec.alternate && magnitude != 0 => b"0x", + Radix::HexUpper if spec.alternate && magnitude != 0 => b"0X", + _ => b"", + }; + let mut leading = spec + .precision + .map_or(0, |precision| precision.saturating_sub(digits.len())); + // `#` with `o` raises the precision just enough for a leading zero. + if spec.alternate && radix == Radix::Octal && leading == 0 && digits.first() != Some(&b'0') { + leading = 1; + } + let fill = spec + .width + .saturating_sub(prefix.len() + leading + digits.len()); + // `0` pads between the prefix and the digits, unless a precision was + // given or the field is left-justified. + let zero_fill = spec.zero && !spec.left && spec.precision.is_none(); + if !spec.left && !zero_fill { + out.pad(&SPACES, fill); + } + out.put(prefix); + if zero_fill { + out.pad(&ZEROS, fill); + } + out.pad(&ZEROS, leading); + out.put(digits); + if spec.left { + out.pad(&SPACES, fill); + } +} + +/// A text conversion (`%c`, `%s`, `%p`): the bytes and the field's padding. +/// The `0` flag means nothing here, as in C. +fn text(out: &mut Out<'_, impl Sink>, spec: Spec, bytes: &[u8]) { + let fill = spec.width.saturating_sub(bytes.len()); + if !spec.left { + out.pad(&SPACES, fill); + } + out.put(bytes); + if spec.left { + out.pad(&SPACES, fill); + } +} + +#[cfg(test)] +mod tests { + use super::*; + extern crate std; + use std::{collections::VecDeque, string::String, vec::Vec}; + + /// One scripted argument. + #[derive(Clone, Copy, Debug)] + enum Arg { + Int(i64), + Uint(u64), + Ptr(usize), + Double(f64), + LongDouble(Float), + } + + /// A NUL-terminated (or, for `%.*s`, counted) string argument. + fn text(bytes: &'static [u8]) -> Arg { + Arg::Ptr(bytes.as_ptr() as usize) + } + + struct Script { + args: VecDeque, + /// Each consumption, so a test can prove alignment. + taken: Vec<&'static str>, + } + + impl Script { + fn take(&mut self, kind: &'static str) -> Arg { + self.taken.push(kind); + self.args + .pop_front() + .expect("the format consumed more arguments than given") + } + } + + impl Args for Script { + fn signed(&mut self, length: Length) -> i64 { + let Arg::Int(value) = self.take("signed") else { + panic!("a signed conversion met another type") + }; + // The truncation a C callee applies to a promoted argument. + match length { + Length::Char => value as i8 as i64, + Length::Short => value as i16 as i64, + Length::Int => value as i32 as i64, + _ => value, + } + } + fn unsigned(&mut self, length: Length) -> u64 { + let value = match self.take("unsigned") { + Arg::Uint(value) => value, + Arg::Int(value) => value as u64, + other => panic!("an unsigned conversion met {other:?}"), + }; + match length { + Length::Char => value as u8 as u64, + Length::Short => value as u16 as u64, + Length::Int => value as u32 as u64, + _ => value, + } + } + fn int(&mut self) -> c_int { + let Arg::Int(value) = self.take("int") else { + panic!("an int met another type") + }; + value as c_int + } + fn pointer(&mut self) -> usize { + let Arg::Ptr(value) = self.take("pointer") else { + panic!("a pointer met another type") + }; + value + } + fn double(&mut self) -> f64 { + let Arg::Double(value) = self.take("double") else { + panic!("a double met another type") + }; + value + } + fn long_double(&mut self) -> Float { + let Arg::LongDouble(value) = self.take("long double") else { + panic!("a long double met another type") + }; + value + } + } + + struct Collect(Vec); + + impl Sink for Collect { + fn put(&mut self, bytes: &[u8]) { + self.0.extend_from_slice(bytes); + } + } + + /// Format a NUL-terminated `format`; return the text, count and script. + fn run(format: &[u8], args: &[Arg]) -> (String, usize, Vec<&'static str>) { + assert_eq!(format.last(), Some(&0), "test formats are NUL-terminated"); + let mut script = Script { + args: args.iter().copied().collect(), + taken: Vec::new(), + }; + let mut sink = Collect(Vec::new()); + // SAFETY: the format is NUL-terminated and the script matches it. + let count = unsafe { super::format(&mut sink, format.as_ptr(), &mut script) }; + assert!( + script.args.is_empty(), + "the format left arguments unconsumed" + ); + (String::from_utf8(sink.0).unwrap(), count, script.taken) + } + + fn check(format: &[u8], args: &[Arg], expected: &str) { + let (out, count, _) = run(format, args); + assert_eq!( + out, + expected, + "format {:?}", + String::from_utf8_lossy(format) + ); + assert_eq!(count, expected.len()); + } + + // Integer, character and string expectations match the host C library's + // printf for the same format and arguments. + + #[test] + fn literal_text_and_percent() { + check(b"plain\0", &[], "plain"); + check(b"100%% sure\0", &[], "100% sure"); + check(b"\0", &[], ""); + } + + #[test] + fn signed_decimal_with_flags_width_and_precision() { + check(b"%d\0", &[Arg::Int(42)], "42"); + check(b"%d\0", &[Arg::Int(-42)], "-42"); + check(b"%i\0", &[Arg::Int(0)], "0"); + check(b"%5d|\0", &[Arg::Int(42)], " 42|"); + check(b"%-5d|\0", &[Arg::Int(42)], "42 |"); + check(b"%05d\0", &[Arg::Int(-42)], "-0042"); + check(b"%+d %+d\0", &[Arg::Int(7), Arg::Int(-7)], "+7 -7"); + check(b"% d % d\0", &[Arg::Int(7), Arg::Int(-7)], " 7 -7"); + check(b"%.3d\0", &[Arg::Int(7)], "007"); + check(b"%6.3d|\0", &[Arg::Int(-7)], " -007|"); + check(b"%06.3d|\0", &[Arg::Int(7)], " 007|"); + check(b"%.0d|\0", &[Arg::Int(0)], "|"); + check(b"%-2d|\0", &[Arg::Int(3)], "3 |"); + check(b"%d\0", &[Arg::Int(i32::MIN as i64)], "-2147483648"); + check(b"%lld\0", &[Arg::Int(i64::MIN)], "-9223372036854775808"); + check(b"%ld\0", &[Arg::Int(1 << 40)], "1099511627776"); + check(b"%hhd\0", &[Arg::Int(300)], "44"); + check(b"%hd\0", &[Arg::Int(70000)], "4464"); + } + + #[test] + fn unsigned_octal_and_hexadecimal() { + check(b"%u\0", &[Arg::Uint(4294967295)], "4294967295"); + check( + b"%x %X\0", + &[Arg::Uint(0xbeef), Arg::Uint(0xbeef)], + "beef BEEF", + ); + check(b"%08x\0", &[Arg::Uint(0x1a)], "0000001a"); + check(b"%#x %#X\0", &[Arg::Uint(255), Arg::Uint(255)], "0xff 0XFF"); + check(b"%#x\0", &[Arg::Uint(0)], "0"); + check(b"%#010x\0", &[Arg::Uint(255)], "0x000000ff"); + check( + b"%o %#o %#o\0", + &[Arg::Uint(8), Arg::Uint(8), Arg::Uint(0)], + "10 010 0", + ); + check(b"%#.0o\0", &[Arg::Uint(0)], "0"); + check(b"%.8x\0", &[Arg::Uint(0x1234)], "00001234"); + check(b"%3X|\0", &[Arg::Uint(0xa)], " A|"); + check(b"%8.2X|\0", &[Arg::Uint(0x5)], " 05|"); + check(b"%lx\0", &[Arg::Uint(0xdead_beef_cafe)], "deadbeefcafe"); + check(b"%08llx\0", &[Arg::Uint(0x12)], "00000012"); + check(b"%llu\0", &[Arg::Uint(u64::MAX)], "18446744073709551615"); + check( + b"%zu %zx\0", + &[Arg::Uint(4096), Arg::Uint(4096)], + "4096 1000", + ); + check(b"%hhx\0", &[Arg::Uint(0x1ff)], "ff"); + // `+` and space belong to signed conversions only. + check(b"%+u % u\0", &[Arg::Uint(5), Arg::Uint(5)], "5 5"); + } + + #[test] + fn characters_strings_and_pointers() { + check(b"[%c]\0", &[Arg::Int(b'A' as i64)], "[A]"); + check( + b"[%3c][%-3c]\0", + &[Arg::Int(b'x' as i64), Arg::Int(b'y' as i64)], + "[ x][y ]", + ); + check(b"%s\0", &[text(b"hello\0")], "hello"); + check( + b"[%8s][%-8s]\0", + &[text(b"ab\0"), text(b"cd\0")], + "[ ab][cd ]", + ); + check(b"%.2s\0", &[text(b"hello\0")], "he"); + check(b"%s\0", &[Arg::Ptr(0)], "(null)"); + check( + b"%lc %lc\0", + &[Arg::Int(b'q' as i64), Arg::Int(0x263a)], + "q ?", + ); + // `%p` is this crate's own spelling, not the host's. + check(b"%p\0", &[Arg::Ptr(0x1000)], "0x1000"); + check(b"%p\0", &[Arg::Ptr(0)], "0x0"); + check(b"[%10p]\0", &[Arg::Ptr(0xabc)], "[ 0xabc]"); + } + + #[test] + fn a_precision_bounds_the_read_of_an_unterminated_string() { + // No NUL in these bytes: `%.*s` must stop at its count. + check(b"[%.*s]\0", &[Arg::Int(3), text(b"abcd")], "[abc]"); + check(b"[%.4s]\0", &[text(b"abcd")], "[abcd]"); + } + + #[test] + fn star_width_and_precision() { + check(b"[%*d]\0", &[Arg::Int(5), Arg::Int(42)], "[ 42]"); + check(b"[%*d]\0", &[Arg::Int(-5), Arg::Int(42)], "[42 ]"); + check(b"[%*s]\0", &[Arg::Int(4), text(b"x\0")], "[ x]"); + check(b"[%.*s]\0", &[Arg::Int(2), text(b"xyz\0")], "[xy]"); + // A negative precision is as if it were absent. + check(b"[%.*s]\0", &[Arg::Int(-1), text(b"xyz\0")], "[xyz]"); + check(b"[%.*d]\0", &[Arg::Int(4), Arg::Int(7)], "[0007]"); + } + + #[test] + fn floats_are_formatted_and_the_arguments_after_them_line_up() { + let (out, _, taken) = run(b"%0.3f s, %u lines\0", &[Arg::Double(1.5), Arg::Uint(9)]); + assert_eq!(out, "1.500 s, 9 lines"); + assert_eq!(taken, ["double", "unsigned"]); + // `L` reads a long double, in the source's own format. + let x87_two_and_a_half = Float::from_x87(0x4000_a000_0000_0000_0000); + let (out, _, taken) = run( + b"a %Lf then %d\0", + &[Arg::LongDouble(x87_two_and_a_half), Arg::Int(7)], + ); + assert_eq!(out, "a 2.500000 then 7"); + assert_eq!(taken, ["long double", "signed"]); + check( + b"[%8.2e|%-8g|%+a]\0", + &[Arg::Double(-0.0), Arg::Double(1e-5), Arg::Double(0.5)], + "[-0.00e+00|1e-05 |+0x1p-1]", + ); + } + + #[test] + fn n_stores_nothing_and_unknown_conversions_consume_nothing() { + let (out, _, taken) = run(b"ab%ncd\0", &[Arg::Ptr(0x10)]); + assert_eq!(out, "abcd"); + assert_eq!(taken, ["pointer"]); + let (out, _, taken) = run(b"(%rip) %d\0", &[Arg::Int(1)]); + assert_eq!(out, "(%rip) 1"); + assert_eq!(taken, ["signed"]); + } + + #[test] + fn a_specification_cut_off_by_the_terminator_is_spelled_back() { + check(b"tail %-05\0", &[], "tail %-05"); + check(b"tail %\0", &[], "tail %"); + } + + #[test] + fn a_field_width_saturates_at_int_max() { + let digits = b"99999999999999999999x\0"; + let format = Format { + base: digits.as_ptr(), + _bytes: PhantomData, + }; + let mut at = 0; + // SAFETY: NUL-terminated; the scan stops at the `x`. + assert_eq!(unsafe { format.decimal(&mut at) }, MAX_FIELD); + assert_eq!(at, 20); + } +} diff --git a/native/crates/c-shim/src/format/float.rs b/native/crates/c-shim/src/format/float.rs new file mode 100644 index 0000000..2766af5 --- /dev/null +++ b/native/crates/c-shim/src/format/float.rs @@ -0,0 +1,866 @@ +//! The float conversions of the `printf` engine ([`super::format`]): `%e %E +//! %f %F %g %G %a %A`, of a `double` and of a `long double` in either format +//! kore's C ABIs give it — x87 double-extended on x86_64, IEEE binary128 on +//! aarch64 and riscv64. +//! +//! ## Every digit is the value's +//! +//! A finite float is `mantissa × 2^exponent` with an integer mantissa, so its +//! decimal expansion is finite, and a conversion prints that expansion +//! rounded once, to nearest with ties to even, at the last digit it keeps: +//! the correctly rounded conversion C asks for in the default rounding mode +//! (C11 7.21.6.1p13, F.5), and what glibc and musl print. A [`Decimal`] holds +//! the expansion in base-10⁹ limbs, built from the mantissa by doubling 29 +//! bits or halving 9 bits at a time (musl's scheme, kept in integers), so no +//! digit is estimated. +//! +//! Halving moves digits only toward the low end: a limb below the last digit +//! a conversion can keep never changes one above it. Such limbs are folded +//! into one sticky bit as they appear, which is all a round to nearest asks +//! of them, so a conversion stores at most the digits it prints and two limbs +//! more, and never more than the expansion has: a `%.1000000f` of a `double` +//! costs what its 1074 binary places give. Doubling carries upward, so an +//! integer part is built whole: at most 4933 digits, for a `long double`. +//! +//! ## Spellings +//! +//! An infinity is `inf` and a NaN `nan`, upper case under `E F G A`, each +//! signed as any value is: a NaN's sign bit is its sign, as glibc and musl +//! print it. `%a` normalizes every nonzero value to a leading `1`, subnormals +//! included, as musl does (glibc spells a subnormal `double` with a leading +//! `0`, and an x87 value with its explicit integer bit); with a precision it +//! rounds to nearest-even, which can carry the leading digit to `2`. + +use core::cmp::Ordering; + +use super::{Out, SPACES, Sink, Spec, ZEROS}; + +/// A float as a conversion reads it: its sign bit, and what it holds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Float { + negative: bool, + class: Class, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Class { + /// `mantissa × 2^exponent`; a zero mantissa is a zero of that sign. + Finite { + mantissa: u128, + exponent: i32, + }, + Infinite, + Nan, +} + +/// IEEE binary64: 52 stored fraction bits under an 11-bit exponent, the sign +/// on top. +const F64_FRACTION: u32 = 52; +const F64_BIAS: i32 = 1023; +const F64_EXPONENT_MASK: u64 = 0x7ff; +const F64_SIGN: u32 = u64::BITS - 1; + +/// x87 double-extended: a 64-bit significand whose integer bit is stored, +/// under a 15-bit exponent and the sign, in the low 80 bits. +const X87_SIGNIFICAND: u32 = u64::BITS; +const X87_FRACTION: u32 = X87_SIGNIFICAND - 1; +const X87_BIAS: i32 = 16383; +const X87_SIGN: u32 = X87_SIGNIFICAND + 15; + +/// IEEE binary128: 112 stored fraction bits under a 15-bit exponent, the +/// sign on top. +const QUAD_FRACTION: u32 = 112; +const QUAD_BIAS: i32 = 16383; +const QUAD_SIGN: u32 = u128::BITS - 1; + +/// The 15-bit exponent field both `long double` formats share. +const WIDE_EXPONENT_MASK: u128 = 0x7fff; + +impl Float { + /// A `double`. + pub fn from_f64(value: f64) -> Self { + let bits = value.to_bits(); + let biased = (bits >> F64_FRACTION) & F64_EXPONENT_MASK; + let fraction = u128::from(bits & ((1 << F64_FRACTION) - 1)); + let class = match biased { + F64_EXPONENT_MASK if fraction == 0 => Class::Infinite, + F64_EXPONENT_MASK => Class::Nan, + // Subnormal: the least exponent, and no implicit bit. + 0 => Class::Finite { + mantissa: fraction, + exponent: 1 - F64_BIAS - F64_FRACTION as i32, + }, + _ => Class::Finite { + mantissa: fraction | 1 << F64_FRACTION, + exponent: biased as i32 - F64_BIAS - F64_FRACTION as i32, + }, + }; + Self { + negative: bits >> F64_SIGN != 0, + class, + } + } + + /// An x87 double-extended `long double`: its 80 bits, low-aligned (any + /// padding above them is ignored). + /// + /// The encodings no x87 since the 387 produces are read as musl reads + /// them: an unnormal (a nonzero exponent and a clear integer bit) is a + /// NaN, a pseudo-infinity (the top exponent, a clear integer bit and a + /// zero fraction) an infinity, and a pseudo-denormal (a zero exponent and + /// a set integer bit) the value its bits spell. + pub fn from_x87(bits: u128) -> Self { + let negative = (bits >> X87_SIGN) & 1 != 0; + let biased = (bits >> X87_SIGNIFICAND) & WIDE_EXPONENT_MASK; + let significand = bits as u64; + let integer = significand >> X87_FRACTION != 0; + let class = match biased { + WIDE_EXPONENT_MASK if significand << 1 == 0 => Class::Infinite, + WIDE_EXPONENT_MASK => Class::Nan, + 0 => Class::Finite { + mantissa: u128::from(significand), + exponent: 1 - X87_BIAS - X87_FRACTION as i32, + }, + _ if !integer => Class::Nan, + _ => Class::Finite { + mantissa: u128::from(significand), + exponent: biased as i32 - X87_BIAS - X87_FRACTION as i32, + }, + }; + Self { negative, class } + } + + /// An IEEE binary128 `long double`. + pub fn from_quad(bits: u128) -> Self { + let biased = (bits >> QUAD_FRACTION) & WIDE_EXPONENT_MASK; + let fraction = bits & ((1 << QUAD_FRACTION) - 1); + let class = match biased { + WIDE_EXPONENT_MASK if fraction == 0 => Class::Infinite, + WIDE_EXPONENT_MASK => Class::Nan, + 0 => Class::Finite { + mantissa: fraction, + exponent: 1 - QUAD_BIAS - QUAD_FRACTION as i32, + }, + _ => Class::Finite { + mantissa: fraction | 1 << QUAD_FRACTION, + exponent: biased as i32 - QUAD_BIAS - QUAD_FRACTION as i32, + }, + }; + Self { + negative: bits >> QUAD_SIGN != 0, + class, + } + } +} + +/// The precision `e`, `f` and `g` take when none is given (C11 7.21.6.1p8). +const DEFAULT_PRECISION: i64 = 6; + +/// One base-10⁹ limb's radix. +const BASE: u32 = 1_000_000_000; + +/// Decimal digits per limb. +const LIMB_DIGITS: i64 = 9; + +/// `POW10[k]` is `10^k`, up to a limb's radix. +const POW10: [u32; 10] = [ + 1, + 10, + 100, + 1_000, + 10_000, + 100_000, + 1_000_000, + 10_000_000, + 100_000_000, + 1_000_000_000, +]; + +/// Bits one doubling pass shifts in: a limb below 10⁹ times 2²⁹, plus the +/// carry below 2²⁹ + 1, stays under 2⁶³, and the carry out fits one limb. +const DOUBLE_STEP: u32 = 29; + +/// Bits one halving pass shifts out. 10⁹ = 2⁹·5⁹, so a limb's remainder +/// below 2⁹ passes to the next limb as exactly `remainder × 10⁹ / 2⁹`, and +/// a pass adds at most one limb below the last. +const HALVE_STEP: u32 = 9; + +/// Limbs kept free in front of an expansion: rounding at the digit just +/// above it materializes one limb, and a carry out of the top one more. +const HEADROOM: usize = 2; + +/// Decimal digits an integer of `bits` bits can have: `⌊bits·log₁₀2⌋ + 1`, +/// by 30103/100000, which exceeds log₁₀2. +const fn decimal_digits(bits: u32) -> usize { + bits as usize * 30103 / 100_000 + 1 +} + +/// Limbs a mantissa occupies before it is scaled: any `u128`. +const MANTISSA_LIMBS: usize = decimal_digits(u128::BITS).div_ceil(LIMB_DIGITS as usize); + +/// The limbs a format's widest expansion needs: its largest integer part +/// (`mantissa_bits + max_exponent` bits, which doubling builds whole), or its +/// longest fraction (one digit per binary place below the point, and the +/// window's one limb more) beside the integer part a fraction can have; and +/// the rounding headroom. +const fn capacity(mantissa_bits: u32, max_exponent: i32, min_exponent: i32) -> usize { + let whole = decimal_digits(mantissa_bits + max_exponent as u32).div_ceil(LIMB_DIGITS as usize); + let fraction = (min_exponent.unsigned_abs() as usize).div_ceil(LIMB_DIGITS as usize) + + 1 + + decimal_digits(mantissa_bits).div_ceil(LIMB_DIGITS as usize); + (if whole > fraction { whole } else { fraction }) + HEADROOM +} + +const fn max(a: usize, b: usize) -> usize { + if a > b { a } else { b } +} + +/// A `double`'s expansion: 125 limbs. +pub(super) const DOUBLE_LIMBS: usize = capacity( + F64_FRACTION + 1, + (2 * F64_BIAS) - F64_BIAS - F64_FRACTION as i32, + 1 - F64_BIAS - F64_FRACTION as i32, +); + +/// A `long double`'s expansion, in either format: 1840 limbs. +pub(super) const LONG_DOUBLE_LIMBS: usize = max( + capacity( + X87_FRACTION + 1, + (2 * X87_BIAS) - X87_BIAS - X87_FRACTION as i32, + 1 - X87_BIAS - X87_FRACTION as i32, + ), + capacity( + QUAD_FRACTION + 1, + (2 * QUAD_BIAS) - QUAD_BIAS - QUAD_FRACTION as i32, + 1 - QUAD_BIAS - QUAD_FRACTION as i32, + ), +); + +/// The decimal expansion of a finite nonzero value, most significant limb +/// first: `limbs[start..start + len]`, the first of which is `10^(9·top)`'s. +/// Digits below the last stored limb are zero, or — when `sticky` is set — +/// include a nonzero one the window let go. +struct Decimal { + limbs: [u32; N], + start: usize, + len: usize, + top: i32, + sticky: bool, +} + +impl Decimal { + /// `mantissa × 2^exponent` (`mantissa` nonzero), exact in every limb whose + /// base-10⁹ exponent is at least `cut`. + fn new(mantissa: u128, exponent: i32, cut: i64) -> Self { + let mut low_first = [0u32; MANTISSA_LIMBS]; + let mut count = 0; + let mut rest = mantissa; + while rest != 0 { + low_first[count] = (rest % u128::from(BASE)) as u32; + rest /= u128::from(BASE); + count += 1; + } + // Doubling grows toward the front and halving toward the back. + let start = if exponent > 0 { N - count } else { HEADROOM }; + let mut decimal = Self { + limbs: [0; N], + start, + len: count, + top: count as i32 - 1, + sticky: false, + }; + for (slot, &limb) in decimal.limbs[start..start + count] + .iter_mut() + .zip(low_first[..count].iter().rev()) + { + *slot = limb; + } + match exponent.cmp(&0) { + Ordering::Greater => decimal.double(exponent.unsigned_abs()), + Ordering::Less => decimal.halve(exponent.unsigned_abs(), cut), + Ordering::Equal => {} + } + decimal + } + + fn double(&mut self, mut shift: u32) { + while shift > 0 { + let step = shift.min(DOUBLE_STEP); + let mut carry = 0u64; + for limb in self.limbs[self.start..self.start + self.len] + .iter_mut() + .rev() + { + let wide = (u64::from(*limb) << step) + carry; + *limb = (wide % u64::from(BASE)) as u32; + carry = wide / u64::from(BASE); + } + if carry != 0 { + self.start -= 1; + self.len += 1; + self.top += 1; + self.limbs[self.start] = carry as u32; + } + shift -= step; + } + } + + fn halve(&mut self, mut shift: u32, cut: i64) { + self.fold_below(cut); + while shift > 0 && self.len > 0 { + let step = shift.min(HALVE_STEP); + let mask = (1u32 << step) - 1; + let share = BASE >> step; + let mut carry = 0u32; + for limb in &mut self.limbs[self.start..self.start + self.len] { + let rest = *limb & mask; + *limb = (*limb >> step) + carry; + carry = share * rest; + } + // Only the leading limb can empty in a pass: the one after it + // gains the leading limb's whole remainder. + if self.limbs[self.start] == 0 { + self.start += 1; + self.len -= 1; + self.top -= 1; + } + if carry != 0 { + if i64::from(self.top) - self.len as i64 >= cut { + self.limbs[self.start + self.len] = carry; + self.len += 1; + } else { + self.sticky = true; + } + } + shift -= step; + } + } + + /// Fold every limb below the base-10⁹ exponent `cut` into `sticky`. + fn fold_below(&mut self, cut: i64) { + let keep = (i64::from(self.top) - cut + 1).clamp(0, self.len as i64) as usize; + let end = self.start + self.len; + if self.limbs[self.start + keep..end] + .iter() + .any(|&limb| limb != 0) + { + self.sticky = true; + } + self.len = keep; + } + + fn is_zero(&self) -> bool { + self.len == 0 + } + + /// The position of the leading digit: `p` for the digit worth `10^p`. + /// The expansion is nonzero. + fn leading(&self) -> i64 { + let first = self.limbs[self.start]; + let digits = POW10 + .iter() + .position(|&power| first < power) + .unwrap_or(POW10.len()); + LIMB_DIGITS * i64::from(self.top) + digits as i64 - 1 + } + + /// The digit at `position`. + #[cfg(test)] + fn digit(&self, position: i64) -> u32 { + let back = i64::from(self.top) - position.div_euclid(LIMB_DIGITS); + if !(0..self.len as i64).contains(&back) { + return 0; + } + let limb = self.limbs[self.start + back as usize]; + limb / POW10[position.rem_euclid(LIMB_DIGITS) as usize] % 10 + } + + /// Keep the digits at `position` and above, rounded to nearest, ties to + /// even, on everything below. + fn round(&mut self, position: i64) { + let below_everything = self.is_zero() || position - 1 > self.leading(); + let limb = position.div_euclid(LIMB_DIGITS); + let lowest = i64::from(self.top) - self.len as i64 + 1; + if below_everything { + // Under a tenth of the last kept digit's unit: zero. An empty + // expansion here had every digit folded below its window, which + // lies under the digit after `position`. + self.len = 0; + self.sticky = false; + return; + } + if limb < lowest { + // Every stored digit is kept, and what lies below them is under a + // tenth of the last kept digit's unit (the window reaches below + // `position`'s limb whenever anything was folded). + self.sticky = false; + return; + } + if limb > i64::from(self.top) { + // `position - 1` is the top digit of the leading limb, so the + // kept digit is the first of the limb above: materialize it. + self.start -= 1; + self.len += 1; + self.top += 1; + self.limbs[self.start] = 0; + } + let at = self.start + (i64::from(self.top) - limb) as usize; + let end = self.start + self.len; + let k = position.rem_euclid(LIMB_DIGITS) as usize; + let (round_digit, rest_nonzero) = if k > 0 { + let under = self.limbs[at] % POW10[k]; + ( + under / POW10[k - 1], + under % POW10[k - 1] != 0 || self.limbs[at + 1..end].iter().any(|&l| l != 0), + ) + } else if at + 1 < end { + let next = self.limbs[at + 1]; + let top_digit = POW10[LIMB_DIGITS as usize - 1]; + ( + next / top_digit, + next % top_digit != 0 || self.limbs[at + 2..end].iter().any(|&l| l != 0), + ) + } else { + (0, false) + }; + let rest_nonzero = rest_nonzero || self.sticky; + self.limbs[at] -= self.limbs[at] % POW10[k]; + self.len = at + 1 - self.start; + self.sticky = false; + let odd = self.limbs[at] / POW10[k] % 2 == 1; + if round_digit > 5 || (round_digit == 5 && (rest_nonzero || odd)) { + let mut carry_at = at; + self.limbs[carry_at] += POW10[k]; + while self.limbs[carry_at] >= BASE { + self.limbs[carry_at] -= BASE; + if carry_at == self.start { + self.start -= 1; + self.len += 1; + self.top += 1; + self.limbs[self.start] = 0; + } + carry_at -= 1; + self.limbs[carry_at] += 1; + } + } + while self.len > 0 && self.limbs[self.start] == 0 { + self.start += 1; + self.len -= 1; + self.top -= 1; + } + } + + /// The position of the lowest nonzero stored digit. + fn lowest(&self) -> Option { + let stored = &self.limbs[self.start..self.start + self.len]; + let back = stored.iter().rposition(|&limb| limb != 0)?; + let limb = stored[back]; + let zeros = POW10[1..] + .iter() + .take_while(|&&power| limb % power == 0) + .count(); + Some(LIMB_DIGITS * (i64::from(self.top) - back as i64) + zeros as i64) + } + + /// Write the digits at positions `from` down to `to`, inclusive. + fn write(&self, out: &mut Out<'_, S>, from: i64, to: i64) { + if from < to { + return; + } + let mut position = from; + let (high, low) = if self.is_zero() { + (i64::MIN, i64::MAX) + } else { + ( + LIMB_DIGITS * i64::from(self.top) + LIMB_DIGITS - 1, + LIMB_DIGITS * (i64::from(self.top) - self.len as i64 + 1), + ) + }; + if position > high { + let zeros = position - high.max(to - 1); + out.pad(&ZEROS, zeros as usize); + position -= zeros; + } + while position >= to && position >= low { + let limb = position.div_euclid(LIMB_DIGITS); + let at = self.start + (i64::from(self.top) - limb) as usize; + let mut text = [b'0'; LIMB_DIGITS as usize]; + let mut value = self.limbs[at]; + for byte in text.iter_mut().rev() { + *byte = b'0' + (value % 10) as u8; + value /= 10; + } + let hi = position.rem_euclid(LIMB_DIGITS) as usize; + let lo = if to.div_euclid(LIMB_DIGITS) == limb { + to.rem_euclid(LIMB_DIGITS) as usize + } else { + 0 + }; + let last = LIMB_DIGITS as usize - 1; + out.put(&text[last - hi..=last - lo]); + position -= (hi - lo + 1) as i64; + } + if position >= to { + out.pad(&ZEROS, (position - to + 1) as usize); + } + } +} + +/// Write the digits of `digits` (all zeros for a zero) from `from` down to +/// `to`. +fn write_digits( + out: &mut Out<'_, S>, + digits: Option<&Decimal>, + from: i64, + to: i64, +) { + match digits { + Some(digits) => digits.write(out, from, to), + None if from >= to => out.pad(&ZEROS, (from - to + 1) as usize), + None => {} + } +} + +/// The lowest base-10⁹ limb a rounding at `position` must read exactly: the +/// one holding the digit after it, and one more. Nothing below the +/// expansion's last digit is ever needed: a fraction of `-exponent` binary +/// places has exactly that many decimal ones. +fn window(position: i64, exponent: i32) -> i64 { + let cut = (position - 1).div_euclid(LIMB_DIGITS) - 1; + cut.max(i64::from(exponent).div_euclid(LIMB_DIGITS) - 1) +} + +/// log₁₀2 = 0.30102999566398…, truncated to eleven places: below the truth, +/// by less than 10⁻¹¹. +const LOG10_2_NUMERATOR: i64 = 30_102_999_566; +const LOG10_2_DENOMINATOR: i64 = 100_000_000_000; + +/// A position at or below that of `mantissa × 2^exponent`'s leading digit. +/// The value is at least `2^power` for `power = bits − 1 + exponent`, whose +/// leading digit sits at `⌊power·log₁₀2⌋`; the truncated logarithm moves +/// that floor by at most one over every exponent a `long double` has, and +/// one digit of slack absorbs it. +fn leading_floor(mantissa: u128, exponent: i32) -> i64 { + let power = i64::from(u128::BITS - mantissa.leading_zeros()) - 1 + i64::from(exponent); + (power * LOG10_2_NUMERATOR).div_euclid(LOG10_2_DENOMINATOR) - 1 +} + +/// `spec`'s precision for `e`, `f` and `g`. +fn precision(spec: Spec) -> i64 { + spec.precision + .map_or(DEFAULT_PRECISION, |precision| precision as i64) +} + +/// Format `value` by `conversion` (`e E f F g G a A`) under `spec`, its +/// expansion held in at most `N` limbs. Never inlined into the engine: a +/// `long double`'s limbs are 7 KiB of stack, which only its conversions pay. +#[inline(never)] +pub(super) fn convert( + out: &mut Out<'_, S>, + spec: Spec, + conversion: u8, + value: Float, +) { + let upper = conversion.is_ascii_uppercase(); + let sign: &[u8] = if value.negative { + b"-" + } else if spec.plus { + b"+" + } else if spec.space { + b" " + } else { + b"" + }; + let (mantissa, exponent) = match value.class { + Class::Infinite => return special(out, spec, sign, if upper { b"INF" } else { b"inf" }), + Class::Nan => return special(out, spec, sign, if upper { b"NAN" } else { b"nan" }), + Class::Finite { mantissa, exponent } => (mantissa, exponent), + }; + match conversion.to_ascii_lowercase() { + b'a' => hexadecimal(out, spec, sign, upper, mantissa, exponent), + b'f' => fixed::(out, spec, sign, mantissa, exponent), + b'e' => scientific::(out, spec, sign, upper, mantissa, exponent), + _ => general::(out, spec, sign, upper, mantissa, exponent), + } +} + +/// Lay out a number of `body` bytes after `sign` and `prefix` in `spec`'s +/// field: spaces before it, or after it under `-`, or zeros between the +/// prefix and the body under `0` (which `-` overrides). +fn field( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + prefix: &[u8], + body: usize, + write: impl FnOnce(&mut Out<'_, S>), +) { + let fill = spec.width.saturating_sub(sign.len() + prefix.len() + body); + let zeros = spec.zero && !spec.left; + if !spec.left && !zeros { + out.pad(&SPACES, fill); + } + out.put(sign); + out.put(prefix); + if zeros { + out.pad(&ZEROS, fill); + } + write(out); + if spec.left { + out.pad(&SPACES, fill); + } +} + +/// An infinity or a NaN: a word, which the `0` flag never pads. +fn special(out: &mut Out<'_, S>, spec: Spec, sign: &[u8], word: &[u8; 3]) { + let spec = Spec { + zero: false, + ..spec + }; + field(out, spec, sign, b"", word.len(), |out| out.put(word)); +} + +/// `%f`: the digits down to the precision's last. +fn fixed( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + mantissa: u128, + exponent: i32, +) { + let precision = precision(spec); + let digits = (mantissa != 0).then(|| { + let mut digits = Decimal::::new(mantissa, exponent, window(-precision, exponent)); + digits.round(-precision); + digits + }); + fixed_layout(out, spec, sign, digits.as_ref(), precision); +} + +/// `%e`: one digit, the precision's after the point, and the exponent. +fn scientific( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + upper: bool, + mantissa: u128, + exponent: i32, +) { + let precision = precision(spec); + let (digits, leading) = significant::(mantissa, exponent, precision); + exponent_layout(out, spec, sign, upper, digits.as_ref(), leading, precision); +} + +/// `%g`: `%e`'s rounding to the precision's significant digits, laid out as +/// `%f` when the exponent is at least −4 and below the precision, without +/// trailing fraction zeros unless `#` keeps them (C11 7.21.6.1p8). +fn general( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + upper: bool, + mantissa: u128, + exponent: i32, +) { + let significant_digits = match spec.precision { + Some(0) => 1, + Some(precision) => precision as i64, + None => DEFAULT_PRECISION, + }; + let (digits, leading) = significant::(mantissa, exponent, significant_digits - 1); + let lowest = digits.as_ref().and_then(Decimal::lowest); + if significant_digits > leading && leading >= -4 { + let precision = significant_digits - 1 - leading; + let fraction = if spec.alternate { + precision + } else { + lowest.map_or(0, |low| (-low).clamp(0, precision)) + }; + fixed_layout(out, spec, sign, digits.as_ref(), fraction); + } else { + let precision = significant_digits - 1; + let fraction = if spec.alternate { + precision + } else { + lowest.map_or(0, |low| (leading - low).clamp(0, precision)) + }; + exponent_layout(out, spec, sign, upper, digits.as_ref(), leading, fraction); + } +} + +/// The expansion rounded to `precision` digits after its leading one, and +/// that leading digit's position (`0` for a zero). The window reaches from a +/// bound below the leading digit, so the expansion is never empty. +fn significant( + mantissa: u128, + exponent: i32, + precision: i64, +) -> (Option>, i64) { + if mantissa == 0 { + return (None, 0); + } + let floor = leading_floor(mantissa, exponent); + let mut digits = Decimal::::new(mantissa, exponent, window(floor - precision, exponent)); + let leading = digits.leading(); + digits.round(leading - precision); + // A carry can open a new leading digit: 9.96 at one place is 1.0e1. + let leading = digits.leading(); + (Some(digits), leading) +} + +/// `[-]d…d[.d…d]`: the integer digits (one `0` when there are none), then the +/// point and `fraction` digits when there are any or `#` asks for the point. +fn fixed_layout( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + digits: Option<&Decimal>, + fraction: i64, +) { + let leading = digits + .filter(|digits| !digits.is_zero()) + .map(Decimal::leading); + let whole = leading.map_or(1, |leading| leading.max(0) + 1); + let point = fraction > 0 || spec.alternate; + let body = whole as usize + usize::from(point) + fraction as usize; + field(out, spec, sign, b"", body, |out| { + write_digits(out, digits, whole - 1, 0); + if point { + out.put(b"."); + } + write_digits(out, digits, -1, -fraction); + }); +} + +/// `[-]d[.d…d]e±dd`: the leading digit, then the point and `fraction` digits +/// when there are any or `#` asks for the point, then the exponent in at +/// least two digits. +fn exponent_layout( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + upper: bool, + digits: Option<&Decimal>, + leading: i64, + fraction: i64, +) { + let point = fraction > 0 || spec.alternate; + let mut text = [0u8; EXPONENT_TEXT]; + let power = exponent_text(&mut text, if upper { b'E' } else { b'e' }, leading, 2); + let body = 1 + usize::from(point) + fraction as usize + power.len(); + field(out, spec, sign, b"", body, |out| { + write_digits(out, digits, leading, leading); + if point { + out.put(b"."); + } + write_digits(out, digits, leading - 1, leading - fraction); + out.put(power); + }); +} + +/// The letter, the sign and up to six digits: a binary exponent of a +/// `long double` subnormal (−16494) is the widest. +const EXPONENT_TEXT: usize = 8; + +/// `letter`, a sign, and `value`'s decimal digits, at least `min_digits`. +fn exponent_text( + text: &mut [u8; EXPONENT_TEXT], + letter: u8, + value: i64, + min_digits: usize, +) -> &[u8] { + let mut digits = [b'0'; EXPONENT_TEXT - 2]; + let mut at = digits.len(); + let mut rest = value.unsigned_abs(); + while rest != 0 || digits.len() - at < min_digits { + at -= 1; + digits[at] = b'0' + (rest % 10) as u8; + rest /= 10; + } + let count = digits.len() - at; + text[0] = letter; + text[1] = if value < 0 { b'-' } else { b'+' }; + text[2..2 + count].copy_from_slice(&digits[at..]); + &text[..2 + count] +} + +/// `%a`: `0x1.h…hp±d`, the value normalized to a leading `1` and every +/// fraction digit it has — or, with a precision, that many, rounded to +/// nearest-even. +fn hexadecimal( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + upper: bool, + mantissa: u128, + exponent: i32, +) { + let alphabet: &[u8; 16] = if upper { + b"0123456789ABCDEF" + } else { + b"0123456789abcdef" + }; + let prefix: &[u8] = if upper { b"0X" } else { b"0x" }; + let (mut lead, mut fraction, mut count, power) = if mantissa == 0 { + (0u8, 0u128, 0usize, 0i64) + } else { + // The bits below the leading one, padded out to whole hex digits, + // without the trailing zero digits a precision did not ask for. + let tail = u128::BITS - 1 - mantissa.leading_zeros(); + let digits = tail.div_ceil(4); + let mut fraction = (mantissa ^ (1 << tail)) << (4 * digits - tail); + let mut count = digits as usize; + while count > 0 && fraction & 0xf == 0 { + fraction >>= 4; + count -= 1; + } + (1u8, fraction, count, i64::from(exponent) + i64::from(tail)) + }; + let mut zeros = 0usize; + if let Some(precision) = spec.precision { + if precision < count { + let dropped = 4 * (count - precision) as u32; + let half = 1u128 << (dropped - 1); + let rest = fraction & ((1u128 << dropped) - 1); + fraction >>= dropped; + let odd = if precision == 0 { + lead & 1 == 1 + } else { + fraction & 1 == 1 + }; + if rest > half || (rest == half && odd) { + fraction += 1; + if precision == 0 || fraction >> (4 * precision) != 0 { + lead += 1; + fraction = 0; + } + } + count = precision; + } else { + zeros = precision - count; + } + } + let point = count + zeros > 0 || spec.alternate; + let mut text = [0u8; EXPONENT_TEXT]; + let power = exponent_text(&mut text, if upper { b'P' } else { b'p' }, power, 1); + let body = 1 + usize::from(point) + count + zeros + power.len(); + field(out, spec, sign, prefix, body, |out| { + out.put(&[alphabet[usize::from(lead)]]); + if point { + out.put(b"."); + } + let mut digits = [0u8; HEX_FRACTION_DIGITS]; + for (i, digit) in digits[..count].iter_mut().enumerate() { + *digit = alphabet[((fraction >> (4 * (count - 1 - i))) & 0xf) as usize]; + } + out.put(&digits[..count]); + out.pad(&ZEROS, zeros); + out.put(power); + }); +} + +/// Hex digits below the leading one any mantissa can have: a `u128`'s. +const HEX_FRACTION_DIGITS: usize = (u128::BITS as usize - 1).div_ceil(4); + +#[cfg(test)] +mod tests; diff --git a/native/crates/c-shim/src/format/float/tests.rs b/native/crates/c-shim/src/format/float/tests.rs new file mode 100644 index 0000000..db777af --- /dev/null +++ b/native/crates/c-shim/src/format/float/tests.rs @@ -0,0 +1,620 @@ +//! The float conversions against the host C library's own. glibc's +//! `snprintf` is the reference for `double`, through every flag, width and +//! precision. For the two `long double` formats it is glibc's `strfroml` +//! (x87, on an x86_64 host) and `strfromf128` (binary128), reached through +//! trampolines: Rust has no x87 type to pass, nor a binary128 one that +//! crosses into C. Where musl and glibc disagree — `%a` of a subnormal, of an +//! x87 value, and x87's invalid encodings — the cases are musl's spellings, +//! written out. + +extern crate std; + +use core::ffi::{c_char, c_int}; +use std::ffi::CString; +use std::string::String; +use std::vec::Vec; + +use super::super::{Args, Length, Sink, format}; +use super::{DOUBLE_LIMBS, Decimal, Float, LONG_DOUBLE_LIMBS, window}; + +/// The one float argument a test format converts. +enum Value { + Double(f64), + Long(Float), +} + +struct One(Option); + +impl Args for One { + fn signed(&mut self, _: Length) -> i64 { + unreachable!("a float test formats one float") + } + fn unsigned(&mut self, _: Length) -> u64 { + unreachable!("a float test formats one float") + } + fn int(&mut self) -> c_int { + unreachable!("a float test formats one float") + } + fn pointer(&mut self) -> usize { + unreachable!("a float test formats one float") + } + fn double(&mut self) -> f64 { + match self.0.take() { + Some(Value::Double(value)) => value, + _ => panic!("the format read a double it was not given"), + } + } + fn long_double(&mut self) -> Float { + match self.0.take() { + Some(Value::Long(value)) => value, + _ => panic!("the format read a long double it was not given"), + } + } +} + +struct Collect(Vec); + +impl Sink for Collect { + fn put(&mut self, bytes: &[u8]) { + self.0.extend_from_slice(bytes); + } +} + +fn ours(spec: &str, value: Value) -> String { + let spec = CString::new(spec).unwrap(); + let mut args = One(Some(value)); + let mut sink = Collect(Vec::new()); + // SAFETY: NUL-terminated, and the one conversion matches the argument. + let count = unsafe { format(&mut sink, spec.as_ptr().cast(), &mut args) }; + assert!(args.0.is_none(), "the format did not read its argument"); + assert_eq!(count, sink.0.len(), "the count is the output's length"); + String::from_utf8(sink.0).unwrap() +} + +/// Room for the widest `%f` of a `long double` with a long precision. +const HOST_BUFFER: usize = 1 << 14; + +unsafe extern "C" { + fn snprintf(buffer: *mut c_char, size: usize, format: *const c_char, ...) -> c_int; +} + +fn host_double(spec: &str, value: f64) -> String { + let spec = CString::new(spec).unwrap(); + let mut buffer = std::vec![0u8; HOST_BUFFER]; + // SAFETY: a buffer of its stated size and a matching double. + let written = unsafe { + snprintf( + buffer.as_mut_ptr().cast(), + buffer.len(), + spec.as_ptr(), + value, + ) + }; + let written = usize::try_from(written).unwrap(); + assert!(written < buffer.len()); + String::from_utf8(buffer[..written].to_vec()).unwrap() +} + +/// xorshift64*: fixed seeds, so a failure reproduces. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) + } +} + +/// Values the conversions turn on: zeros and ones, exact ties at every +/// place, the powers of ten either side of a rounding carry, the subnormal +/// and normal edges, the largest finite value, and the non-finite ones. +fn edge_doubles() -> Vec { + let mut values = std::vec![ + 0.0, + -0.0, + 1.0, + -1.0, + 0.5, + 1.5, + 2.5, + 3.5, + -2.5, + 0.125, + 0.375, + 0.625, + 0.05, + 0.15, + 0.25, + 0.35, + 9.5, + 10.5, + 1e-7, + 1e-5, + 1e-4, + 0.000099999995, + 9.9999995, + 99999.95, + 999999.5, + 123456.0, + 1234567.0, + 1e15, + 1e16, + 1e17, + 1e21, + 1e22, + 1e23, + 1e100, + 9007199254740992.0, + 9007199254740994.0, + 1.0 / 3.0, + 2.0 / 3.0, + 0.1, + 0.2, + 0.3, + f64::EPSILON, + f64::MIN_POSITIVE, + f64::from_bits(1), + f64::from_bits(3), + f64::from_bits(0x000f_ffff_ffff_ffff), + f64::MAX, + -f64::MAX, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + -f64::NAN, + ]; + for exponent in -30..30 { + values.push(10f64.powi(exponent)); + values.push(5.0 * 10f64.powi(exponent)); + } + values +} + +/// Random bit patterns (every exponent, so mostly huge and tiny values) +/// and random values of everyday size. +fn random_doubles(count: usize) -> Vec { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let mut values = Vec::with_capacity(2 * count); + for _ in 0..count { + values.push(f64::from_bits(rng.next())); + let everyday = (rng.next() & 0x800f_ffff_ffff_ffff) | ((1023 - 40 + rng.next() % 80) << 52); + values.push(f64::from_bits(everyday)); + } + values +} + +const DOUBLE_SPECS: &[&str] = &[ + "%e", + "%.0e", + "%.1e", + "%.3e", + "%.16e", + "%.17e", + "%.40e", + "%E", + "%#.0e", + "%+e", + "% .2e", + "[%15.3e]", + "[%-15.3e]", + "%015.3e", + "%f", + "%.0f", + "%.1f", + "%.3f", + "%.20f", + "%.1100f", + "%F", + "%#.0f", + "%+08.2f", + "[%-12.4f]", + "%g", + "%.0g", + "%.1g", + "%.2g", + "%.10g", + "%.17g", + "%.40g", + "%G", + "%+g", + "%012g", + "[%-12g]", + "%a", + "%.0a", + "%.1a", + "%.3a", + "%.13a", + "%.20a", + "%A", + "%#.0a", + "%+a", + "%020a", + "[%-20a]", +]; + +/// `%a` is one conversion musl and glibc spell differently for a `double`: +/// glibc keeps a subnormal's leading `0`. +fn hex_differs(spec: &str, value: f64) -> bool { + spec.contains(['a', 'A']) && value != 0.0 && value.abs() < f64::MIN_POSITIVE +} + +/// `%#g` is the other: when the rounding carries the exponent up to the +/// precision, glibc drops the trailing zeros `#` keeps (musl keeps them, as +/// C 7.21.6.1p8 says). The reference is built from glibc's `%e` and `%#f` or +/// `%#e`, as that paragraph defines `%g`. +const ALTERNATE_GENERAL: &[(&str, Option)] = + &[("%#g", None), ("%#.3g", Some(3)), ("%#.0g", Some(0))]; + +fn host_alternate_general(precision: Option, value: f64) -> String { + if !value.is_finite() { + return host_double("%#g", value); + } + let significant = match precision { + Some(0) => 1, + Some(precision) => precision, + None => 6, + }; + let scientific = host_double(&std::format!("%.{}e", significant - 1), value); + let exponent: i64 = scientific.rsplit_once('e').unwrap().1.parse().unwrap(); + if significant > exponent && exponent >= -4 { + host_double(&std::format!("%#.{}f", significant - 1 - exponent), value) + } else { + host_double(&std::format!("%#.{}e", significant - 1), value) + } +} + +#[test] +fn doubles_print_as_the_host_prints_them() { + let mut values = edge_doubles(); + values.extend(random_doubles(1500)); + for &value in &values { + for &spec in DOUBLE_SPECS { + if hex_differs(spec, value) { + continue; + } + assert_eq!( + ours(spec, Value::Double(value)), + host_double(spec, value), + "{spec} of {value:e} ({:#018x})", + value.to_bits() + ); + } + for &(spec, precision) in ALTERNATE_GENERAL { + assert_eq!( + ours(spec, Value::Double(value)), + host_alternate_general(precision, value), + "{spec} of {value:e} ({:#018x})", + value.to_bits() + ); + } + } +} + +#[test] +fn an_alternate_general_keeps_its_zeros_through_a_carry_as_musl_does() { + let cases: &[(&str, f64, &str)] = &[ + ("%#g", 999999.5, "1.00000e+06"), + ("%#.3g", 999.5, "1.00e+03"), + ("%#.2g", 99.5, "1.0e+02"), + ("%#.1g", 9.5, "1.e+01"), + ("%#.0g", 9.5, "1.e+01"), + ("%#.6g", 0.0009999995, "0.00100000"), + ]; + for &(spec, value, expected) in cases { + assert_eq!( + ours(spec, Value::Double(value)), + expected, + "{spec} of {value:e}" + ); + } +} + +#[test] +fn a_subnormal_double_in_hex_is_normalized_as_musl_spells_it() { + let cases: &[(&str, u64, &str)] = &[ + ("%a", 1, "0x1p-1074"), + ("%a", 3, "0x1.8p-1073"), + ("%.0a", 3, "0x2p-1073"), + ("%a", 0x000f_ffff_ffff_ffff, "0x1.ffffffffffffep-1023"), + ("%.1A", 0x0008_0000_0000_0001, "0X1.0P-1023"), + ]; + for &(spec, bits, expected) in cases { + assert_eq!( + ours(spec, Value::Double(f64::from_bits(bits))), + expected, + "{spec} of {bits:#x}" + ); + } +} + +#[test] +fn every_format_holds_its_widest_expansion() { + // The widest integer part, built whole by doubling. + let max = Decimal::::new((1 << 53) - 1, 971, 0); + assert_eq!(max.leading(), 308); + // The longest fraction a window can reach, whatever the precision. + let least = Decimal::::new(1, -1074, window(-1_000_000, -1074)); + assert_eq!(least.leading(), -324); + assert_eq!( + (-339..=-324) + .rev() + .map(|p| least.digit(p)) + .collect::>(), + [4, 9, 4, 0, 6, 5, 6, 4, 5, 8, 4, 1, 2, 4, 6, 5] + ); + assert!(!least.sticky, "nothing of the expansion was let go"); + assert_eq!(least.lowest(), Some(-1074)); + let quad_max = Decimal::::new((1 << 113) - 1, 16271, 0); + assert_eq!(quad_max.leading(), 4932); + let quad_least = Decimal::::new(1, -16494, window(-1_000_000, -16494)); + assert_eq!(quad_least.leading(), -4966); + assert_eq!(quad_least.lowest(), Some(-16494)); + let x87_least = Decimal::::new(1, -16445, window(-1_000_000, -16445)); + assert_eq!(x87_least.lowest(), Some(-16445)); +} + +#[test] +fn a_huge_precision_prints_its_zeros_without_storing_them() { + let text = ours("%.100000f", Value::Double(0.5)); + assert_eq!(text.len(), 2 + 100_000); + assert!(text.starts_with("0.5000")); + assert!(text[3..].bytes().all(|b| b == b'0')); +} + +/// The `(precision, conversion)` pairs `strfrom*` accepts, which name no +/// flag or width: those are the engine's own, which the `double` sweep has +/// covered. +const WIDE_SPECS: &[(Option, char)] = &[ + (None, 'e'), + (Some(0), 'e'), + (Some(1), 'e'), + (Some(5), 'e'), + (Some(18), 'e'), + (Some(25), 'e'), + (Some(36), 'e'), + (Some(60), 'E'), + (None, 'f'), + (Some(0), 'f'), + (Some(3), 'f'), + (Some(30), 'f'), + (Some(60), 'F'), + (None, 'g'), + (Some(0), 'g'), + (Some(1), 'g'), + (Some(5), 'g'), + (Some(18), 'g'), + (Some(21), 'g'), + (Some(36), 'G'), + (Some(50), 'g'), +]; + +fn wide_spec(precision: Option, conversion: char, long: bool) -> String { + let length = if long { "L" } else { "" }; + match precision { + Some(precision) => std::format!("%.{precision}{length}{conversion}"), + None => std::format!("%{length}{conversion}"), + } +} + +/// Random `long double` encodings of `format`: every exponent, every +/// everyday one, and the subnormals; each a valid encoding. +fn random_wide(rng: &mut Rng, x87: bool) -> u128 { + let sign = u128::from(rng.next() >> 63) << if x87 { 79 } else { 127 }; + let roll = rng.next() % 8; + let biased: u128 = match roll { + 0 => 0, + 1..=3 => u128::from(16383 - 200 + rng.next() % 400), + _ => u128::from(1 + rng.next() % 0x7ffd), + }; + let fraction = (u128::from(rng.next()) << 64) | u128::from(rng.next()); + if x87 { + let integer = u128::from(biased != 0) << 63; + sign | biased << 64 | integer | (fraction & ((1 << 63) - 1)) + } else { + sign | biased << 112 | (fraction & ((1 << 112) - 1)) + } +} + +#[cfg(target_arch = "x86_64")] +mod x86_64_host { + use super::*; + + unsafe extern "C" { + fn strfroml(); + fn strfromf128(); + } + + /// glibc's `strfroml(buffer, size, format, value)`: the x87 value rides + /// the stack, as System V passes every `long double`. + fn host_x87(spec: &str, bits: u128) -> String { + let spec = CString::new(spec).unwrap(); + let mut buffer = std::vec![0u8; HOST_BUFFER]; + let value = bits.to_le_bytes(); + let written: i32; + // SAFETY: the stack stays 16-aligned for the call (`asm!` enters + // aligned and the frame is sixteen bytes), the arguments are live, + // and every caller-saved register is declared clobbered. + unsafe { + core::arch::asm!( + "sub rsp, 16", + "mov rax, qword ptr [r10]", + "mov qword ptr [rsp], rax", + "mov rax, qword ptr [r10 + 8]", + "mov qword ptr [rsp + 8], rax", + "call {f}", + "add rsp, 16", + f = sym strfroml, + in("r10") value.as_ptr(), + in("rdi") buffer.as_mut_ptr(), + in("rsi") buffer.len(), + in("rdx") spec.as_ptr(), + lateout("eax") written, + clobber_abi("C"), + ); + } + let written = usize::try_from(written).unwrap(); + assert!(written < buffer.len()); + String::from_utf8(buffer[..written].to_vec()).unwrap() + } + + /// glibc's `strfromf128(buffer, size, format, value)`: `_Float128` + /// rides `xmm0`. + fn host_quad(spec: &str, bits: u128) -> String { + let spec = CString::new(spec).unwrap(); + let mut buffer = std::vec![0u8; HOST_BUFFER]; + let value = bits.to_le_bytes(); + let written: i32; + // SAFETY: as `host_x87`; the value is loaded into `xmm0` from live + // bytes, and `xmm0` is caller-saved. + unsafe { + core::arch::asm!( + "movdqu xmm0, xmmword ptr [r10]", + "call {f}", + f = sym strfromf128, + in("r10") value.as_ptr(), + in("rdi") buffer.as_mut_ptr(), + in("rsi") buffer.len(), + in("rdx") spec.as_ptr(), + lateout("eax") written, + clobber_abi("C"), + ); + } + let written = usize::try_from(written).unwrap(); + assert!(written < buffer.len()); + String::from_utf8(buffer[..written].to_vec()).unwrap() + } + + fn edge_wide(x87: bool) -> Vec { + if x87 { + std::vec![ + 0, + 1 << 79, + 0x3fff_8000_0000_0000_0000, + 0x3ffb_cccc_cccc_cccc_cccd, + 0x4000_a000_0000_0000_0000, + 0x7ffe_ffff_ffff_ffff_ffff, + 0x0001_8000_0000_0000_0000, + 0x0000_0000_0000_0000_0001, + 0x0000_7fff_ffff_ffff_ffff, + 0x7fff_8000_0000_0000_0000, + 0xffff_8000_0000_0000_0000, + 0x7fff_c000_0000_0000_0000, + ] + } else { + std::vec![ + 0, + 1 << 127, + 0x3fff_0000_0000_0000_0000_0000_0000_0000, + 0x3ffb_9999_9999_9999_9999_9999_9999_999a, + 0x7ffe_ffff_ffff_ffff_ffff_ffff_ffff_ffff, + 0x0001_0000_0000_0000_0000_0000_0000_0000, + 0x0000_0000_0000_0000_0000_0000_0000_0001, + 0x7fff_0000_0000_0000_0000_0000_0000_0000, + 0xffff_0000_0000_0000_0000_0000_0000_0000, + 0x7fff_8000_0000_0000_0000_0000_0000_0000, + ] + } + } + + fn sweep(x87: bool) { + let mut rng = Rng(if x87 { + 0x0123_4567_89ab_cdef + } else { + 0xfedc_ba98_7654_3210 + }); + let mut values = edge_wide(x87); + values.extend((0..400).map(|_| random_wide(&mut rng, x87))); + for &bits in &values { + let value = if x87 { + Float::from_x87(bits) + } else { + Float::from_quad(bits) + }; + for &(precision, conversion) in WIDE_SPECS { + let host = if x87 { + host_x87(&wide_spec(precision, conversion, false), bits) + } else { + host_quad(&wide_spec(precision, conversion, false), bits) + }; + let spec = wide_spec(precision, conversion, true); + assert_eq!( + ours(&spec, Value::Long(value)), + host, + "{spec} of {bits:#034x}" + ); + } + // glibc and musl agree on `%a` of a normal binary128 value only. + let subnormal = (bits >> 112) & 0x7fff == 0 && bits << 1 != 0; + if !x87 && !subnormal { + for spec in ["%a", "%.0a", "%.5a", "%.28a", "%.40A"] { + let long = spec.replace('a', "La").replace('A', "LA"); + assert_eq!( + ours(&long, Value::Long(value)), + host_quad(spec, bits), + "{long} of {bits:#034x}" + ); + } + } + } + } + + #[test] + fn x87_long_doubles_print_as_the_host_prints_them() { + sweep(true); + } + + #[test] + fn binary128_long_doubles_print_as_the_host_prints_them() { + sweep(false); + } +} + +#[test] +fn an_x87_value_in_hex_is_normalized_as_musl_spells_it() { + let cases: &[(u128, &str)] = &[ + (0x3fff_8000_0000_0000_0000, "0x1p+0"), + (0x3ffb_cccc_cccc_cccc_cccd, "0x1.999999999999999ap-4"), + (0x7ffe_ffff_ffff_ffff_ffff, "0x1.fffffffffffffffep+16383"), + (0x0000_0000_0000_0000_0001, "0x1p-16445"), + (0xbfff_c000_0000_0000_0000, "-0x1.8p+0"), + ]; + for &(bits, expected) in cases { + assert_eq!( + ours("%La", Value::Long(Float::from_x87(bits))), + expected, + "{bits:#x}" + ); + } +} + +#[test] +fn x87_encodings_no_fpu_produces_are_read_as_musl_reads_them() { + let spell = |bits: u128| ours("%Lg", Value::Long(Float::from_x87(bits))); + // An unnormal: a clear integer bit under a nonzero exponent. + assert_eq!(spell(0x3fff_4000_0000_0000_0000), "nan"); + // A pseudo-NaN and a pseudo-infinity: the top exponent, integer bit clear. + assert_eq!(spell(0x7fff_4000_0000_0000_0000), "nan"); + assert_eq!(spell(0xffff_0000_0000_0000_0000), "-inf"); + // A pseudo-denormal is the value its bits spell: the least normal. + let least_normal = ours( + "%.30Le", + Value::Long(Float::from_x87(0x0001_8000_0000_0000_0000)), + ); + assert_eq!( + ours( + "%.30Le", + Value::Long(Float::from_x87(0x0000_8000_0000_0000_0000)) + ), + least_normal + ); + // The padding above the 80 bits is not the value's. + assert_eq!( + ours( + "%Lg", + Value::Long(Float::from_x87(0xdead_0000_3fff_8000_0000_0000_0000)) + ), + "1" + ); +} diff --git a/native/crates/c-shim/src/format_symbols.rs b/native/crates/c-shim/src/format_symbols.rs new file mode 100644 index 0000000..da703e9 --- /dev/null +++ b/native/crates/c-shim/src/format_symbols.rs @@ -0,0 +1,342 @@ +//! The `` writes: the `printf` family over [`crate::format`], and +//! the character and block writes over [`crate::stdio`]'s streams. +//! +//! `snprintf` returns what ISO says — the length the complete output would +//! have had — writing at most `n - 1` bytes and a NUL. A count past `INT_MAX` +//! returns `-1`, C's `EOVERFLOW` case. The stream writes return `EOF` when +//! their stream's write fails, which on `stdout`/`stderr` means no +//! [`StandardSink`](crate::stdio::StandardSink) was installed. + +use core::ffi::{VaList, c_char, c_int, c_long, c_longlong, c_uint, c_ulong, c_ulonglong, c_void}; + +use crate::format::{Args, Length, Sink, format}; +use crate::stdio::{self, File, stdout}; + +const EOF: c_int = -1; + +/// A C `va_list` as the engine's argument source. +struct VaArgs<'a, 'f>(&'a mut VaList<'f>); + +impl Args for VaArgs<'_, '_> { + fn signed(&mut self, length: Length) -> i64 { + // SAFETY (each `next_arg`): the format's conversion names this C + // type, the variadic caller's half of the contract. + unsafe { + match length { + // `char` and `short` arrive promoted to `int`. + Length::Char => i64::from(self.0.next_arg::() as i8), + Length::Short => i64::from(self.0.next_arg::() as i16), + Length::Int => i64::from(self.0.next_arg::()), + Length::Long => self.0.next_arg::(), + // glibc reads `%Ld` as `long long`; so does this. + Length::LongLong | Length::Max | Length::LongDouble => { + self.0.next_arg::() + } + Length::Size | Length::Diff => self.0.next_arg::() as i64, + } + } + } + + fn unsigned(&mut self, length: Length) -> u64 { + // SAFETY: as `signed`. + unsafe { + match length { + Length::Char => u64::from(self.0.next_arg::() as u8), + Length::Short => u64::from(self.0.next_arg::() as u16), + Length::Int => u64::from(self.0.next_arg::()), + Length::Long => self.0.next_arg::(), + Length::LongLong | Length::Max | Length::LongDouble => { + self.0.next_arg::() + } + Length::Size | Length::Diff => self.0.next_arg::() as u64, + } + } + } + + fn int(&mut self) -> c_int { + // SAFETY: as `signed`. + unsafe { self.0.next_arg::() } + } + + fn pointer(&mut self) -> usize { + // SAFETY: as `signed`. + unsafe { self.0.next_arg::<*const c_void>() as usize } + } + + fn double(&mut self) -> f64 { + // SAFETY: as `signed`. + unsafe { self.0.next_arg::() } + } + + fn long_double(&mut self) -> crate::format::Float { + // SAFETY: as `signed`; the conversion's `L` names a `long double`. + unsafe { crate::va_long_double::next(self.0) } + } +} + +/// A C buffer that keeps counting past its end, as `snprintf` must. +struct Bounded { + target: *mut u8, + /// Bytes that fit before the terminator. + room: usize, + stored: usize, +} + +impl Sink for Bounded { + fn put(&mut self, bytes: &[u8]) { + let take = bytes.len().min(self.room - self.stored); + // SAFETY: `stored + take <= room`, inside the caller's buffer. + unsafe { + core::ptr::copy_nonoverlapping(bytes.as_ptr(), self.target.add(self.stored), take) + }; + self.stored += take; + } +} + +/// `vsnprintf` with an explicit room: `None` is `vsprintf`'s unbounded +/// buffer, whose size the C caller vouches for. +/// +/// # Safety +/// C's `vsnprintf` contract. +unsafe fn bounded( + target: *mut c_char, + len: Option, + format_: *const c_char, + args: &mut VaList, +) -> c_int { + let room = match len { + Some(0) => 0, + Some(len) => len - 1, + None => usize::MAX, + }; + let mut out = Bounded { + target: target.cast(), + room, + stored: 0, + }; + // SAFETY: the caller's contract. + let count = unsafe { format(&mut out, format_.cast(), &mut VaArgs(args)) }; + if len != Some(0) { + // SAFETY: `stored <= room`, which left one byte for the NUL. + unsafe { *out.target.add(out.stored) = 0 }; + } + c_int::try_from(count).unwrap_or(-1) +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn vsnprintf( + target: *mut c_char, + len: usize, + format: *const c_char, + mut args: VaList, +) -> c_int { + // SAFETY: the caller's contract is vsnprintf's. + unsafe { bounded(target, Some(len), format, &mut args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn snprintf( + target: *mut c_char, + len: usize, + format: *const c_char, + args: ... +) -> c_int { + // SAFETY: the caller's contract is vsnprintf's. + unsafe { vsnprintf(target, len, format, args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn vsprintf( + target: *mut c_char, + format: *const c_char, + mut args: VaList, +) -> c_int { + // SAFETY: the caller vouches that the buffer holds the output. + unsafe { bounded(target, None, format, &mut args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn sprintf(target: *mut c_char, format: *const c_char, args: ...) -> c_int { + // SAFETY: as `vsprintf`. + unsafe { vsprintf(target, format, args) } +} + +/// A stream write, batched: the engine's runs collect here and reach the +/// stream a buffer at a time. +struct Streamed { + file: *mut File, + buffer: [u8; 256], + held: usize, + failed: bool, +} + +impl Streamed { + fn drain(&mut self) { + // SAFETY: the stream is live for the whole call. + if self.held > 0 && !unsafe { stdio::write(self.file, &self.buffer[..self.held]) } { + self.failed = true; + } + self.held = 0; + } +} + +impl Sink for Streamed { + fn put(&mut self, bytes: &[u8]) { + if self.held + bytes.len() > self.buffer.len() { + self.drain(); + } + if bytes.len() > self.buffer.len() { + // A run bigger than the buffer goes straight through. + // SAFETY: as `drain`. + if !unsafe { stdio::write(self.file, bytes) } { + self.failed = true; + } + return; + } + self.buffer[self.held..self.held + bytes.len()].copy_from_slice(bytes); + self.held += bytes.len(); + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn vfprintf(file: *mut File, format_: *const c_char, mut args: VaList) -> c_int { + // One call, one lock: every batch reaches the stream before another + // thread's call does. + // SAFETY: C's vfprintf contract: a live stream and a matching format. + unsafe { + stdio::locked(file, || { + let mut out = Streamed { + file, + buffer: [0; 256], + held: 0, + failed: false, + }; + let count = format(&mut out, format_.cast(), &mut VaArgs(&mut args)); + out.drain(); + if out.failed { + return EOF; + } + c_int::try_from(count).unwrap_or(-1) + }) + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fprintf(file: *mut File, format: *const c_char, args: ...) -> c_int { + // SAFETY: as `vfprintf`. + unsafe { vfprintf(file, format, args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn vprintf(format: *const c_char, args: VaList) -> c_int { + // SAFETY: as `vfprintf`, on `stdout`. + unsafe { vfprintf(stdout.file(), format, args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn printf(format: *const c_char, args: ...) -> c_int { + // SAFETY: as `vfprintf`, on `stdout`. + unsafe { vfprintf(stdout.file(), format, args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fputc(byte: c_int, file: *mut File) -> c_int { + let byte = byte as u8; + // SAFETY: C's contract: a live stream. + if unsafe { stdio::write(file, &[byte]) } { + c_int::from(byte) + } else { + EOF + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn putc(byte: c_int, file: *mut File) -> c_int { + // SAFETY: as `fputc`. + unsafe { fputc(byte, file) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn putchar(byte: c_int) -> c_int { + // SAFETY: `stdout` is live. + unsafe { fputc(byte, stdout.file()) } +} + +/// The bytes of a C string, without its NUL. +/// +/// # Safety +/// `text` is NUL-terminated. +unsafe fn c_str<'a>(text: *const c_char) -> &'a [u8] { + // SAFETY: the caller's contract. + unsafe { core::ffi::CStr::from_ptr(text) }.to_bytes() +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fputs(text: *const c_char, file: *mut File) -> c_int { + // SAFETY: C's contract: a NUL-terminated string and a live stream. + if unsafe { stdio::write(file, c_str(text)) } { + 0 + } else { + EOF + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn puts(text: *const c_char) -> c_int { + let file = stdout.file(); + // The line and its newline under one lock, as one call. + // SAFETY: as `fputs`, on `stdout`. + let written = unsafe { + stdio::locked(file, || { + stdio::write(file, c_str(text)) && stdio::write(file, b"\n") + }) + }; + if written { 0 } else { EOF } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fwrite( + source: *const c_void, + size: usize, + count: usize, + file: *mut File, +) -> usize { + let Some(total) = size.checked_mul(count) else { + return 0; + }; + if total == 0 { + return 0; + } + // SAFETY: C promises `size * count` readable bytes and a live stream. + let bytes = unsafe { core::slice::from_raw_parts(source.cast::(), total) }; + if unsafe { stdio::write(file, bytes) } { + count + } else { + 0 + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fflush(file: *mut File) -> c_int { + if file.is_null() { + // "Every output stream": the standard pair holds nothing back, and a + // consumer's streams flush on their own close. + return 0; + } + // SAFETY: C's contract: a live stream. + if unsafe { stdio::flush(file) } { + 0 + } else { + EOF + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fclose(file: *mut File) -> c_int { + // SAFETY: C's contract: a live stream, not used again. + if unsafe { stdio::close(file) } { + 0 + } else { + EOF + } +} diff --git a/native/crates/c-shim/src/lib.rs b/native/crates/c-shim/src/lib.rs new file mode 100644 index 0000000..68bb647 --- /dev/null +++ b/native/crates/c-shim/src/lib.rs @@ -0,0 +1,83 @@ +//! The C symbols a vendored C library imports on a kore user target. +//! +//! A C library built by `tools/kore-cc` is freestanding: Zig's musl triple +//! supplies ISO C headers and nothing links musl's runtime. What such a +//! library still calls is defined here, over the task's own heap and Rust's +//! Unicode tables, so each consumer does not grow its own copy: +//! +//! - allocation (`malloc`, `calloc`, `realloc`, `free`) and `abort`, each +//! behind its own feature for a C runtime that exports its own; +//! - the `` routines beyond `compiler_builtins`' `memcpy`, +//! `memmove`, `memset`, `memcmp` and `strlen` (its `mem` feature, which +//! every kore user build enables); +//! - with `wide-classes` (a default), the wide character classes; +//! - the `` writes: the `printf` family over one conversion engine +//! ([`format`]), every float conversion exact for `double` and both +//! `long double` formats, whose `va_arg` each ABI's own rule reads +//! (`va_long_double`); and `stdout`/`stderr` plus consumer-built streams +//! ([`stdio`]); +//! - the `` parsers (`strtol` … `strtod`), `qsort`, `strerror`, +//! and a per-thread `errno`; +//! - with `long-double`, the byte-level halves of `strtold` and `ldexpl` +//! ([`long_double`]) a consumer's C layer wraps, since only C can spell +//! the type. +//! +//! The set began as tree-sitter's (its `wasm-stdlib/imports.txt`) and grew +//! with TCC, the second C consumer (`res/TCC.md` §2.2), and with the C +//! programs libkore.so runs, the third: their `printf` is this crate's. +//! +//! Linking this crate is what supplies the symbols: a consumer names it +//! (`use kore_c_shim as _;`) so the linker keeps it. On the host the crate's +//! symbols are absent — the host's libc serves host tests, and a second +//! `malloc` there would replace the process allocator — while the pure +//! engines behind them ([`format`], [`number`], [`sort`], [`long_double`]) +//! are compiled for the host suite, so it tests the code the target runs. +//! +//! The string routines and wide character classes are pure, so a shared +//! library may carry its own (`libkore_grammars.so` takes the classes alone). Everything that touches the process — the +//! standard streams, `errno` (and with it the parsers that set it), and with +//! the default `malloc` and `abort` features the heap's C names and `abort` — +//! sits behind the default `runtime` feature, and a shared library built +//! without it imports those from the executable it links into +//! (`libkore_grammars.so` does), so the heap stays the task's one heap. A C +//! runtime that is itself the process's C library (libkore.so) takes +//! `runtime` alone and exports its own allocator and `abort`. +#![no_std] +#![cfg_attr(all(target_os = "none", feature = "runtime"), feature(c_variadic))] + +#[cfg(test)] +extern crate std; + +#[cfg(any(test, all(target_os = "none", feature = "runtime")))] +pub mod format; +#[cfg(any(test, all(target_os = "none", feature = "long-double")))] +pub mod long_double; +#[cfg(any(test, all(target_os = "none", feature = "runtime")))] +pub mod number; +#[cfg(any(test, all(target_os = "none", feature = "runtime")))] +pub mod sort; + +#[cfg(all(target_os = "none", feature = "malloc"))] +mod alloc_symbols; +#[cfg(all(target_os = "none", feature = "runtime"))] +pub mod errno; +#[cfg(all(target_os = "none", feature = "runtime"))] +mod format_symbols; +#[cfg(all(target_os = "none", feature = "runtime"))] +pub mod stdio; +#[cfg(all(target_os = "none", feature = "runtime"))] +mod stdlib_symbols; +#[cfg(target_os = "none")] +mod string_symbols; +#[cfg(all(target_os = "none", feature = "runtime"))] +mod va_long_double; +#[cfg(all(target_os = "none", feature = "wide-classes"))] +mod wide_symbols; + +/// The C `abort`: the library hit a state it cannot leave. The task ends +/// with the aborted status, which its controller reports. +#[cfg(all(target_os = "none", feature = "abort"))] +#[unsafe(no_mangle)] +extern "C" fn abort() -> ! { + kore_rt::thread::exit(kore_rt::ExitCode::ABORTED) +} diff --git a/native/crates/c-shim/src/long_double.rs b/native/crates/c-shim/src/long_double.rs new file mode 100644 index 0000000..70601b1 --- /dev/null +++ b/native/crates/c-shim/src/long_double.rs @@ -0,0 +1,341 @@ +//! `long double`'s two conversions, `strtold` and `ldexpl`, as bytes. +//! +//! The type is the C compiler's, never Rust's: x87 double-extended (a 64-bit +//! significand with an explicit integer bit) on x86_64, IEEE binary128 on +//! aarch64 and riscv64. Rust has no type for the first and no ABI promise +//! that its `f128` crosses `extern "C"` as C's `long double` for the second, +//! so the consumer's C layer owns the type: it passes its value's bytes and +//! `LDBL_MANT_DIG`, and this side computes on them. Nothing here names an +//! architecture — the format is the C side's word, checked. +//! +//! The arithmetic is LLVM's APFloat ([`rustc_apfloat`]), the code clang +//! folds `long double` literals with: decimal conversion is correctly +//! rounded for any length of input, and scaling rounds once, subnormals and +//! overflow included. The subject grammar is [`crate::number`]'s, so +//! `strtold` reads exactly the sequence `strtod` does. + +use rustc_apfloat::ieee::{Quad, X87DoubleExtended}; +use rustc_apfloat::{Float, Round, Status}; + +use crate::number::{FloatSubject, float_subject}; + +/// `sizeof(long double)` on every kore C ABI: x87's ten bytes padded to +/// sixteen, or binary128's sixteen. The C side asserts its own type agrees. +pub const BYTES: usize = 16; + +/// A `long double` format, by the `LDBL_MANT_DIG` that names it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Format { + /// x87 double-extended (`LDBL_MANT_DIG == 64`). + X87, + /// IEEE binary128 (`LDBL_MANT_DIG == 113`). + Quad, +} + +impl Format { + /// The format `LDBL_MANT_DIG` names, or `None` for one kore has no ABI + /// with. + pub const fn from_mant_dig(mant_dig: i32) -> Option { + match mant_dig { + 64 => Some(Self::X87), + 113 => Some(Self::Quad), + _ => None, + } + } +} + +/// A `long double` value as its little-endian bytes: the significant bits +/// low (x87's ten, binary128's sixteen), any padding zero. +pub const fn to_bytes(bits: u128) -> [u8; BYTES] { + bits.to_le_bytes() +} + +/// The bits of a `long double`'s bytes. x87's six padding bytes are +/// whatever the C side's storage held, so they are cleared here. +pub const fn from_bytes(format: Format, bytes: [u8; BYTES]) -> u128 { + let bits = u128::from_le_bytes(bytes); + match format { + Format::X87 => bits & ((1u128 << 80) - 1), + Format::Quad => bits, + } +} + +/// What a `strtold` parse found. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Parsed { + /// The value, as [`to_bytes`] lays it out. + pub bits: u128, + /// Overflow to infinity, or a nonzero number rounded to zero: `ERANGE`. + pub range: bool, + /// Bytes consumed; zero when there was no number. + pub end: usize, +} + +/// Parse a `long double` from the start of `text`. +pub fn parse(format: Format, text: &[u8]) -> Parsed { + match format { + Format::X87 => parse_as::(text), + Format::Quad => parse_as::(text), + } +} + +fn parse_as(text: &[u8]) -> Parsed { + let (value, range, end) = match float_subject(text) { + FloatSubject::None => (F::ZERO, false, 0), + FloatSubject::Special { nan, negative, end } => { + let value = if nan { F::NAN } else { F::INFINITY }; + (if negative { -value } else { value }, false, end) + } + FloatSubject::Decimal { + negative, + digits, + nonzero, + end, + } => { + // The subject is ASCII digits, a `.` and an exponent — always + // UTF-8, and always inside APFloat's decimal grammar (a subject + // has at least one digit, and an exponent only with digits). + let converted = core::str::from_utf8(digits) + .ok() + .and_then(|digits| F::from_str_r(digits, Round::NearestTiesToEven).ok()); + let Some(converted) = converted else { + // Unreachable by the grammar; refuse the conversion rather + // than invent a value. + return Parsed { + bits: F::ZERO.to_bits(), + range: false, + end: 0, + }; + }; + let value = if negative { + -converted.value + } else { + converted.value + }; + let range = + converted.status.intersects(Status::OVERFLOW) || (value.is_zero() && nonzero); + (value, range, end) + } + }; + Parsed { + bits: value.to_bits(), + range, + end, + } +} + +/// `ldexpl`: `bits × 2^exponent`, rounded once to nearest-even. +pub fn scale(format: Format, bits: u128, exponent: i32) -> u128 { + match format { + Format::X87 => scale_as::(bits, exponent), + Format::Quad => scale_as::(bits, exponent), + } +} + +fn scale_as(bits: u128, exponent: i32) -> u128 { + F::from_bits(bits) + .scalbn_r(exponent, Round::NearestTiesToEven) + .to_bits() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + use std::vec::Vec; + + // The host's own C library is the reference: on an x86_64 host its + // `long double` is x87 and glibc's `strtold` rounds correctly, and + // `strtof128` is binary128. + unsafe extern "C" { + fn strtold(text: *const core::ffi::c_char, end: *mut *mut core::ffi::c_char) -> X87Bytes; + fn ldexpl(value: X87Bytes, exponent: i32) -> X87Bytes; + } + + /// `long double` as the host ABI moves it: Rust has no x87 type, so + /// these tests read the host's answer through its bytes. + #[repr(C, align(16))] + #[derive(Clone, Copy)] + struct X87Bytes([u8; 16]); + + fn host(text: &str) -> (u128, usize) { + let c = CString::new(text).unwrap(); + let mut end = core::ptr::null_mut(); + // SAFETY: a NUL-terminated string and a valid end pointer. + let value = unsafe { host_strtold(c.as_ptr(), &mut end) }; + let consumed = end as usize - c.as_ptr() as usize; + (from_bytes(Format::X87, value), consumed) + } + + /// glibc's `strtold` returns in `st(0)`; this trampoline stores it. + /// The destination rides `r12`, callee-saved, so the call keeps it. + unsafe fn host_strtold( + text: *const core::ffi::c_char, + end: *mut *mut core::ffi::c_char, + ) -> [u8; 16] { + let mut out = [0u8; 16]; + // SAFETY: the caller's pointers; `fstp` stores the ten bytes and + // leaves the x87 stack empty, as the ABI requires. + unsafe { + core::arch::asm!( + "call {f}", + "fstp tbyte ptr [r12]", + f = sym strtold, + in("r12") out.as_mut_ptr(), + in("rdi") text, + in("rsi") end, + clobber_abi("C"), + ); + } + out + } + + fn ours(text: &str) -> (u128, usize) { + let parsed = parse(Format::X87, text.as_bytes()); + (parsed.bits, parsed.end) + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn x87_parses_as_the_host_does() { + let mut corpus: Vec = [ + "0", + "1", + "0.1", + "1e4932", + "1e4933", + "1e-4951", + "3.6e-4951", + "3.7e-4951", + "1.18973149535723176502e4932", + "1.18973149535723176508e4932", + "3.36210314311209350626e-4932", + "6.95e-4950", + "0.30000000000000000001", + "123456789012345678901234567890", + " -2.5e3x", + ".5", + "5.", + "1e+", + "inf", + "-INFINITY", + "nan", + "18446744073709551615", + "18446744073709551616.5", + "9.999999999999999999999e999", + "2.2250738585072011e-308", + "1.00000000000000000005421010862427522170037264004349708557128906250", + "1.0000000000000000000542101086242752217003726400434970855712890625", + "1.00000000000000000016263032587282567", + "4.9406564584124654e-324", + ] + .iter() + .map(|s| std::string::String::from(*s)) + .collect(); + // A deterministic spread of digit strings and exponents. + let mut seed = 0x9e37_79b9_7f4a_7c15u64; + for _ in 0..4000 { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + let digits = 1 + (seed % 40) as usize; + let mut s = std::string::String::new(); + let mut x = seed; + for i in 0..digits { + if i == digits / 2 && seed & 1 == 0 { + s.push('.'); + } + s.push((b'0' + (x % 10) as u8) as char); + x = x / 10 ^ x.rotate_left(29); + } + let exp = ((seed >> 20) % 9900) as i64 - 4950; + s.push_str(&std::format!("e{exp}")); + corpus.push(s); + } + for text in &corpus { + assert_eq!(ours(text), host(text), "{text}"); + } + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn x87_scales_as_the_host_does() { + let values = [ + "1", + "0.1", + "3.0e-4950", + "1.18973149535723176502e4932", + "-7.25", + "inf", + "0", + ]; + for value in values { + let (bits, _) = host(value); + for exponent in [ + -20000, -16445, -16444, -16383, -100, -1, 0, 1, 100, 16383, 16384, 20000, + ] { + let x = X87Bytes(to_bytes(bits)); + // SAFETY: glibc's `ldexpl` on an x87 value passed in memory. + let expected = unsafe { host_ldexpl(x, exponent) }; + assert_eq!( + scale(Format::X87, bits, exponent), + from_bytes(Format::X87, expected), + "{value} {exponent}" + ); + } + } + } + + /// `ldexpl` takes its `long double` on the stack and returns in `st(0)`. + unsafe fn host_ldexpl(value: X87Bytes, exponent: i32) -> [u8; 16] { + let mut out = [0u8; 16]; + // SAFETY: SysV passes an x87 argument in memory, in order, aligned + // 16 (an asm block without `nostack` enters call-aligned); the + // destination rides callee-saved `r12` across the call. + unsafe { + core::arch::asm!( + "sub rsp, 16", + "movdqu xmm0, [{value}]", + "movdqu [rsp], xmm0", + "call {f}", + "add rsp, 16", + "fstp tbyte ptr [r12]", + f = sym ldexpl, + value = in(reg) value.0.as_ptr(), + in("r12") out.as_mut_ptr(), + in("edi") exponent, + clobber_abi("C"), + ); + } + out + } + + #[test] + fn quad_parses_round_numbers_exactly() { + // binary128 of 1.0: biased exponent 0x3fff, zero fraction. + assert_eq!(parse(Format::Quad, b"1").bits, 0x3fff_u128 << 112); + assert_eq!(parse(Format::Quad, b"-2").bits, (0xc000_u128) << 112); + // 0.1 is the nearest binary128, rounded up in its last bit. + assert_eq!( + parse(Format::Quad, b"0.1").bits, + 0x3ffb_9999_9999_9999_9999_9999_9999_999a + ); + let huge = parse(Format::Quad, b"1e5000"); + assert!(huge.range && huge.bits == 0x7fff_u128 << 112); + let tiny = parse(Format::Quad, b"1e-5000"); + assert!(tiny.range && tiny.bits == 0); + assert_eq!( + scale(Format::Quad, parse(Format::Quad, b"1").bits, 3), + parse(Format::Quad, b"8").bits + ); + } + + #[test] + fn formats_are_named_by_mant_dig() { + assert_eq!(Format::from_mant_dig(64), Some(Format::X87)); + assert_eq!(Format::from_mant_dig(113), Some(Format::Quad)); + assert_eq!(Format::from_mant_dig(53), None); + // x87's padding never leaks into the value. + assert_eq!(from_bytes(Format::X87, [0xff; 16]), (1u128 << 80) - 1); + } +} diff --git a/native/crates/c-shim/src/number.rs b/native/crates/c-shim/src/number.rs new file mode 100644 index 0000000..85e3f8f --- /dev/null +++ b/native/crates/c-shim/src/number.rs @@ -0,0 +1,474 @@ +//! The parsing behind `strtol`, `strtoul`, `strtod` and their siblings: ISO C +//! 7.22.1's grammar over the bytes before a string's NUL, pure so the host +//! suite drives the code the target runs. +//! +//! Integers follow the standard exactly: leading C-locale whitespace, an +//! optional sign, base `0` choosing 16 for `0x`, 8 for a leading `0` and 10 +//! otherwise, overflow reported rather than wrapped, and an end offset that +//! is zero when no digit was found. +//! +//! Decimal floats are rounded correctly — the digits go to core's parser, +//! which rounds any length of input exactly — and `inf`, `infinity` and +//! `nan[(chars)]` are recognized in any case. **Hexadecimal floats are not +//! parsed**: `0x1p3` reads as `0` and ends at the `x`, which is C89's +//! reading. No consumer needs more; TCC, the one that compiles C, parses +//! hexadecimal float literals itself. +//! +//! The subject grammar is public ([`float_subject`]) so a parser for a type +//! core has no parser for — TCC's `long double` — reads exactly the sequence +//! `strtod` reads. + +/// ISO C locale whitespace (`isspace`): space, `\t \n \v \f \r`. +pub const fn c_space(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r') +} + +/// What an integer parse found. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Integer { + /// The value's magnitude, saturated at `u64::MAX` when `overflow`. + pub magnitude: u64, + /// A `-` preceded the digits. + pub negative: bool, + /// The magnitude did not fit 64 bits. + pub overflow: bool, + /// Bytes consumed through the last digit; zero when there was no number. + pub end: usize, +} + +/// Parse an integer in `base` (0 or 2..=36) from the start of `text`. +/// `None` for any other base — C's `EINVAL`, with no conversion. +pub fn integer(text: &[u8], base: u32) -> Option { + if base == 1 || base > 36 { + return None; + } + let mut at = 0; + while text.get(at).copied().is_some_and(c_space) { + at += 1; + } + let negative = text.get(at) == Some(&b'-'); + if matches!(text.get(at), Some(b'+' | b'-')) { + at += 1; + } + let digit = |byte: u8, base: u32| char::from(byte).to_digit(base); + let hex_prefix = matches!(text.get(at..at + 2), Some([b'0', b'x' | b'X'])) + && text + .get(at + 2) + .is_some_and(|&byte| digit(byte, 16).is_some()); + let base = match base { + 0 if hex_prefix => 16, + 0 if text.get(at) == Some(&b'0') => 8, + 0 => 10, + base => base, + }; + // `0x` is skipped only when a hexadecimal digit follows it; otherwise + // the `0` is the number and parsing ends at the `x`. + if base == 16 && hex_prefix { + at += 2; + } + let first = at; + let mut magnitude = 0u64; + let mut overflow = false; + while let Some(value) = text.get(at).and_then(|&byte| digit(byte, base)) { + match magnitude + .checked_mul(u64::from(base)) + .and_then(|scaled| scaled.checked_add(u64::from(value))) + { + Some(next) => magnitude = next, + None => { + overflow = true; + magnitude = u64::MAX; + } + } + at += 1; + } + let end = if at == first { 0 } else { at }; + Some(Integer { + magnitude, + negative, + overflow, + end, + }) +} + +impl Integer { + /// The value as a signed integer no wider than `bits`, and whether it + /// was clamped (C's `ERANGE`). + pub fn signed(self, bits: u32) -> (i64, bool) { + let max = (1u64 << (bits - 1)) - 1; + if self.negative { + if self.overflow || self.magnitude > max + 1 { + (-(max as i64) - 1, true) + } else { + (0i64.wrapping_sub(self.magnitude as i64), false) + } + } else if self.overflow || self.magnitude > max { + (max as i64, true) + } else { + (self.magnitude as i64, false) + } + } + + /// The value as an unsigned integer no wider than `bits`, and whether it + /// was clamped. A `-` negates in the unsigned type, as C says. + pub fn unsigned(self, bits: u32) -> (u64, bool) { + let max = if bits == 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + }; + if self.overflow || self.magnitude > max { + (max, true) + } else if self.negative { + (self.magnitude.wrapping_neg() & max, false) + } else { + (self.magnitude, false) + } + } +} + +/// What a float parse found. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Float { + pub value: T, + /// Overflow to infinity, or a nonzero number rounded to zero: `ERANGE`. + pub range: bool, + /// Bytes consumed; zero when there was no number. + pub end: usize, +} + +/// The floating subject sequence at the start of `text` (ISO C 7.22.1.3), +/// leading whitespace skipped: what `strtod` and its siblings convert. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FloatSubject<'t> { + /// No subject: nothing is converted, and the end pointer is the input. + None, + /// `inf`, `infinity` or `nan[(n-char-sequence)]`, in any case. + Special { + nan: bool, + negative: bool, + end: usize, + }, + /// A decimal number. `digits` is the sign-free subject — digits, an + /// optional `.`, an optional exponent — in the grammar a + /// correctly-rounding parser takes; `nonzero` says a mantissa digit was + /// not `0`, so a zero result is an underflow rather than a zero. + Decimal { + negative: bool, + digits: &'t [u8], + nonzero: bool, + end: usize, + }, +} + +/// Scan the floating subject at the start of `text`. +pub fn float_subject(text: &[u8]) -> FloatSubject<'_> { + if let Some((nan, negative, end)) = special(text) { + return FloatSubject::Special { nan, negative, end }; + } + match decimal_subject(text) { + Some((negative, digits, nonzero, end)) => FloatSubject::Decimal { + negative, + digits, + nonzero, + end, + }, + None => FloatSubject::None, + } +} + +/// The decimal subject sequence at the start of `text`: its sign, the slice +/// core's parser takes, whether any mantissa digit was nonzero, and the end. +fn decimal_subject(text: &[u8]) -> Option<(bool, &[u8], bool, usize)> { + let mut at = 0; + while text.get(at).copied().is_some_and(c_space) { + at += 1; + } + let negative = text.get(at) == Some(&b'-'); + if matches!(text.get(at), Some(b'+' | b'-')) { + at += 1; + } + let start = at; + let mut digits = 0usize; + let mut nonzero = false; + while let Some(&byte) = text.get(at).filter(|byte| byte.is_ascii_digit()) { + nonzero |= byte != b'0'; + digits += 1; + at += 1; + } + if text.get(at) == Some(&b'.') { + at += 1; + while let Some(&byte) = text.get(at).filter(|byte| byte.is_ascii_digit()) { + nonzero |= byte != b'0'; + digits += 1; + at += 1; + } + } + if digits == 0 { + return None; + } + // An exponent counts only when at least one digit follows it. + if matches!(text.get(at), Some(b'e' | b'E')) { + let mut exponent = at + 1; + if matches!(text.get(exponent), Some(b'+' | b'-')) { + exponent += 1; + } + if text.get(exponent).is_some_and(u8::is_ascii_digit) { + at = exponent; + while text.get(at).is_some_and(u8::is_ascii_digit) { + at += 1; + } + } + } + Some((negative, &text[start..at], nonzero, at)) +} + +/// `inf`, `infinity` or `nan[(n-char-sequence)]`, case-insensitively, at the +/// start of `text`: `(is_nan, negative, end)`. +fn special(text: &[u8]) -> Option<(bool, bool, usize)> { + let mut at = 0; + while text.get(at).copied().is_some_and(c_space) { + at += 1; + } + let negative = text.get(at) == Some(&b'-'); + if matches!(text.get(at), Some(b'+' | b'-')) { + at += 1; + } + let word = |at: usize, word: &[u8]| { + text.get(at..at + word.len()) + .is_some_and(|got| got.eq_ignore_ascii_case(word)) + }; + if word(at, b"infinity") { + return Some((false, negative, at + 8)); + } + if word(at, b"inf") { + return Some((false, negative, at + 3)); + } + if word(at, b"nan") { + let mut end = at + 3; + // `nan(` followed by alphanumerics and `_`, then `)`. + if text.get(end) == Some(&b'(') { + let mut close = end + 1; + while text + .get(close) + .is_some_and(|&byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + close += 1; + } + if text.get(close) == Some(&b')') { + end = close + 1; + } + } + return Some((true, negative, end)); + } + None +} + +macro_rules! float_parser { + ($name:ident, $ty:ty) => { + /// Parse a float from the start of `text`. + pub fn $name(text: &[u8]) -> Float<$ty> { + let (negative, subject, nonzero, end) = match float_subject(text) { + FloatSubject::Special { nan, negative, end } => { + let value = if nan { <$ty>::NAN } else { <$ty>::INFINITY }; + return Float { + value: if negative { -value } else { value }, + range: false, + end, + }; + } + FloatSubject::None => { + return Float { + value: 0.0, + range: false, + end: 0, + }; + } + FloatSubject::Decimal { + negative, + digits, + nonzero, + end, + } => (negative, digits, nonzero, end), + }; + // The subject is ASCII digits, `.` and an exponent: always UTF-8, + // and always in core's grammar (a lone `.` was refused above). + let value: $ty = core::str::from_utf8(subject) + .ok() + .and_then(|digits| digits.parse().ok()) + .unwrap_or(0.0); + let range = value.is_infinite() || (value == 0.0 && nonzero); + Float { + value: if negative { -value } else { value }, + range, + end, + } + } + }; +} + +float_parser!(double, f64); +float_parser!(single, f32); + +#[cfg(test)] +mod tests { + use super::*; + + fn int(text: &str, base: u32) -> Integer { + integer(text.as_bytes(), base).expect("a valid base") + } + + #[test] + fn integers_follow_the_c_grammar() { + assert_eq!( + int(" 42abc", 10), + Integer { + magnitude: 42, + negative: false, + overflow: false, + end: 4 + } + ); + assert_eq!(int("-17", 10).signed(64), (-17, false)); + assert_eq!(int("+9", 0).signed(64), (9, false)); + assert_eq!(int("0x1F", 0).magnitude, 31); + assert_eq!(int("0x1F", 16).magnitude, 31); + assert_eq!(int("017", 0).magnitude, 15); + assert_eq!(int("z", 36).magnitude, 35); + // `0x` without a hex digit after it: the `0` is the number. + assert_eq!( + int("0xg", 0), + Integer { + magnitude: 0, + negative: false, + overflow: false, + end: 1 + } + ); + assert_eq!(int("0x", 16).end, 1); + // No digits: no conversion, end zero. + assert_eq!(int(" -", 10).end, 0); + assert_eq!(int("", 10).end, 0); + assert!(integer(b"1", 1).is_none()); + assert!(integer(b"1", 37).is_none()); + } + + #[test] + fn integer_overflow_clamps_and_reports() { + assert_eq!(int("9223372036854775807", 10).signed(64), (i64::MAX, false)); + assert_eq!(int("9223372036854775808", 10).signed(64), (i64::MAX, true)); + assert_eq!( + int("-9223372036854775808", 10).signed(64), + (i64::MIN, false) + ); + assert_eq!(int("-9223372036854775809", 10).signed(64), (i64::MIN, true)); + assert_eq!( + int("99999999999999999999999", 10).signed(64), + (i64::MAX, true) + ); + assert_eq!(int("2147483648", 10).signed(32), (i32::MAX as i64, true)); + assert_eq!( + int("18446744073709551615", 10).unsigned(64), + (u64::MAX, false) + ); + assert_eq!( + int("18446744073709551616", 10).unsigned(64), + (u64::MAX, true) + ); + // A minus sign negates in the unsigned type. + assert_eq!(int("-1", 10).unsigned(64), (u64::MAX, false)); + assert_eq!(int("-1", 10).unsigned(32), (u32::MAX as u64, false)); + // Overflowed digits are still consumed. + assert_eq!(int("99999999999999999999999x", 10).end, 23); + } + + #[test] + fn decimal_floats_round_correctly() { + assert_eq!( + double(b"1.5"), + Float { + value: 1.5, + range: false, + end: 3 + } + ); + assert_eq!(double(b" -2.5e3x").value, -2500.0); + assert_eq!(double(b" -2.5e3x").end, 8); + assert_eq!(double(b".5").value, 0.5); + assert_eq!(double(b"5.").value, 5.0); + assert_eq!(double(b"5.").end, 2); + // An exponent with no digits is not part of the number. + assert_eq!(double(b"1e+").end, 1); + assert_eq!(double(b"0.1").value, 0.1); + assert_eq!(single(b"0.1").value, 0.1f32); + assert_eq!(double(b"").end, 0); + assert_eq!(double(b".").end, 0); + assert_eq!(double(b"-").end, 0); + } + + #[test] + fn float_range_and_specials() { + let huge = double(b"1e400"); + assert!(huge.value.is_infinite() && huge.range); + let tiny = double(b"1e-400"); + assert!(tiny.value == 0.0 && tiny.range); + let zero = double(b"0.000"); + assert!(zero.value == 0.0 && !zero.range); + assert!(single(b"1e39").range); + assert_eq!(double(b"INF").value, f64::INFINITY); + assert_eq!(double(b"-Infinity").value, f64::NEG_INFINITY); + assert_eq!(double(b"-Infinity").end, 9); + assert!(double(b"nan(0x1)").value.is_nan()); + assert_eq!(double(b"nan(abc)").end, 8); + assert_eq!(double(b"nan(").end, 3); + // Hexadecimal floats read as C89 does: the `0`, ending at the `x`. + assert_eq!( + double(b"0x1p3"), + Float { + value: 0.0, + range: false, + end: 1 + } + ); + } + + #[test] + fn the_subject_is_the_parsers_own() { + assert_eq!( + float_subject(b" -1.25e3x"), + FloatSubject::Decimal { + negative: true, + digits: b"1.25e3", + nonzero: true, + end: 8 + } + ); + assert_eq!( + float_subject(b"0.000"), + FloatSubject::Decimal { + negative: false, + digits: b"0.000", + nonzero: false, + end: 5 + } + ); + assert_eq!( + float_subject(b"-INF"), + FloatSubject::Special { + nan: false, + negative: true, + end: 4 + } + ); + assert_eq!( + float_subject(b"nan(7)"), + FloatSubject::Special { + nan: true, + negative: false, + end: 6 + } + ); + assert_eq!(float_subject(b" ."), FloatSubject::None); + assert_eq!(float_subject(b"+"), FloatSubject::None); + } +} diff --git a/native/crates/c-shim/src/sort.rs b/native/crates/c-shim/src/sort.rs new file mode 100644 index 0000000..7b16c60 --- /dev/null +++ b/native/crates/c-shim/src/sort.rs @@ -0,0 +1,115 @@ +//! `qsort`'s algorithm: heapsort over raw elements of a C-supplied size. +//! +//! Heapsort because it is O(n log n) in the worst case, iterative (no +//! recursion to exhaust a 64 KiB worker stack), and in place. C's `qsort` +//! promises no stability, and this keeps none. + +/// Sort `count` elements of `size` bytes at `base` by `before(a, b)`, which +/// answers whether `a` sorts strictly after `b` (a C comparator's `> 0`). +/// +/// # Safety +/// `base` points to `count * size` writable bytes, and `after` reads only +/// the `size` bytes at each pointer it is given. +pub unsafe fn heapsort( + base: *mut u8, + count: usize, + size: usize, + after: &mut impl FnMut(*const u8, *const u8) -> bool, +) { + if count < 2 || size == 0 { + return; + } + let at = |index: usize| -> *mut u8 { + // SAFETY: every index the algorithm forms is below `count`. + unsafe { base.add(index * size) } + }; + // SAFETY (the swaps): distinct indices name disjoint elements. + let sift = + |mut root: usize, end: usize, after: &mut dyn FnMut(*const u8, *const u8) -> bool| { + loop { + let left = 2 * root + 1; + if left >= end { + return; + } + let right = left + 1; + let child = if right < end && after(at(right), at(left)) { + right + } else { + left + }; + if !after(at(child), at(root)) { + return; + } + unsafe { core::ptr::swap_nonoverlapping(at(root), at(child), size) }; + root = child; + } + }; + let mut start = count / 2; + while start > 0 { + start -= 1; + sift(start, count, after); + } + let mut end = count; + while end > 1 { + end -= 1; + unsafe { core::ptr::swap_nonoverlapping(at(0), at(end), size) }; + sift(0, end, after); + } +} + +#[cfg(test)] +mod tests { + use super::*; + extern crate std; + use std::vec::Vec; + + fn sort_u32(values: &mut [u32]) { + let len = values.len(); + // SAFETY: the slice's bytes, elements of four bytes. + unsafe { + heapsort(values.as_mut_ptr().cast(), len, 4, &mut |a, b| { + a.cast::().read_unaligned() > b.cast::().read_unaligned() + }) + }; + } + + #[test] + fn sorts_every_shape() { + for len in 0..40 { + let mut ascending: Vec = (0..len).collect(); + let mut descending: Vec = (0..len).rev().collect(); + let mut repeated: Vec = (0..len).map(|i| i % 3).collect(); + let mut scrambled: Vec = + (0..len).map(|i| i.wrapping_mul(2654435761) % 97).collect(); + for values in [ + &mut ascending, + &mut descending, + &mut repeated, + &mut scrambled, + ] { + let mut expected = values.clone(); + expected.sort_unstable(); + sort_u32(values); + assert_eq!(*values, expected); + } + } + } + + #[test] + fn sorts_wide_elements_by_their_key() { + // 24-byte records keyed on their first word, as C sorts structs. + let mut records: Vec<[u64; 3]> = (0..17u64).map(|i| [(i * 7) % 17, i, !i]).collect(); + let len = records.len(); + // SAFETY: the vector's bytes, elements of 24 bytes. + unsafe { + heapsort(records.as_mut_ptr().cast(), len, 24, &mut |a, b| { + a.cast::().read_unaligned() > b.cast::().read_unaligned() + }) + }; + for (index, record) in records.iter().enumerate() { + assert_eq!(record[0], index as u64); + // Each record moved whole. + assert_eq!(record[2], !record[1]); + } + } +} diff --git a/native/crates/c-shim/src/stdio.rs b/native/crates/c-shim/src/stdio.rs new file mode 100644 index 0000000..f004ee4 --- /dev/null +++ b/native/crates/c-shim/src/stdio.rs @@ -0,0 +1,210 @@ +//! `FILE` streams: the standard pair, and the streams a consumer builds over +//! its own file layer. +//! +//! A [`File`] is a write operation and its state. C only ever holds a +//! `FILE *` to one (musl's `FILE` is incomplete in the headers every kore C +//! build uses), so the layout is this crate's own. +//! +//! - **`stdout` and `stderr`** go to the [`StandardSink`] the process +//! installs once ([`install_standard_sink`]) — its terminal, or a result +//! buffer it reports when the task ends. Until one is installed a write +//! reports `EOF` and goes nowhere, which is what every consumer that never +//! installs one has always seen. +//! - **Any other stream** is built by the consumer that owns a file layer +//! (`fdopen` over its own descriptors) with [`File::new`] and its own +//! [`FileOps`]. This crate never opens a file: kore has no ambient +//! filesystem for it to open one in. + +use core::ffi::c_void; +use core::ptr; +use core::sync::atomic::{AtomicPtr, Ordering}; + +/// A C `FILE`. +#[repr(C)] +pub struct File { + ops: &'static FileOps, + state: *mut c_void, +} + +// SAFETY: the standard streams' state is an inert tag, never dereferenced; +// a consumer that places its own stream in a `static` owns that stream's +// synchronization, as C's own `FILE` locking would. +unsafe impl Sync for File {} + +/// What a stream does with its bytes. +pub struct FileOps { + /// Accept `bytes` whole. `false` is a write error: C's `EOF`. + pub write: unsafe fn(state: *mut c_void, bytes: &[u8]) -> bool, + /// Push anything buffered onward. `false` is an error. + pub flush: unsafe fn(state: *mut c_void) -> bool, + /// `fclose`'s last act: flush and release the stream, the `File` itself + /// included when its owner allocated it. `false` is an error; the stream + /// is closed either way, as C's `fclose` leaves it. + pub close: unsafe fn(file: *mut File) -> bool, +} + +impl File { + /// A stream over `ops`, carrying the consumer's `state`. + pub const fn new(ops: &'static FileOps, state: *mut c_void) -> Self { + Self { ops, state } + } + + /// The consumer's state, for its [`FileOps`]. + pub const fn state(&self) -> *mut c_void { + self.state + } +} + +/// Write `bytes` to the stream at `file`. +/// +/// # Safety +/// `file` is a live `FILE *` this crate or a consumer constructed. +pub(crate) unsafe fn write(file: *mut File, bytes: &[u8]) -> bool { + if bytes.is_empty() { + return true; + } + // SAFETY: the caller's contract; only the fields are read. + unsafe { ((*file).ops.write)((*file).state, bytes) } +} + +/// # Safety +/// As [`write`]. +pub(crate) unsafe fn flush(file: *mut File) -> bool { + // SAFETY: the caller's contract. + unsafe { ((*file).ops.flush)((*file).state) } +} + +/// # Safety +/// As [`write`]; `file` is not used again. +pub(crate) unsafe fn close(file: *mut File) -> bool { + // SAFETY: the caller's contract. + unsafe { ((*file).ops.close)(file) } +} + +/// The two standard output streams. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Standard { + /// `stdout`. + Output, + /// `stderr`. + Error, +} + +/// Where the standard streams' bytes go, and the lock one stream call holds +/// across every write it makes. +/// +/// POSIX has each stdio call lock its stream (`flockfile`), so one call's +/// output reaches the stream together even when the call writes it in +/// pieces, as `vfprintf`'s batches and `puts`' line and newline are. The +/// process decides what the lock is: a sink only one thread writes takes +/// none. +pub struct StandardSink { + /// Write `bytes` to the stream whole. `false` reports a write error. + pub write: fn(Standard, &[u8]) -> bool, + /// Take the standard streams' lock for one call. A sink reached with + /// the lock already held by the calling thread takes it again: the lock + /// is the sink's to make reentrant. + pub lock: fn(), + /// Release one [`lock`](Self::lock). + pub unlock: fn(), +} + +/// The installed sink; null until a process installs one. +static SINK: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +/// Send `stdout` and `stderr` to `sink` from now on. A later install +/// replaces an earlier one; a call in flight finishes on the sink it read. +pub fn install_standard_sink(sink: &'static StandardSink) { + SINK.store(ptr::from_ref(sink).cast_mut(), Ordering::Release); +} + +fn installed() -> Option<&'static StandardSink> { + // SAFETY: only `install_standard_sink` stores here, and it stores a + // `&'static StandardSink`. + unsafe { SINK.load(Ordering::Acquire).as_ref() } +} + +/// Run one stream call on `file` holding its lock: around every write it +/// makes, the standard streams' sink's lock; nothing for a consumer's +/// stream, whose owner synchronizes it. +/// +/// # Safety +/// `file` is a live `FILE *` this crate or a consumer constructed. +pub(crate) unsafe fn locked(file: *mut File, call: impl FnOnce() -> R) -> R { + // SAFETY: the caller's contract. + let standard = ptr::eq(unsafe { (*file).ops }, &STANDARD_OPS); + let sink = if standard { installed() } else { None }; + if let Some(sink) = sink { + (sink.lock)(); + } + let result = call(); + if let Some(sink) = sink { + (sink.unlock)(); + } + result +} + +/// The standard streams' state: an inert tag naming which one. +const OUTPUT_TAG: *mut c_void = ptr::without_provenance_mut(1); +const ERROR_TAG: *mut c_void = ptr::without_provenance_mut(2); + +unsafe fn standard_write(state: *mut c_void, bytes: &[u8]) -> bool { + let Some(sink) = installed() else { + return false; + }; + let stream = if state == OUTPUT_TAG { + Standard::Output + } else { + Standard::Error + }; + (sink.write)(stream, bytes) +} + +unsafe fn standard_flush(_: *mut c_void) -> bool { + // The sink receives every write whole; there is nothing buffered here. + true +} + +unsafe fn standard_close(_: *mut File) -> bool { + // C lets a program close `stdout`; the stream is static, so closing it + // only ends this program's use of it. + true +} + +static STANDARD_OPS: FileOps = FileOps { + write: standard_write, + flush: standard_flush, + close: standard_close, +}; + +static STDOUT: File = File::new(&STANDARD_OPS, OUTPUT_TAG); +static STDERR: File = File::new(&STANDARD_OPS, ERROR_TAG); + +/// A `FILE *` value: musl declares `extern FILE *const stdout`, so the +/// symbol holds a pointer to the stream, not the stream. +#[repr(transparent)] +pub struct Stream(*mut File); + +// SAFETY: a pointer to an immutable static; nothing writes through it. +unsafe impl Sync for Stream {} + +impl Stream { + pub(crate) fn file(&self) -> *mut File { + self.0 + } +} + +/// The `FILE *` of a standard stream — what a consumer's `fdopen` answers +/// for descriptor 1 or 2. +pub fn standard_stream(which: Standard) -> *mut File { + match which { + Standard::Output => stdout.file(), + Standard::Error => stderr.file(), + } +} + +#[unsafe(no_mangle)] +pub static stdout: Stream = Stream(&STDOUT as *const File as *mut File); + +#[unsafe(no_mangle)] +pub static stderr: Stream = Stream(&STDERR as *const File as *mut File); diff --git a/native/crates/c-shim/src/stdlib_symbols.rs b/native/crates/c-shim/src/stdlib_symbols.rs new file mode 100644 index 0000000..fa0a508 --- /dev/null +++ b/native/crates/c-shim/src/stdlib_symbols.rs @@ -0,0 +1,240 @@ +//! The `` and `` routines that are pure computation: +//! number parsing, `qsort`, `strerror`. +//! +//! The parsers set `errno` as ISO C requires, so they sit with it behind the +//! `runtime` feature; `qsort` and `strerror` are pure but travel with them. + +use core::ffi::{ + c_char, c_double, c_float, c_int, c_long, c_longlong, c_ulong, c_ulonglong, c_void, +}; + +use crate::errno::{self, EINVAL, ERANGE}; +use crate::number; + +/// The bytes before `text`'s NUL. +/// +/// # Safety +/// `text` is NUL-terminated. +unsafe fn bytes<'a>(text: *const c_char) -> &'a [u8] { + // SAFETY: the caller's contract. + unsafe { core::ffi::CStr::from_ptr(text) }.to_bytes() +} + +/// Store `text + end` through `end_out`, as the parsers' `endptr`. +/// +/// # Safety +/// `end_out` is null or writable; `end` lies within `text`'s string. +unsafe fn report_end(text: *const c_char, end: usize, end_out: *mut *mut c_char) { + if !end_out.is_null() { + // SAFETY: the caller's contract. + unsafe { *end_out = text.add(end).cast_mut() }; + } +} + +/// The shared front of the integer parsers: parse, report the end, and set +/// `EINVAL` for a base C does not define. +/// +/// # Safety +/// C's `strtol` contract. +unsafe fn parse( + text: *const c_char, + end_out: *mut *mut c_char, + base: c_int, +) -> Option { + // SAFETY: the caller's contract. + let subject = unsafe { bytes(text) }; + let parsed = u32::try_from(base) + .ok() + .and_then(|base| number::integer(subject, base)); + let end = parsed.map_or(0, |parsed| parsed.end); + // SAFETY: the end lies within the string. + unsafe { report_end(text, end, end_out) }; + if parsed.is_none() { + errno::set(EINVAL); + } + parsed +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtoll( + text: *const c_char, + end: *mut *mut c_char, + base: c_int, +) -> c_longlong { + // SAFETY: C's contract. + let Some(parsed) = (unsafe { parse(text, end, base) }) else { + return 0; + }; + let (value, clamped) = parsed.signed(c_longlong::BITS); + if clamped { + errno::set(ERANGE); + } + value +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtol(text: *const c_char, end: *mut *mut c_char, base: c_int) -> c_long { + // SAFETY: C's contract. + let Some(parsed) = (unsafe { parse(text, end, base) }) else { + return 0; + }; + let (value, clamped) = parsed.signed(c_long::BITS); + if clamped { + errno::set(ERANGE); + } + value as c_long +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtoull( + text: *const c_char, + end: *mut *mut c_char, + base: c_int, +) -> c_ulonglong { + // SAFETY: C's contract. + let Some(parsed) = (unsafe { parse(text, end, base) }) else { + return 0; + }; + let (value, clamped) = parsed.unsigned(c_ulonglong::BITS); + if clamped { + errno::set(ERANGE); + } + value +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtoul(text: *const c_char, end: *mut *mut c_char, base: c_int) -> c_ulong { + // SAFETY: C's contract. + let Some(parsed) = (unsafe { parse(text, end, base) }) else { + return 0; + }; + let (value, clamped) = parsed.unsigned(c_ulong::BITS); + if clamped { + errno::set(ERANGE); + } + value as c_ulong +} + +/// `atoi`/`atol`: `strtol` in base 10, as glibc defines them — out-of-range +/// input is undefined in C; here it is `strtol`'s clamp, truncated. +#[unsafe(no_mangle)] +unsafe extern "C" fn atoi(text: *const c_char) -> c_int { + // SAFETY: C's contract. + unsafe { strtol(text, core::ptr::null_mut(), 10) as c_int } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn atol(text: *const c_char) -> c_long { + // SAFETY: C's contract. + unsafe { strtol(text, core::ptr::null_mut(), 10) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtod(text: *const c_char, end: *mut *mut c_char) -> c_double { + // SAFETY: C's contract. + let parsed = number::double(unsafe { bytes(text) }); + unsafe { report_end(text, parsed.end, end) }; + if parsed.range { + errno::set(ERANGE); + } + parsed.value +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtof(text: *const c_char, end: *mut *mut c_char) -> c_float { + // SAFETY: C's contract. + let parsed = number::single(unsafe { bytes(text) }); + unsafe { report_end(text, parsed.end, end) }; + if parsed.range { + errno::set(ERANGE); + } + parsed.value +} + +/// A C comparator. +type Compare = unsafe extern "C" fn(*const c_void, *const c_void) -> c_int; + +#[unsafe(no_mangle)] +unsafe extern "C" fn qsort(base: *mut c_void, count: usize, size: usize, compare: Compare) { + // SAFETY: C's contract: `count * size` bytes at `base`, and a comparator + // that reads only the elements it is handed. + unsafe { + crate::sort::heapsort(base.cast(), count, size, &mut |a, b| { + compare(a.cast(), b.cast()) > 0 + }) + }; +} + +#[unsafe(no_mangle)] +extern "C" fn strerror(code: c_int) -> *mut c_char { + errno::describe(code).as_ptr().cast_mut() +} + +/// `strtold`'s byte-level half (`long_double`): parse `text` as the +/// `long double` `mant_dig` names, store its [`long_double::BYTES`] bytes at +/// `out`, and the end at `*end`. Answers the `errno` the C wrapper sets — +/// `ERANGE` as `strtod` would, `EINVAL` for a format or buffer this crate +/// has no ABI for (the value is then `+0` and nothing is consumed) — or `0`. +/// +/// # Safety +/// C's `strtold` contract for `text` and `end`; `out` is writable for +/// `out_len` bytes. +#[cfg(feature = "long-double")] +#[unsafe(no_mangle)] +unsafe extern "C" fn __kore_strtold_bytes( + text: *const c_char, + end: *mut *mut c_char, + mant_dig: c_int, + out: *mut u8, + out_len: usize, +) -> c_int { + use crate::long_double; + let Some(format) = + long_double::Format::from_mant_dig(mant_dig).filter(|_| out_len == long_double::BYTES) + else { + // SAFETY: C's contract. + unsafe { report_end(text, 0, end) }; + if !out.is_null() { + // SAFETY: `out_len` writable bytes. + unsafe { core::ptr::write_bytes(out, 0, out_len) }; + } + return EINVAL; + }; + // SAFETY: C's contract. + let parsed = long_double::parse(format, unsafe { bytes(text) }); + unsafe { + report_end(text, parsed.end, end); + out.cast::<[u8; long_double::BYTES]>() + .write_unaligned(long_double::to_bytes(parsed.bits)); + } + if parsed.range { ERANGE } else { 0 } +} + +/// `ldexpl`'s byte-level half: replace the `long double` at `value` with +/// itself times `2^exponent`, rounded once. `EINVAL` (value untouched) for a +/// format or buffer this crate has no ABI for, else `0`. +/// +/// # Safety +/// `value` is readable and writable for `len` bytes. +#[cfg(feature = "long-double")] +#[unsafe(no_mangle)] +unsafe extern "C" fn __kore_ldexpl_bytes( + value: *mut u8, + len: usize, + mant_dig: c_int, + exponent: c_int, +) -> c_int { + use crate::long_double; + let Some(format) = + long_double::Format::from_mant_dig(mant_dig).filter(|_| len == long_double::BYTES) + else { + return EINVAL; + }; + let slot = value.cast::<[u8; long_double::BYTES]>(); + // SAFETY: the caller's `len` bytes. + let bits = long_double::from_bytes(format, unsafe { slot.read_unaligned() }); + let scaled = long_double::scale(format, bits, exponent); + // SAFETY: as above. + unsafe { slot.write_unaligned(long_double::to_bytes(scaled)) }; + 0 +} diff --git a/native/crates/c-shim/src/string_symbols.rs b/native/crates/c-shim/src/string_symbols.rs new file mode 100644 index 0000000..92206b4 --- /dev/null +++ b/native/crates/c-shim/src/string_symbols.rs @@ -0,0 +1,158 @@ +//! The ISO string routines a C library imports beyond `compiler_builtins`' +//! `memcpy`/`memmove`/`memset`/`memcmp`/`strlen`. + +use core::ffi::{CStr, c_char, c_int, c_void}; + +/// The bytes of a C string, without its NUL. +/// +/// # Safety +/// `text` is NUL-terminated. +unsafe fn bytes<'a>(text: *const c_char) -> &'a [u8] { + // SAFETY: the caller's contract. + unsafe { CStr::from_ptr(text) }.to_bytes() +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strchr(text: *const c_char, value: c_int) -> *mut c_char { + let needle = value as u8; + // SAFETY: C promises a NUL-terminated string. + let body = unsafe { bytes(text) }; + // The terminator itself is part of the searchable string: `strchr(s, 0)` + // answers the NUL's address. + let found = if needle == 0 { + Some(body.len()) + } else { + body.iter().position(|&byte| byte == needle) + }; + match found { + // SAFETY: the offset lies within the string or on its NUL. + Some(offset) => unsafe { text.add(offset) }.cast_mut(), + None => core::ptr::null_mut(), + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strrchr(text: *const c_char, value: c_int) -> *mut c_char { + let needle = value as u8; + // SAFETY: C promises a NUL-terminated string. + let body = unsafe { bytes(text) }; + let found = if needle == 0 { + Some(body.len()) + } else { + body.iter().rposition(|&byte| byte == needle) + }; + match found { + // SAFETY: as `strchr`. + Some(offset) => unsafe { text.add(offset) }.cast_mut(), + None => core::ptr::null_mut(), + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strcpy(target: *mut c_char, source: *const c_char) -> *mut c_char { + // SAFETY: C promises `source` NUL-terminated and `target` large enough + // for it and its NUL, not overlapping. + unsafe { + let len = bytes(source).len(); + core::ptr::copy_nonoverlapping(source, target, len + 1); + } + target +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strstr(haystack: *const c_char, needle: *const c_char) -> *mut c_char { + // SAFETY: C promises both NUL-terminated. + let (hay, pin) = unsafe { (bytes(haystack), bytes(needle)) }; + let Some((&first, rest)) = pin.split_first() else { + return haystack.cast_mut(); + }; + if pin.len() > hay.len() { + return core::ptr::null_mut(); + } + // Scan for the first byte, then compare the rest in place. + let last_start = hay.len() - pin.len(); + let mut at = 0; + while let Some(skip) = hay[at..=last_start].iter().position(|&byte| byte == first) { + let start = at + skip; + if &hay[start + 1..start + pin.len()] == rest { + // SAFETY: `start` lies within the haystack. + return unsafe { haystack.add(start) }.cast_mut(); + } + at = start + 1; + if at > last_start { + break; + } + } + core::ptr::null_mut() +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn memchr(source: *const c_void, value: c_int, len: usize) -> *mut c_void { + let bytes = source.cast::(); + let needle = value as u8; + for index in 0..len { + // SAFETY: C promises `len` readable bytes at `source`. + if unsafe { *bytes.add(index) } == needle { + return unsafe { bytes.add(index) }.cast_mut().cast(); + } + } + core::ptr::null_mut() +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strcmp(left: *const c_char, right: *const c_char) -> c_int { + // SAFETY: both are NUL-terminated, so the bounded compare stops at a NUL. + unsafe { strncmp(left, right, usize::MAX) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strncmp(left: *const c_char, right: *const c_char, len: usize) -> c_int { + for index in 0..len { + // SAFETY: C promises both strings readable up to their NUL or `len`. + let (a, b) = unsafe { (*left.add(index) as u8, *right.add(index) as u8) }; + if a != b { + return c_int::from(a) - c_int::from(b); + } + if a == 0 { + break; + } + } + 0 +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strncpy(target: *mut c_char, source: *const c_char, len: usize) -> *mut c_char { + let mut index = 0; + // SAFETY: C promises `len` writable bytes at `target` and `source` + // readable up to its NUL or `len`; the tail is NUL-padded, as ISO says. + unsafe { + while index < len && *source.add(index) != 0 { + *target.add(index) = *source.add(index); + index += 1; + } + while index < len { + *target.add(index) = 0; + index += 1; + } + } + target +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strncat(target: *mut c_char, source: *const c_char, len: usize) -> *mut c_char { + // SAFETY: C promises `target` NUL-terminated with room for at most `len` + // more bytes and a NUL, and `source` readable up to its NUL or `len`. + unsafe { + let mut end = 0; + while *target.add(end) != 0 { + end += 1; + } + let mut index = 0; + while index < len && *source.add(index) != 0 { + *target.add(end + index) = *source.add(index); + index += 1; + } + *target.add(end + index) = 0; + } + target +} diff --git a/native/crates/c-shim/src/va_long_double/aarch64.rs b/native/crates/c-shim/src/va_long_double/aarch64.rs new file mode 100644 index 0000000..3aaf12a --- /dev/null +++ b/native/crates/c-shim/src/va_long_double/aarch64.rs @@ -0,0 +1,47 @@ +//! AAPCS64: a `long double` is a quad-precision floating-point type, passed in +//! the next SIMD and floating-point register while any remains, its save slot +//! in the register save area, and on the stack aligned to sixteen after that +//! (AAPCS64 §B.4, `va_arg` for a floating-point type). + +use super::{BYTES, Float, VaList, align_up, read}; + +/// `va_list` as AAPCS64 declares it (`__va_list`). +#[repr(C)] +struct List { + stack: *mut u8, + gr_top: *mut u8, + vr_top: *mut u8, + gr_offs: i32, + vr_offs: i32, +} + +const _: () = assert!(core::mem::size_of::>() == core::mem::size_of::()); + +/// One vector register's save slot. +const VECTOR_SLOT: i32 = 16; + +/// The next argument, a binary128 `long double`. +/// +/// # Safety +/// The next argument `list` holds is a `long double`. +pub(crate) unsafe fn next(list: &mut VaList<'_>) -> Float { + // SAFETY: `VaList` is the platform's `va_list` on this target (the size + // is asserted), and both areas are the caller's. + let list = unsafe { &mut *(list as *mut VaList<'_>).cast::() }; + let offset = list.vr_offs; + let at = if offset < 0 && offset + VECTOR_SLOT <= 0 { + list.vr_offs = offset + VECTOR_SLOT; + list.vr_top.wrapping_offset(offset as isize) + } else { + if offset < 0 { + // The registers ran out under this argument: the standard still + // counts it against them before it reads the stack. + list.vr_offs = offset + VECTOR_SLOT; + } + let at = align_up(list.stack); + list.stack = at.wrapping_add(BYTES); + at + }; + // SAFETY: the caller's contract puts the argument here. + Float::from_quad(unsafe { read(at) }) +} diff --git a/native/crates/c-shim/src/va_long_double/mod.rs b/native/crates/c-shim/src/va_long_double/mod.rs new file mode 100644 index 0000000..5f6922a --- /dev/null +++ b/native/crates/c-shim/src/va_long_double/mod.rs @@ -0,0 +1,54 @@ +//! `va_arg(ap, long double)`, which Rust's [`VaList`] cannot spell: Rust has +//! no type for x87's 80-bit format and no `VaArgSafe` binary128, so each ABI's +//! own rule reads the argument's bytes and the `printf` engine decodes them +//! ([`Float`]). +//! +//! [`VaList`] is laid out as the platform's `va_list`, the struct or pointer +//! its procedure-call standard defines, and the rule for a 16-byte, +//! 16-aligned argument comes from that same standard. One file per ABI, +//! selected here; the engine never names one. + +use core::ffi::VaList; + +use crate::format::Float; + +#[cfg(target_arch = "aarch64")] +mod aarch64; +#[cfg(target_arch = "riscv64")] +mod riscv64; +#[cfg(target_arch = "x86_64")] +mod x86_64; + +#[cfg(target_arch = "aarch64")] +pub(crate) use aarch64::next; +#[cfg(target_arch = "riscv64")] +pub(crate) use riscv64::next; +#[cfg(target_arch = "x86_64")] +pub(crate) use x86_64::next; + +#[cfg(not(any( + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "x86_64" +)))] +compile_error!("kore's C ABIs are x86_64, aarch64 and riscv64: name this one's long double rule"); + +/// Every kore C ABI's `long double`: sixteen bytes, aligned to sixteen (x87's +/// ten padded out, or binary128's own). +const BYTES: usize = 16; +const ALIGN: usize = 16; + +/// The 16-aligned address at or above `at`. +fn align_up(at: *mut u8) -> *mut u8 { + at.wrapping_add(at.addr().wrapping_neg() & (ALIGN - 1)) +} + +/// The `long double` stored at `at`, as its bits. +/// +/// # Safety +/// `at` is the address of a `long double` argument the caller passed. +unsafe fn read(at: *const u8) -> u128 { + // SAFETY: the caller's contract; unaligned for no reason but uniformity, + // since every rule aligns its slot. + u128::from_le_bytes(unsafe { at.cast::<[u8; BYTES]>().read_unaligned() }) +} diff --git a/native/crates/c-shim/src/va_long_double/riscv64.rs b/native/crates/c-shim/src/va_long_double/riscv64.rs new file mode 100644 index 0000000..d2b872f --- /dev/null +++ b/native/crates/c-shim/src/va_long_double/riscv64.rs @@ -0,0 +1,24 @@ +//! RISC-V LP64D: a `long double` is a 2×XLEN scalar with 2×XLEN alignment, so a +//! variadic one takes an aligned (even, odd) pair of argument registers, or +//! the stack aligned to sixteen (psABI, "Integer Calling Convention" and +//! "va_list, va_start, and va_arg"). The prologue spills the argument +//! registers just below the stack arguments, a pair to a 16-aligned slot, so +//! one pointer walks both and `va_arg` aligns it. + +use super::{BYTES, Float, VaList, align_up, read}; + +const _: () = assert!(core::mem::size_of::>() == core::mem::size_of::<*mut u8>()); + +/// The next argument, a binary128 `long double`. +/// +/// # Safety +/// The next argument `list` holds is a `long double`. +pub(crate) unsafe fn next(list: &mut VaList<'_>) -> Float { + // SAFETY: `VaList` is the platform's `va_list`, one pointer, on this + // target (the size is asserted). + let cursor = unsafe { &mut *(list as *mut VaList<'_>).cast::<*mut u8>() }; + let at = align_up(*cursor); + *cursor = at.wrapping_add(BYTES); + // SAFETY: the caller's contract puts the argument here. + Float::from_quad(unsafe { read(at) }) +} diff --git a/native/crates/c-shim/src/va_long_double/x86_64.rs b/native/crates/c-shim/src/va_long_double/x86_64.rs new file mode 100644 index 0000000..79de2cf --- /dev/null +++ b/native/crates/c-shim/src/va_long_double/x86_64.rs @@ -0,0 +1,32 @@ +//! System V AMD64: a `long double` is class X87, which the caller always +//! passes in memory, in the overflow area, aligned to sixteen (psABI §3.5.7, +//! the `va_arg` algorithm's step for a type that is not passed in +//! registers). + +use super::{BYTES, Float, VaList, align_up, read}; + +/// `va_list`'s one element, as the psABI declares it (`__va_list_tag`). +#[repr(C)] +struct Tag { + gp_offset: u32, + fp_offset: u32, + overflow_arg_area: *mut u8, + reg_save_area: *mut u8, +} + +const _: () = assert!(core::mem::size_of::>() == core::mem::size_of::()); + +/// The next argument, an x87 `long double`. +/// +/// # Safety +/// The next argument `list` holds is a `long double`. +pub(crate) unsafe fn next(list: &mut VaList<'_>) -> Float { + // SAFETY: `VaList` is the platform's `va_list` element on this target + // (the size is asserted), and the overflow area is the caller's frame. + let tag = unsafe { &mut *(list as *mut VaList<'_>).cast::() }; + let at = align_up(tag.overflow_arg_area); + // SAFETY: the caller's contract puts the argument here. + let bits = unsafe { read(at) }; + tag.overflow_arg_area = at.wrapping_add(BYTES); + Float::from_x87(bits) +} diff --git a/native/crates/c-shim/src/wide_symbols.rs b/native/crates/c-shim/src/wide_symbols.rs new file mode 100644 index 0000000..34a3191 --- /dev/null +++ b/native/crates/c-shim/src/wide_symbols.rs @@ -0,0 +1,77 @@ +//! The `` classes and case maps, answered from Rust's Unicode +//! tables: a grammar's scanner asks whether a code point is a letter or a +//! space, and the answer should not depend on a locale kore does not have. + +use core::ffi::c_int; + +/// C's `wint_t`: an unsigned 32-bit code point (or `WEOF`). +type WInt = u32; + +fn class(code: WInt, test: impl Fn(char) -> bool) -> c_int { + c_int::from(char::from_u32(code).is_some_and(test)) +} + +#[unsafe(no_mangle)] +extern "C" fn iswalnum(code: WInt) -> c_int { + class(code, char::is_alphanumeric) +} + +#[unsafe(no_mangle)] +extern "C" fn iswalpha(code: WInt) -> c_int { + class(code, char::is_alphabetic) +} + +#[unsafe(no_mangle)] +extern "C" fn iswblank(code: WInt) -> c_int { + class(code, |c| c == ' ' || c == '\t' || (c.is_whitespace() && !matches!(c, '\n'..='\r' | '\u{85}' | '\u{2028}' | '\u{2029}'))) +} + +#[unsafe(no_mangle)] +extern "C" fn iswdigit(code: WInt) -> c_int { + class(code, |c| c.is_ascii_digit()) +} + +#[unsafe(no_mangle)] +extern "C" fn iswlower(code: WInt) -> c_int { + class(code, char::is_lowercase) +} + +#[unsafe(no_mangle)] +extern "C" fn iswupper(code: WInt) -> c_int { + class(code, char::is_uppercase) +} + +#[unsafe(no_mangle)] +extern "C" fn iswspace(code: WInt) -> c_int { + class(code, char::is_whitespace) +} + +#[unsafe(no_mangle)] +extern "C" fn iswxdigit(code: WInt) -> c_int { + class(code, |c| c.is_ascii_hexdigit()) +} + +/// Printable, not a space, not a letter or digit: ISO's definition. +#[unsafe(no_mangle)] +extern "C" fn iswpunct(code: WInt) -> c_int { + class(code, |c| !c.is_whitespace() && !c.is_alphanumeric() && !c.is_control()) +} + +/// The one-code-point case maps; a mapping that expands to several code +/// points (`ß` → `SS`) leaves the code point as it is, as C's does. +fn single(mut mapped: impl Iterator, code: WInt) -> WInt { + match (mapped.next(), mapped.next()) { + (Some(c), None) => c as WInt, + _ => code, + } +} + +#[unsafe(no_mangle)] +extern "C" fn towlower(code: WInt) -> WInt { + char::from_u32(code).map_or(code, |c| single(c.to_lowercase(), code)) +} + +#[unsafe(no_mangle)] +extern "C" fn towupper(code: WInt) -> WInt { + char::from_u32(code).map_or(code, |c| single(c.to_uppercase(), code)) +} diff --git a/native/crates/dagfs/src/fs.rs b/native/crates/dagfs/src/fs.rs index 55b4b8d..3dd38e4 100644 --- a/native/crates/dagfs/src/fs.rs +++ b/native/crates/dagfs/src/fs.rs @@ -106,6 +106,10 @@ pub struct Fs<'w, 't> { pub(crate) ring_files: u64, /// Why the last whole-file read did or did not take that ring. pub(crate) ring_note: &'static str, + /// This pair's watch plane — the event channel the service formats + /// after the rings in this pair's own window (`crate::watch`) — or + /// `None` on a pair formatted without one. + pub(crate) watch: Option>, } impl<'w, 't> Fs<'w, 't> { @@ -128,11 +132,13 @@ impl<'w, 't> Fs<'w, 't> { // window only through the SPSC protocol. let window: &'static mut [u8] = unsafe { core::slice::from_raw_parts_mut(window.cast::().as_ptr(), window.len()) }; - // An *attach*, not an init: the filesystem row's own `init_window` - // formatted this ring at founding, and `client_rings` fail-closes - // on an unformatted window rather than reading a zeroed page as an - // empty ring. - let rings = client_rings::(window, REQ_CAP, kore_rt::svc::REP_CAP).ok()?; + // An *attach*, not an init: the filesystem row's own formatter + // laid out this window at founding — the rings, then the watch + // plane — and both attaches fail closed on an unformatted window + // rather than reading a zeroed page as an empty ring. + let (pair, tail) = window.split_at_mut(crate::watch::PLANE_AT.min(window.len())); + let rings = client_rings::(pair, REQ_CAP, kore_rt::svc::REP_CAP).ok()?; + let watch = crate::watch::sink(tail).ok(); let kick = rt .bootstrap() .handle(role::SVC_SEND, process_face::REQUEST_KICK)?; @@ -223,9 +229,70 @@ impl<'w, 't> Fs<'w, 't> { ring, ring_files: 0, ring_note: "unused", + watch, }) } + /// This pair's watch plane, if the pair carries one. + pub fn watch_plane(&self) -> Option<&crate::watch::WatchSink<'static>> { + self.watch.as_ref() + } + + /// Arm watch `slot` on the directory `dir`, tagged `tag` (a non-zero + /// name of the client's own for this arming): the plane counts every + /// change to what listing `dir` shows from now on. The view stays the + /// caller's — closing it ends the watch. `Unsupported` on a pair without + /// a plane. + pub async fn watch(&mut self, dir: &crate::View, slot: u8, tag: u32) -> Result<(), Refusal> { + if self.watch.is_none() { + return Err(Refusal::Fs(crate::FsError::Unsupported)); + } + self.checked(&FsRequest::Watch { + handle: dir.handle(), + slot, + tag, + }) + .await + .map(|_| ()) + } + + /// Disarm watch `slot`. + pub async fn unwatch(&mut self, slot: u8) -> Result<(), Refusal> { + self.checked(&FsRequest::Unwatch { slot }).await.map(|_| ()) + } + + /// The plane's wake word, read before a caller drains its sources: a + /// park published against it ([`Self::arm_watch_park`]) is lossless. + /// Zero, and never moving, on a pair without a plane. + pub fn watch_seq(&self) -> u32 { + self.watch.as_ref().map_or(0, crate::watch::WatchSink::seq) + } + + /// The plane's state, `None` on a pair without one (or mid-publish, + /// which the caller treats as nothing new and reads again next pass — + /// the publish ends by moving the wake word). + pub fn watch_state(&self) -> Option { + self.watch.as_ref().and_then(crate::watch::WatchSink::state) + } + + /// Publish this task's intent to wait on the plane, for a caller about to + /// enter one executor wait that every source shares (the multiplexed + /// park, `res/FILES_DESKTOP.md`): the service's next batch then owes + /// this task a notice on its doorbell. `false` when the plane already + /// moved past `seen` — drain again instead of waiting. Always `true` on + /// a pair without a plane, which has nothing to wait for. + pub fn arm_watch_park(&self, seen: u32) -> bool { + self.watch.as_ref().is_none_or(|sink| sink.park(seen)) + } + + /// Withdraw the intent [`Self::arm_watch_park`] published, however the + /// wait ended, so a departed waiter is never charged a notice. + pub fn withdraw_watch_park(&self) { + if let Some(sink) = self.watch.as_ref() { + sink.unpark(); + } + } + /// The bulk data window, read side. Bytes a verb landed here are valid /// until the next request that uses the window. pub fn window(&self) -> &[u8] { diff --git a/native/crates/dagfs/src/lib.rs b/native/crates/dagfs/src/lib.rs index 4800948..0597035 100644 --- a/native/crates/dagfs/src/lib.rs +++ b/native/crates/dagfs/src/lib.rs @@ -67,6 +67,8 @@ mod view; #[cfg(feature = "runtime")] mod walk; #[cfg(feature = "runtime")] +pub mod watch; +#[cfg(feature = "runtime")] pub mod wire; #[cfg(feature = "runtime")] diff --git a/native/crates/dagfs/src/protocol.rs b/native/crates/dagfs/src/protocol.rs index abfbe60..3125092 100644 --- a/native/crates/dagfs/src/protocol.rs +++ b/native/crates/dagfs/src/protocol.rs @@ -193,6 +193,22 @@ pub enum FsRequest { Sync { handle: Handle, }, + /// Arm watch `slot` (below [`FS_WATCH_SLOTS`](kore_abi::svc::FS_WATCH_SLOTS)) + /// on the directory `handle` names, tagged `tag` (non-zero): the pair's + /// watch plane ([`crate::watch`]) counts every change to what a listing + /// of that directory shows, under `tag`, until the slot is re-armed or + /// disarmed or the watch ends. The handle stays the client's; its route + /// is revalidated before every count, and one that stops proving ends the + /// watch rather than reporting through a route the client lost. + Watch { + handle: Handle, + slot: u8, + tag: u32, + }, + /// Disarm watch `slot`, armed or not. + Unwatch { + slot: u8, + }, /// Hold this session's whole-tree commits until the window closes. /// /// The durable contract serializes session zero's entire tree after every @@ -631,6 +647,17 @@ impl SvcProtocol for DagfsProtocol { /// execution pipeline and made boot topology scale with the command set. const SERVE_PAIRS: usize = 2; + /// The rings, then the watch plane ([`crate::watch`]) in the same window: + /// every filesystem pair carries its change channel, so a client never + /// needs a capability beyond the pair to learn that a directory changed. + fn init_pair_window( + window: &mut [u8], + req_cap: u32, + rep_cap: u32, + ) -> Result<(), kore_ring::spsc::RingError> { + crate::watch::init_pair_window(window, req_cap, rep_cap) + } + fn reply_matches(request: &FsReqWire, response: &FsResponse, grants: u8) -> bool { if grants != 0 { return false; diff --git a/native/crates/dagfs/src/text.rs b/native/crates/dagfs/src/text.rs index 49ac01e..3af8d2f 100644 --- a/native/crates/dagfs/src/text.rs +++ b/native/crates/dagfs/src/text.rs @@ -1,6 +1,7 @@ //! The two text helpers every command needs and none should re-spell: a -//! bounded whitespace splitter over one command line, and a bounded line -//! buffer for composing an answer out of pieces. +//! bounded whitespace splitter over one command line (with `"quoted"` words, +//! so a name with spaces can be named), and a bounded line buffer for +//! composing an answer out of pieces. /// A whitespace splitter over one completed command line. /// @@ -18,13 +19,27 @@ impl<'a> Args<'a> { Self { rest: line } } - /// The next space-separated word, or `None` at the end of the line. + /// The next word, or `None` at the end of the line. A word is a run of + /// non-spaces, or — opened by `"` — everything up to the next `"`, + /// spaces included and quotes excluded, so `inspect "Untitled Folder"` + /// names the folder Files makes. A quote left open runs to the end of + /// the line. Borrowed either way: a quoted word is a span of the line. pub fn next(&mut self) -> Option<&'a [u8]> { let start = self.skip_spaces(); if start >= self.rest.len() { self.rest = &self.rest[self.rest.len()..]; return None; } + if self.rest[start] == b'"' { + let open = start + 1; + let close = self.rest[open..] + .iter() + .position(|&byte| byte == b'"') + .map_or(self.rest.len(), |at| open + at); + let word = &self.rest[open..close]; + self.rest = &self.rest[(close + 1).min(self.rest.len())..]; + return Some(word); + } let mut end = start; while end < self.rest.len() && self.rest[end] != b' ' { end += 1; @@ -128,6 +143,21 @@ mod tests { assert_eq!(args.rest(), b""); } + #[test] + fn a_quoted_word_keeps_its_spaces_and_drops_its_quotes() { + let mut args = Args::new(b"remove \"Untitled Folder\" next \"\" \"open to the end"); + assert_eq!(args.next(), Some(&b"remove"[..])); + assert_eq!(args.next(), Some(&b"Untitled Folder"[..])); + assert_eq!(args.next(), Some(&b"next"[..])); + assert_eq!(args.next(), Some(&b""[..]), "an empty quote is an empty word"); + assert_eq!(args.next(), Some(&b"open to the end"[..])); + assert_eq!(args.next(), None); + // A quote inside a word is part of it: only a word's first byte opens. + let mut args = Args::new(b"a\"b c"); + assert_eq!(args.next(), Some(&b"a\"b"[..])); + assert_eq!(args.next(), Some(&b"c"[..])); + } + #[test] fn a_line_buffer_truncates_rather_than_grows() { let mut line = LineBuf::<8>::new(); diff --git a/native/crates/dagfs/src/watch.rs b/native/crates/dagfs/src/watch.rs new file mode 100644 index 0000000..1f9b68a --- /dev/null +++ b/native/crates/dagfs/src/watch.rs @@ -0,0 +1,251 @@ +//! The **watch plane**: a filesystem pair's change channel (`res/EVENT.md`). +//! +//! A client that shows a directory — the Files explorer — needs to learn when +//! what it shows went stale without asking. The plane is the +//! service→client direction of one pair, as the window pair's event channel is +//! the compositor→client one: an event channel formatted after the +//! request/reply rings in the pair's **own** window, so it rides the lease +//! both sides already hold and needs no capability of its own. +//! +//! - **State is level.** [`FsWatchState`] holds, per slot, the tag the slot +//! was armed with and a wrapping count of changes since; the service +//! republishes it whole. Several changes between two service passes advance +//! a count once — a client lists again, it does not replay. +//! - **The wake is conditional.** The service commits after the request (or +//! teardown) that changed a watched directory and owes the client a notice +//! only if it had parked on the plane; a client that is running pays the +//! service nothing. +//! - **No authority, no names.** A watch is armed on a directory handle the +//! client holds; the service revalidates that handle's route before every +//! count and ends the watch the moment it stops proving. The plane says +//! *that* a listing may differ, never *what* it holds — the client lists +//! through its own handle. +//! +//! The record ring is reserved ([`FS_WATCH_RECORDS`], never pushed) so a later +//! per-entry change record lands without moving the state or the offset. + +use kore_abi::svc::{FS_WATCH_RECORDS, FS_WATCH_SLOTS, FsWatchRecord, FsWatchState}; +use kore_ring::event::{EventSink, EventSource}; +use kore_ring::spsc::RingError; +use kore_rt::svc::{REP_CAP, REQ_CAP, SvcProtocol}; + +use crate::DagfsProtocol; + +/// The producer half: the service's. +pub type WatchSource<'a> = EventSource<'a, FsWatchState, FsWatchRecord>; +/// The consumer half: the client's. +pub type WatchSink<'a> = EventSink<'a, FsWatchState, FsWatchRecord>; + +/// Where the plane begins in a pair window: right after the canonical +/// request/reply rings (both 64-byte rounded, so the plane is line-aligned). +pub const PLANE_AT: usize = kore_rt::svc::window_bytes::( + ::REQUEST_CAPACITY, + ::RESPONSE_CAPACITY, +); + +/// Bytes the plane occupies. +pub const PLANE_BYTES: usize = + kore_ring::event::region_bytes::(FS_WATCH_RECORDS); + +/// The smallest pair window that carries the plane. +pub const WINDOW_BYTES: usize = PLANE_AT + PLANE_BYTES; + +// Whoever sizes a filesystem pair window asserts it holds `WINDOW_BYTES` +// against its own constant (exec's `FS_CONTROL_BYTES`); the formatter refuses +// a smaller window at founding. Line alignment is the layout's own invariant. +const _: () = assert!(PLANE_AT % 64 == 0); +const _: () = assert!( + ::REQUEST_CAPACITY == REQ_CAP + && ::RESPONSE_CAPACITY == REP_CAP +); + +/// Format a whole filesystem pair window — the rings, then the plane — before +/// either side maps it. The protocol's [`SvcProtocol::init_pair_window`]. +pub fn init_pair_window(window: &mut [u8], req_cap: u32, rep_cap: u32) -> Result<(), RingError> { + kore_rt::svc::init_window::(window, req_cap, rep_cap)?; + let at = kore_rt::svc::window_bytes::(req_cap, rep_cap); + let plane = window.get_mut(at..).ok_or(RingError::TooSmall)?; + kore_ring::event::init::(plane, FS_WATCH_RECORDS) +} + +/// The service's half over its mapping of the pair window's tail (`aux` is +/// what follows the rings, as `attach_peer_aux` hands it over). +pub fn source(aux: &mut [u8]) -> Result, RingError> { + kore_ring::event::source(aux.get_mut(..PLANE_BYTES).ok_or(RingError::TooSmall)?) +} + +/// The client's half over its mapping of the same tail. +pub fn sink(aux: &mut [u8]) -> Result, RingError> { + kore_ring::event::sink(aux.get_mut(..PLANE_BYTES).ok_or(RingError::TooSmall)?) +} + +/// What one slot's watch did since a [`Cursor`] last looked. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Seen { + /// Nothing the cursor has not seen. + Same, + /// The directory changed: list it again. + Changed, + /// The watch ended (its route stopped proving, or the directory went): + /// what was shown cannot be refreshed through this watch. + Ended, +} + +/// A client's view of one slot: the tag it armed and the last count it acted +/// on. A slot is read only under the cursor's own tag, so the counts the +/// slot carried for an earlier arming — or for nobody — never read as news. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Cursor { + slot: u8, + tag: u32, + seen: u32, + ended: bool, +} + +impl Cursor { + /// A cursor for `slot` armed with `tag`; `None` for a slot past the plane + /// or the zero tag (the plane's "disarmed"). + pub const fn new(slot: u8, tag: u32) -> Option { + if slot as usize >= FS_WATCH_SLOTS || tag == 0 { + return None; + } + Some(Self { + slot, + tag, + seen: 0, + ended: false, + }) + } + + pub const fn slot(&self) -> u8 { + self.slot + } + + pub const fn tag(&self) -> u32 { + self.tag + } + + /// Compare `state` with what this cursor last acted on, and take it as + /// seen. A state still showing another tag in the slot (the arming has + /// not landed, or the slot was re-armed by someone else's request on + /// this pair) is [`Seen::Same`]: nothing of this watch is in it yet. + pub fn observe(&mut self, state: &FsWatchState) -> Seen { + let slot = usize::from(self.slot); + if state.tag[slot] != self.tag { + return Seen::Same; + } + if state.ended & (1 << slot) != 0 { + if self.ended { + return Seen::Same; + } + self.ended = true; + return Seen::Ended; + } + let changes = state.changes[slot]; + if changes == self.seen { + return Seen::Same; + } + self.seen = changes; + Seen::Changed + } +} + +/// The plane's state as a producer edits it — the service's one batch verb +/// over a slot. Kept beside the layout so both halves read one definition. +pub mod state { + use super::{FS_WATCH_SLOTS, FsWatchState}; + + /// Arm `slot` under `tag`: no changes yet, not ended. + pub fn arm(state: &mut FsWatchState, slot: usize, tag: u32) { + if slot < FS_WATCH_SLOTS { + state.tag[slot] = tag; + state.changes[slot] = 0; + state.ended &= !(1 << slot); + } + } + + /// Disarm `slot`. + pub fn disarm(state: &mut FsWatchState, slot: usize) { + if slot < FS_WATCH_SLOTS { + state.tag[slot] = 0; + state.changes[slot] = 0; + state.ended &= !(1 << slot); + } + } + + /// One more change under `slot`'s arming. + pub fn changed(state: &mut FsWatchState, slot: usize) { + if slot < FS_WATCH_SLOTS { + state.changes[slot] = state.changes[slot].wrapping_add(1); + } + } + + /// `slot`'s watch ended; its tag stays, so the client that armed it + /// learns that it did. + pub fn end(state: &mut FsWatchState, slot: usize) { + if slot < FS_WATCH_SLOTS { + state.ended |= 1 << slot; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::vec; + + #[repr(align(64))] + struct Window([u8; WINDOW_BYTES]); + + #[test] + fn the_plane_follows_the_rings_and_both_halves_attach() { + let mut window = Window([0; WINDOW_BYTES]); + init_pair_window(&mut window.0, REQ_CAP, REP_CAP).unwrap(); + // The rings are exactly where every existing client expects them. + let (pair, aux) = window.0.split_at_mut(PLANE_AT); + assert!(kore_rt::svc::client_rings::(pair, REQ_CAP, REP_CAP).is_ok()); + let base = aux.as_mut_ptr(); + // SAFETY: the two halves alias the plane only through the channel + // protocol, as the cross-space form does over two mappings. + let alias = unsafe { core::slice::from_raw_parts_mut(base, PLANE_BYTES) }; + let mut source = source(aux).unwrap(); + let sink = sink(alias).unwrap(); + assert_eq!(sink.state(), Some(FsWatchState::default())); + source.state(|s| state::arm(s, 1, 7)); + assert!(!source.commit(), "nobody parked: no notice owed"); + let seen = sink.seq(); + assert!(sink.park(seen)); + source.state(|s| state::changed(s, 1)); + assert!(source.commit(), "a parked client is owed its notice"); + assert_eq!(sink.state().unwrap().changes[1], 1); + } + + #[test] + fn a_window_too_small_for_the_plane_is_refused() { + let mut short = vec![0u8; PLANE_AT + 64 + 64]; + let at = short.as_ptr().align_offset(64); + let window = &mut short[at..at + PLANE_AT]; + assert_eq!(init_pair_window(window, REQ_CAP, REP_CAP), Err(RingError::TooSmall)); + } + + #[test] + fn a_cursor_reads_only_its_own_arming() { + let mut cursor = Cursor::new(2, 9).unwrap(); + let mut s = FsWatchState::default(); + // The slot's previous watch left counts: not this cursor's news. + state::arm(&mut s, 2, 8); + state::changed(&mut s, 2); + assert_eq!(cursor.observe(&s), Seen::Same); + state::arm(&mut s, 2, 9); + assert_eq!(cursor.observe(&s), Seen::Same); + state::changed(&mut s, 2); + state::changed(&mut s, 2); + assert_eq!(cursor.observe(&s), Seen::Changed, "two changes coalesce"); + assert_eq!(cursor.observe(&s), Seen::Same); + state::end(&mut s, 2); + assert_eq!(cursor.observe(&s), Seen::Ended); + assert_eq!(cursor.observe(&s), Seen::Same, "an ending is reported once"); + assert!(Cursor::new(FS_WATCH_SLOTS as u8, 1).is_none()); + assert!(Cursor::new(0, 0).is_none()); + } +} diff --git a/native/crates/dagfs/src/wire.rs b/native/crates/dagfs/src/wire.rs index 638461e..4cdae6f 100644 --- a/native/crates/dagfs/src/wire.rs +++ b/native/crates/dagfs/src/wire.rs @@ -23,6 +23,8 @@ //! | `OpenAt` | parent | — | — | — | `name` | //! | `OpenGrant` | — | grant id | — | — | — | //! | `Close`/`Stat`/`Sync` | object | — | — | — | — | +//! | `Watch` | directory | — | — | tag (slot in `kind`) | — | +//! | `Unwatch` | — | — | — | — (slot in `kind`) | — | //! | `List` | directory | — | — | cursor | — | //! | `Parents` | object | — | — | cursor | — | //! | `Locate` | anchor | stable identity | — | — | — | @@ -448,6 +450,16 @@ pub fn encode_req(req: &FsRequest) -> FsReqWire { w.op = FsReqWire::OP_SYNC; w.handle = U64::new(handle.to_bits()); } + FsRequest::Watch { handle, slot, tag } => { + w.op = FsReqWire::OP_WATCH; + w.handle = U64::new(handle.to_bits()); + w.kind = slot; + w.length = U32::new(tag); + } + FsRequest::Unwatch { slot } => { + w.op = FsReqWire::OP_UNWATCH; + w.kind = slot; + } FsRequest::DeferCommits { hold } => { w.op = FsReqWire::OP_DEFER_COMMITS; w.kind = u8::from(hold); @@ -713,6 +725,8 @@ pub fn decode_req(w: &FsReqWire) -> Option { | FsReqWire::OP_DEFER_COMMITS | FsReqWire::OP_SET_SYSTEM_SEED | FsReqWire::OP_STOR_MAINTAIN + | FsReqWire::OP_WATCH + | FsReqWire::OP_UNWATCH ) && w.kind != 0 { return None; @@ -914,6 +928,35 @@ pub fn decode_req(w: &FsReqWire) -> Option { } FsReqWire::OP_REVOKE => FsRequest::Revoke { grant }, FsReqWire::OP_SYNC => FsRequest::Sync { handle }, + FsReqWire::OP_WATCH => { + // `kind` is the slot and `length` the tag; nothing else rides. A + // slot past the plane or a zero tag (the plane's "disarmed") is a + // framing mismatch, not a request to guess at. + if usize::from(w.kind) >= kore_abi::svc::FS_WATCH_SLOTS + || w.length.get() == 0 + || w.window_offset.get() != 0 + || w.handle2.get() != 0 + || w.offset.get() != 0 + || w.expected_revision.get() != 0 + || !names_are_empty(w) + { + return None; + } + FsRequest::Watch { + handle, + slot: w.kind, + tag: w.length.get(), + } + } + FsReqWire::OP_UNWATCH => { + if usize::from(w.kind) >= kore_abi::svc::FS_WATCH_SLOTS + || !control_payload_is_empty(w) + || w.window_offset.get() != 0 + { + return None; + } + FsRequest::Unwatch { slot: w.kind } + } FsReqWire::OP_DEFER_COMMITS => { // `kind` is the whole payload: a deferral names no object, and a // value outside the two-state vocabulary is a framing mismatch. @@ -1737,6 +1780,8 @@ pub fn decode_rep_for(request: &FsRequest, w: &FsRepWire) -> Option<(FsResponse, | FsRequest::DetachEdge { .. } | FsRequest::Revoke { .. } | FsRequest::Sync { .. } + | FsRequest::Watch { .. } + | FsRequest::Unwatch { .. } | FsRequest::BeginReplace { .. } | FsRequest::WriteReplacement { .. } | FsRequest::AbortReplace { .. } @@ -1876,6 +1921,8 @@ pub fn reply_matches(request: &FsRequest, response: &FsResponse) -> bool { | FsRequest::DetachEdge { .. } | FsRequest::Revoke { .. } | FsRequest::Sync { .. } + | FsRequest::Watch { .. } + | FsRequest::Unwatch { .. } | FsRequest::BeginReplace { .. } | FsRequest::WriteReplacement { .. } | FsRequest::AbortReplace { .. } @@ -2020,6 +2067,66 @@ mod tests { assert_eq!(decode_req(&stray), None); } + /// A watch carries its directory, its slot and its tag and nothing else; + /// an unwatch carries its slot alone. Both answer `Done`, and a slot past + /// the plane, the zero tag, or any stray operand is refused. + #[test] + fn watch_verbs_round_trip_and_refuse_stray_operands() { + let slots = kore_abi::svc::FS_WATCH_SLOTS as u8; + for (slot, tag) in [(0, 1), (slots - 1, u32::MAX)] { + let watch = FsRequest::Watch { + handle: Handle::from_bits(0x1_0000_0003), + slot, + tag, + }; + let unwatch = FsRequest::Unwatch { slot }; + for request in [watch, unwatch] { + assert!(!request.needs_window()); + assert_eq!(decode_req(&encode_req(&request)), Some(request)); + assert!(reply_matches(&request, &FsResponse::Done)); + assert_eq!( + decode_rep_for(&request, &encode_rep(&FsResponse::Done, 0)), + Some((FsResponse::Done, 0)) + ); + } + } + let wire = encode_req(&FsRequest::Watch { + handle: Handle::from_bits(9), + slot: 1, + tag: 5, + }); + let strays: [fn(&mut FsReqWire); 8] = [ + |w| w.kind = kore_abi::svc::FS_WATCH_SLOTS as u8, + |w| w.length = U32::new(0), + |w| w.window_offset = U32::new(4), + |w| w.handle2 = U64::new(1), + |w| w.offset = U64::new(1), + |w| w.expected_revision = U64::new(1), + |w| w.rights = 1, + |w| { + w.name[0] = b'x'; + w.name_len = 1; + }, + ]; + for stray in strays { + let mut bad = wire; + stray(&mut bad); + assert_eq!(decode_req(&bad), None); + } + let wire = encode_req(&FsRequest::Unwatch { slot: 2 }); + let strays: [fn(&mut FsReqWire); 4] = [ + |w| w.kind = kore_abi::svc::FS_WATCH_SLOTS as u8, + |w| w.handle = U64::new(1), + |w| w.length = U32::new(1), + |w| w.window_offset = U32::new(1), + ]; + for stray in strays { + let mut bad = wire; + stray(&mut bad); + assert_eq!(decode_req(&bad), None); + } + } + /// Every maintenance action survives the wire; only a forced refresh /// carries operands, and an operand on any other action — or a range /// past `u32` — is refused. diff --git a/native/crates/ld/tests/built_images.rs b/native/crates/ld/tests/built_images.rs index f196515..d428f84 100644 --- a/native/crates/ld/tests/built_images.rs +++ b/native/crates/ld/tests/built_images.rs @@ -339,6 +339,8 @@ fn check_variant(root: &Path, variant: &str, required: bool) -> us ("sleep", ImageKind::Executable), ("status", ImageKind::NeedsLibrary("libpci_names.so")), ("lspci", ImageKind::NeedsLibrary("libpci_names.so")), + ("ioctl", ImageKind::NeedsLibrary("libpci_names.so")), + ("vdev-test", ImageKind::Executable), ("help", ImageKind::Executable), ("window", ImageKind::Executable), ("clear", ImageKind::Executable), diff --git a/native/crates/ld/tests/shipping_list.rs b/native/crates/ld/tests/shipping_list.rs index 10dbf43..93dc1af 100644 --- a/native/crates/ld/tests/shipping_list.rs +++ b/native/crates/ld/tests/shipping_list.rs @@ -69,9 +69,9 @@ const UNSHIPPED: &[(&str, &str)] = &[ ( "accel-spmd", "backend crate: the CPU-SIMD `Accelerator` impl over kore-rt's SPMD \ - region launch, the third tenant crates/abi/src/svc.rs declares. No boot \ - row yet — its compute step (`SpmdBackend::run_pending`) needs a serve \ - loop to drive it, and that lands with the row", + region launch. It is the backend of the virtual accelerator \ + (crates/vdev-accel), linked into user/services/vdev, whose founded \ + image is kore-svc-vdev; it is not a boot row of its own", ), ( "swap-nvme", diff --git a/native/crates/rt/src/cap/mod.rs b/native/crates/rt/src/cap/mod.rs index 2266846..e5e1d39 100644 --- a/native/crates/rt/src/cap/mod.rs +++ b/native/crates/rt/src/cap/mod.rs @@ -1754,19 +1754,20 @@ pub enum QuiesceFault { Unenforced, } -/// Why a writable lease did not become an executable successor. +/// Why a consuming lease transition did not yield its successor: a +/// finalization, the return to writable, or a freeze. /// -/// Finalization has one abort door: a refusal before commit returns the exact -/// affine source. Once the kernel has retired that source and begun clearing +/// Each has one abort door: a refusal before commit returns the exact affine +/// source. Once the kernel has retired that source and begun clearing /// aliases, rollback is impossible; a later containment or publication /// failure is therefore reported as [`Consumed`](Self::Consumed), never as a -/// writable lease that no longer exists. +/// lease that no longer exists. #[must_use] #[derive(Debug)] pub enum FinalizeExecError { - /// The session did not commit; the writable source is unchanged. + /// The session did not commit; the source is unchanged. Refused(T), - /// The source was consumed, but no executable successor was published. + /// The source was consumed, but no successor was published. Consumed, } @@ -2854,17 +2855,6 @@ pub struct StashedLease { owned: OwnedSlot, } -impl StashedLease { - /// Adopt an already-activated owned slot the kernel just published as a - /// writable lease — the executable → writable return's landing - /// (`res/IMAGE.md` §6a). Crate-internal: the caller must have proved - /// the record's shape first (`writable_result_state`). - #[cfg(target_os = "none")] - pub(crate) fn from_owned(owned: OwnedSlot) -> Self { - Self { owned } - } -} - /// Transferable memory custody: an allocation lease or a scoped mapping loan. /// A route never acquires the allocation/release authority of a MemLease. #[derive(Debug)] diff --git a/native/crates/rt/src/cap/verbs.rs b/native/crates/rt/src/cap/verbs.rs index d6603ef..f07b911 100644 --- a/native/crates/rt/src/cap/verbs.rs +++ b/native/crates/rt/src/cap/verbs.rs @@ -4178,6 +4178,102 @@ mod metal { None } + /// Move a capability a client delivered into a fixed landing slot — a + /// service's convey bank, whose post must stand again for the next + /// request — into a fresh runtime-owned slot, so the service can keep it + /// past the request that carried it. + /// + /// The move rides a channel minted from `space` for it alone: the + /// landing is exact-moved through that channel's send into a post over + /// the new owner, and the channel is then retired. An exact move keeps + /// the record's rights and lineage, so retiring the private channel + /// reaches nothing it carried. Only the kinds the kernel moves exactly + /// relocate: memory leases and views, fabric routes, channel endpoints. + #[cfg(target_os = "none")] + pub fn relocate_landed( + space: &ChannelSpaceToken, + landed: SlotRef, + ) -> Result { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + use kore_abi::session::Status; + + fn committed(session: &SessionBuilder) -> bool { + TrapTransport::new() + .commit_now(session) + .is_ok_and(|done| done.status() == Status::Ok && done.failing_index() == 0) + } + + let crate::cap::ChannelPair { root, send, recv } = + mint_channel(space).map_err(|_| RelocateError::Unmoved)?; + let mut owner = crate::cap::ReturnedCapability::reserve(); + let mut armed = false; + let mut sent = false; + if let Some(owner) = owner.as_mut() { + let mut post = SessionBuilder::new(); + if post.post_recv(recv.slot(), owner.target()).is_ok() && committed(&post) { + owner.arm_post(); + armed = true; + let mut moving = SessionBuilder::new(); + sent = moving.send_exact(send.slot(), landed).is_ok() && committed(&moving); + } + } + // The root is the private channel's only pivot: revoking it ends + // both endpoints, and with them any post still naming the owner. + let closed = match root.try_retire() { + Ok(outcome) => outcome.is_closed(), + Err(root) => { + core::mem::forget(root); + false + } + }; + let _ = send.try_retire(); + let _ = recv.try_retire(); + let Some(mut owner) = owner else { + return Err(RelocateError::Unmoved); + }; + if sent { + if matches!( + take_delivery(owner.target()), + Some(crate::cap::RawDelivery::Cap) + ) { + owner.activate(); + return Ok(owner); + } + // A committed exact move into a standing post lands, so this is + // a record the runtime cannot classify: keep its name out of the + // allocator for the task's life rather than reopen it. + core::mem::forget(owner); + return Err(RelocateError::Lost); + } + if armed && !closed { + // The post may still stand over this name. + core::mem::forget(owner); + } else if armed { + owner.disarm_post(); + } + Err(RelocateError::Unmoved) + } + + #[cfg(not(target_os = "none"))] + pub fn relocate_landed( + _space: &ChannelSpaceToken, + _landed: SlotRef, + ) -> Result { + Err(RelocateError::Unmoved) + } + + /// Why [`relocate_landed`] produced no owner. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum RelocateError { + /// Nothing moved: the delivery is still in its landing, for the + /// caller's own custody to keep or retire. + Unmoved, + /// The move committed but its result could not be read back; the + /// landing is vacant and the would-be owner is quarantined. + Lost, + } + /// **The metal take-view** (`res/USERLAND.md` §5.6; Phase-A checkpoint /// C) — read a delivered `LeasedRegion` view record off the R/O /// cap-slot alias at [`SLOTS`](kore_abi::task::geom::SLOTS) alias and return @@ -5226,7 +5322,7 @@ mod metal { } ExecFinalizeCompletion::Replaced => { code.release_after_proven_move(); - Ok(StashedLease::from_owned(output)) + Ok(StashedLease::new(output)) } } } @@ -5238,6 +5334,78 @@ mod metal { Err(FinalizeExecError::Refused(code)) } + /// Freeze a runtime-owned writable lease (`res/IMAGE.md` §6b): consume + /// it with every alias and descendant capability of it, and return the + /// frozen successor — the same window and rights, which no lessor may + /// write again while it lives. This is a registrar's exclusive intake: + /// what it then validates is what every later reader of the window reads. + /// + /// Affine slot ownership is preserved on either outcome, as + /// [`finalize_stashed_exec`] preserves it: a refusal returns the exact + /// source, and a consumed source reports + /// [`Consumed`](FinalizeExecError::Consumed) with no authority back. + #[cfg(target_os = "none")] + pub fn freeze_stashed( + mut lease: StashedLease, + ) -> Result> { + let mut output = match crate::cap::OwnedSlot::reserve() { + Some(output) => output, + None => return Err(FinalizeExecError::Refused(lease)), + }; + let Some(source) = lease.slot() else { + return Err(FinalizeExecError::Refused(lease)); + }; + match freeze_slot(source, &mut output) { + ExecFinalizeCompletion::Refused => Err(FinalizeExecError::Refused(lease)), + ExecFinalizeCompletion::Consumed => { + lease.owned.release_after_proven_move(); + Err(FinalizeExecError::Consumed) + } + ExecFinalizeCompletion::Replaced => { + // The committed kernel transition cleared `source`; only its + // dynamic allocator reservation remains to release. + lease.owned.release_after_proven_move(); + Ok(StashedLease::new(output)) + } + } + } + + /// Off metal there is no freeze session to issue. + #[cfg(not(target_os = "none"))] + pub fn freeze_stashed( + lease: StashedLease, + ) -> Result> { + Err(FinalizeExecError::Refused(lease)) + } + + /// [`freeze_stashed`] for a lease a caller delivered into a fixed landing + /// slot (a service's convey bank): the frozen successor moves into a + /// fresh runtime-owned slot, and the landing is left vacant for its post + /// to stand again. A registrar that validates bytes a client wrote takes + /// them this way, so no alias the client kept can write them afterwards. + /// + /// `Refused` leaves the delivery in `slot`, untouched, for the caller's + /// own custody to retire; `Consumed` reports a vacant `slot` with no + /// successor. + #[cfg(target_os = "none")] + pub fn freeze_landed(slot: SlotRef) -> Result> { + let mut output = match crate::cap::OwnedSlot::reserve() { + Some(output) => output, + None => return Err(FinalizeExecError::Refused(())), + }; + match freeze_slot(slot, &mut output) { + ExecFinalizeCompletion::Refused => Err(FinalizeExecError::Refused(())), + ExecFinalizeCompletion::Consumed => Err(FinalizeExecError::Consumed), + ExecFinalizeCompletion::Replaced => Ok(StashedLease::new(output)), + } + } + + /// Off metal there is no freeze session to issue. + #[cfg(not(target_os = "none"))] + pub fn freeze_landed(_slot: SlotRef) -> Result> { + Err(FinalizeExecError::Refused(())) + } + #[cfg(target_os = "none")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ExecFinalizeCompletion { @@ -5274,10 +5442,10 @@ mod metal { } } - /// The writable successor's shape proof — [`exec_result_state`]'s twin. - /// The rights differ (`MODIFY` is back), and checking the exact set is - /// what makes a wrong-shaped record `Malformed` rather than silently - /// accepted. + /// The full-lifecycle successor's shape proof — a return's or a + /// freeze's, [`exec_result_state`]'s twin. The rights differ (`MODIFY` + /// is there), and checking the exact set is what makes a wrong-shaped + /// record `Malformed` rather than silently accepted. #[cfg(target_os = "none")] fn writable_result_state(output: SlotRef) -> ExecResultState { use kore_abi::cap::{CapRecord, ResourceKind}; @@ -5331,6 +5499,34 @@ mod metal { } } + /// Freeze `source` into `output`, mirroring [`return_writable_slot`]: + /// the successor carries the full lifecycle grant, so the same shape + /// proof classifies it. + #[cfg(target_os = "none")] + fn freeze_slot(source: SlotRef, output: &mut crate::cap::OwnedSlot) -> ExecFinalizeCompletion { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + + let mut b = SessionBuilder::new(); + if b.freeze(source, output.slot()).is_err() { + return ExecFinalizeCompletion::Refused; + } + let Ok(committed) = TrapTransport::new().commit_now(&b) else { + return ExecFinalizeCompletion::Refused; + }; + if committed.status() != kore_abi::session::Status::Ok || committed.failing_index() != 0 { + return ExecFinalizeCompletion::Refused; + } + match writable_result_state(output.slot()) { + ExecResultState::Replaced => ExecFinalizeCompletion::Replaced, + ExecResultState::Vacant => ExecFinalizeCompletion::Consumed, + ExecResultState::Malformed => { + output.quarantine_unknown_publish(); + ExecFinalizeCompletion::Consumed + } + } + } + #[cfg(target_os = "none")] fn finalize_exec_slot( source: SlotRef, diff --git a/native/crates/rt/src/clock/mod.rs b/native/crates/rt/src/clock/mod.rs index a249927..ba1dbc9 100644 --- a/native/crates/rt/src/clock/mod.rs +++ b/native/crates/rt/src/clock/mod.rs @@ -92,6 +92,24 @@ pub(crate) fn monotonic_nanos() -> Option { uptime().ok().map(Uptime::as_nanos) } +/// Monotonic nanoseconds now, for a thread that names a deadline to the +/// kernel: the calibrated counter where the task has one, and the kernel's +/// own stamp where it has none. +/// +/// A target without a calibrated user clock (an emulator with no invariant +/// counter, a thread founded before the clock page) can still park against +/// a deadline, because the kernel keeps the time: a sleep to an instant +/// already past resolves at once and returns the kernel's clock, in whole +/// milliseconds. That costs a trap and is a millisecond coarse, so the +/// answer may be up to a millisecond behind; a deadline made from it is +/// never early by more than that. `None` only where the thread cannot trap +/// at all (a host build). +pub fn deadline_base_nanos() -> Option { + monotonic_nanos().or_else(|| { + crate::cap::verbs::sleep_until(1).map(|stamp| (stamp >> 32).saturating_mul(1_000_000)) + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/native/crates/rt/src/crash.rs b/native/crates/rt/src/crash.rs index b06b58e..f61c060 100644 --- a/native/crates/rt/src/crash.rs +++ b/native/crates/rt/src/crash.rs @@ -112,10 +112,23 @@ pub fn register( /// Returning executables share the same recursion-safe diagnostic path. pub fn panic(info: &core::panic::PanicInfo<'_>) -> ! { - if crate::entry::begin_panic() { + report(info); + crate::thread::exit(crate::ExitCode::PANIC) +} + +/// [`panic`]'s report without its exit: publish this thread's panic record +/// once (a panic raised while reporting reports nothing more), and return. +/// +/// For a panic handler that must act between the report and the thread's +/// end — a host that ends the whole process on a worker's panic tells the +/// host only once the record is published, so the host's barrier cannot cut +/// the report short. `true` when this call published the record. +pub fn report(info: &core::panic::PanicInfo<'_>) -> bool { + let first = crate::entry::begin_panic(); + if first { report_panic_info(info); } - crate::thread::exit(crate::ExitCode::PANIC) + first } /// Best-effort software report. The entry panic handler has claimed recursion diff --git a/native/crates/rt/src/entry.rs b/native/crates/rt/src/entry.rs index aff740d..00b9b80 100644 --- a/native/crates/rt/src/entry.rs +++ b/native/crates/rt/src/entry.rs @@ -176,6 +176,28 @@ const PROCESS_STDIO_ATTACHMENT: usize = 1 << 1; #[cfg(any(test, target_os = "none"))] const PROCESS_STDIO_HOST_ATTACHMENT: usize = 1 << 2; +/// The free part of a task's link window ([`Rt::link_free_span`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LinkSpan { + /// The task's whole link window. + pub base: u64, + /// Its end below the module page. + pub limit: u64, + /// The first free address: above everything the startup linker placed. + pub free: u64, +} + +/// Why a task has no free link span ([`Rt::link_free_span`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LinkSpanError { + /// The task has no `geom::LINK` window. + NoWindow, + /// The window has no module page. + NoModulePage, + /// Nothing free above what the startup linker placed. + Full, +} + /// Atomically spend one runtime attachment name. /// /// This is deliberately a bit in the process-root @@ -255,6 +277,48 @@ impl Rt { None } + /// **The free part of this task's link window** (`geom::LINK`): where a + /// program may place mappings of its own after startup — a Wasmtime + /// engine's guest mappings, a C program's `mmap`. + /// + /// The floor is the larger of the module page's two marks: the recorded + /// segments' top ([`ModulePage::high_water`]) and how far the startup + /// placement reached ([`ModulePage::placed_high_water`]: the copied + /// template views and the TLS block, which no record names). Neither + /// alone is the floor. The limit is the module page itself, the + /// window's last page. + /// + /// This names a span and claims nothing. **One placer per window**: a + /// program places everything it maps there from one cursor + /// (`kore_ld::place::SharedPlacer`), and one that also links native + /// modules late (`kore-late`) partitions the span between the two + /// itself, since two cursors over one window hand out each other's + /// addresses. + /// + /// [`ModulePage::high_water`]: kore_abi::image::ModulePage::high_water + /// [`ModulePage::placed_high_water`]: kore_abi::image::ModulePage::placed_high_water + pub fn link_free_span(&self) -> Result { + let (base, len) = self + .geom() + .and_then(|geom| geom.window(kore_abi::task::geom::LINK)) + .ok_or(LinkSpanError::NoWindow)?; + let limit = + kore_abi::image::module_page_va(base, len).ok_or(LinkSpanError::NoModulePage)?; + let page = self.modules(); + let free = page + .and_then(|page| page.high_water()) + .unwrap_or(base) + .max( + page.and_then(|page| page.placed_high_water(limit)) + .unwrap_or(base), + ) + .max(base); + if free >= limit { + return Err(LinkSpanError::Full); + } + Ok(LinkSpan { base, limit, free }) + } + /// The `.kore.geom` window table, if the founder stamped it — where the /// task learns its founder-mapped `(va, len)` windows (scratch, slots, …) /// instead of hardcoded constants (`res/USERLAND.md` §4). `None` for a @@ -504,7 +568,7 @@ const fn report_slot_install(current_p1: usize, requested: usize) -> ReportSlotI #[cfg(target_os = "none")] pub(crate) mod geometry { - use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + use core::sync::atomic::{AtomicI32, AtomicU32, AtomicU64, AtomicUsize, Ordering}; use kore_abi::task::{GeomBlock, geom}; @@ -631,6 +695,31 @@ pub(crate) mod geometry { finalizer_state: AtomicUsize, finalizers: [AtomicUsize; 32], exit_started: AtomicUsize, + /// This thread's C `errno` ([`c_errno`]): ISO C gives `errno` + /// thread storage duration, and this block is the runtime's + /// per-thread home, so the word lives here rather than in an ELF + /// TLS block — which would give every C-carrying image a TLS + /// segment, and with it a TCB migration and a template every spawn + /// must place (`kore-late` refuses such a host outright). Zero on + /// a fresh page, as C requires at thread start; deliberately *not* + /// inherited by a spawned thread, whose block is its own. Only the + /// owning thread reads or writes it. + c_errno: AtomicI32, + /// Sends into peers' doorbells this thread came to hold after its + /// founding ([`futex::adopt_peer`](crate::futex)), indexed by the + /// peer's band, each its arena slot plus one (`0` = none). Arena + /// names are this thread's alone, so they live here and a band a + /// later thread reuses starts with none. Only the owning thread + /// reads or writes them. + late_kicks: [AtomicU32; crate::thread::MAX_THREADS as usize], + /// The [`futex::Hub`](crate::futex) this thread relays an + /// undeliverable wake through, as an address (`0` = none). Set once. + futex_hub: AtomicUsize, + /// What [`thread::exit`](crate::thread::exit) runs before it commits + /// ([`set_exit_hook`]): a `fn(usize)` as an address (`0` = none), + /// and its argument. Taken by the exit, so it runs once. + exit_call: AtomicUsize, + exit_argument: AtomicUsize, } // The block must fit the founder's one-page window. @@ -732,6 +821,75 @@ pub(crate) mod geometry { block().panicking.swap(1, Ordering::AcqRel) == 0 } + /// This thread's C `errno` word (the block's `c_errno` field). + /// + /// Lawful wherever [`block`] is: C code runs only on an anchored + /// thread, since anchoring is crt0's first act and a spawned thread's + /// trampoline's first act. + pub(crate) fn c_errno() -> *mut i32 { + block().c_errno.as_ptr() + } + + /// The send this thread adopted into band `peer`'s doorbell, if any. + pub(crate) fn late_kick(peer: usize) -> Option { + let slot = block().late_kicks.get(peer)?.load(Ordering::Relaxed); + let index = slot.checked_sub(1)?; + Some(kore_abi::session::SlotRef::from_wire(index)) + } + + /// Record (`Some`) or withdraw (`None`) this thread's adopted send into + /// band `peer`'s doorbell. `false` when `peer` is outside the band. + pub(crate) fn set_late_kick(peer: usize, send: Option) -> bool { + let Some(cell) = block().late_kicks.get(peer) else { + return false; + }; + let word = match send { + Some(slot) => match slot.index().checked_add(1) { + Some(word) => word, + None => return false, + }, + None => 0, + }; + cell.store(word, Ordering::Relaxed); + true + } + + /// The futex hub this thread relays through, as an address (`0` = none). + pub(crate) fn futex_hub() -> usize { + block().futex_hub.load(Ordering::Relaxed) + } + + /// Adopt the futex hub at `addr`; the first adoption stands. + pub(crate) fn set_futex_hub(addr: usize) -> bool { + addr != 0 + && block() + .futex_hub + .compare_exchange(0, addr, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + } + + /// Have [`thread::exit`](crate::thread::exit) run `call(argument)` on + /// this thread before it commits, replacing any hook set before. + pub(crate) fn set_exit_hook(call: fn(usize), argument: usize) { + let b = block(); + b.exit_argument.store(argument, Ordering::Relaxed); + b.exit_call.store(call as usize, Ordering::Relaxed); + } + + /// Take this thread's exit hook, leaving none: a hook that itself ends + /// the thread runs once. + pub(crate) fn take_exit_hook() -> Option<(fn(usize), usize)> { + let b = block(); + let call = b.exit_call.swap(0, Ordering::Relaxed); + if call == 0 { + return None; + } + // SAFETY: only `set_exit_hook` stores a nonzero `exit_call`, and it + // stores a `fn(usize)` cast to its address. + let call = unsafe { core::mem::transmute::(call) }; + Some((call, b.exit_argument.load(Ordering::Relaxed))) + } + /// Where this thread's block lives — the value the thread pointer holds, /// recovered the same way every other accessor recovers it. pub(crate) fn block_va() -> usize { diff --git a/native/crates/rt/src/futex.rs b/native/crates/rt/src/futex.rs index 04fe2d6..bebb188 100644 --- a/native/crates/rt/src/futex.rs +++ b/native/crates/rt/src/futex.rs @@ -70,15 +70,26 @@ //! `i`'s send end is looked up at `role::DOORBELL@(2 + i)`, and index 2 is //! the self-kick a founder already carves — so a thread endowed with it can //! wake **thread 0** with nothing added, which is the endowment the input -//! service's relay thread already takes. Higher indices are the peer mesh, -//! and `thread::spawn_with` does not provision one yet: [`wake`] returns -//! `false` for a peer this thread holds no send end for, rather than -//! reporting a kick it did not deliver. +//! service's relay thread already takes. Higher indices are the peer mesh; +//! [`wake`] returns `false` for a peer this thread holds no send end for, +//! rather than reporting a kick it did not deliver. //! -//! That asymmetry is a provisioning gap, not a design one — filling it is -//! minting `MAX_THREADS` doorbell pairs in the spawning thread and endowing -//! each child with the ends, which needs a `role::CHANNEL_SPACE` grant in the -//! task that spawns. Nothing in the API changes when it lands. +//! The mesh is provisioned by whoever founds the threads, from a +//! `role::CHANNEL_SPACE` grant: the star (`res/SPMD.md` §3) for an SPMD +//! region, and the [`thread_controller`](crate::thread_controller) for the +//! threads it owns. A founding stamps the sends it can name in advance; a +//! send a thread comes to hold *after* its founding — the channel a +//! controller mints for each child it founds, or the one a starter mints for +//! the controller it founds — is **adopted** ([`adopt_peer`]) into the +//! thread's own runtime block, and [`wake`] finds it there exactly as it +//! finds a stamped one. +//! +//! A waiter that no send this thread holds can reach is handed to a +//! **relay**: the star's hub (one notice on `DOORBELL@`[`RELAY`], answered by +//! a kick to every peer), or a [`Hub`] this thread adopted — a hub that is +//! itself a waiter, woken through its own word, which kicks only the threads +//! with waiters registered ([`sweep`]). Nothing in the wait/wake API changes +//! with either. //! //! **For a word in a shared window the gap is closed** by the kernel //! address-wait (`res/EVENT.md` §6). A word that @@ -349,6 +360,15 @@ pub(crate) const PEER_BASE: u16 = 2; /// band peers so it can never collide with one. pub(crate) const RELAY: u16 = PEER_BASE + MAX_THREADS as u16; +/// A thread's **own kick**: a send into its own doorbell that it may derive +/// from — the authority it hands a thread it founds so that thread can wake +/// it ([`thread::own_kick`](crate::thread)). One past [`RELAY`], so neither +/// the peer arithmetic nor the relay can name it: a thread never kicks itself +/// through the mesh. Thread 0's own kick predates this index — it is the +/// founder-carved self-kick at [`PEER_BASE`], its own peer-0 entry. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) const OWN_KICK: u16 = RELAY + 1; + /// Wait until another thread wakes this one, or `timeout` elapses. /// /// Parks **only if** `word` still holds `expected` at the moment the wait is @@ -825,6 +845,13 @@ pub(crate) fn notify_resident_completion() -> bool { /// parked is still woken correctly — its post already stands, so the notice /// resolves the park it is about to enter. pub fn wake(word: &AtomicU32) -> bool { + wake_one(word, true) +} + +/// [`wake`]'s body. `relay` says whether a waiter this thread cannot reach +/// is handed on ([`relay_missed`]); a [`Hub`]'s own signal is not, since the +/// hub is the relay. +fn wake_one(word: &AtomicU32, relay: bool) -> bool { let addr = address_of(word); let route = crate::cap::mapped::locate(word as *const AtomicU32 as *const u8); let mut unreachable_waiter = false; @@ -852,12 +879,12 @@ pub fn wake(word: &AtomicU32) -> bool { } } // A registered waiter this caller holds no peer send for is the - // hub's to reach (`res/SPMD.md` §3 — the star): one relay notice, - // counted as a wake because a notice is now in flight toward the - // waiter. No hub wired → today's honest miss. + // hub's to reach (`res/SPMD.md` §3 — the star, or this thread's own + // `Hub`): one relay, counted as a wake because a wake is now in + // flight toward the waiter. No hub wired → today's honest miss. if unreachable_waiter { STAT_MISSED.fetch_add(1, Ordering::Relaxed); - if imp::relay() { + if relay && relay_missed() { STAT_RELAYS.fetch_add(1, Ordering::Relaxed); return true; } @@ -865,6 +892,16 @@ pub fn wake(word: &AtomicU32) -> bool { false } +/// Hand the waiters this thread could not reach to the thread that can: +/// its adopted [`Hub`], through the hub's own word, or else the star's hub +/// through one raw notice on `DOORBELL@`[`RELAY`]. `true` when either went. +fn relay_missed() -> bool { + match imp::hub() { + Some(hub) => hub.relay(), + None => imp::relay(), + } +} + /// Wake **every** thread waiting on `word`; returns how many were kicked. /// /// The count is of notices successfully posted, not of threads that will @@ -904,17 +941,204 @@ pub fn wake_all(word: &AtomicU32) -> usize { missed = true; } } - // The relay hop (the star, `res/SPMD.md` §3): every waiter this - // caller could not kick is the hub's job — one notice covers them - // all, counted once. With the intra-chunk kick mesh this is the - // exception path (cross-chunk waiters), not the release path. - if missed && imp::relay() { + // The relay hop (the star, `res/SPMD.md` §3, or this thread's `Hub`): + // every waiter this caller could not kick is the hub's job — one relay + // covers them all, counted once. With the intra-chunk kick mesh this + // is the exception path (cross-chunk waiters), not the release path. + if missed && relay_missed() { STAT_RELAYS.fetch_add(1, Ordering::Relaxed); woken += 1; } woken } +/// A **relay hub that is also a waiter**: the thread that founds a group of +/// threads and holds a send into each of their doorbells — the +/// [`thread_controller`](crate::thread_controller) — parked on its +/// [`word`](Self::word) like any other waiter. +/// +/// A thread that adopted the hub ([`adopt_hub`]) relays a wake it cannot +/// deliver by counting it in [`relays`](Self::relays), then [`signal`]ing +/// the hub: an ordinary store-then-wake of the hub's word, with every +/// property [`wake`] has. The hub reads `relays` after each wake and, when +/// it moved, [`sweep`]s. +/// +/// Why a count and not the star's raw notice: this hub is woken for work of +/// its own, and a notice says nothing about why it came. A hub that took +/// every notice for a relay would kick every parked thread on every +/// unrelated event; one that took none would lose a relay whose notice +/// coalesced with another on its doorbell. The count is bumped before the +/// word, so a hub that sees the word move sees the relay behind it. +/// +/// [`signal`]: Self::signal +#[repr(align(128))] +pub(crate) struct Hub { + word: AtomicU32, + relays: AtomicU32, +} + +impl Hub { + pub(crate) const fn new() -> Self { + Self { + word: AtomicU32::new(0), + relays: AtomicU32::new(0), + } + } + + /// The word the hub thread waits on. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + pub(crate) fn word(&self) -> &AtomicU32 { + &self.word + } + + /// Relays owed so far, wrapping: the hub sweeps whenever this moved + /// since its last sweep. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + pub(crate) fn relays(&self) -> u32 { + self.relays.load(Ordering::SeqCst) + } + + /// Tell the hub something changed — store, then wake, the order this + /// module requires. Delivered directly or not at all: the hub has no + /// relay but itself. `true` when a kick reached it. + pub(crate) fn signal(&self) -> bool { + self.word.fetch_add(1, Ordering::SeqCst); + wake_one(&self.word, false) + } + + /// One relay: counted before the word moves. + fn relay(&self) -> bool { + self.relays.fetch_add(1, Ordering::SeqCst); + self.signal() + } +} + +/// Relay through `hub` from now on: a wake this thread cannot deliver goes +/// to it ([`Hub`]). The first adoption stands — `false` when this thread +/// already relays through a hub, or has no runtime block to record it in. +pub(crate) fn adopt_hub(hub: &'static Hub) -> bool { + imp::adopt_hub(hub) +} + +/// Record that this thread holds `send`, a send into band `peer`'s +/// doorbell its founding did not stamp — the channel a controller mints for +/// each child it founds, or a starter for the controller it founds. From +/// here on [`wake`] reaches `peer` directly. `false` when `peer` is outside +/// the band or this thread has no runtime block. +/// +/// The record is this thread's alone — it names a slot in its own arena — so +/// it lives in its runtime block, not beside [`WAITING_ON`]: a band reused +/// by a later thread starts from a zeroed block and inherits nothing. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) fn adopt_peer(peer: u32, send: kore_abi::session::SlotRef) -> bool { + imp::adopt_peer(peer as usize, send) +} + +/// Withdraw `peer`'s adopted send. Called before the send is retired, so no +/// wake presents a name the arena is about to reissue. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) fn forget_peer(peer: u32) { + imp::forget_peer(peer as usize); +} + +/// Whether this thread holds a send into band `peer`'s doorbell — stamped +/// at its founding or adopted since. +pub(crate) fn reaches(peer: u32) -> bool { + imp::reaches(peer as usize) +} + +/// The hub's answer to a relay ([`Hub`]): kick every thread this one can +/// reach that has a doorbell waiter registered, itself excepted, and return +/// how many were kicked. +/// +/// A relay does not say which word it was for, so every such thread is a +/// candidate and each kick is a wake its target re-checks — spurious for one +/// whose word did not move, which every waiter tolerates by contract, and +/// paid only when a relay was owed. A word-parked waiter is not kicked: the +/// kernel's address-wait, not its doorbell, holds it, and its slice bounds +/// a relay it misses. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) fn sweep() -> usize { + let Some(me) = current_thread() else { + return 0; + }; + imp::sweep(me) +} + +/// [`sweep`]'s selection over the registry, with the kick injected so the +/// host suite can hold it to account. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +fn sweep_with(me: usize, mut kick: impl FnMut(usize) -> bool) -> usize { + let mut kicked = 0; + for thread in 0..MAX_THREADS as usize { + if thread != me && waits_on_bell(thread) && kick(thread) { + STAT_KICKS.fetch_add(1, Ordering::Relaxed); + kicked += 1; + } + } + kicked +} + +/// Whether `thread` has any waiter a kick reaches — a doorbell-mode entry. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +fn waits_on_bell(thread: usize) -> bool { + WAITING_ON[thread].iter().any(|cell| { + let entry = cell.load(Ordering::SeqCst); + entry != NOT_WAITING && entry & WORD_MODE == 0 + }) +} + +/// [`wait`] that also ends when the task `task` ends: one doorbell park +/// whose join arm (`SessionFlags::WAIT_TASK`) watches that task beside the +/// word, for a thread that must see both a futex wake and a sibling's end +/// without polling either — the [`thread_controller`](crate::thread_controller) +/// reaping a child that has announced its exit. `report` is a landing the +/// caller owns (a `thread::JoinScratch`); an exit leaves its report there. +/// +/// [`Woken`](WaitOutcome::Woken) covers the task's end too: the caller +/// re-checks its word *and* the task. There is no monitored spin, because +/// the edge waited for is a kernel event no store announces, and the park +/// stays on the doorbell even for a word in a noted window — the kernel's +/// word wait does not compose with a join arm. Uncounted in [`stats`], like +/// the star hub's housekeeping wait: the threads that use it are not the +/// ones a bench measures. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) fn wait_or_end( + word: &AtomicU32, + expected: u32, + timeout: Option, + task: kore_abi::session::SlotRef, + report: kore_abi::session::SlotRef, +) -> WaitOutcome { + let Some(me) = current_thread() else { + return WaitOutcome::Unsupported; + }; + // Arm, register, check — `wait`'s order and for its reasons. + if !imp::arm() { + return WaitOutcome::Unsupported; + } + let Some(slot) = claim_wait(me, address_of(word)) else { + return WaitOutcome::Unsupported; + }; + let _slot = WaitSlot { thread: me, slot }; + if word.load(Ordering::SeqCst) != expected { + return WaitOutcome::Retry; + } + imp::park_or_end(timeout, task, report) +} + +/// How long a park that could not raise its join arm rides the doorbell +/// alone. A join arm the kernel did not install answers `WouldBlock` at +/// once, with nothing parked; riding the doorbell for this long instead +/// keeps a caller that loops on [`wait_or_end`] from trapping in a ring. +/// The joiner's own pacing for an unestablished park +/// (`thread::join_park`, two milliseconds), so the two degrade alike. +#[cfg_attr( + not(target_os = "none"), + expect(dead_code, reason = "the join arm is a metal-only trap") +)] +const END_SLICE: Duration = Duration::from_millis(2); + /// Whether this thread can [`wait`] at all — a doorbell to park on and an /// identity in the thread band. /// @@ -1182,15 +1406,115 @@ mod imp { let Some(rt) = crate::entry::thread_context() else { return false; }; - let Ok(peer) = u16::try_from(index) else { - return false; - }; - let Some(send) = rt.bootstrap().handle(role::DOORBELL, PEER_BASE + peer) else { + kick_with(&rt, index) + } + + /// [`kick`] against a context the caller already resolved — the sweep + /// kicks many peers off one boot-page parse. + fn kick_with(rt: &crate::entry::Rt, index: usize) -> bool { + let Some(send) = send_for(rt, index) else { return false; }; verbs::post_notice(send, COOKIE_FUTEX, index as u64) } + /// The send this thread holds into band `index`'s doorbell: the one its + /// founding stamped at `DOORBELL@(PEER_BASE + index)`, else the one it + /// adopted since ([`super::adopt_peer`]). + fn send_for(rt: &crate::entry::Rt, index: usize) -> Option { + let peer = u16::try_from(index).ok()?; + rt.bootstrap() + .handle(role::DOORBELL, PEER_BASE + peer) + .or_else(|| crate::entry::geometry::late_kick(index)) + } + + pub(super) fn reaches(index: usize) -> bool { + crate::entry::thread_context().is_some_and(|rt| send_for(&rt, index).is_some()) + } + + pub(super) fn adopt_peer(index: usize, send: kore_abi::session::SlotRef) -> bool { + crate::entry::geometry::set_late_kick(index, Some(send)) + } + + pub(super) fn forget_peer(index: usize) { + let _ = crate::entry::geometry::set_late_kick(index, None); + } + + pub(super) fn sweep(me: usize) -> usize { + let Some(rt) = crate::entry::thread_context() else { + return 0; + }; + super::sweep_with(me, |thread| kick_with(&rt, thread)) + } + + /// The hub this thread relays through, if it adopted one. + pub(super) fn hub() -> Option<&'static super::Hub> { + let addr = crate::entry::geometry::futex_hub(); + // SAFETY: `adopt_hub` records only the address of a `&'static Hub`, + // and nothing ever clears or rewrites it to another value. + (addr != 0).then(|| unsafe { &*(addr as *const super::Hub) }) + } + + pub(super) fn adopt_hub(hub: &'static super::Hub) -> bool { + crate::entry::geometry::set_futex_hub(hub as *const super::Hub as usize) + } + + /// Park on the standing post with the join arm watching `task` beside + /// it — the composite `WAIT_DELIVERY | WAIT_TASK` (± `WAIT_SLEEP`) + /// shape the kernel resolves first-wins (`SessionFlags::WAIT_TASK`). + pub(super) fn park_or_end( + timeout: Option, + task: kore_abi::session::SlotRef, + report: kore_abi::session::SlotRef, + ) -> WaitOutcome { + let Some(db) = doorbell() else { + return WaitOutcome::Unsupported; + }; + let deadline = match timeout { + Some(d) => match crate::clock::uptime() { + Ok(now) => Some((now + d).as_nanos()), + Err(_) => return WaitOutcome::Unsupported, + }, + None => None, + }; + let mut p = SessionBuilder::new(); + p.park_standing(db.chan, db.land.index() as u16, db.recv_tag); + if let Some(at) = deadline { + p.wait_sleep(at); + } + if p.join(task, report).is_err() { + return WaitOutcome::Unsupported; + } + p.wait_task(); + let Ok(committed) = TrapTransport::new().commit_join_park(&p) else { + return WaitOutcome::Unsupported; + }; + match committed.status() { + // The doorbell won: consume and re-stand, as `park` does. + Status::Ok => { + let _ = verbs::take_notice(db.land); + post(&db); + WaitOutcome::Woken + } + // The task ended (`Exited`), or the post or the task was reaped + // (`Revoked`) — the caller's re-check reads which; a reaped + // doorbell refuses its next arm. An OCC loss parked nothing and + // is retried by the caller's loop. + Status::Exited | Status::Revoked | Status::Conflict => WaitOutcome::Woken, + Status::Deadline => WaitOutcome::TimedOut, + // Nobody won, or the join arm could not be raised and nothing + // parked: ride the doorbell alone, bounded (`END_SLICE`). + Status::WouldBlock => { + let slice = timeout.map_or(super::END_SLICE, |t| t.min(super::END_SLICE)); + match park(Some(slice)) { + WaitOutcome::TimedOut if timeout.is_none() => WaitOutcome::Woken, + outcome => outcome, + } + } + _ => WaitOutcome::Unsupported, + } + } + // The relay slot sits exactly one past the last band peer — the // module-level constant and this mesh's arithmetic must agree. const _: () = assert!(super::RELAY == PEER_BASE + super::MAX_THREADS as u16); @@ -1261,6 +1585,37 @@ mod imp { false } + pub(super) fn reaches(_index: usize) -> bool { + false + } + + pub(super) fn adopt_peer(_index: usize, _send: kore_abi::session::SlotRef) -> bool { + false + } + + pub(super) fn forget_peer(_index: usize) {} + + pub(super) fn sweep(_me: usize) -> usize { + 0 + } + + /// No runtime block off metal: no hub was ever adopted. + pub(super) fn hub() -> Option<&'static super::Hub> { + None + } + + pub(super) fn adopt_hub(_hub: &'static super::Hub) -> bool { + false + } + + pub(super) fn park_or_end( + _timeout: Option, + _task: kore_abi::session::SlotRef, + _report: kore_abi::session::SlotRef, + ) -> super::WaitOutcome { + super::WaitOutcome::Unsupported + } + pub(super) fn relay() -> bool { false } @@ -1621,4 +1976,62 @@ mod tests { assert!(!wake(&word)); assert_eq!(wake_all(&word), 0); } + + #[test] + fn a_relay_is_counted_before_the_hub_word_moves() { + // The hub's lost-relay argument: it reads `relays` after it sees its + // word move, so the count must already have moved by then. Off metal + // the kick refuses, but the two stores are the shared half. + static HUB: Hub = Hub::new(); + let (word, relays) = (HUB.word().load(Ordering::SeqCst), HUB.relays()); + assert!(!HUB.relay(), "off metal no kick reaches the hub"); + assert_eq!(HUB.relays(), relays.wrapping_add(1)); + assert_eq!(HUB.word().load(Ordering::SeqCst), word.wrapping_add(1)); + // A plain signal moves the word and owes no relay. + assert!(!HUB.signal()); + assert_eq!(HUB.relays(), relays.wrapping_add(1)); + assert_eq!(HUB.word().load(Ordering::SeqCst), word.wrapping_add(2)); + } + + #[test] + fn a_hub_word_owns_its_interference_span() { + // The hub thread's monitored spin watches its word's line; every + // client of the hub stores to it. Sharing the line with neighbours + // would end that spin on their traffic. + assert_eq!(core::mem::align_of::(), 128); + } + + #[test] + fn a_sweep_kicks_only_doorbell_waiters_other_than_itself() { + // Rows 8..=11: the hub (8), a doorbell waiter (9), a word-parked + // waiter (10), and a thread waiting on nothing (11). + for row in 8..=11 { + reset_thread(row); + } + let a = AtomicU32::new(0); + let b = AtomicU32::new(0); + claim_wait(8, address_of(&a)).expect("the hub's own wait"); + claim_wait(9, address_of(&a)).expect("a doorbell waiter"); + claim_wait_on_word(10, address_of(&b)).expect("a word waiter"); + let mut kicked = std::vec::Vec::new(); + let n = sweep_with(8, |thread| { + kicked.push(thread); + true + }); + assert_eq!(kicked, [9], "the sweep kicked a thread no kick can wake"); + assert_eq!(n, 1); + // A kick that refuses is not counted as a wake. + assert_eq!(sweep_with(8, |_| false), 0); + for row in 8..=11 { + reset_thread(row); + } + } + + #[test] + fn the_own_kick_is_named_past_every_peer_and_the_relay() { + // No peer arithmetic (`PEER_BASE + band`) and no relay lookup may + // ever resolve a thread's own kick. + assert_eq!(OWN_KICK, RELAY + 1); + assert!(OWN_KICK > PEER_BASE + MAX_THREADS as u16 - 1); + } } diff --git a/native/crates/rt/src/process.rs b/native/crates/rt/src/process.rs index 4417d58..03733ae 100644 --- a/native/crates/rt/src/process.rs +++ b/native/crates/rt/src/process.rs @@ -53,6 +53,13 @@ pub fn initialize_with_context( page.initialize_with_context(args, context, TelemetrySnapshot::EMPTY) } +/// Publish the name the launcher resolved the program by +/// ([`ProcessIo::command`]), after the page is initialized and before START. +pub fn write_command(window: &mut [u8], name: &[u8]) -> Result<(), ProcessIoError> { + let (page, _) = ProcessIo::mut_from_prefix(window).map_err(|_| ProcessIoError::Invalid)?; + page.write_command(name) +} + /// Publish the launcher's filesystem snapshot, same window as the telemetry /// snapshot and the same lifecycle point: after founding, before START. pub fn write_filesystem( diff --git a/native/crates/rt/src/service/mod.rs b/native/crates/rt/src/service/mod.rs index bf11006..a4cc04a 100644 --- a/native/crates/rt/src/service/mod.rs +++ b/native/crates/rt/src/service/mod.rs @@ -413,6 +413,21 @@ pub trait SystemService: crate::svc::SvcProtocol + Sized { /// retryable domain custody still closes in `peer_closed` afterward. fn peer_detaching(&mut self, _peer: crate::svc::PeerId) {} + /// Recyclable peers this service owes one wake: it published a batch into + /// an event channel it keeps in their pair's tail (`attach_peer_aux`, + /// `res/EVENT.md`) and the batch's commit found the consumer parked. + /// + /// [`ServeHost`](crate::svc::ServeHost) asks after every unit of work — + /// a request, a teardown, an idle recheck — and sends each peer named in + /// `owed[..n]` one notice through the endpoint it owns, so a service never + /// copies an affine endpoint out of its peer bundle to wake a client. Name + /// each peer once per owed wake and clear it as it is handed over; a full + /// batch is asked again. The default owes nothing: the request/reply shape, + /// whose only wake is the reply's. + fn take_owed_wakes(&mut self, _owed: &mut [Option]) -> usize { + 0 + } + /// Observe completion of an admission requested through /// [`Serve::admit_peer`]. The host, not the wire request, chooses the /// generational identity and reports only the authenticated result. diff --git a/native/crates/rt/src/session/builder.rs b/native/crates/rt/src/session/builder.rs index fa60e93..089a589 100644 --- a/native/crates/rt/src/session/builder.rs +++ b/native/crates/rt/src/session/builder.rs @@ -1264,6 +1264,27 @@ impl SessionBuilder { Ok(Staged(result)) } + /// Consume a writable lease and return its **frozen** successor + /// (`Revoke × REVOKE_KIND_FREEZE`) — the exclusive intake of + /// `res/IMAGE.md` §6b. + /// + /// The kernel requires `ACCESS|MODIFY|DERIVE|REVOKE` on `subject`, kills + /// its complete descendant lineage, removes every CPU/DMA/guest alias, + /// and writes an `ACCESS|MODIFY|DERIVE|REVOKE` lease over the same window + /// into `result`. No lessor may write that window again while the + /// successor lives — the successor and its carves included. + pub fn freeze(&mut self, subject: SlotRef, result: SlotRef) -> Result { + self.push(Inv { + op: OpCode::Revoke as u16, + requested: 0, + kind: kore_abi::session::REVOKE_KIND_FREEZE, + subject: subject.index(), + result: result.index(), + arg: Invocation::NO_ARG, + })?; + Ok(Staged(result)) + } + /// `Derive × DERIVE_KIND_ARENA_EXTEND` on the task's own space /// self-view (slice E): grow the caller's cap arena by one chunk. /// An **effect**, not a mint — the request is empty, the arg is the diff --git a/native/crates/rt/src/session/tests.rs b/native/crates/rt/src/session/tests.rs index 8446618..7cfcd4a 100644 --- a/native/crates/rt/src/session/tests.rs +++ b/native/crates/rt/src/session/tests.rs @@ -235,6 +235,36 @@ fn the_executable_return_is_the_finalizations_wire_twin() { ); } +/// The freeze is the third turn and the same wire shape: one no-argument +/// consuming revoke, distinct from both of the others. +#[test] +fn the_freeze_is_a_no_argument_consuming_revoke() { + let source = SlotRef::from_wire(12); + let result = SlotRef::from_wire(13); + let mut builder = SessionBuilder::new(); + let staged = builder.freeze(source, result).expect("the freeze stages"); + assert_eq!(staged.slot(), result); + + let mut out = [0u8; DESCRIPTOR_MAX]; + let len = builder.finish(&mut out).expect("descriptor finishes"); + assert_eq!(len, 8 + 16, "the same one-invocation descriptor"); + let (header, invocations, args) = parse(&out[..len]); + assert_eq!(header.count.get(), 1); + assert_eq!(invocations[0].op.get(), OpCode::Revoke as u16); + assert_eq!(invocations[0].kind, kore_abi::session::REVOKE_KIND_FREEZE); + assert_eq!(invocations[0].requested, 0, "rights come from the lease"); + assert_eq!(invocations[0].subject.get(), source.index()); + assert_eq!(invocations[0].result.get(), result.index()); + assert_eq!(invocations[0].arg.get(), Invocation::NO_ARG); + assert!(args.is_empty(), "the freeze names no second capability"); + for other in [ + kore_abi::session::REVOKE_KIND_EXEC, + kore_abi::session::REVOKE_KIND_EXEC_RETURN, + ] { + assert_ne!(kore_abi::session::REVOKE_KIND_FREEZE, other); + } +} + /// Length selection happens before the transport chooses registers or /// scratch, but both carriers must receive byte-identical output from the one /// encoder. `finish` writes exactly that prefix and leaves spare carrier bytes diff --git a/native/crates/rt/src/spmd.rs b/native/crates/rt/src/spmd.rs index 135b2c9..5c8b23a 100644 --- a/native/crates/rt/src/spmd.rs +++ b/native/crates/rt/src/spmd.rs @@ -847,6 +847,7 @@ fn launch( // ticket whose founding never happened refuses its join and // detaches (the band stays spent — visible, never hidden). ── decide(ctx, ABORT); + let mut detached = false; for staged in workers.iter_mut().filter_map(Option::take) { // A refused join here is a transient observation failure on // a worker that is already exiting; one bounded retry, then @@ -855,7 +856,10 @@ fn launch( Ok(_) => {} Err(e) => match crate::thread::join_reserved(e.into_thread(), &mut join_scratch) { Ok(_) => {} - Err(e) => e.into_thread().detach(), + Err(e) => { + detached = true; + e.into_thread().detach() + } }, } } @@ -866,6 +870,15 @@ fn launch( // invariant changes later. let _ = star.finish(); } + // Every worker this launch founded is joined, so none can read the + // gate again: restage it, and the next launch over `ctx` may run. + // A detached worker may still be parked on it, and a restaged gate + // would release it into the next launch's body: `ctx` then stays + // aborted, refusing every launch over it, rather than hand one + // region's job to another's worker. + if !detached { + ctx.restage(); + } return None; } decide(ctx, RUN); @@ -1539,6 +1552,10 @@ fn dispatch_resident( .map(|m| m.fitting_width()) .unwrap_or(1) .clamp(1, DISPATCH_CAP as u32); + // No wider than this thread's arena can found: a resident gang + // holds its founder's names for the task's life, and a thread + // other than thread 0 founds from a fixed arena. + let fit = affordable(fit, false); let founded = (fit >= 2) .then(|| { found_resident::( @@ -1556,7 +1573,15 @@ fn dispatch_resident( // contract says; an executable takes `PROFILE_CHANNELS` // to found the star. Some(region) if !region.star_active() => { - let _ = region.retire(); + // The retire told every worker to end before its first + // join, so each join it could not complete is one still + // to finish: collect them all, as a dispatch collects its + // own. Dropping the region instead would leave the gang + // running, its bands, stacks and slots spent for the + // task's life. + if let Err(refused) = region.retire() { + join_dispatched(refused.spmd); + } gang.state.store(RESIDENT_REFUSED, Relaxed); return ResidentOutcome::Unavailable; } @@ -1764,6 +1789,104 @@ fn block_split(span: u32, workers: u32, worker: u32) -> (u32, u32) { (start, start + len) } +/// What a [`launch`] founds beside its workers: what its arena budget +/// ([`launch_slots`]) depends on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +struct LaunchShape { + /// The star a launch here mints: `Some(founder_kick)` — whether each + /// member's mesh carries the founder's kick — or `None` for a starless + /// launch. + star: Option, + /// Whether the launch founds the star's relay hub: a one-shot region does, + /// a resident one does not. + hub: bool, + /// Whether each worker takes a TLS block. + tls: bool, + /// Whether each compute worker is endowed the process heap. + heap: bool, +} + +/// The arena names a [`launch`] of `width` workers holds at its peak. +/// +/// The launch holds its join landing and the star's wires +/// ([`star::wires_slots`](crate::star::wires_slots)) throughout. Each gang +/// session then holds every one of its workers' founding names at its commit +/// ([`thread::gang_founding_slots`](crate::thread::gang_founding_slots)), on +/// top of what the sessions before it kept ([`RETAINED_SLOTS`] a worker), and +/// the hub's session holds its own on top of all of them. A launch whose peak +/// fits the arena cannot run out of names part way through, where a refusal +/// leaks what it claimed. The sessions are [`launch`]'s own split +/// ([`founding_sessions`]): a starred session carries at most +/// [`MESH_CEILING`](crate::star::MESH_CEILING) workers. +/// +/// [`RETAINED_SLOTS`]: crate::thread::RETAINED_SLOTS +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +fn launch_slots(width: u32, shape: LaunchShape) -> usize { + use crate::thread::{RETAINED_SLOTS, gang_founding_slots}; + let wires = shape + .star + .map_or(0, |_| crate::star::wires_slots(width as usize)); + let mut held = 1 + wires; + let mut peak = held; + let ceiling = match shape.star { + Some(_) => crate::star::MESH_CEILING, + None => crate::session::MAX_SESSION_FOUNDS, + }; + for (start, end) in founding_sessions(width, ceiling as u32) { + let chunk = (end - start) as usize; + let mesh = shape + .star + .map_or(0, |kick| crate::star::mesh_caps(kick, chunk)); + let endowments = chunk * (mesh + usize::from(shape.heap)); + peak = peak.max(held + gang_founding_slots(chunk, shape.tls, endowments)); + held += chunk * (RETAINED_SLOTS + usize::from(shape.tls)); + } + if shape.hub && shape.star.is_some() { + // The hub's endowment: its own recv and a send to every worker. + peak = peak.max(held + gang_founding_slots(1, shape.tls, 1 + width as usize)); + } + peak +} + +/// The widest launch of at most `want` workers whose peak +/// ([`launch_slots`]) fits `available` arena names, or 0 when not one worker +/// fits. +/// +/// Every width is weighed, widest first: the peak is not monotonic in the +/// width, because a wider gang splits into more sessions whose meshes are +/// smaller (seven meshed workers cost one session of seven; eight cost two of +/// four). +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +fn affordable_width(want: u32, shape: LaunchShape, available: usize) -> u32 { + (1..=want) + .rev() + .find(|&width| launch_slots(width, shape) <= available) + .unwrap_or(0) +} + +/// The shape a launch over this thread would take, read before it mints +/// anything. +#[cfg(target_os = "none")] +fn launch_shape(hub: bool) -> LaunchShape { + LaunchShape { + star: crate::star::shape().map(|star| star.founder_kick), + hub, + tls: crate::entry::geometry::tls_template().is_some(), + heap: worker_heap().is_some(), + } +} + +/// [`affordable_width`] on this thread's arena, now. +#[cfg(target_os = "none")] +fn affordable(want: u32, hub: bool) -> u32 { + affordable_width( + want, + launch_shape(hub), + crate::entry::geometry::available_cap_slots(), + ) +} + /// **The parallel-for** — run `body` over `0..tiles`, sized to the /// machine, with no thread configuration at all. /// @@ -1855,6 +1978,12 @@ fn dispatch_erased( ResidentOutcome::Busy => return None, ResidentOutcome::Unavailable => {} } + // The one-shot region, as wide as asked and no wider than this thread's + // arena can found whole; `None` only when not one worker fits. + let width = affordable(width, true); + if width == 0 { + return None; + } let env = &body as *const &(dyn Fn(&'static SpmdCtx, u32) + Sync) as *mut (); let region = launch::( &DISPATCH_CTX, @@ -2319,6 +2448,100 @@ mod tests { assert_eq!(split(12), [6, 6]); } + const fn shape(star: Option, hub: bool, tls: bool) -> LaunchShape { + LaunchShape { + star, + hub, + tls, + heap: true, + } + } + + /// The budget counts what the founding holds, step by step: the join + /// landing and the star's wires, every worker's founding names and mesh + /// at the session's commit, what each session keeps, then the hub's + /// session on top. Three meshed workers founded off thread 0 peak at 46 + /// names: two more than a 64-slot worker's arena had left, the C + /// `main` that `kore_spmd_dispatch` refused on. + #[test] + fn a_launch_budget_counts_what_its_founding_holds() { + // Resident, starred, no founder kick: 1 + 9 wires, then 3 × 7 + // founding names and 3 × (4 mesh + heap) endowments. + assert_eq!(launch_slots(3, shape(Some(false), false, false)), 46); + // The one-shot's hub founds after the gang returned its plan: 10 + // held + 3 × 4 kept + 7 + (1 + 3) endowments = 33, under the gang's + // peak. + assert_eq!(launch_slots(3, shape(Some(false), true, false)), 46); + // Thread 0's star carries its kick: one more mesh cap a worker. + assert_eq!(launch_slots(3, shape(Some(true), false, false)), 49); + // Starless: the landing, then 3 × (7 + heap). + assert_eq!(launch_slots(3, shape(None, true, false)), 25); + // A TLS block is one more founding name and one more kept name. + assert_eq!(launch_slots(3, shape(None, false, true)), 28); + // Two sessions: the second commits over what the first kept. + // 8 meshed workers: 1 + 19, then 4 × 7 + 4 × 6 = 72, keeping 16; + // then 36 + 52 = 88. + assert_eq!(launch_slots(8, shape(Some(false), false, false)), 88); + } + + /// A wider gang can cost less at its peak: past the mesh ceiling it + /// splits into sessions whose meshes are smaller. The clamp weighs every + /// width for that reason. + #[test] + fn a_launch_peak_is_not_monotonic_in_its_width() { + let meshed = crate::star::MESH_CEILING as u32; + let starred = shape(Some(false), false, false); + assert!(launch_slots(meshed + 1, starred) < launch_slots(meshed, starred)); + let between = launch_slots(meshed + 1, starred); + assert_eq!(affordable_width(meshed + 1, starred, between), meshed + 1); + assert!(affordable_width(meshed, starred, between) < meshed); + } + + /// The clamp takes the widest width that fits and refuses only when one + /// worker does not: the 44 names a 64-slot C `main` had left found two of + /// its three workers, and a founder's arena founds every one. + #[test] + fn a_launch_is_clamped_to_its_arena() { + let resident = shape(Some(false), false, false); + assert_eq!(affordable_width(3, resident, 44), 2); + assert_eq!(affordable_width(3, resident, 46), 3); + assert_eq!( + affordable_width(3, resident, launch_slots(1, resident) - 1), + 0 + ); + assert_eq!(affordable_width(0, resident, usize::MAX), 0); + // A founder's dynamic names: the founding granule below the linker + // band, after a doorbell worker's boot names and runtime layout. + let founder = crate::entry::STARTUP_LINK_SLOT_BASE as usize - (3 + 5 + 1 + 10 + 2); + for tls in [false, true] { + for hub in [false, true] { + let widest = crate::thread::MAX_THREADS - 1; + let width = affordable_width(widest, shape(Some(false), hub, tls), founder); + assert!(width >= 16, "tls {tls} hub {hub}: {width}"); + } + } + } + + /// The starless degrade (a refused mint) never needs more than the + /// starred launch its budget was weighed for, at any width, so a launch + /// sized for its star cannot run out of names when the star fails. + #[test] + fn the_starless_degrade_fits_its_starred_budget() { + for width in 1..crate::thread::MAX_THREADS { + for kick in [false, true] { + for tls in [false, true] { + for hub in [false, true] { + assert!( + launch_slots(width, shape(None, hub, tls)) + <= launch_slots(width, shape(Some(kick), hub, tls)), + "width {width} kick {kick} tls {tls} hub {hub}" + ); + } + } + } + } + } + /// A scripted worker: joins yield the scripted outcomes in order. struct MockWorker { outcomes: &'static [Result, crate::cap::TaskWaitError>], diff --git a/native/crates/rt/src/star.rs b/native/crates/rt/src/star.rs index 7be7730..2476ce3 100644 --- a/native/crates/rt/src/star.rs +++ b/native/crates/rt/src/star.rs @@ -110,6 +110,49 @@ pub(crate) struct StarPlan { wires: Wires, } +/// Arena names a star of `workers` holds in its founder for the region's +/// life: each worker's channel root and the parent's send into it (the mint's +/// third name, the parent's recv, is retired at once), and the hub channel's +/// root, send and recv. [`provision`]'s peak is the same count: the last +/// mint's recv is the hub's, kept. +pub(crate) const fn wires_slots(workers: usize) -> usize { + 2 * workers + 3 +} + +/// The star a thread could mint, before it mints one: whether it holds a +/// channel space to mint from, and whether its members would carry the +/// founder's kick. [`provision`] mints exactly this shape, and a launch +/// budgets its arena by it (`spmd::launch_slots`). +#[cfg(target_os = "none")] +#[derive(Clone, Copy, Debug)] +pub(crate) struct StarShape { + /// The founder's own kick rides every member's mesh: only thread 0 + /// holds a send into its own doorbell (`DOORBELL@PEER_BASE`). + pub(crate) founder_kick: bool, +} + +/// This thread's [`StarShape`], or `None` with no `CHANNEL_SPACE` grant. +#[cfg(target_os = "none")] +pub(crate) fn shape() -> Option { + let rt = crate::entry::thread_context()?; + rt.bootstrap() + .handle(kore_abi::task::role::CHANNEL_SPACE, 0)?; + Some(StarShape { + founder_kick: founder_kick(&rt).is_some(), + }) +} + +/// The founder kick a member's mesh carries (see [`StarShape`]). +#[cfg(target_os = "none")] +fn founder_kick(rt: &crate::entry::Rt) -> Option { + if crate::entry::geometry::shard_index() == Some(0) { + rt.bootstrap() + .handle(kore_abi::task::role::DOORBELL, crate::futex::PEER_BASE) + } else { + None + } +} + /// Mint the region's whole forest, all-or-nothing: `workers` channels /// plus the hub's. `None` — no `CHANNEL_SPACE` boot grant, dynamic /// arena slots exhausted, a refused mint — releases everything minted @@ -164,12 +207,7 @@ pub(crate) fn provision(workers: usize) -> Option { // doorbell — a spawned thread's `DOORBELL@PEER_BASE` names thread 0's — // so a region founded elsewhere carries no founder kick and its // launcher rides the park's slice (`res/SPMD.md` §14.2). - let founder_kick = if crate::entry::geometry::shard_index() == Some(0) { - rt.bootstrap() - .handle(kore_abi::task::role::DOORBELL, crate::futex::PEER_BASE) - } else { - None - }; + let founder_kick = founder_kick(&rt); Some(StarPlan { wires: Wires { @@ -186,6 +224,11 @@ pub(crate) fn provision(workers: usize) -> Option { }) } +/// [`StarPlan::mesh_caps`] before a plan exists, for a launch's budget. +pub(crate) const fn mesh_caps(founder_kick: bool, chunk: usize) -> usize { + 2 + founder_kick as usize + chunk.saturating_sub(1) +} + /// Descriptor invocations one meshed worker costs its gang session in a /// chunk of `chunk`: two views, the boot delegate and the Found, plus its /// endowment of `chunk + 3` caps (own recv, hub send, the founder's send, @@ -219,6 +262,14 @@ pub(crate) const MESH_CEILING: usize = { #[cfg(target_os = "none")] impl StarPlan { + /// Capabilities one member of a chunk of `chunk` carries in its mesh: + /// its own recv, the hub's send, the founder's kick when the plan has + /// one, and a send to every chunk sibling — the count + /// [`worker_endow_mesh`](Self::worker_endow_mesh) fills. + pub(crate) const fn mesh_caps(&self, chunk: usize) -> usize { + mesh_caps(self.wires.founder_kick.is_some(), chunk) + } + /// Worker `w`'s endowments — **the intra-chunk kick mesh**: its own /// recv (→ `DOORBELL@0`, which the stamped landing name at /// `DOORBELL@1` completes into a working park), the hub send diff --git a/native/crates/rt/src/startup.rs b/native/crates/rt/src/startup.rs index 800d2be..186a3b0 100644 --- a/native/crates/rt/src/startup.rs +++ b/native/crates/rt/src/startup.rs @@ -29,12 +29,23 @@ pub fn drain() -> Result ! { +pub fn barrier(finalize: bool) { use kore_abi::session::Status; let report = loop { match drain() { @@ -46,9 +57,19 @@ pub fn finish(code: ExitCode) -> ! { Err(_) => crate::thread::exit(ExitCode::INTERNAL), } }; - if crate::entry::geometry::begin_process_exit() && !report.forced { + // Marked whether or not this end finalizes, so no later end can run them. + let first = crate::entry::geometry::begin_process_exit(); + if finalize && first && !report.forced { crate::entry::geometry::run_finalizers(); } +} + +/// End a returning program. Live workers are stopped without their cooperation; +/// their stacks and shared storage are reclaimed by kernel task teardown. +/// User finalizers run only after a clean worker barrier ([`barrier`]). +#[cfg(target_os = "none")] +pub fn finish(code: ExitCode) -> ! { + barrier(true); crate::thread::exit(code) } diff --git a/native/crates/rt/src/svc/client.rs b/native/crates/rt/src/svc/client.rs index 8d187dc..994d5c8 100644 --- a/native/crates/rt/src/svc/client.rs +++ b/native/crates/rt/src/svc/client.rs @@ -487,6 +487,21 @@ pub enum AffineBundleCallError { AfterTransfer(CallError), } +/// Failure of a context-bundle call +/// ([`call_with_context_bundle`](SvcClient::call_with_context_bundle)): the +/// [`AffineBundleCallError`] discipline for a bundle with no executable +/// member, so no witness and no first input. +#[derive(Debug)] +pub enum ContextBundleCallError { + NotMoved { + error: CallError, + second: B, + third: C, + auxiliaries: [Option; MAX_INVOCATION_AUXILIARIES], + }, + AfterTransfer(CallError), +} + /// Failure of a return call whose transferred capability is freshly staged /// inside the transfer session. /// @@ -2264,6 +2279,170 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { } } + /// Move a service request and its context bundle in one atomic session + /// — [`call_with_invocation_bundle`](Self::call_with_invocation_bundle) + /// without the executable member. Such a request names no generation to + /// invoke, so it carries no witness and no input sent through an + /// executable endpoint; the receiver correlates the bundle with the + /// request by the data the request itself carries. Every other step is + /// the invocation's, with its helpers: the same preflights before the + /// request record is published, the same single transfer session, and + /// the same fail-closed poisoning once the record is visible. + pub async fn call_with_context_bundle( + &mut self, + req: &S::Request, + second: B, + second_rights: Rights, + process_tx: SlotRef, + third: C, + third_rights: Rights, + third_tx: SlotRef, + auxiliaries: [Option; MAX_INVOCATION_AUXILIARIES], + auxiliary_tx: [SlotRef; MAX_INVOCATION_AUXILIARIES], + ) -> Result, ContextBundleCallError> { + macro_rules! not_moved { + ($error:expr) => { + ContextBundleCallError::NotMoved { + error: $error, + second, + third, + auxiliaries, + } + }; + } + + if let Err(error) = self.require_idle() { + return Err(not_moved!(error)); + } + // Check every destination before the record is published: see the + // invocation bundle's note on why a post-publication rollback cannot + // be undone. + if ![process_tx, third_tx] + .into_iter() + .all(live_send_destination) + || auxiliaries + .iter() + .zip(auxiliary_tx) + .any(|(auxiliary, destination)| { + auxiliary.is_some() && !live_send_destination(destination) + }) + { + return Err(not_moved!(CallError::Refused(Status::Revoked))); + } + let request = S::encode_req(req); + let grants = match self.prepare_call(&request) { + Ok(grants) => grants, + Err(error) => return Err(not_moved!(error)), + }; + if self.rings.req.capacity_left() == 0 { + return Err(not_moved!(CallError::RingFull)); + } + if let Err(error) = self.ensure_grant_posts(grants).await { + return Err(not_moved!(error)); + } + let second_slot = match preflight_return_send(&second, second_rights) { + Ok(slot) => slot, + Err(error) => return Err(not_moved!(error)), + }; + let third_slot = match preflight_return_send(&third, third_rights) { + Ok(slot) => slot, + Err(error) => return Err(not_moved!(error)), + }; + let mut source_slots = [SlotRef::from_wire(0); 2 + MAX_INVOCATION_AUXILIARIES]; + source_slots[0] = second_slot; + source_slots[1] = third_slot; + let mut source_count = 2; + for auxiliary in auxiliaries.iter().flatten() { + let checked = match auxiliary.transfer { + AuxiliaryTransfer::Derived => { + preflight_return_send(&auxiliary.capability, auxiliary.rights) + } + AuxiliaryTransfer::Exact => preflight_exact_return(&auxiliary.capability), + }; + let slot = match checked { + Ok(slot) => slot, + Err(error) => return Err(not_moved!(error)), + }; + source_slots[source_count] = slot; + source_count += 1; + } + if !invocation_sources_are_distinct(&source_slots[..source_count]) { + return Err(not_moved!(CallError::Refused(Status::Malformed))); + } + + let mut transfer = SessionBuilder::new(); + let mut malformed = transfer + .send(process_tx, second_slot, second_rights) + .and_then(|_| transfer.send(third_tx, third_slot, third_rights)) + .is_err(); + for (auxiliary, destination) in auxiliaries.iter().zip(auxiliary_tx) { + if malformed { + break; + } + let Some(auxiliary) = auxiliary.as_ref() else { + continue; + }; + let Some(slot) = return_capability_sealed::Sealed::return_slot(&auxiliary.capability) + else { + return Err(not_moved!(CallError::Refused(Status::Revoked))); + }; + malformed = match auxiliary.transfer { + AuxiliaryTransfer::Derived => { + transfer.send(destination, slot, auxiliary.rights).is_err() + } + AuxiliaryTransfer::Exact => transfer.send_exact(destination, slot).is_err(), + }; + } + if malformed { + return Err(not_moved!(CallError::Refused(Status::Malformed))); + } + + let observation = ReplyObservation::begin(None).await; + if self.rings.req.push(request).is_err() { + return Err(not_moved!(CallError::RingFull)); + } + let committed = match self.transport.commit(&transfer).await { + Ok(committed) => committed, + Err(_) => { + drop(second); + drop(third); + drop(auxiliaries); + self.poison(); + return Err(ContextBundleCallError::AfterTransfer(CallError::Transport)); + } + }; + match classify_return_commit(committed, transfer.staged(), ReturnSource::Held) { + ReturnCommit::Moved => { + return_capability_sealed::Sealed::on_proven_moved(second); + return_capability_sealed::Sealed::on_proven_moved(third); + for auxiliary in auxiliaries.into_iter().flatten() { + return_capability_sealed::Sealed::on_proven_moved(auxiliary.capability); + } + self.state = ClientState::AwaitingReply { request }; + self.kick_published_request() + .await + .map_err(ContextBundleCallError::AfterTransfer)?; + self.collect_reply(observation) + .await + .map_err(ContextBundleCallError::AfterTransfer) + } + ReturnCommit::NotMoved(error) => { + // Custody stayed local, but the record is already visible + // and the ring has no rollback: fail closed, as the + // invocation bundle does. + self.poison(); + Err(not_moved!(error)) + } + ReturnCommit::AfterTransfer(error) => { + drop(second); + drop(third); + drop(auxiliaries); + self.poison(); + Err(ContextBundleCallError::AfterTransfer(error)) + } + } + } + /// Issue a protocol-declared implicit return request as one L4-style /// **send-and-wait** crossing. /// diff --git a/native/crates/rt/src/svc/mod.rs b/native/crates/rt/src/svc/mod.rs index 86aa8fb..41177f8 100644 --- a/native/crates/rt/src/svc/mod.rs +++ b/native/crates/rt/src/svc/mod.rs @@ -48,7 +48,8 @@ mod recyclable; mod serve; pub use client::{ - AffineBundleCallError, CallError, CallReply, ClientEndpoints, GrantPin, InvocationAuxiliary, + AffineBundleCallError, CallError, CallReply, ClientEndpoints, ContextBundleCallError, GrantPin, + InvocationAuxiliary, InvocationAuxiliaryCapability, MAX_INVOCATION_AUXILIARIES, Observation, ReplyWaitStats, RetainedReturnCallError, ReturnCallError, ReturnCapability, ReturnDeriveError, ReturnDeriver, ReturnLeaseSource, @@ -497,6 +498,8 @@ const _: () = { assert!(!kernel_reserved(COOKIE_HOMES)); assert!(COOKIE_HOMES != COOKIE_STORE); assert!(!kernel_reserved(COOKIE_SPMD)); + assert!(!kernel_reserved(COOKIE_VDEV)); + assert!(COOKIE_VDEV != COOKIE_SPMD); assert!(!kernel_reserved(COOKIE_LUA)); assert!(!kernel_reserved(COOKIE_LUA_REPL)); assert!(!kernel_reserved(COOKIE_MPY)); @@ -998,6 +1001,17 @@ pub const COOKIE_EXEC_CLEANUP_C: u64 = 0x6B72_5F65_7843_6C43; // "kr_exClC" /// `spmd-witness` prints the detailed line instead). pub const COOKIE_SPMD: u64 = 0x6B72_5F73_706D_6457; // "kr_spmdW" +/// Completion cookie for the boot virtual-device run (`"kr_vdevT"`), posted +/// by INIT beside [`COOKIE_SPMD`]: it launches `vdev-test` through the +/// ordinary generation-bound spawn path, with the face exec mints for the +/// virtual-device profile, and forwards the verdict. Payload 0 is the clean +/// run; 900 no row/capability; 1000+s is a nonzero exit s, where application +/// slots 256..262 are `vdev-test`'s stages (open, identity, lend, bring-up, +/// command, result, close) and 4 a launch that was given no face. Anything +/// else is the launch's spawn rung. Non-fatal, as every boot verdict is: the +/// gates key on it, and a terminal rerun of `vdev-test` prints the stage. +pub const COOKIE_VDEV: u64 = 0x6B72_5F76_6465_7654; // "kr_vdevT" + /// Completion cookie for INIT's production Lua VM validation (`"kr_luaOK"`). /// The same archive generation exposed to each shell is launched through the /// ordinary profile-driven path with `--selftest`; payload zero is success and diff --git a/native/crates/rt/src/svc/serve.rs b/native/crates/rt/src/svc/serve.rs index 1b79832..985f785 100644 --- a/native/crates/rt/src/svc/serve.rs +++ b/native/crates/rt/src/svc/serve.rs @@ -2358,6 +2358,39 @@ impl<'w, 't, T: Transport, S: SystemService> ServeHost<'w, 't, T, S> { // work: a pair a serve step just minted is attached before // the next wake, and a retired one stops now. self.settle_minted(); + self.deliver_owed_wakes().await; + } + } + + /// Send each peer the service owes a wake its one notice + /// ([`SystemService::take_owed_wakes`]), through the endpoint this host + /// owns. A service names a peer once per owed wake, so the batches end + /// within the peer table; the bound only keeps a service that kept + /// answering full batches from holding the loop. + async fn deliver_owed_wakes(&mut self) { + // Peers asked for per call: a stack array, so the ask allocates + // nothing; its size only trades calls for stack, and the founder + // roster's whole recyclable band fits one batch. + const BATCH: usize = 8; + for _ in 0..=self.pairs.len().div_ceil(BATCH) { + let mut owed = [None; BATCH]; + let count = self.svc.take_owed_wakes(&mut owed).min(BATCH); + for peer in owed[..count].iter().flatten() { + // A peer retired since its batch was published has no one + // left to wake; `notify_peer` refuses it by generation. + let _ = self + .notify_peer( + *peer, + Notice { + cookie: crate::event::COOKIE_EVENT, + payload: 0, + }, + ) + .await; + } + if count < BATCH { + break; + } } } @@ -2406,6 +2439,7 @@ impl<'w, 't, T: Transport, S: SystemService> ServeHost<'w, 't, T, S> { // The handshake drains even on an idle pass, so a host test can // attach a freshly minted pair without a request in flight. self.settle_minted(); + self.deliver_owed_wakes().await; served } } @@ -2820,6 +2854,116 @@ mod tests { ); } + /// A service that publishes into its peers' auxiliary planes and owes + /// some of them a wake (`SystemService::take_owed_wakes`). + struct Owing { + /// Peers still owed, or `None` for a service that answers full + /// batches forever. + owed: Option, + asks: u32, + } + + impl SystemService for Owing { + const NAME: &'static str = "owing-test"; + type Endowment = NoCaps; + + fn bind(_: NoCaps) -> Self { + Self { owed: Some(0), asks: 0 } + } + + fn serve(&mut self, _: u64, sv: &mut Serve<'_, Reply>) { + sv.reply(Reply::Granted); + } + + fn take_owed_wakes(&mut self, owed: &mut [Option]) -> usize { + self.asks += 1; + let count = match self.owed.as_mut() { + Some(left) => { + let count = (*left).min(owed.len()); + *left -= count; + count + } + None => owed.len(), + }; + for peer in &mut owed[..count] { + *peer = PeerId::from_arrival(0x102); + } + count + } + } + + impl SvcProtocol for Owing { + type Request = u64; + type Response = Reply; + type ReqWire = u64; + type RepWire = u64; + + const REPLY_GRANTS: usize = 0; + + fn encode_req(req: &u64) -> u64 { + *req + } + + fn decode_req(wire: &u64) -> Option { + Some(*wire) + } + + fn encode_rep(rep: &Reply, grants: u8) -> u64 { + DropAware::encode_rep(rep, grants) + } + + fn decode_rep(wire: &u64) -> Option<(Reply, u8)> { + DropAware::decode_rep(wire) + } + } + + /// The host asks for owed wakes after every pass — a served request and + /// an idle one alike — asks again while batches come back full, and stops + /// within the peer table even for a service that never stops owing. (A + /// notice to a peer that is not a live recyclable generation is refused + /// by generation, which is what a peer retired since its batch gets.) + #[test] + fn the_host_drains_owed_wakes_after_every_pass_and_bounds_the_batches() { + let window = { + let bytes = window_bytes::(REQ_CAP, REP_CAP); + let layout = std::alloc::Layout::from_size_align(bytes, 64).expect("window layout"); + // SAFETY: as `window()` above — one leaked aligned allocation. + unsafe { + let ptr = std::alloc::alloc_zeroed(layout); + assert!(!ptr.is_null()); + core::slice::from_raw_parts_mut(ptr, bytes) + } + }; + let (mut client, rings) = split_window::(window, REQ_CAP, REP_CAP).expect("rings"); + let endpoints = ServeEndpoints { + kick: SlotRef::from_wire(60), + grant_tx: [None; MAX_GRANTS], + scratch: [None; MAX_GRANTS], + return_post: None, + }; + let transport = OkTransport::new(); + let mut host = ServeHost::new(Owing::bind(NoCaps), &transport, rings, endpoints); + + client.req.push(1).expect("request fits"); + assert!(drive(host.serve_next())); + assert_eq!(host.service().asks, 1, "a pass that owes nothing asks once"); + + // Every peer the table holds owed at once: full batches, then the + // short ask that ends them. + let peers = host.pairs.len(); + host.service_mut().owed = Some(peers); + assert!(!drive(host.serve_next())); + assert_eq!(host.service().asks, 1 + peers as u32 / 8 + 1); + assert_eq!(host.service().owed, Some(0)); + + // A service that never stops owing is bounded by the peer table. + host.service_mut().owed = None; + host.service_mut().asks = 0; + assert!(!drive(host.serve_next())); + let bound = host.pairs.len().div_ceil(8) as u32 + 1; + assert_eq!(host.service().asks, bound); + } + /// Walking return posts are host-owned reply-and-wait state: the first /// stands before readiness, and every successor is acknowledged before /// `serve_next` returns with its reply visible. diff --git a/native/crates/rt/src/thread.rs b/native/crates/rt/src/thread.rs index bbbe78a..b4d6606 100644 --- a/native/crates/rt/src/thread.rs +++ b/native/crates/rt/src/thread.rs @@ -415,6 +415,22 @@ fn release_thread_index(n: u32) { } } +/// **How many thread bands are free now**: how many more threads this task +/// could found before its band plan is full. An upper bound — a founding also +/// needs its founder's arena names and, for a strip stack, room in the stack +/// window — and a snapshot, which a concurrent founding or join moves. +#[cfg(target_os = "none")] +pub fn free_bands() -> u32 { + let used = THREAD_SLOTS.load(Ordering::Acquire); + (!used & (((1u64 << MAX_THREADS) - 1) as u32)).count_ones() +} + +/// Off the metal no thread founds another: no band is free. +#[cfg(not(target_os = "none"))] +pub fn free_bands() -> u32 { + 0 +} + /// How a spawned thread enters Rust: the plain `fn()` every service /// worker uses, or an **spmd entry** carrying a region launch's context /// and the index it assigned (`res/SPMD.md` §2) — the CUDA @@ -480,18 +496,21 @@ pub struct Thread { /// The parent-arena slot holding the child's task handle: the join /// subject, and (revoked) the reap. handle: SlotRef, - /// The parent-held `Region`s the provisioning session produced — - /// stack, runtime block, boot staging page. Revoking them is what - /// actually unmaps the band; the parent holds them precisely so that - /// a dead thread cannot keep its own stack alive. - provisioned: [SlotRef; 3], + /// The parent-held `Region`s the provisioning session produced that + /// outlive the founding: the stack and the runtime block. Revoking them + /// is what actually unmaps the band; the parent holds them precisely so + /// that a dead thread cannot keep its own stack alive. The third, the + /// boot staging page, is the founding's delegation source, which its + /// commit consumes ([`HELD_REGIONS`]). + regions: [SlotRef; HELD_REGIONS], tls_region: Option, - /// Founding result slots the kernel already scrubbed (the staged - /// space, the delivered view, the boot delegation, one per - /// endowment): no capability is left in them, but the *userland* slot - /// reservation is, and a task that never returned it would run out of - /// arena long before it ran out of bands. - spent: [SlotRef; 3 + MAX_WORKER_CAPS], + /// Founding names whose return waits for [`join`]: always the staged + /// space, whose founder record the `Found` leaves standing until the reap + /// ([`STAGED_SPACE`]); and, for a gang founding that did not commit whole, + /// whose outcome only the join learns, what a whole commit returns at + /// once ([`release_committed`]: [`RELEASED_AT_COMMIT`] names and one per + /// endowment). + spent: [SlotRef; SPENT_CAPACITY], /// Live entries in [`spent`](Self::spent). spent_len: u8, /// The arena strip this stack was carved from — `Some` for SPMD @@ -629,12 +648,155 @@ pub(crate) fn heap_endowment() -> Option { Some(WorkerCap::heap(rt.bootstrap().handle(role::HEAP, 0)?)) } -/// Off metal no thread founds another; there is no heap to endow. +/// The rights of a wake send a thread may pass on: the notice itself +/// (`ACCESS`, [`verbs::NOTICE_RIGHTS`](crate::cap::verbs::NOTICE_RIGHTS)) and +/// `DERIVE`, so the thread can found children holding the same end. A send +/// into a wake channel conveys nothing but the right to kick it. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) const WAKE_SEND_RIGHTS: kore_abi::cap::Rights = + kore_abi::cap::Rights::ACCESS.union(kore_abi::cap::Rights::DERIVE); + +/// [`WorkerCap`] handing a child this thread's own capability at +/// `(role, index)`: a derive of it, of the same `kind`, holding `rights`, +/// which the child finds under the same coordinate. +/// +/// `None` when this thread holds nothing there, something of another kind, +/// or a capability that cannot give `rights` to a child: its own must cover +/// them and `DERIVE`. A share this thread cannot make is not offered, so it +/// never fails the spawn it would have rode on. The arena record decides, +/// which is the kernel's own word for the slot's kind and rights. +/// +/// This is how a capability reaches every thread of a process without any +/// thread inheriting another's holdings: each founder names what it hands on +/// and re-derives it. Ask for `DERIVE` in `rights` when the child founds +/// threads that are to hold it too. +pub fn share_endowment( + role: u16, + index: u16, + kind: kore_abi::cap::ResourceKind, + rights: kore_abi::cap::Rights, +) -> Option { + share_endowment_of_kind(role, index, kind as u16, rights) +} + +/// [`share_endowment`] with the kind as its wire discriminant, for a caller +/// that stored it. +#[cfg(target_os = "none")] +pub(crate) fn share_endowment_of_kind( + role: u16, + index: u16, + kind: u16, + rights: kore_abi::cap::Rights, +) -> Option { + use kore_abi::cap::{CapRecord, Rights}; + let rt = crate::entry::thread_context()?; + let subject = rt.bootstrap().handle(role, index)?; + let record_va = crate::entry::geometry::slot_record_va(subject)?; + // SAFETY: the slot alias is this thread's kernel-authored, read-only view + // of its own arena record for a slot its boot block names. + let record = unsafe { core::ptr::read_volatile(record_va as *const CapRecord) }; + if record.kind != kind || !record.rights.contains(rights | Rights::DERIVE) { + return None; + } + Some(WorkerCap { + role, + index, + subject, + kind: u8::try_from(kind).ok()?, + rights, + }) +} + +/// Off metal no thread founds another. #[cfg(not(target_os = "none"))] -pub(crate) fn heap_endowment() -> Option { +pub(crate) fn share_endowment_of_kind( + _role: u16, + _index: u16, + _kind: u16, + _rights: kore_abi::cap::Rights, +) -> Option { + None +} + +/// [`WorkerCap`] re-deriving this thread's own `DOORBELL@PEER_BASE` send for a +/// child, so the child's [`kick_parent`] reaches the same thread this one's +/// does: a [`DoorbellWorker`]'s wake end, or one inherited the same way. +/// +/// `None` when this thread holds no such send, or holds one it may not derive +/// from (a band peer's notice-only end, [`spmd`](crate::spmd)'s mesh): see +/// [`share_endowment`]. +#[cfg(target_os = "none")] +pub(crate) fn wake_endowment() -> Option { + share_endowment( + role::DOORBELL, + crate::futex::PEER_BASE, + kore_abi::cap::ResourceKind::ChanSend, + WAKE_SEND_RIGHTS, + ) +} + +impl WorkerCap { + /// **Channel space, endowed** — `role::CHANNEL_SPACE@0`: a revocable + /// child of the mint grant `space`, so the thread can mint the channels + /// that wake the threads *it* founds (a `DoorbellWorker` of its own, or + /// the [`thread_controller`](crate::thread_controller) a C program's + /// `pthread_create` starts on its main thread). The parent keeps its + /// grant; revoking it ends the child's. + pub fn channel_space(space: SlotRef) -> Self { + use kore_abi::cap::ResourceKind; + Self { + role: role::CHANNEL_SPACE, + index: 0, + subject: space, + kind: ResourceKind::ChannelSpace as u16 as u8, + rights: crate::cap::CHANNEL_SPACE_RIGHTS, + } + } +} + +/// [`WorkerCap::channel_space`] over this thread's own `role::CHANNEL_SPACE@0` +/// — `None` when it holds no mint grant. +#[cfg(target_os = "none")] +pub fn channel_space_endowment() -> Option { + let rt = crate::entry::thread_context()?; + Some(WorkerCap::channel_space( + rt.bootstrap().handle(role::CHANNEL_SPACE, 0)?, + )) +} + +/// Off metal no thread founds another. +#[cfg(not(target_os = "none"))] +pub fn channel_space_endowment() -> Option { None } +/// This thread's **own kick**: a send into its own doorbell it may derive +/// from, as a slot — `DOORBELL@`[`OWN_KICK`](crate::futex::OWN_KICK), or for +/// thread 0 the founder-carved self-kick at `DOORBELL@PEER_BASE`. +/// +/// `None` when it holds none, or one it may not derive from: the authority a +/// thread hands a peer so the peer can wake it must itself be derivable. The +/// arena record decides, as for [`wake_endowment`]. +#[cfg(target_os = "none")] +pub(crate) fn own_kick() -> Option { + use kore_abi::cap::{CapRecord, ResourceKind}; + let rt = crate::entry::thread_context()?; + let boot = rt.bootstrap(); + let send = boot + .handle(role::DOORBELL, crate::futex::OWN_KICK) + .or_else(|| { + (crate::entry::geometry::shard_index() == Some(0)) + .then(|| boot.handle(role::DOORBELL, crate::futex::PEER_BASE)) + .flatten() + })?; + let record_va = crate::entry::geometry::slot_record_va(send)?; + // SAFETY: the slot alias is this thread's kernel-authored, read-only view + // of its own arena record for a slot its boot block names. + let record = unsafe { core::ptr::read_volatile(record_va as *const CapRecord) }; + (record.kind == ResourceKind::ChanSend as u16 && record.rights.contains(WAKE_SEND_RIGHTS)) + .then_some(send) +} + /// Build exactly the capability namespace a spawned worker receives. /// /// This stays target-independent so host tests can pin a subtle shared-space @@ -694,7 +856,12 @@ fn push_worker_thread_blocks(gblock: &mut GeomBlock, local: Window) -> bool { /// view. [`spawn_with`] when the thread must hold capabilities. #[cfg(target_os = "none")] pub fn spawn(entry: fn()) -> Option { - spawn_entry(ThreadEntry::Plain(entry), &[]) + spawn_entry( + ThreadEntry::Plain(entry), + &[], + WorkerStack::Band, + WorkerArena::Task, + ) } // (The per-worker `spawn_worker` shim retired with G5: workers found as a @@ -711,7 +878,158 @@ pub fn spawn(entry: fn()) -> Option { /// into [`exit`]`(0)`. #[cfg(target_os = "none")] pub fn spawn_with(entry: fn(), endow: &[WorkerCap]) -> Option { - spawn_entry(ThreadEntry::Plain(entry), endow) + spawn_entry( + ThreadEntry::Plain(entry), + endow, + WorkerStack::Band, + WorkerArena::Task, + ) +} + +/// Where a spawned worker's stack lives. +/// +/// A plain thread's stack is its band slot by default: `STACK_BYTES` +/// (64 KiB) live under a guard page, tiled by [`MAX_THREADS`] — what a +/// service worker's bounded frames need. A synchronous engine whose +/// recursion depth follows its *input* — a C compiler's expression and block +/// nesting — needs more than a slot holds, so it asks for a **guarded +/// strip** of the task's stack window instead (`res/SPMD.md` §8's +/// kernel-thread class, the shape SPMD gang workers already take: usable +/// pages with one guard granule below, mapped by the founding and given +/// back by `join`). Overflow takes a fault either way, never a +/// neighbour's memory. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WorkerStack { + /// The thread's band slot. + Band, + /// A guarded strip of at least this many bytes, rounded up to whole + /// pages. The strip comes from the task's one stack window (16 MiB of + /// reservation, shared with SPMD gangs); a request the window cannot + /// place — or a zero-byte one — refuses the spawn. + Bytes(usize), +} + +/// How many capability slots a spawned thread's arena holds: the one sizing +/// question a founding leaves to its founder (`FoundReq.arena_slots`, which +/// the kernel accepts in `1..=ARENA_SLOTS`). +/// +/// Every thread's arena holds its boot names, the runtime's serve scratch, +/// report and carve slots, and whatever it reserves while it runs. A thread +/// that **founds** holds more, for as long as each child lives: the child's +/// handle and the regions its stack, runtime block and boot page live in +/// (what [`join`] revokes), and, while a founding is in flight, that +/// founding's whole result plan. A thread that founds a gang, or a thread +/// controller that founds a process's threads, therefore needs an arena that +/// scales with its children, the case the kernel's founding granule is sized +/// for ("founders are the arena's real sizing case", `ARENA_SLOTS`). A thread +/// controller founded with a task's 64 slots held two live threads; with the +/// founding granule it holds over two dozen, within a couple of the band +/// plan's own limit ([`MAX_THREADS`]). +/// +/// Spawned arenas cannot grow: only thread 0 carries the extension window, so +/// the size is chosen once, here. +#[cfg(any(test, target_os = "none"))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum WorkerArena { + /// A thread that founds nothing: [`TASK_ARENA_SLOTS`]. + Task, + /// A thread that founds others: the founding granule, + /// [`ARENA_SLOT_CEILING`](crate::entry::ARENA_SLOT_CEILING). + Founder, +} + +/// A task's arena: one word of the runtime's slot ledger. +/// +/// It holds the widest endowment a worker can take, the serve scratch, the +/// report and carve slots behind it, and the short-lived reservations of a +/// thread that founds nothing (a channel mint, a join landing). The assert +/// below proves the first three fit; a widening of [`MAX_WORKER_CAPS`] or of +/// the serve scratch that broke that would fail the build here, not a +/// founding at run time. +#[cfg(any(test, target_os = "none"))] +pub(crate) const TASK_ARENA_SLOTS: u32 = u64::BITS; + +#[cfg(any(test, target_os = "none"))] +const _: () = { + // The highest boot name a worker can carry, then the runtime layout the + // entry derives after it: serve scratch, the report slot, the carve slot. + let top = child_slot::ENDOW_BASE as usize + MAX_WORKER_CAPS; + let carve = top + crate::entry::SERVE_SCRATCH_SLOTS + 2; + assert!(carve < TASK_ARENA_SLOTS as usize); + assert!(TASK_ARENA_SLOTS <= crate::entry::ARENA_SLOT_CEILING); + // Both sizes fit the per-thread slot alias band: the slot alias of a + // founder's arena is the band's whole width. + assert!(kore_abi::execution::alias_bytes(crate::entry::ARENA_SLOT_CEILING) <= SLOTS_BYTES); +}; + +#[cfg(any(test, target_os = "none"))] +impl WorkerArena { + /// The `FoundReq.arena_slots` this arena is founded with. + pub(crate) const fn slots(self) -> u32 { + match self { + Self::Task => TASK_ARENA_SLOTS, + Self::Founder => crate::entry::ARENA_SLOT_CEILING, + } + } +} + +/// The regions a founder provisions for each thread: the stack, the runtime +/// block and the boot staging page (a TLS block is one more, apart). +const PROVISIONED_REGIONS: usize = 3; + +/// The provisioned regions a founder holds until [`join`] revokes them: the +/// stack and the runtime block ([`Thread`]'s `regions`). The boot staging +/// page is the founding's delegation source: the commit moves its page into +/// the child and retires its founder record, so its name is free from then +/// on, a dead record any later landing or result replaces. +const HELD_REGIONS: usize = 2; + +/// The staged space a thread's `Found` consumes, whose founder name stays +/// spent until [`join`]'s reap. The commit moves the space into the child but +/// scrubs from the founder only what it delivers and delegates, so the +/// founder's record of the space stands — an `AddrSpace` with every right, +/// bound to nothing the founder holds — until the reap retires it. A name +/// returned before that is not vacant: the next post landing there is refused +/// as occupied, and the next result written there overwrites a live record. +const STAGED_SPACE: usize = 1; + +/// The founding results a committed `Found` scrubs from the founder, beside +/// one per endowment: the child's delivered view and the boot delegation's. +const SCRUBBED_RESULTS: usize = 2; + +/// The founding names a committed `Found` leaves without a live record, +/// beside one per endowment: the consumed staging page and the scrubbed +/// results ([`release_committed`]). +const RELEASED_AT_COMMIT: usize = PROVISIONED_REGIONS - HELD_REGIONS + SCRUBBED_RESULTS; + +/// [`Thread`]'s `spent`: the staged space, and what a gang founding that did +/// not commit whole could not return. +const SPENT_CAPACITY: usize = STAGED_SPACE + RELEASED_AT_COMMIT + MAX_WORKER_CAPS; + +/// Arena names one thread's founding holds in its founder while it is in +/// flight, beside one per endowment: its provisioned regions, the staged +/// space, the results the commit scrubs, and the handle. One more with a TLS +/// block. +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +pub(crate) const FOUNDING_SLOTS: usize = PROVISIONED_REGIONS + STAGED_SPACE + SCRUBBED_RESULTS + 1; + +/// Arena names a founded thread keeps in its founder until [`join`]: its +/// handle, its held regions and its staged space (one more with a TLS block). +/// The rest of the founding's plan returns at commit ([`release_committed`]). +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +pub(crate) const RETAINED_SLOTS: usize = 1 + HELD_REGIONS + STAGED_SPACE; + +// Every founding name is kept until the reap or returned at commit, never +// both and never neither: the launch budgets ([`crate::spmd`]) count the two +// halves apart. +const _: () = assert!(FOUNDING_SLOTS == RETAINED_SLOTS + RELEASED_AT_COMMIT); + +/// The arena names a gang session founding `workers` threads holds at its +/// peak, just before its founding commits: every worker's claims and every +/// endowment result. +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +pub(crate) const fn gang_founding_slots(workers: usize, tls: bool, endowments: usize) -> usize { + workers * (FOUNDING_SLOTS + tls as usize) + endowments } /// A worker that can **park and be kicked**, and kick back: two channels @@ -719,7 +1037,9 @@ pub fn spawn_with(entry: fn(), endow: &[WorkerCap]) -> Option { /// doorbell (its recv; the parent keeps the send that kicks it) and a /// **wake** channel the other way (the worker holds the send at /// `DOORBELL@PEER_BASE`; the parent parks on the recv) — plus a heap -/// position over the task's supply so it may allocate. +/// position over the task's supply so it may allocate, and the worker's +/// own kick, a derivable send into its doorbell it can hand a thread it +/// founds. /// /// The wake channel exists because a launched task holds no self-kick: /// the founder carves `DOORBELL@2` for service rows that declare workers, @@ -795,6 +1115,19 @@ impl DoorbellWorker { } } +/// The calling thread's C `errno` word — what a C runtime's +/// `__errno_location` answers. +/// +/// Per thread, as ISO C requires (C11 7.5p2): the word lives in this +/// thread's runtime block rather than an ELF TLS block, so an image that +/// carries C code needs no TLS segment for it. Zero when the thread starts. +/// The pointer is valid for the calling thread's life, and only that thread +/// may read or write through it. +#[cfg(target_os = "none")] +pub fn c_errno() -> *mut i32 { + crate::entry::geometry::c_errno() +} + /// Worker side of [`DoorbellWorker`]: post one notice into the parent's /// wake channel (the send founded at `DOORBELL@PEER_BASE`). `false` when /// this thread holds no such send or the post was refused — the latter @@ -815,11 +1148,31 @@ pub fn kick_parent() -> bool { } /// Spawn `entry` on a worker founded holding its own doorbell, a send -/// into thread 0's doorbell, a heap position, and `extra`. `None` when -/// the task holds no `CHANNEL_SPACE` (its profile lacks `channels`), the -/// mint or a derive refuses, or [`spawn_with`] does. +/// into thread 0's doorbell, its own kick ([`own_kick`]), a heap position, +/// and `extra`. `None` when the task holds no `CHANNEL_SPACE` (its profile +/// lacks `channels`), the mint or a derive refuses, or [`spawn_with`] does. +/// +/// The worker takes a founder's arena, the founding granule of capability +/// slots rather than a task's 64: a doorbell worker is where a host runs a +/// program's own code (libkore's C `main`, a +/// language engine), and that code founds threads of its own — its first +/// `pthread_create` starts the [thread controller](crate::thread_controller) +/// and an SPMD dispatch founds a gang, each holding the founder's arena for +/// as long as its children live. #[cfg(target_os = "none")] pub fn spawn_with_doorbell(entry: fn(), extra: &[WorkerCap]) -> Option { + spawn_with_doorbell_on(entry, extra, WorkerStack::Band) +} + +/// [`spawn_with_doorbell`] on the stack `stack` names — a guarded strip +/// for an engine whose depth its input decides ([`WorkerStack`]). `None` +/// also when the stack window cannot place the strip. +#[cfg(target_os = "none")] +pub fn spawn_with_doorbell_on( + entry: fn(), + extra: &[WorkerCap], + stack: WorkerStack, +) -> Option { use kore_abi::cap::{ResourceKind, Rights}; use crate::cap::verbs; @@ -854,7 +1207,8 @@ pub fn spawn_with_doorbell(entry: fn(), extra: &[WorkerCap]) -> Option caps.len() { + const OWN: usize = 4; + if OWN + extra.len() > caps.len() { let _ = pair.send.try_retire(); let _ = pair.root.try_retire(); let _ = wake.send.try_retire(); @@ -869,16 +1223,34 @@ pub fn spawn_with_doorbell(entry: fn(), extra: &[WorkerCap]) -> Option Option Option { - spawn_entry(ThreadEntry::Argument(entry, argument), endow) + spawn_entry( + ThreadEntry::Argument(entry, argument), + endow, + WorkerStack::Band, + WorkerArena::Task, + ) +} + +/// [`spawn_with_arg`] founding a thread that founds others, with a founder's +/// arena ([`WorkerArena::Founder`]): the thread controller, and the threads +/// it founds, any of which may found a gang or ask for more threads. +#[cfg(target_os = "none")] +pub(crate) fn spawn_founder_with_arg( + entry: fn(usize), + argument: usize, + endow: &[WorkerCap], +) -> Option { + spawn_entry( + ThreadEntry::Argument(entry, argument), + endow, + WorkerStack::Band, + WorkerArena::Founder, + ) } #[cfg(not(target_os = "none"))] pub fn spawn_with_arg(_entry: fn(usize), _argument: usize, _endow: &[WorkerCap]) -> Option { None } +/// Return to the slot ledger the founding names a committed `Found` left +/// without a live record. +/// +/// A `Found` that commits delivers the child's view and every endowment out +/// of the founder (the delivery map scrubs each one's founder slot), clears +/// the founder's delegated `Region` results, the boot page's among them, and +/// retires the tag of the staging page its delegation consumed. No live +/// record is left under these names, and none can have been copied: the +/// founder was inside its own trap from apply to scrub. They are free the +/// moment the commit reports `Ok`, and only that proves it: a refused or +/// unreadable commit keeps its names (the founding's leak-on-refusal rule), +/// as a gang founding that did not commit whole keeps them in its tickets for +/// `join`. The staged space is not among them: the commit consumes it without +/// scrubbing its founder record ([`STAGED_SPACE`]). +#[cfg(target_os = "none")] +fn release_committed(slots: impl IntoIterator) { + for slot in slots { + crate::entry::geometry::release_cap_slot(slot); + } +} + /// The one founding every public spawn shape resolves to — the entry /// shape is the only thing the shapes disagree on, and it rides the /// start record untouched by everything below. #[cfg(target_os = "none")] -fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap]) -> Option { +fn spawn_entry( + entry: ThreadEntry, + endow: &[WorkerCap], + stack: WorkerStack, + arena: WorkerArena, +) -> Option { if endow.len() > MAX_WORKER_CAPS { return None; } @@ -927,7 +1347,7 @@ fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap]) -> Option { // ordinary slot exhaustion before spending a band or provisioning memory, // and leave a report slot available to poll/join existing children. // Session A owns three regions (+ TLS); B owns four results + endowments. - let slots_needed = 3 + usize::from(template.is_some()) + 4 + endow.len() + 1; + let slots_needed = FOUNDING_SLOTS + usize::from(template.is_some()) + endow.len() + 1; if crate::entry::geometry::available_cap_slots() < slots_needed { return None; } @@ -952,17 +1372,22 @@ fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap]) -> Option { w.map = p.remaining; } - // ── The stack: an arena strip for SPMD workers (G4b), the band - // slot for plain threads. Carved here and — exactly like the + // ── The stack: an arena strip for SPMD workers (G4b) and for a + // plain thread that asked for one ([`WorkerStack::Bytes`]), the + // band slot otherwise. Carved here and — exactly like the // band index above — left spent if a later founding step // refuses: the teardown that gives both back is join's, never // spawn's, and the leak-on-refusal is the safe direction for // the same reason the band's is. ── - let strip = match &entry { - ThreadEntry::Spmd(..) => { + let strip = match (&entry, stack) { + (ThreadEntry::Spmd(..), _) => { Some(crate::spmd::STACKS.carve_guarded(crate::spmd::STACK_GRANULES)?) } - ThreadEntry::Plain(_) | ThreadEntry::Argument(..) => None, + (ThreadEntry::Plain(_) | ThreadEntry::Argument(..), WorkerStack::Band) => None, + (ThreadEntry::Plain(_) | ThreadEntry::Argument(..), WorkerStack::Bytes(bytes)) => { + let granules = u32::try_from(bytes.div_ceil(crate::stack_arena::GRANULE)).ok()?; + Some(crate::spmd::STACKS.carve_guarded(granules)?) + } }; let (stack_base, stack_len) = match &strip { Some(s) => (s.base(), s.bytes() as u64), @@ -1034,7 +1459,7 @@ fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap]) -> Option { // ── Stamp the boot page (at its staging VA — the delegation moves // it to `boot_va`, which is what the child receives as `arg`). ── - let arena_slots: u32 = 64; + let arena_slots = arena.slots(); if let (Some(t), Some(p)) = (template, tls) { unsafe { t.initialize(p.mapping.base()) }; } @@ -1098,22 +1523,25 @@ fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap]) -> Option { if committed.status() != kore_abi::session::Status::Ok { return None; } - // The join ticket: what the thread cost, in the order `join` must - // give it back. - let mut spent = [space_res; 3 + MAX_WORKER_CAPS]; - spent[1] = view_res; - spent[2] = boot_res; - for (i, e) in staged[..endow.len()].iter().enumerate() { - spent[3 + i] = e.result; - } + // The founding's plan is spent, and the names it left without a live + // record come back now: the commit moved the view and every endowment + // into the child, scrubbing their founder slots, cleared the boot + // delegation's, and consumed the staging page (`release_committed`). The + // staged space's record stands until the reap ([`STAGED_SPACE`]), so its + // name rides the join ticket beside what `join` revokes. + release_committed( + [stage_res, view_res, boot_res] + .into_iter() + .chain(staged[..endow.len()].iter().map(|e| e.result)), + ); Some(Thread { owner, index, handle: handle_res, - provisioned: [stack_res, tblock_res, stage_res], + regions: [stack_res, tblock_res], tls_region, - spent, - spent_len: (3 + endow.len()) as u8, + spent: [space_res; SPENT_CAPACITY], + spent_len: STAGED_SPACE as u8, strip, }) } @@ -1317,6 +1745,19 @@ pub(crate) fn spawn_gang_workers( if star.is_some() && n > crate::star::MESH_CEILING { return false; } + // Refuse a session this arena cannot finish before claiming anything: + // past here a refusal leaks what it claimed. A planned endowment is the + // star's mesh closed with the heap; an explicit one is as given. + let endowments = if endows.is_empty() { + n * (star.map_or(0, |plan| plan.mesh_caps(n)) + usize::from(heap.is_some())) + } else { + endows.iter().map(|caps| caps.len()).sum() + }; + if crate::entry::geometry::available_cap_slots() + < gang_founding_slots(n, template.is_some(), endowments) + { + return false; + } let Some(self_space) = crate::entry::geometry::self_space_slot() else { return false; }; @@ -1339,7 +1780,8 @@ pub(crate) fn spawn_gang_workers( tls_region: Option, } let mut claims: [Option; MAX_SESSION_FOUNDS] = [const { None }; MAX_SESSION_FOUNDS]; - let arena_slots: u32 = 64; + // A gang worker runs a region's body and founds nothing. + let arena_slots = WorkerArena::Task.slots(); let transport = TrapTransport::new(); // ── Session A′: provision every worker's three pages, one commit. ── let mut a = SessionBuilder::new(); @@ -1609,23 +2051,35 @@ pub(crate) fn spawn_gang_workers( let founded_all = committed.status() == kore_abi::session::Status::Ok; // Mint every ticket now that the founding committed: whichever // workers exist are joinable, and a vacant handle's join refuses in - // the caller's abort arm rather than orphaning anyone. + // the caller's abort arm rather than orphaning anyone. A whole commit + // returns what it left of each worker's founding without a live record + // now (`release_committed`); a partial one leaves those names to the + // tickets, for the join that learns which workers exist. Either way the + // staged space's name waits for the reap ([`STAGED_SPACE`]). for (k, claim) in claims.iter_mut().take(n).enumerate() { let Some(c) = claim.take() else { return false }; - let mut spent = [c.space_res; 3 + MAX_WORKER_CAPS]; - spent[1] = c.view_res; - spent[2] = c.boot_res; - for (i, e) in endow_store[k][..endow_lens[k]].iter().enumerate() { - spent[3 + i] = e.result; - } + let endowed = &endow_store[k][..endow_lens[k]]; + let committed = [c.stage_res, c.view_res, c.boot_res] + .into_iter() + .chain(endowed.iter().map(|e| e.result)); + let mut spent = [c.space_res; SPENT_CAPACITY]; + let spent_len = if founded_all { + release_committed(committed); + STAGED_SPACE + } else { + for (name, slot) in spent[STAGED_SPACE..].iter_mut().zip(committed) { + *name = slot; + } + STAGED_SPACE + RELEASED_AT_COMMIT + endowed.len() + }; out[k] = Some(Thread { owner, index: c.index, handle: c.handle_res, - provisioned: [c.stack_res, c.tblock_res, c.stage_res], + regions: [c.stack_res, c.tblock_res], tls_region: c.tls_region, spent, - spent_len: (3 + endow_lens[k]) as u8, + spent_len: spent_len as u8, strip: Some(c.strip), }); } @@ -1827,6 +2281,13 @@ pub fn exit(code: ExitCode) -> ! { let Some(rt) = crate::entry::thread_context() else { crate::entry::refuse(); }; + // A thread whose end is awaited announces it first ([`on_exit`]): every + // way a thread ends that runs code — its entry returning, a panic, an + // explicit exit — comes through here. Taken, so a hook that itself ends + // the thread runs once. + if let Some((call, argument)) = crate::entry::geometry::take_exit_hook() { + call(argument); + } let Some(slot) = rt.bootstrap().handle(role::SELF, 0) else { crate::entry::refuse(); }; @@ -1852,6 +2313,22 @@ pub fn exit(_code: ExitCode) -> ! { crate::entry::refuse() } +/// Have this thread's [`exit`] run `call(argument)` first, on this thread, +/// before the exit commits — how the thread that reaps it learns it is +/// ending without polling for it (the +/// [`thread_controller`](crate::thread_controller)'s children). The last hook +/// set stands. `call` runs after the thread's own code, so it must not block, +/// and must not end the thread itself. +/// +/// The kernel's exit commits after the hook returns, so a reaper woken by +/// the hook can still find the thread live for a moment; it parks on the +/// thread's end for that moment (`futex::wait_or_end`), which no store could +/// announce. +#[cfg(target_os = "none")] +pub(crate) fn on_exit(call: fn(usize), argument: usize) { + crate::entry::geometry::set_exit_hook(call, argument); +} + /// One caller-owned report landing reserved for a complete worker-join fold. /// /// The slot is affine for the same reason a [`Thread`] is: once worker @@ -1908,8 +2385,9 @@ impl Drop for JoinScratch { /// 2. **the parent-held regions** — stack, runtime block, boot staging. /// These are the parent's precisely so a dead thread cannot keep its /// own stack alive; -/// 3. **the arena slots** — the founding results the kernel scrubbed, plus -/// the handle; +/// 3. **the arena slots** — the held regions, the staged space, whose +/// founder record the reap has just retired, whatever a partial gang +/// commit left in the ticket, and the handle; /// 4. **the band index — last.** Every window in that band is now /// unmapped, so the next [`spawn`] can be handed it. Released any /// earlier and a new thread would map its stack over a corpse's: the @@ -1978,21 +2456,21 @@ pub(crate) fn join_reserved_at( }); } // The stack and runtime-block regions are parent-held and die here. - // The STAGE region is deliberately NOT presented: the founding's + // The STAGE region is deliberately not among them: the founding's // delegate consumed it (the boot page re-rooted under the child, its // parent record retired with the move), so revoking it refuses the // WHOLE session as `Revoked` — which un-revokes the handle too and // silently leaves `torn_down` false. That exact shape leaked every // joined thread's strip and band index since #407 (bounded, and - // invisible until the G7 retirement witness demanded a real reap); - // the slot itself is still released below, corpse and all. - for slot in [ - Some(thread.provisioned[0]), - Some(thread.provisioned[1]), - thread.tls_region, - ] - .into_iter() - .flatten() + // invisible until the G7 retirement witness demanded a real reap). + // Its name came back when the founding committed, or rides `spent` + // from a gang founding that did not commit whole. + for slot in thread + .regions + .into_iter() + .map(Some) + .chain([thread.tls_region]) + .flatten() { if b.revoke(slot).is_err() { return Err(ThreadJoinError { @@ -2019,7 +2497,7 @@ pub(crate) fn join_reserved_at( if let Some(slot) = thread.tls_region { crate::entry::geometry::release_cap_slot(slot); } - for slot in thread.provisioned { + for slot in thread.regions { crate::entry::geometry::release_cap_slot(slot); } for slot in &thread.spent[..thread.spent_len as usize] { @@ -2187,6 +2665,19 @@ fn join_park( } } +/// [`Thread::poll`] through a landing reserved for it: a repeat observer — +/// the [`thread_controller`](crate::thread_controller), between a child's +/// announced end and its reap — probes without spending an arena slot each +/// time, and so without the one refusal that is about the observer rather +/// than the thread. +#[cfg(target_os = "none")] +pub(crate) fn poll_reserved( + thread: &Thread, + scratch: &JoinScratch, +) -> Result { + poll_task_handle_in(&TrapTransport::new(), thread.handle, scratch.report) +} + /// Issue one unflagged join probe. `WouldBlock` is the one positive live-task /// result; unlike the blocking path, an OCC conflict or unrelated park status /// is surfaced so a nonblocking caller controls its own retry policy. diff --git a/native/crates/rt/src/thread_controller.rs b/native/crates/rt/src/thread_controller.rs index 65c6289..dbee9b4 100644 --- a/native/crates/rt/src/thread_controller.rs +++ b/native/crates/rt/src/thread_controller.rs @@ -4,13 +4,56 @@ //! records. Only the controller touches native tickets. A detached record is //! reusable only after native exit and successful reclamation. Storage is //! supplied by the process and must outlive every client and child. +//! +//! # Every wait is a futex park +//! +//! Nothing here polls. The controller thread waits on its hub word +//! ([`futex::Hub`]), and everything that needs it — a create request, a +//! child's end, a wake relayed through it — stores, then wakes that word, the +//! order [`futex`] requires. A client waits on its record's event word, which +//! the controller bumps and wakes after every transition a client can wait +//! for. A child announces its end through the hook [`thread::exit`] runs, on +//! every way a thread ends that runs code; the kernel's exit commits a moment +//! later, an edge no store can announce, so the controller reaps through a +//! park whose join arm watches that child beside its word +//! ([`futex::wait_or_end`]). +//! +//! # The controller is its threads' hub +//! +//! The star's rule (`res/SPMD.md` §3) for threads that come and go. The +//! starter mints the controller's doorbell, keeps a send into it (an adopted +//! peer), and founds it holding that doorbell, its own kick, the starter's +//! kick, the host's wake, the heap and a channel-space grant. For each child +//! the controller mints a doorbell, keeps the send (an adopted peer, withdrawn +//! before the child's band can be reused), and founds the child holding it, a +//! send into the controller, the starter's kick, the host's wake and the heap. +//! Children and the starter relay through the controller's hub, so a wake +//! between two threads neither can kick directly still arrives: the controller +//! sweeps ([`futex::sweep`]). +//! +//! Both the controller and every child take a founder's arena (the +//! founding granule of capability slots): the controller holds each live child's handle, +//! its stack and runtime-block regions, its staged space and the root and send +//! of its doorbell until the reap, a whole process's worth of them, and a +//! child may found an SPMD gang of its own. +//! +//! # Degrade, never wedge +//! +//! A starter with no channel space founds a controller without a doorbell: +//! neither it nor its children can park, and each of their waits takes the +//! deadline park every thread can ([`pace::cede`]). A client the controller +//! cannot kick waits in bounded slices ([`UNREACHED_SLICE`]). A thread that +//! cannot wake a parking controller is refused a create, which would +//! otherwise wait on a request nothing announced. +//! +//! [`thread::exit`]: crate::thread::exit +//! [`pace::cede`]: crate::pace::cede use core::cell::UnsafeCell; -use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use core::time::Duration; -// Each owner derives the process heap into its children using its own local -// slot. No creator's slot number is ever forwarded across the controller seam. -use super::thread::heap_endowment; +use super::futex::{self, Hub, WaitOutcome}; const FREE: u64 = 0; const BUILD: u64 = 1; @@ -26,6 +69,37 @@ const RETIRED: u64 = 15; const MASK: u64 = 15; const COUNT: usize = super::thread::MAX_THREADS as usize - 2; +/// Lifecycle of the controller thread itself, in [`Controller::started`]: +/// not started, being founded, or started — parking on its doorbell, or +/// without one, taking the deadline park. +const IDLE: u32 = 0; +const STARTING: u32 = 1; +const PARKING: u32 = 2; +#[cfg_attr( + not(target_os = "none"), + expect(dead_code, reason = "only a metal founding starts") +)] +const POLLING: u32 = 3; + +/// How long one park lasts for a client the controller cannot kick — a +/// thread it holds no send into, such as an initial thread launched without +/// a self-kick. Nothing can end such a park early, so its slice is the +/// latency it pays, and this is the cadence those waits paced themselves +/// at before the controller parked (`pace::CEDE_NS`): a client the wake +/// fabric cannot reach is no worse off than it was. Shorter would wake an +/// unreachable joiner more often for nothing; longer would delay it. +const UNREACHED_SLICE: Duration = Duration::from_nanos(crate::pace::CEDE_NS); + +/// How long the controller rides its doorbell after a reap the kernel did +/// not complete — a refused lifecycle observation or reclamation, which no +/// later event is bound to retry. The joiner's own pacing for a join that +/// cannot establish its park (`thread::join_park`, two milliseconds). +#[cfg_attr( + not(target_os = "none"), + expect(dead_code, reason = "reaping is metal-only") +)] +const REAP_RETRY: Duration = Duration::from_millis(2); + /// An identifier within one controller, never a capability-slot number. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Id { @@ -49,14 +123,30 @@ pub enum Error { struct Start { call: fn(usize) -> usize, argument: usize, + /// Read by the controller thread, after the reap. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] retire: Option, + /// The controller this record belongs to, for the child's hooks. + home: usize, } fn empty(_: usize) -> usize { 0 } +/// One managed thread's shared record, on its own interference span: its +/// creator, its joiner, the child and the controller each write it, and a +/// waiting client's monitored spin sleeps on its event word's line, which a +/// neighbour's traffic would end for nothing. The span is the one the +/// barrier isolates against (`barrier::Line`). +#[repr(align(128))] struct Record { state: AtomicU64, + /// Bumped by the controller after each transition a client can wait + /// for; clients park on it. An event count: the state is the truth. + event: AtomicU32, + /// Set by the child's exit hook: it has stopped running its own code + /// and is committing its end. + ending: AtomicU32, start: UnsafeCell, value: AtomicUsize, identity: AtomicUsize, @@ -69,10 +159,13 @@ impl Record { const fn new() -> Self { Self { state: AtomicU64::new(FREE), + event: AtomicU32::new(0), + ending: AtomicU32::new(0), start: UnsafeCell::new(Start { call: empty, argument: 0, retire: None, + home: 0, }), value: AtomicUsize::new(0), identity: AtomicUsize::new(0), @@ -85,13 +178,101 @@ impl Record { Ordering::Release, ); } + /// Publish a transition a client may be parked on: the state was + /// stored; now the event moves, then its waiters are woken. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + fn announce(&self) { + self.event.fetch_add(1, Ordering::SeqCst); + futex::wake_all(&self.event); + } +} + +/// A capability every thread a controller founds is founded holding, beyond +/// the runtime's own: a share of the one its founder holds at the same +/// boot-block coordinate ([`thread::share_endowment`]). +/// +/// Capabilities are per thread, so a program whose threads must all act on +/// one (park on a word of a shared window, ring a peer's doorbell) says so +/// here, once, and each founding hands it on: the starter to the controller, +/// the controller to each thread. A thread whose founder does not hold it is +/// founded without it. +/// +/// [`thread::share_endowment`]: crate::thread::share_endowment +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Handed { + /// The boot-block role every holder finds it under. + pub role: u16, + /// The instance within that role. + pub index: u16, + /// The kind of capability there. + pub kind: kore_abi::cap::ResourceKind, + /// The rights a thread holds it with. The controller holds `DERIVE` as + /// well, which is what lets it hand the share on. + pub rights: kore_abi::cap::Rights, +} + +/// How many capabilities a controller hands on. Each costs a derive at +/// every founding, and what a program's threads share is a face or two, not +/// a namespace: two is a device's interrupt word and its doorbell. +pub const MAX_HANDED: usize = 2; + +impl Handed { + /// Marks a taken entry: a vacant one is zero. + const TAKEN: u64 = 1 << 63; + + fn pack(self) -> Option { + let kind = u8::try_from(self.kind as u16).ok()?; + Some( + Self::TAKEN + | u64::from(self.role) << 32 + | u64::from(self.index) << 16 + | u64::from(kind) << 8 + | u64::from(self.rights.bits()), + ) + } + + /// `(role, index, kind, rights)` of a taken entry. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + fn unpack(word: u64) -> Option<(u16, u16, u16, kore_abi::cap::Rights)> { + (word & Self::TAKEN != 0).then(|| { + ( + (word >> 32) as u16, + (word >> 16) as u16, + u16::from((word >> 8) as u8), + kore_abi::cap::Rights::from_bits_truncate(word as u8), + ) + }) + } + + /// The coordinate alone: two entries for one coordinate would stamp it + /// twice in a thread's boot block. + const fn coordinate(word: u64) -> u64 { + word & (0xffff_ffff << 16) + } } /// Named process-owned storage. Start once during application startup and /// share its static reference with workers. This consumes one native thread /// for the process lifetime. No native ticket escapes its owner. pub struct Controller { - started: AtomicUsize, + /// [`IDLE`], [`STARTING`], [`PARKING`] or [`POLLING`]; clients park on + /// it while the controller is being founded. + started: AtomicU32, + /// What the controller thread waits on, and what its threads relay + /// through. + hub: Hub, + /// The controller thread's band, plus one (`0` until it is founded). + band: AtomicU32, + /// The band of the thread that started the controller, plus one. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + starter: AtomicU32, + /// The bands whose doorbells the controller can kick, a bit each: the + /// clients its wakes reach. A child's bit is set once it is founded and + /// cleared before its band can be reused. + reach: AtomicU32, + /// What every founding here hands on ([`Handed`], packed; zero is + /// vacant). + handed: [AtomicU64; MAX_HANDED], records: [Record; COUNT], } impl Default for Controller { @@ -99,35 +280,79 @@ impl Default for Controller { Self::new() } } +// The reach bitmap has one bit per band. +const _: () = assert!(super::thread::MAX_THREADS <= u32::BITS); + impl Controller { pub const fn new() -> Self { Self { - started: AtomicUsize::new(0), + started: AtomicU32::new(IDLE), + hub: Hub::new(), + band: AtomicU32::new(0), + starter: AtomicU32::new(0), + reach: AtomicU32::new(0), + handed: [const { AtomicU64::new(0) }; MAX_HANDED], records: [const { Record::new() }; COUNT], } } + /// Have every thread this controller founds hold `share`. + /// + /// Call before [`start`](Self::start), from any thread: the controller + /// thread is founded holding what was named by then, and can hand on + /// nothing it was not founded with. `false` when the controller has + /// started, the share's kind has no place in an endowment, another + /// share already names the coordinate, or [`MAX_HANDED`] are taken. + /// Naming the same share again is `true`. + pub fn hand_on(&self, share: Handed) -> bool { + let Some(word) = share.pack() else { + return false; + }; + if self.started.load(Ordering::Acquire) != IDLE { + return false; + } + for entry in &self.handed { + match entry.compare_exchange(0, word, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => return true, + Err(taken) if taken == word => return true, + Err(taken) if Handed::coordinate(taken) == Handed::coordinate(word) => { + return false; + } + Err(_) => {} + } + } + false + } + /// Start the owner. Concurrent/repeated calls do not create another owner. + /// + /// The calling thread becomes the controller's starter: it mints the + /// controller's wake channel from its channel-space grant and holds that + /// channel's root, as a `DoorbellWorker`'s parent holds its worker's, so + /// start from a thread that outlives the controller's use — the initial + /// thread, or a program's main. pub fn start(&'static self) -> Result<(), Error> { self.started - .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) + .compare_exchange(IDLE, STARTING, Ordering::AcqRel, Ordering::Acquire) .map_err(|_| Error::Busy)?; - let heap = heap_endowment(); - match super::thread::spawn_with_arg( - controller_entry, - self as *const Self as usize, - heap.as_slice(), - ) { - Some(ticket) => { - // This owner deliberately lives until process teardown. Managed - // children never use native detach: their tickets are retained. - ticket.detach(); - self.started.store(2, Ordering::Release); - Ok(()) - } - None => { - self.started.store(0, Ordering::Release); - Err(Error::Unavailable) + let mode = self.found(); + self.started.store(mode.unwrap_or(IDLE), Ordering::Release); + futex::wake_all(&self.started); + mode.map(|_| ()).ok_or(Error::Unavailable) + } + + /// How the started controller waits, waiting out its founding; `None` + /// when it is not started. + fn mode(&self) -> Option { + loop { + match self.started.load(Ordering::Acquire) { + IDLE => return None, + STARTING => { + // The starter wakes every thread it can when it is done; + // a thread it cannot reach rides its slice. + park(&self.started, STARTING, Some(UNREACHED_SLICE)); + } + mode => return Some(mode), } } } @@ -143,13 +368,19 @@ impl Controller { /// runs once on the controller after native reclamation, including when the /// callback exits without returning. On error custody remains with caller. /// The cleanup must not make blocking calls back into this controller. + /// + /// Call it from a thread that can wake the controller — its starter, or a + /// thread it created. A parking controller sleeps until something wakes + /// it, so any other thread is refused [`Error::Unavailable`] rather than + /// left waiting on a request nothing announced. pub fn create_with_cleanup( &'static self, call: fn(usize) -> usize, argument: usize, retire: Option, ) -> Result { - if self.started.load(Ordering::Acquire) != 2 { + let mode = self.mode().ok_or(Error::Unavailable)?; + if mode == PARKING && !self.wakes_controller() { return Err(Error::Unavailable); } for (slot, record) in self.records.iter().enumerate() { @@ -170,31 +401,54 @@ impl Controller { call, argument, retire, + home: self as *const Self as usize, }; } record.identity.store(0, Ordering::Relaxed); record.value.store(0, Ordering::Relaxed); record.returned.store(0, Ordering::Relaxed); + record.ending.store(0, Ordering::Relaxed); record.state.store(old | REQUEST, Ordering::Release); - loop { - let state = record.state.load(Ordering::Acquire); - if state == old | FAILED { - record.release(old); - return Err(Error::Unavailable); - } - if state != old | REQUEST { - return Ok(Id { - controller: self as *const Self as usize, - slot, - generation: old, - }); - } - pause(); + self.hub.signal(); + let state = self.settle(record, |state| state != old | REQUEST); + if state == old | FAILED { + record.release(old); + return Err(Error::Unavailable); } + return Ok(Id { + controller: self as *const Self as usize, + slot, + generation: old, + }); } Err(Error::Full) } + /// Whether this thread holds a send into the controller's doorbell. + fn wakes_controller(&self) -> bool { + self.band + .load(Ordering::Acquire) + .checked_sub(1) + .is_some_and(futex::reaches) + } + + /// Park until `record`'s state satisfies `done`, and return that state. + fn settle(&self, record: &Record, done: impl Fn(u64) -> bool) -> u64 { + loop { + // The event count before the state: a transition published after + // this load moves the count, which the park re-reads once it is + // registered (`futex`'s order), so none is missed. + let seen = record.event.load(Ordering::SeqCst); + let state = record.state.load(Ordering::Acquire); + if done(state) { + return state; + } + let reached = futex::current_thread() + .is_some_and(|band| self.reach.load(Ordering::SeqCst) & (1 << band) != 0); + park(&record.event, seen, (!reached).then_some(UNREACHED_SLICE)); + } + } + fn record(&self, id: Id) -> Result<&Record, Error> { if id.controller != self as *const Self as usize { return Err(Error::Invalid); @@ -263,9 +517,7 @@ impl Controller { continue; } if next == JOIN { - while record.state.load(Ordering::Acquire) != id.generation | JOIN_READY { - pause(); - } + self.settle(record, |state| state == id.generation | JOIN_READY); } // Acquire completion before reading the value it publishes. The owner // is a different thread from the callback on weakly ordered ISAs. @@ -314,96 +566,533 @@ impl Controller { } } +/// One wait on `word`, still holding `expected`: the futex park, or — on a +/// thread that cannot park at all — the deadline park every thread can take, +/// which a spin hint stands in for only off metal. +fn park(word: &AtomicU32, expected: u32, slice: Option) { + if futex::wait(word, expected, slice) == WaitOutcome::Unsupported { + pause(); + } +} + fn pause() { if !crate::pace::cede() { core::hint::spin_loop(); } } +/// The managed thread's entry: announce its end however it comes, relay +/// through the controller, run the callback. fn child_entry(argument: usize) { // SAFETY: controller storage is static; record cannot be reused until join. let record = unsafe { &*(argument as *const Record) }; #[cfg(target_os = "none")] - record.identity.store( - crate::entry::geometry::thread_owner_identity().unwrap_or(0), - Ordering::Release, - ); + { + crate::thread::on_exit(child_ending, argument); + record.identity.store( + crate::entry::geometry::thread_owner_identity().unwrap_or(0), + Ordering::Release, + ); + } // Acquire the request publication (or the owner's subsequent state) // before reading callback storage. It remains immutable through native join. let _ = record.state.load(Ordering::Acquire); let start = unsafe { *record.start.get() }; + // SAFETY: `home` names the static controller whose record this is. + let _ = futex::adopt_hub(&unsafe { &*(start.home as *const Controller) }.hub); record .value .store((start.call)(start.argument), Ordering::Release); record.returned.store(1, Ordering::Release); } -fn controller_entry(argument: usize) { - // SAFETY: start requires static storage. - let controller = unsafe { &*(argument as *const Controller) }; - let heap = heap_endowment(); - let mut tickets: [Option; COUNT] = [const { None }; COUNT]; - loop { - for (record, ticket) in controller.records.iter().zip(tickets.iter_mut()) { - let state = record.state.load(Ordering::Acquire); - let generation = state & !MASK; - if state & MASK == REQUEST { - *ticket = super::thread::spawn_with_arg( - child_entry, - record as *const Record as usize, - heap.as_slice(), - ); - record.state.store( - generation | if ticket.is_some() { RUNNING } else { FAILED }, - Ordering::Release, - ); - } - let Some(child) = ticket.as_ref() else { +/// The child's exit hook: say the thread is ending, then wake the +/// controller to reap it. +#[cfg(target_os = "none")] +fn child_ending(argument: usize) { + // SAFETY: as in `child_entry`; the record outlives its child. + let record = unsafe { &*(argument as *const Record) }; + record.ending.store(1, Ordering::SeqCst); + // SAFETY: `start` is immutable until the controller has reaped this + // thread, which it cannot do before this hook returns. + let start = unsafe { *record.start.get() }; + let controller = unsafe { &*(start.home as *const Controller) }; + controller.hub.signal(); +} + +#[cfg(target_os = "none")] +mod owner { + //! The controller thread and its founding. + + use core::sync::atomic::Ordering; + + use kore_abi::cap::{ResourceKind, Rights}; + use kore_abi::session::SlotRef; + use kore_abi::task::role; + + use super::{ + COMPLETE, COUNT, Controller, DETACHED, JOIN, JOIN_READY, MASK, PARKING, POLLING, + REAP_RETRY, RUNNING, Record, child_entry, + }; + use crate::cap::{ + CHANNEL_ENDPOINT_RIGHTS, ChanSendToken, ChannelRoot, ChannelSpaceToken, verbs, + }; + use crate::futex::{self, PEER_BASE, WaitOutcome}; + use crate::thread::{self, JoinScratch, Thread, WAKE_SEND_RIGHTS, WorkerCap}; + + /// The most capabilities either founding here endows: the controller's + /// doorbell, its own kick, the heap, the host's wake, a channel-space + /// grant and the starter's kick, and what the program has it hand on. + const MAX_ENDOW: usize = 6 + super::MAX_HANDED; + const _: () = assert!(MAX_ENDOW <= thread::MAX_WORKER_CAPS); + + /// The shares `controller` hands on, as this thread can make them: + /// derives of its own holdings, with `DERIVE` too when `derivable`. + fn handed(controller: &Controller, derivable: bool, endowment: &mut Endowment) { + for entry in &controller.handed { + let Some((role, index, kind, rights)) = + super::Handed::unpack(entry.load(Ordering::Acquire)) + else { continue; }; - if !matches!( - child.poll(), - Ok(crate::cap::TaskPollOutcome::Exited(_) | crate::cap::TaskPollOutcome::Reaped) - ) { - continue; + let rights = if derivable { + rights | Rights::DERIVE + } else { + rights + }; + endowment.push(thread::share_endowment_of_kind(role, index, kind, rights)); + } + } + + /// A founding's endowment, filled in order. + struct Endowment { + caps: [WorkerCap; MAX_ENDOW], + len: usize, + } + + impl Endowment { + fn new() -> Self { + let vacant = WorkerCap { + role: 0, + index: 0, + subject: SlotRef::from_wire(0), + kind: 0, + rights: Rights::empty(), + }; + Self { + caps: [vacant; MAX_ENDOW], + len: 0, } - match super::thread::join(ticket.take().unwrap()) { - Err(error) => *ticket = Some(error.into_thread()), - Ok(_) => { - // Child can no longer access its context; cleanup also runs - // for abrupt native exit and detached completion. - let start = unsafe { *record.start.get() }; - if let Some(retire) = start.retire { - retire(start.argument); + } + + /// Append `cap`, if any; `true` when it is now in the endowment. + fn push(&mut self, cap: Option) -> bool { + let (Some(cap), Some(at)) = (cap, self.caps.get_mut(self.len)) else { + return false; + }; + *at = cap; + self.len += 1; + true + } + + fn as_slice(&self) -> &[WorkerCap] { + &self.caps[..self.len] + } + } + + /// A send at `DOORBELL@index`, derived from `subject` with `rights`. + fn send(index: u16, subject: SlotRef, rights: Rights) -> WorkerCap { + WorkerCap { + role: role::DOORBELL, + index, + subject, + kind: ResourceKind::ChanSend as u16 as u8, + rights, + } + } + + /// A thread's own doorbell (`DOORBELL@0`), derived from its channel's root. + fn doorbell(root: &ChannelRoot) -> WorkerCap { + WorkerCap { + role: role::DOORBELL, + index: 0, + subject: root.slot(), + kind: ResourceKind::ChanRecv as u16 as u8, + rights: CHANNEL_ENDPOINT_RIGHTS, + } + } + + /// One thread's wake channel: its root and the send its founder keeps. + /// The founder's own recv is surplus — the thread's recv is a fresh + /// derive of the root at its founding (the star's rule). + struct Wires { + root: ChannelRoot, + kick: ChanSendToken, + } + + impl Wires { + fn mint(space: &ChannelSpaceToken) -> Option { + let pair = verbs::mint_channel(space).ok()?; + let _ = pair.recv.try_retire(); + Some(Self { + root: pair.root, + kick: pair.send, + }) + } + + fn retire(self) { + let _ = self.kick.try_retire(); + let _ = self.root.try_retire(); + } + } + + impl Controller { + /// Found the controller thread from the calling thread: the mode it + /// will wait in, or `None` when it could not be founded. + pub(super) fn found(&'static self) -> Option { + let starter = futex::current_thread()? as u32; + self.starter.store(starter + 1, Ordering::Release); + let rt = crate::entry::thread_context()?; + let space = rt + .bootstrap() + .handle(role::CHANNEL_SPACE, 0) + .and_then(verbs::claim_channel_space); + let wires = space.as_ref().and_then(Wires::mint); + let own_kick = thread::own_kick(); + + let mut endowment = Endowment::new(); + if let Some(wires) = &wires { + endowment.push(Some(doorbell(&wires.root))); + // The controller derives every child's send into itself from + // this one, so it keeps DERIVE. + endowment.push(Some(send( + futex::OWN_KICK, + wires.root.slot(), + WAKE_SEND_RIGHTS, + ))); + } + endowment.push(thread::heap_endowment()); + // Peer 0. For a starter that is thread 0 it is that thread's own + // kick, and the starter is reached through it. + let peer_zero = endowment.push(thread::wake_endowment()); + endowment.push(thread::channel_space_endowment()); + // The controller hands these on to every thread it founds. + handed(self, true, &mut endowment); + // Any other starter is reached at its own band's peer index. + let reached = if starter == 0 { + own_kick.is_some() && peer_zero + } else { + endowment.push( + own_kick.map(|kick| send(PEER_BASE + starter as u16, kick, WAKE_SEND_RIGHTS)), + ) + }; + let Some(ticket) = thread::spawn_founder_with_arg( + super::controller_entry, + self as *const Self as usize, + endowment.as_slice(), + ) else { + if let Some(wires) = wires { + wires.retire(); + } + return None; + }; + let band = ticket.band_index(); + self.band.store(band + 1, Ordering::Release); + if reached { + self.reach.fetch_or(1 << starter, Ordering::SeqCst); + } + // This owner deliberately lives until process teardown. Managed + // children never use native detach: their tickets are retained. + ticket.detach(); + // The claimed grant is a witness, not a reservation: dropping it + // retires nothing (the star's rule, `star::Wires`). + drop(space); + let Some(wires) = wires else { + return Some(POLLING); + }; + // Wake the controller directly, and relay through it. + if !futex::adopt_peer(band, wires.kick.slot()) { + wires.retire(); + return Some(POLLING); + } + let _ = futex::adopt_hub(&self.hub); + // The channel lives as long as the controller does: its root and + // this thread's send stay in this thread's arena for good. + core::mem::forget(wires); + Some(PARKING) + } + } + + /// A managed child the controller founded and will reap. + struct Child { + thread: Thread, + band: u32, + /// Its doorbell's root and the controller's send into it; `None` + /// when the controller has no channel space to mint them. + wires: Option, + } + + /// The controller thread's own state, on its stack for the process life. + pub(super) struct Owner { + controller: &'static Controller, + band: u32, + /// The grant children's doorbells are minted from. + space: Option, + /// This thread's own kick: every child's send into it derives from it. + kick: Option, + /// The starter's band and the send into it, when the starter is not + /// thread 0 (which children reach as peer 0). + starter: Option<(u32, SlotRef)>, + /// One report landing, reused serially by every probe, join arm and + /// reap: the controller never spends an arena slot to observe. + scratch: Option, + children: [Option; COUNT], + /// The hub's relay count at the last sweep. + swept: u32, + } + + /// What one reap attempt came to. + enum Reap { + /// Nothing left to reap. + Done, + /// The child announced its end and has not committed it. + Pending, + /// The kernel refused an observation or the reclamation. + Failed, + } + + impl Owner { + pub(super) fn new(controller: &'static Controller) -> Self { + let band = futex::current_thread().map_or(u32::MAX, |band| band as u32); + let rt = crate::entry::thread_context(); + let boot = rt.as_ref().map(|rt| rt.bootstrap()); + let space = boot + .as_ref() + .and_then(|boot| boot.handle(role::CHANNEL_SPACE, 0)) + .and_then(verbs::claim_channel_space); + // Derivable, or no child is handed a send into this thread. + let kick = thread::own_kick(); + let starter = controller + .starter + .load(Ordering::Acquire) + .checked_sub(1) + .filter(|&starter| starter != 0) + .and_then(|starter| { + let send = boot + .as_ref()? + .handle(role::DOORBELL, PEER_BASE + starter as u16)?; + Some((starter, send)) + }); + Self { + controller, + band, + space, + kick, + starter, + scratch: JoinScratch::reserve(), + children: [const { None }; COUNT], + swept: controller.hub.relays(), + } + } + + /// Serve the records for the process's life. + pub(super) fn serve(mut self) -> ! { + let controller = self.controller; + loop { + let seen = controller.hub.word().load(Ordering::SeqCst); + let owed = controller.hub.relays(); + if owed != self.swept { + self.swept = owed; + futex::sweep(); + } + let mut watch = None; + let mut failed = false; + for (slot, record) in controller.records.iter().enumerate() { + let state = record.state.load(Ordering::Acquire); + if state & MASK == super::REQUEST { + self.found_child(slot, record, state & !MASK); } - loop { - let old = record.state.load(Ordering::Acquire); - let next = match old & MASK { - RUNNING => COMPLETE, - JOIN => JOIN_READY, - DETACHED => { - record.release(generation); - break; + if record.ending.load(Ordering::Acquire) != 0 { + match self.reap(slot, record) { + Reap::Done => {} + Reap::Pending => { + watch.get_or_insert(slot); } - _ => unreachable!(), - }; - if record - .state - .compare_exchange( - old, - generation | next, - Ordering::Release, - Ordering::Acquire, - ) - .is_ok() - { - break; + Reap::Failed => failed = true, } } } + let word = controller.hub.word(); + let outcome = match (watch, &self.scratch) { + (Some(slot), Some(scratch)) if !failed => match &self.children[slot] { + Some(child) => futex::wait_or_end( + word, + seen, + None, + child.thread.handle(), + scratch.report_slot(), + ), + None => futex::wait_quiet(word, seen, None), + }, + _ => futex::wait_quiet(word, seen, failed.then_some(REAP_RETRY)), + }; + if outcome == WaitOutcome::Unsupported { + super::pause(); + } } } - pause(); + + /// Found the child `record` asks for, then publish the outcome. + fn found_child(&mut self, slot: usize, record: &'static Record, generation: u64) { + let child = self.spawn(record); + let next = if child.is_some() { + RUNNING + } else { + super::FAILED + }; + self.children[slot] = child; + record.state.store(generation | next, Ordering::Release); + record.announce(); + } + + fn spawn(&mut self, record: &'static Record) -> Option { + let wires = self.space.as_ref().and_then(Wires::mint); + let mut endowment = Endowment::new(); + if let Some(wires) = &wires { + endowment.push(Some(doorbell(&wires.root))); + } + // A child hands these on to no one (the threads it asks for are + // founded here, not by it): notice rights only. + endowment.push( + self.kick + .map(|kick| send(PEER_BASE + self.band as u16, kick, verbs::NOTICE_RIGHTS)), + ); + endowment.push(self.starter.map(|(starter, kick)| { + send(PEER_BASE + starter as u16, kick, verbs::NOTICE_RIGHTS) + })); + endowment.push(thread::heap_endowment()); + endowment.push(thread::wake_endowment()); + handed(self.controller, false, &mut endowment); + let Some(thread) = thread::spawn_founder_with_arg( + child_entry, + record as *const Record as usize, + endowment.as_slice(), + ) else { + if let Some(wires) = wires { + wires.retire(); + } + return None; + }; + let band = thread.band_index(); + if let Some(wires) = &wires + && futex::adopt_peer(band, wires.kick.slot()) + { + self.controller.reach.fetch_or(1 << band, Ordering::SeqCst); + } + Some(Child { + thread, + band, + wires, + }) + } + + /// Reap `slot`'s child if it has ended, then publish its end. + fn reap(&mut self, slot: usize, record: &Record) -> Reap { + if self.scratch.is_none() { + // The landing was not available when this thread started. + self.scratch = JoinScratch::reserve(); + } + let (Some(child), Some(scratch)) = + (self.children[slot].as_ref(), self.scratch.as_mut()) + else { + return if self.children[slot].is_none() { + Reap::Done + } else { + Reap::Failed + }; + }; + match thread::poll_reserved(&child.thread, scratch) { + Ok(crate::cap::TaskPollOutcome::Running) => return Reap::Pending, + Ok(_) => {} + Err(_) => return Reap::Failed, + } + // Withdraw the child's kick before its band can be handed to + // another thread: no wake may present this send, and no client + // at that band may count on this controller reaching it. + futex::forget_peer(child.band); + self.controller + .reach + .fetch_and(!(1 << child.band), Ordering::SeqCst); + let Some(Child { + thread, + band, + wires, + }) = self.children[slot].take() + else { + return Reap::Done; + }; + if let Err(error) = thread::join_reserved(thread, scratch) { + self.children[slot] = Some(Child { + thread: error.into_thread(), + band, + wires, + }); + return Reap::Failed; + } + if let Some(wires) = wires { + wires.retire(); + } + // Child can no longer access its context; cleanup also runs + // for abrupt native exit and detached completion. + let start = unsafe { *record.start.get() }; + if let Some(retire) = start.retire { + retire(start.argument); + } + record.ending.store(0, Ordering::Relaxed); + loop { + let old = record.state.load(Ordering::Acquire); + let generation = old & !MASK; + let next = match old & MASK { + RUNNING => COMPLETE, + JOIN => JOIN_READY, + DETACHED => { + record.release(generation); + break; + } + // Only this thread moves a founded child's record out of + // these three. + _ => break, + }; + if record + .state + .compare_exchange(old, generation | next, Ordering::Release, Ordering::Acquire) + .is_ok() + { + break; + } + } + record.announce(); + Reap::Done + } + } +} + +fn controller_entry(argument: usize) { + // SAFETY: start requires static storage. + let controller = unsafe { &*(argument as *const Controller) }; + #[cfg(target_os = "none")] + owner::Owner::new(controller).serve(); + #[cfg(not(target_os = "none"))] + let _ = controller; +} + +#[cfg(not(target_os = "none"))] +impl Controller { + /// Off metal no thread founds another: the controller never starts. + fn found(&'static self) -> Option { + let _ = child_entry; + let _ = controller_entry; + None } } @@ -411,6 +1100,80 @@ fn controller_entry(argument: usize) { mod tests { extern crate std; use super::*; + use kore_abi::cap::{ResourceKind, Rights}; + + fn share(role: u16, index: u16, rights: Rights) -> Handed { + Handed { + role, + index, + kind: ResourceKind::MemLease, + rights, + } + } + + #[test] + fn a_handed_share_survives_its_packing() { + let wanted = Handed { + role: 0xbeef, + index: 0x0108, + kind: ResourceKind::ChanSend, + rights: Rights::ACCESS | Rights::DERIVE, + }; + let word = wanted.pack().expect("a channel send packs"); + assert_ne!(word, 0, "a taken entry is never the vacant word"); + assert_eq!( + Handed::unpack(word), + Some(( + wanted.role, + wanted.index, + ResourceKind::ChanSend as u16, + wanted.rights + )) + ); + assert_eq!(Handed::unpack(0), None, "a vacant entry hands nothing on"); + // Role and index zero with no rights is still a taken entry. + let bare = share(0, 0, Rights::empty()).pack().expect("packs"); + assert!(Handed::unpack(bare).is_some()); + } + + #[test] + fn a_controller_hands_on_each_coordinate_once_and_only_so_many() { + let controller = Controller::new(); + let first = share(7, 8, Rights::ACCESS); + assert!(controller.hand_on(first)); + assert!(controller.hand_on(first), "naming the same share again"); + assert!( + !controller.hand_on(share(7, 8, Rights::ACCESS | Rights::MODIFY)), + "one coordinate, two shares: a boot block names a coordinate once" + ); + assert!(controller.hand_on(share(7, 9, Rights::ACCESS))); + assert!( + !controller.hand_on(share(9, 9, Rights::ACCESS)), + "the table is full" + ); + let taken = controller + .handed + .iter() + .filter(|entry| entry.load(Ordering::Acquire) != 0) + .count(); + assert_eq!(taken, MAX_HANDED); + } + + #[test] + fn a_started_controller_takes_no_more_shares() { + let controller = Controller::new(); + controller.started.store(PARKING, Ordering::Release); + assert!( + !controller.hand_on(share(7, 8, Rights::ACCESS)), + "its thread was founded without the share and cannot hand it on" + ); + assert!( + controller + .handed + .iter() + .all(|entry| entry.load(Ordering::Acquire) == 0) + ); + } fn completed(controller: &Controller, generation: u64) -> Id { let record = &controller.records[0]; record.value.store(73, Ordering::Relaxed); @@ -456,4 +1219,48 @@ mod tests { ); assert_eq!(controller.join(final_id), Err(Error::Invalid)); } + #[test] + fn a_parked_joiner_is_released_by_the_announced_transition() { + // The joiner's half of the protocol, with a host thread standing in + // for the controller: it moves the record JOIN → JOIN_READY and + // announces it; the joiner returns the value only after that. + let controller = Controller::new(); + let record = &controller.records[0]; + record.state.store(RUNNING, Ordering::Release); + let id = Id { + controller: &controller as *const Controller as usize, + slot: 0, + generation: 0, + }; + std::thread::scope(|scope| { + let joiner = scope.spawn(|| controller.join(id)); + while record.state.load(Ordering::Acquire) != JOIN { + std::thread::yield_now(); + } + record.value.store(5, Ordering::Relaxed); + record.returned.store(1, Ordering::Release); + let before = record.event.load(Ordering::SeqCst); + record.state.store(JOIN_READY, Ordering::Release); + record.announce(); + assert_eq!(record.event.load(Ordering::SeqCst), before.wrapping_add(1)); + assert_eq!(joiner.join().unwrap(), Ok(5)); + }); + assert_eq!(record.state.load(Ordering::Acquire), 16); + } + #[test] + fn an_unstarted_controller_refuses_and_never_waits() { + let controller: &'static Controller = + std::boxed::Box::leak(std::boxed::Box::new(Controller::new())); + assert_eq!(controller.create(|_| 0, 0), Err(Error::Unavailable)); + // Off metal the founding cannot happen; the refusal leaves it idle + // and restartable rather than stuck in its founding state. + assert_eq!(controller.start(), Err(Error::Unavailable)); + assert_eq!(controller.started.load(Ordering::Acquire), IDLE); + assert_eq!(controller.create(|_| 0, 0), Err(Error::Unavailable)); + } + #[test] + fn records_own_their_interference_span() { + assert_eq!(core::mem::align_of::(), 128); + assert_eq!(core::mem::size_of::(), 128); + } } diff --git a/native/crates/vdev-accel/Cargo.toml b/native/crates/vdev-accel/Cargo.toml new file mode 100644 index 0000000..f30cdeb --- /dev/null +++ b/native/crates/vdev-accel/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "kore-vdev-accel" +version = "0.0.1" +edition = "2024" +description = "kore's virtual accelerator: a virtio-accel 1.0 device with no hardware behind it, computing six functions of FP8 (E4M3) values on the CPU backend." +publish = false + +[dependencies] +# The CPU backend this device is an instance of: its registered-kernel +# program format, its admission rules and its SPMD launch. The device adds a +# kernel registry and nothing else. +kore-svc-accel-spmd = { path = "../../services/accel-spmd" } +# The device model's contract, and the transport that carries a driver to +# an implementation of it. +kore-vdev = { path = "../vdev" } +# The frozen protocol (MicroPerceptron/virtio-accel 1.0), at the pin every +# accelerator row uses. `-device` is the command engine a device runs: it +# decodes one request frame, drives the backend and writes the response. +# `-proto` is the frame layouts and the configuration a driver reads. +virtio-accel-core = { version = "=0.4.1", default-features = false } +virtio-accel-device = { version = "=0.4.1", default-features = false } +virtio-accel-proto = { version = "=0.4.1", default-features = false } +# The configuration a driver reads is the protocol's `WireConfig`, as bytes. +zerocopy = { version = "0.8", default-features = false, features = ["derive"] } + +[dev-dependencies] +kore-driver = { path = "../driver", features = ["accel"] } +kore-rt = { path = "../rt", features = ["std"] } +# The protocol's reference driver: the session test is that driver, over +# kore's registers and queue, against this device. +virtio-accel-guest = { version = "=0.4.1", default-features = false } +virtio-accel-transport = { version = "=0.4.1", default-features = false } diff --git a/native/crates/vdev-accel/src/device.rs b/native/crates/vdev-accel/src/device.rs new file mode 100644 index 0000000..ba3ca3a --- /dev/null +++ b/native/crates/vdev-accel/src/device.rs @@ -0,0 +1,226 @@ +//! **The device**: the protocol's command engine over the CPU backend. +//! +//! A virtio-accel device has one job per driver request: take a request +//! frame, act on it, and write a response frame. The protocol crate ships the +//! engine that does this for any backend (`virtio_accel_device`'s +//! `CommandProcessor`): it decodes and validates the frame, keeps the object +//! graph (contexts, buffers, programs, queues, events) and refuses a request +//! before the backend sees it. This module is that engine instantiated over +//! [`SpmdBackend`] with the FP8 registry, which is the whole device. +//! +//! Three things a transport needs, and nothing about any transport: +//! +//! - [`AccelDevice::CONFIG`], the configuration a driver reads before it +//! sends a frame: the protocol version and the largest frames and chains +//! this device takes; +//! - [`AccelDevice::command`], one request in and one response out; +//! - [`AccelDevice::drive`], the device's own turn. A submission is admitted +//! and answered at once, as the protocol requires, and its work runs here. +//! On hardware this is the silicon's; for this device it is a call the +//! host makes between commands, so a driver that submits and then polls +//! sees the event complete. +//! +//! Those are the device model's contract in `kore-vdev` +//! ([`VirtualDevice`]), which is how a driver reaches this device: through +//! that crate's register block and queue, with nothing here knowing of +//! either. +//! +//! Two of the protocol's rules that a driver writer meets first: a buffer +//! bound to a submission is the event's until the driver destroys the event, +//! so results are read after `DestroyEvent`, not after the poll that saw it +//! complete; and a reset gives every object a new namespace, so an id a +//! driver kept from before the reset names nothing afterwards. + +use kore_svc_accel_spmd::SpmdBackend; +use kore_vdev::device::{ReadableBytes, Unrecoverable, VirtualDevice, WritableBytes}; +use virtio_accel_core::{ByteSink, ByteSource, TransportByteSink, TransportByteSource}; +use virtio_accel_device::{ + ChainRegion, CommandOutcome, CommandProcessError, CommandProcessor, CommandProcessorInitError, + DeviceHealth, ObjectNamespace, ResetDisposition, ResourcePolicy, +}; +use virtio_accel_proto::{ + BASELINE_COMMAND_QUEUES, Le16, Le32, PROTOCOL_MAJOR, PROTOCOL_MINOR, WireConfig, +}; +use zerocopy::IntoBytes; + +use crate::fp8; + +/// Descriptors one command chain may have. A request is at most a header, a +/// fixed payload and one run of data, and a response a header and one run: +/// sixteen leaves a driver room to split either without making a chain a +/// place to hide unbounded work. +pub const MAX_CHAIN_DESCRIPTORS: u16 = 16; + +/// The largest request or response frame, header included. +/// +/// A buffer transfer rides inside a frame, so this bounds one transfer, not +/// a buffer: a driver moves a larger buffer in pieces. 64 KiB is sixteen +/// pages of a driver's shared memory per frame in flight. +pub const MAX_FRAME_BYTES: u32 = 64 * 1024; + +/// Buffer backing the device will hold for a driver at once: 2 MiB, two +/// million E4M3 values. The backing is the host's heap, so this is what one +/// driver can make the device's host allocate, and a host that serves +/// several drivers from one heap sizes that heap by it. +pub const MAX_BUFFER_BACKING_BYTES: u64 = 2 << 20; + +/// Resident program bytes the device will hold. A program here is a +/// registered kernel's handle, a few bytes; 64 KiB admits every program the +/// backend's own count limit allows many times over. +pub const MAX_PROGRAM_RESIDENT_BYTES: u64 = 64 * 1024; + +/// What a driver reads from the register block's device id: `"accl"`. The +/// protocol assigns itself no Virtio device id, so the name is kore's. +pub const DEVICE_ID: u32 = u32::from_le_bytes(*b"accl"); + +/// The virtual accelerator. +pub struct AccelDevice { + processor: CommandProcessor, + /// The namespace the device's objects live in. Each reset takes the + /// next, which is what makes an id from before a reset name nothing + /// after it. + namespace: ObjectNamespace, +} + +impl AccelDevice { + /// What a driver reads before its first frame. + pub const CONFIG: WireConfig = WireConfig { + protocol_major: Le16::new(PROTOCOL_MAJOR), + protocol_minor: Le16::new(PROTOCOL_MINOR), + command_queue_count: Le16::new(BASELINE_COMMAND_QUEUES), + max_chain_descriptors: Le16::new(MAX_CHAIN_DESCRIPTORS), + max_request_bytes: Le32::new(MAX_FRAME_BYTES), + max_response_bytes: Le32::new(MAX_FRAME_BYTES), + }; + + /// A device in its first state. + pub fn new() -> Result { + const FIRST: ObjectNamespace = ObjectNamespace::new(1).unwrap(); + Self::in_namespace(FIRST) + } + + fn in_namespace(namespace: ObjectNamespace) -> Result { + const POLICY: ResourcePolicy = + ResourcePolicy::new(MAX_BUFFER_BACKING_BYTES, MAX_PROGRAM_RESIDENT_BYTES).unwrap(); + Ok(Self { + processor: CommandProcessor::new( + SpmdBackend::new(fp8::KERNELS), + &Self::CONFIG, + namespace, + POLICY, + )?, + namespace, + }) + } + + /// One command chain: `request` is its device-readable bytes, `response` + /// its device-writable bytes and `regions` their layout in the chain. + /// + /// The outcome says how many response bytes were written, or that the + /// chain cannot carry a response at all. An `Err` is the transport's own + /// memory failing under the engine, not a protocol error: those are + /// responses. + pub fn command( + &mut self, + regions: &[ChainRegion], + request: &dyn ByteSource, + response: &mut dyn ByteSink, + ) -> Result { + self.processor.process(regions, request, response) + } + + /// Run the work submissions admitted, oldest first. Answers whether any + /// ran, which is when an event a driver holds may have become terminal. + pub fn drive(&mut self) -> bool { + let mut ran = false; + // One admitted launch is in flight at a time, so this loop runs it + // and ends: the backend refuses a second admission until then. + while self.processor.accelerator().run_pending() { + ran = true; + } + ran + } + + /// Whether the device still serves, or needs the reset a transport + /// offers its driver. + pub fn health(&self) -> DeviceHealth { + self.processor.health() + } +} + +impl core::fmt::Debug for AccelDevice { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + // The engine's object graph is its own business, and a driver's. + f.debug_struct("AccelDevice") + .field("namespace", &self.namespace.get()) + .field("health", &self.health()) + .finish_non_exhaustive() + } +} + +impl VirtualDevice for AccelDevice { + const DEVICE_ID: u32 = DEVICE_ID; + /// The largest ring there is: four of the longest chains, or thirty-two + /// commands of two descriptors, in flight at once. + const MAX_QUEUE_SIZE: u16 = kore_vdev::ring::MAX_QUEUE_SIZE; + const MAX_CHAIN_DESCRIPTORS: u16 = MAX_CHAIN_DESCRIPTORS; + + fn config(&self) -> &[u8] { + const CONFIG: &WireConfig = &AccelDevice::CONFIG; + CONFIG.as_bytes() + } + + fn command( + &mut self, + regions: &[ChainRegion], + request: &dyn ReadableBytes, + response: &mut dyn WritableBytes, + ) -> u32 { + // The engine's byte ports over the transport's, with no copy + // between: a frame is read from and written to the driver's memory. + let request = TransportByteSource::new(request); + let mut response = TransportByteSink::new(response); + match AccelDevice::command(self, regions, &request, &mut response) { + Ok(CommandOutcome::Response { used, .. }) => used, + // A frame with no usable response, or the chain's memory failing + // under the engine. Either way nothing was written the driver + // may read; if the failure came after the engine had acted, the + // engine has marked itself, and `needs_reset` says so. + Ok(CommandOutcome::Unusable(_)) | Err(_) => 0, + } + } + + fn drive(&mut self) -> bool { + AccelDevice::drive(self) + } + + fn needs_reset(&self) -> bool { + self.health() != DeviceHealth::Running + } + + fn reset(&mut self) -> Result<(), Unrecoverable> { + // The namespace is sixteen bits and never reused: a device reset + // sixty-five thousand times is not brought back, because an id from + // its first life would name an object of its last. + let next = self + .namespace + .get() + .checked_add(1) + .and_then(ObjectNamespace::new) + .ok_or(Unrecoverable)?; + let report = self.processor.reset(next).map_err(|_| Unrecoverable)?; + match report.disposition { + ResetDisposition::BackendReusable => self.namespace = next, + // The backend could not account for something it held. The + // protocol forbids calling it again: it is dropped whole, and a + // new engine over a new backend takes its place. + ResetDisposition::BackendDiscardRequired => { + *self = Self::in_namespace(next).map_err(|_| Unrecoverable)?; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/native/crates/vdev-accel/src/device/tests.rs b/native/crates/vdev-accel/src/device/tests.rs new file mode 100644 index 0000000..0350327 --- /dev/null +++ b/native/crates/vdev-accel/src/device/tests.rs @@ -0,0 +1,372 @@ +use super::*; +use crate::fp8::Op; +use kore_svc_accel_spmd::{ + FORMAT_REGISTERED, IDENTITY_MAGIC, REGISTERED_MANIFEST, REGISTERED_RESIDENT_BYTES, +}; +use std::vec::Vec; +use virtio_accel_device::{SegmentedSink, SegmentedSource}; +use virtio_accel_proto::{ + AllocateBufferRequest, CreateContextRequest, CreateQueueRequest, KnownEventState, KnownOpcode, + Le64, LoadProgramRequest, ObjectPayload, RequestFlags, RequestHeader, ResponseHeader, + StatusCode, SubmitRequest, SubmitResponse, TransferBufferRequest, WireBinding, WireDeviceInfo, + WireEventState, read_exact, +}; +use zerocopy::IntoBytes; + +const HEADER: usize = core::mem::size_of::(); +/// The registered-kernel manifest's length. Spelled through `size_of_val`: +/// in this module an array's `len` is also the byte ports'. +const MANIFEST_BYTES: u64 = core::mem::size_of_val(®ISTERED_MANIFEST) as u64; + +/// The driver's side of one command: a request frame in one readable region +/// and room for the response in one writable region, as a two-descriptor +/// chain carries them. +struct Driver { + device: AccelDevice, + next_request: u64, +} + +impl Driver { + fn new() -> Self { + Self { + device: AccelDevice::new().expect("device"), + next_request: 1, + } + } + + /// Send `opcode` with `payload`; answer the status and response payload. + fn call(&mut self, opcode: KnownOpcode, payload: &[u8], room: usize) -> (StatusCode, Vec) { + let id = self.next_request; + self.next_request += 1; + let header = RequestHeader::new(opcode, RequestFlags::empty(), payload.len() as u32, id); + let mut request = header.as_bytes().to_vec(); + request.extend_from_slice(payload); + let mut response = std::vec![0u8; HEADER + room]; + let regions = [ + ChainRegion::readable(request.len() as u64), + ChainRegion::writable(response.len() as u64), + ]; + let outcome = { + let segments = [request.as_slice()]; + let source = SegmentedSource::new(&segments).unwrap(); + let mut parts = [response.as_mut_slice()]; + let mut sink = SegmentedSink::new(&mut parts).unwrap(); + self.device + .command(®ions, &source, &mut sink) + .expect("the chain's memory holds") + }; + let CommandOutcome::Response { + request_id, + status, + used, + } = outcome + else { + panic!("the chain could not carry a response: {outcome:?}"); + }; + assert_eq!(request_id, id, "a response answers its request"); + let header: ResponseHeader = read_exact(&response[..HEADER]).unwrap(); + assert_eq!(header.request_id.get(), id); + assert_eq!(header.status.get(), status.0); + let payload_bytes = header.payload_bytes.get() as usize; + assert_eq!(used as usize, HEADER + payload_bytes); + (status, response[HEADER..HEADER + payload_bytes].to_vec()) + } + + fn ok(&mut self, opcode: KnownOpcode, payload: &[u8], room: usize) -> Vec { + let (status, payload) = self.call(opcode, payload, room); + assert_eq!(status, StatusCode::OK, "{opcode:?}"); + payload + } + + fn object(&mut self, opcode: KnownOpcode, payload: &[u8]) -> u64 { + let reply = self.ok(opcode, payload, 8); + read_exact::(&reply).unwrap().object_id.get() + } + + fn buffer(&mut self, context: u64, bytes: u64, usage: u32) -> u64 { + self.object( + KnownOpcode::AllocateBuffer, + AllocateBufferRequest { + context_id: Le64::new(context), + bytes: Le64::new(bytes), + alignment: Le64::new(1), + memory_domain: MEMORY_DOMAIN_HOST, + reserved0: [0; 7], + usage: Le32::new(usage), + reserved1: Le32::new(0), + } + .as_bytes(), + ) + } + + fn program(&mut self, context: u64, kernel: u32) -> u64 { + let mut target = [Le32::new(0); 12]; + target[0] = Le32::new(IDENTITY_MAGIC); + target[1] = Le32::new(kernel); + let mut payload = LoadProgramRequest { + context_id: Le64::new(context), + format: Le32::new(FORMAT_REGISTERED), + flags: Le32::new(0), + target, + payload_bytes: Le64::new(MANIFEST_BYTES), + resident_bytes: Le64::new(REGISTERED_RESIDENT_BYTES), + } + .as_bytes() + .to_vec(); + payload.extend_from_slice(®ISTERED_MANIFEST); + self.object(KnownOpcode::LoadProgram, &payload) + } +} + +/// Provider memory a driver reaches by transfer: `MemoryDomain::Host`. +const MEMORY_DOMAIN_HOST: u8 = 1; +const TRANSFER_SOURCE: u32 = 1 << 0; +const TRANSFER_DESTINATION: u32 = 1 << 1; +const PROGRAM_INPUT: u32 = 1 << 2; +const PROGRAM_OUTPUT: u32 = 1 << 3; +const ACCESS_READ: u8 = 1; +const ACCESS_WRITE: u8 = 2; + +fn transfer(buffer: u64, bytes: u64) -> TransferBufferRequest { + TransferBufferRequest { + buffer_id: Le64::new(buffer), + offset: Le64::new(0), + bytes: Le64::new(bytes), + } +} + +fn binding(buffer: u64, slot: u32, access: u8) -> WireBinding { + WireBinding { + buffer_id: Le64::new(buffer), + offset: Le64::new(0), + bytes: Le64::new(256), + slot: Le32::new(slot), + access, + reserved: [0; 3], + } +} + +/// The configuration is one the protocol accepts, and is what the engine +/// was built with. +#[test] +fn the_configuration_is_the_baseline_protocol() { + assert_eq!(AccelDevice::CONFIG.validate(), Ok(())); + assert_eq!(AccelDevice::CONFIG.protocol_major.get(), 1); + assert_eq!(AccelDevice::CONFIG.command_queue_count.get(), 1); +} + +/// A whole driver session in the protocol's own frames: discover the device, +/// make a context, two buffers, a program and a queue, write every E4M3 +/// value, submit, poll, and read the results back. Then again for each +/// function, on the same buffers. +#[test] +fn a_driver_session_computes_each_function_through_protocol_frames() { + let mut driver = Driver::new(); + + let info: WireDeviceInfo = read_exact(&driver.ok( + KnownOpcode::GetDeviceInfo, + &[], + core::mem::size_of::(), + )) + .unwrap(); + assert_eq!(&info.uuid[..13], b"kore-spmd-cpu"); + assert_eq!(info.max_contexts.get(), 1); + assert!(info.max_bindings_per_submission.get() >= 2); + + let context = driver.object( + KnownOpcode::CreateContext, + CreateContextRequest { + flags: Le32::new(0), + reserved: Le32::new(0), + } + .as_bytes(), + ); + let operands = driver.buffer(context, 256, PROGRAM_INPUT | TRANSFER_DESTINATION); + let results = driver.buffer(context, 256, PROGRAM_OUTPUT | TRANSFER_SOURCE); + let queue = driver.object( + KnownOpcode::CreateQueue, + CreateQueueRequest { + context_id: Le64::new(context), + flags: Le32::new(0), + reserved: Le32::new(0), + } + .as_bytes(), + ); + + let every_byte: [u8; 256] = core::array::from_fn(|i| i as u8); + let mut write = transfer(operands, 256).as_bytes().to_vec(); + write.extend_from_slice(&every_byte); + driver.ok(KnownOpcode::WriteBuffer, &write, 0); + + for op in Op::ALL { + let program = driver.program(context, op.kernel_id()); + let mut submit = SubmitRequest { + queue_id: Le64::new(queue), + program_id: Le64::new(program), + binding_count: Le32::new(2), + flags: Le32::new(0), + // Zero: no deadline. + timeout_ns: Le64::new(0), + } + .as_bytes() + .to_vec(); + submit.extend_from_slice(binding(operands, 0, ACCESS_READ).as_bytes()); + submit.extend_from_slice(binding(results, 1, ACCESS_WRITE).as_bytes()); + let event = read_exact::(&driver.ok(KnownOpcode::Submit, &submit, 8)) + .unwrap() + .event_id + .get(); + + // Admitted and answered, not yet run: the work is the device's turn. + let poll = |driver: &mut Driver| { + let reply = driver.ok( + KnownOpcode::PollEvent, + ObjectPayload { + object_id: Le64::new(event), + } + .as_bytes(), + 8, + ); + read_exact::(&reply) + .unwrap() + .known_state() + .unwrap() + }; + assert_eq!(poll(&mut driver), KnownEventState::Pending); + assert!(driver.device.drive(), "the admitted submission runs"); + assert!(!driver.device.drive(), "and nothing is left to run"); + assert_eq!(poll(&mut driver), KnownEventState::Complete); + + // A buffer a submission bound stays the event's until the driver + // destroys the event, terminal or not: the protocol's rule, and the + // order a driver must follow. + let read = transfer(results, 256); + assert_eq!( + driver.call(KnownOpcode::ReadBuffer, read.as_bytes(), 256).0, + StatusCode::BUSY + ); + let object = |id: u64| ObjectPayload { + object_id: Le64::new(id), + }; + driver.ok(KnownOpcode::DestroyEvent, object(event).as_bytes(), 0); + let read = driver.ok(KnownOpcode::ReadBuffer, read.as_bytes(), 256); + assert_eq!(read.as_slice(), op.table(), "{}", op.name()); + driver.ok(KnownOpcode::UnloadProgram, object(program).as_bytes(), 0); + } + assert_eq!(driver.device.health(), DeviceHealth::Running); +} + +/// What a driver gets wrong is answered in a response, and the device goes +/// on serving. +#[test] +fn a_drivers_mistakes_are_answered_and_the_device_keeps_serving() { + let mut driver = Driver::new(); + let context = driver.object( + KnownOpcode::CreateContext, + CreateContextRequest { + flags: Le32::new(0), + reserved: Le32::new(0), + } + .as_bytes(), + ); + + // A program this device does not have: another registry's kernel. + let mut target = [Le32::new(0); 12]; + target[0] = Le32::new(IDENTITY_MAGIC); + target[1] = Le32::new(1); + let mut load = LoadProgramRequest { + context_id: Le64::new(context), + format: Le32::new(FORMAT_REGISTERED), + flags: Le32::new(0), + target, + payload_bytes: Le64::new(MANIFEST_BYTES), + resident_bytes: Le64::new(REGISTERED_RESIDENT_BYTES), + } + .as_bytes() + .to_vec(); + load.extend_from_slice(®ISTERED_MANIFEST); + assert_eq!( + driver.call(KnownOpcode::LoadProgram, &load, 8).0, + StatusCode::INCOMPATIBLE + ); + + // An id that names nothing. + let stranger = ObjectPayload { + object_id: Le64::new(0xdead_beef), + }; + let (status, _) = driver.call(KnownOpcode::PollEvent, stranger.as_bytes(), 8); + assert_ne!(status, StatusCode::OK); + + // A second context, where the device has one. + assert_eq!( + driver + .call( + KnownOpcode::CreateContext, + CreateContextRequest { + flags: Le32::new(0), + reserved: Le32::new(0), + } + .as_bytes(), + 8 + ) + .0, + StatusCode::RESOURCE_LIMIT + ); + + // A frame past the configured limit is refused before it is read. + let oversized = std::vec![0u8; MAX_FRAME_BYTES as usize]; + let (status, _) = driver.call(KnownOpcode::WriteBuffer, &oversized, 0); + assert_ne!(status, StatusCode::OK); + + assert_eq!(driver.device.health(), DeviceHealth::Running); + driver.ok(KnownOpcode::GetDeviceInfo, &[], 128); +} + +/// A reset releases what the driver made and renames every object: an id +/// from before it names nothing after. +#[test] +fn a_reset_releases_everything_and_stales_every_id() { + let mut driver = Driver::new(); + let create = CreateContextRequest { + flags: Le32::new(0), + reserved: Le32::new(0), + }; + let context = driver.object(KnownOpcode::CreateContext, create.as_bytes()); + let buffer = driver.buffer(context, 4096, TRANSFER_SOURCE | TRANSFER_DESTINATION); + + VirtualDevice::reset(&mut driver.device).expect("reset"); + assert_eq!(driver.device.health(), DeviceHealth::Running); + + let (status, _) = driver.call(KnownOpcode::ReadBuffer, transfer(buffer, 16).as_bytes(), 16); + assert_eq!(status, StatusCode::STALE_OBJECT); + // The one context the device has is free again. + let again = driver.object(KnownOpcode::CreateContext, create.as_bytes()); + assert_ne!(again, context); + + // And again: each reset is a new namespace, so the id from between the + // two is as stale as the one from before the first. + VirtualDevice::reset(&mut driver.device).expect("a second reset"); + assert_eq!(driver.device.health(), DeviceHealth::Running); + let object = ObjectPayload { + object_id: Le64::new(again), + }; + let (status, _) = driver.call(KnownOpcode::DestroyContext, object.as_bytes(), 0); + assert_eq!(status, StatusCode::STALE_OBJECT); + let third = driver.object(KnownOpcode::CreateContext, create.as_bytes()); + assert!(third != again && third != context); +} + +/// A namespace is never reused, and there are sixteen bits of them. A +/// device reset past the last is not brought back: its transport then +/// reports it dead, and the driver is given a new device. +#[test] +fn a_device_out_of_namespaces_is_not_brought_back() { + let mut device = AccelDevice::new().expect("device"); + // The first life is namespace one; each reset takes the next. + for reset in 1..u32::from(u16::MAX) { + assert_eq!(VirtualDevice::reset(&mut device), Ok(()), "reset {reset}"); + } + assert_eq!(device.health(), DeviceHealth::Running); + assert_eq!(VirtualDevice::reset(&mut device), Err(Unrecoverable)); + assert_eq!(VirtualDevice::reset(&mut device), Err(Unrecoverable)); +} diff --git a/native/crates/vdev-accel/src/fp8.rs b/native/crates/vdev-accel/src/fp8.rs new file mode 100644 index 0000000..d8bd5f8 --- /dev/null +++ b/native/crates/vdev-accel/src/fp8.rs @@ -0,0 +1,138 @@ +//! **FP8 (E4M3) elementwise kernels**: the programs the virtual accelerator +//! runs. +//! +//! One operand, one result, each an E4M3 byte: OCP 8-bit floating point +//! (`torch.float8_e4m3fn`), a sign, four exponent bits biased by 7 and three +//! mantissa bits, with no infinities, one NaN (`S.1111.111`), a largest +//! magnitude of 448 and subnormals in steps of 2⁻⁹. +//! +//! Each kernel is a table lookup. The tables are generated by +//! `tools/fp8_tables.py`, whose documentation defines every entry: the exact +//! real result rounded to nearest, ties to even; saturating at ±448, since +//! there is no infinity; the canonical NaN `0x7f` for a NaN operand and for a +//! result outside the reals. The generator derives them in exact and +//! 120-digit arithmetic and proves each rounding decision by its margin, so +//! a result does not depend on any machine's math library. That is the point +//! of defining a device by a table: a driver writer needs one answer. +//! +//! A program is named the way every kernel of the CPU backend is: the +//! registered-kernel artifact format, with the kernel's id in the target +//! identity ([`Kernel::identity`]). A submission binds the operand buffer to +//! slot 0, read, and the result buffer to slot 1, written; the result's +//! length is the element count. + +use kore_svc_accel_spmd::{Kernel, TileCtx}; + +mod tables; + +/// One of the device's functions. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Op { + Sin, + Cos, + Tan, + Exp, + Log, + Sqrt, +} + +impl Op { + /// Every function, in kernel-id order. + pub const ALL: [Self; 6] = [ + Self::Sin, + Self::Cos, + Self::Tan, + Self::Exp, + Self::Log, + Self::Sqrt, + ]; + + /// The kernel id a program names this function by. + /// + /// The ids are this device's own: a registry is per backend instance, + /// so they share nothing with another CPU row's kernels. They start at + /// `0x0100` so that a driver which confuses the two registries names a + /// kernel neither has. + pub const fn kernel_id(self) -> u32 { + 0x0100 + self as u32 + } + + /// The table that defines this function: its result for each byte. + pub const fn table(self) -> &'static [u8; 256] { + match self { + Self::Sin => &tables::SIN, + Self::Cos => &tables::COS, + Self::Tan => &tables::TAN, + Self::Exp => &tables::EXP, + Self::Log => &tables::LOG, + Self::Sqrt => &tables::SQRT, + } + } + + /// The name a status line prints. + pub const fn name(self) -> &'static str { + match self { + Self::Sin => "sin.e4m3", + Self::Cos => "cos.e4m3", + Self::Tan => "tan.e4m3", + Self::Exp => "exp.e4m3", + Self::Log => "log.e4m3", + Self::Sqrt => "sqrt.e4m3", + } + } + + /// This function of one value. + #[inline] + pub const fn apply(self, operand: u8) -> u8 { + self.table()[operand as usize] + } +} + +/// Run one span: `out[i] = table[operand[first + i]]`. +/// +/// A byte indexes a 256-byte table, which is the whole computation and has +/// no wider form: the table fits four cache lines and the span is already +/// one worker's share of the grid. +#[inline] +fn map(ctx: TileCtx<'_>, table: &[u8; 256]) { + let first = ctx.tiles.start as usize; + let operands = &ctx.inputs[0][first..first + ctx.out.len()]; + for (result, operand) in ctx.out.iter_mut().zip(operands) { + *result = table[usize::from(*operand)]; + } +} + +macro_rules! kernels { + ($($op:ident => $run:ident),+ $(,)?) => { + $(fn $run(ctx: TileCtx<'_>) { + map(ctx, Op::$op.table()); + })+ + + /// The device's kernel registry: what [`SpmdBackend::new`] is given. + /// + /// [`SpmdBackend::new`]: kore_svc_accel_spmd::SpmdBackend::new + pub static KERNELS: &[Kernel] = &[ + $(Kernel { + id: Op::$op.kernel_id(), + name: Op::$op.name(), + inputs: 1, + elem_bytes: 1, + // A lookup has no lanes: one element a step. + lanes: 1, + run: $run, + }),+ + ]; + }; +} + +kernels! { + Sin => sin, + Cos => cos, + Tan => tan, + Exp => exp, + Log => log, + Sqrt => sqrt, +} + +#[cfg(test)] +mod tests; diff --git a/native/crates/vdev-accel/src/fp8/tables.rs b/native/crates/vdev-accel/src/fp8/tables.rs new file mode 100644 index 0000000..92f6254 --- /dev/null +++ b/native/crates/vdev-accel/src/fp8/tables.rs @@ -0,0 +1,225 @@ +// Generated by tools/fp8_tables.py; do not edit. The generator's +// documentation is the definition of these values, and +// `tools/fp8_tables.py --check ` proves the file current. + +/// `sin` of each E4M3 byte, as an E4M3 byte. +#[rustfmt::skip] +pub static SIN: [u8; 256] = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2e, + 0x2f, 0x31, 0x31, 0x32, 0x33, 0x34, 0x34, 0x35, + 0x35, 0x36, 0x37, 0x38, 0x38, 0x38, 0x38, 0x37, + 0x37, 0x34, 0x32, 0x2c, 0x21, 0x9e, 0xab, 0xb1, + 0xb4, 0xb8, 0xb7, 0xb3, 0xa9, 0x26, 0x33, 0x37, + 0x38, 0x2d, 0xb1, 0xb8, 0xb1, 0x2d, 0x38, 0x32, + 0xa9, 0xb4, 0x37, 0x85, 0xb6, 0x34, 0x29, 0xb8, + 0x31, 0xb8, 0x34, 0x09, 0xb4, 0x38, 0xb0, 0xaa, + 0x37, 0x28, 0xb8, 0x11, 0x38, 0xaa, 0xb6, 0x31, + 0x34, 0xb0, 0x26, 0x19, 0xab, 0x32, 0xb5, 0x37, + 0xb8, 0xb6, 0xae, 0x21, 0x33, 0x37, 0x37, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xae, + 0xaf, 0xb1, 0xb1, 0xb2, 0xb3, 0xb4, 0xb4, 0xb5, + 0xb5, 0xb6, 0xb7, 0xb8, 0xb8, 0xb8, 0xb8, 0xb7, + 0xb7, 0xb4, 0xb2, 0xac, 0xa1, 0x1e, 0x2b, 0x31, + 0x34, 0x38, 0x37, 0x33, 0x29, 0xa6, 0xb3, 0xb7, + 0xb8, 0xad, 0x31, 0x38, 0x31, 0xad, 0xb8, 0xb2, + 0x29, 0x34, 0xb7, 0x05, 0x36, 0xb4, 0xa9, 0x38, + 0xb1, 0x38, 0xb4, 0x89, 0x34, 0xb8, 0x30, 0x2a, + 0xb7, 0xa8, 0x38, 0x91, 0xb8, 0x2a, 0x36, 0xb1, + 0xb4, 0x30, 0xa6, 0x99, 0x2b, 0xb2, 0x35, 0xb7, + 0x38, 0x36, 0x2e, 0xa1, 0xb3, 0xb7, 0xb7, 0x7f, +]; + +/// `cos` of each E4M3 byte, as an E4M3 byte. +#[rustfmt::skip] +pub static COS: [u8; 256] = [ + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x37, 0x37, 0x37, 0x37, 0x37, 0x36, 0x36, + 0x36, 0x36, 0x35, 0x34, 0x34, 0x33, 0x32, 0x31, + 0x31, 0x2e, 0x2a, 0x24, 0x19, 0x96, 0xa3, 0xaa, + 0xad, 0xb2, 0xb5, 0xb7, 0xb8, 0xb8, 0xb7, 0xb5, + 0xb2, 0xa5, 0x29, 0x33, 0x37, 0x38, 0x34, 0x2b, + 0xa1, 0xb7, 0xb5, 0x02, 0x36, 0x37, 0x21, 0xb4, + 0xb7, 0x33, 0x2d, 0xb8, 0x2e, 0x32, 0xb7, 0x22, + 0x35, 0xa0, 0xb3, 0x38, 0xb2, 0xa2, 0x36, 0xb7, + 0x2d, 0xb7, 0x9e, 0x38, 0xa4, 0xb7, 0x2f, 0x35, + 0xb3, 0x36, 0xb8, 0x38, 0xb7, 0x35, 0xb1, 0x2a, + 0x92, 0x30, 0x36, 0x38, 0x34, 0x28, 0xaa, 0x7f, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x37, 0x37, 0x37, 0x37, 0x37, 0x36, 0x36, + 0x36, 0x36, 0x35, 0x34, 0x34, 0x33, 0x32, 0x31, + 0x31, 0x2e, 0x2a, 0x24, 0x19, 0x96, 0xa3, 0xaa, + 0xad, 0xb2, 0xb5, 0xb7, 0xb8, 0xb8, 0xb7, 0xb5, + 0xb2, 0xa5, 0x29, 0x33, 0x37, 0x38, 0x34, 0x2b, + 0xa1, 0xb7, 0xb5, 0x02, 0x36, 0x37, 0x21, 0xb4, + 0xb7, 0x33, 0x2d, 0xb8, 0x2e, 0x32, 0xb7, 0x22, + 0x35, 0xa0, 0xb3, 0x38, 0xb2, 0xa2, 0x36, 0xb7, + 0x2d, 0xb7, 0x9e, 0x38, 0xa4, 0xb7, 0x2f, 0x35, + 0xb3, 0x36, 0xb8, 0x38, 0xb7, 0x35, 0xb1, 0x2a, + 0x92, 0x30, 0x36, 0x38, 0x34, 0x28, 0xaa, 0x7f, +]; + +/// `tan` of each E4M3 byte, as an E4M3 byte. +#[rustfmt::skip] +pub static TAN: [u8; 256] = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2d, 0x2e, 0x2f, 0x30, + 0x31, 0x32, 0x34, 0x35, 0x37, 0x38, 0x3a, 0x3b, + 0x3c, 0x40, 0x44, 0x4a, 0x56, 0xd9, 0xcb, 0xc5, + 0xc1, 0xba, 0xb4, 0xad, 0xa1, 0x1e, 0x2c, 0x33, + 0x39, 0x49, 0xc6, 0xb8, 0xa9, 0x26, 0x36, 0x43, + 0xce, 0xae, 0x32, 0xf6, 0xb2, 0x2f, 0x4e, 0xb6, + 0x2a, 0xb9, 0x41, 0x05, 0xc1, 0x39, 0xa9, 0xcd, + 0x33, 0x50, 0xb9, 0x09, 0x3a, 0xcc, 0xb2, 0x2a, + 0x41, 0xa8, 0x51, 0x11, 0xcb, 0x2b, 0xc0, 0x33, + 0xb8, 0xb1, 0xa6, 0x19, 0x2c, 0x34, 0x3b, 0x44, + 0x5d, 0xbd, 0xaf, 0x21, 0x36, 0x47, 0xc4, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xad, 0xae, 0xaf, 0xb0, + 0xb1, 0xb2, 0xb4, 0xb5, 0xb7, 0xb8, 0xba, 0xbb, + 0xbc, 0xc0, 0xc4, 0xca, 0xd6, 0x59, 0x4b, 0x45, + 0x41, 0x3a, 0x34, 0x2d, 0x21, 0x9e, 0xac, 0xb3, + 0xb9, 0xc9, 0x46, 0x38, 0x29, 0xa6, 0xb6, 0xc3, + 0x4e, 0x2e, 0xb2, 0x76, 0x32, 0xaf, 0xce, 0x36, + 0xaa, 0x39, 0xc1, 0x85, 0x41, 0xb9, 0x29, 0x4d, + 0xb3, 0xd0, 0x39, 0x89, 0xba, 0x4c, 0x32, 0xaa, + 0xc1, 0x28, 0xd1, 0x91, 0x4b, 0xab, 0x40, 0xb3, + 0x38, 0x31, 0x26, 0x99, 0xac, 0xb4, 0xbb, 0xc4, + 0xdd, 0x3d, 0x2f, 0xa1, 0xb6, 0xc7, 0x44, 0x7f, +]; + +/// `exp` of each E4M3 byte, as an E4M3 byte. +#[rustfmt::skip] +pub static EXP: [u8; 256] = [ + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, + 0x39, 0x39, 0x39, 0x3a, 0x3a, 0x3a, 0x3a, 0x3a, + 0x3a, 0x3b, 0x3b, 0x3b, 0x3c, 0x3c, 0x3c, 0x3d, + 0x3d, 0x3e, 0x3f, 0x40, 0x40, 0x41, 0x42, 0x42, + 0x43, 0x44, 0x46, 0x48, 0x49, 0x4a, 0x4c, 0x4d, + 0x4f, 0x51, 0x54, 0x58, 0x5a, 0x5d, 0x60, 0x63, + 0x66, 0x6b, 0x71, 0x77, 0x7d, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7f, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x37, 0x37, 0x37, 0x37, 0x37, 0x37, 0x37, + 0x37, 0x37, 0x37, 0x37, 0x37, 0x36, 0x36, 0x36, + 0x36, 0x36, 0x36, 0x35, 0x35, 0x35, 0x35, 0x35, + 0x34, 0x34, 0x34, 0x33, 0x33, 0x33, 0x32, 0x32, + 0x32, 0x31, 0x31, 0x30, 0x2f, 0x2e, 0x2d, 0x2d, + 0x2c, 0x2a, 0x29, 0x28, 0x26, 0x25, 0x23, 0x22, + 0x21, 0x1d, 0x1b, 0x18, 0x15, 0x12, 0x0f, 0x0c, + 0x09, 0x06, 0x03, 0x02, 0x01, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, +]; + +/// `log` of each E4M3 byte, as an E4M3 byte. +#[rustfmt::skip] +pub static LOG: [u8; 256] = [ + 0xfe, 0xcc, 0xcb, 0xca, 0xca, 0xc9, 0xc9, 0xc9, + 0xc8, 0xc8, 0xc8, 0xc7, 0xc7, 0xc7, 0xc6, 0xc6, + 0xc6, 0xc5, 0xc5, 0xc5, 0xc4, 0xc4, 0xc4, 0xc3, + 0xc3, 0xc3, 0xc2, 0xc2, 0xc1, 0xc1, 0xc1, 0xc1, + 0xc0, 0xc0, 0xbf, 0xbe, 0xbd, 0xbd, 0xbc, 0xbc, + 0xbb, 0xba, 0xb9, 0xb9, 0xb8, 0xb6, 0xb5, 0xb4, + 0xb3, 0xb1, 0xaf, 0xac, 0xa9, 0xa5, 0xa1, 0x98, + 0x00, 0x1f, 0x26, 0x2a, 0x2d, 0x30, 0x31, 0x32, + 0x33, 0x35, 0x37, 0x38, 0x39, 0x39, 0x3a, 0x3b, + 0x3b, 0x3c, 0x3d, 0x3e, 0x3e, 0x3f, 0x40, 0x40, + 0x40, 0x41, 0x41, 0x42, 0x42, 0x42, 0x43, 0x43, + 0x43, 0x44, 0x44, 0x44, 0x45, 0x45, 0x45, 0x46, + 0x46, 0x46, 0x47, 0x47, 0x47, 0x48, 0x48, 0x48, + 0x48, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x4a, + 0x4a, 0x4a, 0x4a, 0x4a, 0x4b, 0x4b, 0x4b, 0x4b, + 0x4b, 0x4b, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x7f, + 0xfe, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, +]; + +/// `sqrt` of each E4M3 byte, as an E4M3 byte. +#[rustfmt::skip] +pub static SQRT: [u8; 256] = [ + 0x00, 0x13, 0x18, 0x1a, 0x1b, 0x1d, 0x1e, 0x1f, + 0x20, 0x20, 0x21, 0x21, 0x22, 0x22, 0x23, 0x23, + 0x23, 0x24, 0x25, 0x25, 0x26, 0x26, 0x27, 0x27, + 0x28, 0x28, 0x29, 0x29, 0x2a, 0x2a, 0x2b, 0x2b, + 0x2b, 0x2c, 0x2d, 0x2d, 0x2e, 0x2e, 0x2f, 0x2f, + 0x30, 0x30, 0x31, 0x31, 0x32, 0x32, 0x33, 0x33, + 0x33, 0x34, 0x35, 0x35, 0x36, 0x36, 0x37, 0x37, + 0x38, 0x38, 0x39, 0x39, 0x3a, 0x3a, 0x3b, 0x3b, + 0x3b, 0x3c, 0x3d, 0x3d, 0x3e, 0x3e, 0x3f, 0x3f, + 0x40, 0x40, 0x41, 0x41, 0x42, 0x42, 0x43, 0x43, + 0x43, 0x44, 0x45, 0x45, 0x46, 0x46, 0x47, 0x47, + 0x48, 0x48, 0x49, 0x49, 0x4a, 0x4a, 0x4b, 0x4b, + 0x4b, 0x4c, 0x4d, 0x4d, 0x4e, 0x4e, 0x4f, 0x4f, + 0x50, 0x50, 0x51, 0x51, 0x52, 0x52, 0x53, 0x53, + 0x53, 0x54, 0x55, 0x55, 0x56, 0x56, 0x57, 0x57, + 0x58, 0x58, 0x59, 0x59, 0x5a, 0x5a, 0x5b, 0x7f, + 0x80, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, +]; diff --git a/native/crates/vdev-accel/src/fp8/tests.rs b/native/crates/vdev-accel/src/fp8/tests.rs new file mode 100644 index 0000000..1bfe6e3 --- /dev/null +++ b/native/crates/vdev-accel/src/fp8/tests.rs @@ -0,0 +1,286 @@ +use super::*; +use kore_svc_accel_spmd::{ + FORMAT_REGISTERED, IDENTITY_MAGIC, REGISTERED_MANIFEST, REGISTERED_RESIDENT_BYTES, SpmdBackend, + SpmdBuffer, SpmdContext, +}; +use virtio_accel_core::{ + Accelerator, AccessMode, ArtifactFormat, ArtifactRef, BackendError, BindingRef, BufferDesc, + BufferRange, BufferUsage, ContextDesc, ContextFlags, EventState, MemoryDomain, QueueDesc, + QueueFlags, TargetIdentity, Timeout, +}; + +const NAN: u8 = 0x7f; +const LARGEST: u8 = 0x7e; + +/// The value of one E4M3 byte; `None` for the NaN. +fn decode(code: u8) -> Option { + let sign = if code & 0x80 != 0 { -1.0 } else { 1.0 }; + let exponent = i32::from((code >> 3) & 0xf); + let mantissa = f64::from(code & 0x7); + if exponent == 0xf && code & 0x7 == 0x7 { + return None; + } + Some(if exponent == 0 { + sign * mantissa * 2f64.powi(-9) + } else { + sign * (8.0 + mantissa) * 2f64.powi(exponent - 10) + }) +} + +/// Round a real to E4M3 by the device's rules: nearest, ties to even, +/// saturating, the sign kept on a zero. +fn encode(value: f64) -> u8 { + if value.is_nan() { + return NAN; + } + let sign = if value.is_sign_negative() { 0x80 } else { 0x00 }; + let magnitude = value.abs(); + if magnitude >= 448.0 { + return sign | LARGEST; + } + let mut best = 0u8; + for code in 1..=LARGEST { + let (candidate, held) = (decode(code).unwrap(), decode(best).unwrap()); + let (near, far) = ((candidate - magnitude).abs(), (held - magnitude).abs()); + if near < far || (near == far && code % 2 == 0) { + best = code; + } + } + sign | best +} + +/// What this machine's own math library says the function is, in binary64. +/// The tables were not made with it; agreeing with it is the check. +fn reference(op: Op, x: f64) -> f64 { + match op { + Op::Sin => x.sin(), + Op::Cos => x.cos(), + Op::Tan => x.tan(), + Op::Exp => x.exp(), + Op::Log => x.ln(), + Op::Sqrt => x.sqrt(), + } +} + +/// The generator's tables against an independent computation of all 1536 +/// entries. Its closest rounding decision is 2.5e-5 of a boundary away +/// (`tools/fp8_tables.py --margin`), eleven orders of magnitude past +/// binary64's error, so agreement here is conclusive and not luck. +#[test] +fn every_entry_is_the_correctly_rounded_result() { + for op in Op::ALL { + for code in 0..=u8::MAX { + let want = match decode(code) { + None => NAN, + // The limit of log at either zero is negative infinity, + // which saturates: `ln(-0.0)` is that in binary64 too. + Some(x) => encode(reference(op, x)), + }; + assert_eq!( + op.apply(code), + want, + "{}({code:#04x} = {:?})", + op.name(), + decode(code) + ); + } + } +} + +/// The rules a driver writer reads, each on a value they can check by hand. +#[test] +fn the_stated_rules_hold_on_values_one_can_check_by_hand() { + let code = |value: f64| encode(value); + let one = code(1.0); + assert_eq!(one, 0x38); + assert_eq!(decode(Op::Sin.apply(one)), Some(0.8125), "sin 1 = 0.8414…"); + assert_eq!(decode(Op::Cos.apply(one)), Some(0.5625), "cos 1 = 0.5403…"); + assert_eq!(decode(Op::Tan.apply(one)), Some(1.5), "tan 1 = 1.5574…"); + assert_eq!(decode(Op::Exp.apply(one)), Some(2.75), "e = 2.7182…"); + assert_eq!(Op::Log.apply(one), 0x00, "log 1 is exactly zero"); + assert_eq!(decode(Op::Sqrt.apply(code(4.0))), Some(2.0)); + assert_eq!(Op::Cos.apply(0x00), one); + assert_eq!(Op::Exp.apply(0x00), one); + + // No infinity: a result past the largest magnitude is that magnitude. + assert_eq!(decode(LARGEST), Some(448.0)); + assert_eq!(Op::Exp.apply(LARGEST), LARGEST); + assert_eq!(Op::Exp.apply(code(8.0)), LARGEST, "e^8 = 2980.9…"); + assert_eq!(Op::Log.apply(0x00), 0x80 | LARGEST, "log 0 is -448"); + assert_eq!(Op::Log.apply(0x80), 0x80 | LARGEST, "and so is log -0"); + assert_eq!(Op::Exp.apply(0x80 | LARGEST), 0x00, "e^-448 rounds to zero"); + + // Outside the reals, and a NaN operand, give the one canonical NaN. + for op in Op::ALL { + assert_eq!(op.apply(0x7f), NAN, "{}", op.name()); + assert_eq!(op.apply(0xff), NAN, "{}", op.name()); + } + assert_eq!(Op::Log.apply(code(-1.0)), NAN); + assert_eq!(Op::Sqrt.apply(code(-1.0)), NAN); + + // A zero keeps its sign where IEEE 754 keeps it. + for op in [Op::Sin, Op::Tan, Op::Sqrt] { + assert_eq!(op.apply(0x00), 0x00, "{}", op.name()); + assert_eq!(op.apply(0x80), 0x80, "{}", op.name()); + } + // sin and tan are odd, cos is even, over every value. + for code in 0..=LARGEST { + assert_eq!(Op::Sin.apply(code | 0x80), Op::Sin.apply(code) ^ 0x80); + assert_eq!(Op::Tan.apply(code | 0x80), Op::Tan.apply(code) ^ 0x80); + assert_eq!(Op::Cos.apply(code | 0x80), Op::Cos.apply(code)); + } +} + +#[test] +fn the_registry_names_each_function_once() { + assert_eq!(KERNELS.len(), Op::ALL.len()); + for (kernel, op) in KERNELS.iter().zip(Op::ALL) { + assert_eq!(kernel.id, op.kernel_id()); + assert_eq!(kernel.name, op.name()); + assert_eq!((kernel.inputs, kernel.elem_bytes), (1, 1)); + assert_eq!(kernel.identity().0[..2], [IDENTITY_MAGIC, op.kernel_id()]); + } + for (index, kernel) in KERNELS.iter().enumerate() { + assert!( + !KERNELS[..index].iter().any(|prior| prior.id == kernel.id), + "duplicate kernel id {:#x}", + kernel.id + ); + } +} + +/// A span is its own share of the grid: two disjoint spans equal one run. +#[test] +fn a_tiled_run_equals_the_whole_run() { + let operands: [u8; 256] = core::array::from_fn(|i| i as u8); + let run = KERNELS[Op::Exp as usize].run; + let launch = kore_driver::accel::LaunchShape::ArtifactDefined; + let mut whole = [0u8; 256]; + run(TileCtx { + tiles: 0..256, + total: 256, + launch, + inputs: &[&operands], + out: &mut whole, + }); + assert_eq!(whole, *Op::Exp.table()); + + let mut split = [0u8; 256]; + let (low, high) = split.split_at_mut(100); + run(TileCtx { + tiles: 0..100, + total: 256, + launch, + inputs: &[&operands], + out: low, + }); + run(TileCtx { + tiles: 100..256, + total: 256, + launch, + inputs: &[&operands], + out: high, + }); + assert_eq!(split, whole); +} + +const BYTES: u64 = 256; + +fn buffer(backend: &SpmdBackend, context: &SpmdContext, usage: BufferUsage) -> SpmdBuffer { + let usage = usage | BufferUsage::TRANSFER_SOURCE | BufferUsage::TRANSFER_DESTINATION; + let desc = BufferDesc::new(BYTES, 1, MemoryDomain::Host, usage).expect("descriptor"); + backend + .allocate_buffer(context, desc) + .expect("allocated") + .into_parts() + .0 +} + +fn artifact(id: u32) -> ArtifactRef<'static> { + let mut words = [0u32; 12]; + words[0] = IDENTITY_MAGIC; + words[1] = id; + ArtifactRef { + format: ArtifactFormat::new(FORMAT_REGISTERED).expect("format"), + target: TargetIdentity(words), + payload: ®ISTERED_MANIFEST, + resident_bytes: REGISTERED_RESIDENT_BYTES, + } +} + +/// The device, as the protocol reaches it: every function over every byte, +/// through the CPU backend's context, buffers, program, queue and event. +#[test] +fn the_backend_runs_each_function_over_every_value() { + let backend = SpmdBackend::new(KERNELS); + let context = backend + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = backend + .create_queue( + &context, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let mut operands = buffer(&backend, &context, BufferUsage::PROGRAM_INPUT); + let results = buffer(&backend, &context, BufferUsage::PROGRAM_OUTPUT); + let every_byte: [u8; 256] = core::array::from_fn(|i| i as u8); + backend + .write_buffer(&mut operands, 0, &every_byte) + .expect("written"); + let whole = BufferRange::new(0, BYTES).expect("range"); + + for op in Op::ALL { + let program = backend + .load_program(&context, artifact(op.kernel_id())) + .expect("loaded"); + let event = backend + .submit( + &queue, + &program, + &[ + BindingRef { + slot: 0, + buffer: &operands, + range: whole, + access: AccessMode::Read, + }, + BindingRef { + slot: 1, + buffer: &results, + range: whole, + access: AccessMode::Write, + }, + ], + Timeout::Infinite, + ) + .map_err(|_| "refused") + .expect("admitted"); + assert_eq!(backend.poll_event(&event), Ok(EventState::Pending)); + assert!(backend.run_pending(), "the admitted launch runs"); + assert_eq!(backend.poll_event(&event), Ok(EventState::Complete)); + let mut read = [0u8; 256]; + backend.read_buffer(&results, 0, &mut read).expect("read"); + assert_eq!(read, *op.table(), "{}", op.name()); + backend.destroy_event(event).map_err(|_| "event").unwrap(); + backend + .unload_program(program) + .map_err(|_| "program") + .unwrap(); + } + + // A kernel of another registry is not one this device has. + assert_eq!( + backend + .load_program( + &context, + artifact(kore_svc_accel_spmd::kernel::KERNEL_ADD_F32) + ) + .err(), + Some(BackendError::Incompatible) + ); +} diff --git a/native/crates/vdev-accel/src/lib.rs b/native/crates/vdev-accel/src/lib.rs new file mode 100644 index 0000000..fe4f5bf --- /dev/null +++ b/native/crates/vdev-accel/src/lib.rs @@ -0,0 +1,25 @@ +//! # kore-vdev-accel: the virtual accelerator +//! +//! A virtio-accel 1.0 device with no hardware behind it. It exists so a +//! driver can be written, run and debugged on a machine that has no +//! accelerator, against a device whose every answer is known in advance. +//! +//! What it computes is deliberately small and exact: six functions of one +//! FP8 (E4M3) value, elementwise over a buffer. See [`fp8`]. An E4M3 value +//! is a byte, so each function is a 256-entry table, and the table *is* the +//! device's definition. A driver is right when it reads back those bytes. +//! +//! The device is an instance of the CPU backend, `kore-svc-accel-spmd`: its +//! programs are that backend's registered kernels, and this crate supplies +//! the registry ([`fp8::KERNELS`]). What a driver exchanges with it is the +//! protocol's own frames, decoded by the protocol's own command engine +//! ([`device::AccelDevice`]). Nothing here knows how a frame reaches the +//! device; that is the transport's business. + +#![no_std] + +pub mod device; +pub mod fp8; + +#[cfg(test)] +extern crate std; diff --git a/native/crates/vdev-accel/tests/session.rs b/native/crates/vdev-accel/tests/session.rs new file mode 100644 index 0000000..ee0d653 --- /dev/null +++ b/native/crates/vdev-accel/tests/session.rs @@ -0,0 +1,438 @@ +//! A driver's whole session with the virtual accelerator, end to end and +//! with nothing of the driver written here: the driver is the protocol's +//! reference guest client (`virtio-accel-guest`), over kore's register +//! block and shared-memory queue (`kore-vdev`'s driver half), against this +//! crate's device behind `kore-vdev`'s transport. +//! +//! What it shows is that the pieces are one device. The reference client +//! knows only the protocol's queue contract; the device model knows only +//! requests and responses; the registers and the ring between them are what +//! a driver in C will map. + +use core::num::{NonZeroU32, NonZeroU64}; +use core::ptr::NonNull; + +use kore_svc_accel_spmd::{ + FORMAT_REGISTERED, IDENTITY_MAGIC, REGISTERED_MANIFEST, REGISTERED_RESIDENT_BYTES, +}; +use kore_vdev::device::VirtualDevice; +use kore_vdev::driver::{self, DriverChain, DriverRing, Layout, Slots}; +use kore_vdev::mem::Window; +use kore_vdev::regs; +use kore_vdev::transport::Transport; +use kore_vdev_accel::device::{AccelDevice, DEVICE_ID}; +use kore_vdev_accel::fp8::Op; +use virtio_accel_guest::{ + AccessMode, Binding, Buffer, BufferDesc, BufferRange, BufferUsage, Completion, Context, + EventState, ExecutionQueue, GuestClient, GuestConfig, MemoryDomain, Operation, ProgramDesc, + RequestPoll, StartResult, SubmissionOutcome, +}; +use virtio_accel_proto::{ + AllocateBufferRequest, CreateContextRequest, CreateQueueRequest, LoadProgramRequest, + ObjectPayload, RequestHeader, ResponseHeader, StatusCode, SubmitRequest, SubmitResponse, + TransferBufferRequest, WireBinding, WireConfig, WireDeviceInfo, WireEventState, read_exact, +}; +use virtio_accel_transport::{DriverChainBuffer, QueueSize}; + +type Ring = DriverRing; +type Chain = DriverChain; +type Client = GuestClient; + +const BUDGET: NonZeroU32 = NonZeroU32::new(8).unwrap(); +const REQUEST: u32 = core::mem::size_of::() as u32; +const RESPONSE: u32 = core::mem::size_of::() as u32; +/// Every E4M3 value, once. +const VALUES: u32 = 256; + +const fn bytes_of() -> u32 { + core::mem::size_of::() as u32 +} + +/// Host memory standing for a lease both sides map. +fn window(bytes: u64) -> Window { + let words: &'static mut [u64] = + Box::leak(vec![0u64; (bytes as usize).div_ceil(8)].into_boxed_slice()); + // SAFETY: a leaked, eight-byte aligned allocation of at least `bytes` + // bytes that nothing else holds a reference into. + unsafe { + Window::new( + NonNull::new(words.as_mut_ptr().cast()).unwrap(), + bytes as usize, + ) + } +} + +/// A driver process and the device's host, in one test. +struct Session { + registers: Window, + memory: Window, + layout: Layout, + transport: Transport, + /// How often the device raised the driver's interrupt. + interrupts: u32, +} + +/// What a bring-up gives the driver. +struct Driver { + client: Client, + slots: Slots, +} + +impl Session { + fn new() -> Self { + // Sixteen descriptors, the device's longest chain, so eight commands + // in flight; a slot holds a request or a response of a page. + let layout = Layout::new(QueueSize::new(16).unwrap(), 4096, 4096).unwrap(); + let registers = window(regs::BYTES); + let memory = window(layout.bytes()); + let device = AccelDevice::new().expect("the device"); + let mut transport = Transport::new(registers, device).expect("a transport"); + // The driver lends its memory before it brings the device up. + transport.lend(memory).expect("a device in its first state"); + Self { + registers, + memory, + layout, + transport, + interrupts: 0, + } + } + + /// The doorbell: the host services the transport. + fn ring(&mut self) { + loop { + let serviced = self.transport.service(BUDGET); + self.interrupts += u32::from(serviced.interrupt); + if !serviced.pending { + return; + } + } + } + + /// What every driver does first: find the device, bring it up and read + /// its configuration. + fn bring_up(&mut self) -> Driver { + let identity = driver::identify(&self.registers).expect("a kore virtual device"); + assert_eq!(identity.device_id, DEVICE_ID); + assert_eq!(identity.vendor_id, regs::VENDOR_KORE); + let mut config = [0u8; core::mem::size_of::()]; + assert_eq!(identity.config_bytes as usize, config.len()); + driver::read_config(&self.registers, &mut config).unwrap(); + let wire: WireConfig = read_exact(&config).unwrap(); + assert_eq!(wire, AccelDevice::CONFIG); + + let (registers, memory, layout) = (self.registers, self.memory, self.layout); + let (ring, slots) = driver::bring_up(®isters, memory, layout, || self.ring()) + .expect("the device comes up"); + // The protocol has no feature of its own: what is left of the offer + // once the transport's bit is taken away must be nothing. + let config = GuestConfig::new(wire, identity.features & !regs::F_VERSION_1, layout.slots()) + .expect("a configuration the reference driver accepts"); + Driver { + client: GuestClient::new(ring, config).expect("the reference driver"), + slots, + } + } + + /// One command: take a slot for a request of `request` payload bytes + /// and a response of `response`, start the operation, ring, and take + /// the answer. + fn call( + &mut self, + driver: &mut Driver, + request: u32, + response: u32, + start: impl FnOnce(&mut Client, Chain) -> StartResult, + ) -> Completion::Error> { + let chain = driver + .slots + .chain(REQUEST + request, RESPONSE + response) + .expect("a free slot"); + let pending = match start(&mut driver.client, chain) { + Ok(pending) => pending, + Err(error) => panic!("the request was not started: {:?}", error.into_parts().2), + }; + let before = self.interrupts; + self.ring(); + assert_eq!(self.interrupts, before + 1, "one command, one interrupt"); + match driver.client.poll(pending) { + RequestPoll::Ready(completion) => completion, + _ => panic!("a serviced command has its answer"), + } + } + + /// A command that must succeed: its output, with the slot returned. + fn ok( + &mut self, + driver: &mut Driver, + request: u32, + response: u32, + start: impl FnOnce(&mut Client, Chain) -> StartResult, + ) -> O::Output { + let (output, chain) = self.ok_with_chain(driver, request, response, start); + driver.slots.recycle(chain); + output + } + + fn ok_with_chain( + &mut self, + driver: &mut Driver, + request: u32, + response: u32, + start: impl FnOnce(&mut Client, Chain) -> StartResult, + ) -> (O::Output, Chain) { + match self.call(driver, request, response, start) { + Completion::Success { output, chain } => (output, chain), + Completion::DeviceError { status, .. } => panic!("the device answered {status:?}"), + Completion::InvalidResponse { error, .. } => panic!("a malformed answer: {error:?}"), + } + } + + /// A command the device must refuse: the status it refused with. + fn refused( + &mut self, + driver: &mut Driver, + request: u32, + response: u32, + start: impl FnOnce(&mut Client, Chain) -> StartResult, + ) -> StatusCode { + match self.call(driver, request, response, start) { + Completion::DeviceError { status, chain, .. } => { + driver.slots.recycle(chain); + status + } + _ => panic!("the device was expected to refuse"), + } + } +} + +/// The objects a session computes with. +struct Objects { + context: Context, + operands: Buffer, + results: Buffer, + queue: ExecutionQueue, +} + +fn objects(session: &mut Session, driver: &mut Driver) -> Objects { + let info = session.ok(driver, 0, bytes_of::(), |client, chain| { + client.get_device_info(chain) + }); + assert_eq!(&info.uuid[..13], b"kore-spmd-cpu"); + + let context = session.ok( + driver, + bytes_of::(), + bytes_of::(), + |client, chain| client.create_context(chain), + ); + let mut buffer = |usage| { + let desc = BufferDesc::new(u64::from(VALUES), 1, MemoryDomain::Host, usage).unwrap(); + session.ok( + driver, + bytes_of::(), + bytes_of::(), + |client, chain| client.allocate_buffer(chain, &context, desc), + ) + }; + let operands = buffer(BufferUsage::PROGRAM_INPUT | BufferUsage::TRANSFER_DESTINATION); + let results = buffer(BufferUsage::PROGRAM_OUTPUT | BufferUsage::TRANSFER_SOURCE); + let queue = session.ok( + driver, + bytes_of::(), + bytes_of::(), + |client, chain| client.create_execution_queue(chain, &context), + ); + Objects { + context, + operands, + results, + queue, + } +} + +/// Run `op` over the operands buffer and answer what the results buffer +/// then holds. +fn compute(session: &mut Session, driver: &mut Driver, objects: &Objects, op: Op) -> Vec { + let mut target = [0u32; 12]; + target[0] = IDENTITY_MAGIC; + target[1] = op.kernel_id(); + let desc = ProgramDesc::new( + NonZeroU32::new(FORMAT_REGISTERED).unwrap(), + target, + NonZeroU64::new(REGISTERED_RESIDENT_BYTES).unwrap(), + ); + let program = session.ok( + driver, + bytes_of::() + REGISTERED_MANIFEST.len() as u32, + bytes_of::(), + |client, chain| { + client.load_program( + chain, + &objects.context, + desc, + REGISTERED_MANIFEST.as_slice(), + ) + }, + ); + + let whole = BufferRange::new(0, u64::from(VALUES)).unwrap(); + let bindings = [ + Binding { + buffer: &objects.operands, + range: whole, + slot: 0, + access: AccessMode::Read, + }, + Binding { + buffer: &objects.results, + range: whole, + slot: 1, + access: AccessMode::Write, + }, + ]; + let outcome = session.ok( + driver, + bytes_of::() + 2 * bytes_of::(), + bytes_of::(), + // Zero: no deadline. + |client, chain| client.submit(chain, &objects.queue, &program, &bindings, 0), + ); + let SubmissionOutcome::Accepted(event) = outcome else { + panic!("the submission was not accepted: {outcome:?}"); + }; + + // The submission was answered and then run in the same service: the + // first poll finds it complete. + let state = session.ok( + driver, + bytes_of::(), + bytes_of::(), + |client, chain| client.poll_event(chain, &event), + ); + assert_eq!(state, EventState::Complete, "{}", op.name()); + + // The results are the event's until the event is destroyed. + let read = |client: &mut Client, chain| client.read_buffer(chain, &objects.results, whole); + assert_eq!( + session.refused(driver, bytes_of::(), VALUES, read), + StatusCode::BUSY + ); + session.ok(driver, bytes_of::(), 0, |client, chain| { + client.destroy_event(chain, event) + }); + let (output, chain) = + session.ok_with_chain(driver, bytes_of::(), VALUES, read); + assert_eq!(output.bytes, u64::from(VALUES)); + let mut results = vec![0u8; VALUES as usize]; + chain + .read_device_writable(u64::from(RESPONSE), &mut results) + .unwrap(); + driver.slots.recycle(chain); + + session.ok(driver, bytes_of::(), 0, |client, chain| { + client.unload_program(chain, program) + }); + results +} + +/// The session a driver is written to hold: bring the device up, make its +/// objects, write every E4M3 value once, and for each of the six functions +/// submit, poll, and read back exactly the function's table. +#[test] +fn the_reference_driver_computes_every_function() { + let mut session = Session::new(); + let mut driver = session.bring_up(); + let objects = objects(&mut session, &mut driver); + + let every_value: [u8; VALUES as usize] = core::array::from_fn(|value| value as u8); + session.ok( + &mut driver, + bytes_of::() + VALUES, + 0, + |client, chain| client.write_buffer(chain, &objects.operands, 0, every_value.as_slice()), + ); + + for op in Op::ALL { + let results = compute(&mut session, &mut driver, &objects, op); + assert_eq!(results.as_slice(), op.table(), "{}", op.name()); + for (value, result) in results.iter().enumerate() { + assert_eq!(*result, op.apply(value as u8)); + } + } + assert_eq!(driver.slots.free(), u32::from(session.layout.slots())); + assert!(!session.transport.device().needs_reset()); + assert!(!session.transport.is_dead()); +} + +/// A second bring-up is a reset. The device forgets every object, an id +/// kept from before names nothing, and the device computes again from its +/// first state. +#[test] +fn a_reset_gives_the_driver_a_device_in_its_first_state() { + let mut session = Session::new(); + let mut driver = session.bring_up(); + let before = objects(&mut session, &mut driver); + + let mut driver = session.bring_up(); + // Discovery again: the client is new, as the device is. + session.ok( + &mut driver, + 0, + bytes_of::(), + |client, chain| client.get_device_info(chain), + ); + let stale = session.refused( + &mut driver, + bytes_of::(), + VALUES, + |client, chain| { + client.read_buffer(chain, &before.results, BufferRange::new(0, 16).unwrap()) + }, + ); + assert_eq!(stale, StatusCode::STALE_OBJECT); + + // The one context the device has is free again, and everything works. + let objects = objects(&mut session, &mut driver); + assert_ne!(objects.context.raw(), before.context.raw()); + let sevens = [0x77u8; VALUES as usize]; + session.ok( + &mut driver, + bytes_of::() + VALUES, + 0, + |client, chain| client.write_buffer(chain, &objects.operands, 0, sevens.as_slice()), + ); + let results = compute(&mut session, &mut driver, &objects, Op::Sqrt); + assert_eq!(results, [Op::Sqrt.apply(0x77); VALUES as usize]); +} + +/// The reference driver with every slot in flight at once: the commands are +/// served in the order they were published, and each answer reaches the +/// request it belongs to. +#[test] +fn every_slot_in_flight_is_answered() { + let mut session = Session::new(); + let mut driver = session.bring_up(); + let slots = session.layout.slots(); + + let pending: Vec<_> = (0..slots) + .map(|_| { + let chain = driver + .slots + .chain(REQUEST, RESPONSE + bytes_of::()) + .unwrap(); + driver.client.get_device_info(chain).ok().expect("started") + }) + .collect(); + assert_eq!(driver.slots.free(), 0); + session.ring(); + for pending in pending { + let RequestPoll::Ready(Completion::Success { output, chain }) = driver.client.poll(pending) + else { + panic!("every command was answered"); + }; + assert_eq!(&output.uuid[..13], b"kore-spmd-cpu"); + driver.slots.recycle(chain); + } + assert_eq!(driver.slots.free(), u32::from(slots)); +} diff --git a/native/crates/vdev/Cargo.toml b/native/crates/vdev/Cargo.toml new file mode 100644 index 0000000..ed6c513 --- /dev/null +++ b/native/crates/vdev/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "kore-vdev" +version = "0.0.1" +edition = "2024" +description = "kore virtual devices: the transport a driver reaches a device with no hardware behind it, as memory it shares with the device's host." +publish = false + +[dependencies] +# Bounded per-chain metadata: a chain's spans and regions are at most the +# queue's size, so they need no heap. +heapless = { version = "0.9.3", default-features = false } +# The queue contracts the ring implements (MicroPerceptron/virtio-accel 1.0): +# `DeviceQueue`, `DeviceChain`, the chain-layout rules and the reset epoch. +# The crate is dependency-free and names no memory model, which is why the +# ring over shared memory lives here and not there. +virtio-accel-transport = { version = "=0.4.1", default-features = false } + +[dev-dependencies] +# The protocol's own in-memory split queue: the oracle the shared-memory +# ring is held to, chain shape by chain shape. +virtio-accel-split-queue = { version = "=0.4.1", default-features = false } diff --git a/native/crates/vdev/src/device.rs b/native/crates/vdev/src/device.rs new file mode 100644 index 0000000..92c49be --- /dev/null +++ b/native/crates/vdev/src/device.rs @@ -0,0 +1,69 @@ +//! **The device model's contract**: what a device with no hardware is, to +//! the transport that carries a driver to it. +//! +//! A model answers commands. A driver sends one command as one chain of +//! descriptors, the bytes it gives the device to read followed by the bytes +//! it gives the device to write, and the model reads a request from the +//! first and writes a response into the second. That is the whole exchange, +//! and it is all a model sees of a driver: it is handed lengths and byte +//! ports, never a device address, a descriptor or a ring. +//! +//! Everything else a driver meets is the transport's +//! ([`Transport`](crate::transport::Transport)): the register block, the +//! status it steps through, the queue, the doorbell and the interrupt. A +//! model is written once and knows none of it. + +pub use virtio_accel_transport::{ChainRegion, ReadableBytes, WritableBytes}; + +/// A reset could not return the model to its first state. The transport +/// stops serving for good, and the host drops the instance. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Unrecoverable; + +/// One device model. +pub trait VirtualDevice { + /// What the driver reads from [`DEVICE_ID`](crate::regs::DEVICE_ID) to + /// tell which model it has been given. + const DEVICE_ID: u32; + + /// The largest queue the model takes: a power of two, and at most + /// [`MAX_QUEUE_SIZE`](crate::ring::MAX_QUEUE_SIZE). + const MAX_QUEUE_SIZE: u16; + + /// The longest chain one command may be, in descriptors: at least two + /// (a request and a response) and at most the queue's size. + const MAX_CHAIN_DESCRIPTORS: u16; + + /// The device configuration a driver reads at + /// [`CONFIG`](crate::regs::CONFIG) before its first command. Fixed for + /// the life of the model. + fn config(&self) -> &[u8]; + + /// One command. `regions` is the chain's layout, `request` its readable + /// bytes and `response` its writable bytes, each as one run however the + /// driver split it. + /// + /// Returns the number of response bytes written, from the start of + /// `response`: what the driver is told the device used. Zero says the + /// chain could not carry a response at all, and nothing was written. + fn command( + &mut self, + regions: &[ChainRegion], + request: &dyn ReadableBytes, + response: &mut dyn WritableBytes, + ) -> u32; + + /// The model's own turn, taken after each command: work a command + /// admitted and answered runs here. Hardware does this by itself; a + /// model does it when the transport says so. Answers whether any ran. + fn drive(&mut self) -> bool; + + /// Whether the model can no longer answer truthfully. The transport + /// then stops taking commands and tells the driver to reset the device. + fn needs_reset(&self) -> bool; + + /// Forget everything a driver made and return to the first state. The + /// transport has stopped taking and completing chains before it calls + /// this, and nothing the driver held from before names anything after. + fn reset(&mut self) -> Result<(), Unrecoverable>; +} diff --git a/native/crates/vdev/src/driver.rs b/native/crates/vdev/src/driver.rs new file mode 100644 index 0000000..d6e3687 --- /dev/null +++ b/native/crates/vdev/src/driver.rs @@ -0,0 +1,624 @@ +//! **The driver's half**: the same registers and the same queue, from the +//! side that owns the memory. +//! +//! A driver written in C does these steps by hand against the layout in +//! [`regs`]. This is those steps in Rust, behind the driver-side queue +//! contract of `virtio-accel-transport` ([`DriverQueue`]), so the protocol's +//! reference guest client runs over a kore virtual device unchanged. +//! +//! # The memory a driver lends +//! +//! One window, laid out by [`Layout`]: the descriptor table, the available +//! ring and the used ring, then a fixed pool of command slots. A slot is two +//! descriptors and the bytes they name, room for one request the device +//! reads and one response it writes. A command is therefore always a chain +//! of two, the shortest the protocol allows, and a driver has as many +//! commands in flight as it has slots. +//! +//! A [`DriverChain`] is one slot, sized for one command. It is taken from +//! the pool ([`Slots::chain`]), filled, published on the ring, taken back +//! used and returned ([`Slots::recycle`]). It is not `Clone`: a slot has +//! one holder at a time, the driver or the device. The pool is apart from +//! the ring because the protocol's client owns the queue it is given and is +//! handed each chain by its caller. +//! +//! # Reset +//! +//! A driver that wants the device back in its first state calls +//! [`bring_up`] again and drops what the last one gave it. The reset is +//! the first step of a bring-up, and the rings are cleared after the device +//! has acknowledged it and before the device is told the queue is ready, so +//! neither side ever sees the other's indices from before. +//! +//! # What the driver trusts +//! +//! The device is the more trusted side, but a used entry is still checked +//! before it is believed: it must name a chain the driver published and has +//! not yet taken back, and a length no longer than the room that chain gave. + +use core::sync::atomic::{Ordering, fence}; + +use virtio_accel_transport::{ + ChainId, DriverChainBuffer, DriverQueue, NotificationHint, NotificationRecheck, PublishError, + PublishErrorKind, PublishedChain, QueueConfigError, QueueEpoch, QueueError, QueuePort, + QueueSize, QueueState, ReclaimedChain, UsedChain, UsedLength, +}; + +use crate::mem::{Memory, MemoryFault}; +use crate::regs::{ + self, F_VERSION_1, STATUS_ACKNOWLEDGE, STATUS_DEVICE_NEEDS_RESET, STATUS_DRIVER, + STATUS_DRIVER_OK, STATUS_FAILED, STATUS_FEATURES_OK, +}; +use crate::ring::{ + AVAIL_ALIGN, AVAIL_ENTRY, DESC_BYTES, MAX_QUEUE_SIZE, RING_FLAGS, RING_HEADER, RING_IDX, + RingAddresses, USED_ALIGN, USED_ENTRY, USED_LEN, VIRTQ_AVAIL_F_NO_INTERRUPT, VIRTQ_DESC_F_NEXT, + VIRTQ_DESC_F_WRITE, VIRTQ_USED_F_NO_NOTIFY, +}; + +/// Descriptors one command slot takes: its request and its response. +const SLOT_DESCRIPTORS: u16 = 2; + +/// The most command slots a queue has: half the largest queue. +pub const MAX_SLOTS: usize = (MAX_QUEUE_SIZE / SLOT_DESCRIPTORS) as usize; + +/// A slot's bytes start on a word, so a frame's fields are aligned for a +/// driver that reads them in place. +const SLOT_ALIGN: u64 = 8; + +/// What the device's half of the register block says it is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Identity { + /// The device model ([`regs::DEVICE_ID`]). + pub device_id: u32, + /// Who made it ([`regs::VENDOR_ID`]). + pub vendor_id: u32, + /// The features it offers. + pub features: u64, + /// The largest queue it takes. + pub queue_num_max: u32, + /// Bytes of device configuration at [`regs::CONFIG`]. + pub config_bytes: u32, +} + +/// Why a device could not be brought up. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BringUpError { + /// The block does not begin with [`regs::MAGIC_VALUE`]. + NotADevice, + /// The block's layout is a version this driver does not know. + Version(u32), + /// The device did not return to status zero when asked to reset. + Reset, + /// The device does not offer [`F_VERSION_1`], or refused it. + Features, + /// The layout does not fit: the queue is larger than the device takes, + /// or the memory is smaller than the layout. + Layout, + /// The device refused the queue. + Queue, + /// A register or the driver's own memory did not answer. + Memory(MemoryFault), +} + +impl From for BringUpError { + fn from(fault: MemoryFault) -> Self { + Self::Memory(fault) + } +} + +/// Read the device's identity, refusing a block that is not a kore virtual +/// device of a layout this driver knows. +pub fn identify(registers: &M) -> Result { + if registers.load_u32(regs::MAGIC)? != regs::MAGIC_VALUE { + return Err(BringUpError::NotADevice); + } + let version = registers.load_u32(regs::VERSION)?; + if version != regs::VERSION_VALUE { + return Err(BringUpError::Version(version)); + } + Ok(Identity { + device_id: registers.load_u32(regs::DEVICE_ID)?, + vendor_id: registers.load_u32(regs::VENDOR_ID)?, + features: registers.load_u64(regs::DEVICE_FEATURES)?, + queue_num_max: registers.load_u32(regs::QUEUE_NUM_MAX)?, + config_bytes: registers.load_u32(regs::CONFIG_BYTES)?, + }) +} + +/// Read the first `target.len()` bytes of the device configuration. +pub fn read_config(registers: &M, target: &mut [u8]) -> Result<(), BringUpError> { + if target.len() as u64 > u64::from(registers.load_u32(regs::CONFIG_BYTES)?) { + return Err(BringUpError::Layout); + } + Ok(registers.read(regs::CONFIG, target)?) +} + +/// How a driver lays out the memory it lends. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Layout { + size: QueueSize, + request_bytes: u32, + response_bytes: u32, +} + +impl Layout { + /// A queue of `size` descriptors, `size / 2` command slots, each with + /// room for a request of `request_bytes` and a response of + /// `response_bytes`. `None` when the queue is larger than a ring holds + /// or a slot has no room. + pub const fn new(size: QueueSize, request_bytes: u32, response_bytes: u32) -> Option { + if size.get() > MAX_QUEUE_SIZE + || size.get() < SLOT_DESCRIPTORS + || request_bytes == 0 + || response_bytes == 0 + { + return None; + } + Some(Self { + size, + request_bytes, + response_bytes, + }) + } + + /// The queue's size. + pub const fn size(&self) -> QueueSize { + self.size + } + + /// Command slots: commands the driver can have in flight. + pub const fn slots(&self) -> u16 { + self.size.get() / SLOT_DESCRIPTORS + } + + /// Where the queue's three parts are. + pub const fn addresses(&self) -> RingAddresses { + let size = self.size.get() as u64; + let available = (size * DESC_BYTES).next_multiple_of(AVAIL_ALIGN); + let used = (available + RING_HEADER + size * AVAIL_ENTRY).next_multiple_of(USED_ALIGN); + RingAddresses { + descriptors: 0, + available, + used, + } + } + + const fn slot_bytes(&self) -> u64 { + (self.request_bytes as u64).next_multiple_of(SLOT_ALIGN) + + (self.response_bytes as u64).next_multiple_of(SLOT_ALIGN) + } + + const fn slots_at(&self) -> u64 { + let size = self.size.get() as u64; + (self.addresses().used + RING_HEADER + size * USED_ENTRY).next_multiple_of(SLOT_ALIGN) + } + + /// Where `slot`'s request and response bytes are. + const fn slot(&self, slot: u16) -> (u64, u64) { + let request = self.slots_at() + slot as u64 * self.slot_bytes(); + ( + request, + request + (self.request_bytes as u64).next_multiple_of(SLOT_ALIGN), + ) + } + + /// Bytes of memory the layout takes. + pub const fn bytes(&self) -> u64 { + self.slots_at() + self.slots() as u64 * self.slot_bytes() + } +} + +/// Bring a device up: reset it, agree features, describe the queue and tell +/// it the driver is ready. `kick` rings the doorbell and waits for the +/// interrupt that answers it. +/// +/// The steps are Virtio 1.3 §3.1.1's. On a refusal the driver marks the +/// device [`STATUS_FAILED`], as that section requires, and says which step +/// was refused. +pub fn bring_up( + registers: &M, + memory: M, + layout: Layout, + mut kick: impl FnMut(), +) -> Result<(DriverRing, Slots), BringUpError> { + let identity = identify(registers)?; + let status = |value| registers.store_u32(regs::DRIVER_STATUS, value); + let answered = || registers.load_u32(regs::DEVICE_STATUS); + let fail = |error| { + // Best effort: the refusal is what the caller needs to hear. + let _ = status(STATUS_FAILED); + Err(error) + }; + + // A device in its reset state, which the driver has announced nothing + // to, has nothing to undo and answers no reset: there is nothing to + // wait for. Any other is reset first, and that is answered. + if registers.load_u32(regs::DRIVER_STATUS)? != 0 || answered()? != 0 { + status(0)?; + kick(); + if answered()? != 0 { + return Err(BringUpError::Reset); + } + } + let mut reached = STATUS_ACKNOWLEDGE | STATUS_DRIVER; + status(reached)?; + + if identity.features & F_VERSION_1 == 0 { + return fail(BringUpError::Features); + } + registers.store_u64(regs::DRIVER_FEATURES, F_VERSION_1)?; + reached |= STATUS_FEATURES_OK; + status(reached)?; + kick(); + if answered()? & STATUS_FEATURES_OK == 0 { + return fail(BringUpError::Features); + } + + if u32::from(layout.size.get()) > identity.queue_num_max || memory.len() < layout.bytes() { + return fail(BringUpError::Layout); + } + let ring = DriverRing::new(memory.clone(), layout)?; + let at = layout.addresses(); + registers.store_u32(regs::QUEUE_NUM, u32::from(layout.size.get()))?; + registers.store_u64(regs::QUEUE_DESC, at.descriptors)?; + registers.store_u64(regs::QUEUE_DRIVER, at.available)?; + registers.store_u64(regs::QUEUE_DEVICE, at.used)?; + registers.store_u32(regs::QUEUE_READY, 1)?; + reached |= STATUS_DRIVER_OK; + status(reached)?; + kick(); + let device = answered()?; + if device & STATUS_DRIVER_OK == 0 || device & STATUS_DEVICE_NEEDS_RESET != 0 { + return fail(BringUpError::Queue); + } + Ok((ring, Slots::new(memory, layout))) +} + +/// Why the driver's ring could not go on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DriverRingError { + /// A used entry names no chain the driver has with the device, or a + /// length longer than the chain's room. + UsedEntry, + /// The driver's own memory did not answer. + Memory(MemoryFault), +} + +impl From for QueueError { + fn from(fault: MemoryFault) -> Self { + Self::Transport(DriverRingError::Memory(fault)) + } +} + +/// One command slot, sized for one command: `readable` bytes the driver +/// writes and the device reads, `writable` bytes the device writes. +#[derive(Debug)] +pub struct DriverChain { + memory: M, + slot: u16, + request: u64, + readable: u32, + response: u64, + writable: u32, +} + +impl DriverChainBuffer for DriverChain { + type Error = MemoryFault; + + fn device_readable_len(&self) -> u64 { + u64::from(self.readable) + } + + fn device_writable_len(&self) -> u64 { + u64::from(self.writable) + } + + fn write_device_readable(&mut self, offset: u64, source: &[u8]) -> Result<(), MemoryFault> { + within(offset, source.len(), self.readable)?; + self.memory.write(self.request + offset, source) + } + + fn read_device_writable(&self, offset: u64, target: &mut [u8]) -> Result<(), MemoryFault> { + within(offset, target.len(), self.writable)?; + self.memory.read(self.response + offset, target) + } +} + +fn within(offset: u64, len: usize, room: u32) -> Result<(), MemoryFault> { + if offset + .checked_add(len as u64) + .is_some_and(|end| end <= u64::from(room)) + { + Ok(()) + } else { + Err(MemoryFault::Range) + } +} + +/// The pool of command slots in the driver's memory. +#[derive(Debug)] +pub struct Slots { + memory: M, + layout: Layout, + /// Slots no chain holds, one bit each. + free: u32, +} + +impl Slots { + fn new(memory: M, layout: Layout) -> Self { + let slots = u32::from(layout.slots()); + Self { + memory, + layout, + free: if slots == u32::BITS { + u32::MAX + } else { + (1 << slots) - 1 + }, + } + } + + /// Slots no chain holds. + pub const fn free(&self) -> u32 { + self.free.count_ones() + } + + /// A free slot sized for a request of `readable` bytes and a response of + /// up to `writable`. `None` when every slot is taken, or the sizes are + /// zero or more than a slot has room for. + pub fn chain(&mut self, readable: u32, writable: u32) -> Option> { + if readable == 0 + || writable == 0 + || readable > self.layout.request_bytes + || writable > self.layout.response_bytes + || self.free == 0 + { + return None; + } + let slot = self.free.trailing_zeros() as u16; + self.free &= !(1 << slot); + let (request, response) = self.layout.slot(slot); + Some(DriverChain { + memory: self.memory.clone(), + slot, + request, + readable, + response, + writable, + }) + } + + /// Return a chain's slot: one the device has used, or one that was + /// never published. + pub fn recycle(&mut self, chain: DriverChain) { + self.free |= 1 << chain.slot; + } +} + +/// The driver's half of one split virtqueue in its own memory. +#[derive(Debug)] +pub struct DriverRing { + memory: M, + layout: Layout, + state: QueueState, + /// Available entries published, and used entries taken back. + published: u16, + taken: u16, + /// The chains the device has, by slot. + held: [Option>; MAX_SLOTS], +} + +impl DriverRing { + /// A ready queue laid out by `layout` in `memory`, with both rings + /// cleared: the memory may have held a queue before a reset. + fn new(memory: M, layout: Layout) -> Result { + let at = layout.addresses(); + for ring in [at.available, at.used] { + memory.store_u16(ring + RING_FLAGS, 0)?; + memory.store_u16(ring + RING_IDX, 0)?; + } + Ok(Self { + memory, + layout, + state: ready(layout.size, QueueEpoch::INITIAL), + published: 0, + taken: 0, + held: [const { None }; MAX_SLOTS], + }) + } + + /// The layout the queue was made with. + pub const fn layout(&self) -> Layout { + self.layout + } + + fn descriptor( + &self, + index: u16, + at: u64, + len: u32, + flags: u16, + next: u16, + ) -> Result<(), MemoryFault> { + let mut bytes = [0u8; DESC_BYTES as usize]; + bytes[..8].copy_from_slice(&at.to_le_bytes()); + bytes[8..12].copy_from_slice(&len.to_le_bytes()); + bytes[12..14].copy_from_slice(&flags.to_le_bytes()); + bytes[14..].copy_from_slice(&next.to_le_bytes()); + self.memory.write( + self.layout.addresses().descriptors + u64::from(index) * DESC_BYTES, + &bytes, + ) + } + + /// Write the chain's two descriptors and publish its head. + fn expose(&mut self, chain: &DriverChain) -> Result { + let at = self.layout.addresses(); + let head = chain.slot * SLOT_DESCRIPTORS; + self.descriptor( + head, + chain.request, + chain.readable, + VIRTQ_DESC_F_NEXT, + head + 1, + )?; + self.descriptor( + head + 1, + chain.response, + chain.writable, + VIRTQ_DESC_F_WRITE, + 0, + )?; + let entry = u64::from(self.published % self.layout.size.get()); + self.memory.write( + at.available + RING_HEADER + entry * AVAIL_ENTRY, + &head.to_le_bytes(), + )?; + self.published = self.published.wrapping_add(1); + // Release: the request bytes, the descriptors and the entry are + // visible to a device that has loaded this index. + self.memory + .store_u16(at.available + RING_IDX, self.published)?; + // The index is published before the device's suppression flag is + // read: a device that clears the flag and rechecks the index cannot + // miss this chain. + fence(Ordering::SeqCst); + Ok( + if self.memory.load_u16(at.used + RING_FLAGS)? & VIRTQ_USED_F_NO_NOTIFY == 0 { + NotificationHint::Notify + } else { + NotificationHint::Suppressed + }, + ) + } +} + +fn ready(size: QueueSize, epoch: QueueEpoch) -> QueueState { + // A configured size no larger than the maximum, which is itself: the + // one shape `QueueState::new` refuses is not this one. + QueueState::new(size, Some(size), true, epoch) + .unwrap_or_else(|_| QueueState::unconfigured(size, epoch)) +} + +impl QueuePort for DriverRing { + fn state(&self) -> QueueState { + self.state + } +} + +impl DriverQueue for DriverRing { + type Chain = DriverChain; + type Reclaimed = heapless::Vec>, MAX_SLOTS>; + type Error = DriverRingError; + + fn publish( + &mut self, + chain: Self::Chain, + ) -> Result> { + if !self.state.ready() { + return Err(PublishError::new(chain, PublishErrorKind::NotReady)); + } + let slot = usize::from(chain.slot); + if self.held[slot].is_some() { + // The slot's descriptors are the device's until it uses them. + return Err(PublishError::new(chain, PublishErrorKind::QueueFull)); + } + match self.expose(&chain) { + Ok(hint) => { + let id = ChainId::new(self.state.epoch(), u64::from(chain.slot * SLOT_DESCRIPTORS)); + self.held[slot] = Some(chain); + Ok(PublishedChain::new(id, hint)) + } + Err(fault) => Err(PublishError::new( + chain, + PublishErrorKind::Transport(DriverRingError::Memory(fault)), + )), + } + } + + fn pop_used(&mut self) -> Result>, QueueError> { + if !self.state.ready() { + return Err(QueueError::NotReady); + } + let at = self.layout.addresses(); + // Acquire: the entry and the response bytes the device wrote before + // it published this index are visible to the reads below. + if self.memory.load_u16(at.used + RING_IDX)? == self.taken { + return Ok(None); + } + let entry = + at.used + RING_HEADER + u64::from(self.taken % self.layout.size.get()) * USED_ENTRY; + let head = self.memory.load_u32(entry)?; + let len = self.memory.load_u32(entry + USED_LEN)?; + self.taken = self.taken.wrapping_add(1); + + let slot = head / u32::from(SLOT_DESCRIPTORS); + let chain = if head % u32::from(SLOT_DESCRIPTORS) == 0 { + self.held + .get_mut(slot as usize) + .and_then(Option::take) + .filter(|chain| len <= chain.writable) + } else { + None + }; + let Some(chain) = chain else { + return Err(QueueError::Transport(DriverRingError::UsedEntry)); + }; + Ok(Some(UsedChain::new( + ChainId::new(self.state.epoch(), u64::from(head)), + UsedLength::new(len), + chain, + ))) + } + + fn disable_used_notifications(&mut self) -> Result<(), QueueError> { + self.memory.store_u16( + self.layout.addresses().available + RING_FLAGS, + VIRTQ_AVAIL_F_NO_INTERRUPT, + )?; + Ok(()) + } + + fn enable_used_notifications( + &mut self, + ) -> Result> { + let at = self.layout.addresses(); + self.memory.store_u16(at.available + RING_FLAGS, 0)?; + // The flag is cleared before the index is read: a device that used + // a chain while interrupts were off is seen here, and one that uses + // a chain after sees the flag clear and interrupts. + fence(Ordering::SeqCst); + Ok(if self.memory.load_u16(at.used + RING_IDX)? == self.taken { + NotificationRecheck::Idle + } else { + NotificationRecheck::WorkPending + }) + } + + fn reset( + &mut self, + next_epoch: QueueEpoch, + ) -> Result> { + if next_epoch <= self.state.epoch() { + return Err(QueueError::InvalidConfiguration( + QueueConfigError::NonIncreasingEpoch, + )); + } + let epoch = self.state.epoch(); + let mut reclaimed = heapless::Vec::new(); + for (slot, held) in self.held.iter_mut().enumerate() { + if let Some(chain) = held.take() { + let head = slot as u64 * u64::from(SLOT_DESCRIPTORS); + // One entry a slot at most, and the vector holds every slot. + let _ = reclaimed.push(ReclaimedChain::new(ChainId::new(epoch, head), chain)); + } + } + // Unconfigured, and it stays so: the queue a driver uses after a + // reset is the one the next `bring_up` makes. + self.state = QueueState::unconfigured(self.layout.size, next_epoch); + Ok(reclaimed) + } +} + +#[cfg(test)] +mod tests; diff --git a/native/crates/vdev/src/driver/tests.rs b/native/crates/vdev/src/driver/tests.rs new file mode 100644 index 0000000..b23cb95 --- /dev/null +++ b/native/crates/vdev/src/driver/tests.rs @@ -0,0 +1,392 @@ +use super::*; +use crate::device::VirtualDevice; +use crate::mem::{Window, tests::window}; +use crate::transport::Transport; +use crate::transport::tests::{ECHO_CONFIG, Echo}; +use core::num::NonZeroU32; +use std::vec; +use std::vec::Vec; + +const BUDGET: NonZeroU32 = NonZeroU32::new(16).unwrap(); + +fn layout() -> Layout { + Layout::new(QueueSize::new(8).unwrap(), 64, 32).unwrap() +} + +/// A device and the two windows a driver is given to it. +struct Bench { + registers: Window, + memory: Window, + transport: Transport, +} + +impl Bench { + fn new() -> Self { + let registers = window(regs::BYTES as usize); + let memory = window(layout().bytes() as usize); + let mut transport = Transport::new(registers, Echo::default()).unwrap(); + transport.lend(memory).unwrap(); + Self { + registers, + memory, + transport, + } + } + + fn bring_up(&mut self) -> Result<(DriverRing, Slots), BringUpError> { + self.bring_up_with(layout()) + } + + fn bring_up_with( + &mut self, + layout: Layout, + ) -> Result<(DriverRing, Slots), BringUpError> { + let transport = &mut self.transport; + bring_up(&self.registers, self.memory, layout, || { + transport.service(BUDGET); + }) + } + + /// Send `request` with `room` for the answer and take the answer back. + fn exchange( + &mut self, + ring: &mut DriverRing, + slots: &mut Slots, + request: &[u8], + room: u32, + ) -> Vec { + let mut chain = slots.chain(request.len() as u32, room).expect("a slot"); + chain.write_device_readable(0, request).unwrap(); + let published = ring.publish(chain).expect("published"); + assert_eq!(published.notification(), NotificationHint::Notify); + assert!(self.transport.service(BUDGET).interrupt); + let used = ring.pop_used().unwrap().expect("used"); + assert_eq!(used.id(), published.id()); + let (_, len, chain) = used.into_parts(); + let mut answer = vec![0u8; len.get() as usize]; + chain.read_device_writable(0, &mut answer).unwrap(); + slots.recycle(chain); + answer + } +} + +#[test] +fn a_layout_keeps_its_parts_apart_and_aligned() { + for size in [2u16, 4, 8, 16, 32, 64] { + for (request, response) in [(1u32, 1u32), (64, 32), (65_536, 65_536), (17, 4093)] { + let layout = Layout::new(QueueSize::new(size).unwrap(), request, response).unwrap(); + let at = layout.addresses(); + let size = u64::from(size); + assert_eq!(at.descriptors % 16, 0); + assert_eq!(at.available % 2, 0); + assert_eq!(at.used % 4, 0); + assert!(at.descriptors + size * 16 <= at.available); + assert!(at.available + 4 + size * 2 <= at.used); + let mut end = at.used + 4 + size * 8; + for slot in 0..layout.slots() { + let (request_at, response_at) = layout.slot(slot); + assert!(end <= request_at, "slot {slot} begins after what precedes"); + assert_eq!(request_at % 8, 0); + assert_eq!(response_at % 8, 0); + assert!(request_at + u64::from(request) <= response_at); + end = response_at + u64::from(response); + } + assert!(end <= layout.bytes()); + assert_eq!(layout.slots(), (size / 2) as u16); + } + } + let size = |size| QueueSize::new(size).unwrap(); + assert!(Layout::new(size(1), 8, 8).is_none(), "no room for a chain"); + assert!( + Layout::new(size(128), 8, 8).is_none(), + "more than a ring holds" + ); + assert!(Layout::new(size(8), 0, 8).is_none()); + assert!(Layout::new(size(8), 8, 0).is_none()); +} + +#[test] +fn a_driver_identifies_brings_up_and_exchanges() { + let mut bench = Bench::new(); + let identity = identify(&bench.registers).unwrap(); + assert_eq!( + identity, + Identity { + device_id: Echo::DEVICE_ID, + vendor_id: regs::VENDOR_KORE, + features: F_VERSION_1, + queue_num_max: 8, + config_bytes: ECHO_CONFIG.len() as u32, + } + ); + let mut config = [0u8; ECHO_CONFIG.len()]; + read_config(&bench.registers, &mut config).unwrap(); + assert_eq!(config, ECHO_CONFIG); + assert_eq!( + read_config(&bench.registers, &mut [0u8; ECHO_CONFIG.len() + 1]), + Err(BringUpError::Layout), + "no more than the device has" + ); + + let (mut ring, mut slots) = bench.bring_up().unwrap(); + assert!(ring.state().ready()); + assert_eq!(ring.layout(), layout()); + assert_eq!(slots.free(), 4); + assert_eq!( + bench.transport.status(), + STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK | STATUS_DRIVER_OK + ); + assert_eq!(bench.transport.features(), F_VERSION_1); + + assert_eq!( + bench.exchange(&mut ring, &mut slots, b"hello", 32), + b"olleh" + ); + assert_eq!(bench.exchange(&mut ring, &mut slots, b"device", 3), b"eci"); + // Enough exchanges to take both rings' indices round the queue. + for round in 0..40u8 { + let request = [round, round.wrapping_mul(7), 3]; + let answer = bench.exchange(&mut ring, &mut slots, &request, 8); + assert_eq!(answer, [3, round.wrapping_mul(7), round]); + } + assert_eq!(slots.free(), 4); + assert_eq!(bench.transport.device().commands, 42); +} + +#[test] +fn a_slot_has_one_holder() { + let mut bench = Bench::new(); + let (mut ring, mut slots) = bench.bring_up().unwrap(); + + assert!(slots.chain(0, 8).is_none()); + assert!(slots.chain(8, 0).is_none()); + assert!(slots.chain(65, 8).is_none(), "more than a slot's request"); + assert!(slots.chain(8, 33).is_none(), "more than a slot's response"); + assert_eq!(slots.free(), 4, "a refusal takes nothing"); + + let mut chains: Vec<_> = (0..4).map(|_| slots.chain(64, 32).unwrap()).collect(); + assert!(slots.chain(1, 1).is_none(), "every slot is taken"); + + // A chain's bytes are its slot's and no other's. + for (index, chain) in chains.iter_mut().enumerate() { + chain.write_device_readable(0, &[index as u8; 64]).unwrap(); + assert_eq!( + chain.write_device_readable(1, &[0; 64]), + Err(MemoryFault::Range) + ); + assert_eq!( + chain.read_device_writable(30, &mut [0; 3]), + Err(MemoryFault::Range) + ); + assert_eq!(chain.device_readable_len(), 64); + assert_eq!(chain.device_writable_len(), 32); + } + // All four in flight at once, answered in order. + let ids: Vec<_> = chains + .drain(..) + .map(|chain| ring.publish(chain).unwrap().id()) + .collect(); + bench.transport.service(BUDGET); + for (index, id) in ids.iter().enumerate() { + let (used, len, chain) = ring.pop_used().unwrap().unwrap().into_parts(); + assert_eq!(used, *id); + assert_eq!(len.get(), 32); + let mut answer = [0u8; 32]; + chain.read_device_writable(0, &mut answer).unwrap(); + assert_eq!(answer, [index as u8; 32]); + slots.recycle(chain); + } + assert!(ring.pop_used().unwrap().is_none()); + assert_eq!(slots.free(), 4); +} + +/// VQ-017 from the driver's side: suppression in either direction loses +/// nothing. +#[test] +fn suppressed_notifications_lose_no_work() { + let mut bench = Bench::new(); + let (mut ring, mut slots) = bench.bring_up().unwrap(); + let one = NonZeroU32::new(1).unwrap(); + + // The driver asks not to be interrupted, and is not. + ring.disable_used_notifications().unwrap(); + let mut chain = slots.chain(2, 8).unwrap(); + chain.write_device_readable(0, b"ab").unwrap(); + ring.publish(chain).unwrap(); + assert!(!bench.transport.service(BUDGET).interrupt); + // Enabling them again finds what was used meanwhile. + assert_eq!( + ring.enable_used_notifications().unwrap(), + NotificationRecheck::WorkPending + ); + let (_, len, chain) = ring.pop_used().unwrap().unwrap().into_parts(); + assert_eq!(len.get(), 2); + slots.recycle(chain); + assert_eq!( + ring.enable_used_notifications().unwrap(), + NotificationRecheck::Idle + ); + + // The device stops listening while its budget is spent: a chain + // published then needs no doorbell, and is served all the same. + for _ in 0..2 { + let mut chain = slots.chain(2, 8).unwrap(); + chain.write_device_readable(0, b"cd").unwrap(); + ring.publish(chain).unwrap(); + } + assert!(bench.transport.service(one).pending); + let mut chain = slots.chain(2, 8).unwrap(); + chain.write_device_readable(0, b"ef").unwrap(); + assert_eq!( + ring.publish(chain).unwrap().notification(), + NotificationHint::Suppressed + ); + while bench.transport.service(one).pending {} + let answers: Vec<_> = core::iter::from_fn(|| ring.pop_used().unwrap()) + .map(|used| { + let (_, _, chain) = used.into_parts(); + let mut answer = [0u8; 2]; + chain.read_device_writable(0, &mut answer).unwrap(); + slots.recycle(chain); + answer + }) + .collect(); + assert_eq!(answers, [*b"dc", *b"dc", *b"fe"]); +} + +#[test] +fn a_used_entry_is_checked_before_it_is_believed() { + // Each forgery: the entry the device's side of the memory is made to + // hold, as `(head, length)`. + for (what, head, len) in [ + ("a head never published", 2u32, 1u32), + ("a slot's second descriptor", 1, 1), + ("no descriptor at all", 4000, 1), + ("more than the room given", 0, 9), + ] { + let mut bench = Bench::new(); + let (mut ring, mut slots) = bench.bring_up().unwrap(); + let mut chain = slots.chain(2, 8).unwrap(); + chain.write_device_readable(0, b"ab").unwrap(); + ring.publish(chain).unwrap(); + + let used = layout().addresses().used; + bench.memory.store_u32(used + 4, head).unwrap(); + bench.memory.store_u32(used + 8, len).unwrap(); + bench.memory.store_u16(used + 2, 1).unwrap(); + assert_eq!( + ring.pop_used().map(|used| used.is_some()), + Err(QueueError::Transport(DriverRingError::UsedEntry)), + "{what}" + ); + } +} + +#[test] +fn a_refused_bring_up_says_which_step() { + // Not a device at all. + let blank = window(regs::BYTES as usize); + let memory = window(layout().bytes() as usize); + let bring = |registers: &Window| bring_up(registers, memory, layout(), || {}).map(|_| ()); + assert_eq!(bring(&blank), Err(BringUpError::NotADevice)); + blank.store_u32(regs::MAGIC, regs::MAGIC_VALUE).unwrap(); + blank.store_u32(regs::VERSION, 2).unwrap(); + assert_eq!(bring(&blank), Err(BringUpError::Version(2))); + + // A device that does not answer a reset. + blank.store_u32(regs::VERSION, regs::VERSION_VALUE).unwrap(); + blank + .store_u32(regs::DEVICE_STATUS, STATUS_DRIVER_OK) + .unwrap(); + assert_eq!(bring(&blank), Err(BringUpError::Reset)); + + // One that offers nothing a driver can accept, and one that never + // answers the features it was sent. The driver gives up on both. + blank.store_u32(regs::DEVICE_STATUS, 0).unwrap(); + assert_eq!(bring(&blank), Err(BringUpError::Features)); + assert_eq!(blank.load_u32(regs::DRIVER_STATUS), Ok(STATUS_FAILED)); + blank.store_u64(regs::DEVICE_FEATURES, F_VERSION_1).unwrap(); + assert_eq!(bring(&blank), Err(BringUpError::Features)); + assert_eq!(blank.load_u32(regs::DRIVER_STATUS), Ok(STATUS_FAILED)); + + // A queue larger than the device takes, and memory smaller than the + // layout. + let mut bench = Bench::new(); + let wide = Layout::new(QueueSize::new(16).unwrap(), 8, 8).unwrap(); + assert_eq!( + bench.bring_up_with(wide).map(|_| ()), + Err(BringUpError::Layout) + ); + let deep = Layout::new(QueueSize::new(8).unwrap(), 4096, 4096).unwrap(); + assert_eq!( + bench.bring_up_with(deep).map(|_| ()), + Err(BringUpError::Layout) + ); + assert_eq!( + bench.registers.load_u32(regs::DRIVER_STATUS), + Ok(STATUS_FAILED) + ); + // The device heard the driver give up, and a new bring-up starts over. + bench.transport.service(BUDGET); + assert_ne!(bench.transport.status() & STATUS_FAILED, 0); + bench.bring_up().expect("after a reset"); + + // A queue the device refuses: its chains are longer than a queue of + // two can hold. + let mut bench = Bench::new(); + let short = Layout::new(QueueSize::new(2).unwrap(), 8, 8).unwrap(); + assert_eq!( + bench.bring_up_with(short).map(|_| ()), + Err(BringUpError::Queue) + ); + assert_ne!(bench.transport.status() & STATUS_DEVICE_NEEDS_RESET, 0); + bench.bring_up().expect("after a reset"); +} + +/// A second bring-up is a reset: the device forgets, both rings start from +/// nothing, and what the first one gave is dead. +#[test] +fn bringing_up_again_resets_the_device() { + let mut bench = Bench::new(); + let (mut ring, mut slots) = bench.bring_up().unwrap(); + for _ in 0..3 { + bench.exchange(&mut ring, &mut slots, b"before", 8); + } + // One left with the device across the reset. + let mut chain = slots.chain(4, 8).unwrap(); + chain.write_device_readable(0, b"lost").unwrap(); + ring.publish(chain).unwrap(); + + let (mut again, mut pool) = bench.bring_up().unwrap(); + assert_eq!(bench.transport.device().resets, 1); + assert_eq!( + bench.transport.device().commands, + 3, + "the lost one never ran" + ); + assert_eq!(pool.free(), 4); + assert!(again.pop_used().unwrap().is_none(), "nothing from before"); + assert_eq!(bench.exchange(&mut again, &mut pool, b"after", 8), b"retfa"); + + // The old ring gives its chain back and serves no more. + let epoch = ring.state().epoch(); + assert_eq!( + ring.reset(epoch).map(|_| ()), + Err(QueueError::InvalidConfiguration( + QueueConfigError::NonIncreasingEpoch + )) + ); + let reclaimed = ring.reset(epoch.checked_next().unwrap()).unwrap(); + assert_eq!(reclaimed.len(), 1); + let (id, chain) = reclaimed.into_iter().next().unwrap().into_parts(); + assert_eq!(id.epoch(), epoch); + assert_eq!(chain.device_readable_len(), 4); + assert!(!ring.state().ready()); + assert_eq!( + ring.pop_used().map(|used| used.is_some()), + Err(QueueError::NotReady) + ); + let (returned, kind) = ring.publish(chain).unwrap_err().into_parts(); + assert_eq!(kind, PublishErrorKind::NotReady); + assert_eq!(returned.device_readable_len(), 4, "handed back unchanged"); +} diff --git a/native/crates/vdev/src/lib.rs b/native/crates/vdev/src/lib.rs new file mode 100644 index 0000000..29924f3 --- /dev/null +++ b/native/crates/vdev/src/lib.rs @@ -0,0 +1,43 @@ +//! # kore-vdev: virtual devices +//! +//! A virtual device is a device with no hardware behind it, hosted by a +//! userland service. A driver reaches it the way it reaches hardware, and +//! each of the three things a driver holds is ordinary shared memory or an +//! ordinary notice: +//! +//! - **registers** are a lease both sides map: the driver reads and writes +//! them as it would a BAR; +//! - **DMA** is a lease the driver owns and lends to the device. A device +//! address is an offset into it, and the device can touch nothing outside +//! it ([`mem`]); +//! - **an interrupt** is a notice the driver parks on. +//! +//! No kernel mechanism is added for any of it, and nothing is emulated: the +//! mappings are real, so the MMU is what a driver's access goes through. +//! +//! What this crate holds is the part every virtual device shares: +//! +//! - [`mem`], the device's view of memory it shares with a driver; +//! - [`regs`], the register block a driver maps and steps through; +//! - [`ring`], the split virtqueue laid out in the driver's memory; +//! - [`transport`], which runs the three for one driver on each doorbell; +//! - [`device`], the contract a device model implements. A model is a +//! separate crate (`kore-vdev-accel` is the first) and sees none of the +//! above: it is handed a request to read and a response to write; +//! - [`driver`], the driver's half of the same registers and queue, for a +//! driver written in Rust. A driver in C writes those steps by hand +//! against the layout in [`regs`]. + +#![no_std] + +extern crate alloc; + +pub mod device; +pub mod driver; +pub mod mem; +pub mod regs; +pub mod ring; +pub mod transport; + +#[cfg(test)] +extern crate std; diff --git a/native/crates/vdev/src/mem.rs b/native/crates/vdev/src/mem.rs new file mode 100644 index 0000000..3312483 --- /dev/null +++ b/native/crates/vdev/src/mem.rs @@ -0,0 +1,294 @@ +//! **The device's view of the driver's memory.** +//! +//! A driver lends a virtual device one window of memory and names places in +//! it by offset: the offset is the *device address*, the only address a +//! driver ever hands a device. The device reaches the window through +//! [`Memory`], which refuses anything outside it, so a driver's mistake, or +//! its malice, stays inside what it lent. +//! +//! The other side of the window is another task that may write any byte at +//! any moment. Nothing here ever forms a Rust reference into it: every +//! access is an atomic load or store, a snapshot the device then validates. +//! A snapshot is not coherent across words; the ring's publication points +//! ([`Memory::load_u16`], [`Memory::store_u16`]) are what order one side's +//! writes before the other's reads. + +use core::ptr::NonNull; +use core::sync::atomic::{AtomicU8, AtomicU16, AtomicU32, AtomicU64, Ordering}; + +/// Why a device address did not name memory the driver lent. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MemoryFault { + /// Some byte of the range is outside the window. + Range, + /// A word access named an address that is not a multiple of its size. + Alignment, +} + +/// A window of driver memory, addressed from zero. +pub trait Memory { + /// Bytes in the window: device addresses are `0..len`. + fn len(&self) -> u64; + + /// Whether the window is empty. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Whether `len` bytes at `at` lie inside the window. + fn contains(&self, at: u64, len: u64) -> bool { + at.checked_add(len).is_some_and(|end| end <= self.len()) + } + + /// Snapshot `target.len()` bytes at `at`. + fn read(&self, at: u64, target: &mut [u8]) -> Result<(), MemoryFault>; + + /// Store `source` at `at`. + fn write(&self, at: u64, source: &[u8]) -> Result<(), MemoryFault>; + + /// Load the little-endian `u16` at `at` with acquire ordering: every + /// write the other side made before it published this word is visible + /// to reads after this one. + fn load_u16(&self, at: u64) -> Result; + + /// Store a little-endian `u16` at `at` with release ordering: every + /// write this side made before it is visible to the other side once it + /// has loaded this word. + fn store_u16(&self, at: u64, value: u16) -> Result<(), MemoryFault>; + + /// Load the little-endian `u32` at `at`, acquire. + fn load_u32(&self, at: u64) -> Result; + + /// Store a little-endian `u32` at `at`, release. + fn store_u32(&self, at: u64, value: u32) -> Result<(), MemoryFault>; + + /// Load the little-endian `u64` at `at`, acquire. + fn load_u64(&self, at: u64) -> Result; + + /// Store a little-endian `u64` at `at`, release. + fn store_u64(&self, at: u64, value: u64) -> Result<(), MemoryFault>; +} + +/// A mapped window: `len` bytes at `base` in this task, shared with a driver. +/// +/// A plain view, copied freely: it owns nothing, and whoever made it keeps +/// the mapping alive ([`Window::new`]). +#[derive(Clone, Copy, Debug)] +pub struct Window { + base: NonNull, + len: usize, +} + +impl Window { + /// A view of `len` bytes at `base`. + /// + /// # Safety + /// `base` is aligned to eight bytes and `len` bytes from it are mapped + /// readable and writable in this task for as long as this view or any + /// copy of it is used. The bytes may be written by another task at any + /// time; nothing else in this task holds a Rust reference into them. + pub const unsafe fn new(base: NonNull, len: usize) -> Self { + Self { base, len } + } + + /// Where the window is mapped in this task. A host uses it to name a + /// word of the window to the kernel, never to form a reference. + pub const fn base(&self) -> NonNull { + self.base + } + + fn range(&self, at: u64, len: usize) -> Result<*mut u8, MemoryFault> { + if !self.contains(at, len as u64) { + return Err(MemoryFault::Range); + } + // SAFETY: `at + len <= self.len`, so the offset stays inside the + // mapping `new` was promised. + Ok(unsafe { self.base.as_ptr().add(at as usize) }) + } + + fn word(&self, at: u64, size: usize) -> Result<*mut u8, MemoryFault> { + if at % size as u64 != 0 { + return Err(MemoryFault::Alignment); + } + // `base` is eight-byte aligned, so an offset aligned to `size` is an + // address aligned to it. + self.range(at, size) + } +} + +/// Bytes moved a word at a time once both sides are word aligned. +const WORD: usize = core::mem::size_of::(); + +impl Memory for Window { + fn len(&self) -> u64 { + self.len as u64 + } + + fn read(&self, at: u64, target: &mut [u8]) -> Result<(), MemoryFault> { + let source = self.range(at, target.len())?; + // Each load is atomic, so a concurrent writer in the driver cannot + // make this a data race; what it can do is change the bytes, which + // is why the caller validates the copy and never the window. + let mut index = 0; + // SAFETY (all three loops): `source.add(index)` is inside the range + // checked above for every `index < target.len()`. + while index < target.len() && (source as usize + index) % WORD != 0 { + target[index] = + unsafe { AtomicU8::from_ptr(source.add(index)) }.load(Ordering::Relaxed); + index += 1; + } + while index + WORD <= target.len() { + let word = + unsafe { AtomicU64::from_ptr(source.add(index).cast()) }.load(Ordering::Relaxed); + target[index..index + WORD].copy_from_slice(&word.to_ne_bytes()); + index += WORD; + } + while index < target.len() { + target[index] = + unsafe { AtomicU8::from_ptr(source.add(index)) }.load(Ordering::Relaxed); + index += 1; + } + Ok(()) + } + + fn write(&self, at: u64, source: &[u8]) -> Result<(), MemoryFault> { + let target = self.range(at, source.len())?; + let mut index = 0; + // SAFETY (all three loops): as in `read`. + while index < source.len() && (target as usize + index) % WORD != 0 { + unsafe { AtomicU8::from_ptr(target.add(index)) } + .store(source[index], Ordering::Relaxed); + index += 1; + } + while index + WORD <= source.len() { + let mut word = [0u8; WORD]; + word.copy_from_slice(&source[index..index + WORD]); + unsafe { AtomicU64::from_ptr(target.add(index).cast()) } + .store(u64::from_ne_bytes(word), Ordering::Relaxed); + index += WORD; + } + while index < source.len() { + unsafe { AtomicU8::from_ptr(target.add(index)) } + .store(source[index], Ordering::Relaxed); + index += 1; + } + Ok(()) + } + + fn load_u16(&self, at: u64) -> Result { + let word = self.word(at, core::mem::size_of::())?; + // SAFETY: in range and aligned for a `u16`. + Ok(u16::from_le( + unsafe { AtomicU16::from_ptr(word.cast()) }.load(Ordering::Acquire), + )) + } + + fn store_u16(&self, at: u64, value: u16) -> Result<(), MemoryFault> { + let word = self.word(at, core::mem::size_of::())?; + // SAFETY: in range and aligned for a `u16`. + unsafe { AtomicU16::from_ptr(word.cast()) }.store(value.to_le(), Ordering::Release); + Ok(()) + } + + fn load_u32(&self, at: u64) -> Result { + let word = self.word(at, core::mem::size_of::())?; + // SAFETY: in range and aligned for a `u32`. + Ok(u32::from_le( + unsafe { AtomicU32::from_ptr(word.cast()) }.load(Ordering::Acquire), + )) + } + + fn store_u32(&self, at: u64, value: u32) -> Result<(), MemoryFault> { + let word = self.word(at, core::mem::size_of::())?; + // SAFETY: in range and aligned for a `u32`. + unsafe { AtomicU32::from_ptr(word.cast()) }.store(value.to_le(), Ordering::Release); + Ok(()) + } + + fn load_u64(&self, at: u64) -> Result { + let word = self.word(at, core::mem::size_of::())?; + // SAFETY: in range and aligned for a `u64`. + Ok(u64::from_le( + unsafe { AtomicU64::from_ptr(word.cast()) }.load(Ordering::Acquire), + )) + } + + fn store_u64(&self, at: u64, value: u64) -> Result<(), MemoryFault> { + let word = self.word(at, core::mem::size_of::())?; + // SAFETY: in range and aligned for a `u64`. + unsafe { AtomicU64::from_ptr(word.cast()) }.store(value.to_le(), Ordering::Release); + Ok(()) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use std::boxed::Box; + use std::vec; + + /// A window over host memory that lives for the test run. + pub(crate) fn window(bytes: usize) -> Window { + let words: &'static mut [u64] = Box::leak(vec![0u64; bytes.div_ceil(8)].into_boxed_slice()); + // SAFETY: a leaked, eight-byte aligned allocation of at least + // `bytes` bytes that nothing else references. + unsafe { Window::new(NonNull::new(words.as_mut_ptr().cast()).unwrap(), bytes) } + } + + #[test] + fn a_window_reaches_exactly_what_was_lent() { + let memory = window(4096); + assert_eq!(memory.len(), 4096); + assert!(memory.contains(4090, 6)); + assert!(!memory.contains(4090, 7)); + assert!(!memory.contains(u64::MAX, 2), "the end does not wrap"); + + let mut out = [0u8; 4]; + assert_eq!(memory.read(4093, &mut out), Err(MemoryFault::Range)); + assert_eq!(memory.write(4096, &[1]), Err(MemoryFault::Range)); + assert_eq!(memory.write(4096, &[]), Ok(()), "nothing, at the end"); + assert_eq!(memory.load_u16(4095), Err(MemoryFault::Alignment)); + assert_eq!(memory.load_u16(4096), Err(MemoryFault::Range)); + assert_eq!(memory.store_u32(2, 0), Err(MemoryFault::Alignment)); + assert_eq!(memory.load_u32(4096), Err(MemoryFault::Range)); + assert_eq!(memory.load_u64(4), Err(MemoryFault::Alignment)); + assert_eq!(memory.store_u64(4096, 0), Err(MemoryFault::Range)); + } + + /// Every alignment of a copy takes the byte head, the word body and the + /// byte tail in some mix, and all of them move the same bytes. + #[test] + fn a_copy_moves_the_same_bytes_at_every_alignment() { + let memory = window(256); + let pattern: std::vec::Vec = (0..64u8).map(|i| i.wrapping_mul(37) ^ 0x5a).collect(); + for at in 0..17u64 { + for len in [0usize, 1, 7, 8, 9, 15, 16, 17, 31, 40] { + memory.write(0, &[0xee; 96]).unwrap(); + memory.write(at, &pattern[..len]).unwrap(); + let mut back = vec![0u8; len]; + memory.read(at, &mut back).unwrap(); + assert_eq!(back, pattern[..len], "at {at}, {len} bytes"); + // The bytes either side are untouched. + let mut around = [0u8; 96]; + memory.read(0, &mut around).unwrap(); + assert!(around[..at as usize].iter().all(|b| *b == 0xee)); + assert!(around[at as usize + len..].iter().all(|b| *b == 0xee)); + } + } + } + + #[test] + fn words_are_little_endian_whatever_the_host_is() { + let memory = window(16); + memory.store_u16(2, 0x1234).unwrap(); + memory.store_u32(4, 0x89ab_cdef).unwrap(); + memory.store_u64(8, 0x0123_4567_89ab_cdef).unwrap(); + let mut bytes = [0u8; 16]; + memory.read(0, &mut bytes).unwrap(); + assert_eq!(bytes[..8], [0, 0, 0x34, 0x12, 0xef, 0xcd, 0xab, 0x89]); + assert_eq!(bytes[8..], [0xef, 0xcd, 0xab, 0x89, 0x67, 0x45, 0x23, 0x01]); + assert_eq!(memory.load_u16(2), Ok(0x1234)); + assert_eq!(memory.load_u32(4), Ok(0x89ab_cdef)); + assert_eq!(memory.load_u64(8), Ok(0x0123_4567_89ab_cdef)); + } +} diff --git a/native/crates/vdev/src/regs.rs b/native/crates/vdev/src/regs.rs new file mode 100644 index 0000000..c4717bc --- /dev/null +++ b/native/crates/vdev/src/regs.rs @@ -0,0 +1,119 @@ +//! **The register block**: what a driver maps where hardware would have a +//! BAR. +//! +//! One page both sides map. Each register belongs to one side: the driver +//! never writes a device register and the device never writes a driver +//! register, so no word has two writers. +//! +//! The layout follows Virtio's own transports in what it carries (identity, +//! feature words, a status the driver steps through, the queue's size and +//! three addresses, a device configuration area) and departs from +//! `virtio-mmio` in one way that is forced: that layout selects what a +//! register shows by a write to another register, which only works when a +//! write traps. A write to shared memory does not, so nothing here is behind +//! a selector, and the status is two words: the one the driver writes and +//! the one the device answers with. +//! +//! # The doorbell +//! +//! The device does not see a write happen. It looks at the driver's +//! registers and at the queue when the driver rings its **doorbell**, the +//! transport's one notification, and it raises the driver's **interrupt** +//! when it has changed [`DEVICE_STATUS`] or used a chain. Virtio leaves +//! notification to each transport (a register on `virtio-mmio`, a hypercall +//! on channel I/O); here it is a notice on a channel, in both directions. +//! +//! The interrupt is a count the driver can sleep on: [`INTERRUPTS`] moves +//! each time the device raises one, so a driver reads it, does its work, +//! and parks until it differs from what it read. An interrupt raised +//! between the read and the park is not lost, because the park compares. +//! +//! A doorbell that announces a write to [`DRIVER_STATUS`] is always answered +//! with an interrupt, whether or not the device took the step: a driver +//! writes its status, rings, waits for the interrupt and reads +//! [`DEVICE_STATUS`]. A step the device refused is one its status does not +//! carry. +//! +//! # Bringing a device up +//! +//! 1. Check [`MAGIC`] and [`VERSION`], read [`DEVICE_ID`]. +//! 2. Write [`STATUS_ACKNOWLEDGE`]` | `[`STATUS_DRIVER`] to +//! [`DRIVER_STATUS`]. +//! 3. Read [`DEVICE_FEATURES`], write the accepted subset to +//! [`DRIVER_FEATURES`], add [`STATUS_FEATURES_OK`], ring the doorbell, +//! and wait for the interrupt. [`DEVICE_STATUS`] carries +//! `FEATURES_OK` only if the device accepted the subset. +//! 4. Lay the queue out in DMA memory; write [`QUEUE_NUM`], the three +//! addresses and [`QUEUE_READY`]. +//! 5. Add [`STATUS_DRIVER_OK`], ring, wait. The device is live when +//! [`DEVICE_STATUS`] carries it. +//! +//! Writing zero to [`DRIVER_STATUS`] and ringing resets the device; the +//! reset is done when [`DEVICE_STATUS`] reads zero. A device that can no +//! longer serve sets [`STATUS_DEVICE_NEEDS_RESET`] there. + +/// Bytes in the register block: one page. +pub const BYTES: u64 = 0x1000; + +/// `u32`, device. [`MAGIC_VALUE`]: this is a kore virtual device. +pub const MAGIC: u64 = 0x00; +/// `u32`, device. [`VERSION_VALUE`]: the layout in this module. +pub const VERSION: u64 = 0x04; +/// `u32`, device. Which device model this is; see the model's own crate. +pub const DEVICE_ID: u64 = 0x08; +/// `u32`, device. Who made it: [`VENDOR_KORE`] for a device kore hosts. +pub const VENDOR_ID: u64 = 0x0c; +/// `u64`, device. The features the device offers. +pub const DEVICE_FEATURES: u64 = 0x10; +/// `u64`, driver. The features the driver accepts, a subset of the offer. +pub const DRIVER_FEATURES: u64 = 0x18; +/// `u32`, driver. The status the driver has reached (`STATUS_*`). +pub const DRIVER_STATUS: u64 = 0x20; +/// `u32`, device. The status the device has accepted, and its own +/// [`STATUS_DEVICE_NEEDS_RESET`]. +pub const DEVICE_STATUS: u64 = 0x24; +/// `u32`, device. The largest queue the device takes. +pub const QUEUE_NUM_MAX: u64 = 0x28; +/// `u32`, driver. The queue's size: a power of two, at most the maximum. +pub const QUEUE_NUM: u64 = 0x2c; +/// `u32`, driver. Nonzero once the queue's size and addresses are written. +pub const QUEUE_READY: u64 = 0x30; +/// `u32`, device. Changes whenever the configuration area does. +pub const CONFIG_GENERATION: u64 = 0x34; +/// `u64`, driver. The descriptor table's device address. +pub const QUEUE_DESC: u64 = 0x38; +/// `u64`, driver. The available ring's device address. +pub const QUEUE_DRIVER: u64 = 0x40; +/// `u64`, driver. The used ring's device address. +pub const QUEUE_DEVICE: u64 = 0x48; +/// `u32`, device. Bytes of device configuration at [`CONFIG`]. +pub const CONFIG_BYTES: u64 = 0x50; +/// `u32`, device. Interrupts the device has raised, wrapping: the word a +/// driver parks on. +pub const INTERRUPTS: u64 = 0x54; +/// The device configuration area, device-written, to the end of the page. +pub const CONFIG: u64 = 0x100; + +/// `"kvdv"`, little-endian: a kore virtual device. +pub const MAGIC_VALUE: u32 = u32::from_le_bytes(*b"kvdv"); +/// The layout's version. +pub const VERSION_VALUE: u32 = 1; +/// `"kore"`, little-endian. +pub const VENDOR_KORE: u32 = u32::from_le_bytes(*b"kore"); + +/// The driver has noticed the device (Virtio 1.3 §2.1). +pub const STATUS_ACKNOWLEDGE: u32 = 1; +/// The driver knows how to drive it. +pub const STATUS_DRIVER: u32 = 2; +/// The driver is set up and the device may serve. +pub const STATUS_DRIVER_OK: u32 = 4; +/// Feature negotiation is complete. +pub const STATUS_FEATURES_OK: u32 = 8; +/// The device cannot go on and must be reset. Device-written. +pub const STATUS_DEVICE_NEEDS_RESET: u32 = 64; +/// The driver has given up on the device. +pub const STATUS_FAILED: u32 = 128; + +/// The device and driver follow Virtio 1.0 or later (feature bit 32). A +/// device offers it and a driver must accept it. +pub const F_VERSION_1: u64 = 1 << 32; diff --git a/native/crates/vdev/src/ring.rs b/native/crates/vdev/src/ring.rs new file mode 100644 index 0000000..2b5da72 --- /dev/null +++ b/native/crates/vdev/src/ring.rs @@ -0,0 +1,630 @@ +//! **The split virtqueue, in the driver's memory**: the device's half. +//! +//! A driver lays a queue out in the memory it lends: a descriptor table, an +//! available ring it writes and a used ring the device writes, in the split +//! layout of Virtio 1.3 §2.7. It tells the device where the three are by +//! device address, publishes a chain of descriptors by adding its head to the +//! available ring, and takes the chain back from the used ring with the +//! number of bytes the device wrote. +//! +//! [`DeviceRing`] is the device's half of that, behind the queue contracts of +//! `virtio-accel-transport` ([`DeviceQueue`], [`DeviceChain`]). The protocol +//! ships a queue for the same contracts, `virtio-accel-split-queue`, but it +//! is a model that owns its rings and every descriptor's bytes on the heap of +//! one task. A driver and a device in two address spaces need the rings in +//! memory both map, which is this. The model is the oracle: the tests here +//! put each chain shape through both and require the same answer. +//! +//! # What the device trusts +//! +//! Nothing the driver wrote. Every index is checked against the queue's size, +//! a chain is walked at most `size` descriptors with a visited set so a loop +//! ends, every span must lie inside the window ([`Memory`]), and a chain is +//! read **once**: the walk snapshots each descriptor, and the ports then use +//! the snapshot, so a driver that rewrites a descriptor after publishing it +//! changes nothing the device does. A chain that is malformed is not an +//! error of the queue: it is handed up like any other, its [`io`] says why, +//! and the device completes it with a used length of zero, as the protocol's +//! command-chain rules require. +//! +//! The profile is the protocol's: direct descriptors only, no indirect +//! tables, no event indices, and the two suppression flags. +//! +//! [`io`]: DeviceChain::io + +use alloc::rc::Rc; +use core::sync::atomic::{AtomicU64, Ordering, fence}; + +use virtio_accel_transport::{ + ByteAccessError, ChainError, ChainId, ChainIo, ChainIoResult, ChainRegion, DeviceChain, + DeviceQueue, MalformedChain, NotificationHint, NotificationRecheck, QueueConfigError, + QueueControl, QueueEpoch, QueueError, QueuePort, QueueSize, QueueState, ReadableBytes, + UsedLength, WritableBytes, validate_chain_layout, +}; + +use crate::mem::{Memory, MemoryFault}; + +/// The largest queue a device offers. +/// +/// A chain is walked with a visited set one machine word wide, so the size +/// is bounded by it; sixty-four descriptors is four of the longest command +/// chains, or thirty-two two-descriptor commands, in flight at once. +pub const MAX_QUEUE_SIZE: u16 = u64::BITS as u16; + +/// One descriptor: `addr: u64, len: u32, flags: u16, next: u16` +/// (Virtio 1.3 §2.7.5). +pub(crate) const DESC_BYTES: u64 = 16; +const DESC_LEN: usize = 8; +const DESC_FLAGS: usize = 12; +const DESC_NEXT: usize = 14; +/// The chain continues at `next`. +pub const VIRTQ_DESC_F_NEXT: u16 = 1 << 0; +/// The device writes this descriptor's bytes; otherwise it reads them. +pub const VIRTQ_DESC_F_WRITE: u16 = 1 << 1; +/// The descriptor names an indirect table: not in this profile. +pub const VIRTQ_DESC_F_INDIRECT: u16 = 1 << 2; + +/// Both rings begin `flags: u16, idx: u16` (§2.7.6, §2.7.8). +pub(crate) const RING_FLAGS: u64 = 0; +pub(crate) const RING_IDX: u64 = 2; +pub(crate) const RING_HEADER: u64 = 4; +/// An available-ring entry is a descriptor index. +pub(crate) const AVAIL_ENTRY: u64 = 2; +/// A used-ring entry is `id: u32, len: u32`. +pub(crate) const USED_ENTRY: u64 = 8; +pub(crate) const USED_LEN: u64 = 4; +/// The driver asks not to be interrupted when the device uses a chain. +pub const VIRTQ_AVAIL_F_NO_INTERRUPT: u16 = 1 << 0; +/// The device asks not to be notified when the driver publishes a chain. +pub const VIRTQ_USED_F_NO_NOTIFY: u16 = 1 << 0; + +/// Alignments the layout requires of each part (§2.7's table). +pub(crate) const DESC_ALIGN: u64 = 16; +pub(crate) const AVAIL_ALIGN: u64 = 2; +pub(crate) const USED_ALIGN: u64 = 4; + +/// Where a driver put a queue's three parts, as device addresses. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RingAddresses { + pub descriptors: u64, + pub available: u64, + pub used: u64, +} + +/// Why the ring could not go on. Each is the driver's doing, and none is one +/// chain's: a device that sees one stops serving the queue and asks for a +/// reset. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RingError { + /// A part of the queue is not where the layout allows: outside the + /// window, or not aligned for its entries. + Layout, + /// The queue was made ready smaller than the longest chain it must hold. + ChainLimitExceedsQueue, + /// The queue cannot be reconfigured while it is ready. + Busy, + /// The available index is further ahead than the queue is long. + AvailableIndex, + /// A head was published that the device already holds. + HeadInFlight, + /// The window stopped answering for a ring the queue validated. + Memory(MemoryFault), +} + +impl From for QueueError { + fn from(fault: MemoryFault) -> Self { + Self::Transport(RingError::Memory(fault)) + } +} + +/// Whether a ring can hold a queue of at most `max_size` whose chains are at +/// most `max_chain_descriptors` long: a chain is two descriptors at least, +/// and no longer than the queue. +pub const fn limits_fit(max_size: QueueSize, max_chain_descriptors: u16) -> bool { + max_size.get() <= MAX_QUEUE_SIZE + && max_chain_descriptors >= 2 + && max_chain_descriptors <= max_size.get() +} + +/// One descriptor's bytes, snapshotted when its chain was taken. +#[derive(Clone, Copy, Debug)] +struct Span { + at: u64, + len: u32, +} + +/// The chain's spans in one direction, as one run of bytes. +#[derive(Clone, Debug)] +struct Port { + memory: M, + spans: heapless::Vec, + len: u64, + epoch: QueueEpoch, + current: Rc, +} + +impl Port { + /// The spans `len` bytes at `offset` fall in, as `(address, from, count)` + /// with `from` the offset into the caller's buffer. + fn each( + &self, + offset: u64, + len: usize, + mut visit: impl FnMut(u64, usize, usize) -> Result<(), MemoryFault>, + ) -> Result<(), ByteAccessError> { + if self.current.load(Ordering::Acquire) != self.epoch.get() { + return Err(ByteAccessError::Reset); + } + if !offset + .checked_add(len as u64) + .is_some_and(|end| end <= self.len) + { + return Err(ByteAccessError::OutOfBounds); + } + let (mut skip, mut done) = (offset, 0usize); + for span in &self.spans { + let span_len = u64::from(span.len); + if skip >= span_len { + skip -= span_len; + continue; + } + if done == len { + break; + } + let count = ((span_len - skip) as usize).min(len - done); + visit(span.at + skip, done, count).map_err(|_| ByteAccessError::Access)?; + done += count; + skip = 0; + } + Ok(()) + } +} + +/// The bytes a chain gives the device to read. +#[derive(Debug)] +pub struct RingSource(Port); + +impl ReadableBytes for RingSource { + fn len(&self) -> u64 { + self.0.len + } + + fn read_at(&self, offset: u64, target: &mut [u8]) -> Result<(), ByteAccessError> { + let memory = &self.0.memory; + let len = target.len(); + self.0.each(offset, len, |at, from, count| { + memory.read(at, &mut target[from..from + count]) + }) + } +} + +/// The bytes a chain gives the device to write. +#[derive(Debug)] +pub struct RingSink(Port); + +impl WritableBytes for RingSink { + fn len(&self) -> u64 { + self.0.len + } + + fn write_at(&mut self, offset: u64, source: &[u8]) -> Result<(), ByteAccessError> { + let memory = &self.0.memory; + self.0.each(offset, source.len(), |at, from, count| { + memory.write(at, &source[from..from + count]) + }) + } +} + +/// One chain the device took from the available ring and has not yet used. +/// +/// Not `Clone`: [`DeviceRing::complete`] consumes it, so a chain is used +/// once. +#[derive(Debug)] +pub struct RingChain { + id: ChainId, + head: u16, + regions: heapless::Vec, + malformed: Option, + request: RingSource, + response: RingSink, +} + +impl DeviceChain for RingChain { + type Request = RingSource; + type Response = RingSink; + type Error = RingError; + + fn id(&self) -> ChainId { + self.id + } + + fn io(&mut self) -> ChainIoResult<'_, Self::Request, Self::Response, Self::Error> { + let current = self.request.0.current.load(Ordering::Acquire); + if current != self.id.epoch().get() { + return Err(ChainError::ResetRace { + chain: self.id.epoch(), + // An epoch is never zero: it starts at one and only grows. + current: QueueEpoch::new(current).unwrap_or(QueueEpoch::INITIAL), + }); + } + if let Some(malformed) = self.malformed { + return Err(ChainError::Malformed(malformed)); + } + Ok(ChainIo::new( + &self.regions, + &self.request, + &mut self.response, + )) + } +} + +/// The device's half of one split virtqueue in a driver's memory. +#[derive(Debug)] +pub struct DeviceRing { + memory: M, + state: QueueState, + max_chain_descriptors: u16, + addresses: RingAddresses, + /// The epoch every chain this queue hands out carries, shared with them + /// so a reset reaches a chain the device still holds. + current: Rc, + /// Available entries taken, and used entries published. + taken: u16, + used: u16, + /// Heads the device holds, one bit each. + in_flight: u64, +} + +impl DeviceRing { + /// A queue in `memory` of at most `max_size` descriptors, whose chains + /// are at most `max_chain_descriptors` long. Unconfigured: the driver + /// sets its size and addresses and makes it ready. + /// + /// `None` when the limits do not fit: a chain is two descriptors at + /// least, and no longer than the queue. + pub fn new(memory: M, max_size: QueueSize, max_chain_descriptors: u16) -> Option { + if !limits_fit(max_size, max_chain_descriptors) { + return None; + } + let epoch = QueueEpoch::INITIAL; + Some(Self { + memory, + state: QueueState::unconfigured(max_size, epoch), + max_chain_descriptors, + addresses: RingAddresses::default(), + current: Rc::new(AtomicU64::new(epoch.get())), + taken: 0, + used: 0, + in_flight: 0, + }) + } + + /// The window the queue is in, given back: the queue is over. + pub fn into_memory(self) -> M { + self.memory + } + + /// Where the driver says the queue's parts are. Taken while the queue is + /// not ready, and checked when it is made so. + pub fn set_addresses(&mut self, addresses: RingAddresses) -> Result<(), QueueError> { + if self.state.ready() { + return Err(QueueError::Transport(RingError::Busy)); + } + self.addresses = addresses; + Ok(()) + } + + /// The addresses the queue was given. + pub const fn addresses(&self) -> RingAddresses { + self.addresses + } + + /// Chains the device holds: taken and not yet used. + pub const fn in_flight(&self) -> u32 { + self.in_flight.count_ones() + } + + fn size(&self) -> Result> { + match (self.state.ready(), self.state.size()) { + (true, Some(size)) => Ok(size.get()), + _ => Err(QueueError::NotReady), + } + } + + /// Whether the three parts lie inside the window where the layout + /// allows them, for a queue of `size`. + fn layout_holds(&self, size: u16) -> bool { + let RingAddresses { + descriptors, + available, + used, + } = self.addresses; + let size = u64::from(size); + descriptors % DESC_ALIGN == 0 + && available % AVAIL_ALIGN == 0 + && used % USED_ALIGN == 0 + && self.memory.contains(descriptors, size * DESC_BYTES) + && self + .memory + .contains(available, RING_HEADER + size * AVAIL_ENTRY) + && self.memory.contains(used, RING_HEADER + size * USED_ENTRY) + } + + /// Walk the chain at `head`, snapshotting each descriptor once. + fn take(&self, head: u16, size: u16) -> Result, QueueError> { + let epoch = self.state.epoch(); + let port = || Port { + memory: self.memory.clone(), + spans: heapless::Vec::new(), + len: 0, + epoch, + current: Rc::clone(&self.current), + }; + let mut chain = RingChain { + id: ChainId::new(epoch, u64::from(head)), + head, + regions: heapless::Vec::new(), + malformed: None, + request: RingSource(port()), + response: RingSink(port()), + }; + chain.malformed = self.walk(&mut chain, head, size)?.err(); + Ok(chain) + } + + /// The walk proper. The outer error is the queue's own memory failing; + /// the inner one is this chain being malformed, which is the chain's + /// business and not the queue's. + fn walk( + &self, + chain: &mut RingChain, + head: u16, + size: u16, + ) -> Result, QueueError> { + let (mut next, mut seen) = (head, 0u64); + // At most `size` descriptors, a hostile loop included: the visited + // set ends it on the first index met twice. + for _ in 0..size { + if next >= size { + return Ok(Err(MalformedChain::DescriptorIndex)); + } + if seen & (1 << next) != 0 { + return Ok(Err(MalformedChain::DescriptorLoop)); + } + seen |= 1 << next; + + let mut descriptor = [0u8; DESC_BYTES as usize]; + self.memory.read( + self.addresses.descriptors + u64::from(next) * DESC_BYTES, + &mut descriptor, + )?; + let word = |at: usize, len: usize| { + let mut bytes = [0u8; 8]; + bytes[..len].copy_from_slice(&descriptor[at..at + len]); + u64::from_le_bytes(bytes) + }; + let (at, len) = (word(0, 8), word(DESC_LEN, 4) as u32); + let flags = word(DESC_FLAGS, 2) as u16; + next = word(DESC_NEXT, 2) as u16; + + if flags & VIRTQ_DESC_F_INDIRECT != 0 { + return Ok(Err(MalformedChain::IndirectUnsupported)); + } + if flags & !(VIRTQ_DESC_F_NEXT | VIRTQ_DESC_F_WRITE) != 0 { + return Ok(Err(MalformedChain::DescriptorFlags)); + } + let writable = flags & VIRTQ_DESC_F_WRITE != 0; + let region = if writable { + ChainRegion::writable(u64::from(len)) + } else { + ChainRegion::readable(u64::from(len)) + }; + // The vectors hold `MAX_QUEUE_SIZE`, which bounds `size`. + let _ = chain.regions.push(region); + let port = if writable { + &mut chain.response.0 + } else { + &mut chain.request.0 + }; + let _ = port.spans.push(Span { at, len }); + port.len += u64::from(len); + + if flags & VIRTQ_DESC_F_NEXT == 0 { + // Count, zero lengths, direction order and totals are the + // protocol's own rule, by its own function. + if let Err(error) = + validate_chain_layout(&chain.regions, self.max_chain_descriptors) + { + return Ok(Err(error.into())); + } + // Every byte the engine may touch is mapped before it runs. + let mapped = |port: &Port| { + port.spans + .iter() + .all(|span| self.memory.contains(span.at, u64::from(span.len))) + }; + if !mapped(&chain.request.0) || !mapped(&chain.response.0) { + return Ok(Err(MalformedChain::Address)); + } + return Ok(Ok(())); + } + } + // `size` descriptors and still a `next`: longer than any chain. + Ok(Err(MalformedChain::DescriptorCount)) + } +} + +impl QueuePort for DeviceRing { + fn state(&self) -> QueueState { + self.state + } +} + +impl QueueControl for DeviceRing { + type Error = RingError; + + fn configure(&mut self, size: QueueSize) -> Result<(), QueueError> { + if self.state.ready() { + return Err(QueueError::Transport(RingError::Busy)); + } + if size > self.state.max_size() { + return Err(QueueError::InvalidConfiguration( + QueueConfigError::SizeExceedsMaximum, + )); + } + if size.get() < self.max_chain_descriptors { + return Err(QueueError::Transport(RingError::ChainLimitExceedsQueue)); + } + self.state = QueueState::new(self.state.max_size(), Some(size), false, self.state.epoch()) + .map_err(QueueError::InvalidConfiguration)?; + Ok(()) + } + + fn set_ready(&mut self, ready: bool) -> Result<(), QueueError> { + if ready { + let Some(size) = self.state.size() else { + return Err(QueueError::InvalidConfiguration( + QueueConfigError::ReadyWithoutSize, + )); + }; + if !self.layout_holds(size.get()) { + return Err(QueueError::Transport(RingError::Layout)); + } + } + self.state = QueueState::new( + self.state.max_size(), + self.state.size(), + ready, + self.state.epoch(), + ) + .map_err(QueueError::InvalidConfiguration)?; + Ok(()) + } +} + +impl DeviceQueue for DeviceRing { + type Chain = RingChain; + type Error = RingError; + + fn pop_available(&mut self) -> Result, QueueError> { + let size = self.size()?; + // Acquire: the entry and the descriptors the driver wrote before it + // published this index are visible to the reads below. + let published = self.memory.load_u16(self.addresses.available + RING_IDX)?; + let pending = published.wrapping_sub(self.taken); + if pending == 0 { + return Ok(None); + } + if pending > size { + return Err(QueueError::Transport(RingError::AvailableIndex)); + } + let slot = u64::from(self.taken % size); + let mut entry = [0u8; AVAIL_ENTRY as usize]; + self.memory.read( + self.addresses.available + RING_HEADER + slot * AVAIL_ENTRY, + &mut entry, + )?; + let head = u16::from_le_bytes(entry); + self.taken = self.taken.wrapping_add(1); + + let chain = self.take(head, size)?; + if head < size { + // The head is a real descriptor: the device holds it until it + // is used, and a second publication of it is the driver reusing + // a descriptor it does not own. + if self.in_flight & (1 << head) != 0 { + return Err(QueueError::Transport(RingError::HeadInFlight)); + } + self.in_flight |= 1 << head; + } + Ok(Some(chain)) + } + + fn complete( + &mut self, + chain: Self::Chain, + used: UsedLength, + ) -> Result> { + let current = self.state.epoch(); + if chain.id.epoch() != current { + return Err(QueueError::ResetRace { + operation: chain.id.epoch(), + current, + }); + } + let size = self.size()?; + let capacity = chain.response.0.len; + if u64::from(used.get()) > capacity { + return Err(QueueError::UsedLengthExceeded { used, capacity }); + } + let slot = u64::from(self.used % size); + let entry = self.addresses.used + RING_HEADER + slot * USED_ENTRY; + self.memory.store_u32(entry, u32::from(chain.head))?; + self.memory.store_u32(entry + USED_LEN, used.get())?; + self.used = self.used.wrapping_add(1); + // Release: the response bytes and the entry above are visible to a + // driver that has loaded this index. + self.memory + .store_u16(self.addresses.used + RING_IDX, self.used)?; + if u32::from(chain.head) < u64::BITS { + self.in_flight &= !(1 << chain.head); + } + // The index is published before the driver's suppression flag is + // read: a driver that clears the flag and then rechecks the index + // cannot miss this entry, and one that set it is not interrupted. + fence(Ordering::SeqCst); + let flags = self + .memory + .load_u16(self.addresses.available + RING_FLAGS)?; + Ok(if flags & VIRTQ_AVAIL_F_NO_INTERRUPT == 0 { + NotificationHint::Notify + } else { + NotificationHint::Suppressed + }) + } + + fn disable_available_notifications(&mut self) -> Result<(), QueueError> { + self.size()?; + self.memory + .store_u16(self.addresses.used + RING_FLAGS, VIRTQ_USED_F_NO_NOTIFY)?; + Ok(()) + } + + fn enable_available_notifications( + &mut self, + ) -> Result> { + self.size()?; + self.memory.store_u16(self.addresses.used + RING_FLAGS, 0)?; + // The flag is cleared before the index is read: a driver that + // published while notifications were off is seen here, and one that + // publishes after sees the flag clear and notifies. + fence(Ordering::SeqCst); + let published = self.memory.load_u16(self.addresses.available + RING_IDX)?; + Ok(if published == self.taken { + NotificationRecheck::Idle + } else { + NotificationRecheck::WorkPending + }) + } + + fn reset(&mut self, next_epoch: QueueEpoch) -> Result<(), QueueError> { + if next_epoch <= self.state.epoch() { + return Err(QueueError::InvalidConfiguration( + QueueConfigError::NonIncreasingEpoch, + )); + } + // Published first: a chain the device still holds refuses its bytes + // and its completion from here on, so nothing of the old epoch is + // written to memory the driver may now reuse. + self.current.store(next_epoch.get(), Ordering::Release); + self.state = QueueState::unconfigured(self.state.max_size(), next_epoch); + self.addresses = RingAddresses::default(); + self.taken = 0; + self.used = 0; + self.in_flight = 0; + Ok(()) + } +} + +#[cfg(test)] +pub(crate) mod tests; diff --git a/native/crates/vdev/src/ring/tests.rs b/native/crates/vdev/src/ring/tests.rs new file mode 100644 index 0000000..c69997c --- /dev/null +++ b/native/crates/vdev/src/ring/tests.rs @@ -0,0 +1,683 @@ +use super::*; +use crate::mem::{Window, tests::window}; +use std::vec; +use std::vec::Vec; +use virtio_accel_split_queue::{Descriptor, DriverChain, SplitQueue}; +use virtio_accel_transport::RegionDirection; + +/// One descriptor a test driver writes: readable bytes, or room to write. +#[derive(Clone, Copy)] +pub(crate) enum Part<'a> { + Read(&'a [u8]), + Write(u32), +} + +/// A driver's half of the queue, written the way a C driver would write it: +/// the three parts laid out in its own memory, and descriptors, available +/// entries and used entries moved by hand. +pub(crate) struct Guest { + pub(crate) memory: Window, + pub(crate) size: u16, + pub(crate) at: RingAddresses, + /// Where the next buffer goes: buffers are never reused in a test. + bump: u64, + published: u16, + seen: u16, +} + +pub(crate) const WINDOW: usize = 64 * 1024; +const BUFFERS: u64 = 0x2000; + +impl Guest { + /// A queue of `size` whose chains are at most `max_chain` long, ready. + fn ready(size: u16, max_chain: u16) -> (Self, DeviceRing) { + let (guest, mut ring) = Self::unready(size, max_chain); + ring.set_ready(true).expect("a layout inside the window"); + (guest, ring) + } + + fn unready(size: u16, max_chain: u16) -> (Self, DeviceRing) { + let guest = Self::over(window(WINDOW), size); + let mut ring = DeviceRing::new( + guest.memory, + QueueSize::new(MAX_QUEUE_SIZE).unwrap(), + max_chain, + ) + .expect("limits fit"); + ring.configure(QueueSize::new(size).unwrap()) + .expect("configured"); + ring.set_addresses(guest.at).expect("addresses"); + (guest, ring) + } + + /// A driver's queue of `size` laid out in `memory`, which no device has + /// been told about yet. + pub(crate) fn over(memory: Window, size: u16) -> Self { + Self { + memory, + size, + at: RingAddresses { + descriptors: 0, + available: 0x800, + used: 0x1000, + }, + bump: BUFFERS, + published: 0, + seen: 0, + } + } + + pub(crate) fn descriptor(&self, index: u16, addr: u64, len: u32, flags: u16, next: u16) { + let mut bytes = [0u8; 16]; + bytes[..8].copy_from_slice(&addr.to_le_bytes()); + bytes[8..12].copy_from_slice(&len.to_le_bytes()); + bytes[12..14].copy_from_slice(&flags.to_le_bytes()); + bytes[14..].copy_from_slice(&next.to_le_bytes()); + self.memory + .write(self.at.descriptors + u64::from(index) * 16, &bytes) + .unwrap(); + } + + fn space(&mut self, len: u32) -> u64 { + let at = self.bump; + self.bump += u64::from(len.max(1)).next_multiple_of(8); + at + } + + /// Write `parts` as a direct chain in descriptors `first..`, and answer + /// where each part's bytes are. + pub(crate) fn chain(&mut self, first: u16, parts: &[Part<'_>]) -> Vec { + let mut places = Vec::new(); + for (offset, part) in parts.iter().enumerate() { + let index = first + offset as u16; + let last = offset + 1 == parts.len(); + let (len, mut flags) = match part { + Part::Read(bytes) => (bytes.len() as u32, 0), + Part::Write(len) => (*len, VIRTQ_DESC_F_WRITE), + }; + let at = self.space(len); + if let Part::Read(bytes) = part { + self.memory.write(at, bytes).unwrap(); + } + if !last { + flags |= VIRTQ_DESC_F_NEXT; + } + self.descriptor(index, at, len, flags, if last { 0 } else { index + 1 }); + places.push(at); + } + places + } + + /// Add `head` to the available ring and publish it. + pub(crate) fn publish(&mut self, head: u16) { + let slot = u64::from(self.published % self.size); + self.memory + .write(self.at.available + 4 + slot * 2, &head.to_le_bytes()) + .unwrap(); + self.published = self.published.wrapping_add(1); + self.memory + .store_u16(self.at.available + 2, self.published) + .unwrap(); + } + + /// The next used entry the driver has not seen: `(head, bytes written)`. + pub(crate) fn used(&mut self) -> Option<(u32, u32)> { + if self.memory.load_u16(self.at.used + 2).unwrap() == self.seen { + return None; + } + let slot = u64::from(self.seen % self.size); + let entry = self.at.used + 4 + slot * 8; + self.seen = self.seen.wrapping_add(1); + Some(( + self.memory.load_u32(entry).unwrap(), + self.memory.load_u32(entry + 4).unwrap(), + )) + } + + pub(crate) fn read(&self, at: u64, len: usize) -> Vec { + let mut bytes = vec![0u8; len]; + self.memory.read(at, &mut bytes).unwrap(); + bytes + } +} + +fn request(chain: &mut RingChain) -> Vec { + let (_, source, _) = chain.io().expect("a valid chain").into_parts(); + let mut bytes = vec![0u8; source.len() as usize]; + source.read_at(0, &mut bytes).unwrap(); + bytes +} + +/// VQ-001, VQ-003, VQ-013: a command crosses in readable descriptors and its +/// response comes back in writable ones, each side one run of bytes whatever +/// the descriptors' split, with the used length exactly what was written. +#[test] +fn a_chain_carries_a_request_out_and_a_response_back() { + let (mut guest, mut ring) = Guest::ready(8, 8); + assert!(ring.pop_available().unwrap().is_none()); + + let places = guest.chain( + 2, + &[ + Part::Read(b"hello, "), + Part::Read(b"device"), + Part::Write(5), + Part::Write(16), + ], + ); + guest.publish(2); + + let mut chain = ring.pop_available().unwrap().expect("one chain"); + assert_eq!(ring.in_flight(), 1); + assert_eq!(request(&mut chain), b"hello, device"); + { + let (regions, source, sink) = chain.io().unwrap().into_parts(); + let shape: Vec<_> = regions.iter().map(|r| (r.direction, r.bytes)).collect(); + assert_eq!( + shape, + [ + (RegionDirection::DeviceReadable, 7), + (RegionDirection::DeviceReadable, 6), + (RegionDirection::DeviceWritable, 5), + (RegionDirection::DeviceWritable, 16), + ] + ); + // A read may start and end anywhere, across the split. + let mut middle = [0u8; 6]; + source.read_at(4, &mut middle).unwrap(); + assert_eq!(&middle, b"o, dev"); + assert_eq!( + source.read_at(8, &mut middle), + Err(ByteAccessError::OutOfBounds) + ); + // The response crosses its split the same way. + assert_eq!(sink.len(), 21); + sink.write_at(0, b"hi, driver").unwrap(); + assert_eq!(sink.write_at(20, b"xy"), Err(ByteAccessError::OutOfBounds)); + } + assert_eq!( + ring.complete(chain, UsedLength::new(10)), + Ok(NotificationHint::Notify) + ); + assert_eq!(ring.in_flight(), 0); + assert_eq!(guest.used(), Some((2, 10))); + assert_eq!(guest.used(), None); + assert_eq!(guest.read(places[2], 5), b"hi, d"); + assert_eq!(guest.read(places[3], 5), b"river"); + // Room past the response is untouched. + assert_eq!(guest.read(places[3] + 5, 11), [0u8; 11]); +} + +/// A chain is read once. A driver that rewrites a descriptor after it +/// published the chain changes nothing the device does with it. +#[test] +fn a_descriptor_rewritten_after_publication_changes_nothing() { + let (mut guest, mut ring) = Guest::ready(4, 4); + let places = guest.chain(0, &[Part::Read(b"the request"), Part::Write(8)]); + let decoy = guest.space(64); + guest.memory.write(decoy, &[0xaa; 64]).unwrap(); + guest.publish(0); + let mut chain = ring.pop_available().unwrap().unwrap(); + + // Point both descriptors at the decoy, longer, and swap their roles. + guest.descriptor(0, decoy, 64, VIRTQ_DESC_F_WRITE | VIRTQ_DESC_F_NEXT, 1); + guest.descriptor(1, decoy, 64, 0, 0); + + assert_eq!(request(&mut chain), b"the request"); + let (_, _, sink) = chain.io().unwrap().into_parts(); + assert_eq!(sink.len(), 8); + sink.write_at(0, b"answered").unwrap(); + ring.complete(chain, UsedLength::new(8)).unwrap(); + assert_eq!(guest.read(places[1], 8), b"answered"); + assert_eq!(guest.read(decoy, 64), [0xaa; 64]); +} + +/// A chain's shape as plain numbers: `(length, flags, next)` per descriptor. +type Shape = &'static [(u32, u16, u16)]; + +const N: u16 = VIRTQ_DESC_F_NEXT; +const W: u16 = VIRTQ_DESC_F_WRITE; + +/// Shapes a driver may publish, good and bad, with what the protocol calls +/// each. The chain limit in these tests is four descriptors. +const SHAPES: &[(&str, Shape, u16, Option)] = &[ + ( + "one readable, one writable", + &[(24, N, 1), (32, W, 0)], + 0, + None, + ), + ( + "two and two", + &[(8, N, 1), (8, N, 2), (8, N | W, 3), (8, W, 0)], + 0, + None, + ), + ( + "a head that is not descriptor zero", + &[(16, W, 0), (16, N, 0)], + 1, + None, + ), + ( + "no writable descriptor", + &[(8, N, 1), (8, 0, 0)], + 0, + Some(MalformedChain::Direction), + ), + ( + "no readable descriptor", + &[(8, N | W, 1), (8, W, 0)], + 0, + Some(MalformedChain::Direction), + ), + ( + "readable after writable", + &[(8, N, 1), (8, N | W, 2), (8, 0, 0)], + 0, + Some(MalformedChain::Direction), + ), + ( + "a single descriptor", + &[(8, W, 0)], + 0, + Some(MalformedChain::DescriptorCount), + ), + ( + "longer than the chain limit", + &[(8, N, 1), (8, N, 2), (8, N, 3), (8, N | W, 4), (8, W, 0)], + 0, + Some(MalformedChain::DescriptorCount), + ), + ( + "a zero-length descriptor", + &[(0, N, 1), (8, W, 0)], + 0, + Some(MalformedChain::ZeroLength), + ), + ( + "a loop", + &[(8, N, 1), (8, N | W, 0)], + 0, + Some(MalformedChain::DescriptorLoop), + ), + ( + "a descriptor that follows itself", + &[(8, N, 0)], + 0, + Some(MalformedChain::DescriptorLoop), + ), + ( + "an indirect descriptor", + &[(16, VIRTQ_DESC_F_INDIRECT, 0)], + 0, + Some(MalformedChain::IndirectUnsupported), + ), + ( + "a flag the profile does not have", + &[(8, N | 0x80, 1), (8, W, 0)], + 0, + Some(MalformedChain::DescriptorFlags), + ), +]; + +/// What a consumed chain says of itself: its regions, or why it is malformed. +type Verdict = Result, MalformedChain>; + +fn verdict(chain: &mut C, name: &str) -> Verdict +where + C::Error: core::fmt::Debug, +{ + match chain.io() { + Ok(io) => Ok(io.into_parts().0.to_vec()), + Err(ChainError::Malformed(malformed)) => Err(malformed), + Err(other) => panic!("{name}: {other:?}"), + } +} + +/// VQ-004, VQ-005, VQ-006, and the oracle: every shape gets the same answer +/// from this ring as from the protocol's own in-memory split queue, and a +/// malformed one is handed up, says why, and is used with nothing written. +#[test] +fn every_chain_shape_is_judged_as_the_protocols_own_queue_judges_it() { + for &(name, shape, head, expected) in SHAPES { + // This ring, in a driver's memory. + let (mut guest, mut ring) = Guest::ready(8, 4); + for (index, &(len, flags, next)) in shape.iter().enumerate() { + let at = guest.space(len); + guest.descriptor(index as u16, at, len, flags, next); + } + guest.publish(head); + let mut chain = ring.pop_available().unwrap().expect(name); + let ours = verdict(&mut chain, name); + assert_eq!(ours.as_ref().err().copied(), expected, "{name}"); + + // The protocol's model, given the same descriptors. + let mut model = SplitQueue::new(QueueSize::new(8).unwrap(), 4).unwrap(); + model.configure(QueueSize::new(8).unwrap()).unwrap(); + QueueControl::set_ready(&mut model, true).unwrap(); + let descriptors = shape + .iter() + .map(|&(len, flags, next)| Descriptor::raw(vec![0; len as usize], flags, next)) + .collect(); + let raw = DriverChain::raw(descriptors, head).expect(name); + model.inject_available(raw).map_err(|_| name).unwrap(); + let mut modelled = DeviceQueue::pop_available(&mut model).unwrap().expect(name); + assert_eq!( + ours, + verdict(&mut modelled, name), + "{name}: this ring against the model" + ); + + // Either way the driver gets its chain back; a malformed one with + // nothing written. + let used = u32::from(ours.is_ok()); + ring.complete(chain, UsedLength::new(used)).expect(name); + assert_eq!(guest.used(), Some((u32::from(head), used)), "{name}"); + } +} + +/// What only a ring in real memory can get wrong: indices and addresses the +/// driver made up. +#[test] +fn indices_and_addresses_outside_what_was_lent_are_malformed_chains() { + let malformed = |build: &dyn Fn(&mut Guest) -> u16| { + let (mut guest, mut ring) = Guest::ready(8, 4); + let head = build(&mut guest); + guest.publish(head); + let mut chain = ring.pop_available().unwrap().unwrap(); + let reason = verdict(&mut chain, "made up").expect_err("malformed"); + // It is still the driver's chain to get back, with nothing written. + ring.complete(chain, UsedLength::new(0)).unwrap(); + assert_eq!(guest.used(), Some((u32::from(head), 0))); + reason + }; + + // A head past the queue. + assert_eq!(malformed(&|_| 8), MalformedChain::DescriptorIndex); + assert_eq!(malformed(&|_| u16::MAX), MalformedChain::DescriptorIndex); + // A `next` past the queue. + assert_eq!( + malformed(&|guest| { + let at = guest.space(8); + guest.descriptor(0, at, 8, N, 9); + 0 + }), + MalformedChain::DescriptorIndex + ); + // Bytes that begin or end outside the window, readable or writable. + for (at, len) in [ + (WINDOW as u64, 8u32), + (WINDOW as u64 - 4, 8), + (u64::MAX - 3, 8), + (0x1_0000_0000, 8), + ] { + assert_eq!( + malformed(&|guest| { + let good = guest.space(8); + guest.descriptor(0, at, len, N, 1); + guest.descriptor(1, good, 8, W, 0); + 0 + }), + MalformedChain::Address, + "readable at {at:#x}" + ); + assert_eq!( + malformed(&|guest| { + let good = guest.space(8); + guest.descriptor(0, good, 8, N, 1); + guest.descriptor(1, at, len, W, 0); + 0 + }), + MalformedChain::Address, + "writable at {at:#x}" + ); + } +} + +/// A used length is the bytes the device wrote, and it cannot claim more +/// than the chain gave it room for. +#[test] +fn a_used_length_past_the_chains_room_is_refused() { + let (mut guest, mut ring) = Guest::ready(4, 4); + guest.chain(0, &[Part::Read(b"q"), Part::Write(4)]); + guest.publish(0); + let chain = ring.pop_available().unwrap().unwrap(); + assert_eq!( + ring.complete(chain, UsedLength::new(5)).err(), + Some(QueueError::UsedLengthExceeded { + used: UsedLength::new(5), + capacity: 4 + }) + ); + assert_eq!(guest.used(), None, "nothing was published"); +} + +/// VQ-014: the device uses chains in whatever order it finishes them. +#[test] +fn chains_are_used_in_the_order_the_device_finishes_them() { + let (mut guest, mut ring) = Guest::ready(8, 4); + guest.chain(0, &[Part::Read(b"first"), Part::Write(4)]); + guest.chain(4, &[Part::Read(b"second"), Part::Write(4)]); + guest.publish(0); + guest.publish(4); + let first = ring.pop_available().unwrap().unwrap(); + let second = ring.pop_available().unwrap().unwrap(); + assert!(ring.pop_available().unwrap().is_none()); + assert_eq!(ring.in_flight(), 2); + ring.complete(second, UsedLength::new(3)).unwrap(); + ring.complete(first, UsedLength::new(1)).unwrap(); + assert_eq!(guest.used(), Some((4, 3))); + assert_eq!(guest.used(), Some((0, 1))); +} + +/// VQ-017: either side may ask not to be woken, and neither loses work for +/// it. +#[test] +fn suppression_is_honoured_and_loses_no_work() { + let (mut guest, mut ring) = Guest::ready(4, 4); + + // The driver asks not to be interrupted. + guest + .memory + .store_u16(guest.at.available, VIRTQ_AVAIL_F_NO_INTERRUPT) + .unwrap(); + guest.chain(0, &[Part::Read(b"q"), Part::Write(1)]); + guest.publish(0); + let chain = ring.pop_available().unwrap().unwrap(); + assert_eq!( + ring.complete(chain, UsedLength::new(1)), + Ok(NotificationHint::Suppressed) + ); + // The entry is there all the same. + assert_eq!(guest.used(), Some((0, 1))); + + // The device asks not to be notified while it drains, and says so where + // the driver reads it. + ring.disable_available_notifications().unwrap(); + assert_eq!( + guest.memory.load_u16(guest.at.used), + Ok(VIRTQ_USED_F_NO_NOTIFY) + ); + assert!(ring.pop_available().unwrap().is_none()); + // A chain published before it listens again is found by the recheck. + guest.chain(2, &[Part::Read(b"r"), Part::Write(1)]); + guest.publish(2); + assert_eq!( + ring.enable_available_notifications(), + Ok(NotificationRecheck::WorkPending) + ); + assert_eq!(guest.memory.load_u16(guest.at.used), Ok(0)); + let chain = ring.pop_available().unwrap().unwrap(); + ring.complete(chain, UsedLength::new(0)).unwrap(); + assert_eq!( + ring.enable_available_notifications(), + Ok(NotificationRecheck::Idle) + ); +} + +/// VQ-018: after a reset nothing of the old epoch is written. A chain the +/// device still holds can neither read, write nor be used. +#[test] +fn a_reset_silences_every_chain_the_device_still_holds() { + let (mut guest, mut ring) = Guest::ready(4, 4); + let places = guest.chain(0, &[Part::Read(b"before"), Part::Write(8)]); + guest.publish(0); + let mut chain = ring.pop_available().unwrap().unwrap(); + let old = chain.id().epoch(); + let next = old.checked_next().unwrap(); + + assert_eq!( + ring.reset(old).err(), + Some(QueueError::InvalidConfiguration( + QueueConfigError::NonIncreasingEpoch + )) + ); + ring.reset(next).unwrap(); + assert!(!ring.state().ready()); + assert_eq!(ring.in_flight(), 0); + + assert!(matches!( + chain.io().map(|_| ()), + Err(ChainError::ResetRace { chain, current }) if chain == old && current == next + )); + assert_eq!( + chain.response.write_at(0, b"too late"), + Err(ByteAccessError::Reset) + ); + let mut bytes = [0u8; 6]; + assert_eq!( + chain.request.read_at(0, &mut bytes), + Err(ByteAccessError::Reset) + ); + assert_eq!( + ring.complete(chain, UsedLength::new(0)).err(), + Some(QueueError::ResetRace { + operation: old, + current: next + }) + ); + assert_eq!(guest.used(), None, "no used entry from the old epoch"); + assert_eq!(guest.read(places[1], 8), [0u8; 8]); + + // The queue serves again once the driver sets it up again. + assert_eq!( + ring.pop_available().map(|chain| chain.is_some()), + Err(QueueError::NotReady) + ); + ring.configure(QueueSize::new(4).unwrap()).unwrap(); + ring.set_addresses(guest.at).unwrap(); + ring.set_ready(true).unwrap(); + // The driver starts its rings over too, as a reset requires of it. + guest.memory.store_u16(guest.at.available + 2, 0).unwrap(); + guest.memory.store_u16(guest.at.used + 2, 0).unwrap(); + (guest.published, guest.seen) = (0, 0); + guest.chain(0, &[Part::Read(b"after"), Part::Write(8)]); + guest.publish(0); + let mut chain = ring.pop_available().unwrap().unwrap(); + assert_eq!(chain.id().epoch(), next); + assert_eq!(request(&mut chain), b"after"); + ring.complete(chain, UsedLength::new(0)).unwrap(); + assert_eq!(guest.used(), Some((0, 0))); +} + +/// The queue's own shape is checked when it is made ready, and a driver that +/// breaks the ring's rules afterwards stops the queue, not the device. +#[test] +fn a_queue_the_driver_set_up_wrongly_is_refused() { + // Limits that do not fit each other. + let memory = window(4096); + let size = |n| QueueSize::new(n).unwrap(); + assert!(DeviceRing::new(memory, size(128), 4).is_none()); + assert!(DeviceRing::new(memory, size(8), 1).is_none()); + assert!(DeviceRing::new(memory, size(8), 16).is_none()); + + let (guest, mut ring) = Guest::unready(8, 4); + // Smaller than the longest chain. + assert_eq!( + ring.configure(size(2)).err(), + Some(QueueError::Transport(RingError::ChainLimitExceedsQueue)) + ); + for wrong in [ + RingAddresses { + descriptors: 8, + ..guest.at + }, + RingAddresses { + available: 0x801, + ..guest.at + }, + RingAddresses { + used: 0x1002, + ..guest.at + }, + RingAddresses { + descriptors: WINDOW as u64 - 16, + ..guest.at + }, + RingAddresses { + used: WINDOW as u64 - 8, + ..guest.at + }, + RingAddresses { + available: u64::MAX - 1, + ..guest.at + }, + ] { + ring.set_addresses(wrong).unwrap(); + assert_eq!( + ring.set_ready(true).err(), + Some(QueueError::Transport(RingError::Layout)), + "{wrong:?}" + ); + assert!(!ring.state().ready()); + } + ring.set_addresses(guest.at).unwrap(); + ring.set_ready(true).unwrap(); + // Ready: its shape is fixed until it is made unready or reset. + assert_eq!( + ring.set_addresses(guest.at).err(), + Some(QueueError::Transport(RingError::Busy)) + ); + assert_eq!( + ring.configure(size(8)).err(), + Some(QueueError::Transport(RingError::Busy)) + ); + + // An available index further ahead than the queue is long. + guest.memory.store_u16(guest.at.available + 2, 9).unwrap(); + assert_eq!( + ring.pop_available().map(|chain| chain.is_some()), + Err(QueueError::Transport(RingError::AvailableIndex)) + ); + + // A head published twice while the device holds it. + let (mut guest, mut ring) = Guest::ready(8, 4); + guest.chain(0, &[Part::Read(b"once"), Part::Write(4)]); + guest.publish(0); + guest.publish(0); + let held = ring.pop_available().unwrap().unwrap(); + assert_eq!( + ring.pop_available().map(|chain| chain.is_some()), + Err(QueueError::Transport(RingError::HeadInFlight)) + ); + drop(held); +} + +/// The ring's indices are sixteen bits and wrap; the queue goes on across +/// the wrap, slot by slot. +#[test] +fn the_indices_wrap_and_the_queue_goes_on() { + let (mut guest, mut ring) = Guest::ready(2, 2); + let places = guest.chain(0, &[Part::Read(b"ping"), Part::Write(4)]); + for round in 0..70_000u32 { + guest.publish(0); + let mut chain = ring.pop_available().unwrap().expect("published"); + let (_, _, sink) = chain.io().unwrap().into_parts(); + sink.write_at(0, &round.to_le_bytes()).unwrap(); + ring.complete(chain, UsedLength::new(4)).unwrap(); + assert_eq!(guest.used(), Some((0, 4)), "round {round}"); + assert_eq!(guest.read(places[1], 4), round.to_le_bytes()); + } + assert!(ring.pop_available().unwrap().is_none()); +} diff --git a/native/crates/vdev/src/transport.rs b/native/crates/vdev/src/transport.rs new file mode 100644 index 0000000..9e0ce10 --- /dev/null +++ b/native/crates/vdev/src/transport.rs @@ -0,0 +1,460 @@ +//! **The transport**: one driver's path to one device model. +//! +//! It owns the three things between them: the register block +//! ([`regs`](crate::regs)), the queue in the driver's memory +//! ([`ring`](crate::ring)) and the model ([`VirtualDevice`]). The host +//! calls [`Transport::service`] when the driver rings its doorbell and +//! raises the driver's interrupt when the answer says to. That is the whole +//! of what a host does, so a host is a loop around two notices. +//! +//! The register block is there from the start, so a driver can find out +//! what the device is before it commits memory to it. The driver's memory +//! arrives when the driver lends it ([`Transport::lend`]), which it must +//! have done by the time it says the queue is ready. +//! +//! # What one service does +//! +//! 1. Reads the status the driver has written. Zero is a reset: the queue +//! takes a new epoch, the model forgets everything, and the status the +//! device publishes returns to zero. +//! 2. Otherwise accepts each step the driver has reached that the device +//! has not, in Virtio's order and only in that order: `ACKNOWLEDGE`, +//! `DRIVER`, `FEATURES_OK` if the features the driver wrote are ones the +//! device offered, `DRIVER_OK` if the queue the driver described is one +//! the device can serve. +//! 3. Once live, takes each chain the driver published, hands it to the +//! model and returns it used. +//! +//! A status write that changes what the driver has announced is always +//! answered with an interrupt, whether or not the device accepted the step: +//! a driver writes its status, rings, waits, and then reads what the device +//! made of it. A refusal is the device's status not carrying the step, +//! never silence. +//! +//! Writing the status the device last saw announces nothing and is not +//! answered, zero included: a device in its reset state that is told to +//! reset has nothing to do. That matters because a look is not always the +//! driver's doing. Whoever hosts the device looks whenever anything wakes +//! it, another driver's doorbell among them, and a device that answered +//! every look in its reset state would interrupt a driver that had asked +//! for nothing. +//! +//! # What the driver cannot do to the device +//! +//! The device keeps its own copy of every decision. It never reads back a +//! register it wrote, it reads the driver's registers once per step and +//! acts on what it read, and a feature set or a queue it accepted stays +//! what it accepted whatever the driver writes afterwards. A driver that +//! describes a queue the device cannot serve, or breaks the queue once it +//! is live, gets [`STATUS_DEVICE_NEEDS_RESET`] and nothing more until it +//! resets; a chain that is merely malformed is one chain's failure, returned +//! used with a length of zero. +//! +//! One driver cannot hold the host either: a service takes at most the +//! budget of chains it is given, a second look at the queue counting as +//! one, and says whether more may be waiting. + +use core::num::NonZeroU32; + +use virtio_accel_transport::{ + ChainError, DeviceChain, DeviceQueue, NotificationHint, NotificationRecheck, QueueControl, + QueuePort, QueueSize, UsedLength, +}; + +use crate::device::VirtualDevice; +use crate::mem::Memory; +use crate::regs::{ + self, F_VERSION_1, STATUS_ACKNOWLEDGE, STATUS_DEVICE_NEEDS_RESET, STATUS_DRIVER, + STATUS_DRIVER_OK, STATUS_FAILED, STATUS_FEATURES_OK, +}; +use crate::ring::{DeviceRing, RingAddresses, limits_fit}; + +/// The features every device here offers. The profile has no optional ring +/// feature (no indirect tables, no event indices, no packed ring), so the +/// offer is the one bit that says the device is not a legacy one. +pub const OFFERED_FEATURES: u64 = F_VERSION_1; + +/// Why a transport could not be made. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AttachError { + /// The register window is not the page the block needs. + Registers, + /// The model's configuration does not fit the block's configuration + /// area. + Config, + /// The model's queue limits are not ones the ring can hold. + Queue, +} + +/// The driver's memory changes only while the device is in its first state: +/// a device that has accepted any step may hold a queue in the memory it +/// has, and is reset before that memory is taken from it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Busy; + +/// What one [`Transport::service`] left for the host to do. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Serviced { + /// Raise the driver's interrupt: the device answered a status write, + /// changed its own status, or used a chain the driver asked to hear + /// about. [`INTERRUPTS`](regs::INTERRUPTS) has already moved; the host + /// wakes whoever is parked on it. + pub interrupt: bool, + /// The budget ran out with chains possibly still waiting. The host + /// calls again without waiting for a doorbell: the driver was told not + /// to ring one. + pub pending: bool, +} + +/// Something under the transport stopped holding: a register that would not +/// load, the queue refusing a step it had validated, or the model using +/// more than it was given. Each ends in [`STATUS_DEVICE_NEEDS_RESET`]. +struct Broken; + +/// One driver's path to one device model. +#[derive(Debug)] +pub struct Transport { + registers: M, + /// The queue, once the driver has lent the memory it is in. + ring: Option>, + max_queue: QueueSize, + device: D, + /// The status the device has accepted and published. + status: u32, + /// The driver's status as last read: a doorbell that finds another + /// value is announcing a status write. + announced: u32, + /// The features accepted at `FEATURES_OK`. + features: u64, + /// A reset failed: the status stays `DEVICE_NEEDS_RESET` for good. + dead: bool, + /// Interrupts raised, as [`regs::INTERRUPTS`] publishes them. + interrupts: u32, +} + +impl Transport { + /// A transport for `device`, with the register block in `registers`. + /// Writes the device's half of the block: the driver may map and read + /// it as soon as this returns. + pub fn new(registers: M, device: D) -> Result { + if registers.len() < regs::BYTES { + return Err(AttachError::Registers); + } + let config = device.config(); + if config.len() as u64 > regs::BYTES - regs::CONFIG { + return Err(AttachError::Config); + } + let max_queue = QueueSize::new(D::MAX_QUEUE_SIZE) + .filter(|size| limits_fit(*size, D::MAX_CHAIN_DESCRIPTORS)) + .ok_or(AttachError::Queue)?; + + let block = |at, value| registers.store_u32(at, value); + // The length was checked above, and every offset is a constant of + // the layout, aligned for its word: a fault here is a window that + // does not keep `Memory`'s contract. + let written = registers + .write(regs::CONFIG, config) + .and_then(|()| block(regs::CONFIG_BYTES, config.len() as u32)) + .and_then(|()| block(regs::CONFIG_GENERATION, 0)) + .and_then(|()| block(regs::QUEUE_NUM_MAX, u32::from(D::MAX_QUEUE_SIZE))) + .and_then(|()| block(regs::DEVICE_STATUS, 0)) + .and_then(|()| block(regs::INTERRUPTS, 0)) + .and_then(|()| registers.store_u64(regs::DEVICE_FEATURES, OFFERED_FEATURES)) + .and_then(|()| block(regs::VENDOR_ID, regs::VENDOR_KORE)) + .and_then(|()| block(regs::DEVICE_ID, D::DEVICE_ID)) + .and_then(|()| block(regs::VERSION, regs::VERSION_VALUE)) + // Last: a driver that sees the magic sees the rest. + .and_then(|()| block(regs::MAGIC, regs::MAGIC_VALUE)); + if written.is_err() { + return Err(AttachError::Registers); + } + Ok(Self { + registers, + ring: None, + max_queue, + device, + status: 0, + announced: 0, + features: 0, + dead: false, + interrupts: 0, + }) + } + + /// The driver lends `memory`: device addresses are offsets into it from + /// here on. Answers the memory it replaces, if the driver had lent some + /// before, which the host may now unmap. + pub fn lend(&mut self, memory: M) -> Result, Busy> { + if self.status != 0 { + return Err(Busy); + } + let before = self.withdraw()?; + // The limits were checked when the transport was made. + self.ring = DeviceRing::new(memory, self.max_queue, D::MAX_CHAIN_DESCRIPTORS); + Ok(before) + } + + /// Take back the memory the driver lent, leaving the device with none. + pub fn withdraw(&mut self) -> Result, Busy> { + if self.status != 0 { + return Err(Busy); + } + Ok(self.ring.take().map(DeviceRing::into_memory)) + } + + /// Whether the driver has lent memory. + pub const fn has_memory(&self) -> bool { + self.ring.is_some() + } + + /// The status the device has accepted (`STATUS_*`). + pub const fn status(&self) -> u32 { + self.status + } + + /// The features the driver and device agreed on; zero before + /// `FEATURES_OK`. + pub const fn features(&self) -> u64 { + self.features + } + + /// A reset could not bring the device back. The host drops this + /// transport; a driver that attaches again gets a new one. + pub const fn is_dead(&self) -> bool { + self.dead + } + + /// The model. + pub const fn device(&self) -> &D { + &self.device + } + + /// The driver rang its doorbell, or the last service left chains + /// pending: look at what the driver wrote and act on it, taking at most + /// `budget` chains. + pub fn service(&mut self, budget: NonZeroU32) -> Serviced { + let before = self.status; + let mut serviced = Serviced::default(); + match self.step(budget) { + Ok(done) => serviced = done, + // Told to the driver below, as the status changing. + Err(Broken) => self.status |= STATUS_DEVICE_NEEDS_RESET, + } + if self.status != before { + // Nothing is left to try if the status itself will not store: + // the driver sees the old one, and the device still refuses to + // serve past what it accepted. + let _ = self.registers.store_u32(regs::DEVICE_STATUS, self.status); + serviced.interrupt = true; + } + if serviced.interrupt { + // After everything the interrupt is about: a driver woken by + // this word finds the status and the used ring already written. + self.interrupts = self.interrupts.wrapping_add(1); + let _ = self.registers.store_u32(regs::INTERRUPTS, self.interrupts); + } + serviced + } + + fn step(&mut self, budget: NonZeroU32) -> Result { + let driver = self.driver_u32(regs::DRIVER_STATUS)?; + // Only a status the device has not seen is an announcement. A look + // that finds the same one, a device in its reset state included, + // answers nothing: the look may be another driver's doing. + let answer = Serviced { + interrupt: driver != self.announced, + pending: false, + }; + self.announced = driver; + if driver == 0 { + self.reset(); + return Ok(answer); + } + self.advance(driver)?; + if !self.live() { + return Ok(answer); + } + let served = self.serve(budget)?; + Ok(Serviced { + interrupt: answer.interrupt || served.interrupt, + pending: served.pending, + }) + } + + fn driver_u32(&self, at: u64) -> Result { + self.registers.load_u32(at).map_err(|_| Broken) + } + + fn driver_u64(&self, at: u64) -> Result { + self.registers.load_u64(at).map_err(|_| Broken) + } + + /// Whether the device takes chains: the driver finished bring-up and + /// neither side has given up. + const fn live(&self) -> bool { + self.status & (STATUS_DRIVER_OK | STATUS_FAILED | STATUS_DEVICE_NEEDS_RESET) + == STATUS_DRIVER_OK + } + + /// The driver wrote zero. Nothing accepted means nothing to undo, which + /// is also what makes a reset idempotent. + fn reset(&mut self) { + if self.status == 0 || self.dead { + return; + } + // The queue first: from the new epoch on nothing of the old one is + // taken, written or completed, so the model is torn down with no + // chain in its hands and the driver may reuse its memory once it + // reads the status as zero. + let queue = self.ring.as_mut().is_none_or(|ring| { + ring.state() + .epoch() + .checked_next() + .is_some_and(|epoch| ring.reset(epoch).is_ok()) + }); + let model = self.device.reset().is_ok(); + self.features = 0; + if queue && model { + self.status = 0; + } else { + self.dead = true; + self.status = STATUS_DEVICE_NEEDS_RESET; + } + } + + /// Accept each step `driver` has reached that the device has not, in + /// order. A step whose predecessor is missing is not accepted, and a + /// step once accepted is not revisited: only a reset takes it back. + fn advance(&mut self, driver: u32) -> Result<(), Broken> { + if self.status & (STATUS_DEVICE_NEEDS_RESET | STATUS_FAILED) != 0 { + return Ok(()); + } + if driver & STATUS_FAILED != 0 { + // The driver gave up: the device stops with it. + self.status |= STATUS_FAILED; + return Ok(()); + } + let reached = |status: u32, step: u32, after: u32| { + driver & step != 0 && status & step == 0 && status & after == after + }; + if reached(self.status, STATUS_ACKNOWLEDGE, 0) { + self.status |= STATUS_ACKNOWLEDGE; + } + if reached(self.status, STATUS_DRIVER, STATUS_ACKNOWLEDGE) { + self.status |= STATUS_DRIVER; + } + let named = STATUS_ACKNOWLEDGE | STATUS_DRIVER; + if reached(self.status, STATUS_FEATURES_OK, named) { + let accepted = self.driver_u64(regs::DRIVER_FEATURES)?; + // A subset of the offer that keeps the one bit a driver must + // accept. Anything else leaves `FEATURES_OK` unset, which is + // how Virtio tells a driver its choice was refused. + if accepted & !OFFERED_FEATURES == 0 && accepted & F_VERSION_1 != 0 { + self.features = accepted; + self.status |= STATUS_FEATURES_OK; + } + } + if reached(self.status, STATUS_DRIVER_OK, named | STATUS_FEATURES_OK) { + if self.bring_up_queue()? { + self.status |= STATUS_DRIVER_OK; + } else { + // The driver said it was done and described a queue that + // cannot be served. There is no later step at which it + // could be put right. + self.status |= STATUS_DEVICE_NEEDS_RESET; + } + } + Ok(()) + } + + /// Take the queue the driver described. `false` when it is not one the + /// device can serve: not ready, a size that is not a power of two the + /// model takes, parts outside the driver's memory, or no memory lent + /// for it to be in. + fn bring_up_queue(&mut self) -> Result { + if self.driver_u32(regs::QUEUE_READY)? == 0 { + return Ok(false); + } + let Some(size) = u16::try_from(self.driver_u32(regs::QUEUE_NUM)?) + .ok() + .and_then(QueueSize::new) + else { + return Ok(false); + }; + let addresses = RingAddresses { + descriptors: self.driver_u64(regs::QUEUE_DESC)?, + available: self.driver_u64(regs::QUEUE_DRIVER)?, + used: self.driver_u64(regs::QUEUE_DEVICE)?, + }; + let Some(ring) = self.ring.as_mut() else { + return Ok(false); + }; + Ok(ring.configure(size).is_ok() + && ring.set_addresses(addresses).is_ok() + && ring.set_ready(true).is_ok()) + } + + /// Take what the driver published, at most `budget` chains. + fn serve(&mut self, budget: NonZeroU32) -> Result { + let mut serviced = Serviced::default(); + let mut left = budget.get(); + // Live means the queue was brought up, which took memory. + let Some(ring) = self.ring.as_mut() else { + return Err(Broken); + }; + // The driver need not ring again while the device is looking. + ring.disable_available_notifications().map_err(|_| Broken)?; + loop { + while left != 0 { + let Some(mut chain) = ring.pop_available().map_err(|_| Broken)? else { + break; + }; + left -= 1; + let used = match chain.io() { + Ok(io) => { + let (regions, request, response) = io.into_parts(); + self.device.command(regions, request, response) + } + // One chain's failure: returned used, nothing written. + Err(ChainError::Malformed(_)) => 0, + // No reset runs inside a service, and the chain's bytes + // were checked when it was taken. + Err(ChainError::ResetRace { .. } | ChainError::Transport(_)) => { + return Err(Broken); + } + }; + let hint = ring + .complete(chain, UsedLength::new(used)) + .map_err(|_| Broken)?; + serviced.interrupt |= hint == NotificationHint::Notify; + // The command is answered before its work runs: a driver + // that submits is told so, and then sees it complete. + self.device.drive(); + if self.device.needs_reset() { + self.status |= STATUS_DEVICE_NEEDS_RESET; + return Ok(serviced); + } + } + if left == 0 { + // Notifications stay off: the host comes back by itself. + serviced.pending = true; + return Ok(serviced); + } + match ring.enable_available_notifications().map_err(|_| Broken)? { + NotificationRecheck::Idle => return Ok(serviced), + // Published while the device was not listening. + NotificationRecheck::WorkPending => { + ring.disable_available_notifications().map_err(|_| Broken)?; + // A look costs what a chain costs. A driver that shows + // an index and takes it back before the device reads + // the entry would otherwise keep the device looking, + // at no cost to the budget, for as long as it liked. + left -= 1; + } + } + } + } +} + +#[cfg(test)] +pub(crate) mod tests; diff --git a/native/crates/vdev/src/transport/tests.rs b/native/crates/vdev/src/transport/tests.rs new file mode 100644 index 0000000..9c6410f --- /dev/null +++ b/native/crates/vdev/src/transport/tests.rs @@ -0,0 +1,1070 @@ +use super::*; +use crate::device::{ChainRegion, ReadableBytes, Unrecoverable, WritableBytes}; +use crate::mem::{Window, tests::window}; +use crate::ring::tests::{Guest, Part, WINDOW}; +use crate::ring::{VIRTQ_AVAIL_F_NO_INTERRUPT, VIRTQ_USED_F_NO_NOTIFY}; +use std::vec; +use std::vec::Vec; + +/// A model that answers a request with its bytes reversed, and misbehaves +/// on request. +#[derive(Debug, Default)] +pub(crate) struct Echo { + pub(crate) commands: u32, + pub(crate) drives: u32, + pub(crate) resets: u32, + /// Set by the request `sick`: the model can no longer answer. + sick: bool, + /// A reset does not bring this model back. + pub(crate) unrecoverable: bool, +} + +pub(crate) const ECHO_CONFIG: &[u8] = b"echo config"; + +impl VirtualDevice for Echo { + const DEVICE_ID: u32 = u32::from_le_bytes(*b"echo"); + const MAX_QUEUE_SIZE: u16 = 8; + const MAX_CHAIN_DESCRIPTORS: u16 = 4; + + fn config(&self) -> &[u8] { + ECHO_CONFIG + } + + fn command( + &mut self, + _regions: &[ChainRegion], + request: &dyn ReadableBytes, + response: &mut dyn WritableBytes, + ) -> u32 { + self.commands += 1; + let mut bytes = vec![0u8; request.len() as usize]; + request.read_at(0, &mut bytes).unwrap(); + match bytes.as_slice() { + b"sick" => self.sick = true, + // Claims more than the chain gave it. + b"over" => return response.len() as u32 + 1, + _ => {} + } + bytes.reverse(); + bytes.truncate(response.len() as usize); + response.write_at(0, &bytes).unwrap(); + bytes.len() as u32 + } + + fn drive(&mut self) -> bool { + self.drives += 1; + false + } + + fn needs_reset(&self) -> bool { + self.sick + } + + fn reset(&mut self) -> Result<(), Unrecoverable> { + self.resets += 1; + if self.unrecoverable { + return Err(Unrecoverable); + } + self.sick = false; + Ok(()) + } +} + +const BUDGET: NonZeroU32 = NonZeroU32::new(16).unwrap(); +const NAMED: u32 = STATUS_ACKNOWLEDGE | STATUS_DRIVER; +const AGREED: u32 = NAMED | STATUS_FEATURES_OK; +const LIVE: u32 = AGREED | STATUS_DRIVER_OK; + +/// A driver written the way a C driver is: the register block and its own +/// memory, with every step done by hand. +struct Bench { + registers: Window, + guest: Guest, + transport: Transport, +} + +impl Bench { + fn new() -> Self { + Self::with(Echo::default()) + } + + fn with(model: Echo) -> Self { + let registers = window(regs::BYTES as usize); + let guest = Guest::over(window(WINDOW), Echo::MAX_QUEUE_SIZE); + let mut transport = Transport::new(registers, model).expect("a transport"); + assert!(transport.lend(guest.memory).unwrap().is_none()); + Self { + registers, + guest, + transport, + } + } + + /// Ring the doorbell. + fn ring(&mut self) -> Serviced { + self.transport.service(BUDGET) + } + + fn write_status(&self, status: u32) { + self.registers + .store_u32(regs::DRIVER_STATUS, status) + .unwrap(); + } + + /// The status the device publishes, which is also what it holds. + fn status(&self) -> u32 { + let published = self.registers.load_u32(regs::DEVICE_STATUS).unwrap(); + assert_eq!(published, self.transport.status()); + published + } + + fn describe_queue(&self, size: u32, at: RingAddresses) { + let registers = &self.registers; + registers.store_u32(regs::QUEUE_NUM, size).unwrap(); + registers + .store_u64(regs::QUEUE_DESC, at.descriptors) + .unwrap(); + registers + .store_u64(regs::QUEUE_DRIVER, at.available) + .unwrap(); + registers.store_u64(regs::QUEUE_DEVICE, at.used).unwrap(); + registers.store_u32(regs::QUEUE_READY, 1).unwrap(); + } + + /// Everything up to `FEATURES_OK`. + fn agree(&mut self) { + self.write_status(NAMED); + self.registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + self.write_status(AGREED); + assert!(self.ring().interrupt); + assert_eq!(self.status(), AGREED); + } + + /// The whole bring-up, with the queue the guest laid out. + fn bring_up(&mut self) { + self.agree(); + self.describe_queue(u32::from(self.guest.size), self.guest.at); + self.write_status(LIVE); + assert!(self.ring().interrupt); + assert_eq!(self.status(), LIVE); + } + + /// Publish `request` with `room` bytes for the answer in descriptors + /// `head` and `head + 1`, and say where the answer goes. + fn send(&mut self, head: u16, request: &[u8], room: u32) -> u64 { + let places = self + .guest + .chain(head, &[Part::Read(request), Part::Write(room)]); + self.guest.publish(head); + places[1] + } + + /// A reset, and a queue with nothing in it for the bring-up after. + fn reset(&mut self) { + self.write_status(0); + assert!( + self.ring().interrupt, + "a reset of a device that is not is answered" + ); + self.guest = Guest::over(self.guest.memory, self.guest.size); + for ring in [self.guest.at.available, self.guest.at.used] { + self.guest.memory.write(ring, &[0; 4]).unwrap(); + } + } +} + +#[test] +fn the_block_says_what_the_device_is() { + let bench = Bench::new(); + let registers = &bench.registers; + assert_eq!(registers.load_u32(regs::MAGIC), Ok(regs::MAGIC_VALUE)); + assert_eq!(registers.load_u32(regs::VERSION), Ok(regs::VERSION_VALUE)); + assert_eq!(registers.load_u32(regs::DEVICE_ID), Ok(Echo::DEVICE_ID)); + assert_eq!(registers.load_u32(regs::VENDOR_ID), Ok(regs::VENDOR_KORE)); + assert_eq!(registers.load_u64(regs::DEVICE_FEATURES), Ok(F_VERSION_1)); + assert_eq!(registers.load_u32(regs::DEVICE_STATUS), Ok(0)); + assert_eq!(registers.load_u32(regs::QUEUE_NUM_MAX), Ok(8)); + assert_eq!(registers.load_u32(regs::CONFIG_GENERATION), Ok(0)); + assert_eq!(registers.load_u32(regs::INTERRUPTS), Ok(0)); + assert_eq!( + registers.load_u32(regs::CONFIG_BYTES), + Ok(ECHO_CONFIG.len() as u32) + ); + let mut config = [0u8; ECHO_CONFIG.len()]; + registers.read(regs::CONFIG, &mut config).unwrap(); + assert_eq!(config, ECHO_CONFIG); + + // The words are where a C header says they are. + let mut head = [0u8; 16]; + registers.read(0, &mut head).unwrap(); + assert_eq!(&head[..4], b"kvdv"); + assert_eq!(head[4..8], 1u32.to_le_bytes()); + assert_eq!(&head[8..12], b"echo"); + assert_eq!(&head[12..], b"kore"); +} + +#[test] +fn a_transport_is_refused_what_it_cannot_carry() { + struct Wide; + impl VirtualDevice for Wide { + const DEVICE_ID: u32 = 0; + const MAX_QUEUE_SIZE: u16 = 8; + const MAX_CHAIN_DESCRIPTORS: u16 = 4; + fn config(&self) -> &[u8] { + &[0; (regs::BYTES - regs::CONFIG) as usize + 1] + } + fn command( + &mut self, + _: &[ChainRegion], + _: &dyn ReadableBytes, + _: &mut dyn WritableBytes, + ) -> u32 { + 0 + } + fn drive(&mut self) -> bool { + false + } + fn needs_reset(&self) -> bool { + false + } + fn reset(&mut self) -> Result<(), Unrecoverable> { + Ok(()) + } + } + struct Odd; + impl VirtualDevice for Odd { + const DEVICE_ID: u32 = 0; + const MAX_QUEUE_SIZE: u16 = SIZE; + const MAX_CHAIN_DESCRIPTORS: u16 = CHAIN; + fn config(&self) -> &[u8] { + &[] + } + fn command( + &mut self, + _: &[ChainRegion], + _: &dyn ReadableBytes, + _: &mut dyn WritableBytes, + ) -> u32 { + 0 + } + fn drive(&mut self) -> bool { + false + } + fn needs_reset(&self) -> bool { + false + } + fn reset(&mut self) -> Result<(), Unrecoverable> { + Ok(()) + } + } + + let page = regs::BYTES as usize; + let attach = |registers: usize| Transport::new(window(registers), Echo::default()).map(|_| ()); + assert_eq!(attach(page - 8), Err(AttachError::Registers)); + assert_eq!(attach(page), Ok(())); + assert_eq!( + Transport::new(window(page), Wide).map(|_| ()), + Err(AttachError::Config) + ); + let odd = |made: Result<(), AttachError>| assert_eq!(made, Err(AttachError::Queue)); + odd(Transport::new(window(page), Odd::<6, 2>).map(|_| ())); + odd(Transport::new(window(page), Odd::<128, 2>).map(|_| ())); + odd(Transport::new(window(page), Odd::<8, 1>).map(|_| ())); + odd(Transport::new(window(page), Odd::<8, 9>).map(|_| ())); +} + +#[test] +fn a_driver_steps_through_bring_up() { + let mut bench = Bench::new(); + // A look at a device in its reset state announces nothing, however + // often it comes: whoever hosts the device looks on every wake, and a + // driver that asked for nothing is not interrupted. Nothing is undone. + for _ in 0..3 { + assert_eq!( + bench.ring(), + Serviced { + interrupt: false, + pending: false + } + ); + } + assert_eq!(bench.registers.load_u32(regs::INTERRUPTS), Ok(0)); + assert_eq!(bench.transport.device().resets, 0); + // Zero written over zero is the same look. + bench.write_status(0); + assert!(!bench.ring().interrupt); + + bench.write_status(STATUS_ACKNOWLEDGE); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), STATUS_ACKNOWLEDGE); + // The same status again announces nothing. + assert!(!bench.ring().interrupt); + + bench.write_status(NAMED); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), NAMED); + assert_eq!(bench.transport.features(), 0); + + bench + .registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + bench.write_status(AGREED); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), AGREED); + assert_eq!(bench.transport.features(), F_VERSION_1); + + bench.describe_queue(8, bench.guest.at); + bench.write_status(LIVE); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), LIVE); + assert_eq!(bench.transport.device().commands, 0); +} + +/// A driver may write every step before it rings once: the device takes +/// them in order all the same. +#[test] +fn steps_written_together_are_taken_in_order() { + let mut bench = Bench::new(); + bench + .registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + bench.describe_queue(8, bench.guest.at); + bench.write_status(LIVE); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), LIVE); +} + +#[test] +fn a_step_whose_predecessor_is_missing_is_not_accepted() { + for skipped in [ + STATUS_DRIVER, + STATUS_FEATURES_OK, + STATUS_DRIVER_OK, + STATUS_ACKNOWLEDGE | STATUS_FEATURES_OK, + STATUS_ACKNOWLEDGE | STATUS_DRIVER_OK, + NAMED | STATUS_DRIVER_OK, + ] { + let mut bench = Bench::new(); + bench + .registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + bench.describe_queue(8, bench.guest.at); + bench.write_status(skipped); + assert!(bench.ring().interrupt, "refused, and told so"); + let accepted = bench.status(); + assert_eq!( + accepted & !skipped, + 0, + "nothing the driver did not write: {skipped:#x}" + ); + assert_eq!( + accepted & (STATUS_DRIVER_OK | STATUS_DEVICE_NEEDS_RESET), + 0, + "not live from {skipped:#x}" + ); + // And nothing is served. + bench.send(0, b"hello", 8); + bench.ring(); + assert!(bench.guest.used().is_none()); + } +} + +#[test] +fn features_the_device_did_not_offer_are_refused() { + for refused in [0, 1, F_VERSION_1 | 1, F_VERSION_1 | 1 << 33, u64::MAX] { + let mut bench = Bench::new(); + bench.write_status(NAMED); + bench + .registers + .store_u64(regs::DRIVER_FEATURES, refused) + .unwrap(); + bench.write_status(AGREED); + assert!(bench.ring().interrupt, "the refusal is an answer"); + assert_eq!(bench.status(), NAMED, "{refused:#x}"); + assert_eq!(bench.transport.features(), 0); + + // The queue cannot go live behind a refusal. + bench.describe_queue(8, bench.guest.at); + bench.write_status(LIVE); + bench.ring(); + assert_eq!(bench.status(), NAMED); + } +} + +#[test] +fn a_queue_the_device_cannot_serve_asks_for_a_reset() { + let good = Guest::over(window(64), 8).at; + let outside = |part: fn(&mut RingAddresses) -> &mut u64| { + let mut at = good; + *part(&mut at) = WINDOW as u64; + at + }; + let cases: [(&str, u32, RingAddresses, bool); 9] = [ + ("not ready", 8, good, false), + ("no size", 0, good, true), + ("not a power of two", 6, good, true), + ("larger than the device takes", 16, good, true), + ("larger than a queue is", 0x1_0008, good, true), + ("shorter than a chain", 2, good, true), + ( + "descriptors outside", + 8, + outside(|at| &mut at.descriptors), + true, + ), + ( + "available outside", + 8, + outside(|at| &mut at.available), + true, + ), + ("used outside", 8, outside(|at| &mut at.used), true), + ]; + for (what, size, at, ready) in cases { + let mut bench = Bench::new(); + bench.agree(); + bench.describe_queue(size, at); + if !ready { + bench.registers.store_u32(regs::QUEUE_READY, 0).unwrap(); + } + bench.write_status(LIVE); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), AGREED | STATUS_DEVICE_NEEDS_RESET, "{what}"); + + // Nothing is served, and putting the queue right is too late. + bench.describe_queue(8, good); + bench.send(0, b"hello", 8); + assert!(!bench.ring().interrupt); + assert!(bench.guest.used().is_none()); + assert_eq!(bench.status(), AGREED | STATUS_DEVICE_NEEDS_RESET); + + // A reset is the way back. + bench.reset(); + assert_eq!(bench.status(), 0); + bench.bring_up(); + } +} + +#[test] +fn commands_are_served_and_answered() { + let mut bench = Bench::new(); + bench.bring_up(); + + let first = bench.send(0, b"hello", 16); + let second = bench.send(2, b"device", 3); + assert_eq!( + bench.ring(), + Serviced { + interrupt: true, + pending: false + } + ); + assert_eq!(bench.guest.used(), Some((0, 5))); + assert_eq!(bench.guest.used(), Some((2, 3))); + assert_eq!(bench.guest.used(), None); + assert_eq!(bench.guest.read(first, 5), b"olleh"); + assert_eq!(bench.guest.read(second, 3), b"eci", "no more than the room"); + assert_eq!(bench.transport.device().commands, 2); + assert_eq!(bench.transport.device().drives, 2, "a turn after each"); + + // A doorbell with nothing new is not answered. + assert_eq!(bench.ring(), Serviced::default()); + + // VQ-017: a driver that asked not to be interrupted is not, and still + // finds its answer. + let flags = bench.guest.at.available; + bench + .guest + .memory + .store_u16(flags, VIRTQ_AVAIL_F_NO_INTERRUPT) + .unwrap(); + let third = bench.send(4, b"quiet", 8); + assert_eq!(bench.ring(), Serviced::default()); + assert_eq!(bench.guest.used(), Some((4, 5))); + assert_eq!(bench.guest.read(third, 5), b"teiuq"); +} + +#[test] +fn a_service_takes_no_more_than_its_budget() { + let mut bench = Bench::new(); + bench.bring_up(); + // Four chains of two descriptors fill the queue of eight. + for head in [0u16, 2, 4, 6] { + bench.send(head, &[b'a' + head as u8; 4], 4); + } + let two = NonZeroU32::new(2).unwrap(); + let notify = bench.guest.at.used; + let suppressed = + |bench: &Bench| bench.guest.memory.load_u16(notify).unwrap() == VIRTQ_USED_F_NO_NOTIFY; + + assert_eq!( + bench.transport.service(two), + Serviced { + interrupt: true, + pending: true + } + ); + assert_eq!(bench.transport.device().commands, 2); + assert!(suppressed(&bench), "the host comes back by itself"); + + // Exactly the budget again: the device cannot yet tell it is done. + assert!(bench.transport.service(two).pending); + assert_eq!(bench.transport.device().commands, 4); + assert!(suppressed(&bench)); + + assert_eq!(bench.transport.service(two), Serviced::default()); + assert!(!suppressed(&bench), "idle: the driver rings again"); + let heads: Vec<_> = core::iter::from_fn(|| bench.guest.used()).collect(); + assert_eq!(heads, [(0, 4), (2, 4), (4, 4), (6, 4)]); +} + +/// VQ-004..VQ-006 through the transport: a chain that cannot carry a +/// command is returned used with nothing written, the model never sees it, +/// and the chains after it are served. +#[test] +fn a_malformed_chain_fails_alone() { + let mut bench = Bench::new(); + bench.bring_up(); + bench + .guest + .chain(0, &[Part::Read(b"no room for an answer")]); + bench.guest.publish(0); + // A head that is no descriptor at all. + bench.guest.publish(200); + let answer = bench.send(2, b"fine", 8); + + assert!(bench.ring().interrupt); + assert_eq!(bench.guest.used(), Some((0, 0))); + assert_eq!(bench.guest.used(), Some((200, 0))); + assert_eq!(bench.guest.used(), Some((2, 4))); + assert_eq!(bench.guest.read(answer, 4), b"enif"); + assert_eq!(bench.transport.device().commands, 1); + assert_eq!( + bench.status(), + LIVE, + "one chain's failure is not the device's" + ); +} + +#[test] +fn a_queue_broken_while_live_asks_for_a_reset() { + let mut bench = Bench::new(); + bench.bring_up(); + bench.send(0, b"hello", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.guest.used(), Some((0, 5))); + + // The driver claims more chains than the queue is long. + let idx = bench.guest.at.available + 2; + bench.guest.memory.store_u16(idx, 1 + 9).unwrap(); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), LIVE | STATUS_DEVICE_NEEDS_RESET); + + // Nothing more is taken, however the driver rings. + bench.guest.memory.store_u16(idx, 1).unwrap(); + bench.send(2, b"again", 8); + assert_eq!(bench.ring(), Serviced::default()); + assert!(bench.guest.used().is_none()); + assert_eq!(bench.transport.device().commands, 1); + + bench.reset(); + assert_eq!(bench.status(), 0); + bench.bring_up(); + let answer = bench.send(0, b"back", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.guest.used(), Some((0, 4))); + assert_eq!(bench.guest.read(answer, 4), b"kcab"); +} + +#[test] +fn a_model_that_cannot_go_on_asks_for_a_reset() { + // It says so itself: the command that made it sick is still answered. + let mut bench = Bench::new(); + bench.bring_up(); + bench.send(0, b"sick", 8); + bench.send(2, b"after", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), LIVE | STATUS_DEVICE_NEEDS_RESET); + assert_eq!(bench.guest.used(), Some((0, 4))); + assert_eq!(bench.guest.used(), None, "nothing after it is taken"); + assert_eq!(bench.transport.device().commands, 1); + bench.reset(); + assert_eq!(bench.status(), 0); + assert_eq!(bench.transport.device().resets, 1); + bench.bring_up(); + bench.send(0, b"well", 8); + bench.ring(); + assert_eq!(bench.guest.used(), Some((0, 4))); + + // Or it claims to have written more than the chain gave it: nothing is + // returned used that would tell the driver to read bytes it never lent. + let mut bench = Bench::new(); + bench.bring_up(); + bench.send(0, b"over", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), LIVE | STATUS_DEVICE_NEEDS_RESET); + assert_eq!(bench.guest.used(), None); +} + +/// VQ-018, VQ-019 at the transport: a reset returns the device to its first +/// state, and nothing of the queue from before is used afterwards. +#[test] +fn a_reset_returns_the_device_to_its_first_state() { + let mut bench = Bench::new(); + bench.bring_up(); + bench.send(0, b"hello", 8); + bench.ring(); + // Published, and never served: the reset comes first. + let unanswered = bench.send(2, b"never", 8); + let used_before = bench + .guest + .memory + .load_u16(bench.guest.at.used + 2) + .unwrap(); + + bench.write_status(0); + assert_eq!( + bench.ring(), + Serviced { + interrupt: true, + pending: false + } + ); + assert_eq!(bench.status(), 0); + assert_eq!(bench.transport.features(), 0); + assert_eq!(bench.transport.device().resets, 1); + assert_eq!(bench.transport.device().commands, 1); + assert_eq!( + bench + .guest + .memory + .load_u16(bench.guest.at.used + 2) + .unwrap(), + used_before, + "no used entry after the reset" + ); + assert_eq!(bench.guest.read(unanswered, 5), [0; 5], "and no bytes"); + + // Zero again announces nothing: nothing was accepted since, so nothing + // is undone, and a reset that asked nothing is not answered. + assert!(!bench.ring().interrupt); + assert_eq!(bench.transport.device().resets, 1); + + // The old queue is not live: ringing without a bring-up serves nothing. + bench.write_status(NAMED); + bench.ring(); + assert_eq!( + bench + .guest + .memory + .load_u16(bench.guest.at.used + 2) + .unwrap(), + used_before + ); + + bench.reset(); + assert_eq!(bench.transport.device().resets, 2); + bench.bring_up(); + let answer = bench.send(0, b"anew", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.guest.used(), Some((0, 4)), "from the first entry"); + assert_eq!(bench.guest.read(answer, 4), b"wena"); +} + +#[test] +fn a_reset_that_fails_is_final() { + let mut bench = Bench::with(Echo { + unrecoverable: true, + ..Echo::default() + }); + bench.bring_up(); + assert!(!bench.transport.is_dead()); + + bench.write_status(0); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), STATUS_DEVICE_NEEDS_RESET); + assert!(bench.transport.is_dead()); + + // No bring-up and no further reset changes it. + bench.write_status(NAMED); + bench.ring(); + assert_eq!(bench.status(), STATUS_DEVICE_NEEDS_RESET); + bench.write_status(0); + assert!(bench.ring().interrupt, "still answered"); + assert_eq!(bench.status(), STATUS_DEVICE_NEEDS_RESET); + assert_eq!(bench.transport.device().resets, 1, "and not tried again"); +} + +#[test] +fn a_driver_that_gives_up_stops_the_device() { + let mut bench = Bench::new(); + bench.bring_up(); + bench.write_status(LIVE | STATUS_FAILED); + bench.send(0, b"hello", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), LIVE | STATUS_FAILED); + assert!(bench.guest.used().is_none()); + + // Taking the bit back does not undo it. + bench.write_status(LIVE); + bench.ring(); + assert_eq!(bench.status(), LIVE | STATUS_FAILED); + assert!(bench.guest.used().is_none()); + + bench.reset(); + assert_eq!(bench.status(), 0); +} + +/// The device acts on what it accepted, not on what the registers say now. +#[test] +fn what_was_accepted_stays_accepted() { + let mut bench = Bench::new(); + bench.bring_up(); + let registers = bench.registers; + + // The driver scribbles over everything it can, the device's own + // registers included, and takes back steps it reached. + registers + .store_u64(regs::DRIVER_FEATURES, u64::MAX) + .unwrap(); + registers.store_u32(regs::QUEUE_NUM, 2).unwrap(); + registers.store_u32(regs::QUEUE_READY, 0).unwrap(); + for part in [regs::QUEUE_DESC, regs::QUEUE_DRIVER, regs::QUEUE_DEVICE] { + registers.store_u64(part, WINDOW as u64).unwrap(); + } + registers.store_u32(regs::DEVICE_STATUS, 0).unwrap(); + registers + .store_u64(regs::DEVICE_FEATURES, u64::MAX) + .unwrap(); + registers.store_u32(regs::QUEUE_NUM_MAX, 0x8000).unwrap(); + bench.write_status(STATUS_ACKNOWLEDGE); + + let answer = bench.send(0, b"still", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.guest.used(), Some((0, 5))); + assert_eq!(bench.guest.read(answer, 5), b"llits"); + assert_eq!(bench.transport.status(), LIVE); + assert_eq!(bench.transport.features(), F_VERSION_1); +} + +/// Driver memory in which a chain is published at the worst moment: after +/// the device found the queue empty and before it says it is listening +/// again. The driver saw the device not listening and rang no doorbell. +#[derive(Clone, Debug)] +struct Racing { + window: Window, + /// `(the device's listening flag, the available index, its new value)`, + /// until the race has been run. + race: std::rc::Rc>>, +} + +impl Memory for Racing { + fn len(&self) -> u64 { + self.window.len() + } + fn read(&self, at: u64, target: &mut [u8]) -> Result<(), crate::mem::MemoryFault> { + self.window.read(at, target) + } + fn write(&self, at: u64, source: &[u8]) -> Result<(), crate::mem::MemoryFault> { + self.window.write(at, source) + } + fn load_u16(&self, at: u64) -> Result { + self.window.load_u16(at) + } + fn store_u16(&self, at: u64, value: u16) -> Result<(), crate::mem::MemoryFault> { + if let Some((listening, index, published)) = self.race.get() + && at == listening + && value == 0 + { + self.race.set(None); + self.window.store_u16(index, published)?; + } + self.window.store_u16(at, value) + } + fn load_u32(&self, at: u64) -> Result { + self.window.load_u32(at) + } + fn store_u32(&self, at: u64, value: u32) -> Result<(), crate::mem::MemoryFault> { + self.window.store_u32(at, value) + } + fn load_u64(&self, at: u64) -> Result { + self.window.load_u64(at) + } + fn store_u64(&self, at: u64, value: u64) -> Result<(), crate::mem::MemoryFault> { + self.window.store_u64(at, value) + } +} + +/// VQ-017, the half a single thread cannot reach by itself: the device +/// looks once more after it says it is listening, so the chain no doorbell +/// announced is served in the same service. +#[test] +fn a_chain_published_as_the_device_stops_looking_is_served() { + let race = std::rc::Rc::new(core::cell::Cell::new(None)); + let racing = |window| Racing { + window, + race: race.clone(), + }; + let registers = window(regs::BYTES as usize); + let mut guest = Guest::over(window(WINDOW), Echo::MAX_QUEUE_SIZE); + let mut transport = Transport::new(racing(registers), Echo::default()).unwrap(); + transport.lend(racing(guest.memory)).unwrap(); + + registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + registers.store_u32(regs::QUEUE_NUM, 8).unwrap(); + registers + .store_u64(regs::QUEUE_DESC, guest.at.descriptors) + .unwrap(); + registers + .store_u64(regs::QUEUE_DRIVER, guest.at.available) + .unwrap(); + registers + .store_u64(regs::QUEUE_DEVICE, guest.at.used) + .unwrap(); + registers.store_u32(regs::QUEUE_READY, 1).unwrap(); + registers.store_u32(regs::DRIVER_STATUS, LIVE).unwrap(); + transport.service(BUDGET); + assert_eq!(transport.status(), LIVE); + + // One chain announced, and a second written but not yet published. + guest.chain(0, &[Part::Read(b"first"), Part::Write(8)]); + guest.publish(0); + let late = guest.chain(2, &[Part::Read(b"late"), Part::Write(8)])[1]; + guest + .memory + .write(guest.at.available + 4 + 2, &2u16.to_le_bytes()) + .unwrap(); + race.set(Some((guest.at.used, guest.at.available + 2, 2))); + + assert_eq!( + transport.service(BUDGET), + Serviced { + interrupt: true, + pending: false + } + ); + assert!(race.get().is_none(), "the race was run"); + assert_eq!(guest.used(), Some((0, 5))); + assert_eq!(guest.used(), Some((2, 4)), "served without a doorbell"); + assert_eq!(guest.read(late, 4), b"etal"); + assert_eq!( + guest.memory.load_u16(guest.at.used).unwrap(), + 0, + "and the device is listening when it stops" + ); +} + +/// Driver memory whose available index says one chain is published on +/// every second look and none on the looks between: what a driver on +/// another processor can make the device see by writing the index and +/// taking it back. +#[derive(Clone, Debug)] +struct Flapping { + window: Window, + /// The available index, once the queue is live. + index: std::rc::Rc>>, + looks: std::rc::Rc>, +} + +impl Memory for Flapping { + fn len(&self) -> u64 { + self.window.len() + } + fn read(&self, at: u64, target: &mut [u8]) -> Result<(), crate::mem::MemoryFault> { + self.window.read(at, target) + } + fn write(&self, at: u64, source: &[u8]) -> Result<(), crate::mem::MemoryFault> { + self.window.write(at, source) + } + fn load_u16(&self, at: u64) -> Result { + if self.index.get() == Some(at) { + // The device looks to take a chain, then looks again as it + // says it is listening: nothing, then one, and so on. + self.looks.set(self.looks.get() + 1); + return Ok(u16::from(self.looks.get() % 2 == 0)); + } + self.window.load_u16(at) + } + fn store_u16(&self, at: u64, value: u16) -> Result<(), crate::mem::MemoryFault> { + self.window.store_u16(at, value) + } + fn load_u32(&self, at: u64) -> Result { + self.window.load_u32(at) + } + fn store_u32(&self, at: u64, value: u32) -> Result<(), crate::mem::MemoryFault> { + self.window.store_u32(at, value) + } + fn load_u64(&self, at: u64) -> Result { + self.window.load_u64(at) + } + fn store_u64(&self, at: u64, value: u64) -> Result<(), crate::mem::MemoryFault> { + self.window.store_u64(at, value) + } +} + +/// A driver that shows the device a chain and takes it back, over and +/// over, gets the budget's worth of looks from one service and no more. +#[test] +fn a_driver_that_flaps_its_index_cannot_hold_the_device() { + let index = std::rc::Rc::new(core::cell::Cell::new(None)); + let looks = std::rc::Rc::new(core::cell::Cell::new(0)); + let flapping = |window| Flapping { + window, + index: index.clone(), + looks: looks.clone(), + }; + let registers = window(regs::BYTES as usize); + let guest = Guest::over(window(WINDOW), Echo::MAX_QUEUE_SIZE); + let mut transport = Transport::new(flapping(registers), Echo::default()).unwrap(); + transport.lend(flapping(guest.memory)).unwrap(); + + registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + registers.store_u32(regs::QUEUE_NUM, 8).unwrap(); + registers + .store_u64(regs::QUEUE_DESC, guest.at.descriptors) + .unwrap(); + registers + .store_u64(regs::QUEUE_DRIVER, guest.at.available) + .unwrap(); + registers + .store_u64(regs::QUEUE_DEVICE, guest.at.used) + .unwrap(); + registers.store_u32(regs::QUEUE_READY, 1).unwrap(); + registers.store_u32(regs::DRIVER_STATUS, LIVE).unwrap(); + transport.service(BUDGET); + assert_eq!(transport.status(), LIVE); + + index.set(Some(guest.at.available + 2)); + looks.set(0); + let serviced = transport.service(BUDGET); + assert!(serviced.pending, "the host is asked to come back"); + assert_eq!( + looks.get(), + 2 * BUDGET.get(), + "a look to take and a look to listen, the budget's worth" + ); + assert_eq!(transport.device().commands, 0); + assert_eq!(transport.status(), LIVE, "flapping is not a broken queue"); +} + +/// The block is there before any memory is: a driver identifies the device +/// first. A queue made ready with no memory lent is one the device cannot +/// serve. +#[test] +fn a_queue_with_no_memory_lent_asks_for_a_reset() { + let registers = window(regs::BYTES as usize); + let guest = Guest::over(window(WINDOW), Echo::MAX_QUEUE_SIZE); + let mut transport = Transport::new(registers, Echo::default()).unwrap(); + assert!(!transport.has_memory()); + assert_eq!(registers.load_u32(regs::MAGIC), Ok(regs::MAGIC_VALUE)); + assert_eq!(registers.load_u32(regs::DEVICE_ID), Ok(Echo::DEVICE_ID)); + + registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + registers.store_u32(regs::DRIVER_STATUS, AGREED).unwrap(); + assert!(transport.service(BUDGET).interrupt); + assert_eq!(transport.status(), AGREED, "no memory is needed to agree"); + + registers.store_u32(regs::QUEUE_NUM, 8).unwrap(); + registers + .store_u64(regs::QUEUE_DRIVER, guest.at.available) + .unwrap(); + registers + .store_u64(regs::QUEUE_DEVICE, guest.at.used) + .unwrap(); + registers.store_u32(regs::QUEUE_READY, 1).unwrap(); + registers.store_u32(regs::DRIVER_STATUS, LIVE).unwrap(); + assert!(transport.service(BUDGET).interrupt); + assert_eq!(transport.status(), AGREED | STATUS_DEVICE_NEEDS_RESET); + + // Lending now is too late; after the reset it is not. + assert_eq!(transport.lend(guest.memory).map(|_| ()), Err(Busy)); + registers.store_u32(regs::DRIVER_STATUS, 0).unwrap(); + transport.service(BUDGET); + assert_eq!(transport.status(), 0); + assert!(transport.lend(guest.memory).unwrap().is_none()); + registers.store_u32(regs::DRIVER_STATUS, LIVE).unwrap(); + transport.service(BUDGET); + assert_eq!(transport.status(), LIVE); +} + +/// The driver's memory is the device's for as long as the device has +/// accepted anything: it is replaced or taken back only in the first state. +#[test] +fn memory_changes_only_in_the_first_state() { + let mut bench = Bench::new(); + let other = window(WINDOW); + // In the first state it is replaced, and the old is handed back. + assert!(bench.transport.lend(other).unwrap().is_some()); + assert!(bench.transport.lend(bench.guest.memory).unwrap().is_some()); + + bench.write_status(STATUS_ACKNOWLEDGE); + bench.ring(); + assert_eq!(bench.transport.lend(other).map(|_| ()), Err(Busy)); + assert_eq!(bench.transport.withdraw().map(|_| ()), Err(Busy)); + + bench.write_status(0); + bench.ring(); + bench.bring_up(); + let answer = bench.send(0, b"hello", 8); + bench.ring(); + assert_eq!(bench.guest.read(answer, 5), b"olleh"); + assert_eq!(bench.transport.withdraw().map(|_| ()), Err(Busy)); + + // After a reset the device holds no queue, and gives the memory up. + bench.reset(); + assert!(bench.transport.withdraw().unwrap().is_some()); + assert!(!bench.transport.has_memory()); + assert!(bench.transport.withdraw().unwrap().is_none()); +} + +/// The interrupt is a count: it moves exactly when a service says to +/// interrupt, after what the interrupt is about has been written. +#[test] +fn the_interrupt_count_moves_with_each_interrupt() { + let mut bench = Bench::new(); + let count = |bench: &Bench| bench.registers.load_u32(regs::INTERRUPTS).unwrap(); + assert_eq!(count(&bench), 0); + + bench.bring_up(); + let raised = count(&bench); + assert_eq!(raised, 2, "one for the features, one for the queue"); + + // Nothing to answer: no interrupt, and the count stays. + assert!(!bench.ring().interrupt); + assert_eq!(count(&bench), raised); + + // Two commands in one service are one interrupt. + bench.send(0, b"one", 8); + bench.send(2, b"two", 8); + assert!(bench.ring().interrupt); + assert_eq!(count(&bench), raised + 1); + + // A driver that asked not to be interrupted is not counted either. + let flags = bench.guest.at.available; + bench + .guest + .memory + .store_u16(flags, VIRTQ_AVAIL_F_NO_INTERRUPT) + .unwrap(); + bench.send(4, b"quiet", 8); + assert!(!bench.ring().interrupt); + assert_eq!(count(&bench), raised + 1); + assert_eq!(bench.guest.used(), Some((0, 3))); + + // The driver cannot wind the count: the device keeps its own. + bench.registers.store_u32(regs::INTERRUPTS, 0).unwrap(); + bench.write_status(0); + assert!(bench.ring().interrupt); + assert_eq!(count(&bench), raised + 2); +} diff --git a/native/crates/window/src/client.rs b/native/crates/window/src/client.rs index bd6892b..2fc2bd0 100644 --- a/native/crates/window/src/client.rs +++ b/native/crates/window/src/client.rs @@ -53,6 +53,9 @@ pub enum ClientError { EventMalformed, /// A menu the encoder refused before anything reached the compositor. Menu(crate::menu::MenuError), + /// A clipboard text past [`crate::clipboard::CLIPBOARD_BYTES`], or one + /// longer than its reader can hold: refused whole, never cut short. + TooLong, } impl ClientError { @@ -946,6 +949,101 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { } } + /// Put `text` on this window's seat's clipboard (`res/CLIPBOARD.md`), a + /// request plane at a time. Only the seat's focused window may, on a key + /// press, button press or menu choice it was just delivered and has not + /// spent ([`SurfaceRefusal::Denied`] otherwise). An empty text is no + /// copy; one past the clipboard's bound is refused before anything is + /// sent, and one that is not UTF-8 the compositor refuses whole. + /// `OutOfMemory` leaves the clipboard and the gesture as they were, so a + /// retry may succeed. + pub async fn copy(&mut self, text: &[u8]) -> Result<(), ClientError> { + let total = u32::try_from(text.len()) + .ok() + .filter(|&total| total as usize <= crate::clipboard::CLIPBOARD_BYTES) + .ok_or(ClientError::TooLong)?; + if total == 0 { + return Err(ClientError::Protocol); + } + let mut offset = 0_u32; + for piece in text.chunks(crate::clipboard::CHUNK_BYTES.min(self.menu.len())) { + self.menu[..piece.len()].copy_from_slice(piece); + let len = piece.len() as u16; + match self + .surface_request(SurfaceRequest::CopyChunk { offset, len, total }) + .await? + { + SurfaceResponse::Copied { through } if through == offset + u32::from(len) => { + offset = through; + } + _ => return Err(ClientError::Protocol), + } + } + Ok(()) + } + + /// Read this window's seat's clipboard (`res/CLIPBOARD.md`): `begin` + /// learns the text's length before any of it, and may refuse it + /// (`false`: longer than the caller can hold, [`ClientError::TooLong`]); + /// `piece` then takes the text in order, a request plane at a time, and + /// answers whether it wants more (`false` ends the read there: a reader + /// that needs only the text's start reads no more of it). Answers the + /// length, zero when the clipboard holds no text. Gated as [`Self::copy`] + /// is; a copy landing while the read is under way refuses it + /// ([`SurfaceRefusal::Unknown`]) rather than mixing two texts. + pub async fn paste( + &mut self, + begin: impl FnOnce(u32) -> bool, + mut piece: impl FnMut(&[u8]) -> bool, + ) -> Result { + let mut begin = Some(begin); + let mut offset = 0_u32; + let mut buffer = [0_u8; PASTE_STAGE_BYTES]; + loop { + let (at, len, total) = match self.surface_request(SurfaceRequest::Paste { offset }).await? { + SurfaceResponse::Clipboard { offset: at, len, total } => (at, len, total), + SurfaceResponse::ClipboardEmpty if offset == 0 => return Ok(0), + _ => return Err(ClientError::Protocol), + }; + if at != offset || usize::from(len) > self.menu.len() { + return Err(ClientError::Protocol); + } + if let Some(begin) = begin.take() + && !begin(total) + { + return Err(ClientError::TooLong); + } + // The compositor wrote the plane before it answered; read it as + // the shared memory it is, after the reply's acquire. + let plane = shared_plane(self.menu); + for chunk in plane[..usize::from(len)].chunks(buffer.len()) { + for (byte, shared) in buffer.iter_mut().zip(chunk) { + *byte = shared.load(core::sync::atomic::Ordering::Relaxed); + } + if !piece(&buffer[..chunk.len()]) { + return Ok(total); + } + } + offset += u32::from(len); + if offset == total { + return Ok(total); + } + } + } + + /// Ask for the clipboard to be typed into this terminal window's shell + /// line (`res/CLIPBOARD.md` §Shell): queued for its controller, which + /// takes what the line can hold. Gated as [`Self::copy`] is, and only the + /// seat's terminal window may ask. Answers the text's length, zero when + /// the clipboard holds no text. + pub async fn paste_to_shell(&mut self) -> Result { + match self.surface_request(SurfaceRequest::PasteToShell).await? { + SurfaceResponse::PasteQueued { total } => Ok(total), + SurfaceResponse::ClipboardEmpty => Ok(0), + _ => Err(ClientError::Protocol), + } + } + async fn read_configuration(&mut self) -> Result { let reply = self .control @@ -1392,6 +1490,22 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { } } +/// A paste's piece reaches its reader through this much stack at a time: +/// the plane is read atomically, a byte at a time, and handed on as a slice. +/// Small enough for any task's stack, large enough that a 4 KiB piece is +/// sixteen calls. +const PASTE_STAGE_BYTES: usize = 256; + +/// The request plane as the memory it is: shared with the compositor, which +/// writes it while it answers a paste. +fn shared_plane(plane: &[u8]) -> &[core::sync::atomic::AtomicU8] { + // SAFETY: `AtomicU8` has `u8`'s size and alignment, and every access the + // returned view makes is atomic, so the compositor's writes race with none + // of this process's plain accesses: the plane's plain writers run only + // between requests, never while this view is live. + unsafe { &*(core::ptr::from_ref(plane) as *const [core::sync::atomic::AtomicU8]) } +} + fn valid_configuration(previous: Option, next: Configure) -> bool { next.window != 0 && next.serial != 0 diff --git a/native/crates/window/src/clipboard.rs b/native/crates/window/src/clipboard.rs new file mode 100644 index 0000000..eca5220 --- /dev/null +++ b/native/crates/window/src/clipboard.rs @@ -0,0 +1,107 @@ +//! The seat's clipboard (`res/CLIPBOARD.md`): one text the compositor holds +//! for each seat, which every window of that seat can put text on and read +//! text from, and which no window can touch without the user asking. +//! +//! The text crosses a window's own pair lease, never a lent view: a piece at +//! a time through the request plane the window already maps +//! ([`crate::menu`]'s page), copied out by the compositor before it answers a +//! copy and copied in before it answers a paste. A lent view per copy or +//! paste would spend the window's map cursor, which no unmap gives back, and +//! carry a capability and a revocation for a few hundred bytes of text. +//! +//! **Gesture-gated.** Only the seat's focused window may begin a copy or a +//! paste, and only on a key press, button press or menu choice the +//! compositor delivered to it moments before — each such gesture begins one +//! copy or one paste. A window in the background can neither overwrite the +//! clipboard nor read it. + +/// The most the clipboard holds, in bytes: a copy past it is refused, never +/// cut short. Equal to `kore_edit::CLIPBOARD_BYTES`, the bound every text +/// field already copies within, so any selection a field can copy fits. +pub const CLIPBOARD_BYTES: usize = 512 << 10; + +/// The most one piece of a copy or a paste carries: the request plane whole. +pub const CHUNK_BYTES: usize = crate::menu::MENU_PLANE_BYTES; + +const _: () = assert!(CLIPBOARD_BYTES <= u32::MAX as usize && CHUNK_BYTES <= u16::MAX as usize); + +/// What a paste into a shell's line takes from the text (`res/CLIPBOARD.md` +/// §Shell): the printable ASCII before the first line break, and never a line +/// break, so a paste can never run a command the user has not read. Why the +/// rest was held back is told to the user, never silently dropped. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ShellStop { + /// The whole text was taken. + Whole, + /// The text goes on past a line break. + LineBreak, + /// A character the line cannot hold: a control byte, or a character + /// outside ASCII. + Unholdable, + /// The line is full. + Full, +} + +impl ShellStop { + /// What the user is told about the text held back: nothing when the + /// whole text was typed. + pub const fn describe(self) -> Option<&'static str> { + match self { + Self::Whole => None, + Self::LineBreak => Some("typed the first line; the other lines were held back"), + Self::Unholdable => { + Some("stopped at a tab, control or non-ASCII character the line cannot hold") + } + Self::Full => Some("the line is full; the rest was held back"), + } + } +} + +/// Scan `bytes`, the text from where the last scan stopped, for what a shell +/// paste may take: the length of the run it takes, and why it stops there. +/// `last` says `bytes` reaches the text's end; `room` bounds the run. `None` +/// means `bytes` ran out before the text did: scan the next piece with the +/// room that is left, even none, to learn what follows. +pub fn shell_run(bytes: &[u8], last: bool, room: usize) -> (usize, Option) { + let limit = bytes.len().min(room); + let taken = bytes[..limit] + .iter() + .position(|&byte| !(b' '..=b'~').contains(&byte)) + .unwrap_or(limit); + let stop = match bytes.get(taken) { + Some(b'\r' | b'\n') => Some(ShellStop::LineBreak), + Some(_) if taken < room => Some(ShellStop::Unholdable), + Some(_) => Some(ShellStop::Full), + None if last => Some(ShellStop::Whole), + None => None, + }; + (taken, stop) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A shell paste takes printable ASCII up to the first line break, the + /// first character the line cannot hold, or the line's room. + #[test] + fn a_shell_run_stops_before_what_the_line_cannot_take() { + assert_eq!(shell_run(b"ls -la", true, 255), (6, Some(ShellStop::Whole))); + assert_eq!(shell_run(b"ls -la", false, 255), (6, None)); + assert_eq!(shell_run(b"echo hi\nrm -rf /", true, 255), (7, Some(ShellStop::LineBreak))); + assert_eq!(shell_run(b"a\r\nb", true, 255), (1, Some(ShellStop::LineBreak))); + assert_eq!(shell_run(b"tab\there", true, 255), (3, Some(ShellStop::Unholdable))); + assert_eq!(shell_run("caf\u{e9}".as_bytes(), true, 255), (3, Some(ShellStop::Unholdable))); + assert_eq!(shell_run(b"abcdef", true, 4), (4, Some(ShellStop::Full))); + // A run that fills the room exactly at the text's end took it whole; + // short of the end, what follows decides. + assert_eq!(shell_run(b"abcd", true, 4), (4, Some(ShellStop::Whole))); + assert_eq!(shell_run(b"abcd", false, 4), (4, None)); + assert_eq!(shell_run(b"\nmore", true, 0), (0, Some(ShellStop::LineBreak))); + assert_eq!(shell_run(b"more", true, 0), (0, Some(ShellStop::Full))); + assert_eq!(shell_run(b"", true, 4), (0, Some(ShellStop::Whole))); + // A line break at the room's edge is still a line break: the line + // is not full, the text has another line. + assert_eq!(shell_run(b"abcd\n", true, 4), (4, Some(ShellStop::LineBreak))); + } +} diff --git a/native/crates/window/src/control.rs b/native/crates/window/src/control.rs index 8aec8e2..21ccca6 100644 --- a/native/crates/window/src/control.rs +++ b/native/crates/window/src/control.rs @@ -40,10 +40,17 @@ const SET_LAYOUT_HINTS: u8 = 0x4e; const PREPARE_CAPTURE: u8 = 0x4f; const REQUEST_CAPTURE: u8 = 0x50; const RELEASE_CAPTURE: u8 = 0x51; +const READ_CLIPBOARD: u8 = 0x54; const ACCEPTED: u8 = 0x40; const ADMITTED: u8 = 0x41; const REFUSED: u8 = 0x42; const SHELL_ACTION: u8 = 0x4d; +const CLIPBOARD_TEXT: u8 = 0x54; + +/// Bytes of clipboard text one [`DisplayRequest::ReadClipboard`] answer +/// carries inline: the reply record's four data words. A shell paste takes at +/// most one line of the shell's line editor, a few of these. +pub const CLIPBOARD_TEXT_BYTES: usize = 32; /// Bound on one window's published tab strip. pub const MAX_TABS: usize = 8; @@ -310,6 +317,12 @@ pub enum ShellAction { /// with `token` (never zero), by which it claims the pick. Data only; /// the compositor brokers the answer. Pick { token: u64, types: u64 }, + /// The user pasted into this controller's terminal window + /// (`res/CLIPBOARD.md`): read the clipboard with + /// [`DisplayRequest::ReadClipboard`] under `ticket` (never zero) and type + /// what the line can hold. The ticket reads the text the user pasted and + /// no other: a copy made since refuses it. + Paste { ticket: u64 }, } impl ShellAction { @@ -361,6 +374,7 @@ impl ShellAction { refusal: CaptureRefusal::from_code(b as u8)?, })), 6 if a != 0 && b != 0 => Some(Some(Self::Pick { token: a, types: b })), + 7 if a != 0 && b == 0 => Some(Some(Self::Paste { ticket: a })), _ => None, } } @@ -397,6 +411,7 @@ impl ShellAction { (5, mode.code() as u64, refusal.code() as u64) } Some(Self::Pick { token, types }) => (6, token, types), + Some(Self::Paste { ticket }) => (7, ticket, 0), } } } @@ -668,8 +683,65 @@ pub enum DisplayRequest { ReleaseCapture { ticket: u32, }, + /// Controller verb: read the clipboard text the user pasted into this + /// controller's terminal, from `offset`, under the ticket its + /// [`ShellAction::Paste`] carried. + ReadClipboard { ticket: u64, offset: u32 }, +} + +/// Clipboard text read for a controller: bytes `offset..offset + len` of a +/// `total`-byte text, the first `len` of `bytes` (the rest zero). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ClipboardText { + pub offset: u32, + pub len: u8, + pub total: u32, + pub bytes: [u8; CLIPBOARD_TEXT_BYTES], +} + +impl ClipboardText { + /// The text this answer carries. + pub fn text(&self) -> &[u8] { + &self.bytes[..usize::from(self.len)] + } + + /// The answer's word: `offset | total << 24 | len << 48`. + const fn word(&self) -> u64 { + self.offset as u64 | (self.total as u64) << 24 | (self.len as u64) << 48 + } + + fn from_words(word: u64, data: [u64; 4]) -> Option { + let (offset, total, len) = ((word & 0xff_ffff) as u32, (word >> 24 & 0xff_ffff) as u32, (word >> 48) as u16); + let mut bytes = [0; CLIPBOARD_TEXT_BYTES]; + for (word, chunk) in data.into_iter().zip(bytes.chunks_exact_mut(8)) { + chunk.copy_from_slice(&word.to_le_bytes()); + } + let len = u8::try_from(len).ok()?; + (len != 0 + && usize::from(len) <= CLIPBOARD_TEXT_BYTES + && total != 0 + && total as usize <= crate::clipboard::CLIPBOARD_BYTES + && u64::from(offset) + u64::from(len) <= u64::from(total) + && bytes[usize::from(len)..].iter().all(|&byte| byte == 0)) + .then_some(Self { + offset, + len, + total, + bytes, + }) + } + + fn data(&self) -> [u64; 4] { + core::array::from_fn(|word| { + u64::from_le_bytes(self.bytes[word * 8..word * 8 + 8].try_into().unwrap()) + }) + } } +const _: () = assert!( + crate::clipboard::CLIPBOARD_BYTES < 1 << 24 && CLIPBOARD_TEXT_BYTES == 4 * 8 +); + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DisplayResponse { Desktop(DesktopState), @@ -689,6 +761,7 @@ pub enum DisplayResponse { Configured(Configure), ControlRefused(ControlRefusal), ShellAction(Option), + ClipboardText(ClipboardText), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -848,6 +921,11 @@ pub fn encode_req(request: &DisplayRequest) -> SurReqWire { w.op = RELEASE_CAPTURE; w.subject = u64::from(ticket).into(); } + DisplayRequest::ReadClipboard { ticket, offset } => { + w.op = READ_CLIPBOARD; + w.subject = ticket.into(); + w.a = u64::from(offset).into(); + } } w } @@ -981,6 +1059,16 @@ pub fn decode_req(w: &SurReqWire) -> Option { .ok() .filter(|ticket| *ticket != 0)?, }, + READ_CLIPBOARD => { + let offset = u32::try_from(w.a.get()).ok()?; + if w.subject.get() == 0 || offset as usize >= crate::clipboard::CLIPBOARD_BYTES { + return None; + } + DisplayRequest::ReadClipboard { + ticket: w.subject.get(), + offset, + } + } op if op & OUTPUT != 0 => { DisplayRequest::Output(kore_svc_output::wire::decode_req(&OutReqWire { op: op & !OUTPUT, @@ -1053,6 +1141,15 @@ pub fn encode_rep(response: &DisplayResponse, grants: u8) -> SurRepWire { w.a = a.into(); w.b = b.into(); } + DisplayResponse::ClipboardText(text) => { + w.tag = CLIPBOARD_TEXT; + w.id = text.word().into(); + let [a, b, c, d] = text.data(); + w.a = a.into(); + w.b = b.into(); + w.c = c.into(); + w.d = d.into(); + } } w } @@ -1119,6 +1216,9 @@ pub fn decode_rep(w: &SurRepWire) -> Option<(DisplayResponse, u8)> { SHELL_ACTION => { DisplayResponse::ShellAction(ShellAction::from_wire(w.id.get(), w.a.get(), w.b.get())?) } + CLIPBOARD_TEXT if w.id.get() >> 56 == 0 => DisplayResponse::ClipboardText( + ClipboardText::from_words(w.id.get(), [w.a.get(), w.b.get(), w.c.get(), w.d.get()])?, + ), tag if tag & OUTPUT != 0 => { DisplayResponse::Output(kore_svc_output::wire::decode_rep(&OutRepWire { tag: tag & !OUTPUT, @@ -1193,6 +1293,10 @@ pub fn reply_matches(w: &SurReqWire, response: &DisplayResponse, grants: u8) -> DisplayResponse::Accepted, ) => true, (DisplayRequest::TakeShellAction, DisplayResponse::ShellAction(_)) => true, + ( + DisplayRequest::ReadClipboard { offset, .. }, + DisplayResponse::ClipboardText(text), + ) => text.offset == offset, (DisplayRequest::TakeAdmission, DisplayResponse::Admitted { .. }) => true, ( DisplayRequest::CancelAdmission, @@ -1220,7 +1324,8 @@ pub fn reply_matches(w: &SurReqWire, response: &DisplayResponse, grants: u8) -> | DisplayRequest::TakeShellAction | DisplayRequest::PrepareCapture { .. } | DisplayRequest::RequestCapture { .. } - | DisplayRequest::ReleaseCapture { .. }, + | DisplayRequest::ReleaseCapture { .. } + | DisplayRequest::ReadClipboard { .. }, DisplayResponse::ControlRefused(_), ) => true, (DisplayRequest::CancelAdmission, DisplayResponse::ControlRefused(_)) => true, @@ -1819,6 +1924,67 @@ mod tests { assert_eq!(decode_rep(&wire), None); } + /// A shell paste names its ticket; a clipboard read answers its own + /// offset with a piece inside its text, zero past its length, and + /// nothing else; and an unticketed or out-of-bound read refuses. + #[test] + fn a_shell_paste_reads_its_ticketed_text_and_fails_closed() { + let paste = DisplayResponse::ShellAction(Some(ShellAction::Paste { ticket: 0xfeed })); + assert_eq!(decode_rep(&encode_rep(&paste, 0)), Some((paste, 0))); + let mut bad = encode_rep(&paste, 0); + bad.a = 0.into(); + assert_eq!(decode_rep(&bad), None, "no ticket"); + bad.a = 1.into(); + bad.b = 1.into(); + assert_eq!(decode_rep(&bad), None, "a stray word"); + + let read = DisplayRequest::ReadClipboard { + ticket: 0xfeed, + offset: 32, + }; + let wire = encode_req(&read); + assert_eq!(decode_req(&wire), Some(read)); + for (ticket, offset) in [(0, 0), (1, crate::clipboard::CLIPBOARD_BYTES as u32)] { + assert_eq!( + decode_req(&encode_req(&DisplayRequest::ReadClipboard { ticket, offset })), + None, + "ticket {ticket} offset {offset}" + ); + } + let mut bytes = [0; CLIPBOARD_TEXT_BYTES]; + bytes[..5].copy_from_slice(b"ls -l"); + let text = ClipboardText { + offset: 32, + len: 5, + total: 37, + bytes, + }; + let answer = DisplayResponse::ClipboardText(text); + let reply = encode_rep(&answer, 0); + assert_eq!(decode_rep(&reply), Some((answer, 0))); + assert_eq!(text.text(), b"ls -l"); + assert!(reply_matches(&wire, &answer, 0)); + assert!(!reply_matches(&wire, &answer, 1)); + assert!(reply_matches(&wire, &DisplayResponse::ControlRefused(ControlRefusal::Stale), 0)); + let elsewhere = encode_req(&DisplayRequest::ReadClipboard { + ticket: 0xfeed, + offset: 0, + }); + assert!(!reply_matches(&elsewhere, &answer, 0)); + let mut tail = reply; + tail.d = (tail.d.get() | 1 << 56).into(); + assert_eq!(decode_rep(&tail), None, "a live byte past the piece"); + for (offset, len, total) in [(0u32, 0u16, 5u32), (0, 33, 64), (33, 5, 37), (0, 1, 0)] { + let mut bad = reply; + bad.id = (u64::from(offset) | u64::from(total) << 24 | u64::from(len) << 48).into(); + bad.a = 0.into(); + bad.b = 0.into(); + bad.c = 0.into(); + bad.d = 0.into(); + assert_eq!(decode_rep(&bad), None, "{offset} {len} {total}"); + } + } + #[test] fn the_ninth_dock_row_round_trips_publication_and_launch() { let request = DisplayRequest::PublishDockItem { diff --git a/native/crates/window/src/keysplit.rs b/native/crates/window/src/keysplit.rs new file mode 100644 index 0000000..2b3fc08 --- /dev/null +++ b/native/crates/window/src/keysplit.rs @@ -0,0 +1,315 @@ +//! The keys a terminal window keeps from its shell (`res/KEYBOARD.md` §8). +//! +//! A terminal's keys are its shell's line, except the chords that act on the +//! window itself: its [`TerminalKeys`] — by default Command+C to copy the +//! selection, Command+V to paste and Command+A to select all, as on a Mac, so +//! Control stays the shell's (Control+C interrupts), and Shift with Page Up, +//! Page Down, Home or End to scroll the window's history. With Command held +//! every key is the window's, bound or not: Command never types. +//! +//! A key belongs to whoever it was pressed for, until it is released: Command +//! let go while C is still held does not hand the shell a fresh `c`, and a +//! letter held down before Command was pressed stays the shell's. So the +//! split is a state machine over complete snapshots, one per source, run by +//! whatever stands between the keyboard and the line: input-stream for the +//! founded terminal, the terminal app for a program it hosts. + +use kore_abi::keyboard::{modifier, usage}; + +use crate::shortcut::{Chord, Keymap, Mods}; + +/// Keys one snapshot lists. +pub const SNAPSHOT_KEYS: usize = 6; + +/// What a terminal window does with a key it keeps. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TerminalAction { + /// Put the selection on the clipboard. + Copy, + /// Type the clipboard into the line. + Paste, + /// Select every row held. + SelectAll, + /// Scroll history a page back, a page forward, to its oldest row, or + /// back to the live grid. + PageUp, + PageDown, + Top, + Bottom, +} + +/// A terminal window's keys: the shortcuts it acts on, and whether Command +/// keeps every key from the line. Both sides of the split read one table — +/// input-stream, which takes the keys out of the founded shell's line, and +/// the window, which acts on them — so a keymap the user edits changes both +/// at once. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TerminalKeys { + pub keymap: Keymap, + /// With Command held, every key is the window's, bound or not. + pub command_reserved: bool, +} + +impl TerminalKeys { + pub const DEFAULT: Self = Self { + keymap: Keymap::new() + .with(Chord::new(usage::C, Mods::COMMAND), TerminalAction::Copy) + .with(Chord::new(usage::V, Mods::COMMAND), TerminalAction::Paste) + .with(Chord::new(usage::A, Mods::COMMAND), TerminalAction::SelectAll) + .with(Chord::new(usage::PAGE_UP, Mods::SHIFT), TerminalAction::PageUp) + .with(Chord::new(usage::PAGE_DOWN, Mods::SHIFT), TerminalAction::PageDown) + .with(Chord::new(usage::HOME, Mods::SHIFT), TerminalAction::Top) + .with(Chord::new(usage::END, Mods::SHIFT), TerminalAction::Bottom), + command_reserved: true, + }; + + /// Whether `key`, pressed with the report bitmap `modifiers` held, is the + /// window's rather than the line's. + pub fn owns(&self, key: u8, modifiers: u8) -> bool { + if key == 0 || modifier::bit(key).is_some() { + return false; + } + self.keymap.action(key, modifiers).is_some() + || (self.command_reserved && Mods::of(modifiers).contains(Mods::COMMAND)) + } +} + +impl Default for TerminalKeys { + fn default() -> Self { + Self::DEFAULT + } +} + +/// Sources a split tells apart at once: input-stream's own bound on the +/// keyboards one terminal reads (its `PrevTable`). A further source shares +/// the last place, which forgets its holder's keys as it changes hands — +/// degraded the way every per-source table there is, never dropped. +const SOURCES: usize = 4; + +/// What one source holds, by whom it was pressed for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Held { + source: u8, + window: [u8; SNAPSHOT_KEYS], + shell: [u8; SNAPSHOT_KEYS], + /// The window's view as last handed out: the modifiers and its keys. + shown: (u8, [u8; SNAPSHOT_KEYS]), +} + +impl Held { + /// A source as the window first knows it: nothing held, no modifiers — + /// the state a fresh window's keyboard starts in, so plain typing never + /// reaches the window at all. + const FRESH: Self = Self { + source: 0, + window: [0; SNAPSHOT_KEYS], + shell: [0; SNAPSHOT_KEYS], + shown: (0, [0; SNAPSHOT_KEYS]), + }; +} + +/// One snapshot, split. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Split { + /// The keys the shell's line reads: the snapshot without the window's. + pub shell: [u8; SNAPSHOT_KEYS], + /// The keys the window reads, with the snapshot's modifiers, when what + /// the window sees changed: a key of its own pressed or released, or a + /// modifier, which it needs for a Shift-click. + pub window: Option<[u8; SNAPSHOT_KEYS]>, +} + +/// The keys of `keys` that `side` holds, in their order, packed to the front +/// as a keyboard reports them. +fn pack(keys: [u8; SNAPSHOT_KEYS], side: &[u8; SNAPSHOT_KEYS]) -> [u8; SNAPSHOT_KEYS] { + let mut packed = [0; SNAPSHOT_KEYS]; + for (slot, key) in packed + .iter_mut() + .zip(keys.into_iter().filter(|key| *key != 0 && side.contains(key))) + { + *slot = key; + } + packed +} + +/// The split's state, per source, under one window's keys. +pub struct KeySplit { + keys: TerminalKeys, + held: [Held; SOURCES], + used: usize, +} + +impl KeySplit { + /// A split under `keys`: [`TerminalKeys::DEFAULT`] until the user's + /// keymap names others. + pub const fn new(keys: TerminalKeys) -> Self { + Self { + keys, + held: [Held::FRESH; SOURCES], + used: 0, + } + } + + /// `source`'s place in `held`, of which `used` are taken. + fn place<'h>(held: &'h mut [Held; SOURCES], used: &mut usize, source: u8) -> &'h mut Held { + let index = match held[..*used].iter().position(|held| held.source == source) { + Some(index) => index, + None => { + let index = (*used).min(SOURCES - 1); + *used = (*used + 1).min(SOURCES); + held[index] = Held { source, ..Held::FRESH }; + index + } + }; + &mut held[index] + } + + /// Split `source`'s complete snapshot: `modifiers` held, `keys` down. + pub fn split(&mut self, source: u8, modifiers: u8, keys: [u8; SNAPSHOT_KEYS]) -> Split { + let keys_owned = &self.keys; + let held = Self::place(&mut self.held, &mut self.used, source); + // A key no longer down is released, whoever held it. + for owned in held.window.iter_mut().chain(held.shell.iter_mut()) { + if *owned != 0 && !keys.contains(owned) { + *owned = 0; + } + } + // A key newly down is the owner's its press made it. + for &key in keys.iter().filter(|&&key| key != 0) { + if held.window.contains(&key) || held.shell.contains(&key) { + continue; + } + let side = if keys_owned.owns(key, modifiers) { + &mut held.window + } else { + &mut held.shell + }; + // The two sides partition one snapshot's keys, so a free place is + // always there. + if let Some(free) = side.iter_mut().find(|slot| **slot == 0) { + *free = key; + } + } + let shell = pack(keys, &held.shell); + let window = pack(keys, &held.window); + let view = (modifiers, window); + let changed = held.shown != view; + held.shown = view; + Split { + shell, + window: changed.then_some(window), + } + } + + /// A route boundary: whatever was held is forgotten, as the window + /// forgets it on the focus change that made the boundary. + pub fn reset(&mut self) { + *self = Self::new(self.keys); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use kore_abi::keyboard::modifier::{GUI, LEFT_GUI, LEFT_SHIFT}; + + const C: u8 = 0x06; + const V: u8 = 0x19; + const A: u8 = 0x04; + + fn keys_of(down: &[u8]) -> [u8; SNAPSHOT_KEYS] { + keys(down) + } + + fn keys(down: &[u8]) -> [u8; SNAPSHOT_KEYS] { + let mut keys = [0; SNAPSHOT_KEYS]; + keys[..down.len()].copy_from_slice(down); + keys + } + + /// Command chords and Shift with the scrolling keys are the window's; + /// everything else, Control chords included, is the shell's. + #[test] + fn the_window_keeps_command_chords_and_shifted_scrolling() { + let keys = TerminalKeys::DEFAULT; + assert!(keys.owns(C, LEFT_GUI)); + assert!(keys.owns(V, GUI)); + assert!(keys.owns(usage::PAGE_UP, LEFT_SHIFT)); + assert!(keys.owns(usage::END, LEFT_SHIFT | LEFT_GUI), "Command keeps every key"); + assert!(!keys.owns(usage::PAGE_UP, 0)); + assert!(!keys.owns(C, modifier::LEFT_CTRL)); + assert!(!keys.owns(A, LEFT_SHIFT)); + assert!(!keys.owns(usage::LEFT_GUI, LEFT_GUI), "a modifier's own usage is state"); + assert!(!keys.owns(0, LEFT_GUI)); + assert_eq!(keys.keymap.action(V, modifier::RIGHT_GUI), Some(TerminalAction::Paste)); + assert_eq!(keys.keymap.action(usage::HOME, LEFT_SHIFT), Some(TerminalAction::Top)); + } + + /// The split follows the keymap it was given: a user's binding takes its + /// key from the line, and without Command reserved, unbound Command keys + /// stay the line's. + #[test] + fn the_split_reads_the_keymap_it_was_given() { + let mut keys = TerminalKeys::DEFAULT; + keys.command_reserved = false; + keys.keymap + .bind(Chord::new(usage::INSERT, Mods::SHIFT).unwrap(), TerminalAction::Paste) + .unwrap(); + assert!(keys.owns(usage::INSERT, LEFT_SHIFT)); + assert!(!keys.owns(usage::Z, LEFT_GUI)); + let mut split = KeySplit::new(keys); + let paste = split.split(0, LEFT_SHIFT, keys_of(&[usage::INSERT])); + assert_eq!((paste.shell, paste.window), (keys_of(&[]), Some(keys_of(&[usage::INSERT])))); + } + + /// A key stays with whoever it was pressed for until it is released, and + /// the window hears its own keys and every modifier change, nothing else. + #[test] + fn a_key_belongs_to_whoever_it_was_pressed_for() { + let mut split = KeySplit::new(TerminalKeys::DEFAULT); + // `a` held, then Command: the `a` stays the shell's, and no Command+A. + let first = split.split(1, 0, keys(&[A])); + assert_eq!((first.shell, first.window), (keys(&[A]), None), "plain typing is the line's alone"); + let chord = split.split(1, LEFT_GUI, keys(&[A])); + assert_eq!((chord.shell, chord.window), (keys(&[A]), Some(keys(&[])))); + // Command+C, with `a` still down. + let copy = split.split(1, LEFT_GUI, keys(&[A, C])); + assert_eq!((copy.shell, copy.window), (keys(&[A]), Some(keys(&[C])))); + // Command let go while C is held: still the window's, no `c` typed. + let late = split.split(1, 0, keys(&[A, C])); + assert_eq!((late.shell, late.window), (keys(&[A]), Some(keys(&[C])))); + // Released: C pressed again now is the shell's. + let up = split.split(1, 0, keys(&[A])); + assert_eq!((up.shell, up.window), (keys(&[A]), Some(keys(&[])))); + let again = split.split(1, 0, keys(&[A, C])); + assert_eq!((again.shell, again.window), (keys(&[A, C]), None), "nothing changed for the window"); + // Another source is its own. + let other = split.split(2, LEFT_SHIFT, keys(&[usage::PAGE_UP])); + assert_eq!((other.shell, other.window), (keys(&[]), Some(keys(&[usage::PAGE_UP])))); + assert_eq!(split.split(1, 0, keys(&[A, C])).window, None); + } + + /// A source past the table's bound takes the last place, forgetting the + /// keys its holder had there rather than reading them as its own. + #[test] + fn a_source_past_the_bound_shares_the_last_place() { + let mut split = KeySplit::new(TerminalKeys::DEFAULT); + for source in 0..SOURCES as u8 { + let _ = split.split(source, LEFT_GUI, keys(&[C])); + } + let stranger = split.split(9, 0, keys(&[C])); + assert_eq!((stranger.shell, stranger.window), (keys(&[C]), None)); + let first = split.split(0, 0, keys(&[C])); + assert_eq!((first.shell, first.window), (keys(&[]), Some(keys(&[C]))), "the first sources keep theirs"); + } + + /// A boundary forgets every key, and the window hears the next state. + #[test] + fn a_reset_forgets_the_holders() { + let mut split = KeySplit::new(TerminalKeys::DEFAULT); + let _ = split.split(0, LEFT_GUI, keys(&[V])); + split.reset(); + let after = split.split(0, 0, keys(&[V])); + assert_eq!((after.shell, after.window), (keys(&[V]), None)); + } +} diff --git a/native/crates/window/src/lib.rs b/native/crates/window/src/lib.rs index b0e622d..98bc056 100644 --- a/native/crates/window/src/lib.rs +++ b/native/crates/window/src/lib.rs @@ -6,6 +6,7 @@ pub mod appearance; pub mod capture; #[cfg(feature = "runtime")] pub mod channel; +pub mod clipboard; #[cfg(feature = "runtime")] pub mod client; pub mod command; @@ -21,6 +22,8 @@ pub mod configure; pub mod control; #[cfg(feature = "runtime")] pub mod event; +pub mod keysplit; +pub mod shortcut; mod face; #[cfg(feature = "runtime")] pub mod relay; diff --git a/native/crates/window/src/relay.rs b/native/crates/window/src/relay.rs index 6ecf091..e6fe9a8 100644 --- a/native/crates/window/src/relay.rs +++ b/native/crates/window/src/relay.rs @@ -148,7 +148,25 @@ impl RelayProducer<'_> { /// Loss is explicit, bounded and nonblocking. While a Reset is pending, /// no snapshot is published, even if the consumer has freed ring slots. pub fn push(&mut self, focus: Focus, report: InputReport) -> bool { - if focus.target != FocusTarget::Window || focus.encode().is_none() || !keyboard(report) { + if focus.target != FocusTarget::Window { + return false; + } + self.publish(focus, report) + } + + /// On the terminal route, the keys its window keeps from the shell + /// ([`crate::keysplit`]): the same snapshot ring and loss rule, tagged + /// with the terminal route's epoch, so the compositor hands them to the + /// terminal window and to nothing a later route focuses. + pub fn push_window_keys(&mut self, focus: Focus, report: InputReport) -> bool { + if focus.target != FocusTarget::Terminal { + return false; + } + self.publish(focus, report) + } + + fn publish(&mut self, focus: Focus, report: InputReport) -> bool { + if focus.encode().is_none() || !keyboard(report) { return false; } if self.cell.reset_epoch.load(Ordering::Acquire) == 0 @@ -546,6 +564,7 @@ mod tests { malformed.data[0] = 1; assert!(!input.reports.push(focus, malformed)); assert!(!input.reports.push(Focus { epoch: 0, ..focus }, report)); + assert!(!input.reports.push_window_keys(focus, report), "a window route's keys are pushed whole"); assert_eq!(input.reports.dropped(), 0); assert!(!input.reports.needs_wake()); assert!(!compositor.reports.pending()); @@ -553,6 +572,25 @@ mod tests { assert_eq!(compositor.echo.pop(), None); } + /// The terminal route's window keys ride the ring under its own epoch. + #[test] + fn a_terminal_routes_window_keys_ride_under_its_epoch() { + let mut page = Page([0; 4096]); + let (mut input, mut compositor) = split(&mut page.0).unwrap(); + let chord = InputReport::keyboard(1, kore_abi::keyboard::modifier::LEFT_GUI, [6, 0, 0, 0, 0, 0]); + assert!(input.reports.push_window_keys(Focus::INITIAL, chord)); + assert_eq!( + compositor.reports.poll(), + Some(RelayEvent::Snapshot { + epoch: Focus::INITIAL.epoch, + report: chord + }) + ); + let mut malformed = chord; + malformed.data[0] = 1; + assert!(!input.reports.push_window_keys(Focus::INITIAL, malformed)); + } + /// The seat's lock state rides the snapshot to the window unchanged — a /// client resolves Caps Lock exactly as the terminal discipline does. #[test] diff --git a/native/crates/window/src/shortcut.rs b/native/crates/window/src/shortcut.rs new file mode 100644 index 0000000..a7535cc --- /dev/null +++ b/native/crates/window/src/shortcut.rs @@ -0,0 +1,283 @@ +//! Shortcuts as data (`res/KEYBOARD.md` §8): a [`Chord`] — one key and the +//! modifiers held with it — bound to an action in a [`Keymap`]. +//! +//! A window's shortcuts are a table it reads, never a match over usages: the +//! same table answers what a key does, what its menu says the shortcut is, +//! and — for a terminal — which keys leave the shell's line at all +//! ([`crate::keysplit`]). A keymap the user edits (Settings ▸ Keyboard & +//! Pointer) replaces a table; nothing that reads one changes. +//! +//! Modifiers are side-agnostic: either Shift satisfies a chord's Shift. A +//! chord matches its modifiers exactly, so Command+Shift+C is not Command+C. +//! Locks never take part: Caps Lock changes what a letter types, not which +//! shortcut its key is. + +use core::fmt; + +use kore_abi::keyboard::{modifier, usage}; + +/// The modifiers a chord needs, left and right alike. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct Mods(u8); + +impl Mods { + pub const NONE: Self = Self(0); + pub const CTRL: Self = Self(1); + pub const SHIFT: Self = Self(1 << 1); + pub const ALT: Self = Self(1 << 2); + /// The GUI key: Command on a Mac keyboard, the Super (Windows) key on a + /// PC's. + pub const COMMAND: Self = Self(1 << 3); + + /// The modifiers a report's bitmap holds. + pub const fn of(modifiers: u8) -> Self { + const fn held(modifiers: u8, mask: u8, bit: Mods) -> u8 { + if modifiers & mask != 0 { bit.0 } else { 0 } + } + Self( + held(modifiers, modifier::CTRL, Self::CTRL) + | held(modifiers, modifier::SHIFT, Self::SHIFT) + | held(modifiers, modifier::ALT, Self::ALT) + | held(modifiers, modifier::GUI, Self::COMMAND), + ) + } + + pub const fn with(self, other: Self) -> Self { + Self(self.0 | other.0) + } + + pub const fn contains(self, other: Self) -> bool { + self.0 & other.0 == other.0 + } +} + +/// One key and the modifiers held with it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Chord { + key: u8, + mods: Mods, +} + +impl Chord { + /// `key` (a Keyboard/Keypad-page usage) with `mods` held. `None` for no + /// key or a modifier's own usage: a modifier is held with a chord, never + /// its key. + pub const fn new(key: u8, mods: Mods) -> Option { + if key == 0 || modifier::bit(key).is_some() { + return None; + } + Some(Self { key, mods }) + } + + pub const fn key(self) -> u8 { + self.key + } + + pub const fn mods(self) -> Mods { + self.mods + } + + /// Whether `key`, pressed with the report bitmap `modifiers` held, is + /// this chord. + pub const fn matches(self, key: u8, modifiers: u8) -> bool { + self.key == key && Mods::of(modifiers).0 == self.mods.0 + } + + /// The chord as a menu or the settings page spells it: `Cmd+C`, + /// `Shift+Page Up`. + pub fn write_label(self, out: &mut impl fmt::Write) -> fmt::Result { + for (bit, name) in [ + (Mods::CTRL, "Ctrl"), + (Mods::ALT, "Alt"), + (Mods::SHIFT, "Shift"), + (Mods::COMMAND, "Cmd"), + ] { + if self.mods.contains(bit) { + write!(out, "{name}+")?; + } + } + match self.key { + usage::A..=usage::Z => out.write_char(char::from(b'A' + (self.key - usage::A))), + usage::DIGIT_1..=usage::DIGIT_9 => out.write_char(char::from(b'1' + (self.key - usage::DIGIT_1))), + usage::DIGIT_0 => out.write_char('0'), + usage::F1..=usage::F12 => write!(out, "F{}", self.key - usage::F1 + 1), + key => out.write_str(match key { + usage::ENTER => "Enter", + usage::ESCAPE => "Esc", + usage::BACKSPACE => "Backspace", + usage::TAB => "Tab", + usage::SPACE => "Space", + usage::INSERT => "Insert", + usage::HOME => "Home", + usage::PAGE_UP => "Page Up", + usage::DELETE => "Delete", + usage::END => "End", + usage::PAGE_DOWN => "Page Down", + usage::RIGHT => "Right", + usage::LEFT => "Left", + usage::DOWN => "Down", + usage::UP => "Up", + _ => "Key", + }), + } + } +} + +/// The most bindings one keymap holds: a window's shortcuts, with room for a +/// user's own. +pub const MAX_BINDINGS: usize = 16; + +/// A keymap is full. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct KeymapFull; + +/// Chords bound to a window's actions: one action per chord, any number of +/// chords per action. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Keymap { + bindings: [Option<(Chord, A)>; MAX_BINDINGS], +} + +impl Default for Keymap { + fn default() -> Self { + Self::new() + } +} + +impl Keymap { + pub const fn new() -> Self { + Self { + bindings: [None; MAX_BINDINGS], + } + } + + /// This keymap with `chord` bound to `action`, for a table built as a + /// constant; a chord already bound or no room left is a build error. + pub const fn with(mut self, chord: Option, action: A) -> Self { + let Some(chord) = chord else { + panic!("a binding names a key"); + }; + let mut free = None; + let mut index = 0; + while index < MAX_BINDINGS { + match self.bindings[index] { + Some((bound, _)) if bound.key == chord.key && bound.mods.0 == chord.mods.0 => { + panic!("a chord is bound once"); + } + None if free.is_none() => free = Some(index), + _ => {} + } + index += 1; + } + let Some(free) = free else { + panic!("the keymap is full"); + }; + self.bindings[free] = Some((chord, action)); + self + } +} + +impl Keymap { + /// Bind `chord` to `action`, replacing what it was bound to. + pub fn bind(&mut self, chord: Chord, action: A) -> Result<(), KeymapFull> { + if let Some(bound) = self.bindings.iter_mut().flatten().find(|(bound, _)| *bound == chord) { + bound.1 = action; + return Ok(()); + } + let free = self.bindings.iter_mut().find(|slot| slot.is_none()).ok_or(KeymapFull)?; + *free = Some((chord, action)); + Ok(()) + } + + /// Leave `chord` unbound. + pub fn unbind(&mut self, chord: Chord) { + for slot in &mut self.bindings { + if slot.is_some_and(|(bound, _)| bound == chord) { + *slot = None; + } + } + } + + /// What `key` pressed with the report bitmap `modifiers` held does. + pub fn action(&self, key: u8, modifiers: u8) -> Option { + self.bindings + .iter() + .flatten() + .find(|(chord, _)| chord.matches(key, modifiers)) + .map(|&(_, action)| action) + } + + /// The chord a menu shows for `action`: the first bound to it. + pub fn chord(&self, action: A) -> Option { + self.bindings + .iter() + .flatten() + .find(|(_, bound)| *bound == action) + .map(|&(chord, _)| chord) + } +} + +#[cfg(test)] +mod tests { + extern crate std; + use super::*; + use std::string::String; + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum Act { + Copy, + Paste, + } + + const MAP: Keymap = Keymap::new() + .with(Chord::new(usage::C, Mods::COMMAND), Act::Copy) + .with(Chord::new(usage::V, Mods::COMMAND), Act::Paste) + .with(Chord::new(usage::INSERT, Mods::SHIFT), Act::Paste); + + /// A chord matches its modifiers exactly, either side's, and no lock. + #[test] + fn a_chord_matches_its_modifiers_exactly_either_side() { + assert_eq!(MAP.action(usage::C, modifier::LEFT_GUI), Some(Act::Copy)); + assert_eq!(MAP.action(usage::C, modifier::RIGHT_GUI), Some(Act::Copy)); + assert_eq!(MAP.action(usage::C, modifier::LEFT_GUI | modifier::LEFT_SHIFT), None); + assert_eq!(MAP.action(usage::C, modifier::LEFT_CTRL), None); + assert_eq!(MAP.action(usage::INSERT, modifier::RIGHT_SHIFT), Some(Act::Paste)); + assert_eq!(Chord::new(usage::LEFT_SHIFT, Mods::NONE), None, "a modifier is never a chord's key"); + assert_eq!(Chord::new(0, Mods::COMMAND), None); + } + + /// Bindings change as data: rebinding replaces, unbinding removes, and a + /// menu reads the chord an action is bound to. + #[test] + fn a_keymap_is_rebound_as_data() { + let mut map = MAP; + assert_eq!(map.chord(Act::Paste), Chord::new(usage::V, Mods::COMMAND)); + let ctrl_c = Chord::new(usage::C, Mods::CTRL.with(Mods::SHIFT)).unwrap(); + map.bind(ctrl_c, Act::Copy).unwrap(); + assert_eq!(map.action(usage::C, modifier::LEFT_CTRL | modifier::LEFT_SHIFT), Some(Act::Copy)); + map.bind(ctrl_c, Act::Paste).unwrap(); + assert_eq!(map.action(usage::C, modifier::LEFT_CTRL | modifier::LEFT_SHIFT), Some(Act::Paste)); + map.unbind(Chord::new(usage::V, Mods::COMMAND).unwrap()); + assert_eq!(map.action(usage::V, modifier::LEFT_GUI), None); + assert_eq!(map.chord(Act::Paste), Chord::new(usage::INSERT, Mods::SHIFT)); + let mut full = Keymap::::new(); + for key in 0..MAX_BINDINGS as u8 { + full.bind(Chord::new(usage::A + key, Mods::COMMAND).unwrap(), Act::Copy).unwrap(); + } + assert_eq!(full.bind(Chord::new(usage::F1, Mods::NONE).unwrap(), Act::Copy), Err(KeymapFull)); + } + + /// A chord spells itself as menus show it. + #[test] + fn a_chord_labels_itself() { + let label = |chord: Option| { + let mut text = String::new(); + chord.unwrap().write_label(&mut text).unwrap(); + text + }; + assert_eq!(label(Chord::new(usage::C, Mods::COMMAND)), "Cmd+C"); + assert_eq!(label(Chord::new(usage::PAGE_UP, Mods::SHIFT)), "Shift+Page Up"); + assert_eq!(label(Chord::new(usage::DIGIT_0, Mods::CTRL.with(Mods::ALT))), "Ctrl+Alt+0"); + assert_eq!(label(Chord::new(usage::F1 + 11, Mods::NONE)), "F12"); + } +} diff --git a/native/crates/window/src/wire.rs b/native/crates/window/src/wire.rs index 3581b10..0432959 100644 --- a/native/crates/window/src/wire.rs +++ b/native/crates/window/src/wire.rs @@ -16,6 +16,7 @@ use kore_abi::session::DISPLAY_FORMAT_XRGB8888; use kore_abi::svc::{CommitWire, MAX_COMMIT_RECTS, SurRectWire, SurRepWire, SurReqWire}; use crate::capture::{CaptureFrame, CaptureMode}; +use crate::clipboard::{CHUNK_BYTES, CLIPBOARD_BYTES}; use crate::cursor::Cursor; use crate::document::DocumentKind; use crate::menu::MENU_PLANE_BYTES; @@ -210,6 +211,21 @@ pub enum SurfaceRequest { /// stable identity, or zero if the user cancelled; its name is the /// first `len` bytes of this window's menu plane (none when cancelled). Picked { token: u64, object: u64, len: u8 }, + /// Put text on this window's seat's clipboard ([`crate::clipboard`]), a + /// piece at a time: bytes `offset..offset + len` of a `total`-byte text + /// are at the head of this window's request plane. Offset zero begins the + /// copy (focused window, unspent gesture); the piece that reaches `total` + /// ends it, and the text then replaces the clipboard's. + CopyChunk { offset: u32, len: u16, total: u32 }, + /// Read the seat's clipboard from `offset` into this window's request + /// plane. Offset zero begins the read (focused window, unspent gesture); + /// later offsets continue it until it reaches the end. + Paste { offset: u32 }, + /// The seat's terminal window asks for the clipboard to be typed into its + /// shell's line: queued as an inert + /// [`ShellAction::Paste`](crate::control::ShellAction::Paste) for its + /// controller (focused window, unspent gesture). + PasteToShell, } /// The surface-class reply vocabulary. @@ -263,6 +279,17 @@ pub enum SurfaceResponse { /// The answer is delivered and the pick has ended; the plane may be /// reused. The token, echoed. PickAnswered { token: u64 }, + /// The copy has taken the text through `through`; when that is the whole + /// text, the text is the seat's clipboard. + Copied { through: u32 }, + /// Bytes `offset..offset + len` of the clipboard's `total`-byte text are + /// at the head of the request plane. + Clipboard { offset: u32, len: u16, total: u32 }, + /// The seat's clipboard holds no text. + ClipboardEmpty, + /// The shell paste is queued for this window's controller; the text is + /// `total` bytes. + PasteQueued { total: u32 }, /// The refusal, echoing the subject the verb named (zero if none). Refused { refusal: SurfaceRefusal, @@ -288,6 +315,9 @@ pub enum SurfaceRefusal { Unknown, /// Authority delivery or revocation has not settled; retry is safe. Busy, + /// The user did not ask: a clipboard verb from a window that is not its + /// seat's focused window, or with no unspent gesture behind it. + Denied, } impl SurfaceRefusal { @@ -299,6 +329,7 @@ impl SurfaceRefusal { Self::OutOfMemory => SurRepWire::REFUSE_OUT_OF_MEMORY, Self::Unknown => SurRepWire::REFUSE_UNKNOWN, Self::Busy => SurRepWire::REFUSE_BUSY, + Self::Denied => SurRepWire::REFUSE_DENIED, } } @@ -310,6 +341,7 @@ impl SurfaceRefusal { SurRepWire::REFUSE_OUT_OF_MEMORY => Self::OutOfMemory, SurRepWire::REFUSE_UNKNOWN => Self::Unknown, SurRepWire::REFUSE_BUSY => Self::Busy, + SurRepWire::REFUSE_DENIED => Self::Denied, _ => return None, }) } @@ -457,10 +489,49 @@ pub fn encode_req(req: &SurfaceRequest) -> SurReqWire { w.b = U64::new(object); w.c = U64::new(u64::from(len)); } + SurfaceRequest::CopyChunk { offset, len, total } => { + w.op = SurReqWire::OP_COPY_CHUNK; + w.a = U64::new(piece_word(offset, len)); + w.b = U64::new(u64::from(total)); + } + SurfaceRequest::Paste { offset } => { + w.op = SurReqWire::OP_PASTE; + w.a = U64::new(u64::from(offset)); + } + SurfaceRequest::PasteToShell => { + w.op = SurReqWire::OP_PASTE_TO_SHELL; + } } w } +/// The piece word a copy and a paste share: `offset | len << 32`. +const fn piece_word(offset: u32, len: u16) -> u64 { + offset as u64 | (len as u64) << 32 +} + +/// A clipboard text's length: 1 to [`CLIPBOARD_BYTES`]. +fn text_length(word: u64) -> Option { + u32::try_from(word) + .ok() + .filter(|&len| len != 0 && len as usize <= CLIPBOARD_BYTES) +} + +/// A piece of a clipboard text, lifted: `offset..offset + len` of `total` +/// bytes, `len` 1 to [`CHUNK_BYTES`] and `total` 1 to [`CLIPBOARD_BYTES`]. +fn piece(word: u64, total: u64) -> Option<(u32, u16, u32)> { + let (offset, len) = (word as u32, (word >> 32) as u16); + let total = u32::try_from(total).ok()?; + let fits = u64::from(offset) + u64::from(len) <= u64::from(total); + (word >> 48 == 0 + && len != 0 + && usize::from(len) <= CHUNK_BYTES + && total != 0 + && total as usize <= CLIPBOARD_BYTES + && fits) + .then_some((offset, len, total)) +} + /// Structural lift; stray bits are the re-encode comparison's to catch. fn raw_decode_req(w: &SurReqWire) -> Option { Some(match w.op { @@ -532,6 +603,18 @@ fn raw_decode_req(w: &SurReqWire) -> Option { } SurfaceRequest::Picked { token, object, len } } + SurReqWire::OP_COPY_CHUNK => { + let (offset, len, total) = piece(w.a.get(), w.b.get())?; + SurfaceRequest::CopyChunk { offset, len, total } + } + SurReqWire::OP_PASTE => { + let offset = u32::try_from(w.a.get()).ok()?; + if offset as usize >= CLIPBOARD_BYTES { + return None; + } + SurfaceRequest::Paste { offset } + } + SurReqWire::OP_PASTE_TO_SHELL => SurfaceRequest::PasteToShell, _ => return None, }) } @@ -674,6 +757,22 @@ pub fn encode_rep(rep: &SurfaceResponse, grants: u8) -> SurRepWire { w.tag = SurRepWire::TAG_PICK_ANSWERED; w.a = U64::new(token); } + SurfaceResponse::Copied { through } => { + w.tag = SurRepWire::TAG_COPIED; + w.a = U64::new(u64::from(through)); + } + SurfaceResponse::Clipboard { offset, len, total } => { + w.tag = SurRepWire::TAG_CLIPBOARD; + w.a = U64::new(piece_word(offset, len)); + w.b = U64::new(u64::from(total)); + } + SurfaceResponse::ClipboardEmpty => { + w.tag = SurRepWire::TAG_CLIPBOARD_EMPTY; + } + SurfaceResponse::PasteQueued { total } => { + w.tag = SurRepWire::TAG_PASTE_QUEUED; + w.a = U64::new(u64::from(total)); + } SurfaceResponse::Refused { refusal, subject } => { w.tag = SurRepWire::TAG_REFUSED; w.refusal = refusal.code(); @@ -751,6 +850,17 @@ fn raw_decode_rep(w: &SurRepWire) -> Option { }, SurRepWire::TAG_PICK_CLAIMED => SurfaceResponse::PickClaimed { token: w.a.get() }, SurRepWire::TAG_PICK_ANSWERED => SurfaceResponse::PickAnswered { token: w.a.get() }, + SurRepWire::TAG_COPIED => SurfaceResponse::Copied { + through: text_length(w.a.get())?, + }, + SurRepWire::TAG_CLIPBOARD => { + let (offset, len, total) = piece(w.a.get(), w.b.get())?; + SurfaceResponse::Clipboard { offset, len, total } + } + SurRepWire::TAG_CLIPBOARD_EMPTY => SurfaceResponse::ClipboardEmpty, + SurRepWire::TAG_PASTE_QUEUED => SurfaceResponse::PasteQueued { + total: text_length(w.a.get())?, + }, SurRepWire::TAG_REFUSED => SurfaceResponse::Refused { refusal: SurfaceRefusal::from_code(w.refusal)?, subject: w.id.get(), @@ -867,6 +977,20 @@ pub fn reply_matches(request: &SurReqWire, response: &SurfaceResponse, grants: u | (SurfaceRequest::Picked { token, .. }, SurfaceResponse::PickAnswered { token: got }) => { token == got } + // The copy echoes how far it has taken the text: exactly to the end + // of the piece just sent. + (SurfaceRequest::CopyChunk { offset, len, .. }, SurfaceResponse::Copied { through }) => { + u64::from(through) == u64::from(offset) + u64::from(len) + } + // A read answers from the offset asked, and an empty clipboard + // answers only the read's first piece: a read under way that finds + // the clipboard changed is refused instead. + (SurfaceRequest::Paste { offset }, SurfaceResponse::Clipboard { offset: got, .. }) => { + offset == got + } + (SurfaceRequest::Paste { offset }, SurfaceResponse::ClipboardEmpty) => offset == 0, + (SurfaceRequest::PasteToShell, SurfaceResponse::PasteQueued { .. }) => true, + (SurfaceRequest::PasteToShell, SurfaceResponse::ClipboardEmpty) => true, _ => false, } } @@ -1167,7 +1291,7 @@ mod tests { } } - fn every_request() -> [SurfaceRequest; 14] { + fn every_request() -> [SurfaceRequest; 19] { [ SurfaceRequest::Query, SurfaceRequest::CreateSurface { @@ -1217,10 +1341,25 @@ mod tests { object: u64::MAX, len: PICK_NAME_BYTES as u8, }, + SurfaceRequest::CopyChunk { + offset: 0, + len: 1, + total: 1, + }, + SurfaceRequest::CopyChunk { + offset: (CLIPBOARD_BYTES - CHUNK_BYTES) as u32, + len: CHUNK_BYTES as u16, + total: CLIPBOARD_BYTES as u32, + }, + SurfaceRequest::Paste { offset: 0 }, + SurfaceRequest::Paste { + offset: CLIPBOARD_BYTES as u32 - 1, + }, + SurfaceRequest::PasteToShell, ] } - fn every_response() -> [SurfaceResponse; 13] { + fn every_response() -> [SurfaceResponse; 19] { [ SurfaceResponse::Info { width: 1920, @@ -1260,9 +1399,104 @@ mod tests { SurfaceResponse::PickQueued { serial: 3 }, SurfaceResponse::PickClaimed { token: 0x7071_ce4e }, SurfaceResponse::PickAnswered { token: 0x7071_ce4e }, + SurfaceResponse::Copied { through: 1 }, + SurfaceResponse::Copied { + through: CLIPBOARD_BYTES as u32, + }, + SurfaceResponse::Clipboard { + offset: 4096, + len: 17, + total: 4113, + }, + SurfaceResponse::ClipboardEmpty, + SurfaceResponse::PasteQueued { total: 9 }, + SurfaceResponse::Refused { + refusal: SurfaceRefusal::Denied, + subject: 0, + }, ] } + /// A clipboard piece names a span inside its text, the text inside the + /// clipboard's bound and the piece inside the plane; nothing crosses + /// with a grant; and each reply answers only the verb and offset asked. + #[test] + fn clipboard_pieces_stay_inside_their_text_and_answer_their_own_offset() { + let copy = SurfaceRequest::CopyChunk { + offset: 4096, + len: 100, + total: 4196, + }; + let wire = encode_req(©); + assert_eq!(SurfaceWire::request_grants(&wire), 0); + assert_eq!(decode_req(&wire), Some(copy)); + assert!(reply_matches(&wire, &SurfaceResponse::Copied { through: 4196 }, 0)); + assert!(!reply_matches(&wire, &SurfaceResponse::Copied { through: 4096 }, 0)); + assert!(!reply_matches(&wire, &SurfaceResponse::Copied { through: 4196 }, 1)); + for (offset, len, total) in [ + (0, 0, 1), + (0, 1, 0), + (1, 1, 1), + (0, CHUNK_BYTES as u16 + 1, CLIPBOARD_BYTES as u32), + (0, 1, CLIPBOARD_BYTES as u32 + 1), + (u32::MAX, 1, CLIPBOARD_BYTES as u32), + ] { + let wire = encode_req(&SurfaceRequest::CopyChunk { offset, len, total }); + assert_eq!(decode_req(&wire), None, "{offset} {len} {total}"); + } + let mut stray = encode_req(©); + stray.a = (stray.a.get() | 1 << 50).into(); + assert_eq!(decode_req(&stray), None); + + let read = encode_req(&SurfaceRequest::Paste { offset: 0 }); + let piece = SurfaceResponse::Clipboard { + offset: 0, + len: 12, + total: 12, + }; + assert!(reply_matches(&read, &piece, 0)); + assert!(reply_matches(&read, &SurfaceResponse::ClipboardEmpty, 0)); + let later = encode_req(&SurfaceRequest::Paste { offset: 12 }); + assert!(!reply_matches(&later, &piece, 0)); + assert!( + !reply_matches(&later, &SurfaceResponse::ClipboardEmpty, 0), + "a read under way never learns the clipboard emptied: it is refused" + ); + let past = encode_req(&SurfaceRequest::Paste { + offset: CLIPBOARD_BYTES as u32, + }); + assert_eq!(decode_req(&past), None); + for piece in [ + (0, 13, 12), + (0, 0, 12), + (0, CHUNK_BYTES as u16 + 1, CLIPBOARD_BYTES as u32), + ] { + let mut w = encode_rep(&SurfaceResponse::ClipboardEmpty, 0); + w.tag = SurRepWire::TAG_CLIPBOARD; + w.a = piece_word(piece.0, piece.1).into(); + w.b = u64::from(piece.2).into(); + assert_eq!(decode_rep(&w), None, "{piece:?}"); + } + + let shell = encode_req(&SurfaceRequest::PasteToShell); + assert!(reply_matches(&shell, &SurfaceResponse::PasteQueued { total: 5 }, 0)); + assert!(reply_matches(&shell, &SurfaceResponse::ClipboardEmpty, 0)); + assert!(!reply_matches(&shell, &SurfaceResponse::Copied { through: 5 }, 0)); + let denied = SurfaceResponse::Refused { + refusal: SurfaceRefusal::Denied, + subject: 0, + }; + for request in [copy, SurfaceRequest::Paste { offset: 0 }, SurfaceRequest::PasteToShell] { + assert!(reply_matches(&encode_req(&request), &denied, 0)); + } + for zero in [ + SurfaceResponse::Copied { through: 0 }, + SurfaceResponse::PasteQueued { total: 0 }, + ] { + assert_eq!(decode_rep(&encode_rep(&zero, 0)), None); + } + } + /// A pick is answered only by its own serial's echo, a claim and an /// answer only by their token's, none carries a grant, and a pick of no /// types, a zero serial or token, or an answer whose name contradicts diff --git a/native/provenance.json b/native/provenance.json index 6f475c1..77c02e9 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,9 +1,10 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "7074bda28c416ffe34562857ee43cbc883482782", + "revision": "cac17a02528e13f37a363ed4be71aca40f7adc70", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", + "crates/c-shim": "kore-c-shim", "crates/collections": "kore-collections", "crates/dagfs": "kore-dagfs", "crates/driver": "kore-driver", @@ -16,8 +17,11 @@ "crates/scrub": "kore-scrub", "crates/start": "kore-start", "crates/uart": "kore-uart", + "crates/vdev": "kore-vdev", + "crates/vdev-accel": "kore-vdev-accel", "crates/window": "kore-window", "drivers/nvme": "kore-nvme", + "services/accel-spmd": "kore-svc-accel-spmd", "services/block": "kore-svc-block", "services/dagfs": "kore-svc-dagfs", "services/device": "kore-svc-device", @@ -27,19 +31,22 @@ "services/mm": "kore-svc-mm", "services/output": "kore-svc-output", "services/swap": "kore-svc-swap", + "services/vdev": "kore-svc-vdev", "tools/kore-vol": "kore-vol" }, - "workspace_sha256": "17edafdee9c32e592cda34f7db82a7075ab803793d04119a25b42b11a89784e3", + "workspace_sha256": "41c4f719ce5aa21bc6919a26cac488e54a270c0454b84257c4d163dd65576222", "files": { "LICENSE": "c9b2b12d3bad9080090798a442b4b2a77c48312ae75089f041281b7c8ad21738", "crates/abi/Cargo.toml": "f9a4811b4b6d06947a47a763354cad3bc739a5ea387194a4dcf22409351f3c7b", "crates/abi/src/cap.rs": "41ba3318d372ae447a47f1ef05932204057de17c7df3dd6b9d9a3dff4a935b3e", "crates/abi/src/clock.rs": "cce1c2839cba2377d3cc8696d60812f9195f071f422bd97bb4530ea0368972fc", - "crates/abi/src/command.rs": "206b8b150688cff68345c9237b78d1170e92f01c8323951e2a3e54b5fb808b97", + "crates/abi/src/command.rs": "e994b1527c989cc106281aa127df9a54dc03e1c85f3b15c49ba70ad5e5d65721", "crates/abi/src/crash.rs": "f3c9e5a954e06743f6993b0a5eeb6937d2199c50af5fe95b6be446393ccf999b", - "crates/abi/src/device.rs": "ad4578975e8056f375b0338124f497c9b4b5caa6272ea8a9930e2877d44efac6", + "crates/abi/src/device.rs": "9767bc5990be3e7e13c0542e1f96d738f8f0220d38dbf342742cb4dd3dd1a989", "crates/abi/src/device/census.rs": "6fb54ecaade94b116f41e045b098dc5926e8e1c75eeea33870dd4be01bd1ce23", + "crates/abi/src/device/class.rs": "dacb939e32c175cde259ec2b8ba05132b810583fe6af4615479a0843cc8cc2dc", "crates/abi/src/device/i2c_admission.rs": "1ea2fc04cdb0b15186d2f1d80bde53187860e2197263e225f330965d96e40876", + "crates/abi/src/device/state.rs": "a67c721ce8280c77251eac6d63a3513c8503236e89d2e5ce9d1c9e9245440f9b", "crates/abi/src/execution.rs": "106d679a3f993c3f4e7ad4eb625d40f4399f5b9b50947cb0336ba44a1d6b51af", "crates/abi/src/founding.rs": "801b8e9771763ff9223c3537aabb5653713d1883d1df62ff673f43982aa3e802", "crates/abi/src/guest/aarch64/mod.rs": "1fd24be3fa6cbc344e9873a0e3e75fabd1b919326e477ec4ea4cf148c01b66f8", @@ -48,7 +55,7 @@ "crates/abi/src/guest/wire.rs": "bec45a73cee701b07d40b37f7c296cb5e96f96d2994cd512fd1d9aad7bc7432f", "crates/abi/src/guest/x86_64/mod.rs": "58fe4ec322a03c83e30faf3fef4c7ed6ec06b64d42a33a921e977306148be342", "crates/abi/src/identity.rs": "cf69309ca5e0d53b752f427b8d82eddab874973cfa0a8e120633c326f723f264", - "crates/abi/src/image.rs": "9cc689a9f73015f8e2d3429e57fcc3243dc94a6c929ede623c50515f060065ed", + "crates/abi/src/image.rs": "19b3f6ad647cea4f1bb5a95fc305f93595c059e4dc70a03158d6c9250fc0a3ad", "crates/abi/src/input_status.rs": "5e48ead8883878c52b7ccdb22ff56d7bf04e4d74d7c689573278e7acc5e5b79c", "crates/abi/src/iommu_status.rs": "2a2ef44ec698136ebe213db38b66b08b090b8d2c74a91346a38368ba43c20b02", "crates/abi/src/keyboard.rs": "2a4e763ed9bd76151b1f7450394f3a9a4986a4b4e1a1326124db1e38eb11b8a7", @@ -57,17 +64,36 @@ "crates/abi/src/machine.rs": "7427c50ec38f71dbc83818a5075d03096cd34f5509e22646b04839eeb93cc783", "crates/abi/src/mem.rs": "ae502311778be8ef73f7b9edb1c47f157564f96a58db9add4ad74a313eb3a147", "crates/abi/src/monitor.rs": "8f01b9165d493f78127c3e2d37da401e3c969777b69535612ab2bcbd27eb7871", - "crates/abi/src/pkg.rs": "2893ad39761611ddee84d314bebbed45a9c9823ff145c6a8f28d3844cb40155d", + "crates/abi/src/pkg.rs": "da294e9e37fef2decda03dc5601a2a1131e499378bb14853845c2783b8e1a85a", "crates/abi/src/pointer.rs": "9971a292c1d00b275d8030e4f11fdd71206eaf12671cda8775d220b106ebcbf8", - "crates/abi/src/process.rs": "1dc29781e8d5f500e1e44e8ac36ca466aacf7f14a74b0f730af84336a4319cce", + "crates/abi/src/process.rs": "a5deca9b1d379949b98809a3fcb2bb596e2c06ab18cd27b826151374e6d8a634", "crates/abi/src/processor_status.rs": "c963e48f9b07046ab6c82d4cc2735ee2cc6c94561509a4e6e15927c837c0b0c6", - "crates/abi/src/session.rs": "030b2bee9b46426656db7ebde001738a2cfbdbe775a3be1ae892a14330f2d66f", + "crates/abi/src/session.rs": "ba8e90dc3d1167ac81b52c6b4d6369a7a112b9417342f6697793a7c1673e96b3", "crates/abi/src/stor.rs": "eabad4e5588fd343472660e15cccf32cfcc8d8f4c25eb18dffbd1598b6007db2", - "crates/abi/src/svc.rs": "d857cc812bdbc228c5f852a02ed02a349951cdc268291e51abe4628b894610da", - "crates/abi/src/task.rs": "57cb01a29bd2dbf870a1c49e35424f091fad597396ed5d080fc4f4cdd79c5f2a", + "crates/abi/src/svc.rs": "1bab27eafe928e27ebf75166f7f9da555addf71a27befdcf7494ec4b2fd81e18", + "crates/abi/src/task.rs": "17a9c177b6b3d2dd569dd1930a8d49e84c1a6d28c8ca4bd9a14163c26740d599", "crates/abi/src/uring.rs": "e98d9db77fbe00d5547b582700a1104d2c5b2546bbe57742dfff8362c32dcc2e", "crates/app/Cargo.toml": "fb793b25eec583da7b5e404b19bd656a6b8bcee2a86a20b0da399be2210a9398", "crates/app/src/lib.rs": "5e7e8d791236f278ee85a02a3c37f41412d879cc41d67f6f6d3d701811973d7d", + "crates/c-shim/Cargo.toml": "85a4ad7f806ec588631dda39ef312513282217f3b0ddcdeeb3b5cbf6c1223685", + "crates/c-shim/src/alloc_symbols.rs": "e42b67a45271e7af676425cc915655e95153ee1f198864f5f527541e9c7092d8", + "crates/c-shim/src/errno.rs": "3fe0a17c84d6678f193e63e2db3fad89fef843cdd3bec348e064412b1d730352", + "crates/c-shim/src/format.rs": "d49d93e313f1eb0997deb40cd68942e8987152292c42feaf37546a883d21f4a9", + "crates/c-shim/src/format/float.rs": "eca54620399362cb35d09c842b7f3e40bef6af497c1a5b808df7d9e55c91b413", + "crates/c-shim/src/format/float/tests.rs": "8d579334d89afedcc9945c231a713e36b0baf5e99c5eb8a72d134100068dce9c", + "crates/c-shim/src/format_symbols.rs": "638807fa2f1e33095bfd295707d7ad7b63147d9cdd3cb66a132b49b167682a2f", + "crates/c-shim/src/lib.rs": "d52774104d1c0a3761f2e2825d74154b4e26050ee34bd0dcc93e43226ae41ee1", + "crates/c-shim/src/long_double.rs": "6d4434d610e973ff777f72b728a96537539c98bf076033c906d56e606796ab7b", + "crates/c-shim/src/number.rs": "e1fd500132cafdc70289f713f4ec04ae4e1bc9bc44884ebabbc8591fd6bc96ac", + "crates/c-shim/src/sort.rs": "4afb36cbb5c334d05712780bdee4d11ec30ab6be899f86e9828728f415d4f9db", + "crates/c-shim/src/stdio.rs": "d737a7b844aba0dbb852b6388ccc67c2342205992aaf609a9ac5353b97eea7d2", + "crates/c-shim/src/stdlib_symbols.rs": "5bd8645c999dcbe6e3722971d7ff57c654b74257aae48d9e76aba152f6906b72", + "crates/c-shim/src/string_symbols.rs": "cdcb976d8706218f0478de58993744e213d58a5012c0f25e999d96597c8928c0", + "crates/c-shim/src/va_long_double/aarch64.rs": "b954e8316eae7a30cce9823dcebecc7e8a258eb76a0b82bdc82cb03bd7b90c45", + "crates/c-shim/src/va_long_double/mod.rs": "96bb94ca1c36e0d1933e5abc5740c01a4b208b1d69b3102b0562109898ae35d0", + "crates/c-shim/src/va_long_double/riscv64.rs": "f170be5efa22626436e3b37749d828c20f90b75515a236f7bcf61c725b58673b", + "crates/c-shim/src/va_long_double/x86_64.rs": "49719570f536ac6aa665d7c2e6a5dc2d84ba4601f278f95bb9253fdfb0563f02", + "crates/c-shim/src/wide_symbols.rs": "494d37f1325b5a563c75f51c9ced46f6b338a4eb648a630bcdcd6d09a5d1d4a7", "crates/collections/Cargo.toml": "0b570a215ecd3d18b86b0f1b554e95de6636520c513eefd6f02d16cc2f37ebd6", "crates/collections/src/bitmat.rs": "35c1b7e572de0fa4e3dfc8c1c2cb8b7c5a031a812fb9ed7d4c86466b2a8399ec", "crates/collections/src/bitset.rs": "941e147ef204a6f3100c7d06652c6b9c3efb796ca7bdb738660bf6975a7ed815", @@ -88,23 +114,24 @@ "crates/dagfs/src/entry.rs": "ae077f13df5f0949fbc7c867b5f026aa494843ae605dde4fda845c2a531c33a3", "crates/dagfs/src/error.rs": "f8b93e779e3cc7966db4eaee3098fac4d3615d89441a6fc7ed34061eb6cbaf88", "crates/dagfs/src/face.rs": "b22d6f071397395a8ffda7487f5b876a03fc39590e4a1adefea622d5ae737bc9", - "crates/dagfs/src/fs.rs": "0e0f98c347c6e71363e7f022445e1dc3ccfe6921d2e9ad94f092fcc0296ce1b3", + "crates/dagfs/src/fs.rs": "de64b3e50009141d1be13e6cab3f510ab05cd4ea3d3814642ba2a3b39d388549", "crates/dagfs/src/graph.rs": "833a50a1e6a722ac0d5cefb34b101b9553a9ddd59a5c3d7af0fb723141a063c3", "crates/dagfs/src/ids.rs": "a6cd3127adbbfb3e02c7467f5736e2cabe6b53b9047d98f1cee2ba5de2c78227", "crates/dagfs/src/index.rs": "cd20892b18859bcdf19f63152f994003556bc048b0b0de3816bce250e99d9b81", "crates/dagfs/src/io.rs": "a91034153e6f82d32c9e5b5c8ce21f8b53f35c96adaeb2d507b56b07c75ea577", - "crates/dagfs/src/lib.rs": "aee6c0dd3bec2d44667691a110623a4526a50ff1af3133869b2c7b110e578132", + "crates/dagfs/src/lib.rs": "9d01b5ebc80ba67c7698670e3e9da5c2386d7ba2b4cbe16e3ed2ad4538470bd9", "crates/dagfs/src/name.rs": "a59b8464b96e3a4bb65bd1056e3db5ea80bc3a99da528624ee77757cff4efaf9", "crates/dagfs/src/object.rs": "7c6821771008a5f83b45d8fa53fd9aa00bba405f7a26abec2274962adc1cc00c", "crates/dagfs/src/path.rs": "db0bef7330be961a483e7582df14647d043f2a8fe722eb6320ece488d2fc243d", - "crates/dagfs/src/protocol.rs": "e9a35a3a2d28b1b09e44dc45200323fa258c3ef240d5494afd0b07ea19e17ece", + "crates/dagfs/src/protocol.rs": "de2613cacd587fe64bf613ff5d6de53ed8ccb403d5278bccadc7666e484512f4", "crates/dagfs/src/refusal.rs": "71d612406e9a86509056e6a755e3b3530396232007790588868e8756a5797fe8", "crates/dagfs/src/search.rs": "cfc79463a6367d2fc62a9598ca3fe20d6771c3d9aad07709192da93338a9e4ab", "crates/dagfs/src/stor.rs": "06a270d01e2f3a2dcdb41f803926d863d5fca6229aa14e0cc84337c651acdddb", - "crates/dagfs/src/text.rs": "878ee0abc41bfd30ece0f0a7c6be5935487fb945248773d3eebda8066f8f6032", + "crates/dagfs/src/text.rs": "56517a357cc933ff0f485a8a250d2177baa567d95521b107cc38b558101e7119", "crates/dagfs/src/view.rs": "24d661589ce1bb05e79112cda5ce1027166c7794adc3bcbcfdb3d63e663dfd69", "crates/dagfs/src/walk.rs": "3903476532c8f0933a0db3bf7cae9075c6db8dc06953b479709497329ed7e6a4", - "crates/dagfs/src/wire.rs": "3b5da5293e34e8a6c8b6133228e32b5477c6fef8550c58755ec174b7734feaee", + "crates/dagfs/src/watch.rs": "5070799a20aa65f1c074ec6f605973841547bc378731ea667ee48aac28ab995f", + "crates/dagfs/src/wire.rs": "1777e39074f4da19def29359ce4fe7d079377e4b6911489e37ab540959a2b802", "crates/driver/Cargo.toml": "15154ecb6517360f34580dba34799de5a38d546d8be8a64a626562c59e25e158", "crates/driver/src/accel.rs": "c2355916f1f6647d4bcf8414806c3bb9da45799e458133da70e31190316febfb", "crates/driver/src/block/image.rs": "5bb433b99cafc1a51812469ef8ea637dcc84ad2d2b8a79a3385fb8972884ffb8", @@ -168,12 +195,12 @@ "crates/ld/src/store.rs": "ba5a432a2a3e34e64cdd8f02dccc2c69bab5719b6a4223393c72beca0c7a2b18", "crates/ld/src/stub.rs": "b670715026a889f5baaf233a35d3d24c1891223a0e57665782ee326d90e49f5e", "crates/ld/tests/apply_engine.rs": "e01f5f805f34d614717ceb297275d65936c1b48989df90079fcd2f8ec83b6b1c", - "crates/ld/tests/built_images.rs": "7e6716758f34fab028ac67ce2f016b7e8d7ea93966dd2d300ed4ca254872c055", + "crates/ld/tests/built_images.rs": "63da044373f87670aa24ec7797cb9cdcf85c4f9762c09bf36ff4a49c2729e223", "crates/ld/tests/check_gate.rs": "0ead133777f18f929d5e9b0353a289a5b4fa9e2af929155f525524096ee5578c", "crates/ld/tests/common/mod.rs": "1e4bdfa37511848d9e9458d9a550a1ff2410154411c6d026e9106035044ea438", "crates/ld/tests/late_layout.rs": "9bfc460d8036d69c1acdc52c430ee475e25062519cdc7dd30f235b882414a339", "crates/ld/tests/link_engine.rs": "a9daca8bbb29b510da1acaccc9bbb24466f9c693bd114b7a9ccb8188f31f2a59", - "crates/ld/tests/shipping_list.rs": "113bf634f5a8f83103e266149c2c8daf834f3c99b86d7e3e0693f520283a1469", + "crates/ld/tests/shipping_list.rs": "443f0560cf481e58f3b72277fb6f1dfec9cf9b58cfb369320d0c2e85e174ddea", "crates/ld/tests/static_boot_images.rs": "25fb47a0a8e5b11dca5b8c31f64b90e8d6a6edc3c4d81b0fb5e1733afea3ebe3", "crates/ring/Cargo.toml": "b0a6e8d0f919494ef9e2f8c229ff64e24709347ca76c3d7f5a6e413387c20bcb", "crates/ring/src/event.rs": "a6db5f78e8472f09da1312456f488cf7e6f9a244b1a2cc63b5ea107eb7cd9020", @@ -190,24 +217,24 @@ "crates/rt/src/barrier.rs": "c5bc3bef52f2bcc24edb1016fde955bd01e73074d4ce357d79cae9d3b63e6f3b", "crates/rt/src/block.rs": "70c2aef8bf149da2f32c7c995cd0f4c63cb492647896b107fdd5dbecd3d3042a", "crates/rt/src/cap/mapped.rs": "b29ce431638a29b01edfd72500348f5ae201afe97ec4684ea4449d01e75ec1e7", - "crates/rt/src/cap/mod.rs": "959a357dedc329cb84563f2a8677b462e89da379373022d644ca08ccdb157bc5", + "crates/rt/src/cap/mod.rs": "2e7bd2965128172e4f0f700dd8b64b161236ba4091df13e87d753bb71a71bf33", "crates/rt/src/cap/record.rs": "640b0ff60052d4775c259489af5fbff5cd03164370aceb4775186be956817bcd", - "crates/rt/src/cap/verbs.rs": "0de798ceebe67399a5994980cce2a5a49b8152639af2e3758cd84673701d7cee", + "crates/rt/src/cap/verbs.rs": "7d445d43fd71971a313d98b42e39bf4cb21ed528a1f2a15cbcd49ece02ee9007", "crates/rt/src/chan.rs": "fd50f962b23c2bfa15e72a1800790475d5cf6fcc55503ed39269852f025397e5", "crates/rt/src/clock/aarch64.rs": "47828a573f5fc30a988f012a648a8cb55596b45331cdcdfe1b390cd1d8543339", - "crates/rt/src/clock/mod.rs": "d402d793448359338a361563a0fea91ea43d0264a5eb38c75b9abbb87df89af9", + "crates/rt/src/clock/mod.rs": "5eb69e314f8687a265e12e28307b782d2c2cfe9dc10acab4c02e3e0f1cb2f2cc", "crates/rt/src/clock/riscv64.rs": "17d4ecdb2b4f8152c0cf51f71ce28fe331d929a62405810a1b7eb332bb275c3d", "crates/rt/src/clock/x86_64.rs": "ca499d4dc51b7ed0eb196b831295b5fdecb09a1bd2df628a82e749e27011994a", "crates/rt/src/core_id.rs": "cadde39801a1d0ed05d70d0634a9e52d58043878737d790c4130183751b622cf", - "crates/rt/src/crash.rs": "c750a44492cd981bef6a55c6e0891ef1039e1663f3959e2a5f5289fb354ffe86", - "crates/rt/src/entry.rs": "c08657dd5fa6d88774173730db2501c73a4186aff2689ff866125c2e2189260f", + "crates/rt/src/crash.rs": "b6463a490f80124e22fdb5e13652bbaa94461eb1c4aefcffeca2e7dc7e977611", + "crates/rt/src/entry.rs": "499fd1ac8c1161ceb1241f093f167d7fcb8a5ef0405e5b43dc8e7293942058f3", "crates/rt/src/event.rs": "709a5500231735ad5bdb9ab9ab2fcee7b8f72f93e1b4741afba72fdc8b7c54f3", "crates/rt/src/executor/mod.rs": "071e3dc66bca448e6dabf96976702600bc77fdc98933d58847662fb2de073cb8", "crates/rt/src/executor/task.rs": "b3017cb97f287b8796c61d245c49e5c1b0ebfabe8c535cc2b3842cdbc854dc29", "crates/rt/src/executor/time.rs": "62f9ca9e5e17b395ec2b19a52fc6ed1bb53fd4acc3554cfca81eaf20e48fc272", "crates/rt/src/exit.rs": "3bf7e8ce6dac63b4ce365bdf2e743a9fd171851f1c5e40f8afbb4548877cef1b", "crates/rt/src/fiber_stack.rs": "48bf4fedba07ea2826615368e22882301ae4e0bbce0d893d0d7754d86bd02ce6", - "crates/rt/src/futex.rs": "9ec11227ff7260d141589fa63fdbb238eed3f52e9731ed4d9c7a14c9523a6709", + "crates/rt/src/futex.rs": "df5622a1bd811d7d97ec20dc44a568dba3a2c995254fc30405a60d41371e7390", "crates/rt/src/heap/mod.rs": "a9c39d3de56187a14d4633a229cbbb6e7ddb2c438d6f9f154b3b2551031b9f81", "crates/rt/src/heap/pool.rs": "67d42c6ffc7c7e176d21e97dd3179a3e4ff331f46dd457ed4d29914b1a404de8", "crates/rt/src/heap/shard.rs": "d97f91b3ecc71e3ec56fde0017f9d98cb99c5fbeeffe6ff79ddd153f5c6690fa", @@ -221,31 +248,31 @@ "crates/rt/src/mem_source.rs": "41cbfb3200c222c2d2d0094b19445a9cf7a016e80f20241f8f5cf22872c5ce56", "crates/rt/src/pace.rs": "81217909cee01fa148df9e953e6281d7abf328eed7c6257aac8fbcebc824710c", "crates/rt/src/pinned.rs": "9f86a46572178965c3a111363eb20431fc19916eb9e06c2da50f8f918f3eccf8", - "crates/rt/src/process.rs": "b9286ee4cda4417019e607f258aced1e71fa85d677690099b9602267d27f86a6", + "crates/rt/src/process.rs": "a1de9c1d05a18817adc252fb0c15c82077fbf08659c249670a301978ca039974", "crates/rt/src/process/interrupt.rs": "553bb4aa5678b4efa8d7537c96df0369d8fb810d0b3abb398b71dc7edb3db174", "crates/rt/src/process/words.rs": "cccc85ee65665f1a313c79be8aced6e408899a73986e0d8341ae3734ca79ce31", "crates/rt/src/random.rs": "b6d2738f97a9717d3d9481c49003e6c00df545faf52564189ad1cec508d16af6", "crates/rt/src/reserve.rs": "b2a9cb91873cc404555818d66740e54d4517d7676ad1c1efb620cfd3e43fcdc0", "crates/rt/src/room/mod.rs": "c6f56dda5064ebe75ee85ef122d82c81a507f3fe498b6f9a8463a977a0566c04", "crates/rt/src/room/x86_64.rs": "122e283074fa754a49c73beb733a92660a5d8c53701b5223297ded797c61fc70", - "crates/rt/src/service/mod.rs": "b59017ca341af1f2267dfc1b06d1f11a6e505255b8c327bccee1dfe5d04356fe", - "crates/rt/src/session/builder.rs": "3c08d7a5d4c9c085f11163b4fbf715f31fd1895e0ba67869d06ce73a65401255", + "crates/rt/src/service/mod.rs": "2f31433c345c77bb773036ec1ac259d0a81e74df3b8b24fba0769cc31d50cf8b", + "crates/rt/src/session/builder.rs": "4bea448f727d93bca9f956d7d4b5dda6982390ea616d06fb42b0804281e803b1", "crates/rt/src/session/flows.rs": "7205afbbe1d859868b313a24dccff8902e6ba3c1c8c45031e9531f6286259cae", "crates/rt/src/session/mod.rs": "466c2fbd8af228b3277075ad90a6d81031c1cb0a4c877f75669fff80e15338ea", "crates/rt/src/session/slots.rs": "82594125aa491429ed8c10b1316c19a82a162dc99f6dc97cdaa76398b92558e9", - "crates/rt/src/session/tests.rs": "f0f0ccce35ec87e133e3dcd2d29c7f32790e4be094d31aa113efe4010caf03f2", - "crates/rt/src/spmd.rs": "ebaaa608c8177ed80091215b2b00240ae0d689e620f5fd418c860e78034b7e4b", + "crates/rt/src/session/tests.rs": "80aa5e5d3b0ace53cbbb7a8e9f5bd8760d20f93e3655021172319e88097961ae", + "crates/rt/src/spmd.rs": "8c2264263b737af8538b10c278055bb1843e9c01ead56553022aa754e222c69e", "crates/rt/src/stack_arena.rs": "b252d45bfe845038ca9c8dc32941c634e448a2150dd49cf130e537048602f77b", - "crates/rt/src/star.rs": "a6cf5ff1cbf01f18bb3605e3597da6921ca611da087a3a30ecd61cd026ce6d30", - "crates/rt/src/startup.rs": "b1106a10e82659b8526ba7c94b7e40849edfdcbc30b077a77242712da8ebbd93", - "crates/rt/src/svc/client.rs": "72fb7d66de1981ae253512a1f2026922ce618c9c34eb46ccb47bde465a7fecea", - "crates/rt/src/svc/mod.rs": "a0f3497792d67233da0c96a31a33fe9d291f3ae870fc5570a3940e559795406d", + "crates/rt/src/star.rs": "91de20ffe5734e7a3c553408b6866087af179b7deddb2d576f05be0ca709ab66", + "crates/rt/src/startup.rs": "a005158737d337ea8c270a2860d09762048347e65d2b937734f8b59e3890d324", + "crates/rt/src/svc/client.rs": "062b52a0544eb11b9d4f8a938ebe75299bfd6c2ff7f42a1d8d21e8cb487c16d7", + "crates/rt/src/svc/mod.rs": "208291ede2e702e060f0d735d2ef66034605746c5855c6e5c6d9807de8e15f6a", "crates/rt/src/svc/recyclable.rs": "9e5af8ad316bdd89e001fef7fbb6914b5a936dcdfbab2122740e03c1cf0ee7ec", - "crates/rt/src/svc/serve.rs": "a22172638a94e3ab52b88095dff9d78c88a1d693c81c6242af4932881d268706", + "crates/rt/src/svc/serve.rs": "01ead15cbff6004fe7d13990c373cb6d50c659526942d1b1cf883f05ec72c8d6", "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", - "crates/rt/src/thread.rs": "33c50deece11724565c0d5ca0f8843d41156e6b03123f1f565585c1be1ddc8b7", - "crates/rt/src/thread_controller.rs": "d64362e7e42f310b5fb7ad61c166938b05ee9edf2cedf7742ca6eba59a222137", + "crates/rt/src/thread.rs": "86278fc96aa71883905fbf1e851ea408c34256501b3712ba9459d98a16bc5131", + "crates/rt/src/thread_controller.rs": "298456742190090b14c718669b437906d1390037204e7d69a10877870029d63b", "crates/rt/src/tls.rs": "85bd95bafb3de73d249d7a2eb8779b5afc929077e27b4baa5f7c7428063063ab", "crates/rt/src/transport/mod.rs": "d5af890c800dd53abdbea8329383068cf43f0bf294ccb6b46dd7d8592ae5f321", "crates/rt/src/upcall.rs": "c0cbd2a59479dfba006850bb1bc5dd4f20f59317d178af776ba678f0e6404f02", @@ -264,25 +291,47 @@ "crates/uart/src/pl011.rs": "920c1ad2b67197ea6508de8355d966384a72b9f2ffb72cdfc1c2cbc6b056b261", "crates/uart/src/tx.rs": "532e0fef75563bf5294211502fc0da6534e51e96b4da0a71896fe629c3e5d7be", "crates/uart/src/uart16550.rs": "689427e689025653727db7bbc30a9eb4d0dbf12cc8202d4ae1aef1978f11555c", + "crates/vdev-accel/Cargo.toml": "88d35110e153faa7dfe68089d901bddc07964c32b4f0767cb23c93497c0f018c", + "crates/vdev-accel/src/device.rs": "0e285a5c781dd515263e862fd7fd7689b991d9215734b8a88d38fb6613168da8", + "crates/vdev-accel/src/device/tests.rs": "43f1871d0965c75f9ff1be052e827c68e43b6c0ae6203a5451b2eedfce770e9e", + "crates/vdev-accel/src/fp8.rs": "89d1a9e034706742ededf4fd7a89a56a3f64e9a9d03e53389914c959b760f088", + "crates/vdev-accel/src/fp8/tables.rs": "d7f6f5d80428935cf5ff8a41c18cd5e20d477cca53a70a4ad711e10777ad4a14", + "crates/vdev-accel/src/fp8/tests.rs": "2b4f879cada7a02ba986f00c54327d9c28a22199887e48e461c1d62d61cd7db4", + "crates/vdev-accel/src/lib.rs": "3ee829e95ad07e54ebad67d6f0681c2406884ad5f946407f8f3ec3e8f3d5ed50", + "crates/vdev-accel/tests/session.rs": "774c6a11da5c446d8685a7db3b3dea2aaf4f1139d7a70d4f90e581e2f6eca559", + "crates/vdev/Cargo.toml": "ba7bf6f814d0b3a8aec36be489c95b33635ce267f7cb91185b1662e90098dbec", + "crates/vdev/src/device.rs": "aee2413e3eb6710041c0c7b72fb943038625547a362dca43fb70c6e24eee1710", + "crates/vdev/src/driver.rs": "b1681703bff927b16e114586babdd40278094eb93c3fdcaebcc55a724c941f6b", + "crates/vdev/src/driver/tests.rs": "f6edc3ef3651b981438cf5dc765c6f5cb78e42d880741ea3ebb8e295fcf413b1", + "crates/vdev/src/lib.rs": "d5e950117b8e6db3b99b303e128fa3b0ccf4211a349af1dc2e5f255a679eac09", + "crates/vdev/src/mem.rs": "1b10cacbc8882c80d2ba7377366002027dbd56253e3944fb7e28d2825dcc116f", + "crates/vdev/src/regs.rs": "6257d31c9469c23b72c2b4ebe32c2558278aaae1d95c118aef051f0e7f7833e4", + "crates/vdev/src/ring.rs": "b027dc22fc2f7aa390722ffcd4bfadc95a3be684b6ed11fb181dbd6753ab90ab", + "crates/vdev/src/ring/tests.rs": "1f8fdb5b0e6741e2db26d1c5b5a06d853ca456bc4e5e2918fd8dae49894a32e4", + "crates/vdev/src/transport.rs": "a791afcb6fa52de8d0e3f3e14837520a644bea1379198ac07bedb4f2d9bfb7b6", + "crates/vdev/src/transport/tests.rs": "db2b0c9e3e92122491ae15b00eb4559370db85040c44a5a09a2efbd2c21e9fea", "crates/window/Cargo.toml": "96c2d4c89e179c3718d4b4e897bb0a44c55ac7ad66341529da4bfd35e203592e", "crates/window/src/appearance.rs": "b22e65286b94215bc4265a09ef280a5118f368ac45120311b870a56f7cc40a81", "crates/window/src/capture.rs": "d793f70fff35dfb9b13547f016b5abf8d851a23e53cfe28c1c24ad9949e6ef44", "crates/window/src/channel.rs": "6c96578badb21ddf6f3eb587e65b78af275bd8bcc0e03fd359c88eba6d91138f", - "crates/window/src/client.rs": "2e82a529684f4413cbb1d1475f5d6f108a8eb64f961aa0caec441045c4862d35", + "crates/window/src/client.rs": "485d4bc67d4178c38d1ea53ea7b1c979ea0cde6662df888163f6b2b61a0d82c7", + "crates/window/src/clipboard.rs": "c10c8d7d198b3562900e789077d16761db9cdb69e6425c02bb4a97991a44e55e", "crates/window/src/command.rs": "3ad4867ce62f3a430a68e4d105991229938e131e8cfde8afbbb0f522f99355d3", "crates/window/src/configure.rs": "5731662e919827aa6c9b189dce1d4ae06ebc7f7940c57fafd8662fc33bc46154", - "crates/window/src/control.rs": "0ccad3310bc1abc5efa44ff26d929ee7f7e54c6b83d1a3467c8dca5b3b3f8ca3", + "crates/window/src/control.rs": "3ab2400d32503f75b3419310049252167035179e76f8c838a89094d17946c9a5", "crates/window/src/cursor.rs": "2c320f4f19af4a5f9cd0d4059d1452661ac8cea23f0f6a44e0833cccd4d5f4bd", "crates/window/src/desktop.rs": "2aca94383249fac9f770c4edae35f634ad89388c463b17e4484125190757532b", "crates/window/src/document.rs": "1c4fb5a4bd3ce170a8a79ea97e7efd9aa013b1d2678b5ca1dde06ea1508ddaca", "crates/window/src/event.rs": "cf6565ea882e6c77672f31f0b354b03c57f25591a558893bb9466070b3f503a4", "crates/window/src/face.rs": "1078d8664e6da529c9d4fd43a7e3df77f287d0061f418ed21859d289e86765a0", - "crates/window/src/lib.rs": "310eeb5e3fdafe4e881008d311d0de4346bc1f7a78d417cba2c1e59e354670d6", + "crates/window/src/keysplit.rs": "044360a955b04a5229cba55956e9307a04d7d78facf2dfcf9903a43e3012e3fb", + "crates/window/src/lib.rs": "8b61e7f5e21e981069fc7409d2a7b5fb7add9c3a05aa5db29aa5685cbd5dc479", "crates/window/src/menu.rs": "8baf547a2dd58f83eef635e16e8658a011c917b60125b5d041c885b9eb69db1e", "crates/window/src/notify.rs": "8215605e26509fd615a983eab746ddcec18a156543349f52ddff6f301bdfe569", "crates/window/src/pick.rs": "d861daa5137f7513bc142737da8bd104f84c423e9c26af4f48ef19a3afe1b5aa", - "crates/window/src/relay.rs": "8f56568bcf643825906c74180ee13a14acb8299f6fb08f968919370e82669348", - "crates/window/src/wire.rs": "572daa34bc6233540779c542cd47533ec9c294123562de88ea6a5c2fb54572bf", + "crates/window/src/relay.rs": "bd96d827e9074dfdaddc4bb5a0ea5f924b4c63e5fd2bbffba033ab12f5c6b10b", + "crates/window/src/shortcut.rs": "ce72430d30a58833ddb4bc5c95ff12802b09719bd99383296a52a4d55af70e97", + "crates/window/src/wire.rs": "43789b5b72b003ae74b05b79ae4d1c25a3d015a3798dcddd31c664fe6cbb421f", "drivers/nvme/Cargo.toml": "21acf81448d0cc403eb10ada1c407baf7ff8cea4778ccdeb6c4f0d3508a66fbd", "drivers/nvme/src/block.rs": "4df5122b9c8f28ae878bffb6eeacfdba94569f73182c147a555f345a4d877723", "drivers/nvme/src/commands.rs": "a4cdbfb2b7822143beb7f351f4bba7b4643da1bffa80ffe443f065e47545c097", @@ -294,14 +343,18 @@ "drivers/nvme/src/qpair.rs": "45931cd91894c7e0758947e5dd9180ce12796e9698d3c70067b7af08eb53f2d8", "drivers/nvme/src/regs.rs": "d9410b5c64bb70f99e042f7209119c0f0fdb57f859c6a1c07eba90199f1e98c1", "drivers/nvme/src/swap.rs": "f5e1099b83b2cdeb9d288f4d337f1a234d9dfd16c31e9d9303c10f6f14972141", + "services/accel-spmd/Cargo.toml": "9ddcc01b82f3122a4541f9e36cea8b19e9370d56554a98abe82e39e9598f0a39", + "services/accel-spmd/src/backend.rs": "a332da8b5dd33cddcc261e5b2c024d9212900c8116a877e518016382ed6518fc", + "services/accel-spmd/src/kernel.rs": "d68d6e91faa17bc7c9e922012242bbd79a526afe872ec1655164fb81857a0b1e", + "services/accel-spmd/src/lib.rs": "4e41f01f5a34b9fce68cc9e6a51240ee9066dd495a110d49a999cd452062c56e", "services/block/Cargo.toml": "2729b36369486a6f5c1c29200eaa4603b25c26352f06f52301b70d9d57260e43", - "services/block/boot.toml": "a6d8defc624013f6edd2f489cda144375b9e5d94bbaf25f11efd24ecb0ebc68f", + "services/block/boot.toml": "906dbc8502392782b96449b6dcf604d998a59716a3202c1203a789ec146d0da2", "services/block/src/client.rs": "6aed79a404620f89ffa13078f615e846fdf6f3db5629c5cee5fbccbeaa7636cf", "services/block/src/lib.rs": "cf45960460cf6c9dba72f7a6727b51497275c686166a90be531d675097c275f1", "services/block/src/service.rs": "191e9d887d3f3267d6e2aa96e81996f01321bb2febb85e68581159f6e098b4b5", "services/block/src/wire.rs": "a9d64aef9ec3d58d37ca55b841933fc51b76ab94e18b245278b4519e96c65939", "services/dagfs/Cargo.toml": "d229a126616b1d7638f0f5a6edb4ac37d9d116f32afaa1897ffe484fbb286e5e", - "services/dagfs/boot.toml": "60d7176256fbdf96b7d8f6025c8346db9f847f806842bc58b5cfb2432a3051aa", + "services/dagfs/boot.toml": "8af7f22a788922f0fd91e39f701b2a8b36e209727a5feb340e649ef578468b86", "services/dagfs/examples/fsdemo.rs": "1666a14ef62fd4fc46f7f9efa2eaa5021ed79823f273a77f53758307f884e7d0", "services/dagfs/examples/fswire.rs": "994b97d699796b317741e2c87cbe812cc1776ddcd01173323e72a5f74c8ac267", "services/dagfs/examples/model_volume.rs": "0e3623e962da02b2126062a19d198b9e0a8f6e91896fd9a6f7d13f6f30ddc5f1", @@ -310,12 +363,12 @@ "services/dagfs/src/bounds.rs": "c825fcab0f30777e772c21fcf68cd092b10cf57a741eac22747de87ec3742e87", "services/dagfs/src/client.rs": "3532931af9484f548f7ff60bfd58f56fe392d8a770779f766f4e0edb0580d7d4", "services/dagfs/src/digests.rs": "88a256596ab314714ef523a2bea44dd8d5265436b284534d25d4538535454f3a", - "services/dagfs/src/durable.rs": "93df70108d12fd28134dbaf2434a8e067df2a415bbcd0cdb969a84b86403495d", - "services/dagfs/src/engine.rs": "3c52e78d1acf1bdd89d4bc9a77d95e97c139cebdb2009c2cd25921caa634365e", + "services/dagfs/src/durable.rs": "3caf0dc7bd6c7c13cde2c1a83f1db6d70169efb92f448b54ace80f07b6057456", + "services/dagfs/src/engine.rs": "e7cfda6e4f05a532cfc5bf15854b80254f64f40d25a6e7910bee38bfc0ba52c8", "services/dagfs/src/error.rs": "0a0cac522b38baa34221adda4ccca010fd8119c5aaf1f7b6015ab7a5f602ac33", "services/dagfs/src/extraction.rs": "502c7ba6cc8a37016a6c4a43f46664c3f27fe4c01d2cac21f8eb2cc2591a531d", - "services/dagfs/src/lib.rs": "9e4c0e957c81a9504e4217e0c16a83a0dad48fe35ead5a5785e7d139105d6cc8", - "services/dagfs/src/namespace.rs": "638a66619320f580de82b082f5520f35d47d8ab9b39b21ce413365dbf89dacaa", + "services/dagfs/src/lib.rs": "34140702b546448b1b3e326b0112f1be23e1beeb13af836b7f5377b529bb4e76", + "services/dagfs/src/namespace.rs": "72714f185853fbc2e0b67ab59bca00bf4665896adfc120e42fe5d5e3cdb40173", "services/dagfs/src/object.rs": "ce3468a2a536069eddd3598e8593c7eddc8e5107bd96f6e939af0133eb02fb91", "services/dagfs/src/parent_index.rs": "a46346a378ff2e9dd2ffe2d8a9514d8c777a7d0170ff3b90a9345def3b452a02", "services/dagfs/src/path.rs": "98ddd0026e43f6a4957b896bfa0c33ec217b016229662ebc0402283d1e8b0518", @@ -325,11 +378,12 @@ "services/dagfs/src/provider/mod.rs": "1e40047a77fc1f5140626bc25c9d57e681a38302d8b7a30c9e1c01c140cb3db8", "services/dagfs/src/provider/ram.rs": "fa3175f2d98854f2dc99f41cec1eb47ae7e4a560f9307de5f2c3778f95b7f7f1", "services/dagfs/src/reach.rs": "0b72f192ef37320d0a9a6883b7bdba2cd1a291f76dc343f24c52059d588e94ac", - "services/dagfs/src/router.rs": "07e9ceb72b8af7ab3ec944212df2009fe89d276b9179013d32baaa1e70f7f8d4", + "services/dagfs/src/router.rs": "a598a55029a034a95c14bcf242c1bcb86f3d80759dd797122d4f026af5bc0e26", "services/dagfs/src/search.rs": "98e3807b60e039082e2f16cd975d4637520642b380ea5ee3b7a25a4b843de448", - "services/dagfs/src/service/contract.rs": "32219b508a02f9136416618d2e2ab2db1f3bda46f18f8654683caf43e7eb4a66", - "services/dagfs/src/service/mod.rs": "c815acf69647b61a3b907b19b46e1456894d19237d7f19206892f336c9b1b3e0", + "services/dagfs/src/service/contract.rs": "f978333c7da9cb7797e08b9c71690e09c5b1a58b7eedc3a48256671c7f7577d2", + "services/dagfs/src/service/mod.rs": "3403eaf8628cf0e0bc0ce6ff8d2d9ba7a80ea64d550e2822f680757a6d2672e2", "services/dagfs/src/slot.rs": "e8f8751c09a4e6006bcaccd2d189cbb6762da1be6c4ea56bd331fbdc7f5cf827", + "services/dagfs/src/watches.rs": "9ae8277cd4ea1eadbbc8cc992b53edb6e476f247cf208e43ea6e13af3702b74c", "services/dagfs/src/window.rs": "8be4bd5c8f1e51cfa5744801e12a9406e7dafb351d0340a625ca8b8bddeb3364", "services/dagfs/src/wire.rs": "3c85fd1cdf0e35bb3a85cef10093f1b0d08626a9198fe219ff68fe09ba64595b", "services/dagfs/tests/authority.rs": "1432f45aef2bdffd63b83a2e93481673ba3412b14d4dc2abc1ef91f105172641", @@ -343,10 +397,10 @@ "services/dagfs/tests/provider.rs": "ba4070af34ae034c109e391e97bcc4755c6bd70b8e2fb229aaf9d747fef71e2d", "services/dagfs/tests/query_experiment.rs": "6fe147810474b6754ef31ad0fb95493e0b3c0886fbbafd78deb3729d410e8155", "services/device/Cargo.toml": "8f35c9ddd701b58a538f151e95e93ce618f0b346960491adad6d843287165d1c", - "services/device/boot.toml": "18b640a44bda45b29f3fb381477500a989c9f94b77dcb69eac0ec79dddbeed36", + "services/device/boot.toml": "0e0a12ec4f34a65b39ce414661ef2a51c33957a21e5fcef16bbc8d222a7dd48a", "services/device/src/client.rs": "71660deb4a9f51edb7a72fbb3edac94170bf4cccf9ce182cbd478ee3daf406eb", "services/device/src/lib.rs": "215c7c7e81d5d51664ac77b72b2edb0bab739d5a54b30bc9ad08f9dd3a3cfea4", - "services/device/src/service/mod.rs": "817a718222ab737712ba8223bdd5392d1ca59b02d1edcf119a5812d2e8f96aa4", + "services/device/src/service/mod.rs": "6038ba0b9375e3c6bbb25c49bf0849a5878a402f4a3a691dc9df1494849d252b", "services/device/src/wire.rs": "aaec2baa85f4d5ceab233302bd1ba8e54c158f56dcbdef3bbd59e0d99f449d18", "services/extract-worker-c/Cargo.toml": "0a84769f520d5f2735711a21efebfc93380d91724ba2ed690cfe0fe1529e1513", "services/extract-worker-c/build.rs": "66b66d671df56f116176896f7f6c9aaba008ada2d51d0b3dc1f6ca4db08e74bd", @@ -385,7 +439,7 @@ "services/extract/vendor/jsmn/jsmn.h": "babfcb7413354a43dcd539c600a1e00bd2d1579ad0a6f6c7d8b81dccedb16cd0", "services/extract/vendor/jsmn/kore_jsmn.c": "a666060a7af03c819d1faaccc0d6faa19b76bcfe78e4ef61aeab273ad052c95c", "services/mm/Cargo.toml": "25809b6a03916d96b8de64a5988d005c71c49314fafbe0d920c3ec06f2dc5951", - "services/mm/boot.toml": "d4794c49f41698733a0569a0dd0354d17238321b0048ad3f3a348612784f0cfc", + "services/mm/boot.toml": "03aa5d661af5e0e80e730d1b849aabf316a282470db26b8e1faac2bc46e1abd5", "services/mm/src/client.rs": "c6dcc0dc3ed59e0a4cf7532fd306af4c1dc4d4e5ad1edd1d86887def791c5aa6", "services/mm/src/heap.rs": "e55210f3609f2e3e90f30d5497355ddab5801acc60ccbe3f8acbbffff71dbfde", "services/mm/src/lib.rs": "d3820740666ed993e547ee1b3b579231bf439a813d9766860e3b1b57fe815bd6", @@ -397,12 +451,21 @@ "services/output/src/service/mod.rs": "d80c3beb50979b3e1e500abdfa3c0b9d38e354c24e2e34aea443732abbead6f1", "services/output/src/wire.rs": "3752e994eb18714518facae1b78a9ee2b34f99aad8687653f0b89dc8fdeca621", "services/swap/Cargo.toml": "9c0af01403a6ad5a7f4ff69a02b18c8b1b2624a684814d6dc1ac07968c5bbe53", - "services/swap/boot.toml": "046fbdab1a4d21d6a2e623b15a52d38b841fbb44b0e4f70148c2b16356af93d3", + "services/swap/boot.toml": "7bf8b67cf4b04c1fb1ce3e6c6c3bd4e6d0b180b5ab5ebff04d714809020c4389", "services/swap/src/client.rs": "04f735d059b491765d5ba63a4f5f640332735bf336e62dcbae86c17647583e2c", "services/swap/src/extent.rs": "d6705f77951df1c42cda51fc4526ba6262cab6af4a0545405ce06d2d2e1db7b4", "services/swap/src/lib.rs": "883684df54a87ae66aa7895ea19a872cabc8ab7fd4aded4ea65b7e3dd51fe0cf", "services/swap/src/service/mod.rs": "34597aeb2af68085cf129ba6f6641fac52fd9d8fd9eca26dd264c9feadc77dd0", "services/swap/src/wire.rs": "ff1ab3e6b42214d0c009b8cb1bd07ffa5238348b63ca273337b427dd4637ca14", + "services/vdev/Cargo.toml": "e2485be36449450884f91cb8f00741266ca5f2cab3eca9fa21b2bd3bc48bbc05", + "services/vdev/boot.toml": "21765495f72eb2f2950950dc6029028fcd61e061664dea3af503a8ce117c1231", + "services/vdev/src/client.rs": "1bcac2d3c2b8382afdb0d69ea51fbac9198be3c0f7cf71e2da76d331eb5e1002", + "services/vdev/src/face.rs": "e9fc9cc2cf943d13ef9d9e792bee1e872f39dd6c7eb04c00a3cc4cac875fe589", + "services/vdev/src/host.rs": "3c92e97f847393e88b62b2084ca43c8b3122733c5c8761ddd60dee014c087364", + "services/vdev/src/host/tests.rs": "709bb7fd7bb0aca32410abbd776ff8356958f9dcda0438864c7cfb234cee13b0", + "services/vdev/src/lib.rs": "66cb1f43c2e38a87625fa8b8119239b7384b2be86b44aee70d6e69750c6bfbc3", + "services/vdev/src/service.rs": "72772c8f8f94e19afe302ddc8ba977494a22a593ee7c4fcd285391c87ce6ba2c", + "services/vdev/src/wire.rs": "8a9624f64e86fbb0172c6e6b93de9a44eff495575f0a6014e9a2304784ab8478", "tools/kore-vol/Cargo.toml": "d7f3f5c13b1fddd010b5d70d5e16500cec66e8a48221667f5aec84c3ce7af4d7", "tools/kore-vol/src/device.rs": "21011b8ca119bcf987486f0d79bafdd86e7219183577b27b5dd3145f8acb33f2", "tools/kore-vol/src/lib.rs": "41f7fc123e770cc1f34e97de0c2fc6a0ee9211cc29172e23cb42f01e84d2b034", diff --git a/native/services/accel-spmd/Cargo.toml b/native/services/accel-spmd/Cargo.toml new file mode 100644 index 0000000..892974e --- /dev/null +++ b/native/services/accel-spmd/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "kore-svc-accel-spmd" +version = "0.0.1" +edition = "2024" +description = "kore CPU accelerator backend candidate: the CPU-SIMD implementation of the AccelRequest contract, executing registered kernels over kore-rt's SPMD region launch." +publish = false + +[dependencies] +kore-driver = { path = "../../crates/driver", features = ["accel"] } +# The SPMD region launch (`spmd::dispatch_slices`) this row's `submit` is +# built on, and the heap its buffers come from. +kore-rt = { path = "../../crates/rt" } +# The frozen accelerator contracts the backend speaks. Same pin as the +# founded device rows: one protocol and one coherent type family. +virtio-accel-core = { version = "=0.4.1", default-features = false } + +[dev-dependencies] +kore-rt = { path = "../../crates/rt", features = ["std"] } diff --git a/native/services/accel-spmd/src/backend.rs b/native/services/accel-spmd/src/backend.rs new file mode 100644 index 0000000..d3b6e26 --- /dev/null +++ b/native/services/accel-spmd/src/backend.rs @@ -0,0 +1,1563 @@ +//! The **SPMD backend** — `impl Accelerator` over CPU workers. +//! +//! The accel protocol declares one vocabulary for every backend, "NPU / GPU / +//! CPU-SIMD" (`crates/abi/src/svc.rs`). The NPU and GPU have device-side +//! implementations; this crate supplies the CPU backend, but not yet a +//! founded service row. It needs no device, no BAR, no DMA pool and no broker +//! claim — its compute resource is the machine's own cores, reached through +//! [`kore_rt::spmd`]. +//! +//! # Why the compute is not inside `submit` +//! +//! [`Accelerator::submit`]'s contract is explicit: *"synchronous work is +//! limited to validation and admission; the call must not wait for execution +//! to finish."* That rules out running a kernel inline. The R9 resident region +//! now provides the right non-blocking engine shape, but this backend is not +//! yet founded as a service and therefore has no long-lived region owner or +//! event-descriptor storage to bind to it: +//! +//! - [`spmd::dispatch_slices`] is the reusable launch, and it **blocks until +//! done** — it owns a single static anchor and re-stages it only after +//! joining; +//! - [`spmd::found_resident`] can serve a stream of jobs, but its unsafe `env` +//! contract requires each admitted descriptor and everything it borrows to +//! remain owned until the completion generation advances. That owner belongs +//! in the future founded CPU service row, not in a stack-local `submit`. +//! +//! The current unfounded backend therefore keeps the same conformant shape +//! real hardware uses: +//! **admit, then drive, then observe.** `submit` validates and parks the work +//! in an event slot; [`SpmdBackend::run_pending`] performs one admitted +//! launch; `poll_event` reports without blocking. On a device the driving is +//! the device's; here it is the owning service's own turn — which is a +//! scheduling fact about a CPU backend, not a loosening of the contract. +//! +//! Founding the R9 engine changes `run_pending` and the private event owner; +//! no protocol surface moves. +//! +//! # What holds the bound bytes +//! +//! An admitted event records raw spans of its bound buffers rather than +//! borrowing them: the trait requires the caller to keep "every referenced +//! handle alive until the returned event is terminal and destroyed", which is +//! exactly the lifetime a raw span needs and exactly what a Rust borrow cannot +//! express across an admission boundary. The refusals in [`SpmdBackend::submit`] +//! are what make the spans sound — in particular the aliasing check, without +//! which one buffer bound as both input and output would hand the region +//! overlapping `&` and `&mut`. +//! +//! [`Accelerator::submit`]: virtio_accel_core::Accelerator::submit +//! [`spmd::dispatch_slices`]: kore_rt::spmd::dispatch_slices +//! [`spmd::found_resident`]: kore_rt::spmd::found_resident + +extern crate alloc; + +use alloc::vec::Vec; +use core::cell::{Cell, RefCell, UnsafeCell}; +use kore_driver::accel::NativeAccelerator; + +#[allow(deprecated)] +use kore_driver::accel::Dispatch; +use kore_driver::accel::LaunchShape; +use kore_driver::accel::Workgroups; +use virtio_accel_core::{ + AcceleratorClass, AccessMode, AllocatedBuffer, ArtifactRef, BackendError, BindingRef, + BufferDesc, BufferInfo, BufferProperties, Capabilities, ContextDesc, DeviceIdentity, + DeviceInfo, DeviceLimits, EventState, MemoryDomain, QueueDesc, ReleaseFailure, SubmitFailure, + Timeout, +}; + +use crate::kernel::{Kernel, TileCtx, kernel_id, lookup}; + +/// The artifact format this row serves: registered kernels, named by +/// [`TargetIdentity`](virtio_accel_core::TargetIdentity) and carrying a small, +/// fixed manifest. +/// +/// A distinct format rather than a reinterpretation of someone else's, so a +/// client that hands this row an NPU ELF is refused by *format* rather than +/// misparsed. +pub const FORMAT_REGISTERED: u32 = u32::from_le_bytes(*b"KSPM"); + +/// Versioned fixed manifest for a registered-kernel artifact: magic, version, +/// and a reserved-zero word. Keeping it nonzero makes the row admissible +/// through `virtio-accel-device`, whose program accounting correctly refuses +/// zero-byte artifacts. +pub const REGISTERED_MANIFEST: [u8; 8] = [b'K', b'S', b'P', b'M', 1, 0, 0, 0]; + +/// Contexts this backend hosts at once. +const MAX_CONTEXTS: u32 = 1; +/// Buffers per context. +const MAX_BUFFERS: u32 = 256; +/// Resident programs per context — a lookup costs nothing, so this bounds +/// only bookkeeping. +const MAX_PROGRAMS: u32 = 32; +/// Execution queues per context. +const MAX_QUEUES: u32 = 4; +/// Admitted events in flight. +const MAX_EVENTS: usize = 32; +/// Inputs one kernel may bind. Two covers the built-ins with headroom, and a +/// fixed array keeps the admitted record allocation-free. +const MAX_INPUTS: usize = 4; + +/// A host buffer. +/// +/// [`UnsafeCell`] rather than a plain `Vec` because a submission's output is +/// written through a **shared** binding (`BindingRef` carries `&B`), exactly +/// as a device backend writes through a shared DMA mapping. The exclusivity +/// argument is [`SpmdBackend::submit`]'s: one admitted launch at a time, no +/// buffer bound twice in it, and the region's workers touching disjoint +/// element spans of the one output. +pub struct SpmdBuffer { + bytes: UnsafeCell>, + desc: BufferDesc, +} + +impl SpmdBuffer { + fn len(&self) -> usize { + // SAFETY: the length is fixed at allocation and never changes; no + // writer resizes the vector. + unsafe { (*self.bytes.get()).len() } + } + + fn base(&self) -> *mut u8 { + // SAFETY: the allocation outlives every span derived from it — the + // trait pins the handle until its event is terminal and destroyed. + unsafe { (*self.bytes.get()).as_mut_ptr() } + } + + /// Read access for `read_buffer` and for host-side assertions. + fn as_slice(&self) -> &[u8] { + // SAFETY: no launch is in flight when the adapter calls a transfer + // method — `run_pending` is synchronous and the service is + // single-threaded per task. + unsafe { &*self.bytes.get() } + } +} + +/// A context. The CPU has no per-context device state; this exists so the +/// protocol's object graph is honest rather than collapsed. +pub struct SpmdContext(()); + +/// A resident program: one registered kernel. +pub struct SpmdProgram { + kernel: &'static Kernel, +} + +/// Resident bytes retained by a registered program handle. This is the +/// canonical nonzero charge a host should put in `ArtifactRef::resident_bytes`. +pub const REGISTERED_RESIDENT_BYTES: u64 = core::mem::size_of::() as u64; + +impl SpmdProgram { + /// The kernel this program resolved to — its diagnostic name is what a + /// status row prints. + pub const fn name(&self) -> &'static str { + self.kernel.name + } +} + +/// An execution queue. Ordering is per-queue and, with one admitted launch at +/// a time, trivially satisfied. +pub struct SpmdQueue(()); + +/// A submission token — an index into the backend's event table. +pub struct SpmdEvent { + slot: usize, +} + +/// One bound span, recorded at admission. +#[derive(Clone, Copy)] +struct Span { + base: *mut u8, + len: usize, +} + +/// An admitted-but-unrun launch. +struct Admitted { + kernel: &'static Kernel, + inputs: [Span; MAX_INPUTS], + n_inputs: usize, + out: Span, + elems: u32, + launch: LaunchShape, +} + +enum Slot { + Free, + Admitted(Admitted), + Terminal(EventState), +} + +/// The output span, split among the region's workers. +/// +/// [`spmd::dispatch_slices`](kore_rt::spmd::dispatch_slices) hands its body a +/// `Fn(Range) + Sync`, so the body cannot capture a `&mut [u8]`. This +/// type is the disjointness argument made explicit and contained: it carries +/// the whole span once, and [`tile`](Self::tile) hands out exactly one +/// worker's elements. The block split `dispatch_slices` performs is what makes +/// those spans disjoint, so no two workers ever alias — and every kernel above +/// this line stays entirely safe. +struct TileOut { + base: *mut u8, + len: usize, + elem: usize, +} + +// SAFETY: the only access is `tile`, which is called once per worker with that +// worker's own half-open element range. `dispatch_slices` derives those ranges +// from a block split of a single grid, so they are pairwise disjoint; distinct +// disjoint sub-slices of one allocation may be held mutably at once. +unsafe impl Sync for TileOut {} + +impl TileOut { + /// This worker's bytes. + /// + /// # Safety + /// `range` must be one worker's own block-split span for this launch, and + /// no other caller may hold an overlapping span concurrently. + unsafe fn tile(&self, range: core::ops::Range) -> &mut [u8] { + let start = range.start as usize * self.elem; + let end = range.end as usize * self.elem; + debug_assert!(end <= self.len, "a tile span must stay inside the output"); + // SAFETY: the caller's contract gives exclusivity over this span, and + // the bounds were validated at admission (`elems * elem <= out.len`). + unsafe { core::slice::from_raw_parts_mut(self.base.add(start), end - start) } + } +} + +/// The CPU accelerator row. +pub struct SpmdBackend { + registry: &'static [Kernel], + ctx_live: Cell, + buffers: Cell, + programs: Cell, + queues: Cell, + events: RefCell<[Slot; MAX_EVENTS]>, +} + +impl SpmdBackend { + /// A backend serving `registry`. + pub fn new(registry: &'static [Kernel]) -> Self { + Self { + registry, + ctx_live: Cell::new(false), + buffers: Cell::new(0), + programs: Cell::new(0), + queues: Cell::new(0), + events: RefCell::new([const { Slot::Free }; MAX_EVENTS]), + } + } + + /// **Drive one admitted launch**, oldest slot first; `true` when one ran. + /// + /// The execution step `submit` is forbidden to perform (see the module + /// docs). The owning service calls this on its own turn; a test calls it + /// directly. Blocking here is lawful and deliberate — it is the CPU + /// backend's compute, and the trait constrains `submit` and `poll_event`, + /// not the service's own scheduling. + /// + /// Off metal — or when a metal region launch refuses because the shared + /// dispatch anchor is busy or the machine is exhausted — + /// [`spmd::dispatch_slices`](kore_rt::spmd::dispatch_slices) answers + /// `None` and the work runs **serially on this thread**. That is not a + /// degraded mode to be hidden: SPMD is the acceleration here, never the + /// semantics, so the same bytes come out either way and the host suite + /// exercises this backend path. + pub fn run_pending(&self) -> bool { + let admitted = { + let mut slots = self.events.borrow_mut(); + let Some(index) = slots + .iter() + .position(|slot| matches!(slot, Slot::Admitted(_))) + else { + return false; + }; + match core::mem::replace(&mut slots[index], Slot::Free) { + Slot::Admitted(work) => (index, work), + other => { + slots[index] = other; + return false; + } + } + }; + let (index, work) = admitted; + Self::execute(&work); + self.events.borrow_mut()[index] = Slot::Terminal(EventState::Complete); + true + } + + /// Run one admitted launch over the region, or serially where there is + /// none. + fn execute(work: &Admitted) { + let kernel = work.kernel; + let elem = kernel.elem_bytes as usize; + let inputs: [&[u8]; MAX_INPUTS] = core::array::from_fn(|i| { + let span = work.inputs[i]; + if i < work.n_inputs { + // SAFETY: recorded at admission from a live buffer the trait + // pins until this event is terminal; read-only, and the + // aliasing refusal proved it is not the output. + unsafe { core::slice::from_raw_parts(span.base as *const u8, span.len) } + } else { + &[] + } + }); + let inputs = &inputs[..work.n_inputs]; + let out = TileOut { + base: work.out.base, + len: work.out.len, + elem, + }; + let total = work.elems; + let launch = work.launch; + + let body = |range: core::ops::Range| { + // SAFETY: `dispatch_slices` block-splits the grid, so this range + // is this worker's alone and disjoint from every sibling's. + let slice = unsafe { out.tile(range.clone()) }; + (kernel.run)(TileCtx { + tiles: range, + total, + launch, + inputs, + out: slice, + }); + }; + + if kore_rt::spmd::dispatch_slices(total, &body).is_none() { + // The region launch refused: same grid, one CPU worker, this + // thread. The future founded resident row removes this shared- + // anchor contention from steady-state submissions. + body(0..total); + } + } + + fn claim_event(&self) -> Result { + let slots = self.events.borrow(); + if slots.iter().any(|slot| matches!(slot, Slot::Admitted(_))) { + return Err(BackendError::Busy); + } + slots + .iter() + .position(|slot| matches!(slot, Slot::Free)) + .ok_or(BackendError::ResourceLimit) + } + + /// **The admission** both submit paths share. + /// + /// `launch` is R8's exact carried policy. Artifact-defined execution + /// derives its item count from the output; explicit workgroups are checked + /// against it without discarding their six logical axes. + fn admit( + &self, + program: &SpmdProgram, + bindings: &[BindingRef<'_, SpmdBuffer>], + timeout: Timeout, + launch: LaunchShape, + ) -> Result> { + let reject = |e| Err(SubmitFailure::Rejected(e)); + let kernel = program.kernel; + let n_in = kernel.inputs as usize; + + // A deadline would have to be enforced against a launch this backend + // does not yet interrupt, so it is refused rather than accepted and + // ignored — the same posture the reserved wire fields take. + if !matches!(timeout, Timeout::Infinite) { + return reject(BackendError::Unsupported); + } + if n_in > MAX_INPUTS || bindings.len() != n_in + 1 { + return reject(BackendError::InvalidArgument); + } + + let mut inputs = [Span { + base: core::ptr::null_mut(), + len: 0, + }; MAX_INPUTS]; + let mut out: Option = None; + let mut seen = [false; MAX_INPUTS + 1]; + + for binding in bindings { + let slot = binding.slot as usize; + if slot > n_in || seen[slot] { + return reject(BackendError::InvalidArgument); + } + seen[slot] = true; + + let buffer = binding.buffer; + let Ok(offset) = usize::try_from(binding.range.offset) else { + return reject(BackendError::OutOfBounds); + }; + let Ok(len) = usize::try_from(binding.range.bytes()) else { + return reject(BackendError::OutOfBounds); + }; + let end = match offset.checked_add(len) { + Some(end) if end <= buffer.len() => end, + _ => return reject(BackendError::OutOfBounds), + }; + // Defense in depth: the host is required to have checked this, and + // repeating it costs a comparison against a client-shaped mistake + // that would otherwise write through a read-only declaration. + if !buffer.desc.allows_access(binding.access) { + return reject(BackendError::InvalidArgument); + } + let want = if slot == n_in { + AccessMode::Write + } else { + AccessMode::Read + }; + if binding.access != want { + return reject(BackendError::InvalidArgument); + } + let span = Span { + // SAFETY: `offset <= end <= buffer.len()`, so the offset stays + // inside the one allocation this pointer is derived from. + base: unsafe { buffer.base().add(offset) }, + len: end - offset, + }; + if slot == n_in { + out = Some(span); + } else { + inputs[slot] = span; + } + } + + let Some(out) = out else { + return reject(BackendError::InvalidArgument); + }; + + // **The aliasing refusal** — the check the recorded raw spans rest on. + // One buffer bound as both an input and the output would hand the + // region overlapping shared and exclusive views of the same bytes, and + // no amount of care inside the kernel could make that sound. + for input in inputs.iter().take(n_in) { + let lo = input.base as usize; + let hi = lo + input.len; + let olo = out.base as usize; + let ohi = olo + out.len; + if lo < ohi && olo < hi { + return reject(BackendError::InvalidArgument); + } + } + + let elem = kernel.elem_bytes as usize; + if elem == 0 || out.len % elem != 0 { + return reject(BackendError::InvalidArgument); + } + let Ok(derived) = u32::try_from(out.len / elem) else { + return reject(BackendError::OutOfBounds); + }; + let named = match launch { + LaunchShape::ArtifactDefined => None, + LaunchShape::Workgroups(workgroups) => { + let items = workgroups + .group_count() + .into_iter() + .chain(workgroups.local_size()) + .try_fold(1u64, |acc, axis| acc.checked_mul(u64::from(axis))); + let Some(items) = items.and_then(|items| u32::try_from(items).ok()) else { + return reject(BackendError::OutOfBounds); + }; + Some(items) + } + }; + // A carried geometry must describe the launch the bound output can + // actually hold. Silently clamping to the smaller of the two would run + // a different launch than the client asked for and report success. + let elems = match named { + Some(named) if named != derived => return reject(BackendError::InvalidArgument), + Some(named) => named, + None => derived, + }; + // Every input must cover the same grid: a short input would read past + // its own span in the kernel's element indexing. + for input in inputs.iter().take(n_in) { + if input.len < out.len { + return reject(BackendError::OutOfBounds); + } + } + + let slot = self.claim_event().map_err(SubmitFailure::Rejected)?; + self.events.borrow_mut()[slot] = Slot::Admitted(Admitted { + kernel, + inputs, + n_inputs: n_in, + out, + elems, + launch, + }); + Ok(SpmdEvent { slot }) + } +} + +impl virtio_accel_core::Accelerator for SpmdBackend { + type Context = SpmdContext; + type Buffer = SpmdBuffer; + type Program = SpmdProgram; + type Queue = SpmdQueue; + type Event = SpmdEvent; + + fn device_info(&self) -> Result { + Ok(DeviceInfo { + identity: DeviceIdentity { + uuid: *b"kore-spmd-cpu\0\0\0", + // The protocol assigns NPU/GPU/DSP and no CPU value. `OTHER` + // is the honest answer: inventing a discriminant here would + // claim a number the house protocol has not allocated. + class: AcceleratorClass::OTHER, + vendor_id: 0, + device_id: 0, + }, + // Host DRAM is both host-visible and directly bindable; there is + // no device-local domain and no cancellation path. + capabilities: Capabilities::HOST_VISIBLE_MEMORY | Capabilities::SHARED_MEMORY, + limits: DeviceLimits { + max_contexts: MAX_CONTEXTS, + max_buffers_per_context: MAX_BUFFERS, + max_programs_per_context: MAX_PROGRAMS, + max_queues_per_context: MAX_QUEUES, + max_events_per_context: MAX_EVENTS as u32, + max_bindings_per_submission: (MAX_INPUTS + 1) as u32, + max_buffer_bytes: 1 << 31, + max_artifact_bytes: REGISTERED_MANIFEST.len() as u64, + }, + }) + } + + fn create_context(&self, desc: ContextDesc) -> Result { + if !desc.flags.is_empty() { + return Err(BackendError::Unsupported); + } + if self.ctx_live.replace(true) { + return Err(BackendError::ResourceLimit); + } + Ok(SpmdContext(())) + } + + fn destroy_context(&self, context: Self::Context) -> Result<(), ReleaseFailure> { + if self.buffers.get() != 0 || self.programs.get() != 0 || self.queues.get() != 0 { + return Err(ReleaseFailure::Rejected { + error: BackendError::Busy, + resource: context, + }); + } + if self + .events + .borrow() + .iter() + .any(|slot| !matches!(slot, Slot::Free)) + { + return Err(ReleaseFailure::Rejected { + error: BackendError::Busy, + resource: context, + }); + } + self.ctx_live.set(false); + Ok(()) + } + + fn allocate_buffer( + &self, + _context: &Self::Context, + desc: BufferDesc, + ) -> Result, BackendError> { + match desc.domain { + MemoryDomain::Host | MemoryDomain::Shared => {} + MemoryDomain::Device => return Err(BackendError::Unsupported), + } + if self.buffers.get() >= MAX_BUFFERS { + return Err(BackendError::ResourceLimit); + } + let bytes = usize::try_from(desc.bytes()).map_err(|_| BackendError::OutOfBounds)?; + let mut data = Vec::new(); + data.try_reserve_exact(bytes) + .map_err(|_| BackendError::OutOfMemory)?; + data.resize(bytes, 0); + + // The allocator's guarantee, stated rather than assumed: a `Vec` + // is byte-aligned, so a stricter request is refused instead of being + // silently under-served — `BufferInfo::new` would catch it, and + // catching it here names the reason. + let alignment = desc.alignment(); + if alignment > 1 && (data.as_ptr() as u64) % alignment != 0 { + return Err(BackendError::Incompatible); + } + let info = BufferInfo::new( + desc, + desc.bytes(), + alignment, + BufferProperties::HOST_VISIBLE | BufferProperties::DIRECT_BINDING, + )?; + self.buffers.set(self.buffers.get() + 1); + Ok(AllocatedBuffer::new( + SpmdBuffer { + bytes: UnsafeCell::new(data), + desc, + }, + info, + )) + } + + fn write_buffer( + &self, + buffer: &mut Self::Buffer, + offset: u64, + data: &dyn virtio_accel_core::ByteSource, + ) -> Result<(), BackendError> { + if !buffer + .desc + .usage + .contains(virtio_accel_core::BufferUsage::TRANSFER_DESTINATION) + { + return Err(BackendError::PermissionDenied); + } + let at = usize::try_from(offset).map_err(|_| BackendError::OutOfBounds)?; + let len = usize::try_from(data.len()).map_err(|_| BackendError::OutOfBounds)?; + let end = at.checked_add(len).ok_or(BackendError::OutOfBounds)?; + if end > buffer.len() { + return Err(BackendError::OutOfBounds); + } + // SAFETY: `&mut Self::Buffer` is exclusive, and no launch is in flight + // (`run_pending` is synchronous). + let bytes = unsafe { &mut *buffer.bytes.get() }; + data.read_at(0, &mut bytes[at..end]) + } + + fn read_buffer( + &self, + buffer: &Self::Buffer, + offset: u64, + data: &mut dyn virtio_accel_core::ByteSink, + ) -> Result<(), BackendError> { + if !buffer + .desc + .usage + .contains(virtio_accel_core::BufferUsage::TRANSFER_SOURCE) + { + return Err(BackendError::PermissionDenied); + } + let at = usize::try_from(offset).map_err(|_| BackendError::OutOfBounds)?; + let len = usize::try_from(data.len()).map_err(|_| BackendError::OutOfBounds)?; + let end = at.checked_add(len).ok_or(BackendError::OutOfBounds)?; + if end > buffer.len() { + return Err(BackendError::OutOfBounds); + } + data.write_at(0, &buffer.as_slice()[at..end]) + } + + fn free_buffer(&self, buffer: Self::Buffer) -> Result<(), ReleaseFailure> { + drop(buffer); + self.buffers.set(self.buffers.get().saturating_sub(1)); + Ok(()) + } + + fn load_program( + &self, + _context: &Self::Context, + artifact: ArtifactRef<'_>, + ) -> Result { + if artifact.format.get() != FORMAT_REGISTERED { + return Err(BackendError::Unsupported); + } + if artifact.payload.len() != REGISTERED_MANIFEST.len() as u64 { + return Err(BackendError::InvalidArgument); + } + let mut manifest = [0u8; REGISTERED_MANIFEST.len()]; + artifact.payload.read_at(0, &mut manifest)?; + if manifest != REGISTERED_MANIFEST { + return Err(BackendError::InvalidArgument); + } + if artifact.resident_bytes < REGISTERED_RESIDENT_BYTES { + return Err(BackendError::ResourceLimit); + } + let Some(id) = kernel_id(artifact.target) else { + return Err(BackendError::InvalidArgument); + }; + let Some(kernel) = lookup(self.registry, id) else { + return Err(BackendError::Incompatible); + }; + if self.programs.get() >= MAX_PROGRAMS { + return Err(BackendError::ResourceLimit); + } + self.programs.set(self.programs.get() + 1); + Ok(SpmdProgram { kernel }) + } + + fn unload_program(&self, program: Self::Program) -> Result<(), ReleaseFailure> { + drop(program); + self.programs.set(self.programs.get().saturating_sub(1)); + Ok(()) + } + + fn create_queue( + &self, + _context: &Self::Context, + desc: QueueDesc, + ) -> Result { + if !desc.flags.is_empty() { + return Err(BackendError::Unsupported); + } + if self.queues.get() >= MAX_QUEUES { + return Err(BackendError::ResourceLimit); + } + self.queues.set(self.queues.get() + 1); + Ok(SpmdQueue(())) + } + + fn destroy_queue(&self, queue: Self::Queue) -> Result<(), ReleaseFailure> { + drop(queue); + self.queues.set(self.queues.get().saturating_sub(1)); + Ok(()) + } + + fn submit( + &self, + _queue: &Self::Queue, + program: &Self::Program, + bindings: &[BindingRef<'_, Self::Buffer>], + timeout: Timeout, + ) -> Result> { + self.admit(program, bindings, timeout, LaunchShape::ArtifactDefined) + } + + fn poll_event(&self, event: &Self::Event) -> Result { + match &self.events.borrow()[event.slot] { + Slot::Admitted(_) => Ok(EventState::Pending), + Slot::Terminal(state) => Ok(*state), + // A destroyed or never-admitted slot: the adapter is holding an + // event it already retired. + Slot::Free => Err(BackendError::InvalidArgument), + } + } + + fn destroy_event(&self, event: Self::Event) -> Result<(), ReleaseFailure> { + let mut slots = self.events.borrow_mut(); + if matches!(slots[event.slot], Slot::Admitted(_)) { + drop(slots); + return Err(ReleaseFailure::Rejected { + error: BackendError::Busy, + resource: event, + }); + } + slots[event.slot] = Slot::Free; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::kernel::{BUILTIN, IDENTITY_MAGIC, KERNEL_ADD_F32}; + use kore_driver::accel::LaunchShape; + use kore_driver::accel::Workgroups; + use std::vec; + use virtio_accel_core::{ + Accelerator, ArtifactFormat, BufferRange, BufferUsage, ContextDesc, ContextFlags, + QueueFlags, TargetIdentity, + }; + + const ELEMS: usize = 19; // two SIMD steps plus a tail + const WIDE: usize = ELEMS * 4; + const BYTES: u64 = WIDE as u64; + const KERNEL_OBSERVE_LAUNCH: u32 = 0xfeed; + + /// Report the carried launch through the kernel's own output span: + /// element `i` becomes the launch axis at `i % 6`. Each element's value + /// depends only on the launch and its own index, so every tiling of the + /// grid — SPMD workers or the serial fallback — writes the same bytes, + /// and the report lands in the submitting test's buffer rather than + /// process state a concurrently running test could clobber. + fn observe_launch(ctx: TileCtx<'_>) { + let TileCtx { + tiles, launch, out, .. + } = ctx; + let LaunchShape::Workgroups(workgroups) = launch else { + return; + }; + let [gx, gy, gz] = workgroups.group_count(); + let [lx, ly, lz] = workgroups.local_size(); + let axes = [gx, gy, gz, lx, ly, lz]; + for (i, tile) in tiles.enumerate() { + let value = axes[tile as usize % axes.len()] as f32; + out[i * 4..i * 4 + 4].copy_from_slice(&value.to_le_bytes()); + } + } + + static OBSERVING_REGISTRY: &[Kernel] = &[Kernel { + id: KERNEL_OBSERVE_LAUNCH, + name: "observe.launch", + inputs: 0, + elem_bytes: 4, + lanes: 1, + run: observe_launch, + }]; + + /// The six axes [`observe_launch`] reported into `buffer`. + fn observed_launch(b: &SpmdBackend, buffer: &SpmdBuffer) -> [u32; 6] { + let floats = read_floats(b, buffer); + core::array::from_fn(|i| floats[i] as u32) + } + + /// `ByteSource`/`ByteSink` are implemented for sized arrays, so transfers + /// go through one — a `&[u8]` cannot become a `&dyn ByteSource`. + fn packed(values: &[f32]) -> [u8; WIDE] { + let mut out = [0u8; WIDE]; + for (i, v) in values.iter().enumerate() { + out[i * 4..i * 4 + 4].copy_from_slice(&v.to_le_bytes()); + } + out + } + + fn backend() -> SpmdBackend { + SpmdBackend::new(BUILTIN) + } + + fn desc(usage: BufferUsage) -> BufferDesc { + BufferDesc::new(BYTES, 1, MemoryDomain::Host, usage).expect("descriptor") + } + + fn alloc(b: &SpmdBackend, ctx: &SpmdContext, usage: BufferUsage, values: &[f32]) -> SpmdBuffer { + let transfer_usage = + usage | BufferUsage::TRANSFER_SOURCE | BufferUsage::TRANSFER_DESTINATION; + let mut allocated = b + .allocate_buffer(ctx, desc(transfer_usage)) + .expect("allocated"); + b.write_buffer(allocated.buffer_mut(), 0, &packed(values)) + .expect("written"); + allocated.into_parts().0 + } + + fn program(b: &SpmdBackend, ctx: &SpmdContext, id: u32) -> SpmdProgram { + let mut words = [0u32; 12]; + words[0] = IDENTITY_MAGIC; + words[1] = id; + b.load_program( + ctx, + ArtifactRef { + format: ArtifactFormat::new(FORMAT_REGISTERED).expect("format"), + target: TargetIdentity(words), + payload: ®ISTERED_MANIFEST, + resident_bytes: REGISTERED_RESIDENT_BYTES, + }, + ) + .expect("loaded") + } + + fn bind<'a>( + slot: u32, + buffer: &'a SpmdBuffer, + access: AccessMode, + ) -> BindingRef<'a, SpmdBuffer> { + BindingRef { + slot, + buffer, + range: BufferRange::new(0, BYTES).expect("range"), + access, + } + } + + fn read_floats(b: &SpmdBackend, buffer: &SpmdBuffer) -> vec::Vec { + let mut out = [0u8; WIDE]; + b.read_buffer(buffer, 0, &mut out).expect("read"); + out.chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect() + } + + /// The whole path on a machine with no region: context, buffers, program, + /// admission, the drive step, and a terminal poll carrying the right + /// bytes. This is the claim the row exists to make — that the accel + /// protocol admits a non-device tenant — and it runs on any host. + #[test] + fn a_submission_runs_end_to_end_and_produces_the_right_bytes() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + + let lhs: vec::Vec = (0..ELEMS).map(|i| i as f32).collect(); + let rhs: vec::Vec = (0..ELEMS).map(|i| (i * 3) as f32).collect(); + let a = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &lhs); + let c = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &rhs); + let out = alloc(&b, &ctx, BufferUsage::PROGRAM_OUTPUT, &vec![0.0; ELEMS]); + let prog = program(&b, &ctx, KERNEL_ADD_F32); + + let event = b + .submit( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ], + Timeout::Infinite, + ) + .map_err(|_| "submit") + .expect("admitted"); + + assert!( + matches!(b.poll_event(&event), Ok(EventState::Pending)), + "admission must not have executed anything" + ); + assert!(b.run_pending(), "one launch was admitted"); + assert!(!b.run_pending(), "and only one"); + assert!(matches!(b.poll_event(&event), Ok(EventState::Complete))); + + let got = read_floats(&b, &out); + for i in 0..ELEMS { + assert_eq!(got[i], lhs[i] + rhs[i], "element {i}"); + } + b.destroy_event(event) + .map_err(|_| "destroy") + .expect("freed"); + } + + /// `load_program` refuses every shape that is not a registered kernel, + /// and each refusal is distinguishable — a client can tell "wrong format" + /// from "unknown kernel". + #[test] + fn load_program_refuses_everything_that_is_not_a_registered_kernel() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let good = TargetIdentity({ + let mut w = [0u32; 12]; + w[0] = IDENTITY_MAGIC; + w[1] = KERNEL_ADD_F32; + w + }); + let fmt = ArtifactFormat::new(FORMAT_REGISTERED).expect("format"); + + let other_format = ArtifactFormat::new(FORMAT_REGISTERED ^ 1).expect("format"); + assert!(matches!( + b.load_program( + &ctx, + ArtifactRef { + format: other_format, + target: good, + payload: ®ISTERED_MANIFEST, + resident_bytes: REGISTERED_RESIDENT_BYTES + } + ), + Err(BackendError::Unsupported) + )); + + assert!( + matches!( + b.load_program( + &ctx, + ArtifactRef { + format: fmt, + target: good, + payload: b"bytes", + resident_bytes: REGISTERED_RESIDENT_BYTES + } + ), + Err(BackendError::InvalidArgument) + ), + "a malformed manifest must be refused, never silently ignored" + ); + + let mut wrong_version = REGISTERED_MANIFEST; + wrong_version[4] = 2; + assert!(matches!( + b.load_program( + &ctx, + ArtifactRef { + format: fmt, + target: good, + payload: &wrong_version, + resident_bytes: REGISTERED_RESIDENT_BYTES + } + ), + Err(BackendError::InvalidArgument) + )); + + assert!(matches!( + b.load_program( + &ctx, + ArtifactRef { + format: fmt, + target: good, + payload: ®ISTERED_MANIFEST, + resident_bytes: REGISTERED_RESIDENT_BYTES - 1 + } + ), + Err(BackendError::ResourceLimit) + )); + + assert!(matches!( + b.load_program( + &ctx, + ArtifactRef { + format: fmt, + target: TargetIdentity([0; 12]), + payload: ®ISTERED_MANIFEST, + resident_bytes: REGISTERED_RESIDENT_BYTES + } + ), + Err(BackendError::InvalidArgument) + )); + + let unknown = TargetIdentity({ + let mut w = [0u32; 12]; + w[0] = IDENTITY_MAGIC; + w[1] = 0xdead; + w + }); + assert!( + matches!( + b.load_program( + &ctx, + ArtifactRef { + format: fmt, + target: unknown, + payload: ®ISTERED_MANIFEST, + resident_bytes: REGISTERED_RESIDENT_BYTES + } + ), + Err(BackendError::Incompatible) + ), + "a well-formed identity naming no kernel is Incompatible, not invalid" + ); + } + + /// **The refusal the recorded raw spans rest on.** One buffer bound as + /// both an input and the output would give the region overlapping shared + /// and exclusive views of the same bytes. + #[test] + fn submit_refuses_a_buffer_bound_as_both_input_and_output() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let shared = alloc(&b, &ctx, BufferUsage::MUTABLE_STATE, &vec![1.0; ELEMS]); + let other = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![2.0; ELEMS]); + let prog = program(&b, &ctx, KERNEL_ADD_F32); + + let refused = b.submit( + &queue, + &prog, + &[ + bind(0, &shared, AccessMode::Read), + bind(1, &other, AccessMode::Read), + bind(2, &shared, AccessMode::Write), + ], + Timeout::Infinite, + ); + assert!( + matches!( + refused, + Err(SubmitFailure::Rejected(BackendError::InvalidArgument)) + ), + "aliasing the output with an input must refuse" + ); + } + + /// Admission validates shape before it records anything: binding count, + /// access direction, and a deadline this backend cannot enforce. + #[test] + fn submit_refuses_malformed_admissions() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let a = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![1.0; ELEMS]); + let c = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![2.0; ELEMS]); + let out = alloc(&b, &ctx, BufferUsage::PROGRAM_OUTPUT, &vec![0.0; ELEMS]); + let prog = program(&b, &ctx, KERNEL_ADD_F32); + + let short = b.submit( + &queue, + &prog, + &[bind(0, &a, AccessMode::Read), bind(1, &c, AccessMode::Read)], + Timeout::Infinite, + ); + assert!( + matches!( + short, + Err(SubmitFailure::Rejected(BackendError::InvalidArgument)) + ), + "a missing output binding must refuse" + ); + + let wrong_access = b.submit( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Read), + ], + Timeout::Infinite, + ); + assert!( + matches!( + wrong_access, + Err(SubmitFailure::Rejected(BackendError::InvalidArgument)) + ), + "the output slot must be bound writable" + ); + + let deadline = b.submit( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ], + Timeout::AfterNs(core::num::NonZeroU64::new(1_000).expect("nonzero")), + ); + assert!( + matches!( + deadline, + Err(SubmitFailure::Rejected(BackendError::Unsupported)) + ), + "a deadline this backend cannot enforce is refused, not ignored" + ); + } + + /// An admitted event owns its bound spans until it is terminal, so it + /// cannot be retired out from under the launch it authorized. + #[test] + fn an_admitted_event_refuses_destruction_until_it_has_run() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let a = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![1.0; ELEMS]); + let c = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![2.0; ELEMS]); + let out = alloc(&b, &ctx, BufferUsage::PROGRAM_OUTPUT, &vec![0.0; ELEMS]); + let prog = program(&b, &ctx, KERNEL_ADD_F32); + + let event = b + .submit( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ], + Timeout::Infinite, + ) + .map_err(|_| "submit") + .expect("admitted"); + + let event = match b.destroy_event(event) { + Err(ReleaseFailure::Rejected { error, resource }) => { + assert!(matches!(error, BackendError::Busy)); + resource + } + _ => panic!("destroying an unrun event must be rejected, and hand it back"), + }; + assert!(b.run_pending()); + b.destroy_event(event) + .map_err(|_| "destroy") + .expect("terminal events retire"); + } + + /// Raw admitted spans are sound only under the backend's global + /// single-execution rule. A second admission refuses until the first has + /// actually run; a merely terminal event does not consume that execution + /// slot. + #[test] + fn only_one_execution_is_admitted_globally() { + let b = backend(); + let (_ctx, queue, a, c, out, prog) = rig(&b); + let bindings = || { + [ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ] + }; + + let first = b + .submit(&queue, &prog, &bindings(), Timeout::Infinite) + .map_err(|_| "first submit") + .expect("first admission"); + assert!(matches!( + b.submit(&queue, &prog, &bindings(), Timeout::Infinite), + Err(SubmitFailure::Rejected(BackendError::Busy)) + )); + + assert!(b.run_pending()); + let second = b + .submit(&queue, &prog, &bindings(), Timeout::Infinite) + .map_err(|_| "second submit") + .expect("terminal work no longer blocks admission"); + assert!(b.run_pending()); + b.destroy_event(first) + .map_err(|_| "destroy first") + .expect("first terminal event retires"); + b.destroy_event(second) + .map_err(|_| "destroy second") + .expect("second terminal event retires"); + } + + /// Fixture: a context, a queue, two inputs, an output, and a program. + fn rig( + b: &SpmdBackend, + ) -> ( + SpmdContext, + SpmdQueue, + SpmdBuffer, + SpmdBuffer, + SpmdBuffer, + SpmdProgram, + ) { + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let a = alloc(b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![1.0; ELEMS]); + let c = alloc(b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![2.0; ELEMS]); + let out = alloc(b, &ctx, BufferUsage::PROGRAM_OUTPUT, &vec![0.0; ELEMS]); + let prog = program(b, &ctx, KERNEL_ADD_F32); + (ctx, queue, a, c, out, prog) + } + + /// **R8's path**: a geometry whose work-item count matches the bound + /// output admits, and the launch computes the same result the + /// geometry-free path does. + #[test] + fn a_carried_geometry_matching_the_output_admits_and_runs() { + let b = backend(); + let (_ctx, queue, a, c, out, prog) = rig(&b); + // 19 items as a 19×1×1 grid of 1×1×1 groups. + let workgroups = Workgroups::new([ELEMS as u32, 1, 1], [1, 1, 1]).expect("geometry"); + + let event = b + .submit_with_launch( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ], + LaunchShape::Workgroups(workgroups), + Timeout::Infinite, + ) + .map_err(|_| "submit_with_launch") + .expect("admitted"); + assert!(b.run_pending()); + assert!(matches!(b.poll_event(&event), Ok(EventState::Complete))); + assert!( + read_floats(&b, &out).iter().all(|&v| v == 3.0), + "1.0 + 2.0 across the whole carried grid" + ); + } + + /// Equal products are not equal launch shapes. Preserve all six logical + /// axes through admission and the CPU worker scheduler into the registered + /// kernel's `TileCtx`. + #[test] + fn equal_product_workgroup_shapes_reach_the_kernel_distinctly() { + let b = SpmdBackend::new(OBSERVING_REGISTRY); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let out = alloc(&b, &ctx, BufferUsage::PROGRAM_OUTPUT, &vec![0.0; ELEMS]); + let prog = program(&b, &ctx, KERNEL_OBSERVE_LAUNCH); + + let shapes = [ + Workgroups::new([ELEMS as u32, 1, 1], [1, 1, 1]).unwrap(), + Workgroups::new([1, 1, 1], [ELEMS as u32, 1, 1]).unwrap(), + ]; + let expected = [[ELEMS as u32, 1, 1, 1, 1, 1], [1, 1, 1, ELEMS as u32, 1, 1]]; + + for (shape, expected) in shapes.into_iter().zip(expected) { + let event = b + .submit_with_launch( + &queue, + &prog, + &[bind(0, &out, AccessMode::Write)], + LaunchShape::Workgroups(shape), + Timeout::Infinite, + ) + .map_err(|_| "submit_with_launch") + .expect("same-sized explicit shape admits"); + assert!(b.run_pending()); + assert_eq!(observed_launch(&b, &out), expected); + b.destroy_event(event) + .map_err(|_| "destroy") + .expect("terminal event retires"); + } + } + + /// Deployed OP13 geometry stays exact to the CPU backend: equal products + /// retain their distinct axes, while a client SIMD width that differs from + /// the compiled registered kernel is refused before admission. + #[test] + #[allow(deprecated)] + fn legacy_dispatch_requires_compiled_simd_and_preserves_axes() { + let b = SpmdBackend::new(OBSERVING_REGISTRY); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let out = alloc(&b, &ctx, BufferUsage::PROGRAM_OUTPUT, &vec![0.0; ELEMS]); + let prog = program(&b, &ctx, KERNEL_OBSERVE_LAUNCH); + + let exact = Dispatch::new([1, 1, 1], [ELEMS as u32, 1, 1], 1).unwrap(); + let event = b + .submit_dispatch( + &queue, + &prog, + &[bind(0, &out, AccessMode::Write)], + exact, + Timeout::Infinite, + ) + .map_err(|_| "legacy submit") + .expect("compiled width admits"); + assert!(b.run_pending()); + assert_eq!(observed_launch(&b, &out), [1, 1, 1, ELEMS as u32, 1, 1]); + b.destroy_event(event) + .map_err(|_| "destroy") + .expect("terminal event retires"); + + let wrong_simd = Dispatch::new([1, 1, 1], [ELEMS as u32, 1, 1], 2).unwrap(); + assert!(matches!( + b.submit_dispatch( + &queue, + &prog, + &[bind(0, &out, AccessMode::Write)], + wrong_simd, + Timeout::Infinite, + ), + Err(SubmitFailure::Rejected(BackendError::Incompatible)) + )); + } + + /// A grid the bound output cannot hold is refused rather than clamped — + /// clamping would run a different launch than the client asked for and + /// report success. + #[test] + fn a_geometry_the_output_cannot_hold_is_refused() { + let b = backend(); + let (_ctx, queue, a, c, out, prog) = rig(&b); + let too_big = Workgroups::new([ELEMS as u32 + 1, 1, 1], [1, 1, 1]).expect("geometry"); + assert!(matches!( + b.submit_with_launch( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ], + LaunchShape::Workgroups(too_big), + Timeout::Infinite, + ), + Err(SubmitFailure::Rejected(BackendError::InvalidArgument)) + )); + } + + /// Explicit one-item geometry is literal and cannot alias the separate + /// artifact-defined mode. + #[test] + fn explicit_one_item_launch_does_not_mean_artifact_defined() { + let b = backend(); + let (_ctx, queue, a, c, out, prog) = rig(&b); + assert!(matches!( + b.submit_with_launch( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ], + LaunchShape::Workgroups(Workgroups::new([1; 3], [1; 3]).unwrap()), + Timeout::Infinite, + ), + Err(SubmitFailure::Rejected(BackendError::InvalidArgument)) + )); + } + + /// A device-local request has no honest answer on a CPU row. + #[test] + fn device_local_memory_is_refused_rather_than_faked() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let device = BufferDesc::new(BYTES, 1, MemoryDomain::Device, BufferUsage::PROGRAM_INPUT) + .expect("descriptor"); + assert!(matches!( + b.allocate_buffer(&ctx, device), + Err(BackendError::Unsupported) + )); + } + + /// Transfer permissions are a provider contract too, not merely a check + /// supplied by the generic command service. Keeping the check here makes + /// direct providers and future non-virtio callers preserve the descriptor's + /// authority boundary. + #[test] + fn transfers_require_declared_usage() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let mut source_only = b + .allocate_buffer(&ctx, desc(BufferUsage::TRANSFER_SOURCE)) + .expect("source buffer"); + let mut destination_only = b + .allocate_buffer(&ctx, desc(BufferUsage::TRANSFER_DESTINATION)) + .expect("destination buffer"); + let bytes = [0x5au8; WIDE]; + let mut sink = [0u8; WIDE]; + + assert_eq!( + b.write_buffer(source_only.buffer_mut(), 0, &bytes), + Err(BackendError::PermissionDenied) + ); + assert_eq!( + b.read_buffer(destination_only.buffer(), 0, &mut sink), + Err(BackendError::PermissionDenied) + ); + b.write_buffer(destination_only.buffer_mut(), 0, &bytes) + .expect("write-enabled descriptor"); + b.read_buffer(source_only.buffer(), 0, &mut sink) + .expect("read-enabled descriptor"); + } + + /// The row advertises its bounded nonzero manifest and a class the + /// protocol has actually allocated. + #[test] + fn device_info_advertises_the_registered_manifest_bound() { + let info = backend().device_info().expect("info"); + assert_eq!( + info.validate(), + Ok(()), + "the generic host can admit this row" + ); + assert_eq!( + info.limits.max_artifact_bytes, + REGISTERED_MANIFEST.len() as u64 + ); + assert!(REGISTERED_RESIDENT_BYTES > 0); + assert!(::EXPLICIT_WORKGROUPS); + assert_eq!(info.identity.class, AcceleratorClass::OTHER); + } +} + +impl NativeAccelerator for SpmdBackend { + const EXPLICIT_WORKGROUPS: bool = true; + /// Honor the deployed OP13 dialect without letting its client SIMD become + /// the preferred portable contract. The width is accepted only when it is + /// exactly the compiled registered kernel's lane width; only after that + /// proof are the unchanged grid/group axes lowered to logical workgroups. + #[allow(deprecated)] + fn submit_dispatch( + &self, + _queue: &Self::Queue, + program: &Self::Program, + bindings: &[BindingRef<'_, Self::Buffer>], + dispatch: Dispatch, + timeout: Timeout, + ) -> Result> { + if dispatch.is_scalar() { + return self.admit(program, bindings, timeout, LaunchShape::ArtifactDefined); + } + if dispatch.simd_width() != program.kernel.lanes { + return Err(SubmitFailure::Rejected(BackendError::Incompatible)); + } + let workgroups = + Workgroups::new(dispatch.grid(), dispatch.group()).map_err(SubmitFailure::Rejected)?; + self.admit( + program, + bindings, + timeout, + LaunchShape::Workgroups(workgroups), + ) + } + fn submit_with_launch( + &self, + _queue: &Self::Queue, + program: &Self::Program, + bindings: &[BindingRef<'_, Self::Buffer>], + launch: LaunchShape, + timeout: Timeout, + ) -> Result> { + self.admit(program, bindings, timeout, launch) + } +} diff --git a/native/services/accel-spmd/src/kernel.rs b/native/services/accel-spmd/src/kernel.rs new file mode 100644 index 0000000..b6dce53 --- /dev/null +++ b/native/services/accel-spmd/src/kernel.rs @@ -0,0 +1,362 @@ +//! **Registered kernels** — what a "program" is on the CPU row. +//! +//! The NPU row loads a compiled artifact: an ELF the NPU compiler emitted, +//! placed in device memory and relocated. There is no CPU compiler in this +//! tree, so this row takes the other option the protocol already leaves open. +//! [`ArtifactRef::target`] is twelve **opaque** words whose "schema belongs to +//! the artifact format, not this crate" — so a format whose schema says *the +//! words name a kernel this build already carries* is conformant, not a +//! workaround. `load_program` validates a fixed versioned manifest, then does +//! a lookup rather than loading executable bytes; any other payload is refused +//! rather than ignored. +//! +//! That choice is a v1 posture, not a ceiling: an artifact-loading CPU row can +//! land beside this one under a second [`ArtifactFormat`] without moving a +//! byte of protocol, and the W^X executable-lease path (#660) is what it would +//! stand on. +//! +//! # The tiling contract +//! +//! A kernel declares the size of one **output element**, and that is the unit +//! the region's workers are split over (`res/SPMD.md` R1 — the launch extent +//! is the *work* grid, not a thread count). The backend does the split; a +//! kernel body receives its own element span and a `&mut [u8]` that is exactly +//! those elements' bytes. Two workers therefore cannot alias, and a kernel +//! never reasons about the region at all. +//! +//! Inputs are shared for the whole grid: an elementwise body indexes them by +//! the same element numbers it was given. +//! +//! # What this shape deliberately does not cover +//! +//! Reductions. A sum has one output element, so the tiling above degenerates +//! to a single worker and the parallelism is lost. That is not a gap to paper +//! over here — `spmd::reduce` is a *different launch shape* (map to a partial, +//! then combine), and it wants its own kernel class rather than a reduction +//! squeezed through an elementwise signature. The registry is the place that +//! class will be added. +//! +//! [`ArtifactRef::target`]: virtio_accel_core::ArtifactRef +//! [`ArtifactFormat`]: virtio_accel_core::ArtifactFormat + +use core::ops::Range; +use core::simd::{Simd, num::SimdFloat}; + +use kore_driver::accel::LaunchShape; +use virtio_accel_core::TargetIdentity; + +/// Lanes per SIMD step. Eight `f32` is 256 bits — the v3 user profile's +/// natural width, and the widest the tree's baseline profile can assume. +pub(crate) const LANES: usize = 8; + +/// The identity word every kernel this row serves begins with — `"KSPM"`. +/// +/// A client naming any other first word is naming an artifact for some other +/// format, and gets [`Unsupported`](virtio_accel_core::BackendError::Unsupported) +/// rather than a lookup miss: the two are different failures and a client can +/// act on the difference. +pub const IDENTITY_MAGIC: u32 = u32::from_le_bytes(*b"KSPM"); + +/// One tile span's view of a kernel launch. +/// +/// `out` is **exactly** this span's bytes — the backend has already done the +/// split, so a body writes its whole slice and cannot reach a sibling's. +pub struct TileCtx<'a> { + /// This worker's element span within the grid. + pub tiles: Range, + /// The whole grid's element count. + pub total: u32, + /// The exact launch policy admitted for this execution. + /// + /// Explicit workgroup axes remain intact here even though the CPU worker + /// tier schedules a contiguous linear span. A registered kernel can derive + /// logical group/local ids from the original shape; equal-product shapes + /// are never collapsed into the same kernel invocation. No barrier model + /// is implied by this field (`res/SPMD.md` R8). + pub launch: LaunchShape, + /// The bound inputs, whole (not split) — indexed by grid element. + pub inputs: &'a [&'a [u8]], + /// This span's output bytes, and only this span's. + pub out: &'a mut [u8], +} + +/// A kernel this build carries. +pub struct Kernel { + /// The registry key: `[IDENTITY_MAGIC, id, 0, 0, …]`. + pub id: u32, + /// Diagnostic name — what a status row prints, never a lookup key. + pub name: &'static str, + /// Input bindings this kernel requires, exactly. + pub inputs: u32, + /// Bytes in one output element — the tiling unit. + pub elem_bytes: u32, + /// Lanes this kernel's body runs in lockstep. + /// + /// A property of the compiled body, not a client choice or wire field. + /// Providers may use it for diagnostics and scheduling decisions. + pub lanes: u32, + /// Run one span. Called once per worker, concurrently. + pub run: fn(TileCtx<'_>), +} + +impl Kernel { + /// The twelve-word identity a client names this kernel by. + pub const fn identity(&self) -> TargetIdentity { + let mut words = [0u32; 12]; + words[0] = IDENTITY_MAGIC; + words[1] = self.id; + TargetIdentity(words) + } +} + +/// Split a target identity into its kernel id, or `None` when the words are +/// not this format's. +/// +/// Fail-closed on the reserved tail: words 2..12 must be zero. A client that +/// set one is asking for something this build does not implement, and +/// accepting the request while ignoring the words would be the dishonest +/// reading — the same posture every sibling wire form in the tree takes. +pub fn kernel_id(target: TargetIdentity) -> Option { + let TargetIdentity(words) = target; + if words[0] != IDENTITY_MAGIC { + return None; + } + if words[2..].iter().any(|&w| w != 0) { + return None; + } + Some(words[1]) +} + +/// Find a kernel by id. +pub fn lookup(registry: &'static [Kernel], id: u32) -> Option<&'static Kernel> { + registry.iter().find(|k| k.id == id) +} + +// ── the built-in registry ─────────────────────────────────────────────── + +/// `add.f32` — elementwise `out[i] = in0[i] + in1[i]`. +pub const KERNEL_ADD_F32: u32 = 1; +/// `mul.f32` — elementwise `out[i] = in0[i] * in1[i]`. +pub const KERNEL_MUL_F32: u32 = 2; + +/// The kernels this build carries. +/// +/// Two, deliberately: enough to prove the seam end to end (binding validation, +/// tiling, the region launch, the serial fallback) without this row pretending +/// to be a compute library. A kernel set is a separate question from a backend, +/// and answering it here would make the backend hard to review. +pub static BUILTIN: &[Kernel] = &[ + Kernel { + id: KERNEL_ADD_F32, + name: "add.f32", + inputs: 2, + elem_bytes: 4, + lanes: LANES as u32, + run: add_f32, + }, + Kernel { + id: KERNEL_MUL_F32, + name: "mul.f32", + inputs: 2, + elem_bytes: 4, + lanes: LANES as u32, + run: mul_f32, + }, +]; + +/// Read `count` `f32`s starting at element `first` from a bound input. +/// +/// Bindings arrive as bytes: the wire carries offsets into shared windows and +/// says nothing about element types, so a kernel decodes its own operands. +/// Unaligned by construction — a client's offset need not be 4-byte aligned — +/// which is why this reads through `from_le_bytes` rather than casting. +#[inline] +fn load_f32(src: &[u8], index: usize) -> f32 { + let at = index * 4; + let mut word = [0u8; 4]; + word.copy_from_slice(&src[at..at + 4]); + f32::from_le_bytes(word) +} + +#[inline] +fn store_f32(dst: &mut [u8], index: usize, value: f32) { + let at = index * 4; + dst[at..at + 4].copy_from_slice(&value.to_le_bytes()); +} + +/// The shared body of both elementwise kernels — `op` applied lane-wise. +/// +/// Vectorised through [`core::simd`] rather than a scalar loop +/// (`kore-design-doctrine` §8: byte-at-a-time is a red flag, and the portable +/// API composes with the trait-seam discipline instead of fighting it). The +/// tail below `LANES` runs scalar, which is the ordinary shape and not a +/// fallback. +#[inline] +fn elementwise(ctx: TileCtx<'_>, op: fn(Simd, Simd) -> Simd) { + let a = ctx.inputs[0]; + let b = ctx.inputs[1]; + let span = (ctx.tiles.end - ctx.tiles.start) as usize; + let base = ctx.tiles.start as usize; + + let mut i = 0; + while i + LANES <= span { + let mut lhs = [0f32; LANES]; + let mut rhs = [0f32; LANES]; + for lane in 0..LANES { + lhs[lane] = load_f32(a, base + i + lane); + rhs[lane] = load_f32(b, base + i + lane); + } + let out = op(Simd::from_array(lhs), Simd::from_array(rhs)).to_array(); + for lane in 0..LANES { + store_f32(ctx.out, i + lane, out[lane]); + } + i += LANES; + } + while i < span { + let v = op( + Simd::splat(load_f32(a, base + i)), + Simd::splat(load_f32(b, base + i)), + )[0]; + store_f32(ctx.out, i, v); + i += 1; + } +} + +fn add_f32(ctx: TileCtx<'_>) { + elementwise(ctx, |a, b| a + b); +} + +fn mul_f32(ctx: TileCtx<'_>) { + elementwise(ctx, |a, b| a * b); +} + +/// Keep the `SimdFloat` import load-bearing rather than speculative: the trait +/// is what makes the lane-wise arithmetic above resolve to the portable ops. +const _: fn() = || { + let _ = Simd::::splat(0.0).reduce_sum(); +}; + +#[cfg(test)] +mod tests { + use super::*; + use std::vec; + + fn bytes_of(values: &[f32]) -> vec::Vec { + values.iter().flat_map(|v| v.to_le_bytes()).collect() + } + + fn floats_of(bytes: &[u8]) -> vec::Vec { + bytes + .chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect() + } + + /// The identity round-trips, and every non-conforming shape refuses. + #[test] + fn identities_round_trip_and_fail_closed() { + let kernel = &BUILTIN[0]; + assert_eq!(kernel_id(kernel.identity()), Some(kernel.id)); + + assert_eq!( + kernel_id(TargetIdentity([0; 12])), + None, + "a zero identity is not this format's" + ); + + let mut wrong_magic = kernel.identity(); + wrong_magic.0[0] ^= 1; + assert_eq!(kernel_id(wrong_magic), None, "another format's artifact"); + + let mut reserved_set = kernel.identity(); + reserved_set.0[7] = 1; + assert_eq!( + kernel_id(reserved_set), + None, + "a set reserved word must refuse, not be ignored" + ); + } + + /// A kernel run over one whole span computes the elementwise result — + /// including the tail below the SIMD width. + #[test] + fn add_computes_over_a_full_span_including_the_tail() { + // 19 elements: two full 8-lane steps plus a 3-element tail. + let a: vec::Vec = (0..19).map(|i| i as f32).collect(); + let b: vec::Vec = (0..19).map(|i| (i * 2) as f32).collect(); + let src_a = bytes_of(&a); + let src_b = bytes_of(&b); + let mut out = vec![0u8; 19 * 4]; + + add_f32(TileCtx { + tiles: 0..19, + total: 19, + launch: LaunchShape::ArtifactDefined, + inputs: &[&src_a, &src_b], + out: &mut out, + }); + + let got = floats_of(&out); + for i in 0..19 { + assert_eq!(got[i], a[i] + b[i], "element {i}"); + } + } + + /// The split is the whole safety argument, so it has to compose: running + /// two disjoint spans into their own slices must equal one whole run. + #[test] + fn disjoint_spans_compose_into_the_whole_result() { + let a: vec::Vec = (0..20).map(|i| i as f32).collect(); + let b: vec::Vec = (0..20).map(|i| (i + 7) as f32).collect(); + let src_a = bytes_of(&a); + let src_b = bytes_of(&b); + + let mut whole = vec![0u8; 20 * 4]; + mul_f32(TileCtx { + tiles: 0..20, + total: 20, + launch: LaunchShape::ArtifactDefined, + inputs: &[&src_a, &src_b], + out: &mut whole, + }); + + let mut split = vec![0u8; 20 * 4]; + { + let (lo, hi) = split.split_at_mut(9 * 4); + mul_f32(TileCtx { + tiles: 0..9, + total: 20, + launch: LaunchShape::ArtifactDefined, + inputs: &[&src_a, &src_b], + out: lo, + }); + mul_f32(TileCtx { + tiles: 9..20, + total: 20, + launch: LaunchShape::ArtifactDefined, + inputs: &[&src_a, &src_b], + out: hi, + }); + } + assert_eq!(whole, split, "a tiled run equals the whole run"); + } + + /// Every registered kernel is reachable by its own identity, and ids are + /// unique — a duplicate would make `lookup` silently prefer one. + #[test] + fn the_registry_is_a_function() { + for kernel in BUILTIN { + let found = lookup(BUILTIN, kernel.id).expect("registered"); + assert_eq!(found.name, kernel.name); + } + for (i, k) in BUILTIN.iter().enumerate() { + assert!( + !BUILTIN[..i].iter().any(|prev| prev.id == k.id), + "duplicate kernel id {}", + k.id + ); + assert!(k.elem_bytes > 0, "{} needs a tiling unit", k.name); + } + assert!(lookup(BUILTIN, 0xdead_beef).is_none(), "unknown id misses"); + } +} diff --git a/native/services/accel-spmd/src/lib.rs b/native/services/accel-spmd/src/lib.rs new file mode 100644 index 0000000..494344d --- /dev/null +++ b/native/services/accel-spmd/src/lib.rs @@ -0,0 +1,86 @@ +//! # kore-svc-accel-spmd — the CPU backend for the accel protocol +//! +//! `crates/abi/src/svc.rs` describes the accel request wire form as "one +//! vocabulary for every backend (**NPU / GPU / CPU-SIMD**)". The NPU and GPU +//! have device-side implementations in tree; this crate supplies the CPU +//! backend candidate. It is not yet a founded service row. Its compute +//! resource is the machine's own cores, reached through [`kore_rt::spmd`]'s +//! region launch. +//! +//! ## Why the CPU backend is worth its own tenancy +//! +//! Not because a CPU computes faster than an NPU. The accel protocol targets +//! one client vocabulary across providers, and that target needs a CPU +//! implementation. The host tests here drive this backend through context → +//! buffer → program → submit → poll, while `kore-svc-accel` separately gates +//! the generic service and wire. That is a backend-contract witness, not yet +//! proof of a founded CPU service, common executable artifacts, scheduler +//! placement, or production portability across CPU, GPU, and NPU. +//! `res/AXIOM_INTEGRATION.md` defines the ownership boundary: `.ax` lowering +//! and CPU partition compilation belong to `ax`/`ax-exec`; this crate is the +//! provider tenancy and queue adapter. Its `KSPM` built-ins are bootstrap +//! witnesses, not a parallel graph or kernel catalog. +//! +//! ## R8: this backend is the `CPU-SIMD` half of one geometry +//! +//! `res/SPMD.md` R8 settles where the matrix backend lives: the SPMD launch +//! geometry is *both* a direct CPU API and a device-portable one. Legacy scalar +//! `OP_SUBMIT` is artifact-defined; a non-scalar compatibility record is +//! accepted only when its SIMD width equals the registered kernel's compiled +//! lanes. The additive explicit-launch operation carries only workgroup count +//! and local size, where SIMD remains purely a provider compilation/scheduling +//! choice. +//! +//! [`SpmdBackend`] implements both the deprecated exact compatibility seam and +//! `submit_with_launch`: artifact-defined execution derives its extent from the +//! output, while explicit workgroups must describe exactly the output the +//! registered kernel can hold. +//! +//! ## Two structural findings this row surfaced +//! +//! **A program need not be a compiled artifact.** [`ArtifactRef::target`] is +//! twelve *opaque* words whose "schema belongs to the artifact format, not +//! this crate" — so a format whose schema says the words name a kernel this +//! build already carries is conformant, and needs no CPU compiler to exist. +//! See [`kernel`]. +//! +//! **`submit` and SPMD's one-shot launch did not compose — R9 fixes the +//! mechanism.** The `Accelerator` trait forbids `submit` from waiting for +//! execution, and SPMD's only reusable launch (`dispatch_slices`) blocked +//! on its join. `res/SPMD.md` R9 lands the resident region — the founded- +//! once engine with a non-blocking `submit`, a completion word (`poll`), +//! and a `retire` — which IS the shape this row wants (and the device rows +//! already have). So the compute still lives in +//! [`SpmdBackend::run_pending`] — admit, drive, observe — but that one +//! function becomes a resident client the day this crate is founded as a +//! service: `found_resident` once, `submit` per launch, `poll` for +//! completion, no gang re-founded per submit. It is deliberately NOT +//! rewired yet: with no serve loop driving it, a resident path would be +//! metal code no gate exercises (the metal-real rule), so the rewire lands +//! with the service that drives it. Until then `run_pending` keeps the +//! `dispatch_slices` path, which produces identical bytes. See [`backend`]. +//! +//! ## What is not here +//! +//! A `SystemService` row. This crate is the backend and its kernels; founding +//! it as a boot service (manifest row, endowment, founding order, archive +//! measurement) is its own slice — and it is the slice that turns +//! [`SpmdBackend::run_pending`] into an R9 resident client (above), because +//! only then is that path metal-driven and provable. +//! +//! [`ArtifactRef::target`]: virtio_accel_core::ArtifactRef + +#![no_std] +#![feature(portable_simd)] + +pub mod backend; +pub mod kernel; + +pub use backend::{ + FORMAT_REGISTERED, REGISTERED_MANIFEST, REGISTERED_RESIDENT_BYTES, SpmdBackend, SpmdBuffer, + SpmdContext, SpmdEvent, SpmdProgram, SpmdQueue, +}; +pub use kernel::{BUILTIN, IDENTITY_MAGIC, Kernel, TileCtx}; + +#[cfg(test)] +extern crate std; diff --git a/native/services/block/boot.toml b/native/services/block/boot.toml index dc9f49e..be7e276 100644 --- a/native/services/block/boot.toml +++ b/native/services/block/boot.toml @@ -12,5 +12,5 @@ chain = true endowment = "kore_svc_block::BlockEndowment" protocol = "kore_svc_block::BlockService" endows = ["DMA_POOL@1"] -digest_x86_64 = "blake3:c852d96c707ed0e95aad161be935fcc8bee5310598ec94f0f3bc75849cf586b0" -digest_aarch64 = "blake3:98023e83300526b637f67f62bafb1b523cd8657dabe4c980f5133f282d6f3ed0" +digest_x86_64 = "blake3:e48b79857534c5630964141cadce6985ccfdc9864a819276917efc8ed415ec50" +digest_aarch64 = "blake3:f33c632b85bf9adc9094e709710606d2a56c10449e76d22919697cdba6f08775" diff --git a/native/services/dagfs/boot.toml b/native/services/dagfs/boot.toml index fe9b8b5..15b963a 100644 --- a/native/services/dagfs/boot.toml +++ b/native/services/dagfs/boot.toml @@ -47,5 +47,5 @@ exec_client = true # `dagfs_heap_supply_holds_every_session` holds this row to it; the rest is # room for the proofs open handles carry, allocated per open at their depth. heap_supply_bytes = 0x00D0_0000 -digest_x86_64 = "blake3:2f00411e60e4c43e73c1290e225c8d1908f27e3ff36bf1d3be7709200bb5c454" -digest_aarch64 = "blake3:d8ce59acf3bdf5bb66912e3be2477c3a1de50c97614f66c83138f3fce1995680" +digest_x86_64 = "blake3:10d0cc3c987cd5303a7bdafe82961f3ce30ff9c067a21d96894a7c68fba5c291" +digest_aarch64 = "blake3:981d8318b9a9b415dcbc0e331aa984750134d35828ea499e0a4b53557aa4c548" diff --git a/native/services/dagfs/src/durable.rs b/native/services/dagfs/src/durable.rs index adf8b89..864a3bd 100644 --- a/native/services/dagfs/src/durable.rs +++ b/native/services/dagfs/src/durable.rs @@ -79,7 +79,8 @@ use kore_svc_block::{BlockClient, BlockRefusal, BlockService, ProvisionRequest, use crate::persistence::{PersistentFs, VolumeProbe, bootstrap_data_start}; use crate::{ - FsEndowment, FsError, FsRequest, FsResponse, FsService, ProviderError, RamService, SessionId, + FsEndowment, FsError, FsRequest, FsResponse, FsResult, FsService, Handle, ObjectId, + ProviderError, RamService, SessionId, persistence::{CommitStage, PersistenceError, PersistenceState, commit_engine}, provider::BlockProvider, router::{self, RoutedRequest, Store}, @@ -1248,6 +1249,65 @@ impl MountedEngine { } } + fn attach_watch_plane(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + match self { + Self::Durable { fs, .. } => fs.attach_watch_plane(peer, aux), + Self::Volatile(fs) => fs.attach_watch_plane(peer, aux), + } + } + + fn detach_watch_plane(&mut self, peer: PeerId) { + match self { + Self::Durable { fs, .. } => fs.detach_watch_plane(peer), + Self::Volatile(fs) => fs.detach_watch_plane(peer), + } + } + + fn watch_target(&self, session: SessionId, handle: Handle) -> FsResult<(ObjectId, u64)> { + match self { + Self::Durable { fs, .. } => fs.watch_target(session, handle), + Self::Volatile(fs) => fs.watch_target(session, handle), + } + } + + fn watch_epoch(&self, session: SessionId, handle: Handle, dir: ObjectId) -> FsResult { + match self { + Self::Durable { fs, .. } => fs.watch_epoch(session, handle, dir), + Self::Volatile(fs) => fs.watch_epoch(session, handle, dir), + } + } + + #[allow(clippy::too_many_arguments)] + fn arm_watch( + &mut self, + via: u16, + session: SessionId, + slot: u8, + tag: u32, + store: Store, + handle: Handle, + target: FsResult<(ObjectId, u64)>, + ) -> FsResponse { + match self { + Self::Durable { fs, .. } => fs.arm_watch(via, session, slot, tag, store, handle, target), + Self::Volatile(fs) => fs.arm_watch(via, session, slot, tag, store, handle, target), + } + } + + fn settle_watches(&mut self, system: impl FnMut(SessionId, Handle, ObjectId) -> FsResult) { + match self { + Self::Durable { fs, .. } => fs.settle_watches(system), + Self::Volatile(fs) => fs.settle_watches(system), + } + } + + fn owed_watch_wakes(&mut self, owed: &mut [Option]) -> usize { + match self { + Self::Durable { fs, .. } => fs.owed_watch_wakes(owed), + Self::Volatile(fs) => fs.owed_watch_wakes(owed), + } + } + async fn serve_request_async( &mut self, request: FsRequest, @@ -1417,6 +1477,16 @@ impl MountedFs { self.system.is_some() } + /// Publish what every armed watch saw: the primary frontend holds the + /// planes, and each watch is read through the engine of its own store. + fn settle_watches(&mut self) { + let system = self.system.as_ref(); + self.user.settle_watches(|session, handle, dir| match system { + Some(system) => system.watch_epoch(session, handle, dir), + None => Err(FsError::AccessLost), + }); + } + fn durable(&self) -> bool { self.user.durable() || self.system.as_ref().is_some_and(MountedEngine::durable) } @@ -1711,6 +1781,24 @@ impl MountedFs { } FsResponse::Done } + // A watch on the user volume is the primary frontend's own verb. + RoutedRequest::Watch(Store::User, request) => { + self.user.serve_request_async(request, sv).await + } + // One on the system volume is resolved by the system engine and + // recorded in the pair's own plane: the client holds one plane. + RoutedRequest::Watch(Store::System, FsRequest::Watch { handle, slot, tag }) => { + let Some(system) = self.system.as_ref() else { + return FsResponse::Refused(FsError::InvalidHandle); + }; + let Some(session) = system.session_for_via(sv.via()) else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let target = system.watch_target(session, handle); + self.user + .arm_watch(sv.via(), session, slot, tag, Store::System, handle, target) + } + RoutedRequest::Watch(Store::System, _) => FsResponse::Refused(FsError::Unsupported), RoutedRequest::Both(_) => FsResponse::Refused(FsError::Unsupported), // `DurableService::serve_async` answers the ring verbs before the // engines are consulted; reaching here means this row has none. @@ -3319,6 +3407,28 @@ impl SystemService for DurableService { self.advance_stor(); } + /// A recyclable peer's pair carries its watch plane after the rings; the + /// primary frontend keeps it, whichever volume a watch names. + fn attach_peer_aux(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + self.fs + .as_mut() + .is_some_and(|fs| fs.user.attach_watch_plane(peer, aux)) + } + + fn peer_detaching(&mut self, peer: PeerId) { + if let Some(fs) = self.fs.as_mut() { + fs.user.detach_watch_plane(peer); + } + } + + fn take_owed_wakes(&mut self, owed: &mut [Option]) -> usize { + let Some(fs) = self.fs.as_mut() else { + return 0; + }; + fs.settle_watches(); + fs.user.owed_watch_wakes(owed) + } + fn peer_closed(&mut self, via: u16, reason: PeerClose) -> bool { // A departed session's ring goes first: its targets are client // memory mapped into the controller, and a command still in flight @@ -3481,6 +3591,14 @@ impl SvcProtocol for DurableService { const REPLY_GRANTS: usize = ::REPLY_GRANTS; const SERVE_PAIRS: usize = ::SERVE_PAIRS; + fn init_pair_window( + window: &mut [u8], + req_cap: u32, + rep_cap: u32, + ) -> Result<(), kore_ring::spsc::RingError> { + ::init_pair_window(window, req_cap, rep_cap) + } + fn reply_matches(request: &Self::ReqWire, response: &FsResponse, grants: u8) -> bool { ::reply_matches(request, response, grants) } diff --git a/native/services/dagfs/src/engine.rs b/native/services/dagfs/src/engine.rs index dd2adb0..66a635d 100644 --- a/native/services/dagfs/src/engine.rs +++ b/native/services/dagfs/src/engine.rs @@ -762,6 +762,33 @@ impl Engine

{ }) } + /// What a watch on `handle` follows: the directory it names, and that + /// directory's change epoch now. Watching is observing what a listing + /// shows, so it takes exactly a listing's authority — `READ` on a live + /// route to a directory. + pub(crate) fn watch_target(&self, session: SessionId, handle: Handle) -> FsResult<(ObjectId, u64)> { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + if !self.ns.kind(route.object)?.is_dir() { + return Err(FsError::NotDirectory); + } + Ok((route.object, self.ns.change_epoch(route.object)?)) + } + + /// The change epoch of the directory a watch armed on `handle` follows, + /// read only through a route that still proves to that same directory: + /// a retired edge, a closed handle or a reclaimed directory is + /// `AccessLost`, and the watch ends instead of counting for a client + /// that could no longer list what it counts. + pub(crate) fn watch_epoch(&self, session: SessionId, handle: Handle, dir: ObjectId) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + if route.object != dir { + return Err(FsError::AccessLost); + } + self.ns.change_epoch(dir) + } + /// One page of a directory's entries. pub fn list(&self, session: SessionId, handle: Handle, cursor: u32) -> FsResult { let route = self.route(session, handle)?; diff --git a/native/services/dagfs/src/lib.rs b/native/services/dagfs/src/lib.rs index 0373d1c..ecc50b5 100644 --- a/native/services/dagfs/src/lib.rs +++ b/native/services/dagfs/src/lib.rs @@ -158,6 +158,7 @@ mod router; pub mod search; pub mod service; pub mod slot; +mod watches; pub mod window; pub mod wire; @@ -185,6 +186,9 @@ pub use kore_dagfs::DagfsProtocol; /// The storage-ring client (`res/STORAGE_TRANSPORT.md` §4), for rows that /// read through the filesystem's transport — exec's image loads. pub use kore_dagfs::stor; +/// The watch plane every filesystem pair carries after its rings +/// (`kore_dagfs::watch`), for the founders that format pair windows. +pub use kore_dagfs::watch; pub use namespace::{EdgeId, ListEntry, Name}; pub use object::{ANY_REVISION, ObjectId, ObjectKind, ObjectSubtype, Revision, StableObjectId}; pub use provider::{ diff --git a/native/services/dagfs/src/namespace.rs b/native/services/dagfs/src/namespace.rs index b5ac08d..8adf1db 100644 --- a/native/services/dagfs/src/namespace.rs +++ b/native/services/dagfs/src/namespace.rs @@ -33,7 +33,7 @@ //! while exact edge generations govern route validity. use alloc::{boxed::Box, vec::Vec}; -use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering, fence}; use crate::{ ServiceAlloc, @@ -312,13 +312,29 @@ struct ObjectSnapshotView { /// a metadata commit. A reader never spins behind a preempted writer: it /// returns typed [`FsError::Busy`] and lets the bounded request retry policy /// decide when to revisit the snapshot. +/// +/// `epoch` and `owed` ride the same line but outside the seqlock: each is one +/// word with its own atomic meaning. +/// +/// - `epoch` is a directory's **change epoch** — advanced after every change +/// to what a listing of it shows: an edge made, removed or renamed in it, +/// a child's length, revision or incoming count moved, or an entry made or +/// removed in a child folder (that folder's line: its count, and whether +/// it holds folders). It is never reset, so it stays monotone across slot +/// reuse; the generation is what tells occupants apart. Watches compare it +/// ([`Namespace::change_epoch`]). +/// - `owed` marks a change to what the object's parents show of it — a +/// file's content, a folder's entries — that could not be passed on +/// because another commit held the object's parent row; the holder settles +/// it on release ([`HeldParents`]). #[repr(align(64))] struct ObjectSnapshot { sequence: AtomicU64, generation: AtomicU32, - _pad: AtomicU32, + owed: AtomicU32, len: AtomicU64, dir: AtomicU64, + epoch: AtomicU64, } impl ObjectSnapshot { @@ -326,9 +342,10 @@ impl ObjectSnapshot { Self { sequence: AtomicU64::new(0), generation: AtomicU32::new(0), - _pad: AtomicU32::new(0), + owed: AtomicU32::new(0), len: AtomicU64::new(0), dir: AtomicU64::new(0), + epoch: AtomicU64::new(0), } } @@ -380,6 +397,64 @@ impl ObjectSnapshot { dir: (dir != 0).then(|| DirId(SlotKey::from_bits(dir))), }) } + + /// `id`'s change epoch, validated against the slot's occupant: a slot + /// reused by another object answers `AccessLost`, never its successor's + /// count. + fn read_epoch(&self, id: ObjectId) -> FsResult { + let before = self.sequence.load(Ordering::Acquire); + if before & 1 != 0 { + return Err(FsError::Busy); + } + let generation = self.generation.load(Ordering::Relaxed); + // Acquire: a caller that sees the epoch move sees the change behind + // it (`Namespace::touch` advances it with release order). + let epoch = self.epoch.load(Ordering::Acquire); + let after = self.sequence.load(Ordering::Acquire); + if before != after { + return Err(FsError::Busy); + } + if generation != id.key().generation() || generation == 0 { + return Err(FsError::AccessLost); + } + Ok(epoch) + } +} + +/// A claim on one object's parent row that settles, on release, a change +/// owed to the object's parents ([`Namespace::note_shown`]). +/// +/// Every row claim in this module goes through it, so whichever commit holds +/// the row when a change lands is the one that passes it on: the changer +/// marks the change owed and tries the row; a holder, after letting go, +/// looks for the mark. The two sides store before they look, across a +/// sequentially consistent fence each, so at least one of them sees the +/// other — the change is never lost and never waited for. +pub(crate) struct HeldParents<'a, K> { + ns: &'a Namespace, + object: ObjectId, + claim: Option>, +} + +impl<'a, K> core::ops::Deref for HeldParents<'a, K> { + type Target = ParentClaim<'a>; + fn deref(&self) -> &Self::Target { + self.claim.as_ref().expect("the row is held until the guard drops") + } +} + +impl core::ops::DerefMut for HeldParents<'_, K> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.claim.as_mut().expect("the row is held until the guard drops") + } +} + +impl Drop for HeldParents<'_, K> { + fn drop(&mut self) { + drop(self.claim.take()); + fence(Ordering::SeqCst); + self.ns.settle_owed(self.object); + } } /// One vacant snapshot per object slot, heap-backed for the reason recorded on @@ -396,6 +471,87 @@ fn vacant_snapshots() -> Box<[ObjectSnapshot]> { snapshots.into_boxed_slice() } +/// Change epochs and the parent-row guard. Nothing here depends on the +/// provider's key type: it touches only the snapshot lines and the rows. +impl Namespace { + fn snapshot_line(&self, id: ObjectId) -> Option<&ObjectSnapshot> { + usize::try_from(id.key().index()) + .ok() + .and_then(|index| self.snapshots.get(index)) + } + + /// Advance `dir`'s change epoch, after the change it reports is + /// published (release order; [`ObjectSnapshot::read_epoch`] acquires). + fn touch(&self, dir: ObjectId) { + if let Some(line) = self.snapshot_line(dir) { + line.epoch.fetch_add(1, Ordering::Release); + } + } + + /// Advance the epoch of every directory that names the row's object: what + /// each shows of it — its length, revision, or folder count — moved. + fn touch_parents(&self, row: &ParentClaim<'_>) { + for edge in row.iter() { + self.touch(edge.parent); + } + } + + /// Claim `object`'s parent row through the guard that settles a content + /// change owed while it is held. + fn claim_parents(&self, object: ObjectId) -> FsResult> { + let claim = self.parents_of.claim(object)?; + Ok(HeldParents { + ns: self, + object, + claim: Some(claim), + }) + } + + /// What every directory naming `object` shows of it just changed — a + /// file's length and revision, or a folder's entries (its line's count, + /// and whether it holds folders). Marked owed first, so a commit holding + /// the row passes the change on when it lets go ([`HeldParents`]). + fn note_shown(&self, object: ObjectId) { + let Some(line) = self.snapshot_line(object) else { + return; + }; + line.owed.store(1, Ordering::Relaxed); + fence(Ordering::SeqCst); + self.settle_owed(object); + } + + /// Pass an owed change on to `object`'s parents, if the row is free; a + /// row another commit holds is that commit's to settle. + /// + /// Each pass ends by letting the row go and looking again, because a + /// writer that found the row held during the pass left its mark for this + /// side. Every pass that continues consumed such a mark, so the loop does + /// the writers' work and never waits for anything. + fn settle_owed(&self, object: ObjectId) { + let Some(line) = self.snapshot_line(object) else { + return; + }; + while line.owed.load(Ordering::Relaxed) != 0 { + let Ok(row) = self.parents_of.claim(object) else { + return; + }; + if line.owed.swap(0, Ordering::AcqRel) != 0 { + self.touch_parents(&row); + } + drop(row); + fence(Ordering::SeqCst); + } + } + + /// `dir`'s change epoch: equal readings bracket no change to what a + /// listing of it shows. `AccessLost` once the directory is gone. + pub(crate) fn change_epoch(&self, dir: ObjectId) -> FsResult { + self.snapshot_line(dir) + .ok_or(FsError::AccessLost)? + .read_epoch(dir) + } +} + impl Namespace { pub(crate) fn new() -> Self { // Bootstrap-scale and fixed after startup, exactly like @@ -454,6 +610,7 @@ impl Namespace { debug_assert_eq!(object.id(), id.key()); object.commit_len(len); self.snapshots[id.key().index() as usize].publish(id, len, object.dir()); + self.note_shown(id); object.revision() } @@ -1066,7 +1223,7 @@ impl Namespace { // for the edge, before publishing it, so contention cannot leave a // visible name without its matching count or reverse entry. let mut child_object = self.object_mut(child)?; - let mut parents = self.parents_of.claim(child)?; + let mut parents = self.claim_parents(child)?; parents.reserve()?; let entry = entries .insert(DirEntry { name: *name, child }) @@ -1075,6 +1232,12 @@ impl Namespace { child_object.incoming_inc(); let edge = EdgeId { parent, entry }; parents.add(edge); + // `parent` gained an entry, and every other folder holding the child + // shows one more folder count: the row names them all. `parent`'s own + // folders see its line change (one more entry, perhaps a first + // folder). + self.touch_parents(&parents); + self.note_shown(parent); Ok(edge) } @@ -1093,12 +1256,15 @@ impl Namespace { .ok_or(FsError::NotFound)?; let edge = EdgeId { parent, entry }; let mut child_object = self.object_mut(child)?; - let mut parents = self.parents_of.claim(child)?; + let mut parents = self.claim_parents(child)?; entries .try_remove(edge.entry) .map_err(|error| fs_access(error, FsError::AccessLost))?; child_object.incoming_dec(); parents.remove(edge); + self.touch(parent); + self.touch_parents(&parents); + self.note_shown(parent); Ok((edge, child)) } @@ -1130,7 +1296,7 @@ impl Namespace { return Err(FsError::NotEmpty); } let mut child_object = self.object_mut(child)?; - let mut parents = self.parents_of.claim(child)?; + let mut parents = self.claim_parents(child)?; claims .get_mut(parent)? .try_remove(entry) @@ -1138,6 +1304,9 @@ impl Namespace { child_object.incoming_dec(); let edge = EdgeId { parent, entry }; parents.remove(edge); + self.touch(parent); + self.touch_parents(&parents); + self.note_shown(parent); Ok((edge, child)) } @@ -1166,6 +1335,9 @@ impl Namespace { .try_write(edge.entry) .map_err(|error| fs_access(error, FsError::AccessLost))?; entry.name = *new; + // Only `parent` spells the edge: the child's other folders show it + // under their own names, unchanged. + self.touch(parent); Ok(edge) } @@ -1243,7 +1415,7 @@ impl Namespace { // The child's parent row moves with the edge; it is claimed on its // own, so a file mid-transaction can still be moved. - let mut parents = self.parents_of.claim(child)?; + let mut parents = self.claim_parents(child)?; // Commit: retire the source edge, then create the destination edge. claims @@ -1261,6 +1433,16 @@ impl Namespace { // is deliberately not touched here; the parent row swaps in place. let dst_edge = EdgeId { parent: dst, entry }; parents.replace(src_edge, dst_edge); + // The source lost an entry; the row now names the destination and + // every other folder, whose "located in" for the child moved. + self.touch(src); + self.touch_parents(&parents); + // Across folders, each one's line moved (one entry fewer, one more); + // within one folder only a spelling changed. + if src != dst { + self.note_shown(src); + self.note_shown(dst); + } Ok((src_edge, dst_edge)) } @@ -1360,7 +1542,7 @@ impl Namespace { /// `object`'s incoming edges, copied out of its parent row so no claim /// is held while they are walked. fn incoming_edges(&self, object: ObjectId) -> FsResult> { - let row = self.parents_of.claim(object)?; + let row = self.claim_parents(object)?; let mut edges = BoundedVec::try_new_in(row.len(), crate::service_alloc()) .map_err(|_| FsError::OutOfObjects)?; for edge in row.iter() { @@ -1486,7 +1668,7 @@ impl Namespace { child_ids.as_mut_slice().sort_unstable_by_key(|id| id.key()); let mut children = heapless::Vec::< - (ObjectId, ObjectWrite<'_, K>, ParentClaim<'_>), + (ObjectId, ObjectWrite<'_, K>, HeldParents<'_, K>), MAX_DIR_ENTRIES, >::new(); for child in child_ids { @@ -1494,14 +1676,14 @@ impl Namespace { .objects .try_write(child.key()) .map_err(|error| fs_access(error, FsError::AccessLost))?; - let parents = self.parents_of.claim(child)?; + let parents = self.claim_parents(child)?; children .push((child, claimed, parents)) .map_err(|_| FsError::OutOfObjects)?; } // The reclaimed object names nothing once it is gone; its own row // is emptied with it (it is already empty: nothing names it). - let mut own_parents = self.parents_of.claim(id)?; + let mut own_parents = self.claim_parents(id)?; // All-or-nothing commit: decrement once per removed edge, and drop // its reverse entry, then record only the child generations that @@ -1521,12 +1703,14 @@ impl Namespace { } own_parents.clear(); let mut became_reclaimable = heapless::Vec::new(); - for (id, child, _) in children.iter() { + for (id, child, parents) in children.iter() { if child.is_reclaimable() { became_reclaimable .push(*id) .map_err(|_| FsError::OutOfObjects)?; } + // A child that keeps a folder elsewhere shows one fewer there. + self.touch_parents(parents); } if let Some(entries) = directory.take() { @@ -2743,6 +2927,211 @@ mod tests { assert_eq!(ns.leaked_placements(), 2); } + // ── change epochs ─────────────────────────────────────────────────────── + + /// Whether `dir`'s change epoch moved since `before`, and the new reading. + fn moved(ns: &Ns, dir: ObjectId, before: &mut u64) -> bool { + let now = ns.change_epoch(dir).unwrap(); + let moved = now != *before; + *before = now; + moved + } + + #[test] + fn every_change_a_listing_or_a_folder_line_shows_moves_its_epoch_and_nothing_else_does() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + let b = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + ns.link(root, &name("b"), b).unwrap(); + let file = ns.create_file_object().unwrap(); + let (mut at_a, mut at_b, mut at_root) = ( + ns.change_epoch(a).unwrap(), + ns.change_epoch(b).unwrap(), + ns.change_epoch(root).unwrap(), + ); + + // An entry made in `a`: `a` shows it, and `root` shows `a`'s line + // with one more entry. `b` shows nothing new. + ns.link(a, &name("f"), file).unwrap(); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + assert!(!moved(&ns, b, &mut at_b)); + + // A second folder for the file: `b` gains an entry (and its line in + // `root` moves), and `a` shows one more folder count for it. + ns.link(b, &name("g"), file).unwrap(); + assert!(moved(&ns, b, &mut at_b)); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + + // A rename is one folder's spelling only: its line's count holds. + ns.rename(b, &name("g"), &name("h")).unwrap(); + assert!(moved(&ns, b, &mut at_b)); + assert!(!moved(&ns, a, &mut at_a)); + assert!(!moved(&ns, root, &mut at_root)); + + // A content commit reaches every folder naming the file, and no + // folder line: the folders' entries are unchanged. + { + let mut object = ns.object_mut(file).unwrap(); + ns.commit_file_len(file, &mut object, 12); + } + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, b, &mut at_b)); + assert!(!moved(&ns, root, &mut at_root)); + + // A move: the source loses an entry, the destination gains one, the + // file's other folder sees its location move, and both folders' + // lines move in `root`. + let c = ns.create_dir_object().unwrap(); + ns.link(root, &name("c"), c).unwrap(); + assert!(moved(&ns, root, &mut at_root)); + let mut at_c = ns.change_epoch(c).unwrap(); + ns.relocate(b, &name("h"), c, &name("h")).unwrap(); + assert!(moved(&ns, b, &mut at_b)); + assert!(moved(&ns, c, &mut at_c)); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + + // A move within one folder is a spelling: its line holds. + ns.relocate(c, &name("h"), c, &name("i")).unwrap(); + assert!(moved(&ns, c, &mut at_c)); + assert!(!moved(&ns, root, &mut at_root)); + + // Removing a name: the folder loses it, the other shows one fewer, + // and the folder's line moves. + ns.detach(c, &name("i")).unwrap(); + assert!(moved(&ns, c, &mut at_c)); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + ns.unlink_empty(a, &name("f")).unwrap(); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + + // A refused change moves nothing. + assert_eq!(ns.rename(a, &name("absent"), &name("x")).unwrap_err(), FsError::NotFound); + assert_eq!(ns.detach(a, &name("absent")).unwrap_err(), FsError::NotFound); + assert!(!moved(&ns, a, &mut at_a)); + assert!(!moved(&ns, root, &mut at_root)); + } + + #[test] + fn a_folder_line_change_meeting_its_held_parent_row_is_passed_on_by_the_holder() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + let file = ns.create_file_object().unwrap(); + let mut at_root = ns.change_epoch(root).unwrap(); + + // A commit holds `a`'s own row (a move of `a`, a parents walk) while + // an entry lands in `a`: the linker cannot reach `a`'s folders. + let held = ns.claim_parents(a).unwrap(); + ns.link(a, &name("f"), file).unwrap(); + assert!(!moved(&ns, root, &mut at_root), "the linker left the change owed"); + drop(held); + assert!(moved(&ns, root, &mut at_root), "the holder passed it on"); + drop(ns.claim_parents(a).unwrap()); + assert!(!moved(&ns, root, &mut at_root), "owed once, settled once"); + } + + #[test] + fn a_reclaimed_directory_answers_access_lost_and_its_children_s_other_folders_move() { + let (mut ns, root) = ns_with_root(); + let doomed = ns.create_dir_object().unwrap(); + let kept = ns.create_dir_object().unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("doomed"), doomed).unwrap(); + ns.link(root, &name("kept"), kept).unwrap(); + ns.link(doomed, &name("f"), file).unwrap(); + ns.link(kept, &name("f"), file).unwrap(); + let mut at_kept = ns.change_epoch(kept).unwrap(); + + let (_, child) = ns.detach(root, &name("doomed")).unwrap(); + let mut placements: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + let mut objects: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + assert_eq!(ns.reclaim(child, &mut placements, &mut objects), 1); + assert_eq!(ns.change_epoch(doomed), Err(FsError::AccessLost)); + assert!(moved(&ns, kept, &mut at_kept), "the file is in one folder fewer"); + + // A slot reused by another directory never answers for the old one. + let successor = ns.create_dir_object().unwrap(); + assert_eq!(ns.change_epoch(doomed), Err(FsError::AccessLost)); + assert!(ns.change_epoch(successor).is_ok()); + } + + #[test] + fn a_content_change_meeting_a_held_parent_row_is_passed_on_by_the_holder() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(a, &name("f"), file).unwrap(); + let mut at_a = ns.change_epoch(a).unwrap(); + + // A commit holds the file's row (a move, a parents walk) while the + // file's bytes commit: the writer cannot touch the parents itself. + let held = ns.claim_parents(file).unwrap(); + { + let mut object = ns.object_mut(file).unwrap(); + ns.commit_file_len(file, &mut object, 3); + } + assert!(!moved(&ns, a, &mut at_a), "the writer left the change owed"); + drop(held); + assert!(moved(&ns, a, &mut at_a), "the holder passed it on"); + // Owed once, settled once. + drop(ns.claim_parents(file).unwrap()); + assert!(!moved(&ns, a, &mut at_a)); + } + + #[test] + fn concurrent_writers_and_row_holders_never_lose_a_content_change() { + use std::sync::Arc; + use std::sync::atomic::AtomicBool; + + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(a, &name("f"), file).unwrap(); + let ns = Arc::new(ns); + let stop = Arc::new(AtomicBool::new(false)); + // A holder thread keeps taking and letting go of the row, as parents + // walks and moves do. + let holder = { + let (ns, stop) = (Arc::clone(&ns), Arc::clone(&stop)); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + if let Ok(row) = ns.claim_parents(file) { + core::hint::spin_loop(); + drop(row); + } + } + }) + }; + for len in 0..2_000u64 { + let before = ns.change_epoch(a).unwrap(); + { + let mut object = ns.object_mut(file).unwrap(); + ns.commit_file_len(file, &mut object, len); + } + // Either side settles it; once both have let go, the change is in. + let mut settled = false; + for _ in 0..1_000_000 { + if ns.change_epoch(a).unwrap() != before { + settled = true; + break; + } + std::thread::yield_now(); + } + assert!(settled, "write {len} was never passed on"); + } + stop.store(true, Ordering::Relaxed); + holder.join().unwrap(); + } + /// Write `value` as decimal ASCII into `buf`, returning the byte count. /// Test-local so name generation needs no formatting machinery. fn write_index(buf: &mut [u8; 8], value: usize) -> usize { diff --git a/native/services/dagfs/src/router.rs b/native/services/dagfs/src/router.rs index 8540f77..89c7e10 100644 --- a/native/services/dagfs/src/router.rs +++ b/native/services/dagfs/src/router.rs @@ -58,6 +58,10 @@ pub(crate) enum RoutedRequest { /// A storage-ring verb: neither engine's; served by the row that owns /// the controller (`res/STORAGE_TRANSPORT.md` §4). Stor(FsRequest), + /// Arm a watch on a directory of one store. The request's handle is + /// native to that store; the plane it publishes into is the pair's one, + /// which the primary frontend holds. + Watch(Store, FsRequest), } #[inline] @@ -435,6 +439,13 @@ pub(crate) fn route(request: FsRequest, has_system: bool) -> Result { + let (store, handle) = split_handle(handle); + RoutedRequest::Watch(store, Watch { handle, slot, tag }) + } + // Disarming names no handle: the pair's one plane, in the primary + // frontend, holds every slot, whichever store the slot watched. + Unwatch { .. } => RoutedRequest::One(Store::User, request), LendVersion { handle, window_offset, @@ -541,7 +552,11 @@ pub(crate) fn route(request: FsRequest, has_system: bool) -> Result Option<&'static (dyn core::alloc::Allocator + Sync)> { Some(crate::service_alloc()) } + + /// A recyclable peer's pair carries its watch plane after the rings. + fn attach_peer_aux(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + self.attach_watch_plane(peer, aux) + } + + fn peer_detaching(&mut self, peer: PeerId) { + self.detach_watch_plane(peer); + } + + /// Publish what every armed watch saw since the last unit of work. A lone + /// engine has no system volume, so nothing can be watched on one. + fn take_owed_wakes(&mut self, owed: &mut [Option]) -> usize { + self.settle_watches(|_, _, _| Err(FsError::AccessLost)); + self.owed_watch_wakes(owed) + } } impl FsService

{ @@ -429,6 +445,19 @@ impl FsService

{ Err(error) => FsResponse::Refused(error), }; } + FsRequest::Watch { handle, slot, tag } => { + let target = self.watch_target(session, handle); + self.arm_watch( + sv.via(), + session, + slot, + tag, + crate::router::Store::User, + handle, + target, + ) + } + FsRequest::Unwatch { slot } => self.disarm_watch(sv.via(), session, slot), FsRequest::MatchView { handle, announced } => { let Some(mut input) = sv.returned_input(announced) else { return FsResponse::Refused(FsError::AccessDenied); @@ -501,6 +530,14 @@ impl SvcProtocol for RamService { const REPLY_GRANTS: usize = ::REPLY_GRANTS; const SERVE_PAIRS: usize = ::SERVE_PAIRS; + fn init_pair_window( + window: &mut [u8], + req_cap: u32, + rep_cap: u32, + ) -> Result<(), kore_ring::spsc::RingError> { + DagfsProtocol::init_pair_window(window, req_cap, rep_cap) + } + fn reply_matches(request: &FsReqWire, response: &FsResponse, grants: u8) -> bool { DagfsProtocol::reply_matches(request, response, grants) } diff --git a/native/services/dagfs/src/service/mod.rs b/native/services/dagfs/src/service/mod.rs index 123f0c9..51e04a5 100644 --- a/native/services/dagfs/src/service/mod.rs +++ b/native/services/dagfs/src/service/mod.rs @@ -65,7 +65,7 @@ use kore_rt::{ use kore_svc_extract::schema::InputIdentity; use crate::{ - ServiceAlloc, + ObjectId, ServiceAlloc, authority::{GrantId, Handle, Retired, Rights, SessionId}, block_on, bounds::{FIRST_TASK_VIEW_SESSION, MAX_SESSIONS}, @@ -369,6 +369,9 @@ pub struct FsService { /// immediately before delegating a status request; the default zeros /// honestly describe the bare volatile posture every other harness runs. backend: FsBackendStatus, + /// Which slots' watch planes have a watch armed or owe a wake + /// ([`crate::watches`]); the planes live in the slots' own records. + watches: crate::watches::WatchIndex, } /// What the mounted backend reports about itself for `SystemStatus` — set @@ -484,6 +487,11 @@ struct SessionSlot { /// The session's one private provider placement. The service already /// serializes dispatch; no additional lock or namespace object. replacement: Option>, + /// The recyclable peer's watch plane and its armed watches + /// ([`crate::watches`]), adopted from the pair window's tail at admission + /// and dropped before the host unmaps it. Only the primary frontend's + /// slots carry one: a secondary engine's watches are recorded here too. + watch: Option, } impl SessionSlot { @@ -495,6 +503,7 @@ impl SessionSlot { task_view: None, deferring: false, replacement: None, + watch: None, } } } @@ -601,9 +610,151 @@ impl FsService

{ pending_reclaims: Vec::new_in(crate::service_alloc()), pending_aborts: Vec::new_in(crate::service_alloc()), backend: FsBackendStatus::default(), + watches: crate::watches::WatchIndex::new(), } } + /// Adopt `peer`'s watch plane from its pair window's tail into the slot's + /// record, claiming the record as admission does. Refused for a fixed + /// pair, a heap that cannot hold the record, or a slot whose previous + /// plane never detached — a host/service desync admission must not paper + /// over. A tail without a formatted plane admits the peer with none: the + /// plane is what watches publish into, not what the pair needs to serve, + /// so such a client's `Watch` answers `Unsupported`. + pub(crate) fn attach_watch_plane(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + let slot = peer.slot(); + if slot == 0 { + return false; + } + let Ok(state) = self.slots.claim(slot) else { + return false; + }; + if state.watch.is_some() { + return false; + } + state.watch = crate::watches::Plane::adopt(peer, aux); + true + } + + /// Drop `peer`'s watch plane before its window is unmapped. A late call + /// for an earlier occupant is inert. + pub(crate) fn detach_watch_plane(&mut self, peer: PeerId) { + let slot = peer.slot(); + if let Some(state) = self.slots.get_mut(slot) + && state.watch.as_ref().is_some_and(|plane| plane.peer() == peer) + { + state.watch = None; + self.watches.forget(slot); + } + } + + /// The plane of the pair that delivered a request (`via`) for `session`; + /// `Unsupported` for a fixed pair or a peer admitted without one. + fn watch_plane_mut(&mut self, via: u16, session: SessionId) -> FsResult<&mut crate::watches::Plane> { + self.slots + .get_mut(usize::from(session.index())) + .and_then(|state| state.watch.as_mut()) + .filter(|plane| plane.peer().arrival() == via) + .ok_or(FsError::Unsupported) + } + + /// Arm a watch on a directory of `store`, resolved by that store's engine + /// (`target`), into the plane of the pair that asked. + #[allow(clippy::too_many_arguments)] + pub(crate) fn arm_watch( + &mut self, + via: u16, + session: SessionId, + slot: u8, + tag: u32, + store: crate::router::Store, + handle: Handle, + target: FsResult<(ObjectId, u64)>, + ) -> FsResponse { + let armed = self + .watch_plane_mut(via, session) + .and_then(|plane| { + let owed = plane.arm(slot, tag, store, handle, target?)?; + Ok((plane.armed(), owed)) + }); + match armed { + Ok((armed, owed)) => { + self.watches.note(usize::from(session.index()), armed, owed); + FsResponse::Done + } + Err(error) => FsResponse::Refused(error), + } + } + + pub(crate) fn disarm_watch(&mut self, via: u16, session: SessionId, slot: u8) -> FsResponse { + let disarmed = self.watch_plane_mut(via, session).and_then(|plane| { + let owed = plane.disarm(slot)?; + Ok((plane.armed(), owed)) + }); + match disarmed { + Ok((armed, owed)) => { + self.watches.note(usize::from(session.index()), armed, owed); + FsResponse::Done + } + Err(error) => FsResponse::Refused(error), + } + } + + /// The change epoch a watch on `handle` follows, read through this + /// engine — the secondary half of a dual-router settle. + pub(crate) fn watch_epoch(&self, session: SessionId, handle: Handle, dir: ObjectId) -> FsResult { + self.engine.watch_epoch(session, handle, dir) + } + + /// What a watch on `handle` would follow in this engine. + pub(crate) fn watch_target(&self, session: SessionId, handle: Handle) -> FsResult<(ObjectId, u64)> { + self.engine.watch_target(session, handle) + } + + /// Publish every armed watch whose directory changed. This engine answers + /// for the user store; `system` for the dual router's system volume (a + /// lone engine passes one that refuses, which ends any such watch). + pub(crate) fn settle_watches( + &mut self, + mut system: impl FnMut(SessionId, Handle, ObjectId) -> FsResult, + ) { + if self.watches.idle() { + return; + } + let engine = &self.engine; + for slot in self.watches.armed() { + let Some(plane) = self.slots.get_mut(slot).and_then(|state| state.watch.as_mut()) else { + self.watches.forget(slot); + continue; + }; + let owed = plane.settle(SessionId::new(slot as u16), |store, session, handle, dir| { + match store { + crate::router::Store::User => engine.watch_epoch(session, handle, dir), + crate::router::Store::System => system(session, handle, dir), + } + }); + self.watches.note(slot, plane.armed(), owed); + } + } + + /// Peers owed a watch-plane wake, for the serving host: up to + /// `out.len()`, each handed over once. + pub(crate) fn owed_watch_wakes(&mut self, out: &mut [Option]) -> usize { + let slots = &self.slots; + let mut taken = 0; + self.watches.take_owed(|slot| { + if taken == out.len() { + return false; + } + if let Some(plane) = slots.get(slot).and_then(|state| state.watch.as_ref()) { + out[taken] = Some(plane.peer()); + taken += 1; + } + true + }); + taken + } + /// Replace the backend posture snapshot folded into `SystemStatus` /// answers. The mount owner calls this immediately before delegating a /// status request, so the record reads as one moment. @@ -1140,6 +1291,7 @@ impl FsService

{ if let Some(state) = self.slots.get_mut(slot) { *state = SessionSlot::vacant(); } + self.watches.forget(slot); true } @@ -2466,6 +2618,13 @@ impl FsService

{ self.engine.sync(session, handle).await?; FsResponse::Done } + // A watch publishes into the asking pair's plane, which only the + // pair-aware layer above names (`serve_request_async`, or the dual + // router); reaching the session-only dispatch is a composition + // that forwarded the verb past its owner. + FsRequest::Watch { .. } | FsRequest::Unwatch { .. } => { + return Err(FsError::Unsupported); + } FsRequest::LendVersion { handle, window_offset, @@ -3659,6 +3818,233 @@ mod recyclable_session_tests { assert_eq!(native.stable_object, ls.stable_object); } + /// A recyclable peer's watch plane as exec formats it after the rings, + /// leaked like the host's mapping of it, with the client's half beside + /// the service's. + fn watch_plane() -> ( + &'static mut [u8], + kore_dagfs::watch::WatchSink<'static>, + ) { + #[repr(align(64))] + struct Plane([u8; kore_dagfs::watch::PLANE_BYTES]); + let plane = &mut alloc::boxed::Box::leak(alloc::boxed::Box::new(Plane( + [0; kore_dagfs::watch::PLANE_BYTES], + ))) + .0; + kore_ring::event::init::( + plane, + kore_abi::svc::FS_WATCH_RECORDS, + ) + .unwrap(); + let (base, len) = (plane.as_mut_ptr(), plane.len()); + // SAFETY: the two halves alias the plane only through the channel + // protocol, as the client's and the service's mappings do on metal. + let client = unsafe { core::slice::from_raw_parts_mut(base, len) }; + (plane, kore_dagfs::watch::sink(client).unwrap()) + } + + /// Two peers of one task view: a watcher holding its root, and a writer + /// that changes it. + fn watched_view() -> (RamService, SessionId, SessionId) { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + service + .register_private_invocation_view( + 0xD0, + &Name::new("terminal-0").unwrap(), + Rights::ALL, + None, + ) + .unwrap(); + let watcher = admit_view(&mut service, 0x106, 0xD0, Rights::ALL); + let writer = admit_view(&mut service, 0x105, 0xD0, Rights::ALL); + (service, watcher, writer) + } + + fn settle(service: &mut RamService, owed: &mut [Option]) -> usize { + ::take_owed_wakes(service, owed) + } + + #[test] + fn a_watch_counts_another_peer_s_change_and_owes_a_parked_client_one_wake() { + use kore_dagfs::watch::{Cursor, Seen}; + let (mut service, watcher, writer) = watched_view(); + let peer = PeerId::from_arrival(0x106).unwrap(); + let (plane, sink) = watch_plane(); + assert!(service.attach_watch_plane(peer, plane)); + let root = service.engine.open_root(watcher, Rights::READ).unwrap(); + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 0, 7, crate::router::Store::User, root.handle, target), + FsResponse::Done + ); + let mut cursor = Cursor::new(0, 7).unwrap(); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Same); + + // Nothing changed: nothing published, nobody owed. + let mut owed = [None; 4]; + assert_eq!(settle(&mut service, &mut owed), 0); + + // The client parks; the other peer makes a file in the shared view. + let seen = sink.seq(); + assert!(sink.park(seen)); + let writer_root = service.engine.open_root(writer, Rights::ALL).unwrap(); + crate::block_on(service.engine.create( + writer, + writer_root.handle, + &Name::new("new.txt").unwrap(), + ObjectKind::File, + )) + .unwrap(); + assert_eq!(settle(&mut service, &mut owed), 1); + assert_eq!(owed[0], Some(peer)); + assert_eq!(settle(&mut service, &mut owed), 0, "one change, one wake"); + assert_ne!(sink.seq(), seen); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Changed); + + // A running client is owed nothing for the next change; it reads it. + let created = crate::block_on(service.engine.create( + writer, + writer_root.handle, + &Name::new("more.txt").unwrap(), + ObjectKind::File, + )) + .unwrap(); + assert_eq!(settle(&mut service, &mut owed), 0); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Changed); + + // A child's bytes changing is a change to what the folder shows. + crate::block_on(service.engine.write_at( + writer, + created.handle, + 0, + b"bytes", + crate::ANY_REVISION, + )) + .unwrap(); + assert_eq!(settle(&mut service, &mut owed), 0); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Changed); + + // Closing the watched handle ends the watch: the plane stops + // counting for a route the client let go of. + service.engine.close(watcher, root.handle).unwrap(); + assert_eq!(settle(&mut service, &mut owed), 0); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Ended); + assert_eq!(service.watches.armed_count(), 0); + } + + /// A plane lives in its serving slot's own record, so a peer far past + /// the host's bootstrap table — the seventh program reading files, and + /// the two-hundredth — watches as the first does. + #[test] + fn a_peer_in_a_high_slot_has_its_own_plane() { + use kore_dagfs::watch::{Cursor, Seen}; + let (mut service, _, writer) = watched_view(); + let high = admit_view(&mut service, 0x1C8, 0xD0, Rights::ALL); + assert_eq!(high, SessionId::new(0xC8)); + let peer = PeerId::from_arrival(0x1C8).unwrap(); + let (plane, sink) = watch_plane(); + assert!(service.attach_watch_plane(peer, plane)); + let root = service.engine.open_root(high, Rights::READ).unwrap(); + let target = service.watch_target(high, root.handle); + assert_eq!( + service.arm_watch(0x1C8, high, 3, 11, crate::router::Store::User, root.handle, target), + FsResponse::Done + ); + let mut cursor = Cursor::new(3, 11).unwrap(); + let writer_root = service.engine.open_root(writer, Rights::ALL).unwrap(); + crate::block_on(service.engine.create( + writer, + writer_root.handle, + &Name::new("far.txt").unwrap(), + ObjectKind::File, + )) + .unwrap(); + let mut owed = [None; 4]; + settle(&mut service, &mut owed); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Changed); + // Closing the peer wipes the slot, plane and index together. + service.detach_watch_plane(peer); + assert!(service.peer_closed(0x1C8, PeerClose::Orderly)); + assert_eq!(service.watches.armed_count(), 0); + } + + #[test] + fn a_watch_takes_a_listing_s_authority_and_a_plane_of_the_asking_pair() { + let (mut service, watcher, _) = watched_view(); + let peer = PeerId::from_arrival(0x106).unwrap(); + let root = service.engine.open_root(watcher, Rights::ALL).unwrap(); + let file = crate::block_on(service.engine.create( + watcher, + root.handle, + &Name::new("f").unwrap(), + ObjectKind::File, + )) + .unwrap(); + let user = crate::router::Store::User; + + // No plane attached yet: nothing to publish into. + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 0, 1, user, root.handle, target), + FsResponse::Refused(FsError::Unsupported) + ); + let (plane, sink) = watch_plane(); + assert!(service.attach_watch_plane(peer, plane)); + // A second plane for the occupied slot is refused. + let (again, _) = watch_plane(); + assert!(!service.attach_watch_plane(peer, again)); + // A tail that carries no plane admits its peer without one. + let bare: &'static mut [u8] = alloc::boxed::Box::leak(alloc::boxed::Box::new([0u8; 64])); + assert!(service.attach_watch_plane(PeerId::from_arrival(0x105).unwrap(), bare)); + // Another generation of the slot names no plane of its own. + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x206, watcher, 0, 1, user, root.handle, target), + FsResponse::Refused(FsError::Unsupported) + ); + // A file is not watched: a listing is what a watch follows. + let target = service.watch_target(watcher, file.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 0, 1, user, file.handle, target), + FsResponse::Refused(FsError::NotDirectory) + ); + // A slot past the plane, or the zero tag, is out of range. + for (slot, tag) in [(kore_abi::svc::FS_WATCH_SLOTS as u8, 1), (0, 0)] { + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, slot, tag, user, root.handle, target), + FsResponse::Refused(FsError::Range) + ); + } + assert_eq!(service.watches.armed_count(), 0); + + // Re-arming a slot replaces its watch; disarming clears its tag. + for tag in [3, 4] { + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 1, tag, user, root.handle, target), + FsResponse::Done + ); + } + assert_eq!(service.watches.armed_count(), 1); + assert_eq!(sink.state().unwrap().tag[1], 4); + assert_eq!(service.disarm_watch(0x106, watcher, 1), FsResponse::Done); + assert_eq!(sink.state().unwrap().tag[1], 0); + assert_eq!(service.watches.armed_count(), 0); + + // Detaching the plane before the window goes drops its watches. + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 2, 9, user, root.handle, target), + FsResponse::Done + ); + service.detach_watch_plane(PeerId::from_arrival(0x206).unwrap()); + assert_eq!(service.watches.armed_count(), 1, "a late detach for another generation is inert"); + service.detach_watch_plane(peer); + assert_eq!(service.watches.armed_count(), 0); + } + /// A dead writer's staged replacement is released on the service's next /// dispatch, never inside the synchronous peer-death hook. A release the /// provider refuses stays pending — custody retained — while the dead diff --git a/native/services/dagfs/src/watches.rs b/native/services/dagfs/src/watches.rs new file mode 100644 index 0000000..c7800aa --- /dev/null +++ b/native/services/dagfs/src/watches.rs @@ -0,0 +1,310 @@ +//! Directory watches: the service half of the watch plane +//! (`kore_dagfs::watch`, `res/DAGFS.md` §4). +//! +//! A session arms up to [`FS_WATCH_SLOTS`] watches, each on a directory +//! handle it holds. After every unit of service work each armed directory's +//! change epoch ([`Namespace::change_epoch`]) is read **through the handle's +//! route** — revalidated as every operation's is — and the plane republished +//! when one moved. Only a client parked on its plane is owed a wake; the +//! serving host sends each one notice (`SystemService::take_owed_wakes`). +//! +//! Nothing here is global. A peer's [`Plane`] lives in its serving slot's own +//! record, beside the slot's window and staged replacement, allocated with it +//! and wiped with it; the service keeps only [`WatchIndex`] — which slots have +//! a watch armed, and which owe a wake — so a settle visits exactly the slots +//! that need it, however large the slot space grows. A watch on the dual +//! router's system volume is recorded in the same plane with the store it +//! lives in, so the one plane a client holds carries both. +//! +//! [`Namespace::change_epoch`]: crate::namespace::Namespace::change_epoch + +use kore_abi::svc::{FS_WATCH_SLOTS, FsWatchState}; +use kore_dagfs::watch::{self, WatchSource, state}; +use kore_rt::svc::PeerId; + +use crate::{ + ObjectId, + authority::{Handle, SessionId}, + bounds::PAIR_SESSIONS, + error::{FsError, FsResult}, + router::Store, +}; + +/// One armed slot. +#[derive(Clone, Copy, Debug)] +struct Watch { + store: Store, + handle: Handle, + dir: ObjectId, + /// The epoch the plane last reported for this watch. + seen: u64, +} + +/// One recyclable peer's watch plane and the watches published into it. +pub(crate) struct Plane { + peer: PeerId, + source: WatchSource<'static>, + slots: [Option; FS_WATCH_SLOTS], + /// The plane's state changed since its last commit. + dirty: bool, +} + +impl Plane { + /// `peer`'s plane over its pair window's tail (`aux`, as the host hands it + /// to `attach_peer_aux`), or `None` for a tail without a formatted plane. + /// Starts from nothing armed, whatever the window last carried. + pub(crate) fn adopt(peer: PeerId, aux: &'static mut [u8]) -> Option { + let mut source = watch::source(aux).ok()?; + if source.replace_state(FsWatchState::default()) { + let _ = source.commit(); + } + Some(Self { + peer, + source, + slots: [None; FS_WATCH_SLOTS], + dirty: false, + }) + } + + pub(crate) const fn peer(&self) -> PeerId { + self.peer + } + + /// Whether any slot is armed. + pub(crate) fn armed(&self) -> bool { + self.slots.iter().any(Option::is_some) + } + + fn edit(&mut self, edit: impl FnOnce(&mut FsWatchState)) { + self.source.state(edit); + self.dirty = true; + } + + /// Publish a batch: `true` when its commit found the client parked, and + /// so owes it one notice. + fn commit(&mut self) -> bool { + core::mem::take(&mut self.dirty) && self.source.commit() + } + + /// Arm `slot` under `tag` on the directory `dir` at epoch `seen` (its + /// baseline: only a later change counts), replacing what the slot held. + /// Answers whether a parked client is owed a wake. + pub(crate) fn arm( + &mut self, + slot: u8, + tag: u32, + store: Store, + handle: Handle, + (dir, seen): (ObjectId, u64), + ) -> FsResult { + let slot = usize::from(slot); + if slot >= FS_WATCH_SLOTS || tag == 0 { + return Err(FsError::Range); + } + self.slots[slot] = Some(Watch { + store, + handle, + dir, + seen, + }); + self.edit(|s| state::arm(s, slot, tag)); + Ok(self.commit()) + } + + /// Disarm `slot`, armed or not. Answers whether a parked client is owed + /// a wake. + pub(crate) fn disarm(&mut self, slot: u8) -> FsResult { + let slot = usize::from(slot); + if slot >= FS_WATCH_SLOTS { + return Err(FsError::Range); + } + self.slots[slot] = None; + self.edit(|s| state::disarm(s, slot)); + Ok(self.commit()) + } + + /// Read every armed watch's epoch through `epoch` — the engine of its + /// store, revalidating its route — and publish what moved. A watch whose + /// route no longer proves ends; one whose directory was mid-publish + /// (`Busy`) counts as changed and keeps its baseline, so the next reading + /// reports the change again rather than losing it. Answers whether a + /// parked client is owed a wake. + pub(crate) fn settle( + &mut self, + session: SessionId, + mut epoch: impl FnMut(Store, SessionId, Handle, ObjectId) -> FsResult, + ) -> bool { + for slot in 0..FS_WATCH_SLOTS { + let Some(watch) = self.slots[slot].as_mut() else { + continue; + }; + match epoch(watch.store, session, watch.handle, watch.dir) { + Ok(now) if now == watch.seen => {} + Ok(now) => { + watch.seen = now; + self.edit(|s| state::changed(s, slot)); + } + Err(FsError::Busy) => self.edit(|s| state::changed(s, slot)), + Err(_) => { + self.slots[slot] = None; + self.edit(|s| state::end(s, slot)); + } + } + } + self.commit() + } +} + +/// Words of a set over every pair session's slot. +const WORDS: usize = PAIR_SESSIONS.div_ceil(64); + +/// A set of pair-session slots: one bit each, scanned word by word. +#[derive(Clone, Copy, Debug, Default)] +struct SlotSet([u64; WORDS]); + +impl SlotSet { + fn insert(&mut self, slot: usize) { + if let Some(word) = self.0.get_mut(slot / 64) { + *word |= 1 << (slot % 64); + } + } + + fn remove(&mut self, slot: usize) { + if let Some(word) = self.0.get_mut(slot / 64) { + *word &= !(1 << (slot % 64)); + } + } + + fn is_empty(&self) -> bool { + self.0.iter().all(|word| *word == 0) + } + + /// The members, lowest first, as of the call. + fn members(&self) -> impl Iterator + use<> { + let words = self.0; + (0..WORDS).flat_map(move |at| { + let mut word = words[at]; + core::iter::from_fn(move || { + (word != 0).then(|| { + let bit = word.trailing_zeros() as usize; + word &= word - 1; + at * 64 + bit + }) + }) + }) + } +} + +/// Which slots' planes have a watch armed, and which owe their client a wake. +/// Derived from the planes and kept beside them: a settle and a wake +/// collection cost the slots they name, and nothing when the sets are empty. +#[derive(Debug, Default)] +pub(crate) struct WatchIndex { + armed: SlotSet, + owed: SlotSet, +} + +impl WatchIndex { + pub(crate) const fn new() -> Self { + Self { + armed: SlotSet([0; WORDS]), + owed: SlotSet([0; WORDS]), + } + } + + /// Record what an edit of `slot`'s plane left: whether a watch is still + /// armed there, and whether its client is owed a wake. + pub(crate) fn note(&mut self, slot: usize, armed: bool, owed: bool) { + if armed { + self.armed.insert(slot); + } else { + self.armed.remove(slot); + } + if owed { + self.owed.insert(slot); + } + } + + /// Forget `slot`: its plane is gone. + pub(crate) fn forget(&mut self, slot: usize) { + self.armed.remove(slot); + self.owed.remove(slot); + } + + /// The slots with a watch armed. + pub(crate) fn armed(&self) -> impl Iterator + use<> { + self.armed.members() + } + + pub(crate) fn idle(&self) -> bool { + self.armed.is_empty() + } + + /// Hand `out` each slot owing a wake, lowest first, taking it off the set; + /// `out` answers `false` when its batch is full, and the rest stay owed. + pub(crate) fn take_owed(&mut self, mut out: impl FnMut(usize) -> bool) { + for slot in self.owed.members() { + if !out(slot) { + return; + } + self.owed.remove(slot); + } + } + + #[cfg(test)] + pub(crate) fn armed_count(&self) -> usize { + self.armed.members().count() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_slot_set_names_its_members_lowest_first_across_words() { + let mut set = SlotSet::default(); + for slot in [PAIR_SESSIONS - 1, 3, 64, 65, 0] { + set.insert(slot); + } + set.insert(PAIR_SESSIONS); // past the space: ignored + let members: std::vec::Vec = set.members().collect(); + assert_eq!(members, [0, 3, 64, 65, PAIR_SESSIONS - 1]); + set.remove(64); + set.remove(0); + assert_eq!(set.members().collect::>(), [3, 65, PAIR_SESSIONS - 1]); + assert!(!set.is_empty()); + for slot in [3, 65, PAIR_SESSIONS - 1] { + set.remove(slot); + } + assert!(set.is_empty()); + } + + #[test] + fn owed_wakes_are_taken_once_and_a_full_batch_keeps_the_rest() { + let mut index = WatchIndex::new(); + for slot in [2, 7, 9] { + index.note(slot, true, true); + } + let mut taken = std::vec::Vec::new(); + index.take_owed(|slot| { + if taken.len() == 2 { + return false; + } + taken.push(slot); + true + }); + assert_eq!(taken, [2, 7]); + taken.clear(); + index.take_owed(|slot| { + taken.push(slot); + true + }); + assert_eq!(taken, [9], "the batch's leftover, and nothing twice"); + index.forget(9); + assert_eq!(index.armed().collect::>(), [2, 7]); + index.note(2, false, false); + index.note(7, false, false); + assert!(index.idle()); + } +} diff --git a/native/services/device/boot.toml b/native/services/device/boot.toml index 80737c4..7d444ce 100644 --- a/native/services/device/boot.toml +++ b/native/services/device/boot.toml @@ -47,5 +47,5 @@ protocol = "kore_svc_device::DeviceBroker" # row's own claimant band. instances = 8 endows = ["ROOT_COMPLEX@0", "PLATFORM_BUS@0", "FACE_SUPPLY@0"] -digest_x86_64 = "blake3:6eb5411353e5cf48b9c9497d3b4868207bb8bbade4827537be4a7af3492d4a74" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:cd930094ae3f3fe8145f9a8fd61e5c522bc39b4956f4f45aadd1a66726aa3098" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:f4ef9270f4c48990e242ce05f4d7c7b5c62cb2da959e148290ed073df9075379" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:a4d0b40e6d7ef5d6ce5d270ba91f1c09aee246d5b16e99f864001b92dbe570f4" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/device/src/service/mod.rs b/native/services/device/src/service/mod.rs index 3e1cf26..4e0e6b1 100644 --- a/native/services/device/src/service/mod.rs +++ b/native/services/device/src/service/mod.rs @@ -3,7 +3,7 @@ use kore_abi::device::{DeviceRecord, DeviceSpecies}; use kore_abi::session::SlotRef; -use kore_abi::task::{FACE_RING_STRIDE, FACE_SUPPLY_BYTES, role}; +use kore_abi::task::{CLAIMANT_FACE_SUPPLY_BYTES, FACE_RING_STRIDE, role}; use kore_rt::cap::{PlatformBusToken, RootComplexToken, SupplyToken, verbs}; use kore_rt::entry::FixedSlotBand; use kore_rt::service::{ @@ -34,8 +34,11 @@ pub const DEVICE_GRANT_RIGHTS: Rights = verbs::DEVICE_OWNER_RIGHTS; /// Claimant faces the broker's [`role::FACE_SUPPLY`] band strides into — /// the geometry contract's own quotient, so the count can never disagree -/// with the span the founder mints. -pub const FACE_COUNT: usize = (FACE_SUPPLY_BYTES / FACE_RING_STRIDE) as usize; +/// with the span the founder mints: one face per driver row +/// ([`MAX_CLAIMANT_FACES`](kore_abi::device::MAX_CLAIMANT_FACES)). +pub const FACE_COUNT: usize = (CLAIMANT_FACE_SUPPLY_BYTES / FACE_RING_STRIDE) as usize; + +const _: () = assert!(FACE_COUNT == kore_abi::device::MAX_CLAIMANT_FACES); /// The first minted face's [`Serve::via`] identity. Minted vias start at /// [`MAX_SERVE_PAIRS`](kore_rt::svc::MAX_SERVE_PAIRS) — the host's rule @@ -59,6 +62,14 @@ const FACE_LAND_BASE: u32 = kore_abi::task::MAX_BOOT_HANDLES as u32 + kore_rt::entry::SERVE_SCRATCH_SLOTS as u32 + 2; const FACE_LAND_END: u32 = FACE_LAND_BASE + 2 * FACE_COUNT as u32; +// `claim_fixed_slot_band` admits nothing at or past the startup-link band, +// so a band that crosses it refuses readiness (detail 0x204) on every +// machine. Checked here, where the face count and the band meet. +const _: () = assert!( + FACE_LAND_END <= kore_rt::entry::STARTUP_LINK_SLOT_BASE, + "the claimant landing band must end below the startup-link band" +); + /// A conveyed face leg must at least be postable by the broker. const FACE_LEG_RIGHTS: Rights = Rights::ACCESS; @@ -915,11 +926,11 @@ impl SystemService for DeviceBroker { if ::SERVE_PAIRS > 1 { return Err(StartFailure::new(0x205)); } - if supply.len() != FACE_SUPPLY_BYTES { + if supply.len() != CLAIMANT_FACE_SUPPLY_BYTES { return Err(StartFailure::new(0x200)); } let win = verbs::map_held_rw(supply.slot()).ok_or_else(|| StartFailure::new(0x201))?; - if win.len() < FACE_SUPPLY_BYTES as usize { + if win.len() < CLAIMANT_FACE_SUPPLY_BYTES as usize { return Err(StartFailure::new(0x202)); } let band = rt @@ -1178,7 +1189,7 @@ impl SystemService for DeviceBroker { // image face table's recycled-slot hygiene. // SAFETY: the view is live for the task's life // (mapped once at start, length gated ≥ - // FACE_SUPPLY_BYTES there); this face is not Live, + // CLAIMANT_FACE_SUPPLY_BYTES there); this face is not Live, // so no attached pair aliases this stride // (occupancy is the exclusion discipline); the // `&mut` ends before the spec is exposed. @@ -1370,11 +1381,15 @@ mod face_tests { } /// The band geometry quotient: the face count times the stride is - /// exactly the supply span the founder mints (`FACE_SUPPLY_BYTES`), - /// and the landing band holds exactly two slots per face. + /// exactly the supply span the founder mints + /// (`CLAIMANT_FACE_SUPPLY_BYTES`), and the landing band holds exactly + /// two slots per face. #[test] fn the_face_geometry_is_the_contracts_quotient() { - assert_eq!(FACE_COUNT as u64 * FACE_RING_STRIDE, FACE_SUPPLY_BYTES); + assert_eq!( + FACE_COUNT as u64 * FACE_RING_STRIDE, + CLAIMANT_FACE_SUPPLY_BYTES + ); assert_eq!(FACE_LAND_END - FACE_LAND_BASE, 2 * FACE_COUNT as u32); } } diff --git a/native/services/mm/boot.toml b/native/services/mm/boot.toml index 094ac5a..7771a32 100644 --- a/native/services/mm/boot.toml +++ b/native/services/mm/boot.toml @@ -54,5 +54,5 @@ supply_quota_bytes = 27917287424 # 26 GiB endows = ["MEM_SUPPLY@0", "MEM_REGION@0", "SYSTEM_OBSERVE@0"] # review mirror; asserted === PROFILE.roles shell_client = true exec_client = true -digest_x86_64 = "blake3:0c893dbdf58c6c17dd66d424ad025ae0cc5b4eab41a718940f09a3595c32830a" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:f1ffc4cb9f02cc59ef78781fd8a1cba3722d9518b7abb0adb17bf3f7376b9f57" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:2ebee97309e197da6bcf90b34d7fbdd42eafa679b6afd770e3599500eefed040" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:693bd018cbfffac602028bcbbac3dcbc734b6e3347106f1529a9934b32c92d0b" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/swap/boot.toml b/native/services/swap/boot.toml index b433dd4..314360e 100644 --- a/native/services/swap/boot.toml +++ b/native/services/swap/boot.toml @@ -9,5 +9,5 @@ image = "kore-svc-swap" endowment = "kore_svc_swap::ZramEndowment" protocol = "kore_svc_swap::ZramSwap" endows = ["MEM_SUPPLY@0"] -digest_x86_64 = "blake3:5f77376810266176274a0af803d6b16bb9d750d97780b171e5fba376fcbb58fc" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:b939b2b8bd153674b2bf76dfa83e0941cc3f16139c721c5271f6a75b6fea0874" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:c6363f610b2e977482dca6d876e201b689667ba62f45052a6d2fc49ce425b754" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:7c9b860f6572ab954cebd6579ef63f38a9256e93206127f9cd0c1bfc93a72d7d" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/vdev/Cargo.toml b/native/services/vdev/Cargo.toml new file mode 100644 index 0000000..7d66c36 --- /dev/null +++ b/native/services/vdev/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "kore-svc-vdev" +version = "0.0.0" +edition = "2024" +description = "kore virtual-device service: hosts device models with no hardware behind them, one per driver, over memory the driver shares." +publish = false + +[dependencies] +kore-abi = { path = "../../crates/abi" } +kore-rt = { path = "../../crates/rt" } +# The transport a driver reaches a device model through. +kore-vdev = { path = "../../crates/vdev" } +# The first model. Only the service hosts it. +kore-vdev-accel = { path = "../../crates/vdev-accel", optional = true } + +[features] +default = ["host"] +# The service itself: the device table and the models it hosts. A crate that +# only speaks to the service (a driver's library, which carries no device +# model) builds without it and keeps the wire, the face and the client. +host = ["dep:kore-vdev-accel"] + +[dev-dependencies] +kore-rt = { path = "../../crates/rt", features = ["std"] } +# The driver the host tests drive a hosted device with: the protocol's +# reference guest, over kore-vdev's driver half. +virtio-accel-guest = { version = "=0.4.1", default-features = false } +virtio-accel-proto = { version = "=0.4.1", default-features = false } +virtio-accel-transport = { version = "=0.4.1", default-features = false } diff --git a/native/services/vdev/boot.toml b/native/services/vdev/boot.toml new file mode 100644 index 0000000..9ff6c63 --- /dev/null +++ b/native/services/vdev/boot.toml @@ -0,0 +1,30 @@ +# services/vdev/boot.toml — this service's boot-manifest row (the design +# lives in res/boot.toml's header). The row rides beside the crate it +# endows; whether and WHEN it founds is the root manifest's `services` list. + +# The virtual-device service (res/VIRTUAL_DEVICE.md): hosts a device model +# with no hardware behind it for each driver that opens one. +# +# Unchained, and before the device..exec retained-founder chain for the +# reason power is: exec owns pair one's client face and its retained return +# leg, so the row must exist before exec's founder can publish that face. +# +# Pair zero is the founder's control face. Pair one belongs to exec, which +# uses it only to admit and retire one recyclable face per program launched +# with the virtual-device profile. The shell receives no face. +# +# The row is endowed with nothing: no device, no authority over the machine. +# Its heap is where every open device lives. Four devices may be open at +# once, each holding at most 2 MiB of buffers for its driver +# (`kore_svc_vdev::host::DEVICE_HEAP_BYTES`, 8 MiB); the lease below covers +# that with 4 MiB over for the engines' own tables, the seat table and the +# runtime. The service's tests hold the two numbers together. +[service] +image = "kore-svc-vdev" +endowment = "kore_svc_vdev::VdevEndowment" +protocol = "kore_svc_vdev::VdevService" +endows = [] +exec_client = true # pair 1 administers per-launch driver faces +heap_supply_bytes = 0x00C0_0000 +digest_x86_64 = "blake3:33b4bd37c5e76b836ff3bb9f041b37184483f33ba7c37c4b04b09e11cc4b1b00" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:344e1f6da34c1ef515b0c0b9988b5e3b9e673cf08df6f6065ecd5bc661fe42de" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/vdev/src/client.rs b/native/services/vdev/src/client.rs new file mode 100644 index 0000000..e21b772 --- /dev/null +++ b/native/services/vdev/src/client.rs @@ -0,0 +1,439 @@ +//! The driver's side of the service: the face a program is launched with, +//! and exec's administration of such faces. +//! +//! A driver does four things with its face, and only the first two are +//! requests: +//! +//! 1. [`open`](ProcessVdev::open) a device model. The register block, which +//! the driver has had mapped since it started, is written before the +//! reply. +//! 2. [`lend`](ProcessVdev::lend) the memory its queue and buffers are in. +//! The driver keeps the lease and its own mapping; the service receives +//! a view of exactly the bytes named. +//! 3. Ring the [`doorbell`](ProcessVdev::doorbell): a notice, with no reply. +//! 4. [`wait`](ProcessVdev::wait) for the interrupt: a park on the register +//! block's interrupt count, which any thread of the driver may do. +//! +//! Everything else a driver does with its device it does in the memory the +//! two share, through `kore_vdev::driver` or by hand. +//! +//! # A driver with more than one thread +//! +//! Requests belong to the thread the program was launched on: the face's +//! rings have one producer, and the replies wake that thread. The doorbell +//! and the interrupt are any thread's, but a capability is a thread's own, so +//! a thread that rings or parks must have been founded holding its share of +//! the face ([`THREAD_SHARES`], [`ThreadVdev`]). The register block itself is +//! mapped once, by the launched thread, and every thread of the program sees +//! that mapping. + +use kore_abi::cap::{ResourceKind, Rights}; +use kore_abi::session::SlotRef; +use kore_abi::task::role; +use kore_rt::svc::{ + CallError, PeerBundlePart, PeerId, ReturnDeriveError, ReturnDeriver, StagedReturn, + StagedReturnError, SvcClient, +}; +use kore_rt::thread_controller::Handed; +use kore_rt::transport::Transport; + +use crate::VdevService; +use crate::wire::{VdevRefusal, VdevRequest, VdevResponse}; + +/// Where a program launched with the virtual-device profile finds its face +/// in its own boot block. +/// +/// The layout exec stamps and the program binds, in one place. Indices +/// 0 to 5 of these role namespaces belong to the filesystem, window, +/// memory, accelerator and power faces a launch may also carry; exec pins +/// the disjointness where it sees every face. +pub mod process_face { + /// The doorbell, and the request kick (`role::SVC_SEND`). + pub const REQUEST_KICK: u16 = 8; + /// The face itself: request rings, then the register block + /// (`role::SVC_RING`). + pub const CONTROL_RING: u16 = 8; + /// The return leg a driver lends memory on (`role::SVC_RETURN`). + pub const RETURN_SEND: u16 = 8; + /// Correlation datum for the service's landing on that leg + /// (`role::SVC_RETURN`). + pub const RETURN_LAND: u16 = 9; +} + +/// The cookie of a doorbell notice: `"vdevbell"`. The service looks at its +/// devices on any wake, so the cookie only says what the notice was for. +pub const COOKIE_DOORBELL: u64 = u64::from_le_bytes(*b"vdevbell"); + +/// Rights of the view of lent memory the service receives: it reads and +/// writes, and can pass it nowhere. +pub const LENT_RIGHTS: Rights = Rights::ACCESS.union(Rights::MODIFY); +/// The driver's staged view carries `DERIVE` as well, so the rendezvous can +/// attenuate it to [`LENT_RIGHTS`]. +#[cfg(target_os = "none")] +const LENT_STAGED_RIGHTS: Rights = LENT_RIGHTS.union(Rights::DERIVE); + +/// What a thread of a driver holds of the face, beyond the thread the +/// program was launched on: the face's memory, to park on the interrupt +/// count in it, and the kick, to ring the doorbell. It reads and notifies; +/// it maps nothing and writes nothing through either. +/// +/// A program hands these on to the threads it founds +/// (`kore_rt::thread::share_endowment` for one it founds itself, +/// `Controller::hand_on` for the threads a controller founds). +pub const THREAD_SHARES: [Handed; 2] = [ + Handed { + role: role::SVC_RING, + index: process_face::CONTROL_RING, + kind: ResourceKind::MemLease, + rights: Rights::ACCESS, + }, + Handed { + role: role::SVC_SEND, + index: process_face::REQUEST_KICK, + kind: ResourceKind::ChanSend, + rights: Rights::ACCESS, + }, +]; + +/// How a wait for the interrupt ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Waited { + /// The interrupt count moved, to this. + Interrupt(u32), + /// The deadline passed with the count unmoved. + Deadline, + /// The caller's own condition ended the wait. + Stopped, + /// This thread cannot wait on the device: its share of the face is + /// gone, or the kernel will not park it. Waiting again would not help. + Gone, +} + +/// The longest one park of an open-ended wait: the kernel's word wait is +/// never unbounded, so a wait with no deadline is a loop of bounded parks +/// and each re-reads the count. A quarter of a second, kore-rt's figure for +/// the same loop (`futex`, `event`): long enough that a waiter is not paced +/// by it, short enough that the caller's condition, which no wake +/// announces, is seen soon after it changes. +#[cfg(target_os = "none")] +const PARK_SLICE_NS: u64 = 250_000_000; + +/// The ring client over the generic [`SvcClient`]. +pub struct RingVdev<'w, 't, T: Transport>(pub SvcClient<'w, 't, T, VdevService>); + +impl RingVdev<'_, '_, T> { + /// One request. A call that never reached the service answers + /// [`VdevRefusal::Unavailable`]: to a driver the two are the same, a + /// device it cannot have now. + pub async fn request(&mut self, request: VdevRequest) -> VdevResponse { + match self.0.call(&request).await { + Ok(reply) => reply.response, + Err(_) => VdevResponse::Refused(VdevRefusal::Unavailable), + } + } +} + +/// Exec's fixed administration face, with its retained return leg. +pub struct RingVdevAdmin<'w, 't, T: Transport> { + pub client: RingVdev<'w, 't, T>, + return_tx: SlotRef, + return_land: SlotRef, +} + +impl<'w, 't, T: Transport> RingVdevAdmin<'w, 't, T> { + pub const fn new( + client: RingVdev<'w, 't, T>, + return_tx: SlotRef, + return_land: SlotRef, + ) -> Self { + Self { + client, + return_tx, + return_land, + } + } + + /// Convey one component of a driver's face. + pub async fn admit_component_staged( + &mut self, + part: PeerBundlePart, + rights: Rights, + stage: F, + ) -> Result + where + F: FnOnce(ReturnDeriver<'_>) -> Result, + { + let request = VdevRequest::AdmitPeer { + part, + announced: self.return_land, + }; + self.client + .0 + .call_with_staged_return(&request, self.return_tx, rights, stage) + .await + .map(|reply| reply.response) + } + + pub async fn take_admission(&mut self) -> Result { + self.call(VdevRequest::TakeAdmission).await + } + + pub async fn retire_peer(&mut self, peer: PeerId) -> Result { + self.call(VdevRequest::RetirePeer { peer }).await + } + + pub async fn confirm_retired(&mut self, peer: PeerId) -> Result { + self.call(VdevRequest::ConfirmRetired { peer }).await + } + + async fn call(&mut self, request: VdevRequest) -> Result { + self.client + .0 + .call(&request) + .await + .map(|reply| reply.response) + } +} + +#[cfg(target_os = "none")] +pub use metal::{ProcessVdev, ThreadVdev, attach_process_face, attach_thread_face}; + +#[cfg(target_os = "none")] +mod metal { + use super::*; + use kore_rt::cap::verbs::{WordWait, WordWaitError}; + use kore_rt::cap::{LeaseToken, verbs}; + use kore_rt::service::MAX_GRANTS; + use kore_rt::svc::{ClientEndpoints, REP_CAP, REQ_CAP, ReturnLeaseSource, client_rings}; + use kore_rt::transport::TrapTransport; + use kore_vdev::mem::{Memory, Window}; + use kore_vdev::regs; + + use crate::face::{PEER_BYTES, REGISTERS_AT}; + + // A notice needs exactly what a thread's share of the kick carries. + const _: () = assert!(THREAD_SHARES[1].rights.bits() == verbs::NOTICE_RIGHTS.bits()); + + /// One thread's part of the face: the doorbell and the interrupt. + /// + /// Both name capabilities in the calling thread's own arena, so a value + /// of this is its thread's and is attached on the thread that uses it. + #[derive(Clone, Copy, Debug)] + pub struct ThreadVdev { + kick: SlotRef, + /// The interrupt count, named to the kernel: this thread's holding + /// of the face's memory and the count's offset in it. + interrupts: (SlotRef, u32), + } + + /// Attach the calling thread's part of the face from its own boot + /// block. `None` on a thread founded without it. + pub fn attach_thread_face(rt: &kore_rt::entry::Rt) -> Option { + let boot = rt.bootstrap(); + Some(ThreadVdev { + kick: boot.handle(role::SVC_SEND, process_face::REQUEST_KICK)?, + interrupts: ( + boot.handle(role::SVC_RING, process_face::CONTROL_RING)?, + u32::try_from(REGISTERS_AT + regs::INTERRUPTS).ok()?, + ), + }) + } + + impl ThreadVdev { + /// Ring the doorbell. `false` when the notice was not taken: the + /// service's doorbell is full, which means it has a wake standing + /// and will look anyway. + pub fn doorbell(&self) -> bool { + verbs::post_notice(self.kick, COOKIE_DOORBELL, 0) + } + + /// Park until the interrupt count is no longer `seen`, or until + /// `deadline_ns` (absolute monotonic nanoseconds): one park, and + /// what ended it. The caller reads the count from the register + /// block afterwards, whatever this answers. + /// + /// The park compares in the kernel, so an interrupt raised between + /// the caller's read of `seen` and this call is not lost. + pub fn park(&self, seen: u32, deadline_ns: u64) -> Result { + let (subject, offset) = self.interrupts; + verbs::wait_word(subject, offset, seen, deadline_ns) + } + + /// Wait until the interrupt count in `registers` is no longer + /// `seen`: until `deadline_ns` (absolute monotonic nanoseconds) + /// when one is given, and for as long as `go_on` answers `true`, + /// which is asked before every park. + /// + /// Every wait is a park in the kernel; nothing here spins. A park + /// the kernel has no room for now is a short cede and another look. + pub fn wait( + &self, + registers: Window, + seen: u32, + deadline_ns: Option, + mut go_on: impl FnMut() -> bool, + ) -> Waited { + loop { + match registers.load_u32(regs::INTERRUPTS) { + Ok(count) if count != seen => return Waited::Interrupt(count), + Ok(_) => {} + Err(_) => return Waited::Gone, + } + if !go_on() { + return Waited::Stopped; + } + let Some(now) = kore_rt::clock::deadline_base_nanos() else { + // No way to name an instant, and the kernel takes no + // unbounded park. + return Waited::Gone; + }; + let slice = now.saturating_add(PARK_SLICE_NS); + let bound = match deadline_ns { + Some(at) if now >= at => return Waited::Deadline, + Some(at) => at.min(slice), + None => slice, + }; + match self.park(seen, bound) { + Ok(WordWait::Changed | WordWait::Deadline) => {} + Ok(WordWait::WouldBlock) => { + if !kore_rt::pace::cede() { + return Waited::Gone; + } + } + Ok(WordWait::Revoked) | Err(_) => return Waited::Gone, + } + } + } + } + + /// A program's face with the service: its request rings, the register + /// block of its device, the leg it lends memory on, and the launched + /// thread's own doorbell and interrupt. + pub struct ProcessVdev<'t> { + client: RingVdev<'static, 't, TrapTransport>, + registers: Window, + thread: ThreadVdev, + return_tx: SlotRef, + return_land: SlotRef, + /// Keeps the face mapped for the life of the program. + _ring: LeaseToken, + } + + /// Attach the face from a profiled program's boot block. `None` when the + /// program was launched without one: there is no device to open. + pub fn attach_process_face<'t>( + rt: &kore_rt::entry::Rt, + transport: &'t TrapTransport, + ) -> Option> { + let boot = rt.bootstrap(); + let thread = attach_thread_face(rt)?; + let ring = verbs::take_lease(boot.handle(role::SVC_RING, process_face::CONTROL_RING)?)?; + let window = verbs::map_held_rw(ring.slot())?; + if (window.len() as u64) < PEER_BYTES { + return None; + } + let base = window.cast::(); + // SAFETY: the first page is the face's rings, and this is the + // program's sole client attachment over the task-lifetime lease exec + // delivered. The slice stops short of the register block, so no + // reference covers what the device writes. + let rings: &'static mut [u8] = + unsafe { core::slice::from_raw_parts_mut(base.as_ptr(), REGISTERS_AT as usize) }; + let rings = client_rings::(rings, REQ_CAP, REP_CAP).ok()?; + // SAFETY: the second page of the same mapping, page-aligned, mapped + // read-write for as long as `ring` is held, which is as long as + // this value lives. The service writes it concurrently, which is + // what a `Window` is for. + let registers = + unsafe { Window::new(base.add(REGISTERS_AT as usize), regs::BYTES as usize) }; + Some(ProcessVdev { + client: RingVdev(SvcClient::new( + transport, + rings, + ClientEndpoints { + kick: thread.kick, + grants: [const { None }; MAX_GRANTS], + }, + )), + registers, + thread, + return_tx: boot.handle(role::SVC_RETURN, process_face::RETURN_SEND)?, + return_land: SlotRef::from_wire( + boot.datum(role::SVC_RETURN, process_face::RETURN_LAND)? as u32, + ), + _ring: ring, + }) + } + + impl ProcessVdev<'_> { + /// The register block of this program's device. It says a device is + /// there once [`open`](Self::open) has been accepted. The mapping is + /// the process's: every thread of the program may use it for as long + /// as this value lives. + pub const fn registers(&self) -> Window { + self.registers + } + + /// The launched thread's own doorbell and interrupt. + pub const fn thread(&self) -> ThreadVdev { + self.thread + } + + /// Open the device model with this register-block id. + pub async fn open(&mut self, device: u32) -> VdevResponse { + self.client.request(VdevRequest::Open { device }).await + } + + /// Lend the first `bytes` of `source`: device addresses are offsets + /// into them from here on. The caller keeps the lease and its own + /// mapping of it. + pub async fn lend(&mut self, source: &S, bytes: u64) -> VdevResponse { + let request = VdevRequest::Lend { + announced: self.return_land, + }; + let reply = self + .client + .0 + .call_with_staged_return(&request, self.return_tx, LENT_RIGHTS, |derive| { + derive.lease_window(source, LENT_STAGED_RIGHTS, 0, bytes) + }) + .await; + match reply { + Ok(reply) => reply.response, + Err(_) => VdevResponse::Refused(VdevRefusal::Unavailable), + } + } + + /// Give the device up, and take back the memory lent to it. + pub async fn close(&mut self) -> VdevResponse { + self.client.request(VdevRequest::Close).await + } + + /// Ring the doorbell ([`ThreadVdev::doorbell`]). + pub fn doorbell(&self) -> bool { + self.thread.doorbell() + } + + /// The interrupt count now. + pub fn interrupts(&self) -> u32 { + // The block is a page of a mapping this value keeps: the load + // cannot fault. + self.registers.load_u32(regs::INTERRUPTS).unwrap_or(0) + } + + /// Wait until the interrupt count is no longer `seen`, or until + /// `deadline_ns` (absolute monotonic nanoseconds), and answer the + /// count. `seen` again means no interrupt came: the deadline + /// passed, or this thread cannot wait on the device + /// ([`ThreadVdev::wait`]). + pub fn wait(&self, seen: u32, deadline_ns: u64) -> u32 { + match self + .thread + .wait(self.registers, seen, Some(deadline_ns), || true) + { + Waited::Interrupt(count) => count, + Waited::Deadline | Waited::Stopped | Waited::Gone => seen, + } + } + } +} diff --git a/native/services/vdev/src/face.rs b/native/services/vdev/src/face.rs new file mode 100644 index 0000000..d8e84c4 --- /dev/null +++ b/native/services/vdev/src/face.rs @@ -0,0 +1,29 @@ +//! A driver's face with the service: what both sides agree is where. +//! +//! A face is two pages exec supplies when it launches a program with the +//! virtual-device profile: the request rings, then the register block. The +//! driver has both mapped from its first instruction; the service is handed +//! the second when the face is admitted. + +use kore_rt::svc::{REP_CAP, REQ_CAP, window_bytes}; +use kore_vdev::regs; + +#[cfg(feature = "host")] +use crate::service::VdevService; + +/// Where the register block is in a driver's face: the page after the +/// request rings. +pub const REGISTERS_AT: u64 = 0x1000; + +/// A driver's face: the rings' page and the register block's. +pub const PEER_BYTES: u64 = REGISTERS_AT + regs::BYTES; + +// The rings fit the first page, so the block starts on the second. +const _: () = assert!(window_bytes::(REQ_CAP, REP_CAP) as u64 <= REGISTERS_AT); + +/// The service, to a crate that speaks to it and does not host it: the name +/// its protocol hangs on, which nothing can make a value of. The service's +/// own build has the real one (`service::VdevService`). +#[cfg(not(feature = "host"))] +#[derive(Debug)] +pub enum VdevService {} diff --git a/native/services/vdev/src/host.rs b/native/services/vdev/src/host.rs new file mode 100644 index 0000000..d03e0ea --- /dev/null +++ b/native/services/vdev/src/host.rs @@ -0,0 +1,259 @@ +//! **The devices the service hosts.** +//! +//! Every driver face has a *seat*: the register page of that face, and the +//! device the driver opened over it, if it has opened one. A seat costs a +//! few words, so there is one for every program that holds a face. A device +//! is a model with its own state on the service's heap, so only a few are +//! open at once ([`MAX_DEVICES`]). +//! +//! The table is plain data. What the runtime supplies comes in as +//! arguments: the register window when a face is admitted, the mapped +//! window and its owner when a driver lends memory, and a closure that +//! wakes a driver when its device interrupts. That is what lets the whole +//! of it run off metal. + +use alloc::vec::Vec; +use core::num::NonZeroU32; + +use kore_vdev::mem::{Memory, Window}; +use kore_vdev::regs; +use kore_vdev::transport::Transport; +use kore_vdev_accel::device::{AccelDevice, DEVICE_ID, MAX_BUFFER_BACKING_BYTES}; + +/// Devices open at once. +/// +/// A device may hold [`MAX_BUFFER_BACKING_BYTES`] of buffers for its driver +/// on the service's heap, and the heap is one founded lease +/// (`heap_supply_bytes` in the service's boot manifest row). Four devices +/// are 8 MiB of buffers at most, which that lease covers with room for the +/// engines' own tables; a fifth driver is told the service is full and may +/// ask again when another closes. +pub const MAX_DEVICES: usize = 4; + +/// The heap the open devices can ask for between them. The boot manifest +/// row's `heap_supply_bytes` must cover it; the service's tests hold the +/// two together. +pub const DEVICE_HEAP_BYTES: u64 = MAX_DEVICES as u64 * MAX_BUFFER_BACKING_BYTES; + +/// Seats there can ever be: the runtime names a peer's slot in eight bits. +pub const MAX_SEATS: usize = 1 << u8::BITS; + +/// Chains one device is served per look. A driver has at most thirty-two +/// commands in flight (half the largest queue), so sixteen is two looks for +/// the fullest queue, and no driver keeps the service from the next for +/// longer than sixteen commands take. +pub const SERVICE_BUDGET: NonZeroU32 = NonZeroU32::new(16).unwrap(); + +/// The page size a lent window is counted in on the wire. +pub const PAGE_BYTES: u64 = 0x1000; + +/// Why a driver's request was not done. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refusal { + /// No device model has that id. + NoDevice, + /// The device is not in a state for this: already open, not open, or + /// holding a queue in the memory it has. + Busy, + /// Every device the service hosts at once is taken. + Limit, + /// The model would not start. + Unavailable, + /// What was lent is not memory a device can address. + Memory, +} + +/// One open device. The transport is declared first so that it is dropped +/// first: every view into the lent memory goes before the owner that keeps +/// that memory mapped. +#[derive(Debug)] +struct Hosted { + transport: Transport, + /// What keeps the lent memory mapped, for as long as the device has it. + lent: Option, +} + +/// One driver face. +#[derive(Debug)] +struct Seat { + /// The runtime's name for this occupant of the slot: a later occupant + /// has another, so a request from one is never taken for the other's. + arrival: u16, + registers: Window, + device: Option>, +} + +/// Every seat, by the runtime's slot for the face. `O` is whatever keeps a +/// lent window mapped; dropping it gives the window back. +#[derive(Debug)] +pub struct Devices { + seats: Vec>>, + open: usize, +} + +impl Default for Devices { + fn default() -> Self { + Self::new() + } +} + +impl Devices { + pub const fn new() -> Self { + Self { + seats: Vec::new(), + open: 0, + } + } + + /// Devices open now. + pub const fn open(&self) -> usize { + self.open + } + + /// Whether `slot` is seated. + pub fn seated(&self, slot: usize) -> bool { + self.seats.get(slot).is_some_and(Option::is_some) + } + + /// The arrival seated in `slot`. + pub fn arrival(&self, slot: usize) -> Option { + self.seats.get(slot)?.as_ref().map(|seat| seat.arrival) + } + + /// Seat the face that arrived as `arrival` in `slot`, over its register + /// page. `false` when the slot is taken, is not one the runtime can + /// name, or the table could not grow to hold it. + /// + /// # Safety of the window + /// `registers` stays mapped until [`unseat`](Self::unseat) returns for + /// this slot: the caller owns the mapping and asks before it unmaps. + pub fn seat(&mut self, slot: usize, arrival: u16, registers: Window) -> bool { + if slot >= MAX_SEATS || registers.len() < regs::BYTES || self.seated(slot) { + return false; + } + if slot >= self.seats.len() { + let more = slot + 1 - self.seats.len(); + if self.seats.try_reserve(more).is_err() { + return false; + } + self.seats.resize_with(slot + 1, || None); + } + // A page that held another face's device says nothing of it. + let _ = registers.store_u32(regs::MAGIC, 0); + self.seats[slot] = Some(Seat { + arrival, + registers, + device: None, + }); + true + } + + /// The face in `slot` is going: close its device and forget its + /// register page. After this nothing here touches either window. + pub fn unseat(&mut self, slot: usize) { + if let Some(seat) = self.seats.get_mut(slot).and_then(Option::take) + && seat.device.is_some() + { + self.open -= 1; + } + } + + fn seat_mut(&mut self, slot: usize) -> Option<&mut Seat> { + self.seats.get_mut(slot).and_then(Option::as_mut) + } + + /// Open the device model `device` for the face in `slot`: its register + /// block is written before this returns. + pub fn open_device(&mut self, slot: usize, device: u32) -> Result<(), Refusal> { + let full = self.open >= MAX_DEVICES; + let seat = self.seat_mut(slot).ok_or(Refusal::Unavailable)?; + if device != DEVICE_ID { + return Err(Refusal::NoDevice); + } + if seat.device.is_some() { + return Err(Refusal::Busy); + } + if full { + return Err(Refusal::Limit); + } + let model = AccelDevice::new().map_err(|_| Refusal::Unavailable)?; + let transport = Transport::new(seat.registers, model).map_err(|_| Refusal::Unavailable)?; + seat.device = Some(Hosted { + transport, + lent: None, + }); + self.open += 1; + Ok(()) + } + + /// The driver in `slot` lends `memory`, which `owner` keeps mapped. + /// Answers the pages lent. On a refusal `owner` is dropped, which gives + /// the memory back; so is the owner of any memory this replaces. + pub fn lend(&mut self, slot: usize, memory: Window, owner: O) -> Result { + let hosted = self + .seat_mut(slot) + .and_then(|seat| seat.device.as_mut()) + .ok_or(Refusal::Busy)?; + let pages = memory.len() / PAGE_BYTES; + if pages == 0 || memory.len() % PAGE_BYTES != 0 { + return Err(Refusal::Memory); + } + let pages = u32::try_from(pages).map_err(|_| Refusal::Memory)?; + // The view the device held goes inside `lend`; its owner after. + hosted.transport.lend(memory).map_err(|_| Refusal::Busy)?; + hosted.lent = Some(owner); + Ok(pages) + } + + /// The driver in `slot` gives its device up. The memory it lent goes + /// back, and the register block stops saying a device is there. + pub fn close_device(&mut self, slot: usize) -> Result<(), Refusal> { + let seat = self.seat_mut(slot).ok_or(Refusal::Busy)?; + if seat.device.take().is_none() { + return Err(Refusal::Busy); + } + let _ = seat.registers.store_u32(regs::MAGIC, 0); + self.open -= 1; + Ok(()) + } + + /// A doorbell: look at every open device. `interrupt` is called with the + /// slot and register window of each device that raised one, after its + /// interrupt count moved. Answers whether any device has chains waiting + /// that its budget did not reach, in which case the caller comes back + /// without waiting for another doorbell. + /// + /// A doorbell names no device, so each open one is looked at: at most + /// [`MAX_DEVICES`], and a look at an idle device is three loads. + pub fn service(&mut self, mut interrupt: impl FnMut(usize, &Window)) -> bool { + let mut pending = false; + let mut left = self.open; + for (slot, seat) in self.seats.iter_mut().enumerate() { + if left == 0 { + break; + } + let Some(seat) = seat else { continue }; + let Some(hosted) = seat.device.as_mut() else { + continue; + }; + left -= 1; + let serviced = hosted.transport.service(SERVICE_BUDGET); + let dead = hosted.transport.is_dead(); + if serviced.interrupt { + interrupt(slot, &seat.registers); + } + pending |= serviced.pending; + if dead { + // A model that could not be reset is not brought back. The + // block keeps the status that says so; the driver opens + // again for a new device. + seat.device = None; + self.open -= 1; + } + } + pending + } +} + +#[cfg(test)] +mod tests; diff --git a/native/services/vdev/src/host/tests.rs b/native/services/vdev/src/host/tests.rs new file mode 100644 index 0000000..36c43a4 --- /dev/null +++ b/native/services/vdev/src/host/tests.rs @@ -0,0 +1,397 @@ +use super::*; +use core::cell::Cell; +use core::ptr::NonNull; +use kore_vdev::driver::{self, DriverChain, DriverRing, Layout, Slots}; +use kore_vdev::regs::{ + STATUS_ACKNOWLEDGE, STATUS_DEVICE_NEEDS_RESET, STATUS_DRIVER, STATUS_DRIVER_OK, + STATUS_FEATURES_OK, +}; +use std::boxed::Box; +use std::rc::Rc; +use std::vec; +use std::vec::Vec; +use virtio_accel_guest::{Completion, GuestClient, GuestConfig, RequestPoll}; +use virtio_accel_proto::{RequestHeader, ResponseHeader, WireDeviceInfo}; +use virtio_accel_transport::QueueSize; + +/// Host memory standing for a lease both sides map. +fn window(bytes: u64) -> Window { + let words: &'static mut [u64] = + Box::leak(vec![0u64; (bytes as usize).div_ceil(8)].into_boxed_slice()); + // SAFETY: a leaked, eight-byte aligned allocation of at least `bytes` + // bytes that nothing else holds a reference into. + unsafe { + Window::new( + NonNull::new(words.as_mut_ptr().cast()).unwrap(), + bytes as usize, + ) + } +} + +fn page() -> Window { + window(regs::BYTES) +} + +/// What keeps lent memory mapped, in a test: it counts its own end. +#[derive(Debug)] +struct Owner(Rc>); + +impl Drop for Owner { + fn drop(&mut self) { + self.0.set(self.0.get() + 1); + } +} + +/// A count of owners dropped, and a way to make one. +fn owners() -> (Rc>, impl Fn() -> Owner) { + let dropped = Rc::new(Cell::new(0)); + let make = { + let dropped = dropped.clone(); + move || Owner(dropped.clone()) + }; + (dropped, make) +} + +fn magic(registers: &Window) -> u32 { + registers.load_u32(regs::MAGIC).unwrap() +} + +const LIVE: u32 = STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK | STATUS_DRIVER_OK; + +#[test] +fn a_seat_is_one_faces_and_names_its_occupant() { + let mut devices = Devices::::new(); + let registers = page(); + registers.store_u32(regs::MAGIC, 0xdead_beef).unwrap(); + + assert!(!devices.seat(MAX_SEATS, 7, registers), "no such slot"); + assert!(!devices.seat(2, 7, window(regs::BYTES - 8)), "not a page"); + assert!(!devices.seated(2)); + assert_eq!(devices.arrival(2), None); + + assert!(devices.seat(2, 7, registers)); + assert_eq!( + magic(®isters), + 0, + "an earlier face's device is not shown" + ); + assert!(devices.seated(2)); + assert_eq!(devices.arrival(2), Some(7)); + assert!(!devices.seat(2, 8, page()), "the slot has its occupant"); + assert_eq!(devices.arrival(2), Some(7)); + assert!(!devices.seated(1) && !devices.seated(3)); + + // The table grows to whatever slot the runtime names. + assert!(devices.seat(MAX_SEATS - 1, 9, page())); + assert_eq!(devices.arrival(MAX_SEATS - 1), Some(9)); + + devices.unseat(2); + assert!(!devices.seated(2)); + devices.unseat(2); + devices.unseat(MAX_SEATS + 5); + assert!(devices.seat(2, 8, page()), "and the slot is free again"); + assert_eq!(devices.arrival(2), Some(8)); +} + +#[test] +fn a_driver_opens_the_device_it_names_once() { + let mut devices = Devices::::new(); + let registers = page(); + assert_eq!(devices.open_device(0, DEVICE_ID), Err(Refusal::Unavailable)); + assert!(devices.seat(0, 1, registers)); + + assert_eq!(devices.open_device(0, 0), Err(Refusal::NoDevice)); + assert_eq!( + devices.open_device(0, u32::from_le_bytes(*b"echo")), + Err(Refusal::NoDevice) + ); + assert_eq!(magic(®isters), 0); + assert_eq!(devices.open(), 0); + + assert_eq!(devices.open_device(0, DEVICE_ID), Ok(())); + assert_eq!(devices.open(), 1); + let identity = driver::identify(®isters).expect("the block is written"); + assert_eq!(identity.device_id, DEVICE_ID); + assert_eq!(identity.vendor_id, regs::VENDOR_KORE); + assert_eq!(devices.open_device(0, DEVICE_ID), Err(Refusal::Busy)); + assert_eq!(devices.open(), 1); + + assert_eq!(devices.close_device(0), Ok(())); + assert_eq!(magic(®isters), 0, "no device is shown once it is closed"); + assert_eq!(devices.open(), 0); + assert_eq!(devices.close_device(0), Err(Refusal::Busy)); + assert_eq!(devices.close_device(9), Err(Refusal::Busy)); + assert_eq!(devices.open_device(0, DEVICE_ID), Ok(()), "and opens again"); +} + +#[test] +fn only_so_many_devices_are_open_at_once() { + let mut devices = Devices::::new(); + for slot in 0..=MAX_DEVICES { + assert!(devices.seat(slot, slot as u16, page())); + } + for slot in 0..MAX_DEVICES { + assert_eq!(devices.open_device(slot, DEVICE_ID), Ok(())); + } + assert_eq!(devices.open(), MAX_DEVICES); + assert_eq!( + devices.open_device(MAX_DEVICES, DEVICE_ID), + Err(Refusal::Limit) + ); + // The seat is still the driver's: it asks again when another closes, + // or when another's face is retired with its device open. + assert_eq!(devices.close_device(1), Ok(())); + assert_eq!(devices.open_device(MAX_DEVICES, DEVICE_ID), Ok(())); + assert_eq!(devices.open_device(1, DEVICE_ID), Err(Refusal::Limit)); + devices.unseat(0); + assert_eq!(devices.open(), MAX_DEVICES - 1); + assert_eq!(devices.open_device(1, DEVICE_ID), Ok(())); + // The heap the row is founded with is sized by this. + assert_eq!(DEVICE_HEAP_BYTES, 8 << 20); +} + +#[test] +fn lent_memory_is_kept_exactly_as_long_as_the_device_has_it() { + let (dropped, owner) = owners(); + let mut devices = Devices::::new(); + let registers = page(); + assert!(devices.seat(0, 1, registers)); + let memory = window(4 * PAGE_BYTES); + + // Nothing to lend to, and the memory goes straight back. + assert_eq!(devices.lend(0, memory, owner()), Err(Refusal::Busy)); + assert_eq!(devices.lend(3, memory, owner()), Err(Refusal::Busy)); + assert_eq!(dropped.get(), 2); + + devices.open_device(0, DEVICE_ID).unwrap(); + for bad in [0, 8, PAGE_BYTES - 8, PAGE_BYTES + 8] { + assert_eq!( + devices.lend(0, window(bad), owner()), + Err(Refusal::Memory), + "{bad} bytes" + ); + } + assert_eq!(dropped.get(), 6); + + assert_eq!(devices.lend(0, memory, owner()), Ok(4)); + assert_eq!(dropped.get(), 6, "kept while the device has it"); + // Lent again in the first state: the old memory goes back. + assert_eq!(devices.lend(0, window(PAGE_BYTES), owner()), Ok(1)); + assert_eq!(dropped.get(), 7); + + // Once the driver has taken a step, the device may be using it. + registers + .store_u32(regs::DRIVER_STATUS, STATUS_ACKNOWLEDGE) + .unwrap(); + devices.service(|_, _| {}); + assert_eq!(devices.lend(0, memory, owner()), Err(Refusal::Busy)); + assert_eq!(dropped.get(), 8, "the refused memory, not the lent"); + + // Closing the device gives it back; so does retiring the face. + devices.close_device(0).unwrap(); + assert_eq!(dropped.get(), 9); + devices.open_device(0, DEVICE_ID).unwrap(); + assert_eq!(devices.lend(0, memory, owner()), Ok(4)); + devices.unseat(0); + assert_eq!(dropped.get(), 10); +} + +type Client = GuestClient>; + +/// One driver program: its two windows, and what a bring-up gave it. +struct Program { + slot: usize, + registers: Window, + client: Client, + slots: Slots, +} + +const REQUEST: u32 = core::mem::size_of::() as u32; +const INFO: u32 = + (core::mem::size_of::() + core::mem::size_of::()) as u32; + +/// Seat a driver in `slot`, open the accelerator, lend it memory and bring +/// it up, with the reference guest client over the queue. +fn program(devices: &mut Devices, slot: usize, owner: Owner, size: u16) -> Program { + let registers = page(); + assert!(devices.seat(slot, slot as u16 + 100, registers)); + devices.open_device(slot, DEVICE_ID).unwrap(); + let layout = Layout::new(QueueSize::new(size).unwrap(), 256, 256).unwrap(); + let memory = window(layout.bytes().next_multiple_of(PAGE_BYTES)); + devices.lend(slot, memory, owner).unwrap(); + + let identity = driver::identify(®isters).unwrap(); + let mut config = [0u8; 16]; + driver::read_config(®isters, &mut config).unwrap(); + let wire = virtio_accel_proto::read_exact(&config).unwrap(); + let (ring, slots) = driver::bring_up(®isters, memory, layout, || { + devices.service(|_, _| {}); + }) + .expect("the device comes up"); + let config = GuestConfig::new(wire, identity.features & !regs::F_VERSION_1, layout.slots()) + .expect("a configuration the reference driver accepts"); + Program { + slot, + registers, + client: GuestClient::new(ring, config).unwrap(), + slots, + } +} + +impl Program { + /// Publish one discovery command. + fn ask(&mut self) -> virtio_accel_guest::Pending { + let chain: DriverChain = self.slots.chain(REQUEST, INFO).expect("a free slot"); + self.client.get_device_info(chain).ok().expect("started") + } + + /// Take one answer, which must be there. + fn answer(&mut self, pending: virtio_accel_guest::Pending) { + let RequestPoll::Ready(Completion::Success { output, chain }) = self.client.poll(pending) + else { + panic!("the command was answered"); + }; + assert_eq!(&output.uuid[..13], b"kore-spmd-cpu"); + self.slots.recycle(chain); + } + + fn interrupts(&self) -> u32 { + self.registers.load_u32(regs::INTERRUPTS).unwrap() + } +} + +/// One doorbell serves every open device, and each driver is interrupted +/// for its own work and no other's. +#[test] +fn a_doorbell_reaches_every_open_device() { + let (_, owner) = owners(); + let mut devices = Devices::::new(); + let mut first = program(&mut devices, 1, owner(), 16); + let mut second = program(&mut devices, 4, owner(), 16); + // A face with no device open is not looked at. + assert!(devices.seat(2, 50, page())); + let (before_first, before_second) = (first.interrupts(), second.interrupts()); + + let asked_first = first.ask(); + let asked_second = [second.ask(), second.ask()]; + let mut raised = Vec::new(); + assert!(!devices.service(|slot, registers| { + raised.push((slot, registers.load_u32(regs::INTERRUPTS).unwrap())); + })); + assert_eq!( + raised, + [(1, before_first + 1), (4, before_second + 1)], + "each told once, its count already moved" + ); + first.answer(asked_first); + for asked in asked_second { + second.answer(asked); + } + + // Only the driver with work is interrupted. + let asked = first.ask(); + let mut raised = Vec::new(); + devices.service(|slot, _| raised.push(slot)); + assert_eq!(raised, [first.slot]); + first.answer(asked); + assert_eq!(second.interrupts(), before_second + 1); + + // And an idle doorbell interrupts nobody. + assert!(!devices.service(|slot, _| panic!("seat {slot} was interrupted"))); +} + +/// A queue fuller than the budget is served over as many looks as it +/// takes, and the table says to come back until it is done. +#[test] +fn a_full_queue_is_served_over_several_looks() { + let (_, owner) = owners(); + let mut devices = Devices::::new(); + let mut driver = program(&mut devices, 0, owner(), 64); + let commands = SERVICE_BUDGET.get() + 4; + let asked: Vec<_> = (0..commands).map(|_| driver.ask()).collect(); + + assert!(devices.service(|_, _| {}), "the budget ran out"); + let mut looks = 1; + while devices.service(|_, _| {}) { + looks += 1; + assert!(looks < 8, "the queue drains"); + } + for asked in asked { + driver.answer(asked); + } +} + +/// A device reset past its last namespace cannot be brought back: the +/// table drops it, the block says it needs a reset, and the driver opens a +/// new one. +#[test] +fn a_device_that_cannot_be_reset_is_dropped() { + let (dropped, owner) = owners(); + let mut devices = Devices::::new(); + let registers = page(); + assert!(devices.seat(0, 1, registers)); + devices.open_device(0, DEVICE_ID).unwrap(); + devices.lend(0, window(PAGE_BYTES), owner()).unwrap(); + + let status = |value| registers.store_u32(regs::DRIVER_STATUS, value).unwrap(); + for _ in 1..u32::from(u16::MAX) { + status(STATUS_ACKNOWLEDGE); + devices.service(|_, _| {}); + status(0); + devices.service(|_, _| {}); + } + assert_eq!(devices.open(), 1); + assert_eq!(registers.load_u32(regs::DEVICE_STATUS), Ok(0)); + + status(STATUS_ACKNOWLEDGE); + devices.service(|_, _| {}); + status(0); + let mut told = 0; + devices.service(|_, _| told += 1); + assert_eq!(told, 1, "the driver hears of it"); + assert_eq!( + registers.load_u32(regs::DEVICE_STATUS), + Ok(STATUS_DEVICE_NEEDS_RESET) + ); + assert_eq!(devices.open(), 0, "the dead device is gone"); + assert_eq!(dropped.get(), 1, "and its memory went back"); + + assert_eq!(devices.close_device(0), Err(Refusal::Busy)); + devices.open_device(0, DEVICE_ID).unwrap(); + assert_eq!(registers.load_u32(regs::DEVICE_STATUS), Ok(0)); + registers + .store_u64(regs::DRIVER_FEATURES, regs::F_VERSION_1) + .unwrap(); + registers.store_u32(regs::QUEUE_NUM, 16).unwrap(); + registers.store_u32(regs::QUEUE_READY, 1).unwrap(); + status(LIVE); + devices.service(|_, _| {}); + assert_ne!( + registers.load_u32(regs::DEVICE_STATUS).unwrap() & STATUS_DEVICE_NEEDS_RESET, + 0, + "a new device, which was lent nothing" + ); +} + +/// The row's founded heap covers every device the table will open, with +/// room over for what is not buffers. The two numbers live in two files; +/// this is where they are held together. +#[test] +fn the_founded_heap_covers_every_open_device() { + let row = include_str!("../../boot.toml"); + let value = row + .lines() + .find_map(|line| line.strip_prefix("heap_supply_bytes")) + .and_then(|rest| rest.split('=').nth(1)) + .map(|value| value.split('#').next().unwrap().trim().replace('_', "")) + .expect("the row sizes its heap"); + let heap = u64::from_str_radix(value.trim_start_matches("0x"), 16).expect("a hex size"); + // The engines' tables, the seat table and the runtime's own boxes. + const OVERHEAD: u64 = 2 << 20; + assert!( + heap >= DEVICE_HEAP_BYTES + OVERHEAD, + "{heap:#x} does not cover {MAX_DEVICES} devices of buffers and the overhead" + ); + // One lease, mapped at one stride of the founder's heap band. + assert!(heap <= 16 << 20); +} diff --git a/native/services/vdev/src/lib.rs b/native/services/vdev/src/lib.rs new file mode 100644 index 0000000..d57f05a --- /dev/null +++ b/native/services/vdev/src/lib.rs @@ -0,0 +1,53 @@ +//! # kore-svc-vdev: the virtual-device service +//! +//! A virtual device is a device with no hardware behind it (`kore-vdev`). +//! This service is where one lives: it hosts a device model for each driver +//! that asks for one, and what passes between a driver and its device is +//! memory the two share and two wakes. +//! +//! - **Registers** are a page of the driver's own face with the service, so +//! the driver has them mapped from its first instruction and the device +//! writes them when the driver opens it ([`VdevRequest::Open`]). +//! - **DMA** is memory the driver owns and lends ([`VdevRequest::Lend`]). +//! A device address is an offset into it; the service maps exactly what +//! was lent and can reach nothing else of the driver's. +//! - **The doorbell** is a notice on the face's kick. It carries nothing: +//! the service looks at every device it hosts when one arrives. +//! - **The interrupt** is the register block's interrupt count +//! (`kore_vdev::regs::INTERRUPTS`), which the service moves and then +//! wakes by the memory. A driver parks on the word. +//! +//! No kernel mechanism is specific to any of it. The face is one exec +//! mints for a program at launch and retires when the program ends, as it +//! does the power face ([`service`] documents the administration half). +//! +//! The device table ([`host`]) is plain data over `kore-vdev`'s transport +//! and is tested off metal; [`service`] is the shell that feeds it from the +//! runtime. + +#![no_std] +// The peer table grows with the programs that hold a face, from the +// service's own heap through the `Allocator` seam. +#![cfg_attr(feature = "host", feature(allocator_api))] + +extern crate alloc; + +#[cfg(test)] +extern crate std; + +pub mod client; +pub mod face; +#[cfg(feature = "host")] +pub mod host; +#[cfg(feature = "host")] +pub mod service; +pub mod wire; + +#[cfg(target_os = "none")] +pub use client::{ProcessVdev, ThreadVdev, attach_process_face, attach_thread_face}; +pub use client::{RingVdev, RingVdevAdmin, THREAD_SHARES, Waited, process_face}; +#[cfg(not(feature = "host"))] +pub use face::VdevService; +#[cfg(feature = "host")] +pub use service::{VdevEndowment, VdevService}; +pub use wire::{VdevRefusal, VdevRequest, VdevResponse}; diff --git a/native/services/vdev/src/service.rs b/native/services/vdev/src/service.rs new file mode 100644 index 0000000..06b9162 --- /dev/null +++ b/native/services/vdev/src/service.rs @@ -0,0 +1,389 @@ +//! The service: the device table, fed from the runtime. +//! +//! # Whose face is whose +//! +//! Pair zero is the founder's. Pair one is exec's, the *administration* +//! pair: exec builds a face for each program it launches with the +//! virtual-device profile, conveys its components here one request at a +//! time ([`VdevRequest::AdmitPeer`]), and retires it when the program ends +//! ([`VdevRequest::RetirePeer`]). Every other pair is such a face, a +//! driver's. +//! +//! A driver's face is two pages exec supplies: the request rings, then the +//! register block ([`REGISTERS_AT`]). The driver has both mapped before its +//! first instruction; the service is handed the second when the face is +//! admitted, and seats the driver over it. +//! +//! # What wakes the service +//! +//! A request on any pair, or a doorbell: a bare notice on a driver's kick. +//! Either way the service then looks at every open device +//! ([`Devices::service`]), so a doorbell needs no request and no reply. A +//! device whose budget ran out with chains still waiting is looked at again +//! at once, without waiting for another doorbell. + +use kore_abi::task::geom; +use kore_rt::service::{PeerClose, ReturnMode, Serve, StartFailure, SystemService}; +use kore_rt::svc::{ + PeerAdmissionRefusal, PeerBundleAssembler, PeerBundlePart, PeerId, PeerMapPlan, REP_CAP, + REQ_CAP, window_bytes, +}; +use kore_vdev::mem::Window; +use kore_vdev::regs; + +pub use crate::face::{PEER_BYTES, REGISTERS_AT}; +use crate::host::{Devices, MAX_SEATS}; +pub use crate::wire::{VdevRefusal, VdevRequest, VdevResponse}; + +/// The service is endowed with nothing: it holds no device and no +/// authority over the machine, only its own heap and the faces exec +/// conveys to it. +#[derive(Debug, kore_rt::Bundle)] +#[bundle(name = "vdev")] +pub struct VdevEndowment; + +/// The pair exec administers driver faces on. +const ADMIN_VIA: u16 = 1; + +/// What keeps a lent window mapped in this task: the capability the driver +/// delivered. Dropping it revokes the view, and the mapping with it. +#[cfg(target_os = "none")] +type Lent = kore_rt::cap::ReturnedCapability; +/// Off metal nothing is ever lent: there is no arena to map from. +#[cfg(not(target_os = "none"))] +type Lent = core::convert::Infallible; + +/// The virtual-device service. +#[derive(Debug)] +pub struct VdevService { + devices: Devices, + /// A device ran out of budget with chains waiting: look again at once. + pending: bool, + peer_assembler: PeerBundleAssembler, + peer_map: Option, + last_admission: Option>, +} + +impl VdevService { + /// The seat of the driver a request arrived from, if it is a driver's + /// face and still the occupant of its slot. + fn seat_of(&self, via: u16) -> Option { + let slot = PeerId::from_arrival(via)?.slot(); + (self.devices.arrival(slot) == Some(via)).then_some(slot) + } + + /// The register block inside a face's tail: `aux` is the face after its + /// rings. + fn registers_in(aux: &'static mut [u8]) -> Option { + let rings = window_bytes::(REQ_CAP, REP_CAP) as u64; + let skip = usize::try_from(REGISTERS_AT - rings).ok()?; + let block = aux.get_mut(skip..)?; + if (block.len() as u64) < regs::BYTES { + return None; + } + let base = core::ptr::NonNull::new(block.as_mut_ptr())?; + if base.as_ptr() as usize % REGISTERS_AT as usize != 0 { + return None; + } + // SAFETY: the host mapped the face read-write at a page boundary and + // keeps it mapped until it has called `peer_detaching`, where the + // seat and with it every copy of this view is dropped. The slice + // this was made from is not used again, so nothing in this task + // holds a reference into the page while the driver writes it. + Some(unsafe { Window::new(base, regs::BYTES as usize) }) + } + + fn lend(&mut self, seat: usize, lent: Option) -> VdevResponse { + let Some(owner) = lent else { + // Nothing arrived on the return leg to lend. + return VdevResponse::Refused(VdevRefusal::Memory); + }; + match map_lent(owner) { + Some((memory, owner)) => match self.devices.lend(seat, memory, owner) { + Ok(pages) => VdevResponse::Lent { pages }, + Err(refusal) => VdevResponse::Refused(refusal.into()), + }, + None => VdevResponse::Refused(VdevRefusal::Memory), + } + } +} + +/// Map what a driver lent. The owner comes back with the window so the two +/// are kept, and dropped, together. `None` drops the owner, which gives the +/// memory back. +#[cfg(target_os = "none")] +fn map_lent(owner: Lent) -> Option<(Window, Lent)> { + use kore_rt::cap::verbs; + let view = verbs::take_view(owner.slot()?)?; + // A sink mapping: should the driver die with its memory lent, the + // device's next look reads a sink page, which the transport takes for + // what it is (a status of zero is a reset, an index of zero is an empty + // queue), instead of faulting on a hole before the face is retired. + let mapped = verbs::map_view_sink(view)?; + let base = mapped.cast::(); + if base.as_ptr() as usize % core::mem::align_of::() != 0 { + return None; + } + // SAFETY: the kernel installed `[base, +len)` read-write in this task + // over the lent view and keeps it there while `owner` lives. The window + // is handed to the device table with `owner`, which drops every view + // before it drops the owner. The driver writes the same bytes at will, + // which is what `Window` is for: no reference is ever formed into them. + Some((unsafe { Window::new(base, mapped.len()) }, owner)) +} + +#[cfg(not(target_os = "none"))] +fn map_lent(owner: Lent) -> Option<(Window, Lent)> { + match owner {} +} + +/// Wake the driver parked on its device's interrupt count. +#[cfg(target_os = "none")] +fn interrupt(registers: &Window) { + // The word's address, to name it to the kernel: never dereferenced. + let word = registers + .base() + .as_ptr() + .wrapping_add(regs::INTERRUPTS as usize); + // The count moved before this. A driver not yet parked reads it again + // before it parks; one that is parked is woken here. A wake the kernel + // refuses is a lost kick, which the driver's deadline covers. + if let Some((subject, offset)) = kore_rt::cap::mapped::locate(word) { + let _ = kore_rt::cap::verbs::wake_word(subject, offset, u32::MAX); + } +} + +#[cfg(not(target_os = "none"))] +fn interrupt(_registers: &Window) {} + +impl SystemService for VdevService { + const NAME: &'static str = "vdev"; + type Endowment = VdevEndowment; + + /// Both return legs deliver here: the administration pair's carries a + /// face's components, a driver's carries the memory it lends. + const RETURN_MODE: ReturnMode = ReturnMode::RetainedAny; + + fn bind(_endowment: VdevEndowment) -> Self { + Self { + devices: Devices::new(), + pending: false, + peer_assembler: PeerBundleAssembler::new(0, true), + peer_map: None, + last_admission: None, + } + } + + /// One face for each program that drives a device, and many programs + /// may run at once: the peer table grows from the service's heap. + fn peer_arena(&self) -> Option<&'static (dyn core::alloc::Allocator + Sync)> { + static GLOBAL: alloc::alloc::Global = alloc::alloc::Global; + Some(&GLOBAL) + } + + fn start_on_metal(&mut self, rt: &kore_rt::entry::Rt) -> Result<(), StartFailure> { + // Where driver faces are mapped: one stride a slot, in the + // transient window, for every slot the runtime can name. + let window = rt + .geom() + .and_then(|geometry| geometry.window_of(geom::TRANSIENT)) + .filter(|window| { + window.base() != 0 + && window.base() & 0xfff == 0 + && window.len() >= PEER_BYTES * MAX_SEATS as u64 + }) + .ok_or(StartFailure::new(1))?; + self.peer_map = Some(PeerMapPlan { + base: window.base(), + stride: PEER_BYTES, + }); + Ok(()) + } + + /// Runs on every wake, a doorbell included: look at every open device. + fn poll_background(&mut self) { + self.pending = self.devices.service(|_, registers| interrupt(registers)); + } + + /// A device with chains its budget did not reach is due now. + fn next_deadline(&self) -> Option { + self.pending.then_some(0) + } + + fn drive_deadlines(&mut self, _now_ns: u64) -> u16 { + // The look itself is `poll_background`'s, on the same pass. + 0 + } + + fn drive_host( + host: kore_rt::svc::ServeHost<'_, '_, T, Self>, + ) -> impl core::future::Future { + async move { + host.run_deadline_driven().await; + } + } + + fn serve(&mut self, req: VdevRequest, sv: &mut Serve<'_, VdevResponse>) { + let resp = match req { + VdevRequest::AdmitPeer { part, announced } => { + // A `Ring` begins a face. One arriving while the assembler + // still owns components means the last face was abandoned + // by an asker that gave up: retire what it left, and refuse + // the new one while any of those owners is still closing. + // Administration only, and before anything else touches the + // assembler. + let abandoned = sv.via() == ADMIN_VIA + && matches!(part, PeerBundlePart::Ring) + && !self.peer_assembler.is_empty() + && !self.peer_assembler.retire_incomplete(); + if sv.via() != ADMIN_VIA || self.last_admission.is_some() || abandoned { + VdevResponse::Refused(VdevRefusal::Conflict) + } else if let (Some(map), Some(mut input)) = + (self.peer_map, sv.returned_input(announced)) + { + let accepted = input + .capability() + .and_then(|capability| self.peer_assembler.accepts(part, capability).ok()) + .is_some(); + if !accepted { + VdevResponse::Refused(VdevRefusal::Denied) + } else { + let capability = input.take().expect("validated peer component remains"); + let bundle = self + .peer_assembler + .push(part, capability) + .expect("synchronous peer validation remains stable"); + if let Some(bundle) = bundle { + sv.admit_peer(bundle, map); + } + VdevResponse::Accepted + } + } else { + VdevResponse::Refused(VdevRefusal::Unavailable) + } + } + VdevRequest::TakeAdmission => { + if sv.via() != ADMIN_VIA { + VdevResponse::Refused(VdevRefusal::Denied) + } else { + match self.last_admission.take() { + Some(Ok(peer)) => VdevResponse::PeerAdmitted { peer }, + Some(Err(_)) => VdevResponse::Refused(VdevRefusal::Unavailable), + // No verdict and components still held: the face + // never completed, so no verdict will ever come. + // `Pending` would be a wait with no end. + None if !self.peer_assembler.is_empty() => { + let _ = self.peer_assembler.retire_incomplete(); + VdevResponse::Refused(VdevRefusal::Conflict) + } + // Complete, and the host has not admitted it yet. + None => VdevResponse::Refused(VdevRefusal::Pending), + } + } + } + VdevRequest::RetirePeer { peer } => { + if sv.via() == ADMIN_VIA + && self.devices.arrival(peer.slot()) == Some(peer.arrival()) + { + sv.retire_peer(peer, PeerClose::Orderly); + VdevResponse::Accepted + } else { + VdevResponse::Refused(VdevRefusal::Denied) + } + } + VdevRequest::ConfirmRetired { peer } => { + if sv.via() == ADMIN_VIA + && self.devices.arrival(peer.slot()) != Some(peer.arrival()) + { + VdevResponse::Accepted + } else { + VdevResponse::Refused(VdevRefusal::Conflict) + } + } + VdevRequest::ClosePeer => { + if self.seat_of(sv.via()).is_some() { + sv.close_peer(PeerClose::Orderly); + VdevResponse::Accepted + } else { + VdevResponse::Refused(VdevRefusal::Denied) + } + } + VdevRequest::Open { device } => match self.seat_of(sv.via()) { + Some(seat) => match self.devices.open_device(seat, device) { + Ok(()) => VdevResponse::Accepted, + Err(refusal) => VdevResponse::Refused(refusal.into()), + }, + None => VdevResponse::Refused(VdevRefusal::Denied), + }, + VdevRequest::Lend { announced } => { + // Taken whatever the answer: a capability lent to a request + // that is refused is dropped, which gives it back. + let lent = sv + .returned_input(announced) + .and_then(|mut input| input.take()); + match self.seat_of(sv.via()) { + Some(seat) => self.lend(seat, lent.and_then(lent_owner)), + None => VdevResponse::Refused(VdevRefusal::Denied), + } + } + VdevRequest::Close => match self.seat_of(sv.via()) { + Some(seat) => match self.devices.close_device(seat) { + Ok(()) => VdevResponse::Accepted, + Err(refusal) => VdevResponse::Refused(refusal.into()), + }, + None => VdevResponse::Refused(VdevRefusal::Denied), + }, + }; + sv.reply(resp); + } + + /// The face's tail is its register block: seat the driver over it. + fn attach_peer_aux(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + match Self::registers_in(aux) { + Some(registers) => self.devices.seat(peer.slot(), peer.arrival(), registers), + None => false, + } + } + + /// The host is about to unmap the face: nothing here may touch its + /// register block, or the memory its driver lent, after this. + fn peer_detaching(&mut self, peer: PeerId) { + if self.devices.arrival(peer.slot()) == Some(peer.arrival()) { + self.devices.unseat(peer.slot()); + } + } + + fn peer_admitted(&mut self, result: Result) -> bool { + if let Ok(peer) = result + && self.devices.arrival(peer.slot()) != Some(peer.arrival()) + { + // Admitted with no seat: the face's tail was never attached. + self.last_admission = Some(Err(PeerAdmissionRefusal::PolicyCollision)); + return false; + } + self.last_admission = Some(result); + true + } + + fn peer_closed(&mut self, via: u16, _reason: PeerClose) -> bool { + // `peer_detaching` has run for a face that was seated; this covers + // one that closed before it was. + if let Some(seat) = self.seat_of(via) { + self.devices.unseat(seat); + } + true + } +} + +/// The capability a driver delivered, as what keeps its window mapped. +#[cfg(target_os = "none")] +fn lent_owner(capability: kore_rt::cap::ReturnedCapability) -> Option { + Some(capability) +} + +/// Off metal a delivered capability maps nothing: it is dropped, and the +/// request is answered as one that lent no memory. +#[cfg(not(target_os = "none"))] +fn lent_owner(_capability: kore_rt::cap::ReturnedCapability) -> Option { + None +} diff --git a/native/services/vdev/src/wire.rs b/native/services/vdev/src/wire.rs new file mode 100644 index 0000000..af51e2a --- /dev/null +++ b/native/services/vdev/src/wire.rs @@ -0,0 +1,465 @@ +//! The request vocabulary and its crossing shape. +//! +//! Two kinds of asker share one ring layout. The administration pair +//! (exec's) conveys the components of a driver's face and retires it when +//! the program ends; it never asks for a device. A driver's own face asks +//! for a device and lends it memory; it cannot administer. Which is which +//! is the pair a request arrived on, never a byte of the request. + +use kore_abi::session::SlotRef; +use kore_abi::svc::{VdevRepWire, VdevReqWire}; +use kore_rt::svc::{PeerBundlePart, PeerId, SvcProtocol}; + +use crate::VdevService; + +/// What a face may ask. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VdevRequest { + /// Administration: one component of a driver's face, delivered on the + /// administration pair's return leg. + AdmitPeer { + part: PeerBundlePart, + announced: SlotRef, + }, + /// Administration: the verdict on the face whose last component was + /// conveyed. + TakeAdmission, + /// Administration: the program holding this face has ended. + RetirePeer { peer: PeerId }, + /// Administration: has that face gone? + ConfirmRetired { peer: PeerId }, + /// A driver ends its own face. + ClosePeer, + /// A driver asks for the device model with this register-block id. + Open { device: u32 }, + /// A driver lends the memory its device addresses, delivered on the + /// face's return leg. + Lend { announced: SlotRef }, + /// A driver gives its device up and takes its memory back. + Close, +} + +/// What a face is told. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VdevResponse { + Accepted, + Refused(VdevRefusal), + PeerAdmitted { + peer: PeerId, + }, + /// The device addresses this many 4 KiB pages from here on. + Lent { + pages: u32, + }, +} + +/// Why a request was not done. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VdevRefusal { + /// The request is not this face's to make. + Denied, + /// The admission result is not published yet. + Pending, + /// An earlier admission must be settled first. + Conflict, + /// The service cannot do this now. + Unavailable, + /// No device model has that id. + NoDevice, + /// The device is not in a state for this. + Busy, + /// Every device the service hosts at once is taken. + Limit, + /// What was lent is not memory a device can address. + Memory, +} + +#[cfg(feature = "host")] +impl From for VdevRefusal { + fn from(refusal: crate::host::Refusal) -> Self { + use crate::host::Refusal; + match refusal { + Refusal::NoDevice => Self::NoDevice, + Refusal::Busy => Self::Busy, + Refusal::Limit => Self::Limit, + Refusal::Unavailable => Self::Unavailable, + Refusal::Memory => Self::Memory, + } + } +} + +const PART_RING: u8 = 0; +const PART_KICK: u8 = 1; +const PART_RETURN: u8 = 2; +const PART_GRANT: u8 = 0x80; + +/// Lower a request (total). +pub fn encode_req(req: &VdevRequest) -> VdevReqWire { + let mut wire = VdevReqWire { + op: 0, + reserved: [0; 7], + }; + wire.op = match *req { + VdevRequest::AdmitPeer { part, announced } => { + wire.reserved[0] = match part { + PeerBundlePart::Ring => PART_RING, + PeerBundlePart::Kick => PART_KICK, + PeerBundlePart::ReturnRecv => PART_RETURN, + PeerBundlePart::Grant(index) => PART_GRANT | index, + }; + wire.reserved[1..5].copy_from_slice(&announced.index().to_le_bytes()); + VdevReqWire::OP_ADMIT_PEER + } + VdevRequest::TakeAdmission => VdevReqWire::OP_TAKE_ADMISSION, + VdevRequest::RetirePeer { peer } => { + wire.reserved[..2].copy_from_slice(&peer.arrival().to_le_bytes()); + VdevReqWire::OP_RETIRE_PEER + } + VdevRequest::ConfirmRetired { peer } => { + wire.reserved[..2].copy_from_slice(&peer.arrival().to_le_bytes()); + VdevReqWire::OP_CONFIRM_RETIRED + } + VdevRequest::ClosePeer => VdevReqWire::OP_CLOSE_PEER, + VdevRequest::Open { device } => { + wire.reserved[..4].copy_from_slice(&device.to_le_bytes()); + VdevReqWire::OP_OPEN + } + VdevRequest::Lend { announced } => { + wire.reserved[..4].copy_from_slice(&announced.index().to_le_bytes()); + VdevReqWire::OP_LEND + } + VdevRequest::Close => VdevReqWire::OP_CLOSE, + }; + wire +} + +fn word(bytes: &[u8; 7], at: usize) -> u32 { + u32::from_le_bytes([bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]]) +} + +fn decode_peer(bytes: &[u8; 7]) -> Option { + if bytes[2..].iter().any(|byte| *byte != 0) { + return None; + } + PeerId::from_arrival(u16::from_le_bytes([bytes[0], bytes[1]])) +} + +/// Lift a wire request, fail-closed: only each opcode's declared payload is +/// accepted, and every byte beyond it must be zero. +pub fn decode_req(w: &VdevReqWire) -> Option { + let empty = w.reserved == [0; 7]; + Some(match w.op { + VdevReqWire::OP_ADMIT_PEER if w.reserved[5..] == [0; 2] => VdevRequest::AdmitPeer { + part: match w.reserved[0] { + PART_RING => PeerBundlePart::Ring, + PART_KICK => PeerBundlePart::Kick, + PART_RETURN => PeerBundlePart::ReturnRecv, + value if value & PART_GRANT != 0 => PeerBundlePart::Grant(value & !PART_GRANT), + _ => return None, + }, + announced: SlotRef::from_wire(word(&w.reserved, 1)), + }, + VdevReqWire::OP_TAKE_ADMISSION if empty => VdevRequest::TakeAdmission, + VdevReqWire::OP_RETIRE_PEER => VdevRequest::RetirePeer { + peer: decode_peer(&w.reserved)?, + }, + VdevReqWire::OP_CONFIRM_RETIRED => VdevRequest::ConfirmRetired { + peer: decode_peer(&w.reserved)?, + }, + VdevReqWire::OP_CLOSE_PEER if empty => VdevRequest::ClosePeer, + VdevReqWire::OP_OPEN if w.reserved[4..] == [0; 3] => VdevRequest::Open { + device: word(&w.reserved, 0), + }, + VdevReqWire::OP_LEND if w.reserved[4..] == [0; 3] => VdevRequest::Lend { + announced: SlotRef::from_wire(word(&w.reserved, 0)), + }, + VdevReqWire::OP_CLOSE if empty => VdevRequest::Close, + _ => return None, + }) +} + +/// Lower a response (total). The service grants nothing, so `grants` is +/// written zero whatever was asked. +pub fn encode_rep(rep: &VdevResponse, _grants: u8) -> VdevRepWire { + let (tag, refusal, reserved) = match *rep { + VdevResponse::Accepted => (VdevRepWire::TAG_ACCEPTED, 0, [0; 5]), + VdevResponse::PeerAdmitted { peer } => { + let arrival = peer.arrival().to_le_bytes(); + ( + VdevRepWire::TAG_PEER_ADMITTED, + 0, + [arrival[0], arrival[1], 0, 0, 0], + ) + } + VdevResponse::Lent { pages } => { + let pages = pages.to_le_bytes(); + ( + VdevRepWire::TAG_LENT, + 0, + [pages[0], pages[1], pages[2], pages[3], 0], + ) + } + VdevResponse::Refused(refusal) => ( + VdevRepWire::TAG_REFUSED, + match refusal { + VdevRefusal::Denied => VdevRepWire::REFUSE_DENIED, + VdevRefusal::Pending => VdevRepWire::REFUSE_PENDING, + VdevRefusal::Conflict => VdevRepWire::REFUSE_CONFLICT, + VdevRefusal::Unavailable => VdevRepWire::REFUSE_UNAVAILABLE, + VdevRefusal::NoDevice => VdevRepWire::REFUSE_NO_DEVICE, + VdevRefusal::Busy => VdevRepWire::REFUSE_BUSY, + VdevRefusal::Limit => VdevRepWire::REFUSE_LIMIT, + VdevRefusal::Memory => VdevRepWire::REFUSE_MEMORY, + }, + [0; 5], + ), + }; + VdevRepWire { + tag, + refusal, + grants: 0, + reserved, + } +} + +/// Lift a wire reply, fail-closed: a non-zero `grants`, a refusal byte on +/// a success tag and a set reserved byte each refuse. +pub fn decode_rep(w: &VdevRepWire) -> Option<(VdevResponse, u8)> { + if w.grants != 0 { + return None; + } + let response = match (w.tag, w.refusal) { + (VdevRepWire::TAG_ACCEPTED, 0) if w.reserved == [0; 5] => VdevResponse::Accepted, + (VdevRepWire::TAG_PEER_ADMITTED, 0) if w.reserved[2..] == [0; 3] => { + VdevResponse::PeerAdmitted { + peer: PeerId::from_arrival(u16::from_le_bytes([w.reserved[0], w.reserved[1]]))?, + } + } + (VdevRepWire::TAG_LENT, 0) if w.reserved[4] == 0 => VdevResponse::Lent { + pages: u32::from_le_bytes([w.reserved[0], w.reserved[1], w.reserved[2], w.reserved[3]]), + }, + (VdevRepWire::TAG_REFUSED, refusal) if w.reserved == [0; 5] => { + VdevResponse::Refused(match refusal { + VdevRepWire::REFUSE_DENIED => VdevRefusal::Denied, + VdevRepWire::REFUSE_PENDING => VdevRefusal::Pending, + VdevRepWire::REFUSE_CONFLICT => VdevRefusal::Conflict, + VdevRepWire::REFUSE_UNAVAILABLE => VdevRefusal::Unavailable, + VdevRepWire::REFUSE_NO_DEVICE => VdevRefusal::NoDevice, + VdevRepWire::REFUSE_BUSY => VdevRefusal::Busy, + VdevRepWire::REFUSE_LIMIT => VdevRefusal::Limit, + VdevRepWire::REFUSE_MEMORY => VdevRefusal::Memory, + _ => return None, + }) + } + _ => return None, + }; + Some((response, 0)) +} + +impl SvcProtocol for VdevService { + type Request = VdevRequest; + type Response = VdevResponse; + type ReqWire = VdevReqWire; + type RepWire = VdevRepWire; + + /// The service mints no authority: structural, not a convention. + const REPLY_GRANTS: usize = 0; + /// Pair zero is the founder's control face; pair one is exec's, which + /// administers the driver faces. + const SERVE_PAIRS: usize = 2; + + fn reply_matches(request: &VdevReqWire, response: &VdevResponse, grants: u8) -> bool { + grants == 0 + && match (decode_req(request), response) { + (Some(VdevRequest::TakeAdmission), VdevResponse::PeerAdmitted { .. }) => true, + (Some(VdevRequest::Lend { .. }), VdevResponse::Lent { .. }) => true, + (Some(VdevRequest::Lend { .. }), VdevResponse::Accepted) => false, + (Some(_), VdevResponse::Accepted | VdevResponse::Refused(_)) => true, + _ => false, + } + } + + fn encode_req(req: &VdevRequest) -> VdevReqWire { + encode_req(req) + } + fn decode_req(wire: &VdevReqWire) -> Option { + decode_req(wire) + } + fn encode_rep(rep: &VdevResponse, grants: u8) -> VdevRepWire { + encode_rep(rep, grants) + } + fn decode_rep(wire: &VdevRepWire) -> Option<(VdevResponse, u8)> { + decode_rep(wire) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn peer() -> PeerId { + PeerId::from_arrival(PeerId::SLOT_CAPACITY as u16).expect("the first recyclable pair") + } + + const REQUESTS: [fn() -> VdevRequest; 11] = [ + || VdevRequest::AdmitPeer { + part: PeerBundlePart::Ring, + announced: SlotRef::from_wire(27), + }, + || VdevRequest::AdmitPeer { + part: PeerBundlePart::Kick, + announced: SlotRef::from_wire(28), + }, + || VdevRequest::AdmitPeer { + part: PeerBundlePart::ReturnRecv, + announced: SlotRef::from_wire(u32::MAX), + }, + || VdevRequest::AdmitPeer { + part: PeerBundlePart::Grant(3), + announced: SlotRef::from_wire(0), + }, + || VdevRequest::TakeAdmission, + || VdevRequest::RetirePeer { peer: peer() }, + || VdevRequest::ConfirmRetired { peer: peer() }, + || VdevRequest::ClosePeer, + || VdevRequest::Open { + device: 0x6c63_6361, + }, + || VdevRequest::Lend { + announced: SlotRef::from_wire(0x0102_0304), + }, + || VdevRequest::Close, + ]; + + const RESPONSES: [fn() -> VdevResponse; 12] = [ + || VdevResponse::Accepted, + || VdevResponse::PeerAdmitted { peer: peer() }, + || VdevResponse::Lent { pages: 1 }, + || VdevResponse::Lent { pages: u32::MAX }, + || VdevResponse::Refused(VdevRefusal::Denied), + || VdevResponse::Refused(VdevRefusal::Pending), + || VdevResponse::Refused(VdevRefusal::Conflict), + || VdevResponse::Refused(VdevRefusal::Unavailable), + || VdevResponse::Refused(VdevRefusal::NoDevice), + || VdevResponse::Refused(VdevRefusal::Busy), + || VdevResponse::Refused(VdevRefusal::Limit), + || VdevResponse::Refused(VdevRefusal::Memory), + ]; + + #[test] + fn every_request_and_response_round_trips() { + for request in REQUESTS { + let request = request(); + assert_eq!(decode_req(&encode_req(&request)), Some(request)); + } + for response in RESPONSES { + let response = response(); + assert_eq!(decode_rep(&encode_rep(&response, 0)), Some((response, 0))); + // Whatever was asked, nothing is granted. + assert_eq!(encode_rep(&response, 3).grants, 0); + } + } + + /// A zeroed record, an unknown opcode and a byte set beyond an opcode's + /// payload each refuse: nothing is guessed. + #[test] + fn a_request_decodes_only_from_exactly_its_own_bytes() { + let zero = VdevReqWire { + op: 0, + reserved: [0; 7], + }; + assert_eq!(decode_req(&zero), None, "a zeroed record asks nothing"); + assert_eq!(decode_req(&VdevReqWire { op: 9, ..zero }), None); + assert_eq!(decode_req(&VdevReqWire { op: 255, ..zero }), None); + + for request in REQUESTS { + let request = request(); + let wire = encode_req(&request); + // How many leading bytes of the payload this opcode owns. + let owned = match request { + VdevRequest::AdmitPeer { .. } => 5, + VdevRequest::RetirePeer { .. } | VdevRequest::ConfirmRetired { .. } => 2, + VdevRequest::Open { .. } | VdevRequest::Lend { .. } => 4, + VdevRequest::TakeAdmission | VdevRequest::ClosePeer | VdevRequest::Close => 0, + }; + for at in owned..7 { + let mut stray = wire; + stray.reserved[at] = 1; + assert_eq!(decode_req(&stray), None, "{request:?}, byte {at}"); + } + } + // An unknown part of a face is not a part. + let mut part = encode_req(&REQUESTS[0]()); + part.reserved[0] = 3; + assert_eq!(decode_req(&part), None); + // A fixed pair is not a face exec can retire. + let mut fixed = encode_req(&VdevRequest::RetirePeer { peer: peer() }); + fixed.reserved[..2].copy_from_slice(&1u16.to_le_bytes()); + assert_eq!(decode_req(&fixed), None); + } + + #[test] + fn a_reply_decodes_only_from_exactly_its_own_bytes() { + let zero = VdevRepWire { + tag: 0, + refusal: 0, + grants: 0, + reserved: [0; 5], + }; + assert_eq!(decode_rep(&zero), None); + for response in RESPONSES { + let response = response(); + let wire = encode_rep(&response, 0); + assert_eq!(decode_rep(&VdevRepWire { grants: 1, ..wire }), None); + if !matches!(response, VdevResponse::Refused(_)) { + let refused = VdevRepWire { refusal: 1, ..wire }; + assert_eq!(decode_rep(&refused), None, "{response:?}"); + } + let owned = match response { + VdevResponse::PeerAdmitted { .. } => 2, + VdevResponse::Lent { .. } => 4, + VdevResponse::Accepted | VdevResponse::Refused(_) => 0, + }; + for at in owned..5 { + let mut stray = wire; + stray.reserved[at] = 1; + assert_eq!(decode_rep(&stray), None, "{response:?}, byte {at}"); + } + } + let unknown = VdevRepWire { + tag: VdevRepWire::TAG_REFUSED, + refusal: 9, + ..zero + }; + assert_eq!(decode_rep(&unknown), None); + } + + /// A reply is believed only if it answers what was asked. + #[test] + fn a_reply_must_answer_its_request() { + let matches = |request: VdevRequest, response: VdevResponse| { + ::reply_matches(&encode_req(&request), &response, 0) + }; + let lend = VdevRequest::Lend { + announced: SlotRef::from_wire(1), + }; + assert!(matches(lend, VdevResponse::Lent { pages: 4 })); + assert!(!matches(lend, VdevResponse::Accepted)); + assert!(matches(lend, VdevResponse::Refused(VdevRefusal::Busy))); + assert!(!matches(lend, VdevResponse::PeerAdmitted { peer: peer() })); + assert!(matches( + VdevRequest::TakeAdmission, + VdevResponse::PeerAdmitted { peer: peer() } + )); + assert!(!matches( + VdevRequest::Close, + VdevResponse::Lent { pages: 4 } + )); + assert!(matches(VdevRequest::Close, VdevResponse::Accepted)); + assert!(!::reply_matches( + &encode_req(&VdevRequest::Close), + &VdevResponse::Accepted, + 1 + )); + } +} diff --git a/src/lib.rs b/src/lib.rs index 3396e29..17a396e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,6 +29,17 @@ pub use kore_rt::{thread, thread_controller}; #[cfg(feature = "driver")] pub use kore_driver as driver; +/// A driver's side of a virtual device, a device with no hardware behind it +/// that Kore's virtual-device service hosts for a driver to be written +/// against: the register block and the views of shared memory +/// (`kore-vdev`), and the service's client (`kore-svc-vdev`, built without +/// the service or any device model). +#[cfg(feature = "vdev")] +pub mod vdev { + pub use kore_svc_vdev::{client, face, wire}; + pub use kore_vdev::{driver, mem, regs}; +} + /// Native clocks and deadlines; wall time never coerces into monotonic time. pub mod time { pub use core::time::Duration; diff --git a/tools/check-bindings.py b/tools/check-bindings.py index b389550..2c6c9a6 100644 --- a/tools/check-bindings.py +++ b/tools/check-bindings.py @@ -21,7 +21,8 @@ for arch, c_target, zig_target in TARGETS: subprocess.run([os.environ.get('CLANG', 'clang'), '-target', c_target, '-ffreestanding', '-std=c11', '-Werror', '-fsyntax-only', '-I', 'extern/c/include', - 'crates/ffi/tests/consumer.c', 'crates/ffi/tests/alloc.c'], cwd=ROOT, check=True) + 'crates/ffi/tests/consumer.c', 'crates/ffi/tests/alloc.c', + 'crates/c/tests/program.c'], cwd=ROOT, check=True) translated = destination / f'{arch}-c.zig' with translated.open('w') as output: subprocess.run([zig, 'translate-c', '-target', zig_target, 'extern/c/include/kore.h'], diff --git a/tools/check-program.py b/tools/check-program.py new file mode 100755 index 0000000..0e12146 --- /dev/null +++ b/tools/check-program.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Link libkore.so for every Kore ISA and check the image a C program binds. + +The image is a distribution's one-line cdylib over kore-c (Kore's is +user/runtime/libkore), linked with the contract kore-c documents; this builds +that crate in a temporary workspace, as check-entry.py builds its consumers. +The library is what an independent C program on Kore links with -lkore, so the +checks are the ones its loader and its consumers rely on: + +- a shared object with no entry point: Kore's image gate classifies a module by + its entry, and a library must never register as a program; +- the soname consumers record, nothing it needs, nothing left undefined; +- static initial-exec TLS only (Kore's startup linker resolves no dynamic TLS); +- exactly the declared ABI exported: every function kore.h and kore/program.h + declare, the C names kore/stdlib.h and kore/pthread.h alias, the C + library's two lifecycle entry points, and the C library names Kore's + hosted headers declare and libkore.so implements; no other symbol. +""" +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import tomllib + +ROOT = Path(__file__).resolve().parents[1] +ARCHES = ('x86_64', 'aarch64', 'riscv64gc') +HEADERS = ('extern/c/include/kore.h', 'extern/c/include/kore/program.h') +# kore-ffi's `c-stdlib` names: allocation, the pthread lifecycle, and the +# compiler ABI's thread-exit destructor hooks. +C_STDLIB = {'malloc', 'calloc', 'realloc', 'free', 'pthread_create', 'pthread_join', + 'pthread_detach', 'pthread_cancel', 'pthread_key_create', 'pthread_key_delete', + 'pthread_setspecific', 'pthread_getspecific', 'pthread_setcancelstate', + 'pthread_setcanceltype', 'pthread_testcancel', 'pthread_exit', + '__cxa_thread_atexit', '__cxa_thread_atexit_impl'} +# The C library's start and exit call these when a program links libkore. +LIFECYCLE = {'__kore_program_start', '__kore_program_exit', '__kore_program_arguments'} +# The rest of the C library a program's headers declare (Kore's TCC +# `include/`: , , , ), which libkore.so +# implements: kore-c-shim's printf family, streams, parsers, string routines +# and per-thread errno, and kore-c's own abort; and , , +# and 's open, close, read, write, poll, mmap and munmap +# over the capabilities the program holds. The program's libc.a holds the +# rest: exit, atexit and the routines a compiler assumes. +C_LIBRARY = {'printf', 'fprintf', 'sprintf', 'snprintf', 'vprintf', 'vfprintf', 'vsprintf', + 'vsnprintf', 'fputc', 'putc', 'putchar', 'fputs', 'puts', 'fwrite', 'fflush', + 'fclose', 'stdout', 'stderr', + 'strtol', 'strtoll', 'strtoul', 'strtoull', 'atoi', 'atol', 'strtod', 'strtof', + 'qsort', 'strerror', 'abort', + 'memchr', 'strchr', 'strrchr', 'strcmp', 'strncmp', 'strcpy', 'strncpy', 'strncat', + 'strstr', + '__errno_location', + 'open', 'close', 'read', 'write', 'poll', 'mmap', 'munmap'} +DYNAMIC_TLS = re.compile(r'TLSDESC|DTPMOD|DTPOFF|TLSGD|TLSLD|TLS_DTP') + + +def readelf(*args): + return subprocess.check_output([os.environ.get('LLVM_READELF', 'llvm-readelf'), '-W', *args], + text=True) + + +def declared(): + names = set() + for header in HEADERS: + names |= set(re.findall(r'\b(kore_[a-z][a-z0-9_]*)\s*\(', (ROOT / header).read_text())) + if not names: + raise SystemExit('no foreign functions found in the generated headers') + return names + + +# The shared object as a distribution builds it: a cdylib named `kore` that +# names kore-c, linked with the soname and no undefined symbol. +WRAPPER_MANIFEST = ('[package]\nname = "libkore-so"\nversion = "0.0.0"\nedition = "2024"\n' + '[lib]\nname = "kore"\ncrate-type = ["cdylib"]\n[workspace]\n' + '[dependencies]\nkore-c = {{ path = {path} }}\n') +WRAPPER_SOURCE = '#![no_std]\n// The runtime and its exports come from kore-c.\nuse kore_c as _;\n' +LINK_CONTRACT = ['-C', 'link-arg=--soname=libkore.so', '-C', 'link-arg=--no-undefined'] + + +def build(arch): + channel = tomllib.loads((ROOT / 'rust-toolchain.toml').read_text())['toolchain']['channel'] + target_dir = ROOT / 'target/program-consumer' + env = dict(os.environ, RUSTUP_TOOLCHAIN=channel, CARGO_TARGET_DIR=str(target_dir), + CARGO_INCREMENTAL='0') + with tempfile.TemporaryDirectory(prefix='libkore-so-') as temporary: + work = Path(temporary) + (work / 'src').mkdir() + (work / 'Cargo.toml').write_text( + WRAPPER_MANIFEST.format(path=json.dumps(str(ROOT / 'crates/c')))) + (work / 'src/lib.rs').write_text(WRAPPER_SOURCE) + # Reuse the reviewed dependency versions while Cargo adds this fixture. + (work / 'Cargo.lock').write_bytes((ROOT / 'Cargo.lock').read_bytes()) + subprocess.run([ + 'cargo', 'rustc', '--release', '-Zjson-target-spec', + '--target', str(ROOT / f'targets/{arch}-unknown-kore-user.json'), + '-Zbuild-std=core,alloc,compiler_builtins', + '-Zbuild-std-features=compiler-builtins-mem', '--', *LINK_CONTRACT, + ], cwd=work, env=env, check=True) + return target_dir / f'{arch}-unknown-kore-user/release/libkore.so' + + +def check(arch, expected): + image = build(arch) + header = readelf('--file-header', str(image)) + if not re.search(r'Type:\s+DYN', header): + raise SystemExit(f'{arch}: libkore.so is not a shared object') + if not re.search(r'Entry point address:\s+0x0\b', header): + raise SystemExit(f'{arch}: libkore.so has an entry point; it would register as a program') + dynamic = readelf('--dynamic', str(image)) + if 'Library soname: [libkore.so]' not in dynamic: + raise SystemExit(f'{arch}: libkore.so does not carry its soname') + if needed := re.findall(r'\(NEEDED\)\s+Shared library: \[([^\]]+)\]', dynamic): + raise SystemExit(f'{arch}: libkore.so needs other libraries: {needed}') + exported, undefined = set(), set() + for line in readelf('--dyn-syms', str(image)).splitlines(): + fields = line.split() + if len(fields) < 8 or not fields[0].rstrip(':').isdigit(): + continue + kind, binding, section, name = fields[3], fields[4], fields[6], fields[7] + if section == 'UND': + if name: + undefined.add(name) + elif binding in ('GLOBAL', 'WEAK') and kind in ('FUNC', 'OBJECT', 'TLS'): + exported.add(name.split('@')[0]) + if undefined: + raise SystemExit(f'{arch}: undefined symbols: {sorted(undefined)}') + if missing := expected - exported: + raise SystemExit(f'{arch}: declared but not exported: {sorted(missing)}') + if extra := exported - expected: + raise SystemExit(f'{arch}: exported outside the declared ABI: {sorted(extra)}') + relocations = readelf('--relocations', str(image)) + if found := sorted(set(DYNAMIC_TLS.findall(relocations))): + raise SystemExit(f'{arch}: dynamic TLS relocations {found}; static initial-exec TLS required') + print(f'libkore.so: {arch}: {len(exported)} exported symbols, the declared ABI exactly', flush=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument('--arch', choices=ARCHES) + args = parser.parse_args() + expected = declared() | C_STDLIB | LIFECYCLE | C_LIBRARY + for arch in [args.arch] if args.arch else ARCHES: + check(arch, expected) diff --git a/tools/generate-bindings.py b/tools/generate-bindings.py index 2ba0782..04ee299 100644 --- a/tools/generate-bindings.py +++ b/tools/generate-bindings.py @@ -19,6 +19,14 @@ def generate(check=False): subprocess.run([generator, '--config', 'crates/ffi/cbindgen.toml', '--crate', 'kore-ffi', '--output', str(header)], cwd=ROOT, check=True) source = header.read_text() + # What libkore.so adds for an independent C program; it includes kore.h + # and declares none of its types again. + program_header = Path(directory) / 'program.h' + subprocess.run([generator, '--config', 'crates/c/cbindgen.toml', '--crate', 'kore-c', + '--output', str(program_header)], cwd=ROOT, check=True) + program = program_header.read_text() + if 'kore_fs_run' not in program: + raise SystemExit('missing program export: kore_fs_run') # cbindgen is the declaration authority. This step only groups exported # names; it never reproduces signatures or guesses C/Rust type layouts. functions = sorted(set(re.findall(r'\b(kore_[a-z][a-z0-9_]*)\s*\(', source))) @@ -72,7 +80,8 @@ def generate(check=False): pthread += '#define pthread_cleanup_pop(execute) kore_thread_cleanup_pop(&_kore_cleanup, (execute)); } while (0)\n' pthread += '#endif\n' for relative, data in [('extern/c/include/kore.h', source), ('extern/zig/src/root.zig', zig), - ('extern/c/include/kore/stdlib.h', stdlib), ('extern/c/include/kore/pthread.h', pthread)]: + ('extern/c/include/kore/stdlib.h', stdlib), ('extern/c/include/kore/pthread.h', pthread), + ('extern/c/include/kore/program.h', program)]: path = ROOT / relative if check: if not path.exists() or path.read_text() != data: