From f995d86834b5a8a8d7437c5a52f81ccb72301586 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 19:43:18 +0000 Subject: [PATCH 01/18] native: re-export at Kore eab77f8d, where a host can end its program ## Why `libkore.so` (the next commit) hosts an independent C program on thread 0 and ends the process with the program's own status, from whichever of its threads asks. kore-rt offered that end only as `startup::finish`, with a native `ExitCode`. Its managed threads could not wake thread 0, and a panic's report could not be told apart from its thread's end. Kore `eab77f8d` adds the three pieces to kore-rt, so they arrive here through the exporter rather than as edits to the snapshot. ## What - `native/` is re-exported byte for byte at `eab77f8d904352eabe65fedb84a455d6ccee5b7d` with `tools/sync-native.py`. That commit is on MicroPerceptron/kore#1494's branch, which also brings TinyCC to the Kore shell. Ten of its 23 commits since `7074bda2` reach the closure, which keeps its 26 packages and 382 files: - `kore-rt`: - `startup::barrier(finalize)`: the process barrier without the exit. - Managed threads inherit their founder's wake send. - `crash::report`: a panic's report without the thread's end. - A per-thread C `errno` word and guarded worker stacks sized in bytes. - The verbs and bundle call a shell's private install takes. - `kore-abi`: - The install request and its replies. - The claimant band sized by driver rows (`MAX_CLAIMANT_FACES`), and the device service that uses it. - The services' `boot.toml` digests, resealed three times. - No exported manifest changed, so `native/Cargo.lock` did not. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- native/crates/abi/src/device.rs | 20 +++ native/crates/abi/src/pkg.rs | 37 ++-- native/crates/abi/src/session.rs | 22 +++ native/crates/abi/src/svc.rs | 141 +++++++++++++++ native/crates/abi/src/task.rs | 67 +++++-- native/crates/rt/src/cap/mod.rs | 24 +-- native/crates/rt/src/cap/verbs.rs | 206 +++++++++++++++++++++- native/crates/rt/src/crash.rs | 17 +- native/crates/rt/src/entry.rs | 21 ++- native/crates/rt/src/session/builder.rs | 21 +++ native/crates/rt/src/session/tests.rs | 30 ++++ native/crates/rt/src/startup.rs | 33 +++- native/crates/rt/src/svc/client.rs | 179 +++++++++++++++++++ native/crates/rt/src/svc/mod.rs | 3 +- native/crates/rt/src/thread.rs | 128 ++++++++++++-- native/crates/rt/src/thread_controller.rs | 46 ++++- native/provenance.json | 46 ++--- native/services/block/boot.toml | 4 +- native/services/dagfs/boot.toml | 4 +- native/services/device/boot.toml | 4 +- native/services/device/src/service/mod.rs | 33 +++- native/services/mm/boot.toml | 4 +- native/services/swap/boot.toml | 4 +- 23 files changed, 974 insertions(+), 120 deletions(-) diff --git a/native/crates/abi/src/device.rs b/native/crates/abi/src/device.rs index 60b8e89..d8483f1 100644 --- a/native/crates/abi/src/device.rs +++ b/native/crates/abi/src/device.rs @@ -89,6 +89,26 @@ pub enum DeviceSpecies { /// its banks. pub const MAX_ROOT_COMPLEXES: usize = 8; +/// Claimant faces one device broker can mint: one per driver row, the +/// founder minting face `n` for driver row `n` (`MintFace`, +/// `res/DEVICE_SERVICE.md` §10). Five today: input, block, accel, net and +/// Xe. +/// +/// The bound is shared, not chosen. The kernel founder seats driver row +/// `n`'s one executor bell at a fixed slot of its common pin island and +/// asserts the island holds this many. Exec's manifest gate refuses a +/// driver row past it. The broker sizes its face table, its claimant band +/// ([`CLAIMANT_FACE_SUPPLY_BYTES`](crate::task::CLAIMANT_FACE_SUPPLY_BYTES)) +/// and its landing band (two base-arena slots per face, below the +/// startup-link band, which it asserts) from it. Raising it costs one +/// ring stride per broker, two broker slots and one founder bell slot per +/// face; the founder's pins are the first of those to run out. +/// +/// Deliberately not the image service's face count: that one grows with +/// the executable catalogue, and a broker band sized by it would grow with +/// every program the system can run. +pub const MAX_CLAIMANT_FACES: usize = 5; + impl DeviceSpecies { /// Decode the wire byte; `None` for an unknown species (fail-closed — /// a broker that cannot name the species must not derive). diff --git a/native/crates/abi/src/pkg.rs b/native/crates/abi/src/pkg.rs index bac3b8f..ac109a8 100644 --- a/native/crates/abi/src/pkg.rs +++ b/native/crates/abi/src/pkg.rs @@ -27,13 +27,27 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; /// Layout version. A zeroed record reads `abi == 0` and is *not a /// manifest*; a reader refuses rather than guessing. -pub const PACKAGE_ABI: u16 = 1; -/// Members one manifest may declare. The bootstrap archive today carries -/// 33 executables and 3 libraries (`BOOTSTRAP_EXECUTABLES` / -/// `BOOTSTRAP_LIBRARIES` in exec), and the system package `kore` must list -/// every one of them; 48 leaves a third free before the record grows a -/// second page. `size_of::()` is pinned below. -pub const MAX_PACKAGE_MEMBERS: usize = 48; +/// +/// 2: [`MAX_PACKAGE_MEMBERS`] grew from 48 to 96, so the record grew with +/// it. Version 1 manifests were only ever written — exec rewrites the +/// system package's in place at every boot, and no installer reads one +/// yet — so nothing has to read both shapes; the bump is what lets a future +/// reader tell them apart rather than misread one as the other. +pub const PACKAGE_ABI: u16 = 2; +/// Members one manifest may declare: every bootstrap executable and library +/// (`BOOTSTRAP_EXECUTABLES` / `BOOTSTRAP_LIBRARIES` in exec), which the +/// system package `kore` must list in full. +/// +/// **Where the number comes from.** The record is read whole by one +/// `ReadVersion` transfer (`MAX_TRANSFER`, 4096 bytes): 48 bytes of header +/// and name, then 40 a member, so one page holds 100. 96 is the most of +/// that in whole 32-member steps. 48 was the first ceiling, reached exactly +/// when the catalogue held 45 executables and 3 libraries; the C compiler +/// was the 49th member. **What breaks if it moves.** Larger needs a second +/// transfer to read the record (a reader change, and another +/// [`PACKAGE_ABI`]); smaller fails exec's build, which asserts the +/// catalogue fits. `size_of::()` is pinned below. +pub const MAX_PACKAGE_MEMBERS: usize = 96; /// A member or package name: the store's own name bound /// ([`FS_NAME_BYTES`](crate::svc::FS_NAME_BYTES)), zero-padded, so a /// manifest name is always a legal DagFS name and never needs clamping. @@ -255,12 +269,13 @@ impl PackageManifest { } const _: () = assert!(core::mem::size_of::() == 8 + PACKAGE_NAME_BYTES); -/// 16-byte header + 32-byte name + 48 × 40-byte members = 1968 bytes: one +/// 16-byte header + 32-byte name + 96 × 40-byte members = 3888 bytes: one /// `ReadVersion` transfer (`MAX_TRANSFER` = 4096) reads a whole manifest. const _: () = assert!( core::mem::size_of::() == 16 + PACKAGE_NAME_BYTES + MAX_PACKAGE_MEMBERS * (8 + PACKAGE_NAME_BYTES) ); +const _: () = assert!(core::mem::size_of::() <= 4096); #[cfg(test)] mod tests { @@ -281,7 +296,7 @@ mod tests { fn a_manifest_round_trips_through_its_bytes_and_validates() { let manifest = system_package(); let bytes = manifest.as_bytes(); - assert_eq!(bytes.len(), 1968); + assert_eq!(bytes.len(), 3888); let read = PackageManifest::read_from_bytes(bytes).unwrap(); assert_eq!(read, manifest); assert_eq!(read.validate(), Ok(())); @@ -297,7 +312,7 @@ mod tests { #[test] fn a_zeroed_record_is_not_a_manifest() { - let zero = PackageManifest::read_from_bytes(&[0u8; 1968]).unwrap(); + let zero = PackageManifest::read_from_bytes(&[0u8; 3888]).unwrap(); assert_eq!(zero.validate(), Err(ManifestError::Abi(0))); } @@ -333,7 +348,7 @@ mod tests { } #[test] - fn push_refuses_duplicates_and_the_forty_ninth_member() { + fn push_refuses_duplicates_and_a_member_past_the_ceiling() { let mut manifest = system_package(); assert!( manifest diff --git a/native/crates/abi/src/session.rs b/native/crates/abi/src/session.rs index ac3aa27..8d05539 100644 --- a/native/crates/abi/src/session.rs +++ b/native/crates/abi/src/session.rs @@ -2588,6 +2588,28 @@ pub const REVOKE_KIND_EXEC_RETURN: u8 = 7; /// window survives, the latch re-anchors on it. pub const REVOKE_KIND_UNSHARE_IOVA_LIVE: u8 = 8; +/// The [`Invocation::kind`] selector for the consuming **writable lease → +/// frozen lease** transition through [`OpCode::Revoke`]: the exclusive +/// intake of a registrar that must validate bytes it did not write +/// (`res/IMAGE.md` §6b). +/// +/// The subject is a held or same-session `MemLease` carrying the complete +/// lifecycle grant `ACCESS|MODIFY|DERIVE|REVOKE`. Commit consumes that +/// capability and its complete descendant lineage — every view, carve and +/// route derived from it dies, wherever it travelled — removes every +/// CPU/DMA/guest alias, and writes a fresh `MemLease` over the same window +/// into `result`, carrying the same grant. From then on no lessor of the +/// backing may write the window, the successor and its carves included: the +/// family's claim table holds it immutable for as long as the successor +/// lives. A carve whose parent or sibling still holds a writable view of the +/// window is refused before anything commits, and so is an executable or +/// swap-sealed lease. +/// +/// The successor keeps `MODIFY` because a registrar carves the window's text +/// and finalizes it, which a frozen window admits ([`REVOKE_KIND_EXEC`] +/// leaves it immutable too); `MODIFY` never again reaches a write. +pub const REVOKE_KIND_FREEZE: u8 = 9; + /// Argument payload for a **page delegation** derive (`Derive`, /// [`kind`](Invocation::kind)` = Region` on a held/staged `Region` /// subject; res/IMAGE.md §9a): re-home the subject region diff --git a/native/crates/abi/src/svc.rs b/native/crates/abi/src/svc.rs index 703543e..437e013 100644 --- a/native/crates/abi/src/svc.rs +++ b/native/crates/abi/src/svc.rs @@ -1502,6 +1502,31 @@ impl TaskReqWire { /// `slot` names the face; no plan, no target, no grants. Seal-exempt, /// via==0 only: provisioning is the administrator's, not a task's. pub const OP_PROVISION_HOME: u8 = 25; + /// Install one file from the caller's own tree as a **private** + /// executable generation (`res/EXECUTABLE_CAPABILITY_MIGRATION.md` §7.4): + /// exec reads the file through the caller's conveyed view into the + /// caller's own memory supply, freezes that supply, registers it, and + /// grants an executable capability for it. + /// + /// The request is a client face's atomic context bundle without an + /// executable member: the supply on the ProcessIo lane, the view on the + /// filesystem-view lane, and on the first auxiliary lane the receive end + /// of a channel whose root the caller keeps — the generation's lifetime: + /// revoking that root, or the caller's death, retires it. The proof lane + /// and the other auxiliary lanes are vacant. `slot` carries the supply's + /// capability tag, so exec selects exactly this request's delivery and + /// never a stale one (tags are never reissued). The path rides the + /// supply itself ([`InstallPathHeader`]) and is overwritten by the + /// file's bytes, so the record keeps its size. Every other field is + /// zero. A file longer than the supply is answered + /// [`TAG_INSTALL_SUPPLY_SHORT`](TaskRepWire::TAG_INSTALL_SUPPLY_SHORT) + /// with the size it needs. + /// + /// Reading suffices, by the filesystem's own rule: `BIND_EXECUTABLE` + /// governs the shared executable namespace, not private code a task + /// makes executable from its own lease. The generation binds no global + /// name: only the capability's holder reaches it. + pub const OP_INSTALL_FILE: u8 = 26; /// Every retired `TaskReqWire` discriminant, in one definition the /// decoder cites (the [`derive_kind_known`](crate::session::derive_kind_known) @@ -1568,6 +1593,66 @@ impl TaskReqWire { pub const INSTALL_FROM_TREE: u8 = 1 << 1; } +/// The path an [`OP_INSTALL_FILE`](TaskReqWire::OP_INSTALL_FILE) request +/// installs, which the caller writes at offset 0 of the supply it conveys: +/// this header, then [`len`](Self::len) UTF-8 bytes. The path is relative +/// to the conveyed view and `/`-separated. Exec copies it out before the +/// first file byte lands and the file then overwrites it, so neither the +/// record nor the frozen image carries it. +/// +/// A path rides the supply rather than the record because a legal path is +/// longer than every free field: the record's `target` holds 64 bytes, the +/// deepest legal path [`FS_SEARCH_PATH_BYTES`], and the pair already uses +/// all but 128 bytes of its ring (`kore_svc_exec::wire`). +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct InstallPathHeader { + /// Live path bytes after the header (`1..=MAX_PATH`). + pub len: U16, + /// Reserved; must be zero. + pub reserved: [u8; 6], +} + +impl InstallPathHeader { + /// Wire size of the header (repr(C), 2 + 6). + pub const BYTES: usize = 8; + /// The longest path a request carries: the deepest legal filesystem + /// path, the bound a search hit spells. + pub const MAX_PATH: usize = FS_SEARCH_PATH_BYTES; + /// Bytes a request's path occupies at the head of its supply. + pub const MAX_BYTES: usize = Self::BYTES + Self::MAX_PATH; + + /// The header for `path`, or `None` for an empty path or one past + /// [`MAX_PATH`](Self::MAX_PATH). + pub fn for_path(path: &[u8]) -> Option { + if path.is_empty() || path.len() > Self::MAX_PATH { + return None; + } + Some(Self { + len: U16::new(path.len() as u16), + reserved: [0; 6], + }) + } + + /// The path a supply's head carries: a well-formed header followed by + /// that many UTF-8 bytes, or `None`. Component rules (no empty, `.` or + /// `..` component; the filesystem's name bound) are the reader's to + /// apply, against the filesystem it walks. + pub fn path(head: &[u8]) -> Option<&str> { + let (header, rest) = Self::read_from_prefix(head).ok()?; + let len = usize::from(header.len.get()); + if header.reserved != [0; 6] || len == 0 || len > Self::MAX_PATH { + return None; + } + core::str::from_utf8(rest.get(..len)?).ok() + } +} + +const _: () = assert!(core::mem::size_of::() == InstallPathHeader::BYTES); +const _: () = assert!(InstallPathHeader::MAX_PATH <= u16::MAX as usize); + /// The exec reply wire form (`TaskResponse`'s crossing shape). An owned /// `Spawned` reply carries exactly one task-handle grant. `id` remains /// correlation data only (hash-derived, unordered). A `Spawned` reply uses @@ -1750,6 +1835,20 @@ impl TaskRepWire { /// unprovisioned home is a posture (no store, or a stage that failed and /// is retried next boot), never a refusal. pub const TAG_HOME_PROVISIONED: u8 = 25; + /// Reply to [`TaskReqWire::OP_INSTALL_FILE`]: the private generation was + /// registered, and the reply grants its executable endpoint. `index` + /// carries the segment count and `status` the launch profile word, as + /// [`TAG_TREE_ACQUIRED`](Self::TAG_TREE_ACQUIRED) carries them. + pub const TAG_FILE_INSTALLED: u8 = 26; + /// Reply to [`TaskReqWire::OP_INSTALL_FILE`]: the file is longer than + /// the conveyed supply. `id` is the page-rounded size the file needs, + /// which a caller may retry with; nothing was registered, and the supply + /// was retired, so its memory is the caller's again. + pub const TAG_INSTALL_SUPPLY_SHORT: u8 = 27; + /// The unit [`TAG_INSTALL_SUPPLY_SHORT`](Self::TAG_INSTALL_SUPPLY_SHORT) + /// counts in: mm grants a base-class supply in whole 4 KiB pages, so a + /// size the caller can ask for is a multiple of it. + pub const INSTALL_SUPPLY_GRANULE: u64 = 4096; /// `TAG_HOME_PROVISIONED.index`: the home is recorded at the current /// skeleton version. pub const HOME_PROVISIONED: u8 = 1 << 0; @@ -1854,6 +1953,12 @@ impl TaskRepWire { pub const REFUSE_PEER_ADMISSION_WEDGED: u8 = 22; pub const REFUSE_MODEL_UNAVAILABLE: u8 = 23; pub const REFUSE_MODEL_MEMORY_UNAVAILABLE: u8 = 24; + /// `TaskRefusal::FileUnavailable` — an install's path named no file the + /// presented view could read whole: no such entry, a directory, or a + /// read the filesystem refused. Distinct from + /// [`REFUSE_IMAGE_UNAVAILABLE`](Self::REFUSE_IMAGE_UNAVAILABLE), which + /// says the bytes were read and are not a dynamic executable. + pub const REFUSE_FILE_UNAVAILABLE: u8 = 25; } // Live task discriminants stay disjoint from the frozen retired band on @@ -3963,6 +4068,42 @@ impl InChunk { mod tests { use super::*; + /// An install path round-trips through a supply's head, and every + /// malformed head refuses: empty, past the bound, a reserved byte set, + /// a length past the bytes present, and bytes that are not UTF-8. + #[test] + fn install_paths_ride_a_supply_head() { + let mut head = [0u8; InstallPathHeader::MAX_BYTES + 1]; + let path = b"build/hello"; + let header = InstallPathHeader::for_path(path).unwrap(); + head[..InstallPathHeader::BYTES].copy_from_slice(header.as_bytes()); + head[InstallPathHeader::BYTES..][..path.len()].copy_from_slice(path); + assert_eq!(InstallPathHeader::path(&head), Some("build/hello")); + + assert_eq!(InstallPathHeader::for_path(b""), None); + let longest = [b'a'; InstallPathHeader::MAX_PATH]; + assert!(InstallPathHeader::for_path(&longest).is_some()); + let too_long = [b'a'; InstallPathHeader::MAX_PATH + 1]; + assert_eq!(InstallPathHeader::for_path(&too_long), None); + + let mut reserved = head; + reserved[7] = 1; + assert_eq!(InstallPathHeader::path(&reserved), None); + let mut empty = head; + empty[..2].copy_from_slice(&0u16.to_le_bytes()); + assert_eq!(InstallPathHeader::path(&empty), None); + let mut past = head; + past[..2].copy_from_slice(&((InstallPathHeader::MAX_PATH + 1) as u16).to_le_bytes()); + assert_eq!(InstallPathHeader::path(&past), None); + assert_eq!( + InstallPathHeader::path(&head[..InstallPathHeader::BYTES + path.len() - 1]), + None + ); + let mut invalid = head; + invalid[InstallPathHeader::BYTES] = 0xff; + assert_eq!(InstallPathHeader::path(&invalid), None); + } + /// Every wire form reinterprets in place from arbitrary bytes and /// round-trips through its own bytes — the shared-ring contract /// (any bit pattern is *readable*; the codec's job is refusal). diff --git a/native/crates/abi/src/task.rs b/native/crates/abi/src/task.rs index baed7a9..6958fcf 100644 --- a/native/crates/abi/src/task.rs +++ b/native/crates/abi/src/task.rs @@ -280,9 +280,12 @@ pub mod role { /// the face table's slot arithmetic must never interleave with a /// heap tier's. /// - /// Geometry contract: [`FACE_SUPPLY_BYTES`](super::FACE_SUPPLY_BYTES) - /// = [`FACE_RING_STRIDE`](super::FACE_RING_STRIDE) rings — the - /// founder mints exactly that, the face table carves exactly that. + /// Geometry contract, one per carrying row: the image service's + /// [`FACE_SUPPLY_BYTES`](super::FACE_SUPPLY_BYTES) and a broker's + /// [`CLAIMANT_FACE_SUPPLY_BYTES`](super::CLAIMANT_FACE_SUPPLY_BYTES), + /// each a whole number of + /// [`FACE_RING_STRIDE`](super::FACE_RING_STRIDE) rings — the founder + /// mints exactly that, the face table carves exactly that. pub const FACE_SUPPLY: u16 = 29; /// Read-only machine telemetry authority, endowed only to the monitor /// service. The capability carries `ACCESS | DERIVE`; observation is a @@ -462,35 +465,61 @@ pub const INTERACTIVE_CHILD_LOADER_FACES: usize = DESKTOP_BOOTSTRAP_WINDOW_SLOTS + 2 * DESKTOP_CONTROLLER_CAPACITY; /// Live immutable image-generation peers the boot catalogue can hold at once: -/// the 45-row bootstrap catalogue — the desktop's `files`, its documents +/// the 46-row bootstrap catalogue — the desktop's `files`, its documents /// `editor`, `photos`, `settings` and `power-ui`, the screenshot writer `capture`, beside /// the native filesystem verbs `append`, `sync` and `pack`, the two /// WebAssembly rows, `wasm-witness` and `wasm`, `jsapp`, the JS app host, -/// `inference` / `model-data-witness`, and `shell`, the guest shell `term` -/// opens — plus the founded shell, the graphical terminal, and three -/// extraction parsers. +/// `inference` / `model-data-witness`, `shell`, the guest shell `term` +/// opens, and `tcc`, the C compiler — plus the founded shell, the graphical +/// terminal, and three extraction parsers. /// Exec asserts this covers its catalogue (`kore-svc-exec`), the one place /// that sees both. Named /// separately from the interactive reservation below because the two grow for /// unrelated reasons: this one when a row joins the catalogue, that one when /// the desktop admits another window or controller. -pub const BOOTSTRAP_IMAGE_PEERS: usize = 50; +pub const BOOTSTRAP_IMAGE_PEERS: usize = 51; -/// Bootstrap face-supply sizing. Image and runtime grow their face tables and -/// rings on demand; this reserves enough initial memory for the boot catalogue -/// and for a full desktop that can still run one command per controller. -pub const MAX_MINTED_FACE_PAIRS: usize = BOOTSTRAP_IMAGE_PEERS + INTERACTIVE_CHILD_LOADER_FACES; +/// Private image generations installed at runtime that can live at once, +/// across every session: exec's `InstallFile` rows, each a file a task +/// compiled or copied into its own tree and made executable from its own +/// memory supply. +/// +/// Each one costs a ring stride of face supply here (8 × 16 KiB, 128 KiB in +/// all) and one fixed landing in exec's convey table, which takes it from +/// exec's low convey band: eight leave that band thirty slots, above the +/// twelve the founder claims in it (exec asserts the band still holds +/// them). Exec lets one client face hold at most half of these rows, so no +/// session can exhaust the rest. Raising it costs exactly those two things; +/// lowering it below two would let one session's install starve another's. +pub const INSTALLED_IMAGE_PEERS: usize = 8; -/// Total byte length of a [`role::FACE_SUPPLY`] endowment — exactly -/// [`MAX_MINTED_FACE_PAIRS`] ring strides. +/// Bootstrap face-supply sizing. Image and runtime grow their face tables and +/// rings on demand; this reserves enough initial memory for the boot +/// catalogue, the installed generations, and a full desktop that can still +/// run one command per controller. +pub const MAX_MINTED_FACE_PAIRS: usize = + BOOTSTRAP_IMAGE_PEERS + INSTALLED_IMAGE_PEERS + INTERACTIVE_CHILD_LOADER_FACES; + +/// Total byte length of the image service's [`role::FACE_SUPPLY`] +/// endowment — exactly [`MAX_MINTED_FACE_PAIRS`] ring strides. /// The **cross-boundary geometry contract**: the kernel founder mints a -/// lease (or, for the device broker, a full-span sub of the shelf-held -/// claimant band) of exactly this length; the carrying service's face -/// table carves its slots against the same number (its claim checks the -/// span). One constant, cited by both sides, so a drift refuses at -/// bind instead of carving a phantom ring. +/// lease of exactly this length; the image face table carves its slots +/// against the same number (its claim checks the span). One constant, +/// cited by both sides, so a drift refuses at bind instead of carving a +/// phantom ring. The device broker carries the same role under its own +/// contract, [`CLAIMANT_FACE_SUPPLY_BYTES`]. pub const FACE_SUPPLY_BYTES: u64 = MAX_MINTED_FACE_PAIRS as u64 * FACE_RING_STRIDE; +/// Total byte length of a device broker's [`role::FACE_SUPPLY`] +/// endowment, its claimant band — exactly +/// [`MAX_CLAIMANT_FACES`](crate::device::MAX_CLAIMANT_FACES) ring +/// strides, one per driver row. The same contract as +/// [`FACE_SUPPLY_BYTES`]: the kernel founder mints each broker's band at +/// this length and carves the broker's full-span sub off it, and the +/// broker refuses readiness on any other span. +pub const CLAIMANT_FACE_SUPPLY_BYTES: u64 = + crate::device::MAX_CLAIMANT_FACES as u64 * FACE_RING_STRIDE; + /// **Geometry roles** — the window kinds a task learns its own VAs for from the /// [`GeomBlock`] (`res/USERLAND.md` §4; the fixed-VA elimination rung). Unlike /// [`role`] handle entries (which name an arena *slot*), a geometry entry names diff --git a/native/crates/rt/src/cap/mod.rs b/native/crates/rt/src/cap/mod.rs index 2266846..e5e1d39 100644 --- a/native/crates/rt/src/cap/mod.rs +++ b/native/crates/rt/src/cap/mod.rs @@ -1754,19 +1754,20 @@ pub enum QuiesceFault { Unenforced, } -/// Why a writable lease did not become an executable successor. +/// Why a consuming lease transition did not yield its successor: a +/// finalization, the return to writable, or a freeze. /// -/// Finalization has one abort door: a refusal before commit returns the exact -/// affine source. Once the kernel has retired that source and begun clearing +/// Each has one abort door: a refusal before commit returns the exact affine +/// source. Once the kernel has retired that source and begun clearing /// aliases, rollback is impossible; a later containment or publication /// failure is therefore reported as [`Consumed`](Self::Consumed), never as a -/// writable lease that no longer exists. +/// lease that no longer exists. #[must_use] #[derive(Debug)] pub enum FinalizeExecError { - /// The session did not commit; the writable source is unchanged. + /// The session did not commit; the source is unchanged. Refused(T), - /// The source was consumed, but no executable successor was published. + /// The source was consumed, but no successor was published. Consumed, } @@ -2854,17 +2855,6 @@ pub struct StashedLease { owned: OwnedSlot, } -impl StashedLease { - /// Adopt an already-activated owned slot the kernel just published as a - /// writable lease — the executable → writable return's landing - /// (`res/IMAGE.md` §6a). Crate-internal: the caller must have proved - /// the record's shape first (`writable_result_state`). - #[cfg(target_os = "none")] - pub(crate) fn from_owned(owned: OwnedSlot) -> Self { - Self { owned } - } -} - /// Transferable memory custody: an allocation lease or a scoped mapping loan. /// A route never acquires the allocation/release authority of a MemLease. #[derive(Debug)] diff --git a/native/crates/rt/src/cap/verbs.rs b/native/crates/rt/src/cap/verbs.rs index d6603ef..f07b911 100644 --- a/native/crates/rt/src/cap/verbs.rs +++ b/native/crates/rt/src/cap/verbs.rs @@ -4178,6 +4178,102 @@ mod metal { None } + /// Move a capability a client delivered into a fixed landing slot — a + /// service's convey bank, whose post must stand again for the next + /// request — into a fresh runtime-owned slot, so the service can keep it + /// past the request that carried it. + /// + /// The move rides a channel minted from `space` for it alone: the + /// landing is exact-moved through that channel's send into a post over + /// the new owner, and the channel is then retired. An exact move keeps + /// the record's rights and lineage, so retiring the private channel + /// reaches nothing it carried. Only the kinds the kernel moves exactly + /// relocate: memory leases and views, fabric routes, channel endpoints. + #[cfg(target_os = "none")] + pub fn relocate_landed( + space: &ChannelSpaceToken, + landed: SlotRef, + ) -> Result { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + use kore_abi::session::Status; + + fn committed(session: &SessionBuilder) -> bool { + TrapTransport::new() + .commit_now(session) + .is_ok_and(|done| done.status() == Status::Ok && done.failing_index() == 0) + } + + let crate::cap::ChannelPair { root, send, recv } = + mint_channel(space).map_err(|_| RelocateError::Unmoved)?; + let mut owner = crate::cap::ReturnedCapability::reserve(); + let mut armed = false; + let mut sent = false; + if let Some(owner) = owner.as_mut() { + let mut post = SessionBuilder::new(); + if post.post_recv(recv.slot(), owner.target()).is_ok() && committed(&post) { + owner.arm_post(); + armed = true; + let mut moving = SessionBuilder::new(); + sent = moving.send_exact(send.slot(), landed).is_ok() && committed(&moving); + } + } + // The root is the private channel's only pivot: revoking it ends + // both endpoints, and with them any post still naming the owner. + let closed = match root.try_retire() { + Ok(outcome) => outcome.is_closed(), + Err(root) => { + core::mem::forget(root); + false + } + }; + let _ = send.try_retire(); + let _ = recv.try_retire(); + let Some(mut owner) = owner else { + return Err(RelocateError::Unmoved); + }; + if sent { + if matches!( + take_delivery(owner.target()), + Some(crate::cap::RawDelivery::Cap) + ) { + owner.activate(); + return Ok(owner); + } + // A committed exact move into a standing post lands, so this is + // a record the runtime cannot classify: keep its name out of the + // allocator for the task's life rather than reopen it. + core::mem::forget(owner); + return Err(RelocateError::Lost); + } + if armed && !closed { + // The post may still stand over this name. + core::mem::forget(owner); + } else if armed { + owner.disarm_post(); + } + Err(RelocateError::Unmoved) + } + + #[cfg(not(target_os = "none"))] + pub fn relocate_landed( + _space: &ChannelSpaceToken, + _landed: SlotRef, + ) -> Result { + Err(RelocateError::Unmoved) + } + + /// Why [`relocate_landed`] produced no owner. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum RelocateError { + /// Nothing moved: the delivery is still in its landing, for the + /// caller's own custody to keep or retire. + Unmoved, + /// The move committed but its result could not be read back; the + /// landing is vacant and the would-be owner is quarantined. + Lost, + } + /// **The metal take-view** (`res/USERLAND.md` §5.6; Phase-A checkpoint /// C) — read a delivered `LeasedRegion` view record off the R/O /// cap-slot alias at [`SLOTS`](kore_abi::task::geom::SLOTS) alias and return @@ -5226,7 +5322,7 @@ mod metal { } ExecFinalizeCompletion::Replaced => { code.release_after_proven_move(); - Ok(StashedLease::from_owned(output)) + Ok(StashedLease::new(output)) } } } @@ -5238,6 +5334,78 @@ mod metal { Err(FinalizeExecError::Refused(code)) } + /// Freeze a runtime-owned writable lease (`res/IMAGE.md` §6b): consume + /// it with every alias and descendant capability of it, and return the + /// frozen successor — the same window and rights, which no lessor may + /// write again while it lives. This is a registrar's exclusive intake: + /// what it then validates is what every later reader of the window reads. + /// + /// Affine slot ownership is preserved on either outcome, as + /// [`finalize_stashed_exec`] preserves it: a refusal returns the exact + /// source, and a consumed source reports + /// [`Consumed`](FinalizeExecError::Consumed) with no authority back. + #[cfg(target_os = "none")] + pub fn freeze_stashed( + mut lease: StashedLease, + ) -> Result> { + let mut output = match crate::cap::OwnedSlot::reserve() { + Some(output) => output, + None => return Err(FinalizeExecError::Refused(lease)), + }; + let Some(source) = lease.slot() else { + return Err(FinalizeExecError::Refused(lease)); + }; + match freeze_slot(source, &mut output) { + ExecFinalizeCompletion::Refused => Err(FinalizeExecError::Refused(lease)), + ExecFinalizeCompletion::Consumed => { + lease.owned.release_after_proven_move(); + Err(FinalizeExecError::Consumed) + } + ExecFinalizeCompletion::Replaced => { + // The committed kernel transition cleared `source`; only its + // dynamic allocator reservation remains to release. + lease.owned.release_after_proven_move(); + Ok(StashedLease::new(output)) + } + } + } + + /// Off metal there is no freeze session to issue. + #[cfg(not(target_os = "none"))] + pub fn freeze_stashed( + lease: StashedLease, + ) -> Result> { + Err(FinalizeExecError::Refused(lease)) + } + + /// [`freeze_stashed`] for a lease a caller delivered into a fixed landing + /// slot (a service's convey bank): the frozen successor moves into a + /// fresh runtime-owned slot, and the landing is left vacant for its post + /// to stand again. A registrar that validates bytes a client wrote takes + /// them this way, so no alias the client kept can write them afterwards. + /// + /// `Refused` leaves the delivery in `slot`, untouched, for the caller's + /// own custody to retire; `Consumed` reports a vacant `slot` with no + /// successor. + #[cfg(target_os = "none")] + pub fn freeze_landed(slot: SlotRef) -> Result> { + let mut output = match crate::cap::OwnedSlot::reserve() { + Some(output) => output, + None => return Err(FinalizeExecError::Refused(())), + }; + match freeze_slot(slot, &mut output) { + ExecFinalizeCompletion::Refused => Err(FinalizeExecError::Refused(())), + ExecFinalizeCompletion::Consumed => Err(FinalizeExecError::Consumed), + ExecFinalizeCompletion::Replaced => Ok(StashedLease::new(output)), + } + } + + /// Off metal there is no freeze session to issue. + #[cfg(not(target_os = "none"))] + pub fn freeze_landed(_slot: SlotRef) -> Result> { + Err(FinalizeExecError::Refused(())) + } + #[cfg(target_os = "none")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ExecFinalizeCompletion { @@ -5274,10 +5442,10 @@ mod metal { } } - /// The writable successor's shape proof — [`exec_result_state`]'s twin. - /// The rights differ (`MODIFY` is back), and checking the exact set is - /// what makes a wrong-shaped record `Malformed` rather than silently - /// accepted. + /// The full-lifecycle successor's shape proof — a return's or a + /// freeze's, [`exec_result_state`]'s twin. The rights differ (`MODIFY` + /// is there), and checking the exact set is what makes a wrong-shaped + /// record `Malformed` rather than silently accepted. #[cfg(target_os = "none")] fn writable_result_state(output: SlotRef) -> ExecResultState { use kore_abi::cap::{CapRecord, ResourceKind}; @@ -5331,6 +5499,34 @@ mod metal { } } + /// Freeze `source` into `output`, mirroring [`return_writable_slot`]: + /// the successor carries the full lifecycle grant, so the same shape + /// proof classifies it. + #[cfg(target_os = "none")] + fn freeze_slot(source: SlotRef, output: &mut crate::cap::OwnedSlot) -> ExecFinalizeCompletion { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + + let mut b = SessionBuilder::new(); + if b.freeze(source, output.slot()).is_err() { + return ExecFinalizeCompletion::Refused; + } + let Ok(committed) = TrapTransport::new().commit_now(&b) else { + return ExecFinalizeCompletion::Refused; + }; + if committed.status() != kore_abi::session::Status::Ok || committed.failing_index() != 0 { + return ExecFinalizeCompletion::Refused; + } + match writable_result_state(output.slot()) { + ExecResultState::Replaced => ExecFinalizeCompletion::Replaced, + ExecResultState::Vacant => ExecFinalizeCompletion::Consumed, + ExecResultState::Malformed => { + output.quarantine_unknown_publish(); + ExecFinalizeCompletion::Consumed + } + } + } + #[cfg(target_os = "none")] fn finalize_exec_slot( source: SlotRef, diff --git a/native/crates/rt/src/crash.rs b/native/crates/rt/src/crash.rs index b06b58e..f61c060 100644 --- a/native/crates/rt/src/crash.rs +++ b/native/crates/rt/src/crash.rs @@ -112,10 +112,23 @@ pub fn register( /// Returning executables share the same recursion-safe diagnostic path. pub fn panic(info: &core::panic::PanicInfo<'_>) -> ! { - if crate::entry::begin_panic() { + report(info); + crate::thread::exit(crate::ExitCode::PANIC) +} + +/// [`panic`]'s report without its exit: publish this thread's panic record +/// once (a panic raised while reporting reports nothing more), and return. +/// +/// For a panic handler that must act between the report and the thread's +/// end — a host that ends the whole process on a worker's panic tells the +/// host only once the record is published, so the host's barrier cannot cut +/// the report short. `true` when this call published the record. +pub fn report(info: &core::panic::PanicInfo<'_>) -> bool { + let first = crate::entry::begin_panic(); + if first { report_panic_info(info); } - crate::thread::exit(crate::ExitCode::PANIC) + first } /// Best-effort software report. The entry panic handler has claimed recursion diff --git a/native/crates/rt/src/entry.rs b/native/crates/rt/src/entry.rs index aff740d..8ce8ecd 100644 --- a/native/crates/rt/src/entry.rs +++ b/native/crates/rt/src/entry.rs @@ -504,7 +504,7 @@ const fn report_slot_install(current_p1: usize, requested: usize) -> ReportSlotI #[cfg(target_os = "none")] pub(crate) mod geometry { - use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + use core::sync::atomic::{AtomicI32, AtomicU64, AtomicUsize, Ordering}; use kore_abi::task::{GeomBlock, geom}; @@ -631,6 +631,16 @@ pub(crate) mod geometry { finalizer_state: AtomicUsize, finalizers: [AtomicUsize; 32], exit_started: AtomicUsize, + /// This thread's C `errno` ([`c_errno`]): ISO C gives `errno` + /// thread storage duration, and this block is the runtime's + /// per-thread home, so the word lives here rather than in an ELF + /// TLS block — which would give every C-carrying image a TLS + /// segment, and with it a TCB migration and a template every spawn + /// must place (`kore-late` refuses such a host outright). Zero on + /// a fresh page, as C requires at thread start; deliberately *not* + /// inherited by a spawned thread, whose block is its own. Only the + /// owning thread reads or writes it. + c_errno: AtomicI32, } // The block must fit the founder's one-page window. @@ -732,6 +742,15 @@ pub(crate) mod geometry { block().panicking.swap(1, Ordering::AcqRel) == 0 } + /// This thread's C `errno` word (the block's `c_errno` field). + /// + /// Lawful wherever [`block`] is: C code runs only on an anchored + /// thread, since anchoring is crt0's first act and a spawned thread's + /// trampoline's first act. + pub(crate) fn c_errno() -> *mut i32 { + block().c_errno.as_ptr() + } + /// Where this thread's block lives — the value the thread pointer holds, /// recovered the same way every other accessor recovers it. pub(crate) fn block_va() -> usize { diff --git a/native/crates/rt/src/session/builder.rs b/native/crates/rt/src/session/builder.rs index fa60e93..089a589 100644 --- a/native/crates/rt/src/session/builder.rs +++ b/native/crates/rt/src/session/builder.rs @@ -1264,6 +1264,27 @@ impl SessionBuilder { Ok(Staged(result)) } + /// Consume a writable lease and return its **frozen** successor + /// (`Revoke × REVOKE_KIND_FREEZE`) — the exclusive intake of + /// `res/IMAGE.md` §6b. + /// + /// The kernel requires `ACCESS|MODIFY|DERIVE|REVOKE` on `subject`, kills + /// its complete descendant lineage, removes every CPU/DMA/guest alias, + /// and writes an `ACCESS|MODIFY|DERIVE|REVOKE` lease over the same window + /// into `result`. No lessor may write that window again while the + /// successor lives — the successor and its carves included. + pub fn freeze(&mut self, subject: SlotRef, result: SlotRef) -> Result { + self.push(Inv { + op: OpCode::Revoke as u16, + requested: 0, + kind: kore_abi::session::REVOKE_KIND_FREEZE, + subject: subject.index(), + result: result.index(), + arg: Invocation::NO_ARG, + })?; + Ok(Staged(result)) + } + /// `Derive × DERIVE_KIND_ARENA_EXTEND` on the task's own space /// self-view (slice E): grow the caller's cap arena by one chunk. /// An **effect**, not a mint — the request is empty, the arg is the diff --git a/native/crates/rt/src/session/tests.rs b/native/crates/rt/src/session/tests.rs index 8446618..7cfcd4a 100644 --- a/native/crates/rt/src/session/tests.rs +++ b/native/crates/rt/src/session/tests.rs @@ -235,6 +235,36 @@ fn the_executable_return_is_the_finalizations_wire_twin() { ); } +/// The freeze is the third turn and the same wire shape: one no-argument +/// consuming revoke, distinct from both of the others. +#[test] +fn the_freeze_is_a_no_argument_consuming_revoke() { + let source = SlotRef::from_wire(12); + let result = SlotRef::from_wire(13); + let mut builder = SessionBuilder::new(); + let staged = builder.freeze(source, result).expect("the freeze stages"); + assert_eq!(staged.slot(), result); + + let mut out = [0u8; DESCRIPTOR_MAX]; + let len = builder.finish(&mut out).expect("descriptor finishes"); + assert_eq!(len, 8 + 16, "the same one-invocation descriptor"); + let (header, invocations, args) = parse(&out[..len]); + assert_eq!(header.count.get(), 1); + assert_eq!(invocations[0].op.get(), OpCode::Revoke as u16); + assert_eq!(invocations[0].kind, kore_abi::session::REVOKE_KIND_FREEZE); + assert_eq!(invocations[0].requested, 0, "rights come from the lease"); + assert_eq!(invocations[0].subject.get(), source.index()); + assert_eq!(invocations[0].result.get(), result.index()); + assert_eq!(invocations[0].arg.get(), Invocation::NO_ARG); + assert!(args.is_empty(), "the freeze names no second capability"); + for other in [ + kore_abi::session::REVOKE_KIND_EXEC, + kore_abi::session::REVOKE_KIND_EXEC_RETURN, + ] { + assert_ne!(kore_abi::session::REVOKE_KIND_FREEZE, other); + } +} + /// Length selection happens before the transport chooses registers or /// scratch, but both carriers must receive byte-identical output from the one /// encoder. `finish` writes exactly that prefix and leaves spare carrier bytes diff --git a/native/crates/rt/src/startup.rs b/native/crates/rt/src/startup.rs index 800d2be..186a3b0 100644 --- a/native/crates/rt/src/startup.rs +++ b/native/crates/rt/src/startup.rs @@ -29,12 +29,23 @@ pub fn drain() -> Result ! { +pub fn barrier(finalize: bool) { use kore_abi::session::Status; let report = loop { match drain() { @@ -46,9 +57,19 @@ pub fn finish(code: ExitCode) -> ! { Err(_) => crate::thread::exit(ExitCode::INTERNAL), } }; - if crate::entry::geometry::begin_process_exit() && !report.forced { + // Marked whether or not this end finalizes, so no later end can run them. + let first = crate::entry::geometry::begin_process_exit(); + if finalize && first && !report.forced { crate::entry::geometry::run_finalizers(); } +} + +/// End a returning program. Live workers are stopped without their cooperation; +/// their stacks and shared storage are reclaimed by kernel task teardown. +/// User finalizers run only after a clean worker barrier ([`barrier`]). +#[cfg(target_os = "none")] +pub fn finish(code: ExitCode) -> ! { + barrier(true); crate::thread::exit(code) } diff --git a/native/crates/rt/src/svc/client.rs b/native/crates/rt/src/svc/client.rs index 8d187dc..994d5c8 100644 --- a/native/crates/rt/src/svc/client.rs +++ b/native/crates/rt/src/svc/client.rs @@ -487,6 +487,21 @@ pub enum AffineBundleCallError { AfterTransfer(CallError), } +/// Failure of a context-bundle call +/// ([`call_with_context_bundle`](SvcClient::call_with_context_bundle)): the +/// [`AffineBundleCallError`] discipline for a bundle with no executable +/// member, so no witness and no first input. +#[derive(Debug)] +pub enum ContextBundleCallError { + NotMoved { + error: CallError, + second: B, + third: C, + auxiliaries: [Option; MAX_INVOCATION_AUXILIARIES], + }, + AfterTransfer(CallError), +} + /// Failure of a return call whose transferred capability is freshly staged /// inside the transfer session. /// @@ -2264,6 +2279,170 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { } } + /// Move a service request and its context bundle in one atomic session + /// — [`call_with_invocation_bundle`](Self::call_with_invocation_bundle) + /// without the executable member. Such a request names no generation to + /// invoke, so it carries no witness and no input sent through an + /// executable endpoint; the receiver correlates the bundle with the + /// request by the data the request itself carries. Every other step is + /// the invocation's, with its helpers: the same preflights before the + /// request record is published, the same single transfer session, and + /// the same fail-closed poisoning once the record is visible. + pub async fn call_with_context_bundle( + &mut self, + req: &S::Request, + second: B, + second_rights: Rights, + process_tx: SlotRef, + third: C, + third_rights: Rights, + third_tx: SlotRef, + auxiliaries: [Option; MAX_INVOCATION_AUXILIARIES], + auxiliary_tx: [SlotRef; MAX_INVOCATION_AUXILIARIES], + ) -> Result, ContextBundleCallError> { + macro_rules! not_moved { + ($error:expr) => { + ContextBundleCallError::NotMoved { + error: $error, + second, + third, + auxiliaries, + } + }; + } + + if let Err(error) = self.require_idle() { + return Err(not_moved!(error)); + } + // Check every destination before the record is published: see the + // invocation bundle's note on why a post-publication rollback cannot + // be undone. + if ![process_tx, third_tx] + .into_iter() + .all(live_send_destination) + || auxiliaries + .iter() + .zip(auxiliary_tx) + .any(|(auxiliary, destination)| { + auxiliary.is_some() && !live_send_destination(destination) + }) + { + return Err(not_moved!(CallError::Refused(Status::Revoked))); + } + let request = S::encode_req(req); + let grants = match self.prepare_call(&request) { + Ok(grants) => grants, + Err(error) => return Err(not_moved!(error)), + }; + if self.rings.req.capacity_left() == 0 { + return Err(not_moved!(CallError::RingFull)); + } + if let Err(error) = self.ensure_grant_posts(grants).await { + return Err(not_moved!(error)); + } + let second_slot = match preflight_return_send(&second, second_rights) { + Ok(slot) => slot, + Err(error) => return Err(not_moved!(error)), + }; + let third_slot = match preflight_return_send(&third, third_rights) { + Ok(slot) => slot, + Err(error) => return Err(not_moved!(error)), + }; + let mut source_slots = [SlotRef::from_wire(0); 2 + MAX_INVOCATION_AUXILIARIES]; + source_slots[0] = second_slot; + source_slots[1] = third_slot; + let mut source_count = 2; + for auxiliary in auxiliaries.iter().flatten() { + let checked = match auxiliary.transfer { + AuxiliaryTransfer::Derived => { + preflight_return_send(&auxiliary.capability, auxiliary.rights) + } + AuxiliaryTransfer::Exact => preflight_exact_return(&auxiliary.capability), + }; + let slot = match checked { + Ok(slot) => slot, + Err(error) => return Err(not_moved!(error)), + }; + source_slots[source_count] = slot; + source_count += 1; + } + if !invocation_sources_are_distinct(&source_slots[..source_count]) { + return Err(not_moved!(CallError::Refused(Status::Malformed))); + } + + let mut transfer = SessionBuilder::new(); + let mut malformed = transfer + .send(process_tx, second_slot, second_rights) + .and_then(|_| transfer.send(third_tx, third_slot, third_rights)) + .is_err(); + for (auxiliary, destination) in auxiliaries.iter().zip(auxiliary_tx) { + if malformed { + break; + } + let Some(auxiliary) = auxiliary.as_ref() else { + continue; + }; + let Some(slot) = return_capability_sealed::Sealed::return_slot(&auxiliary.capability) + else { + return Err(not_moved!(CallError::Refused(Status::Revoked))); + }; + malformed = match auxiliary.transfer { + AuxiliaryTransfer::Derived => { + transfer.send(destination, slot, auxiliary.rights).is_err() + } + AuxiliaryTransfer::Exact => transfer.send_exact(destination, slot).is_err(), + }; + } + if malformed { + return Err(not_moved!(CallError::Refused(Status::Malformed))); + } + + let observation = ReplyObservation::begin(None).await; + if self.rings.req.push(request).is_err() { + return Err(not_moved!(CallError::RingFull)); + } + let committed = match self.transport.commit(&transfer).await { + Ok(committed) => committed, + Err(_) => { + drop(second); + drop(third); + drop(auxiliaries); + self.poison(); + return Err(ContextBundleCallError::AfterTransfer(CallError::Transport)); + } + }; + match classify_return_commit(committed, transfer.staged(), ReturnSource::Held) { + ReturnCommit::Moved => { + return_capability_sealed::Sealed::on_proven_moved(second); + return_capability_sealed::Sealed::on_proven_moved(third); + for auxiliary in auxiliaries.into_iter().flatten() { + return_capability_sealed::Sealed::on_proven_moved(auxiliary.capability); + } + self.state = ClientState::AwaitingReply { request }; + self.kick_published_request() + .await + .map_err(ContextBundleCallError::AfterTransfer)?; + self.collect_reply(observation) + .await + .map_err(ContextBundleCallError::AfterTransfer) + } + ReturnCommit::NotMoved(error) => { + // Custody stayed local, but the record is already visible + // and the ring has no rollback: fail closed, as the + // invocation bundle does. + self.poison(); + Err(not_moved!(error)) + } + ReturnCommit::AfterTransfer(error) => { + drop(second); + drop(third); + drop(auxiliaries); + self.poison(); + Err(ContextBundleCallError::AfterTransfer(error)) + } + } + } + /// Issue a protocol-declared implicit return request as one L4-style /// **send-and-wait** crossing. /// diff --git a/native/crates/rt/src/svc/mod.rs b/native/crates/rt/src/svc/mod.rs index 86aa8fb..db6c47e 100644 --- a/native/crates/rt/src/svc/mod.rs +++ b/native/crates/rt/src/svc/mod.rs @@ -48,7 +48,8 @@ mod recyclable; mod serve; pub use client::{ - AffineBundleCallError, CallError, CallReply, ClientEndpoints, GrantPin, InvocationAuxiliary, + AffineBundleCallError, CallError, CallReply, ClientEndpoints, ContextBundleCallError, GrantPin, + InvocationAuxiliary, InvocationAuxiliaryCapability, MAX_INVOCATION_AUXILIARIES, Observation, ReplyWaitStats, RetainedReturnCallError, ReturnCallError, ReturnCapability, ReturnDeriveError, ReturnDeriver, ReturnLeaseSource, diff --git a/native/crates/rt/src/thread.rs b/native/crates/rt/src/thread.rs index bbbe78a..63b8781 100644 --- a/native/crates/rt/src/thread.rs +++ b/native/crates/rt/src/thread.rs @@ -635,6 +635,51 @@ pub(crate) fn heap_endowment() -> Option { None } +/// The rights of a wake send a thread may pass on: the notice itself +/// (`ACCESS`, [`verbs::NOTICE_RIGHTS`](crate::cap::verbs::NOTICE_RIGHTS)) and +/// `DERIVE`, so the thread can found children holding the same end. A send +/// into a wake channel conveys nothing but the right to kick it. +pub(crate) const WAKE_SEND_RIGHTS: kore_abi::cap::Rights = + kore_abi::cap::Rights::ACCESS.union(kore_abi::cap::Rights::DERIVE); + +/// [`WorkerCap`] re-deriving this thread's own `DOORBELL@PEER_BASE` send for a +/// child, so the child's [`kick_parent`] reaches the same thread this one's +/// does: a [`DoorbellWorker`]'s wake end, or one inherited the same way. +/// +/// `None` when this thread holds no such send, or holds one it may not derive +/// from (a band peer's notice-only end, [`spmd`](crate::spmd)'s mesh): an +/// inheritance this thread cannot make is not offered, so it never fails the +/// spawn it would have rode on. The arena record decides, which is the +/// kernel's own word for the slot's kind and rights. +#[cfg(target_os = "none")] +pub(crate) fn wake_endowment() -> Option { + use kore_abi::cap::{CapRecord, ResourceKind}; + let rt = crate::entry::thread_context()?; + let send = rt + .bootstrap() + .handle(role::DOORBELL, crate::futex::PEER_BASE)?; + let record_va = crate::entry::geometry::slot_record_va(send)?; + // SAFETY: the slot alias is this thread's kernel-authored, read-only view + // of its own arena record for a slot its boot block names. + let record = unsafe { core::ptr::read_volatile(record_va as *const CapRecord) }; + if record.kind != ResourceKind::ChanSend as u16 || !record.rights.contains(WAKE_SEND_RIGHTS) { + return None; + } + Some(WorkerCap { + role: role::DOORBELL, + index: crate::futex::PEER_BASE, + subject: send, + kind: ResourceKind::ChanSend as u16 as u8, + rights: WAKE_SEND_RIGHTS, + }) +} + +/// Off metal no thread founds another; there is no wake to inherit. +#[cfg(not(target_os = "none"))] +pub(crate) fn wake_endowment() -> Option { + None +} + /// Build exactly the capability namespace a spawned worker receives. /// /// This stays target-independent so host tests can pin a subtle shared-space @@ -694,7 +739,7 @@ fn push_worker_thread_blocks(gblock: &mut GeomBlock, local: Window) -> bool { /// view. [`spawn_with`] when the thread must hold capabilities. #[cfg(target_os = "none")] pub fn spawn(entry: fn()) -> Option { - spawn_entry(ThreadEntry::Plain(entry), &[]) + spawn_entry(ThreadEntry::Plain(entry), &[], WorkerStack::Band) } // (The per-worker `spawn_worker` shim retired with G5: workers found as a @@ -711,7 +756,30 @@ pub fn spawn(entry: fn()) -> Option { /// into [`exit`]`(0)`. #[cfg(target_os = "none")] pub fn spawn_with(entry: fn(), endow: &[WorkerCap]) -> Option { - spawn_entry(ThreadEntry::Plain(entry), endow) + spawn_entry(ThreadEntry::Plain(entry), endow, WorkerStack::Band) +} + +/// Where a spawned worker's stack lives. +/// +/// A plain thread's stack is its band slot by default: `STACK_BYTES` +/// (64 KiB) live under a guard page, tiled by [`MAX_THREADS`] — what a +/// service worker's bounded frames need. A synchronous engine whose +/// recursion depth follows its *input* — a C compiler's expression and block +/// nesting — needs more than a slot holds, so it asks for a **guarded +/// strip** of the task's stack window instead (`res/SPMD.md` §8's +/// kernel-thread class, the shape SPMD gang workers already take: usable +/// pages with one guard granule below, mapped by the founding and given +/// back by `join`). Overflow takes a fault either way, never a +/// neighbour's memory. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WorkerStack { + /// The thread's band slot. + Band, + /// A guarded strip of at least this many bytes, rounded up to whole + /// pages. The strip comes from the task's one stack window (16 MiB of + /// reservation, shared with SPMD gangs); a request the window cannot + /// place — or a zero-byte one — refuses the spawn. + Bytes(usize), } /// A worker that can **park and be kicked**, and kick back: two channels @@ -795,6 +863,19 @@ impl DoorbellWorker { } } +/// The calling thread's C `errno` word — what a C runtime's +/// `__errno_location` answers. +/// +/// Per thread, as ISO C requires (C11 7.5p2): the word lives in this +/// thread's runtime block rather than an ELF TLS block, so an image that +/// carries C code needs no TLS segment for it. Zero when the thread starts. +/// The pointer is valid for the calling thread's life, and only that thread +/// may read or write through it. +#[cfg(target_os = "none")] +pub fn c_errno() -> *mut i32 { + crate::entry::geometry::c_errno() +} + /// Worker side of [`DoorbellWorker`]: post one notice into the parent's /// wake channel (the send founded at `DOORBELL@PEER_BASE`). `false` when /// this thread holds no such send or the post was refused — the latter @@ -820,6 +901,18 @@ pub fn kick_parent() -> bool { /// mint or a derive refuses, or [`spawn_with`] does. #[cfg(target_os = "none")] pub fn spawn_with_doorbell(entry: fn(), extra: &[WorkerCap]) -> Option { + spawn_with_doorbell_on(entry, extra, WorkerStack::Band) +} + +/// [`spawn_with_doorbell`] on the stack `stack` names — a guarded strip +/// for an engine whose depth its input decides ([`WorkerStack`]). `None` +/// also when the stack window cannot place the strip. +#[cfg(target_os = "none")] +pub fn spawn_with_doorbell_on( + entry: fn(), + extra: &[WorkerCap], + stack: WorkerStack, +) -> Option { use kore_abi::cap::{ResourceKind, Rights}; use crate::cap::verbs; @@ -869,16 +962,20 @@ pub fn spawn_with_doorbell(entry: fn(), extra: &[WorkerCap]) -> Option Option Option { - spawn_entry(ThreadEntry::Argument(entry, argument), endow) + spawn_entry( + ThreadEntry::Argument(entry, argument), + endow, + WorkerStack::Band, + ) } #[cfg(not(target_os = "none"))] pub fn spawn_with_arg(_entry: fn(usize), _argument: usize, _endow: &[WorkerCap]) -> Option { @@ -913,7 +1014,7 @@ pub fn spawn_with_arg(_entry: fn(usize), _argument: usize, _endow: &[WorkerCap]) /// shape is the only thing the shapes disagree on, and it rides the /// start record untouched by everything below. #[cfg(target_os = "none")] -fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap]) -> Option { +fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap], stack: WorkerStack) -> Option { if endow.len() > MAX_WORKER_CAPS { return None; } @@ -952,17 +1053,22 @@ fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap]) -> Option { w.map = p.remaining; } - // ── The stack: an arena strip for SPMD workers (G4b), the band - // slot for plain threads. Carved here and — exactly like the + // ── The stack: an arena strip for SPMD workers (G4b) and for a + // plain thread that asked for one ([`WorkerStack::Bytes`]), the + // band slot otherwise. Carved here and — exactly like the // band index above — left spent if a later founding step // refuses: the teardown that gives both back is join's, never // spawn's, and the leak-on-refusal is the safe direction for // the same reason the band's is. ── - let strip = match &entry { - ThreadEntry::Spmd(..) => { + let strip = match (&entry, stack) { + (ThreadEntry::Spmd(..), _) => { Some(crate::spmd::STACKS.carve_guarded(crate::spmd::STACK_GRANULES)?) } - ThreadEntry::Plain(_) | ThreadEntry::Argument(..) => None, + (ThreadEntry::Plain(_) | ThreadEntry::Argument(..), WorkerStack::Band) => None, + (ThreadEntry::Plain(_) | ThreadEntry::Argument(..), WorkerStack::Bytes(bytes)) => { + let granules = u32::try_from(bytes.div_ceil(crate::stack_arena::GRANULE)).ok()?; + Some(crate::spmd::STACKS.carve_guarded(granules)?) + } }; let (stack_base, stack_len) = match &strip { Some(s) => (s.base(), s.bytes() as u64), diff --git a/native/crates/rt/src/thread_controller.rs b/native/crates/rt/src/thread_controller.rs index 65c6289..688f0cf 100644 --- a/native/crates/rt/src/thread_controller.rs +++ b/native/crates/rt/src/thread_controller.rs @@ -10,7 +10,43 @@ use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; // Each owner derives the process heap into its children using its own local // slot. No creator's slot number is ever forwarded across the controller seam. -use super::thread::heap_endowment; +use super::thread::{WorkerCap, heap_endowment, wake_endowment}; + +/// What a managed thread is founded holding, re-derived by each founder from +/// its own holdings: the heap position its callback allocates through, and a +/// send into the wake channel its founder kicks (`thread::wake_endowment`), so +/// a managed thread reaches the same host thread 0 a `DoorbellWorker` does — +/// the host a language runtime keeps there to serve the process and end it. +/// Either is absent when the founder holds none. +struct Endowment { + caps: [WorkerCap; 2], + len: usize, +} + +impl Endowment { + fn inherited() -> Self { + let vacant = WorkerCap { + role: 0, + index: 0, + subject: kore_abi::session::SlotRef::from_wire(0), + kind: 0, + rights: kore_abi::cap::Rights::empty(), + }; + let mut endowment = Self { + caps: [vacant; 2], + len: 0, + }; + for cap in [heap_endowment(), wake_endowment()].into_iter().flatten() { + endowment.caps[endowment.len] = cap; + endowment.len += 1; + } + endowment + } + + fn as_slice(&self) -> &[WorkerCap] { + &self.caps[..self.len] + } +} const FREE: u64 = 0; const BUILD: u64 = 1; @@ -112,11 +148,11 @@ impl Controller { self.started .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) .map_err(|_| Error::Busy)?; - let heap = heap_endowment(); + let endowment = Endowment::inherited(); match super::thread::spawn_with_arg( controller_entry, self as *const Self as usize, - heap.as_slice(), + endowment.as_slice(), ) { Some(ticket) => { // This owner deliberately lives until process teardown. Managed @@ -341,7 +377,7 @@ fn child_entry(argument: usize) { fn controller_entry(argument: usize) { // SAFETY: start requires static storage. let controller = unsafe { &*(argument as *const Controller) }; - let heap = heap_endowment(); + let endowment = Endowment::inherited(); let mut tickets: [Option; COUNT] = [const { None }; COUNT]; loop { for (record, ticket) in controller.records.iter().zip(tickets.iter_mut()) { @@ -351,7 +387,7 @@ fn controller_entry(argument: usize) { *ticket = super::thread::spawn_with_arg( child_entry, record as *const Record as usize, - heap.as_slice(), + endowment.as_slice(), ); record.state.store( generation | if ticket.is_some() { RUNNING } else { FAILED }, diff --git a/native/provenance.json b/native/provenance.json index 6f475c1..c60b681 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "7074bda28c416ffe34562857ee43cbc883482782", + "revision": "eab77f8d904352eabe65fedb84a455d6ccee5b7d", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", @@ -37,7 +37,7 @@ "crates/abi/src/clock.rs": "cce1c2839cba2377d3cc8696d60812f9195f071f422bd97bb4530ea0368972fc", "crates/abi/src/command.rs": "206b8b150688cff68345c9237b78d1170e92f01c8323951e2a3e54b5fb808b97", "crates/abi/src/crash.rs": "f3c9e5a954e06743f6993b0a5eeb6937d2199c50af5fe95b6be446393ccf999b", - "crates/abi/src/device.rs": "ad4578975e8056f375b0338124f497c9b4b5caa6272ea8a9930e2877d44efac6", + "crates/abi/src/device.rs": "241909f43812f1bd13a4dc325ed3f43c387d80e65193eb15aa276a4c3f7946f5", "crates/abi/src/device/census.rs": "6fb54ecaade94b116f41e045b098dc5926e8e1c75eeea33870dd4be01bd1ce23", "crates/abi/src/device/i2c_admission.rs": "1ea2fc04cdb0b15186d2f1d80bde53187860e2197263e225f330965d96e40876", "crates/abi/src/execution.rs": "106d679a3f993c3f4e7ad4eb625d40f4399f5b9b50947cb0336ba44a1d6b51af", @@ -57,14 +57,14 @@ "crates/abi/src/machine.rs": "7427c50ec38f71dbc83818a5075d03096cd34f5509e22646b04839eeb93cc783", "crates/abi/src/mem.rs": "ae502311778be8ef73f7b9edb1c47f157564f96a58db9add4ad74a313eb3a147", "crates/abi/src/monitor.rs": "8f01b9165d493f78127c3e2d37da401e3c969777b69535612ab2bcbd27eb7871", - "crates/abi/src/pkg.rs": "2893ad39761611ddee84d314bebbed45a9c9823ff145c6a8f28d3844cb40155d", + "crates/abi/src/pkg.rs": "da294e9e37fef2decda03dc5601a2a1131e499378bb14853845c2783b8e1a85a", "crates/abi/src/pointer.rs": "9971a292c1d00b275d8030e4f11fdd71206eaf12671cda8775d220b106ebcbf8", "crates/abi/src/process.rs": "1dc29781e8d5f500e1e44e8ac36ca466aacf7f14a74b0f730af84336a4319cce", "crates/abi/src/processor_status.rs": "c963e48f9b07046ab6c82d4cc2735ee2cc6c94561509a4e6e15927c837c0b0c6", - "crates/abi/src/session.rs": "030b2bee9b46426656db7ebde001738a2cfbdbe775a3be1ae892a14330f2d66f", + "crates/abi/src/session.rs": "8679f287e28556b1caff70126e98cc1b99d76dd9489271f8cc0973990bbd46a1", "crates/abi/src/stor.rs": "eabad4e5588fd343472660e15cccf32cfcc8d8f4c25eb18dffbd1598b6007db2", - "crates/abi/src/svc.rs": "d857cc812bdbc228c5f852a02ed02a349951cdc268291e51abe4628b894610da", - "crates/abi/src/task.rs": "57cb01a29bd2dbf870a1c49e35424f091fad597396ed5d080fc4f4cdd79c5f2a", + "crates/abi/src/svc.rs": "12749268a416577521bf1762cabe0cece8b4adae16572beb13ea3f8228539be3", + "crates/abi/src/task.rs": "1e403147de46734c9f217be6d5fc8edf4dd826d49385a7d4628d0773cd2a0acf", "crates/abi/src/uring.rs": "e98d9db77fbe00d5547b582700a1104d2c5b2546bbe57742dfff8362c32dcc2e", "crates/app/Cargo.toml": "fb793b25eec583da7b5e404b19bd656a6b8bcee2a86a20b0da399be2210a9398", "crates/app/src/lib.rs": "5e7e8d791236f278ee85a02a3c37f41412d879cc41d67f6f6d3d701811973d7d", @@ -190,17 +190,17 @@ "crates/rt/src/barrier.rs": "c5bc3bef52f2bcc24edb1016fde955bd01e73074d4ce357d79cae9d3b63e6f3b", "crates/rt/src/block.rs": "70c2aef8bf149da2f32c7c995cd0f4c63cb492647896b107fdd5dbecd3d3042a", "crates/rt/src/cap/mapped.rs": "b29ce431638a29b01edfd72500348f5ae201afe97ec4684ea4449d01e75ec1e7", - "crates/rt/src/cap/mod.rs": "959a357dedc329cb84563f2a8677b462e89da379373022d644ca08ccdb157bc5", + "crates/rt/src/cap/mod.rs": "2e7bd2965128172e4f0f700dd8b64b161236ba4091df13e87d753bb71a71bf33", "crates/rt/src/cap/record.rs": "640b0ff60052d4775c259489af5fbff5cd03164370aceb4775186be956817bcd", - "crates/rt/src/cap/verbs.rs": "0de798ceebe67399a5994980cce2a5a49b8152639af2e3758cd84673701d7cee", + "crates/rt/src/cap/verbs.rs": "7d445d43fd71971a313d98b42e39bf4cb21ed528a1f2a15cbcd49ece02ee9007", "crates/rt/src/chan.rs": "fd50f962b23c2bfa15e72a1800790475d5cf6fcc55503ed39269852f025397e5", "crates/rt/src/clock/aarch64.rs": "47828a573f5fc30a988f012a648a8cb55596b45331cdcdfe1b390cd1d8543339", "crates/rt/src/clock/mod.rs": "d402d793448359338a361563a0fea91ea43d0264a5eb38c75b9abbb87df89af9", "crates/rt/src/clock/riscv64.rs": "17d4ecdb2b4f8152c0cf51f71ce28fe331d929a62405810a1b7eb332bb275c3d", "crates/rt/src/clock/x86_64.rs": "ca499d4dc51b7ed0eb196b831295b5fdecb09a1bd2df628a82e749e27011994a", "crates/rt/src/core_id.rs": "cadde39801a1d0ed05d70d0634a9e52d58043878737d790c4130183751b622cf", - "crates/rt/src/crash.rs": "c750a44492cd981bef6a55c6e0891ef1039e1663f3959e2a5f5289fb354ffe86", - "crates/rt/src/entry.rs": "c08657dd5fa6d88774173730db2501c73a4186aff2689ff866125c2e2189260f", + "crates/rt/src/crash.rs": "b6463a490f80124e22fdb5e13652bbaa94461eb1c4aefcffeca2e7dc7e977611", + "crates/rt/src/entry.rs": "ec5a610c79a1d5d7f02316bf0b2e77b8203a4d7a09c20f7312da73a7aa7c807d", "crates/rt/src/event.rs": "709a5500231735ad5bdb9ab9ab2fcee7b8f72f93e1b4741afba72fdc8b7c54f3", "crates/rt/src/executor/mod.rs": "071e3dc66bca448e6dabf96976702600bc77fdc98933d58847662fb2de073cb8", "crates/rt/src/executor/task.rs": "b3017cb97f287b8796c61d245c49e5c1b0ebfabe8c535cc2b3842cdbc854dc29", @@ -229,23 +229,23 @@ "crates/rt/src/room/mod.rs": "c6f56dda5064ebe75ee85ef122d82c81a507f3fe498b6f9a8463a977a0566c04", "crates/rt/src/room/x86_64.rs": "122e283074fa754a49c73beb733a92660a5d8c53701b5223297ded797c61fc70", "crates/rt/src/service/mod.rs": "b59017ca341af1f2267dfc1b06d1f11a6e505255b8c327bccee1dfe5d04356fe", - "crates/rt/src/session/builder.rs": "3c08d7a5d4c9c085f11163b4fbf715f31fd1895e0ba67869d06ce73a65401255", + "crates/rt/src/session/builder.rs": "4bea448f727d93bca9f956d7d4b5dda6982390ea616d06fb42b0804281e803b1", "crates/rt/src/session/flows.rs": "7205afbbe1d859868b313a24dccff8902e6ba3c1c8c45031e9531f6286259cae", "crates/rt/src/session/mod.rs": "466c2fbd8af228b3277075ad90a6d81031c1cb0a4c877f75669fff80e15338ea", "crates/rt/src/session/slots.rs": "82594125aa491429ed8c10b1316c19a82a162dc99f6dc97cdaa76398b92558e9", - "crates/rt/src/session/tests.rs": "f0f0ccce35ec87e133e3dcd2d29c7f32790e4be094d31aa113efe4010caf03f2", + "crates/rt/src/session/tests.rs": "80aa5e5d3b0ace53cbbb7a8e9f5bd8760d20f93e3655021172319e88097961ae", "crates/rt/src/spmd.rs": "ebaaa608c8177ed80091215b2b00240ae0d689e620f5fd418c860e78034b7e4b", "crates/rt/src/stack_arena.rs": "b252d45bfe845038ca9c8dc32941c634e448a2150dd49cf130e537048602f77b", "crates/rt/src/star.rs": "a6cf5ff1cbf01f18bb3605e3597da6921ca611da087a3a30ecd61cd026ce6d30", - "crates/rt/src/startup.rs": "b1106a10e82659b8526ba7c94b7e40849edfdcbc30b077a77242712da8ebbd93", - "crates/rt/src/svc/client.rs": "72fb7d66de1981ae253512a1f2026922ce618c9c34eb46ccb47bde465a7fecea", - "crates/rt/src/svc/mod.rs": "a0f3497792d67233da0c96a31a33fe9d291f3ae870fc5570a3940e559795406d", + "crates/rt/src/startup.rs": "a005158737d337ea8c270a2860d09762048347e65d2b937734f8b59e3890d324", + "crates/rt/src/svc/client.rs": "062b52a0544eb11b9d4f8a938ebe75299bfd6c2ff7f42a1d8d21e8cb487c16d7", + "crates/rt/src/svc/mod.rs": "3c62c81946943268c24aee3a0a8e3778442a5475dc7bf3bdb42780aeaaa958e9", "crates/rt/src/svc/recyclable.rs": "9e5af8ad316bdd89e001fef7fbb6914b5a936dcdfbab2122740e03c1cf0ee7ec", "crates/rt/src/svc/serve.rs": "a22172638a94e3ab52b88095dff9d78c88a1d693c81c6242af4932881d268706", "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", - "crates/rt/src/thread.rs": "33c50deece11724565c0d5ca0f8843d41156e6b03123f1f565585c1be1ddc8b7", - "crates/rt/src/thread_controller.rs": "d64362e7e42f310b5fb7ad61c166938b05ee9edf2cedf7742ca6eba59a222137", + "crates/rt/src/thread.rs": "4ee436e8828038c1dcecc196b631c20e4804e787585eaf7c532f85f8353c450d", + "crates/rt/src/thread_controller.rs": "aef46a87fc013df9139f9be508ac21da970d906f2a0e8e3549ea05072f1ce3ae", "crates/rt/src/tls.rs": "85bd95bafb3de73d249d7a2eb8779b5afc929077e27b4baa5f7c7428063063ab", "crates/rt/src/transport/mod.rs": "d5af890c800dd53abdbea8329383068cf43f0bf294ccb6b46dd7d8592ae5f321", "crates/rt/src/upcall.rs": "c0cbd2a59479dfba006850bb1bc5dd4f20f59317d178af776ba678f0e6404f02", @@ -295,13 +295,13 @@ "drivers/nvme/src/regs.rs": "d9410b5c64bb70f99e042f7209119c0f0fdb57f859c6a1c07eba90199f1e98c1", "drivers/nvme/src/swap.rs": "f5e1099b83b2cdeb9d288f4d337f1a234d9dfd16c31e9d9303c10f6f14972141", "services/block/Cargo.toml": "2729b36369486a6f5c1c29200eaa4603b25c26352f06f52301b70d9d57260e43", - "services/block/boot.toml": "a6d8defc624013f6edd2f489cda144375b9e5d94bbaf25f11efd24ecb0ebc68f", + "services/block/boot.toml": "b39c8ff266d2110b706d887770421a732b47ab99b1e84c805a840e439c71511e", "services/block/src/client.rs": "6aed79a404620f89ffa13078f615e846fdf6f3db5629c5cee5fbccbeaa7636cf", "services/block/src/lib.rs": "cf45960460cf6c9dba72f7a6727b51497275c686166a90be531d675097c275f1", "services/block/src/service.rs": "191e9d887d3f3267d6e2aa96e81996f01321bb2febb85e68581159f6e098b4b5", "services/block/src/wire.rs": "a9d64aef9ec3d58d37ca55b841933fc51b76ab94e18b245278b4519e96c65939", "services/dagfs/Cargo.toml": "d229a126616b1d7638f0f5a6edb4ac37d9d116f32afaa1897ffe484fbb286e5e", - "services/dagfs/boot.toml": "60d7176256fbdf96b7d8f6025c8346db9f847f806842bc58b5cfb2432a3051aa", + "services/dagfs/boot.toml": "cfd9a5931d61fd8c4f0240b312d434002d9e33f85da3e9c607dbb26eb7d0c240", "services/dagfs/examples/fsdemo.rs": "1666a14ef62fd4fc46f7f9efa2eaa5021ed79823f273a77f53758307f884e7d0", "services/dagfs/examples/fswire.rs": "994b97d699796b317741e2c87cbe812cc1776ddcd01173323e72a5f74c8ac267", "services/dagfs/examples/model_volume.rs": "0e3623e962da02b2126062a19d198b9e0a8f6e91896fd9a6f7d13f6f30ddc5f1", @@ -343,10 +343,10 @@ "services/dagfs/tests/provider.rs": "ba4070af34ae034c109e391e97bcc4755c6bd70b8e2fb229aaf9d747fef71e2d", "services/dagfs/tests/query_experiment.rs": "6fe147810474b6754ef31ad0fb95493e0b3c0886fbbafd78deb3729d410e8155", "services/device/Cargo.toml": "8f35c9ddd701b58a538f151e95e93ce618f0b346960491adad6d843287165d1c", - "services/device/boot.toml": "18b640a44bda45b29f3fb381477500a989c9f94b77dcb69eac0ec79dddbeed36", + "services/device/boot.toml": "44fc18ed82e7b3e07bcdc0203953dc6ae9f849b762d27b9735f384ab2bf96319", "services/device/src/client.rs": "71660deb4a9f51edb7a72fbb3edac94170bf4cccf9ce182cbd478ee3daf406eb", "services/device/src/lib.rs": "215c7c7e81d5d51664ac77b72b2edb0bab739d5a54b30bc9ad08f9dd3a3cfea4", - "services/device/src/service/mod.rs": "817a718222ab737712ba8223bdd5392d1ca59b02d1edcf119a5812d2e8f96aa4", + "services/device/src/service/mod.rs": "6038ba0b9375e3c6bbb25c49bf0849a5878a402f4a3a691dc9df1494849d252b", "services/device/src/wire.rs": "aaec2baa85f4d5ceab233302bd1ba8e54c158f56dcbdef3bbd59e0d99f449d18", "services/extract-worker-c/Cargo.toml": "0a84769f520d5f2735711a21efebfc93380d91724ba2ed690cfe0fe1529e1513", "services/extract-worker-c/build.rs": "66b66d671df56f116176896f7f6c9aaba008ada2d51d0b3dc1f6ca4db08e74bd", @@ -385,7 +385,7 @@ "services/extract/vendor/jsmn/jsmn.h": "babfcb7413354a43dcd539c600a1e00bd2d1579ad0a6f6c7d8b81dccedb16cd0", "services/extract/vendor/jsmn/kore_jsmn.c": "a666060a7af03c819d1faaccc0d6faa19b76bcfe78e4ef61aeab273ad052c95c", "services/mm/Cargo.toml": "25809b6a03916d96b8de64a5988d005c71c49314fafbe0d920c3ec06f2dc5951", - "services/mm/boot.toml": "d4794c49f41698733a0569a0dd0354d17238321b0048ad3f3a348612784f0cfc", + "services/mm/boot.toml": "461ca1cf03071b333bc2d7a11ba3be273fa4bff923e712b194e263f53c915539", "services/mm/src/client.rs": "c6dcc0dc3ed59e0a4cf7532fd306af4c1dc4d4e5ad1edd1d86887def791c5aa6", "services/mm/src/heap.rs": "e55210f3609f2e3e90f30d5497355ddab5801acc60ccbe3f8acbbffff71dbfde", "services/mm/src/lib.rs": "d3820740666ed993e547ee1b3b579231bf439a813d9766860e3b1b57fe815bd6", @@ -397,7 +397,7 @@ "services/output/src/service/mod.rs": "d80c3beb50979b3e1e500abdfa3c0b9d38e354c24e2e34aea443732abbead6f1", "services/output/src/wire.rs": "3752e994eb18714518facae1b78a9ee2b34f99aad8687653f0b89dc8fdeca621", "services/swap/Cargo.toml": "9c0af01403a6ad5a7f4ff69a02b18c8b1b2624a684814d6dc1ac07968c5bbe53", - "services/swap/boot.toml": "046fbdab1a4d21d6a2e623b15a52d38b841fbb44b0e4f70148c2b16356af93d3", + "services/swap/boot.toml": "1638a88d04cfec0a5a16c7e61f7a43e2284991e63ca253b84ca38a84dc7d2c23", "services/swap/src/client.rs": "04f735d059b491765d5ba63a4f5f640332735bf336e62dcbae86c17647583e2c", "services/swap/src/extent.rs": "d6705f77951df1c42cda51fc4526ba6262cab6af4a0545405ce06d2d2e1db7b4", "services/swap/src/lib.rs": "883684df54a87ae66aa7895ea19a872cabc8ab7fd4aded4ea65b7e3dd51fe0cf", diff --git a/native/services/block/boot.toml b/native/services/block/boot.toml index dc9f49e..1b02999 100644 --- a/native/services/block/boot.toml +++ b/native/services/block/boot.toml @@ -12,5 +12,5 @@ chain = true endowment = "kore_svc_block::BlockEndowment" protocol = "kore_svc_block::BlockService" endows = ["DMA_POOL@1"] -digest_x86_64 = "blake3:c852d96c707ed0e95aad161be935fcc8bee5310598ec94f0f3bc75849cf586b0" -digest_aarch64 = "blake3:98023e83300526b637f67f62bafb1b523cd8657dabe4c980f5133f282d6f3ed0" +digest_x86_64 = "blake3:c55dc025414bbcd3a511a3072095c412aedfee76d69f22bedeb51859bc30ec6b" +digest_aarch64 = "blake3:807f59729471b0b98d09a5b8c8e1d1931986475835c116aaee136fbe5843fdc4" diff --git a/native/services/dagfs/boot.toml b/native/services/dagfs/boot.toml index fe9b8b5..2ba36ca 100644 --- a/native/services/dagfs/boot.toml +++ b/native/services/dagfs/boot.toml @@ -47,5 +47,5 @@ exec_client = true # `dagfs_heap_supply_holds_every_session` holds this row to it; the rest is # room for the proofs open handles carry, allocated per open at their depth. heap_supply_bytes = 0x00D0_0000 -digest_x86_64 = "blake3:2f00411e60e4c43e73c1290e225c8d1908f27e3ff36bf1d3be7709200bb5c454" -digest_aarch64 = "blake3:d8ce59acf3bdf5bb66912e3be2477c3a1de50c97614f66c83138f3fce1995680" +digest_x86_64 = "blake3:88925d674abc8a20dd326c89e6ef95e68f0ac98624520a8cc29213d402cee329" +digest_aarch64 = "blake3:6ce4008a3a308dc0fce3bde1d0c4ee18e650b321c6c683854c5bd55e3606e0da" diff --git a/native/services/device/boot.toml b/native/services/device/boot.toml index 80737c4..8e20284 100644 --- a/native/services/device/boot.toml +++ b/native/services/device/boot.toml @@ -47,5 +47,5 @@ protocol = "kore_svc_device::DeviceBroker" # row's own claimant band. instances = 8 endows = ["ROOT_COMPLEX@0", "PLATFORM_BUS@0", "FACE_SUPPLY@0"] -digest_x86_64 = "blake3:6eb5411353e5cf48b9c9497d3b4868207bb8bbade4827537be4a7af3492d4a74" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:cd930094ae3f3fe8145f9a8fd61e5c522bc39b4956f4f45aadd1a66726aa3098" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:7915aee8c49e13907542815f1f4627ab41f656933a8067e5b247f03b55b0bae2" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:a30ae8ec2936590328807fc1522c906d6ab5afb473437547821899f5df7db4e8" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/device/src/service/mod.rs b/native/services/device/src/service/mod.rs index 3e1cf26..4e0e6b1 100644 --- a/native/services/device/src/service/mod.rs +++ b/native/services/device/src/service/mod.rs @@ -3,7 +3,7 @@ use kore_abi::device::{DeviceRecord, DeviceSpecies}; use kore_abi::session::SlotRef; -use kore_abi::task::{FACE_RING_STRIDE, FACE_SUPPLY_BYTES, role}; +use kore_abi::task::{CLAIMANT_FACE_SUPPLY_BYTES, FACE_RING_STRIDE, role}; use kore_rt::cap::{PlatformBusToken, RootComplexToken, SupplyToken, verbs}; use kore_rt::entry::FixedSlotBand; use kore_rt::service::{ @@ -34,8 +34,11 @@ pub const DEVICE_GRANT_RIGHTS: Rights = verbs::DEVICE_OWNER_RIGHTS; /// Claimant faces the broker's [`role::FACE_SUPPLY`] band strides into — /// the geometry contract's own quotient, so the count can never disagree -/// with the span the founder mints. -pub const FACE_COUNT: usize = (FACE_SUPPLY_BYTES / FACE_RING_STRIDE) as usize; +/// with the span the founder mints: one face per driver row +/// ([`MAX_CLAIMANT_FACES`](kore_abi::device::MAX_CLAIMANT_FACES)). +pub const FACE_COUNT: usize = (CLAIMANT_FACE_SUPPLY_BYTES / FACE_RING_STRIDE) as usize; + +const _: () = assert!(FACE_COUNT == kore_abi::device::MAX_CLAIMANT_FACES); /// The first minted face's [`Serve::via`] identity. Minted vias start at /// [`MAX_SERVE_PAIRS`](kore_rt::svc::MAX_SERVE_PAIRS) — the host's rule @@ -59,6 +62,14 @@ const FACE_LAND_BASE: u32 = kore_abi::task::MAX_BOOT_HANDLES as u32 + kore_rt::entry::SERVE_SCRATCH_SLOTS as u32 + 2; const FACE_LAND_END: u32 = FACE_LAND_BASE + 2 * FACE_COUNT as u32; +// `claim_fixed_slot_band` admits nothing at or past the startup-link band, +// so a band that crosses it refuses readiness (detail 0x204) on every +// machine. Checked here, where the face count and the band meet. +const _: () = assert!( + FACE_LAND_END <= kore_rt::entry::STARTUP_LINK_SLOT_BASE, + "the claimant landing band must end below the startup-link band" +); + /// A conveyed face leg must at least be postable by the broker. const FACE_LEG_RIGHTS: Rights = Rights::ACCESS; @@ -915,11 +926,11 @@ impl SystemService for DeviceBroker { if ::SERVE_PAIRS > 1 { return Err(StartFailure::new(0x205)); } - if supply.len() != FACE_SUPPLY_BYTES { + if supply.len() != CLAIMANT_FACE_SUPPLY_BYTES { return Err(StartFailure::new(0x200)); } let win = verbs::map_held_rw(supply.slot()).ok_or_else(|| StartFailure::new(0x201))?; - if win.len() < FACE_SUPPLY_BYTES as usize { + if win.len() < CLAIMANT_FACE_SUPPLY_BYTES as usize { return Err(StartFailure::new(0x202)); } let band = rt @@ -1178,7 +1189,7 @@ impl SystemService for DeviceBroker { // image face table's recycled-slot hygiene. // SAFETY: the view is live for the task's life // (mapped once at start, length gated ≥ - // FACE_SUPPLY_BYTES there); this face is not Live, + // CLAIMANT_FACE_SUPPLY_BYTES there); this face is not Live, // so no attached pair aliases this stride // (occupancy is the exclusion discipline); the // `&mut` ends before the spec is exposed. @@ -1370,11 +1381,15 @@ mod face_tests { } /// The band geometry quotient: the face count times the stride is - /// exactly the supply span the founder mints (`FACE_SUPPLY_BYTES`), - /// and the landing band holds exactly two slots per face. + /// exactly the supply span the founder mints + /// (`CLAIMANT_FACE_SUPPLY_BYTES`), and the landing band holds exactly + /// two slots per face. #[test] fn the_face_geometry_is_the_contracts_quotient() { - assert_eq!(FACE_COUNT as u64 * FACE_RING_STRIDE, FACE_SUPPLY_BYTES); + assert_eq!( + FACE_COUNT as u64 * FACE_RING_STRIDE, + CLAIMANT_FACE_SUPPLY_BYTES + ); assert_eq!(FACE_LAND_END - FACE_LAND_BASE, 2 * FACE_COUNT as u32); } } diff --git a/native/services/mm/boot.toml b/native/services/mm/boot.toml index 094ac5a..51a97ec 100644 --- a/native/services/mm/boot.toml +++ b/native/services/mm/boot.toml @@ -54,5 +54,5 @@ supply_quota_bytes = 27917287424 # 26 GiB endows = ["MEM_SUPPLY@0", "MEM_REGION@0", "SYSTEM_OBSERVE@0"] # review mirror; asserted === PROFILE.roles shell_client = true exec_client = true -digest_x86_64 = "blake3:0c893dbdf58c6c17dd66d424ad025ae0cc5b4eab41a718940f09a3595c32830a" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:f1ffc4cb9f02cc59ef78781fd8a1cba3722d9518b7abb0adb17bf3f7376b9f57" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:f46d34fd54f1190518157c9b634cf9b5191ec174271dc5f8007bbf8bad824657" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:b871832aa6a5ca4b4d6fee6beed6e0b1625d96bc019c996da1b02eb28c0bec58" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/swap/boot.toml b/native/services/swap/boot.toml index b433dd4..77f463e 100644 --- a/native/services/swap/boot.toml +++ b/native/services/swap/boot.toml @@ -9,5 +9,5 @@ image = "kore-svc-swap" endowment = "kore_svc_swap::ZramEndowment" protocol = "kore_svc_swap::ZramSwap" endows = ["MEM_SUPPLY@0"] -digest_x86_64 = "blake3:5f77376810266176274a0af803d6b16bb9d750d97780b171e5fba376fcbb58fc" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:b939b2b8bd153674b2bf76dfa83e0941cc3f16139c721c5271f6a75b6fea0874" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:9a81892b71b57bc7cea7b2797749293c18ea2fd9cf231bf11d5742d9602c7e7e" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:7cd848899239fa79c8038af480783b1ccf1772488d2b31d72656c6bd587ef06d" # res/aarch64/; make boot-digests-aarch64 From 373a700b55bdc7279fc78742293896af485af9a7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 19:43:36 +0000 Subject: [PATCH 02/18] c: libkore.so, the C ABI as a shared object and an independent program's host ## Why `kore.h` was reachable only from inside a Rust program: `kore-ffi`'s callbacks link into a Rust-founded host, and no crate built a shared object. A C program a compiler writes on Kore (`tcc` at the shell) had no runtime to call into. Nothing gave it a heap for `malloc`, a founded executor for its filesystem session, or a way to stop its threads when it ends. ## What - `crates/c` (`kore-c`) builds `libkore.so`, which a C program links with `-lkore`: - It exports `kore.h`'s functions and `c-stdlib`'s C names (`malloc`, `pthread_create`, ...). - It installs the process heap over the founder's `HEAP` supply. - A panic ends the program with its record published. - The program host (`program.rs`), entered from the C library's start (`__kore_program_start`): - `main` runs on a main thread, a `DoorbellWorker` with a 1 MiB guarded stack (`KORE_MAIN_STACK_BYTES`, `tcc`'s measured size). - Thread 0 keeps the executor and the service faces. It serves what the program's threads ask through one cell and one word, the shape `kore-lua`'s VM bridge has. - Every park is a doorbell park. - `kore_fs_run(step, user, &error)` (`kore/program.h`) runs a `KoreFsStep` program as the program's filesystem session. - It uses the launch's own face and the directory the program was launched in, through `kore-ffi`'s `Session`, attached on first use. - The main thread waits; the callbacks run on the host, as on any native executor. - Any thread ends the program: - The C library's `exit` and `_Exit` call `__kore_program_exit` after their `atexit` handlers, and the first request wins. - The host waits for a main thread that asked, then raises kore-rt's process barrier, which runs the finalizers only after a clean stop. - It ends the process with the C status through the C library's own exit. - An end requested mid-session does not wait for the session. - A launch that cannot carry a host runs the program on thread 0, as the C library alone would. Its filesystem session is refused. - `kore-ffi`: `RunError::foreign` maps a run's failure to the foreign status and error, so the classification keeps one owner. Also a target-only `pthread_testcancel` lint. - Tools: - `generate-bindings.py` generates `kore/program.h` from `kore-c` with cbindgen 0.29.4. It includes `kore.h` and declares none of its types again. - `check-bindings.py` compiles a C consumer of it for every ISA. - `check-program.py` (in `make cross` and CI's target jobs) links `libkore.so` for every ISA. It checks a shared object with no entry point, its soname, nothing needed or undefined, static initial-exec TLS, and exactly the declared ABI exported. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- .github/workflows/ci.yml | 2 + Cargo.lock | 8 + Cargo.toml | 2 +- Makefile | 7 +- crates/c/Cargo.toml | 22 ++ crates/c/build.rs | 10 + crates/c/cbindgen.toml | 20 ++ crates/c/src/api.rs | 105 +++++++ crates/c/src/exit.rs | 114 +++++++ crates/c/src/lib.rs | 46 +++ crates/c/src/program.rs | 521 ++++++++++++++++++++++++++++++++ crates/c/tests/program.c | 30 ++ crates/ffi/src/session.rs | 15 + crates/ffi/src/thread_local.rs | 2 +- extern/c/include/kore/program.h | 58 ++++ tools/check-bindings.py | 3 +- tools/check-program.py | 97 ++++++ tools/generate-bindings.py | 11 +- 18 files changed, 1067 insertions(+), 6 deletions(-) create mode 100644 crates/c/Cargo.toml create mode 100644 crates/c/build.rs create mode 100644 crates/c/cbindgen.toml create mode 100644 crates/c/src/api.rs create mode 100644 crates/c/src/exit.rs create mode 100644 crates/c/src/lib.rs create mode 100644 crates/c/src/program.rs create mode 100644 crates/c/tests/program.c create mode 100644 extern/c/include/kore/program.h create mode 100755 tools/check-program.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94e78e0..3385b16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,8 @@ jobs: sudo apt-get update -qq sudo apt-get install -y llvm python3 tools/check-alloc-symbols.py --arch ${{ matrix.arch }} + - name: Link libkore.so and check its exported ABI + run: python3 tools/check-program.py --arch ${{ matrix.arch }} miri: runs-on: ubuntu-latest timeout-minutes: 30 diff --git a/Cargo.lock b/Cargo.lock index df51275..3967315 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -802,6 +802,14 @@ dependencies = [ "kore-window", ] +[[package]] +name = "kore-c" +version = "0.1.0-alpha.1" +dependencies = [ + "kore-ffi", + "kore-std", +] + [[package]] name = "kore-collections" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index b69f68b..ebc60ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/MicroPerceptron/libkore" [workspace] resolver = "3" -members = ["tests/consumer", "crates/ffi", "crates/entry", "crates/wasi", "crates/lua", "crates/host"] +members = ["tests/consumer", "crates/ffi", "crates/c", "crates/entry", "crates/wasi", "crates/lua", "crates/host"] exclude = ["native"] [features] diff --git a/Makefile b/Makefile index ff12454..99d83ac 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ check: python3 tools/check-native.py fmt: - cargo fmt -p kore-std -p libkore-consumer -p kore-ffi -p kore-entry -p kore-wasi -p kore-lua -p kore-host --check + cargo fmt -p kore-std -p libkore-consumer -p kore-ffi -p kore-c -p kore-entry -p kore-wasi -p kore-lua -p kore-host --check $(ZIG) fmt --check build.zig build.zig.zon crates/ffi/tests/consumer.zig extern/zig/src/root.zig extern/zig/src/allocator.zig docs: @@ -30,7 +30,9 @@ owners: python3 tools/check-owners.py < target/owners.json # Actual Kore user targets, including its hardware-float x86 ABI. A stock -# x86_64-unknown-none check alone cannot certify this interface. `kore-lua` +# x86_64-unknown-none check alone cannot certify this interface. `kore-c` is +# linked rather than checked (`tools/check-program.py`): `libkore.so`'s +# exported ABI, soname and TLS model are properties of the linked image. `kore-lua` # is here because its engine exists only on these targets — its host build # carries the logic and none of the C — so a Kore-only regression in the # amalgamation would otherwise reach a keytest before it reached a @@ -46,6 +48,7 @@ cross: -Z build-std-features=compiler-builtins-mem; \ done python3 tools/check-alloc-symbols.py + python3 tools/check-program.py python3 tools/check-entry.py # kore-host drives real image files under POSIX record locks, which Miri does diff --git a/crates/c/Cargo.toml b/crates/c/Cargo.toml new file mode 100644 index 0000000..4188243 --- /dev/null +++ b/crates/c/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "kore-c" +version = "0.1.0-alpha.1" +edition = "2024" +publish = false +description = "libkore.so: the C ABI as a shared object, and the host of an independent C program on Kore" +build = "build.rs" + +# One artifact on a Kore target: `libkore.so` (dll prefix "lib" + lib name +# "kore"), which a C program links with `-lkore`. Its consumers bind it +# dynamically, so the system holds one copy however many programs run. +[lib] +name = "kore" +path = "src/lib.rs" +crate-type = ["cdylib"] + +[dependencies] +# Named `kore-std` here: this crate's own artifact is `kore`. +kore-std = { package = "kore-std", path = "../..", features = ["heap", "fs"] } +# The C ABI itself, with the standard C allocation and thread names a C +# program calls (`malloc`, `pthread_create`, ...) exported as real symbols. +kore-ffi = { path = "../ffi", features = ["c-stdlib"] } diff --git a/crates/c/build.rs b/crates/c/build.rs new file mode 100644 index 0000000..933cd14 --- /dev/null +++ b/crates/c/build.rs @@ -0,0 +1,10 @@ +//! `libkore.so`'s link contract on a Kore target, carried by the crate so +//! every builder produces the same image: the soname its consumers record as +//! `DT_NEEDED`, and no undefined symbol, since the library binds nothing from +//! the program it links into. +fn main() { + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("none") { + println!("cargo:rustc-cdylib-link-arg=--soname=libkore.so"); + println!("cargo:rustc-cdylib-link-arg=--no-undefined"); + } +} diff --git a/crates/c/cbindgen.toml b/crates/c/cbindgen.toml new file mode 100644 index 0000000..511b74b --- /dev/null +++ b/crates/c/cbindgen.toml @@ -0,0 +1,20 @@ +language = "C" +include_guard = "KORE_PROGRAM_H" +pragma_once = true +cpp_compat = true +usize_is_size_t = true +style = "both" +no_includes = true +includes = ["../kore.h"] +autogen_warning = "/* Generated by cbindgen 0.29.4 from kore-c. Do not edit. */" +header = "/* libkore.so: what an independent C program on Kore adds to kore.h. Link with -lkore. */" +[parse] +parse_deps = false +[export] +# The C library's half of the program lifecycle (its start and exit call +# these), not an application API: the C library declares them itself. +# `KoreFsStep` is kore.h's typedef, which this header includes. +exclude = ["__kore_program_start", "__kore_program_exit", "KoreFsStep"] +[enum] +prefix_with_name = true +rename_variants = "ScreamingSnakeCase" diff --git a/crates/c/src/api.rs b/crates/c/src/api.rs new file mode 100644 index 0000000..8fa4fc8 --- /dev/null +++ b/crates/c/src/api.rs @@ -0,0 +1,105 @@ +//! The C entry points `kore/program.h` declares: what an independent C program +//! on Kore adds to `kore.h` by linking `libkore.so`. +//! +//! Compiled on every target so the header has one source of declarations; +//! off Kore there is no program to host, and each entry refuses. + +use core::ffi::c_void; + +use kore_ffi::{KoreFsError, KoreStatus}; + +/// `kore.h`'s `KoreFsStep` as a C caller may pass it: possibly null. +pub type KoreFsStep = Option; + +/// The stack `main` runs on: a guarded strip of the task's stack window, +/// committed whole when the program starts. A frame deeper than this faults +/// on the guard page below it, as a stack overflow does. +/// +/// The size of the strip `tcc`, a large C program whose depth its input +/// decides, measured its need under Kore's runtime at and runs in; ISO C's +/// own guarantee of 127 nested blocks (C11 §5.2.4.1) needs a small part of +/// it. Larger strips come out of the 16 MiB window the program's SPMD gangs +/// share, and would be committed at every launch. +pub const KORE_MAIN_STACK_BYTES: usize = 1 << 20; + +/// Run `step` as this program's filesystem session, through the filesystem +/// face its launch carried, rooted where it was launched (its working +/// directory). The callbacks and their pointers obey `KoreFsStep`'s contract: +/// each callback runs on the program's host thread, as on any native +/// executor, while the caller waits, so a callback must not depend on the +/// caller's thread-local state. +/// +/// Returns `KORE_STATUS_OK` once the callback program ended and every view it +/// left open closed. `KORE_STATUS_QUARANTINED`: a reply was lost or +/// uncertain, and no session can run again in this program. +/// `KORE_STATUS_REFUSED`: a close was refused (`*error` names why, and the +/// view stays open), the launch carried no filesystem face +/// (`KORE_FS_ERROR_UNSUPPORTED`), or the caller is not the thread `main` runs +/// on, which alone waits for the host. `KORE_STATUS_CAPACITY`: the native +/// owner could not be formed. `KORE_STATUS_INVALID_ARGUMENT`: `step` is null. +/// `*error` is written whenever `error` is not null. +/// +/// # Safety +/// `step` obeys `KoreFsStep`'s pointer, serialization and no-unwind contract; +/// `user` remains valid, and the callbacks' alone, until this returns. `error` +/// is null or valid for one aligned write. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn kore_fs_run( + step: KoreFsStep, + user: *mut c_void, + error: *mut KoreFsError, +) -> KoreStatus { + let (status, cause) = match step { + None => (KoreStatus::InvalidArgument, KoreFsError::None), + // SAFETY: the caller's contract, forwarded whole. + #[cfg(target_os = "none")] + Some(step) => unsafe { crate::program::session(step, user) }, + // No program to host off Kore. + #[cfg(not(target_os = "none"))] + Some(_) => { + let _ = user; + (KoreStatus::Refused, KoreFsError::None) + } + }; + if !error.is_null() { + // SAFETY: the caller's contract: null or valid for one aligned write. + unsafe { error.write(cause) }; + } + status +} + +#[cfg(test)] +mod tests { + use super::*; + + unsafe extern "C" fn never( + _: *mut kore_ffi::KoreFsContext, + _: *const kore_ffi::KoreFsCompletion, + _: *mut c_void, + ) { + panic!("a session ran with no program to host"); + } + + #[test] + fn a_null_step_is_an_invalid_argument_and_the_error_is_written() { + let mut error = KoreFsError::Busy; + // SAFETY: a null step names no callback; `error` is valid. + let status = unsafe { kore_fs_run(None, core::ptr::null_mut(), &mut error) }; + assert_eq!(status, KoreStatus::InvalidArgument); + assert_eq!(error, KoreFsError::None); + } + + #[test] + fn with_no_program_to_host_a_session_is_refused_and_never_runs() { + let mut error = KoreFsError::Busy; + // SAFETY: `never` is never called; `error` is valid. + let status = unsafe { kore_fs_run(Some(never), core::ptr::null_mut(), &mut error) }; + assert_eq!(status, KoreStatus::Refused); + assert_eq!(error, KoreFsError::None); + // A null error pointer is permitted. + // SAFETY: as above. + let status = + unsafe { kore_fs_run(Some(never), core::ptr::null_mut(), core::ptr::null_mut()) }; + assert_eq!(status, KoreStatus::Refused); + } +} diff --git a/crates/c/src/exit.rs b/crates/c/src/exit.rs new file mode 100644 index 0000000..027788b --- /dev/null +++ b/crates/c/src/exit.rs @@ -0,0 +1,114 @@ +//! The program's end request: one word any of the program's threads posts and +//! the host reads. The first request wins; later ones only end their thread. + +use core::ffi::c_int; +use core::sync::atomic::{AtomicU64, Ordering}; + +/// One end request. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Exit { + /// The C status: the wire's `i32`, committed as given. + pub(crate) status: c_int, + /// `exit` rather than `_Exit` or a panic: finalizers may run. + pub(crate) finalize: bool, + /// The main thread asked, and is ending itself; the host waits for it + /// before the barrier, so a program that ran on it alone stops cleanly. + pub(crate) from_main: bool, +} + +const POSTED: u64 = 1 << 63; +const FINALIZE: u64 = 1 << 33; +const FROM_MAIN: u64 = 1 << 32; + +impl Exit { + fn word(self) -> u64 { + POSTED + | (u64::from(self.finalize) * FINALIZE) + | (u64::from(self.from_main) * FROM_MAIN) + | u64::from(self.status as u32) + } + + fn from_word(word: u64) -> Option { + (word & POSTED != 0).then_some(Self { + status: word as u32 as c_int, + finalize: word & FINALIZE != 0, + from_main: word & FROM_MAIN != 0, + }) + } +} + +/// Zero until the first request, then that request, for the program's life. +pub(crate) struct ExitWord(AtomicU64); + +impl ExitWord { + pub(crate) const fn new() -> Self { + Self(AtomicU64::new(0)) + } + + /// The request, once one was posted. + pub(crate) fn posted(&self) -> Option { + Exit::from_word(self.0.load(Ordering::Acquire)) + } + + /// Ask for `exit`. `false` when another request was first; the word keeps + /// that one. + pub(crate) fn post(&self, exit: Exit) -> bool { + self.0 + .compare_exchange(0, exit.word(), Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_request_round_trips_every_status_and_flag() { + for status in [0, 1, 42, -1, c_int::MIN, c_int::MAX] { + for finalize in [false, true] { + for from_main in [false, true] { + let exit = Exit { + status, + finalize, + from_main, + }; + let word = ExitWord::new(); + assert_eq!(word.posted(), None); + assert!(word.post(exit)); + assert_eq!(word.posted(), Some(exit)); + } + } + } + } + + #[test] + fn the_first_request_wins() { + let word = ExitWord::new(); + let first = Exit { + status: 3, + finalize: true, + from_main: false, + }; + let second = Exit { + status: 7, + finalize: false, + from_main: true, + }; + assert!(word.post(first)); + assert!(!word.post(second)); + assert_eq!(word.posted(), Some(first)); + } + + #[test] + fn a_status_of_zero_is_still_a_request() { + let word = ExitWord::new(); + let exit = Exit { + status: 0, + finalize: false, + from_main: false, + }; + assert!(word.post(exit)); + assert_eq!(word.posted(), Some(exit)); + } +} diff --git a/crates/c/src/lib.rs b/crates/c/src/lib.rs new file mode 100644 index 0000000..0de6dd6 --- /dev/null +++ b/crates/c/src/lib.rs @@ -0,0 +1,46 @@ +//! `libkore.so`: libkore's C ABI as a shared object, and the host of an +//! independent C program on Kore. +//! +//! A C program a compiler wrote on Kore (`tcc` at the shell) links this +//! library with `-lkore` and includes `kore.h`. From one image the whole +//! system shares, it then has: +//! +//! - `kore.h`'s functions (`kore-ffi`), and the standard C names that +//! `kore/stdlib.h` and `kore/pthread.h` alias, as real symbols (`malloc` +//! to `free`, `pthread_create` to `pthread_exit`), over the process heap this +//! library installs; +//! - a host: the program's `main` runs on a main thread with a C stack, while +//! thread 0 serves its filesystem session ([`api::kore_fs_run`], declared +//! in `kore/program.h`) and ends the process when any thread exits. +//! +//! This is a C program's runtime, never a Rust program's: a Rust executable +//! links `kore-std` with its own entry, and two runtimes in one process would +//! grow one heap supply behind two cursors. +#![cfg_attr(target_os = "none", no_std)] +#![cfg_attr(target_os = "none", feature(thread_local))] +#![deny(unsafe_op_in_unsafe_fn)] + +// The C ABI's exports are this image's reason to exist; naming the crate keeps +// them in the link. +use kore_ffi as _; + +pub mod api; +#[cfg(any(test, target_os = "none"))] +mod exit; +#[cfg(target_os = "none")] +mod program; + +/// The process heap over the founder's `role::HEAP` supply, grown on demand. +/// The program's allocations (`malloc`, `kore_alloc_*`) and this library's +/// own draw from it; with no supply every allocation answers null. +#[cfg(target_os = "none")] +#[global_allocator] +static PROCESS_HEAP: kore_std::runtime::heap::Heap = + kore_std::runtime::heap::Heap::new_in(kore_std::runtime::heap::HeapSupplySource); + +/// A panic in this library ends the program, recorded; never a silent park. +#[cfg(target_os = "none")] +#[panic_handler] +fn panic(info: &core::panic::PanicInfo<'_>) -> ! { + program::panicked(info) +} diff --git a/crates/c/src/program.rs b/crates/c/src/program.rs new file mode 100644 index 0000000..96b5946 --- /dev/null +++ b/crates/c/src/program.rs @@ -0,0 +1,521 @@ +//! The host of an independent C program: `libkore.so`'s half of the program +//! lifecycle Kore's C library starts and ends. +//! +//! A C program on Kore links its C library's start files whatever else it +//! links. Their `_start` hands the program to the C library, which, when the +//! program also links this library, hands it on to [`__kore_program_start`]. +//! From there: +//! +//! - **The program runs on its main thread**, a worker with a C stack of +//! [`KORE_MAIN_STACK_BYTES`](crate::api::KORE_MAIN_STACK_BYTES), not on +//! thread 0. Thread 0's stack is the +//! founding's, sized for a runtime, and thread 0 is the process owner: the +//! one thread that may stop the others. +//! - **Thread 0 is the host.** It keeps the task's executor and its service +//! faces, and serves what the program's threads ask of it through one +//! request cell and one word, the shape `kore-lua`'s VM bridge and `tcc` +//! share: the program's filesystem session ([`kore_fs_run`](crate::api::kore_fs_run)) +//! and its end. +//! - **Any thread can end the program.** `exit` and `_Exit` reach +//! [`__kore_program_exit`] once the C library has run its `atexit` +//! handlers; the first request wins. The host stops every other thread, +//! runs the finalizers when none was still running, and ends the process +//! with the C status through the C library's own exit. +//! +//! ```text +//! thread 0 (the host) main thread (C) +//! ─────────────────── ─────────────── +//! found the main thread ────────────────────▶ program(): main(), then exit +//! park on the wake channel … kore_fs_run(step, user) +//! ◀──────── SESSION_POSTED, kick_parent ───── futex::wait on the word +//! run the session (its C callbacks run here) +//! SESSION_SERVED, kick ─────────────────────▶ returns the session's status +//! … exit(status): atexit handlers +//! ◀──────── the exit word, kick_parent ────── the thread ends +//! join it, raise the barrier, finalize +//! end(status): the C library's exit +//! ``` +//! +//! Every park is a doorbell park; nothing here spins. The ordering is +//! `kore-lua`'s: a poster stores, then kicks; the host stands its post, +//! re-reads both words, then parks, so a kick that found no post standing +//! announced a store the re-read sees. Every thread of the program can kick +//! the host: the main thread holds the wake channel's send, and each thread it +//! starts is founded holding one too (`thread_controller`). +//! +//! A launch that cannot carry the host (no channel space or heap, a stack +//! window that cannot place the main thread) runs the program on thread 0, as +//! the C library alone would: allocation, threads, time and random work, a +//! filesystem session is refused, and only thread 0's exit ends the program. + +use core::cell::{Cell, UnsafeCell}; +use core::ffi::{c_int, c_void}; +use core::future::poll_fn; +use core::pin::pin; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use core::task::Poll; + +use kore_ffi::{KoreFsError, KoreFsStep, KoreStatus, Session}; + +use crate::exit::{Exit, ExitWord}; +use kore_std::ExitCode; +use kore_std::abi::session::Status; +use kore_std::fs::Fs; +use kore_std::runtime::chan::{NoticeRx, RecvError}; +use kore_std::runtime::entry::{self, Rt}; +use kore_std::runtime::executor::{self, Executor}; +use kore_std::runtime::futex::{self, WaitOutcome}; +use kore_std::runtime::thread::{self, DoorbellWorker, WorkerStack}; +use kore_std::runtime::transport::TrapTransport; +use kore_std::runtime::{crash, startup}; + +/// The C library's program: runs `main`, then `exit`s with its value. +pub type Program = unsafe extern "C" fn() -> !; +/// The C library's exit: commits `status` on the calling thread's own handle. +pub type End = unsafe extern "C" fn(c_int) -> !; + +/// The program, as [`__kore_program_start`] received it, for the main thread. +static PROGRAM: AtomicUsize = AtomicUsize::new(0); +/// Set by the first [`__kore_program_start`]; a second never founds a host. +static STARTED: AtomicBool = AtomicBool::new(false); +/// The host serves: the main thread exists and thread 0 parks for it. Set +/// before the main thread is founded, since its first call may need it. +static HOSTED: AtomicBool = AtomicBool::new(false); + +/// This thread's part in the program. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Role { + /// A thread the program started, or thread 0 before the program runs. + Other, + /// Thread 0, once it holds the program: the host, or the program itself + /// when the launch could not carry a host. + Host, + /// The thread `main` runs on. + Main, +} + +#[thread_local] +static ROLE: Cell = Cell::new(Role::Other); + +// ── The end ───────────────────────────────────────────────────────────── + +/// The program's end, once asked for (see [`ExitWord`]). +static EXIT: ExitWord = ExitWord::new(); + +// ── The session cell ──────────────────────────────────────────────────── + +// The word's states. The main thread owns the cell at IDLE and SERVED, the +// host at POSTED; every hand-over is a Release store matched by an Acquire. +const IDLE: u32 = 0; +const POSTED: u32 = 1; +const SERVED: u32 = 2; + +struct Request { + state: AtomicU32, + payload: UnsafeCell, +} + +struct Payload { + // main thread → host: the session's callback program. + step: Option, + user: *mut c_void, + // host → main thread: how the session ended. + status: KoreStatus, + error: KoreFsError, +} + +// SAFETY: `payload` is touched only by the side the word hands it to (above). +unsafe impl Sync for Request {} + +static SESSION: Request = Request { + state: AtomicU32::new(IDLE), + payload: UnsafeCell::new(Payload { + step: None, + user: core::ptr::null_mut(), + status: KoreStatus::Ok, + error: KoreFsError::None, + }), +}; + +impl Request { + /// # Safety + /// The caller observed (Acquire) a state that hands it the cell, and holds + /// no other reference across the store that hands it back. + #[allow(clippy::mut_from_ref)] + unsafe fn payload(&self) -> &mut Payload { + unsafe { &mut *self.payload.get() } + } +} + +/// Anything the host must answer: an end, or a session. +fn pending() -> bool { + EXIT.posted().is_some() || SESSION.state.load(Ordering::Acquire) == POSTED +} + +// ── Entry points the C library calls ──────────────────────────────────── + +/// Run the program. The C library's start hands it here when the program +/// links libkore: `program` runs `main` and then `exit`; `end` is the C +/// library's exit, which commits a status on the calling thread's own handle. +/// Never returns. +/// +/// # Safety +/// Called once, by the C library's start on the founded thread and before the +/// program has run. Neither `program` nor `end` returns. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn __kore_program_start(program: Option, end: Option) -> ! { + let (Some(program), Some(end)) = (program, end) else { + // A C library that hands over nothing to run has broken its half of + // this contract; say so rather than run on. + thread::exit(ExitCode::INTERNAL) + }; + if STARTED.swap(true, Ordering::AcqRel) { + // A second start founds no second host. + unsafe { program() } + } + ROLE.set(Role::Host); + PROGRAM.store(program as usize, Ordering::Release); + match Founded::found() { + Some(host) => host.serve(end), + None => unsafe { program() }, + } +} + +/// End the program with `status`. The C library's `exit` calls this after its +/// `atexit` handlers with `finalize` nonzero, `_Exit` with zero, on whatever +/// thread called them. +/// +/// Returns only when the caller must commit the exit itself: it is thread 0, +/// every other thread is stopped, and, for `exit`, the finalizers have run if +/// none was still running. On any other thread the request goes to the host +/// and the calling thread ends. +#[unsafe(no_mangle)] +pub extern "C" fn __kore_program_exit(status: c_int, finalize: c_int) { + let finalize = finalize != 0; + match ROLE.get() { + // Thread 0 itself: the program when there is no host, or a callback + // or finalizer the host is running. The barrier is this thread's. + Role::Host => startup::barrier(finalize), + role if HOSTED.load(Ordering::Acquire) => { + let _ = EXIT.post(Exit { + status, + finalize, + from_main: role == Role::Main, + }); + // Refused only when no post stood, and then the host re-reads the + // word before it parks: the store above reaches it either way. + let _ = thread::kick_parent(); + thread::exit(ExitCode::OK) + } + // No host: only thread 0 owns the process. Anywhere else the barrier + // refuses and ends this thread alone. + _ => startup::barrier(finalize), + } +} + +/// The main thread's half of [`kore_fs_run`](crate::api::kore_fs_run): post +/// the session to the host and wait for its answer. +/// +/// # Safety +/// `kore_fs_run`'s contract for `step` and `user`. +pub(crate) unsafe fn session(step: KoreFsStep, user: *mut c_void) -> (KoreStatus, KoreFsError) { + if ROLE.get() != Role::Main || !HOSTED.load(Ordering::Acquire) { + return (KoreStatus::Refused, KoreFsError::None); + } + // The main thread is the only poster, and it waits out each request: the + // cell is idle whenever control is here. + { + // SAFETY: IDLE hands the cell to the main thread. + let payload = unsafe { SESSION.payload() }; + payload.step = Some(step); + payload.user = user; + } + SESSION.state.store(POSTED, Ordering::Release); + // Refused only when no post stood, and then the host re-reads the word + // before it parks. + let _ = thread::kick_parent(); + loop { + match SESSION.state.load(Ordering::Acquire) { + SERVED => break, + state => { + if futex::wait(&SESSION.state, state, None) == WaitOutcome::Unsupported { + // The main thread is founded with its own doorbell, so it + // can always park; if it cannot, spinning is the one + // thing left, and this host will not. End the program. + fatal() + } + } + } + } + let answer = { + // SAFETY: SERVED hands the cell back to the main thread. + let payload = unsafe { SESSION.payload() }; + payload.step = None; + payload.user = core::ptr::null_mut(); + (payload.status, payload.error) + }; + SESSION.state.store(IDLE, Ordering::Release); + answer +} + +/// This library's panic: the record first, then the end. A panic on thread 0 +/// stops the program there; anywhere else it asks the host to, so the process +/// ends with the panic rather than running on without the panicking thread. +pub(crate) fn panicked(info: &core::panic::PanicInfo<'_>) -> ! { + crash::report(info); + match ROLE.get() { + Role::Host => startup::barrier(false), + role if HOSTED.load(Ordering::Acquire) => { + let _ = EXIT.post(Exit { + status: ExitCode::PANIC.get(), + finalize: false, + from_main: role == Role::Main, + }); + let _ = thread::kick_parent(); + } + _ => {} + } + thread::exit(ExitCode::PANIC) +} + +/// A host invariant broke on a program thread: end the program with +/// `INTERNAL`, never by spinning. +fn fatal() -> ! { + let _ = EXIT.post(Exit { + status: ExitCode::INTERNAL.get(), + finalize: false, + from_main: ROLE.get() == Role::Main, + }); + let _ = thread::kick_parent(); + thread::exit(ExitCode::INTERNAL) +} + +// ── The host ──────────────────────────────────────────────────────────── + +/// The host's wake channel broke or a park was refused: it cannot hear the +/// program, so it ends it. +struct Fatal; + +/// Thread 0 with its program founded: the runtime context, the executor's +/// doorbell, and the main thread. +struct Founded { + rt: Rt, + doorbell: executor::Doorbell, + worker: DoorbellWorker, +} + +impl Founded { + /// Found the main thread. `None` when the launch cannot carry the host. + fn found() -> Option { + let rt = entry::thread_context()?; + let doorbell = rt.doorbell()?; + HOSTED.store(true, Ordering::Release); + match thread::spawn_with_doorbell_on( + main_thread, + &[], + WorkerStack::Bytes(crate::api::KORE_MAIN_STACK_BYTES), + ) { + Some(worker) => Some(Self { + rt, + doorbell, + worker, + }), + None => { + HOSTED.store(false, Ordering::Release); + None + } + } + } + + /// Serve the program until it ends, then end the process. + fn serve(self, end: End) -> ! { + let Self { + rt, + doorbell, + worker, + } = self; + let transport = TrapTransport::new(); + let exit = { + let mut executor = Executor::new(entry::transport(), doorbell); + executor.run(host(&rt, &transport, &worker)) + }; + let exit = exit.unwrap_or(Exit { + status: ExitCode::INTERNAL.get(), + finalize: false, + from_main: false, + }); + if exit.from_main { + // The main thread asked and is ending itself: wait for it, so a + // program that ran on it alone is stopped cleanly. + let _ = worker.join(); + } else { + // Still running: the barrier stops it, and its channels end with + // the process. Its join ticket must not try to reclaim it first. + core::mem::forget(worker); + } + startup::barrier(exit.finalize); + // SAFETY: the C library's exit, as `__kore_program_start` received it. + unsafe { end(exit.status) } + } +} + +/// The main thread's entry: the program, as the C library handed it over. +fn main_thread() { + ROLE.set(Role::Main); + // SAFETY: `__kore_program_start` stored a `Program` before founding this + // thread, and the founding's session ordered that store before this load. + let program: Program = unsafe { core::mem::transmute(PROGRAM.load(Ordering::Acquire)) }; + unsafe { program() } +} + +/// The host's loop: answer each session the main thread posts, until any +/// thread asks for the end. +async fn host(rt: &Rt, transport: &TrapTransport, worker: &DoorbellWorker) -> Result { + let mut wake = NoticeRx::new(transport, worker.wake_recv()); + let mut filesystem = Filesystem::new(rt, transport); + loop { + if let Some(exit) = EXIT.posted() { + return Ok(exit); + } + if SESSION.state.load(Ordering::Acquire) == POSTED { + // SAFETY: POSTED hands the cell to the host until it stores SERVED. + let (step, user) = { + let payload = unsafe { SESSION.payload() }; + (payload.step, payload.user) + }; + let Some(step) = step else { + return Err(Fatal); + }; + // An end requested mid-session ends the program without waiting + // for the session: the barrier stops the waiting main thread, and + // the abandoned session's owner is quarantined, as cancellation + // leaves it. + let session = pin!(filesystem.run(step, user)); + let ended = pin!(until_exit(&mut wake)); + let outcome = race(session, ended).await; + let Some((status, error)) = outcome else { + continue; + }; + { + // SAFETY: still POSTED; the host owns the cell. + let payload = unsafe { SESSION.payload() }; + payload.status = status; + payload.error = error; + } + SESSION.state.store(SERVED, Ordering::Release); + let _ = worker.kick(); + continue; + } + park(&mut wake, pending).await?; + } +} + +/// Poll `first` and then `second` until one completes: `Some` with the first's +/// output, or `None` when the second finished first. +async fn race( + mut first: core::pin::Pin<&mut A>, + mut second: core::pin::Pin<&mut B>, +) -> Option { + poll_fn(|cx| { + if let Poll::Ready(output) = first.as_mut().poll(cx) { + return Poll::Ready(Some(output)); + } + if second.as_mut().poll(cx).is_ready() { + return Poll::Ready(None); + } + Poll::Pending + }) + .await +} + +/// Park until a kick, a spurious wake, or `ready` holds, having stood the +/// wake post before re-reading what `ready` reads. +async fn park( + wake: &mut NoticeRx<'_, TrapTransport>, + ready: impl Fn() -> bool, +) -> Result<(), Fatal> { + loop { + match wake.try_recv() { + Ok(Some(_)) => return Ok(()), + Ok(None) => { + if ready() { + return Ok(()); + } + let Some(post) = wake.standing_post() else { + return Err(Fatal); + }; + let _ = executor::next_wake_on(post).await; + } + Err(RecvError::Refused(Status::WouldBlock)) => { + let _ = executor::next_wake().await; + } + Err(_) => return Err(Fatal), + } + } +} + +/// Resolve once an end is posted, parking on the wake channel meanwhile: the +/// session being served is not a reason to wake, only an end is. On a broken +/// channel it never resolves, since the host can no longer hear an end; the +/// session it races then runs to its own completion. +async fn until_exit(wake: &mut NoticeRx<'_, TrapTransport>) { + let ended = || EXIT.posted().is_some(); + while !ended() { + if park(wake, ended).await.is_err() { + core::future::pending::<()>().await; + } + } +} + +/// The program's filesystem: attached on first use, one native owner for the +/// program's life. +struct Filesystem<'r, 't> { + rt: &'r Rt, + transport: &'t TrapTransport, + session: Option>, + /// The launch carried no face, or the owner could not be formed: every + /// later session is refused the same way without asking again. + refused: Option<(KoreStatus, KoreFsError)>, +} + +impl<'r, 't> Filesystem<'r, 't> { + fn new(rt: &'r Rt, transport: &'t TrapTransport) -> Self { + Self { + rt, + transport, + session: None, + refused: None, + } + } + + async fn run(&mut self, step: KoreFsStep, user: *mut c_void) -> (KoreStatus, KoreFsError) { + if let Some(refused) = self.refused { + return refused; + } + if self.session.is_none() { + let Some(fs) = Fs::attach(self.rt, self.transport).await else { + let refused = (KoreStatus::Refused, KoreFsError::Unsupported); + self.refused = Some(refused); + return refused; + }; + match Session::new(fs) { + Ok(session) => self.session = Some(session), + Err(_) => { + let refused = (KoreStatus::Capacity, KoreFsError::None); + self.refused = Some(refused); + return refused; + } + } + } + let Some(session) = self.session.as_mut() else { + return (KoreStatus::Refused, KoreFsError::None); + }; + // SAFETY: `kore_fs_run`'s caller took on `Session::run`'s contract for + // `step` and `user`, and waits until this future completes or is + // dropped with the program's end. + match unsafe { session.run(step, user) }.await { + Ok(()) => (KoreStatus::Ok, KoreFsError::None), + Err(error) => error.foreign(), + } + } +} diff --git a/crates/c/tests/program.c b/crates/c/tests/program.c new file mode 100644 index 0000000..7009957 --- /dev/null +++ b/crates/c/tests/program.c @@ -0,0 +1,30 @@ +/* kore/program.h as an independent C program uses it: a session whose + * callbacks are kore.h's own KoreFsStep, run through kore_fs_run. Compiled + * freestanding for every Kore ISA by tools/check-bindings.py. */ +#include + +struct reader { + int done; +}; + +static void open_root(struct KoreFsContext *context, const struct KoreFsCompletion *completion, + void *user) +{ + struct reader *reader = user; + if (completion->operation == KORE_OP_START) { + (void)kore_fs_open(context, (const uint8_t *)"", 0, KORE_FS_READ); + return; + } + reader->done = 1; +} + +/* The entry's exact type, so a drift in either header fails the build. */ +static KoreStatus (*const run)(KoreFsStep, void *, KoreFsError *) = kore_fs_run; + +int program_session(void) +{ + struct reader reader = {0}; + KoreFsError error = KORE_FS_ERROR_NONE; + KoreStatus status = run(open_root, &reader, &error); + return status == KORE_STATUS_OK && error == KORE_FS_ERROR_NONE && reader.done; +} diff --git a/crates/ffi/src/session.rs b/crates/ffi/src/session.rs index 6176f38..658b83f 100644 --- a/crates/ffi/src/session.rs +++ b/crates/ffi/src/session.rs @@ -25,6 +25,21 @@ pub enum RunError { Cleanup(Refusal), } +impl RunError { + /// What a foreign caller reads for a failed run, in the submission-status + /// vocabulary: `Quarantined` for an owner that may never run again, and + /// `Refused` with the refused close's error, whose view the owner kept. + pub fn foreign(&self) -> (KoreStatus, KoreFsError) { + match self { + Self::Quarantined => (KoreStatus::Quarantined, KoreFsError::None), + Self::Cleanup(Refusal::Fs(error)) => (KoreStatus::Refused, classify(*error)), + // A close answers in the filesystem's vocabulary or quarantines; + // no other refusal reaches cleanup. + Self::Cleanup(_) => (KoreStatus::Refused, KoreFsError::Unknown), + } + } +} + impl<'w, 't> Session<'w, 't> { /// Consume the real native face. Identifier exhaustion returns it unchanged. /// Close any previously opened external views before handing off this face: diff --git a/crates/ffi/src/thread_local.rs b/crates/ffi/src/thread_local.rs index d160bc3..d0e24c3 100644 --- a/crates/ffi/src/thread_local.rs +++ b/crates/ffi/src/thread_local.rs @@ -487,7 +487,7 @@ pub unsafe extern "C" fn pthread_setcanceltype(kind: i32, old: *mut i32) -> i32 /// Same contract as kore_thread_testcancel. #[cfg(all(feature = "c-stdlib", target_os = "none"))] #[unsafe(no_mangle)] -pub unsafe extern "C" fn pthread_testcancel() -> () { +pub unsafe extern "C" fn pthread_testcancel() { unsafe { crate::kore_thread_testcancel() } } diff --git a/extern/c/include/kore/program.h b/extern/c/include/kore/program.h new file mode 100644 index 0000000..ce10f4b --- /dev/null +++ b/extern/c/include/kore/program.h @@ -0,0 +1,58 @@ +/* libkore.so: what an independent C program on Kore adds to kore.h. Link with -lkore. */ + +#ifndef KORE_PROGRAM_H +#define KORE_PROGRAM_H + +#pragma once + +/* Generated by cbindgen 0.29.4 from kore-c. Do not edit. */ + +#include "../kore.h" + +/** + * The stack `main` runs on: a guarded strip of the task's stack window, + * committed whole when the program starts. A frame deeper than this faults + * on the guard page below it, as a stack overflow does. + * + * The size of the strip `tcc`, a large C program whose depth its input + * decides, measured its need under Kore's runtime at and runs in; ISO C's + * own guarantee of 127 nested blocks (C11 §5.2.4.1) needs a small part of + * it. Larger strips come out of the 16 MiB window the program's SPMD gangs + * share, and would be committed at every launch. + */ +#define KORE_MAIN_STACK_BYTES (1 << 20) + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +/** + * Run `step` as this program's filesystem session, through the filesystem + * face its launch carried, rooted where it was launched (its working + * directory). The callbacks and their pointers obey `KoreFsStep`'s contract: + * each callback runs on the program's host thread, as on any native + * executor, while the caller waits, so a callback must not depend on the + * caller's thread-local state. + * + * Returns `KORE_STATUS_OK` once the callback program ended and every view it + * left open closed. `KORE_STATUS_QUARANTINED`: a reply was lost or + * uncertain, and no session can run again in this program. + * `KORE_STATUS_REFUSED`: a close was refused (`*error` names why, and the + * view stays open), the launch carried no filesystem face + * (`KORE_FS_ERROR_UNSUPPORTED`), or the caller is not the thread `main` runs + * on, which alone waits for the host. `KORE_STATUS_CAPACITY`: the native + * owner could not be formed. `KORE_STATUS_INVALID_ARGUMENT`: `step` is null. + * `*error` is written whenever `error` is not null. + * + * # Safety + * `step` obeys `KoreFsStep`'s pointer, serialization and no-unwind contract; + * `user` remains valid, and the callbacks' alone, until this returns. `error` + * is null or valid for one aligned write. + */ +KoreStatus kore_fs_run(KoreFsStep step, void *user, KoreFsError *error); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif /* KORE_PROGRAM_H */ diff --git a/tools/check-bindings.py b/tools/check-bindings.py index b389550..2c6c9a6 100644 --- a/tools/check-bindings.py +++ b/tools/check-bindings.py @@ -21,7 +21,8 @@ for arch, c_target, zig_target in TARGETS: subprocess.run([os.environ.get('CLANG', 'clang'), '-target', c_target, '-ffreestanding', '-std=c11', '-Werror', '-fsyntax-only', '-I', 'extern/c/include', - 'crates/ffi/tests/consumer.c', 'crates/ffi/tests/alloc.c'], cwd=ROOT, check=True) + 'crates/ffi/tests/consumer.c', 'crates/ffi/tests/alloc.c', + 'crates/c/tests/program.c'], cwd=ROOT, check=True) translated = destination / f'{arch}-c.zig' with translated.open('w') as output: subprocess.run([zig, 'translate-c', '-target', zig_target, 'extern/c/include/kore.h'], diff --git a/tools/check-program.py b/tools/check-program.py new file mode 100755 index 0000000..ec2dd26 --- /dev/null +++ b/tools/check-program.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Link libkore.so for every Kore ISA and check the image a C program binds. + +The library is what an independent C program on Kore links with -lkore, so the +checks are the ones its loader and its consumers rely on: + +- a shared object with no entry point: Kore's image gate classifies a module by + its entry, and a library must never register as a program; +- the soname consumers record, nothing it needs, nothing left undefined; +- static initial-exec TLS only (Kore's startup linker resolves no dynamic TLS); +- exactly the declared ABI exported: every function kore.h and kore/program.h + declare, the C names kore/stdlib.h and kore/pthread.h alias, and the C + library's two lifecycle entry points; no other symbol. +""" +import argparse +import os +from pathlib import Path +import re +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +ARCHES = ('x86_64', 'aarch64', 'riscv64gc') +HEADERS = ('extern/c/include/kore.h', 'extern/c/include/kore/program.h') +# kore-ffi's `c-stdlib` names: allocation, the pthread lifecycle, and the +# compiler ABI's thread-exit destructor hooks. +C_STDLIB = {'malloc', 'calloc', 'realloc', 'free', 'pthread_create', 'pthread_join', + 'pthread_detach', 'pthread_cancel', 'pthread_key_create', 'pthread_key_delete', + 'pthread_setspecific', 'pthread_getspecific', 'pthread_setcancelstate', + 'pthread_setcanceltype', 'pthread_testcancel', 'pthread_exit', + '__cxa_thread_atexit', '__cxa_thread_atexit_impl'} +# The C library's start and exit call these when a program links libkore. +LIFECYCLE = {'__kore_program_start', '__kore_program_exit'} +DYNAMIC_TLS = re.compile(r'TLSDESC|DTPMOD|DTPOFF|TLSGD|TLSLD|TLS_DTP') + + +def readelf(*args): + return subprocess.check_output([os.environ.get('LLVM_READELF', 'llvm-readelf'), '-W', *args], + text=True) + + +def declared(): + names = set() + for header in HEADERS: + names |= set(re.findall(r'\b(kore_[a-z][a-z0-9_]*)\s*\(', (ROOT / header).read_text())) + if not names: + raise SystemExit('no foreign functions found in the generated headers') + return names + + +def check(arch, expected): + subprocess.run([ + 'cargo', 'build', '--locked', '-p', 'kore-c', '--release', + '--target', f'targets/{arch}-unknown-kore-user.json', + '-Z', 'build-std=core,alloc,compiler_builtins', + '-Z', 'build-std-features=compiler-builtins-mem', + ], cwd=ROOT, check=True) + image = ROOT / f'target/{arch}-unknown-kore-user/release/libkore.so' + header = readelf('--file-header', str(image)) + if not re.search(r'Type:\s+DYN', header): + raise SystemExit(f'{arch}: libkore.so is not a shared object') + if not re.search(r'Entry point address:\s+0x0\b', header): + raise SystemExit(f'{arch}: libkore.so has an entry point; it would register as a program') + dynamic = readelf('--dynamic', str(image)) + if 'Library soname: [libkore.so]' not in dynamic: + raise SystemExit(f'{arch}: libkore.so does not carry its soname') + if needed := re.findall(r'\(NEEDED\)\s+Shared library: \[([^\]]+)\]', dynamic): + raise SystemExit(f'{arch}: libkore.so needs other libraries: {needed}') + exported, undefined = set(), set() + for line in readelf('--dyn-syms', str(image)).splitlines(): + fields = line.split() + if len(fields) < 8 or not fields[0].rstrip(':').isdigit(): + continue + kind, binding, section, name = fields[3], fields[4], fields[6], fields[7] + if section == 'UND': + if name: + undefined.add(name) + elif binding in ('GLOBAL', 'WEAK') and kind in ('FUNC', 'OBJECT', 'TLS'): + exported.add(name.split('@')[0]) + if undefined: + raise SystemExit(f'{arch}: undefined symbols: {sorted(undefined)}') + if missing := expected - exported: + raise SystemExit(f'{arch}: declared but not exported: {sorted(missing)}') + if extra := exported - expected: + raise SystemExit(f'{arch}: exported outside the declared ABI: {sorted(extra)}') + relocations = readelf('--relocations', str(image)) + if found := sorted(set(DYNAMIC_TLS.findall(relocations))): + raise SystemExit(f'{arch}: dynamic TLS relocations {found}; static initial-exec TLS required') + print(f'libkore.so: {arch}: {len(exported)} exported symbols, the declared ABI exactly', flush=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument('--arch', choices=ARCHES) + args = parser.parse_args() + expected = declared() | C_STDLIB | LIFECYCLE + for arch in [args.arch] if args.arch else ARCHES: + check(arch, expected) diff --git a/tools/generate-bindings.py b/tools/generate-bindings.py index 2ba0782..04ee299 100644 --- a/tools/generate-bindings.py +++ b/tools/generate-bindings.py @@ -19,6 +19,14 @@ def generate(check=False): subprocess.run([generator, '--config', 'crates/ffi/cbindgen.toml', '--crate', 'kore-ffi', '--output', str(header)], cwd=ROOT, check=True) source = header.read_text() + # What libkore.so adds for an independent C program; it includes kore.h + # and declares none of its types again. + program_header = Path(directory) / 'program.h' + subprocess.run([generator, '--config', 'crates/c/cbindgen.toml', '--crate', 'kore-c', + '--output', str(program_header)], cwd=ROOT, check=True) + program = program_header.read_text() + if 'kore_fs_run' not in program: + raise SystemExit('missing program export: kore_fs_run') # cbindgen is the declaration authority. This step only groups exported # names; it never reproduces signatures or guesses C/Rust type layouts. functions = sorted(set(re.findall(r'\b(kore_[a-z][a-z0-9_]*)\s*\(', source))) @@ -72,7 +80,8 @@ def generate(check=False): pthread += '#define pthread_cleanup_pop(execute) kore_thread_cleanup_pop(&_kore_cleanup, (execute)); } while (0)\n' pthread += '#endif\n' for relative, data in [('extern/c/include/kore.h', source), ('extern/zig/src/root.zig', zig), - ('extern/c/include/kore/stdlib.h', stdlib), ('extern/c/include/kore/pthread.h', pthread)]: + ('extern/c/include/kore/stdlib.h', stdlib), ('extern/c/include/kore/pthread.h', pthread), + ('extern/c/include/kore/program.h', program)]: path = ROOT / relative if check: if not path.exists() or path.read_text() != data: From aafea0c8d97ca5a1a01ad8b2790c0097c4d5d7f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 19:48:28 +0000 Subject: [PATCH 03/18] c: the shared object is the distribution's cdylib over kore-c `kore-c` built `libkore.so` itself, as a `cdylib`. A distribution builds its shared libraries inside its own workspace, where its native owners are patched in. Kore does so for each one it ships, and a second `cdylib` named `kore` there would write the same `libkore.so` as this one. `kore-c` is now the library that image is built from, and the distribution owns the one-line `cdylib`: Kore's `user/runtime/libkore`, the pattern its `crates/pci-names` and `user/runtime/pci-names` already have. - `kore-c` is a library named `kore_c`. Its crate documentation states the link contract the shared object carries: the soname, no undefined symbol, no entry point, static TLS and the exported ABI. - `check-program.py` builds that `cdylib` in a temporary workspace, as `check-entry.py` builds its consumers, linked with `--soname=libkore.so --no-undefined`, and checks the image as before. Each ISA still exports exactly the declared ABI (85 symbols). - The soname and `--no-undefined` move from a build script to that link. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- crates/c/Cargo.toml | 15 +++++++------- crates/c/build.rs | 10 ---------- crates/c/src/lib.rs | 29 ++++++++++++++++++++------- tools/check-program.py | 45 +++++++++++++++++++++++++++++++++++------- 4 files changed, 67 insertions(+), 32 deletions(-) delete mode 100644 crates/c/build.rs diff --git a/crates/c/Cargo.toml b/crates/c/Cargo.toml index 4188243..c89498f 100644 --- a/crates/c/Cargo.toml +++ b/crates/c/Cargo.toml @@ -3,19 +3,18 @@ name = "kore-c" version = "0.1.0-alpha.1" edition = "2024" publish = false -description = "libkore.so: the C ABI as a shared object, and the host of an independent C program on Kore" -build = "build.rs" +description = "The runtime of libkore.so: the C ABI and the host of an independent C program on Kore" -# One artifact on a Kore target: `libkore.so` (dll prefix "lib" + lib name -# "kore"), which a C program links with `-lkore`. Its consumers bind it -# dynamically, so the system holds one copy however many programs run. +# The runtime `libkore.so` is built from: a distribution's one-line `cdylib` +# crate names this one and links it under its own `[lib] name = "kore"`, so +# the image is `libkore.so` (Kore's is `user/runtime/libkore`). Link it into +# that shared object alone: it installs the process heap and the panic handler +# a C program's runtime needs, which a Rust executable brings itself. [lib] -name = "kore" +name = "kore_c" path = "src/lib.rs" -crate-type = ["cdylib"] [dependencies] -# Named `kore-std` here: this crate's own artifact is `kore`. kore-std = { package = "kore-std", path = "../..", features = ["heap", "fs"] } # The C ABI itself, with the standard C allocation and thread names a C # program calls (`malloc`, `pthread_create`, ...) exported as real symbols. diff --git a/crates/c/build.rs b/crates/c/build.rs deleted file mode 100644 index 933cd14..0000000 --- a/crates/c/build.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! `libkore.so`'s link contract on a Kore target, carried by the crate so -//! every builder produces the same image: the soname its consumers record as -//! `DT_NEEDED`, and no undefined symbol, since the library binds nothing from -//! the program it links into. -fn main() { - if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("none") { - println!("cargo:rustc-cdylib-link-arg=--soname=libkore.so"); - println!("cargo:rustc-cdylib-link-arg=--no-undefined"); - } -} diff --git a/crates/c/src/lib.rs b/crates/c/src/lib.rs index 0de6dd6..36a22b0 100644 --- a/crates/c/src/lib.rs +++ b/crates/c/src/lib.rs @@ -1,8 +1,8 @@ -//! `libkore.so`: libkore's C ABI as a shared object, and the host of an -//! independent C program on Kore. +//! The runtime of `libkore.so`: libkore's C ABI as one shared object, and the +//! host of an independent C program on Kore. //! -//! A C program a compiler wrote on Kore (`tcc` at the shell) links this -//! library with `-lkore` and includes `kore.h`. From one image the whole +//! A C program a compiler wrote on Kore (`tcc` at the shell) links +//! `libkore.so` with `-lkore` and includes `kore.h`. From one image the whole //! system shares, it then has: //! //! - `kore.h`'s functions (`kore-ffi`), and the standard C names that @@ -13,9 +13,24 @@ //! thread 0 serves its filesystem session ([`api::kore_fs_run`], declared //! in `kore/program.h`) and ends the process when any thread exits. //! -//! This is a C program's runtime, never a Rust program's: a Rust executable -//! links `kore-std` with its own entry, and two runtimes in one process would -//! grow one heap supply behind two cursors. +//! # The shared object +//! +//! A distribution builds the image from a `cdylib` crate whose `[lib]` is +//! named `kore` and whose source names this crate (`use kore_c as _;`); Kore's +//! is `user/runtime/libkore`. Its link carries the contract consumers rely on, +//! which `tools/check-program.py` checks for every ISA: +//! +//! - `--soname=libkore.so`, the name consumers record as `DT_NEEDED`; +//! - `--no-undefined`: the library binds nothing from the program it links into; +//! - a shared object with no entry point, so it registers as a library; +//! - static initial-exec TLS only, which the Kore user targets select; +//! - exactly the declared ABI exported (`kore.h`, `kore/program.h`, the C names +//! `kore/stdlib.h` and `kore/pthread.h` alias, and the C library's two +//! lifecycle entry points). +//! +//! It is a C program's runtime, never a Rust program's: a Rust executable links +//! `kore-std` with its own entry, and two runtimes in one process would grow one +//! heap supply behind two cursors. #![cfg_attr(target_os = "none", no_std)] #![cfg_attr(target_os = "none", feature(thread_local))] #![deny(unsafe_op_in_unsafe_fn)] diff --git a/tools/check-program.py b/tools/check-program.py index ec2dd26..7fda49e 100755 --- a/tools/check-program.py +++ b/tools/check-program.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 """Link libkore.so for every Kore ISA and check the image a C program binds. +The image is a distribution's one-line cdylib over kore-c (Kore's is +user/runtime/libkore), linked with the contract kore-c documents; this builds +that crate in a temporary workspace, as check-entry.py builds its consumers. The library is what an independent C program on Kore links with -lkore, so the checks are the ones its loader and its consumers rely on: @@ -13,10 +16,13 @@ library's two lifecycle entry points; no other symbol. """ import argparse +import json import os from pathlib import Path import re import subprocess +import tempfile +import tomllib ROOT = Path(__file__).resolve().parents[1] ARCHES = ('x86_64', 'aarch64', 'riscv64gc') @@ -47,14 +53,39 @@ def declared(): return names +# The shared object as a distribution builds it: a cdylib named `kore` that +# names kore-c, linked with the soname and no undefined symbol. +WRAPPER_MANIFEST = ('[package]\nname = "libkore-so"\nversion = "0.0.0"\nedition = "2024"\n' + '[lib]\nname = "kore"\ncrate-type = ["cdylib"]\n[workspace]\n' + '[dependencies]\nkore-c = {{ path = {path} }}\n') +WRAPPER_SOURCE = '#![no_std]\n// The runtime and its exports come from kore-c.\nuse kore_c as _;\n' +LINK_CONTRACT = ['-C', 'link-arg=--soname=libkore.so', '-C', 'link-arg=--no-undefined'] + + +def build(arch): + channel = tomllib.loads((ROOT / 'rust-toolchain.toml').read_text())['toolchain']['channel'] + target_dir = ROOT / 'target/program-consumer' + env = dict(os.environ, RUSTUP_TOOLCHAIN=channel, CARGO_TARGET_DIR=str(target_dir), + CARGO_INCREMENTAL='0') + with tempfile.TemporaryDirectory(prefix='libkore-so-') as temporary: + work = Path(temporary) + (work / 'src').mkdir() + (work / 'Cargo.toml').write_text( + WRAPPER_MANIFEST.format(path=json.dumps(str(ROOT / 'crates/c')))) + (work / 'src/lib.rs').write_text(WRAPPER_SOURCE) + # Reuse the reviewed dependency versions while Cargo adds this fixture. + (work / 'Cargo.lock').write_bytes((ROOT / 'Cargo.lock').read_bytes()) + subprocess.run([ + 'cargo', 'rustc', '--release', '-Zjson-target-spec', + '--target', str(ROOT / f'targets/{arch}-unknown-kore-user.json'), + '-Zbuild-std=core,alloc,compiler_builtins', + '-Zbuild-std-features=compiler-builtins-mem', '--', *LINK_CONTRACT, + ], cwd=work, env=env, check=True) + return target_dir / f'{arch}-unknown-kore-user/release/libkore.so' + + def check(arch, expected): - subprocess.run([ - 'cargo', 'build', '--locked', '-p', 'kore-c', '--release', - '--target', f'targets/{arch}-unknown-kore-user.json', - '-Z', 'build-std=core,alloc,compiler_builtins', - '-Z', 'build-std-features=compiler-builtins-mem', - ], cwd=ROOT, check=True) - image = ROOT / f'target/{arch}-unknown-kore-user/release/libkore.so' + image = build(arch) header = readelf('--file-header', str(image)) if not re.search(r'Type:\s+DYN', header): raise SystemExit(f'{arch}: libkore.so is not a shared object') From 3476ee2b9d310b5ab575fe16bc120e3e6d630312 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 19:51:10 +0000 Subject: [PATCH 04/18] c: publish the headers libkore.so implements to the crate that links it A distribution ships `kore.h` and the `kore/` headers beside the `libkore.so` it builds. They have to come from the source revision the image was built from, not from a copy kept in the distribution. Cargo's metadata channel carries them: `kore-c` declares `links = "kore"`, and its build script publishes the generated include directory as `DEP_KORE_INCLUDE` to the dependent's build script. No native library is linked. The build script also refuses a checkout missing any of the four headers. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- crates/c/Cargo.toml | 4 ++++ crates/c/build.rs | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 crates/c/build.rs diff --git a/crates/c/Cargo.toml b/crates/c/Cargo.toml index c89498f..274ab56 100644 --- a/crates/c/Cargo.toml +++ b/crates/c/Cargo.toml @@ -4,6 +4,10 @@ version = "0.1.0-alpha.1" edition = "2024" publish = false description = "The runtime of libkore.so: the C ABI and the host of an independent C program on Kore" +# No native library is linked: the key carries the headers this runtime +# implements to dependents' build scripts as `DEP_KORE_INCLUDE` (`build.rs`). +links = "kore" +build = "build.rs" # The runtime `libkore.so` is built from: a distribution's one-line `cdylib` # crate names this one and links it under its own `[lib] name = "kore"`, so diff --git a/crates/c/build.rs b/crates/c/build.rs new file mode 100644 index 0000000..6928f56 --- /dev/null +++ b/crates/c/build.rs @@ -0,0 +1,29 @@ +//! Publish the C headers `libkore.so` implements to whoever links this crate +//! (`links = "kore"`): a dependent's build script reads `DEP_KORE_INCLUDE`, the +//! directory holding `kore.h` and `kore/`, from this exact source revision, so +//! the headers a distribution ships cannot drift from the image it builds. +use std::path::PathBuf; + +fn main() { + let manifest = + PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); + let include = manifest.join("../../extern/c/include"); + let include = include + .canonicalize() + .unwrap_or_else(|error| panic!("{}: {error}", include.display())); + for header in [ + "kore.h", + "kore/stdlib.h", + "kore/pthread.h", + "kore/program.h", + ] { + let path = include.join(header); + assert!( + path.is_file(), + "missing generated header {}", + path.display() + ); + println!("cargo:rerun-if-changed={}", path.display()); + } + println!("cargo:include={}", include.display()); +} From 2ef937776d52f1bc737704c48431bab92ae16eda Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 20:15:28 +0000 Subject: [PATCH 05/18] docs: libkore.so and the independent C program it hosts - README: `kore-c` joins the package table. `kore-ffi`'s row no longer says there is no standalone C runtime: `kore-c` is one. - `docs/languages.md`: the shared object a distribution builds, what a program that links it gets, the host's two halves (its filesystem session and the program's end), the program without a host, and the two weak entry points a C library calls. It states what `check-program.py` verifies, which evidence is boot evidence (Kore's `keytest-tcc-x86`) and which is not, and what is still Rust-host only (`kore_task_*`). - `docs/provenance.md`: the export at Kore `eab77f8d`, what of MicroPerceptron/kore#1494 reaches the closure, and `kore-c`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- README.md | 5 ++-- docs/languages.md | 75 ++++++++++++++++++++++++++++++++++++++++++++-- docs/provenance.md | 26 ++++++++++++++++ 3 files changed, 101 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index caae6f0..4476d13 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,13 @@ machine's volume through the same exported owners a booted Kore runs. | --- | --- | | `kore-std` | Rust facade: filesystem, prelude, time, random, collections, process and optional driver APIs | | `kore-entry` | Executable-only adapter for a returning Rust `fn main()` on Kore | -| `kore-ffi` | C headers and Zig module over a Rust-founded host, including filesystem, time, random, allocation and worker APIs; no standalone C/Zig CRT | +| `kore-ffi` | C headers and Zig module over a Rust-founded host, including filesystem, time, random, allocation and worker APIs | +| `kore-c` | The runtime of `libkore.so`: `kore.h` as one shared object, and the host of an independent C program (`main` on a worker, its filesystem session, its end) | | `kore-lua` | Embeddable Lua 5.5 engine and bridge; interactive and opt-in restricted worker policies | | `kore-wasi` | WASI 0.2 component host composed by an embedder with its Wasmtime engine and native owners | | `kore-host` | Host-side (Linux/macOS) tooling: a Kore machine's DagFS volume, offline, through the filesystem row's own engine | -All six package manifests set `publish = false`; consume a reviewed Git +All seven package manifests set `publish = false`; consume a reviewed Git revision or a deliberately composed local checkout, not a registry version. The WASI host supports the `wasi:cli/command` world and native-backed I/O, filesystem, clocks and random imports; unsupported imports such as sockets diff --git a/docs/languages.md b/docs/languages.md index 07bd6ae..353a92f 100644 --- a/docs/languages.md +++ b/docs/languages.md @@ -20,9 +20,9 @@ A Rust host attaches `Fs` using its real founding face, then moves it into `kore_ffi::Session`, after closing any external views opened before handoff. The host awaits `Session::run(step, user)` on the existing Kore executor. No second executor, global allocator or global filesystem root is -installed. The C/Zig callback is linked into that same application's image. -This is not yet an independent C `main`/CRT, registry-published static SDK, or a -POSIX syscall veneer. +installed. The C/Zig callback is linked into that same application's image. A C +program with no Rust host gets one from `libkore.so` ([below](#an-independent-c-program-libkoreso)). +Neither is a registry-published static SDK or a POSIX syscall veneer. Each callback receives an opaque context and the preceding completion. Initially that completion is `KORE_OP_START`. Queue one operation and return. Submission @@ -64,6 +64,75 @@ if requests (such as zero-byte reads) complete immediately. C/Zig callbacks must return promptly: the adapter does not preempt arbitrary foreign code inside a callback. +## An independent C program: `libkore.so` + +A C program a compiler writes on Kore (`tcc` at Kore's shell) has no Rust +host. `kore-c` is its runtime. A distribution builds `libkore.so` from a +one-line `cdylib` named `kore` over it, linked with `--soname=libkore.so +--no-undefined` (Kore's `user/runtime/libkore`). It ships the library beside +the headers, which `kore-c` publishes to that crate's build script as +`DEP_KORE_INCLUDE`, so they come from the same revision. A program links it with +`-lkore` and gets: + +- every `kore.h` function, and the C names `kore/stdlib.h` and + `kore/pthread.h` alias as real symbols (`c-stdlib`), over the process heap + `kore-c` installs on the launch's `HEAP` supply; +- a host on thread 0, entered from the C library's start; +- `kore_fs_run(step, user, &error)` (`kore/program.h`), which runs a + `KoreFsStep` program as the program's filesystem session. + +The host is the shape `kore-lua`'s VM bridge has. `main` runs on a main +thread, a `DoorbellWorker` with a 1 MiB guarded stack +(`KORE_MAIN_STACK_BYTES`). Thread 0 is the process owner, and keeps the +executor and the launch's service faces. The two share one request cell and +one word. Every park is a doorbell park; nothing spins. + +- **Filesystem.** `kore_fs_run` from the main thread posts the session and + waits. The host attaches the launch's filesystem face on first use, rooted + at the directory the program was launched in, and runs `kore-ffi`'s + `Session` on its executor. The callbacks run there, as on any native + executor, while the caller waits: they must not depend on its + thread-local state. A thread that cannot park (any but the main thread) + is refused, as is a launch with no face. A quarantined session can never + run again. +- **The end.** The C library's `exit` and `_Exit` call + `__kore_program_exit(status, finalize)` after their `atexit` handlers, on + whatever thread called them. The first request wins and reaches the host, + since threads the program starts inherit a send into its wake channel. + - The host waits for a main thread that asked to end itself. + - It then raises kore-rt's process barrier, stopping every other thread + and running the finalizers only if none was still running. Once a + program has started a thread, `kore-ffi`'s thread controller is one of + them, so such a program's finalizers do not run; `atexit` handlers + always do. + - It commits the C status, the wire's `i32`, through the C library's own + exit. + - An end requested mid-session does not wait for the session. + - A panic in the library ends the program the same way, its record + published first. +- **Without a host.** A launch that cannot carry one (no channel space or + heap) runs the program on thread 0, as its C library alone would. Then the + filesystem session is refused, and only thread 0's exit ends the program. + +The C library's half is two weak references. Its start calls +`__kore_program_start(program, end)` when the program links libkore, and its +exit calls `__kore_program_exit`. A program without libkore is unchanged. On +Kore those are the start files Kore ships with TCC. `tools/check-program.py` +(`make cross`, and CI per ISA) links the shared object as a distribution does +and checks it: + +- a library with no entry point; +- its soname, nothing needed or undefined; +- static initial-exec TLS only; +- exactly the declared ABI exported: `kore.h`, `kore/program.h`, the + `c-stdlib` names and the two lifecycle entry points. + +Kore's `make keytest-tcc-x86` compiles a C program against these headers at +Kore's shell, links `libkore.so`, and runs it through every surface above +(Kore `res/TCC.md` §10). That is boot evidence; this repository's checks are +compiler and link evidence. Task preparation (`kore_task_*`) still needs a +Rust host to admit its batch: the independent host does not admit one. + ## Failure and lifetime Filesystem and provider errors have explicit generated foreign enums. Monotonic diff --git a/docs/provenance.md b/docs/provenance.md index 6554b1d..347f3f9 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -318,3 +318,29 @@ three ISAs. The Zig namespace and the `stdlib.h`/`pthread.h` aliases derive from that header's function list alone, so they are current as well. Not run locally: `zig fmt`, `test-foreign`, the rest of `bindings-check` and `kore-lua`'s Kore-target build; CI's host job runs them. + +## A C program's runtime, over Kore #1494's kore-rt + +The export records Kore `eab77f8d904352eabe65fedb84a455d6ccee5b7d`, a commit +on MicroPerceptron/kore#1494's branch, which brings TinyCC to Kore's shell. +Ten of the 23 commits since `7074bda2` reach the closure, which keeps its 26 +packages and 382 files: + +- `kore-rt` carries what `kore-c`'s host needs: + - `startup::barrier`, the process barrier without the exit; + - managed threads that inherit their founder's wake send; + - `crash::report`, a panic's report without the thread's end; + - also a per-thread C `errno` word, guarded worker stacks sized in bytes, + and the verbs and bundle call a shell's private install takes. +- `kore-abi` carries the install request and its replies, and the claimant + band sized by driver rows (`MAX_CLAIMANT_FACES`), which the device service + uses. +- The services' `boot.toml` digests were resealed three times. + +No exported manifest changed, so `native/Cargo.lock` did not. + +`kore-c` is new: the runtime of `libkore.so`, and the host of an independent +C program (`docs/languages.md`). It is the first libkore package a C program +on Kore links without a Rust host. Its shared object is a distribution's +one-line `cdylib` over it. `tools/check-program.py` links that object for the +three Kore ISAs and checks its shape and its exact exported ABI. From 3a373a265d144904e57c0f6df44b526bcf356c5c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 22:01:38 +0000 Subject: [PATCH 06/18] native: re-export at Kore 0018956a, where the thread controller parks ## Why kore-ffi's pthreads run on kore-rt's thread controller, which waited by ceding a millisecond at a time. Its creators and joiners did the same, and it probed every child for its exit on each pass. Kore `0018956a` makes every one of those waits a futex park and every transition a futex wake. The controller becomes the wake hub of the threads it founds. A thread's end reaches it through an exit hook, and its reap parks on a join arm. That change arrives here through the exporter rather than as edits to the snapshot. ## What - `futex`: sends a thread adopts after its founding (`adopt_peer`), a `Hub` that relays a wake it is counted for and sweeps only then, and `wait_or_end`, a doorbell park whose join arm watches one task. - `thread`: the exit hook `exit` runs first, a thread's own kick at DOORBELL@OWN_KICK (a DoorbellWorker is founded holding one), and `WorkerCap::channel_space`, a revocable mint grant to endow. - `thread_controller`: the parking controller. The closure keeps its 26 packages and 382 files, and no exported manifest changed, so `native/Cargo.lock` did not. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- native/crates/rt/src/entry.rs | 77 +- native/crates/rt/src/futex.rs | 455 +++++++++++- native/crates/rt/src/thread.rs | 132 +++- native/crates/rt/src/thread_controller.rs | 847 ++++++++++++++++++---- native/provenance.json | 10 +- 5 files changed, 1341 insertions(+), 180 deletions(-) diff --git a/native/crates/rt/src/entry.rs b/native/crates/rt/src/entry.rs index 8ce8ecd..babb71e 100644 --- a/native/crates/rt/src/entry.rs +++ b/native/crates/rt/src/entry.rs @@ -504,7 +504,7 @@ const fn report_slot_install(current_p1: usize, requested: usize) -> ReportSlotI #[cfg(target_os = "none")] pub(crate) mod geometry { - use core::sync::atomic::{AtomicI32, AtomicU64, AtomicUsize, Ordering}; + use core::sync::atomic::{AtomicI32, AtomicU32, AtomicU64, AtomicUsize, Ordering}; use kore_abi::task::{GeomBlock, geom}; @@ -641,6 +641,21 @@ pub(crate) mod geometry { /// inherited by a spawned thread, whose block is its own. Only the /// owning thread reads or writes it. c_errno: AtomicI32, + /// Sends into peers' doorbells this thread came to hold after its + /// founding ([`futex::adopt_peer`](crate::futex)), indexed by the + /// peer's band, each its arena slot plus one (`0` = none). Arena + /// names are this thread's alone, so they live here and a band a + /// later thread reuses starts with none. Only the owning thread + /// reads or writes them. + late_kicks: [AtomicU32; crate::thread::MAX_THREADS as usize], + /// The [`futex::Hub`](crate::futex) this thread relays an + /// undeliverable wake through, as an address (`0` = none). Set once. + futex_hub: AtomicUsize, + /// What [`thread::exit`](crate::thread::exit) runs before it commits + /// ([`set_exit_hook`]): a `fn(usize)` as an address (`0` = none), + /// and its argument. Taken by the exit, so it runs once. + exit_call: AtomicUsize, + exit_argument: AtomicUsize, } // The block must fit the founder's one-page window. @@ -751,6 +766,66 @@ pub(crate) mod geometry { block().c_errno.as_ptr() } + /// The send this thread adopted into band `peer`'s doorbell, if any. + pub(crate) fn late_kick(peer: usize) -> Option { + let slot = block().late_kicks.get(peer)?.load(Ordering::Relaxed); + let index = slot.checked_sub(1)?; + Some(kore_abi::session::SlotRef::from_wire(index)) + } + + /// Record (`Some`) or withdraw (`None`) this thread's adopted send into + /// band `peer`'s doorbell. `false` when `peer` is outside the band. + pub(crate) fn set_late_kick(peer: usize, send: Option) -> bool { + let Some(cell) = block().late_kicks.get(peer) else { + return false; + }; + let word = match send { + Some(slot) => match slot.index().checked_add(1) { + Some(word) => word, + None => return false, + }, + None => 0, + }; + cell.store(word, Ordering::Relaxed); + true + } + + /// The futex hub this thread relays through, as an address (`0` = none). + pub(crate) fn futex_hub() -> usize { + block().futex_hub.load(Ordering::Relaxed) + } + + /// Adopt the futex hub at `addr`; the first adoption stands. + pub(crate) fn set_futex_hub(addr: usize) -> bool { + addr != 0 + && block() + .futex_hub + .compare_exchange(0, addr, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + } + + /// Have [`thread::exit`](crate::thread::exit) run `call(argument)` on + /// this thread before it commits, replacing any hook set before. + pub(crate) fn set_exit_hook(call: fn(usize), argument: usize) { + let b = block(); + b.exit_argument.store(argument, Ordering::Relaxed); + b.exit_call.store(call as usize, Ordering::Relaxed); + } + + /// Take this thread's exit hook, leaving none: a hook that itself ends + /// the thread runs once. + pub(crate) fn take_exit_hook() -> Option<(fn(usize), usize)> { + let b = block(); + let call = b.exit_call.swap(0, Ordering::Relaxed); + if call == 0 { + return None; + } + // SAFETY: only `set_exit_hook` stores a nonzero `exit_call`, and it + // stores a `fn(usize)` cast to its address. + let call = unsafe { core::mem::transmute::(call) }; + Some((call, b.exit_argument.load(Ordering::Relaxed))) + } + /// Where this thread's block lives — the value the thread pointer holds, /// recovered the same way every other accessor recovers it. pub(crate) fn block_va() -> usize { diff --git a/native/crates/rt/src/futex.rs b/native/crates/rt/src/futex.rs index 04fe2d6..bebb188 100644 --- a/native/crates/rt/src/futex.rs +++ b/native/crates/rt/src/futex.rs @@ -70,15 +70,26 @@ //! `i`'s send end is looked up at `role::DOORBELL@(2 + i)`, and index 2 is //! the self-kick a founder already carves — so a thread endowed with it can //! wake **thread 0** with nothing added, which is the endowment the input -//! service's relay thread already takes. Higher indices are the peer mesh, -//! and `thread::spawn_with` does not provision one yet: [`wake`] returns -//! `false` for a peer this thread holds no send end for, rather than -//! reporting a kick it did not deliver. +//! service's relay thread already takes. Higher indices are the peer mesh; +//! [`wake`] returns `false` for a peer this thread holds no send end for, +//! rather than reporting a kick it did not deliver. //! -//! That asymmetry is a provisioning gap, not a design one — filling it is -//! minting `MAX_THREADS` doorbell pairs in the spawning thread and endowing -//! each child with the ends, which needs a `role::CHANNEL_SPACE` grant in the -//! task that spawns. Nothing in the API changes when it lands. +//! The mesh is provisioned by whoever founds the threads, from a +//! `role::CHANNEL_SPACE` grant: the star (`res/SPMD.md` §3) for an SPMD +//! region, and the [`thread_controller`](crate::thread_controller) for the +//! threads it owns. A founding stamps the sends it can name in advance; a +//! send a thread comes to hold *after* its founding — the channel a +//! controller mints for each child it founds, or the one a starter mints for +//! the controller it founds — is **adopted** ([`adopt_peer`]) into the +//! thread's own runtime block, and [`wake`] finds it there exactly as it +//! finds a stamped one. +//! +//! A waiter that no send this thread holds can reach is handed to a +//! **relay**: the star's hub (one notice on `DOORBELL@`[`RELAY`], answered by +//! a kick to every peer), or a [`Hub`] this thread adopted — a hub that is +//! itself a waiter, woken through its own word, which kicks only the threads +//! with waiters registered ([`sweep`]). Nothing in the wait/wake API changes +//! with either. //! //! **For a word in a shared window the gap is closed** by the kernel //! address-wait (`res/EVENT.md` §6). A word that @@ -349,6 +360,15 @@ pub(crate) const PEER_BASE: u16 = 2; /// band peers so it can never collide with one. pub(crate) const RELAY: u16 = PEER_BASE + MAX_THREADS as u16; +/// A thread's **own kick**: a send into its own doorbell that it may derive +/// from — the authority it hands a thread it founds so that thread can wake +/// it ([`thread::own_kick`](crate::thread)). One past [`RELAY`], so neither +/// the peer arithmetic nor the relay can name it: a thread never kicks itself +/// through the mesh. Thread 0's own kick predates this index — it is the +/// founder-carved self-kick at [`PEER_BASE`], its own peer-0 entry. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) const OWN_KICK: u16 = RELAY + 1; + /// Wait until another thread wakes this one, or `timeout` elapses. /// /// Parks **only if** `word` still holds `expected` at the moment the wait is @@ -825,6 +845,13 @@ pub(crate) fn notify_resident_completion() -> bool { /// parked is still woken correctly — its post already stands, so the notice /// resolves the park it is about to enter. pub fn wake(word: &AtomicU32) -> bool { + wake_one(word, true) +} + +/// [`wake`]'s body. `relay` says whether a waiter this thread cannot reach +/// is handed on ([`relay_missed`]); a [`Hub`]'s own signal is not, since the +/// hub is the relay. +fn wake_one(word: &AtomicU32, relay: bool) -> bool { let addr = address_of(word); let route = crate::cap::mapped::locate(word as *const AtomicU32 as *const u8); let mut unreachable_waiter = false; @@ -852,12 +879,12 @@ pub fn wake(word: &AtomicU32) -> bool { } } // A registered waiter this caller holds no peer send for is the - // hub's to reach (`res/SPMD.md` §3 — the star): one relay notice, - // counted as a wake because a notice is now in flight toward the - // waiter. No hub wired → today's honest miss. + // hub's to reach (`res/SPMD.md` §3 — the star, or this thread's own + // `Hub`): one relay, counted as a wake because a wake is now in + // flight toward the waiter. No hub wired → today's honest miss. if unreachable_waiter { STAT_MISSED.fetch_add(1, Ordering::Relaxed); - if imp::relay() { + if relay && relay_missed() { STAT_RELAYS.fetch_add(1, Ordering::Relaxed); return true; } @@ -865,6 +892,16 @@ pub fn wake(word: &AtomicU32) -> bool { false } +/// Hand the waiters this thread could not reach to the thread that can: +/// its adopted [`Hub`], through the hub's own word, or else the star's hub +/// through one raw notice on `DOORBELL@`[`RELAY`]. `true` when either went. +fn relay_missed() -> bool { + match imp::hub() { + Some(hub) => hub.relay(), + None => imp::relay(), + } +} + /// Wake **every** thread waiting on `word`; returns how many were kicked. /// /// The count is of notices successfully posted, not of threads that will @@ -904,17 +941,204 @@ pub fn wake_all(word: &AtomicU32) -> usize { missed = true; } } - // The relay hop (the star, `res/SPMD.md` §3): every waiter this - // caller could not kick is the hub's job — one notice covers them - // all, counted once. With the intra-chunk kick mesh this is the - // exception path (cross-chunk waiters), not the release path. - if missed && imp::relay() { + // The relay hop (the star, `res/SPMD.md` §3, or this thread's `Hub`): + // every waiter this caller could not kick is the hub's job — one relay + // covers them all, counted once. With the intra-chunk kick mesh this + // is the exception path (cross-chunk waiters), not the release path. + if missed && relay_missed() { STAT_RELAYS.fetch_add(1, Ordering::Relaxed); woken += 1; } woken } +/// A **relay hub that is also a waiter**: the thread that founds a group of +/// threads and holds a send into each of their doorbells — the +/// [`thread_controller`](crate::thread_controller) — parked on its +/// [`word`](Self::word) like any other waiter. +/// +/// A thread that adopted the hub ([`adopt_hub`]) relays a wake it cannot +/// deliver by counting it in [`relays`](Self::relays), then [`signal`]ing +/// the hub: an ordinary store-then-wake of the hub's word, with every +/// property [`wake`] has. The hub reads `relays` after each wake and, when +/// it moved, [`sweep`]s. +/// +/// Why a count and not the star's raw notice: this hub is woken for work of +/// its own, and a notice says nothing about why it came. A hub that took +/// every notice for a relay would kick every parked thread on every +/// unrelated event; one that took none would lose a relay whose notice +/// coalesced with another on its doorbell. The count is bumped before the +/// word, so a hub that sees the word move sees the relay behind it. +/// +/// [`signal`]: Self::signal +#[repr(align(128))] +pub(crate) struct Hub { + word: AtomicU32, + relays: AtomicU32, +} + +impl Hub { + pub(crate) const fn new() -> Self { + Self { + word: AtomicU32::new(0), + relays: AtomicU32::new(0), + } + } + + /// The word the hub thread waits on. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + pub(crate) fn word(&self) -> &AtomicU32 { + &self.word + } + + /// Relays owed so far, wrapping: the hub sweeps whenever this moved + /// since its last sweep. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + pub(crate) fn relays(&self) -> u32 { + self.relays.load(Ordering::SeqCst) + } + + /// Tell the hub something changed — store, then wake, the order this + /// module requires. Delivered directly or not at all: the hub has no + /// relay but itself. `true` when a kick reached it. + pub(crate) fn signal(&self) -> bool { + self.word.fetch_add(1, Ordering::SeqCst); + wake_one(&self.word, false) + } + + /// One relay: counted before the word moves. + fn relay(&self) -> bool { + self.relays.fetch_add(1, Ordering::SeqCst); + self.signal() + } +} + +/// Relay through `hub` from now on: a wake this thread cannot deliver goes +/// to it ([`Hub`]). The first adoption stands — `false` when this thread +/// already relays through a hub, or has no runtime block to record it in. +pub(crate) fn adopt_hub(hub: &'static Hub) -> bool { + imp::adopt_hub(hub) +} + +/// Record that this thread holds `send`, a send into band `peer`'s +/// doorbell its founding did not stamp — the channel a controller mints for +/// each child it founds, or a starter for the controller it founds. From +/// here on [`wake`] reaches `peer` directly. `false` when `peer` is outside +/// the band or this thread has no runtime block. +/// +/// The record is this thread's alone — it names a slot in its own arena — so +/// it lives in its runtime block, not beside [`WAITING_ON`]: a band reused +/// by a later thread starts from a zeroed block and inherits nothing. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) fn adopt_peer(peer: u32, send: kore_abi::session::SlotRef) -> bool { + imp::adopt_peer(peer as usize, send) +} + +/// Withdraw `peer`'s adopted send. Called before the send is retired, so no +/// wake presents a name the arena is about to reissue. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) fn forget_peer(peer: u32) { + imp::forget_peer(peer as usize); +} + +/// Whether this thread holds a send into band `peer`'s doorbell — stamped +/// at its founding or adopted since. +pub(crate) fn reaches(peer: u32) -> bool { + imp::reaches(peer as usize) +} + +/// The hub's answer to a relay ([`Hub`]): kick every thread this one can +/// reach that has a doorbell waiter registered, itself excepted, and return +/// how many were kicked. +/// +/// A relay does not say which word it was for, so every such thread is a +/// candidate and each kick is a wake its target re-checks — spurious for one +/// whose word did not move, which every waiter tolerates by contract, and +/// paid only when a relay was owed. A word-parked waiter is not kicked: the +/// kernel's address-wait, not its doorbell, holds it, and its slice bounds +/// a relay it misses. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) fn sweep() -> usize { + let Some(me) = current_thread() else { + return 0; + }; + imp::sweep(me) +} + +/// [`sweep`]'s selection over the registry, with the kick injected so the +/// host suite can hold it to account. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +fn sweep_with(me: usize, mut kick: impl FnMut(usize) -> bool) -> usize { + let mut kicked = 0; + for thread in 0..MAX_THREADS as usize { + if thread != me && waits_on_bell(thread) && kick(thread) { + STAT_KICKS.fetch_add(1, Ordering::Relaxed); + kicked += 1; + } + } + kicked +} + +/// Whether `thread` has any waiter a kick reaches — a doorbell-mode entry. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +fn waits_on_bell(thread: usize) -> bool { + WAITING_ON[thread].iter().any(|cell| { + let entry = cell.load(Ordering::SeqCst); + entry != NOT_WAITING && entry & WORD_MODE == 0 + }) +} + +/// [`wait`] that also ends when the task `task` ends: one doorbell park +/// whose join arm (`SessionFlags::WAIT_TASK`) watches that task beside the +/// word, for a thread that must see both a futex wake and a sibling's end +/// without polling either — the [`thread_controller`](crate::thread_controller) +/// reaping a child that has announced its exit. `report` is a landing the +/// caller owns (a `thread::JoinScratch`); an exit leaves its report there. +/// +/// [`Woken`](WaitOutcome::Woken) covers the task's end too: the caller +/// re-checks its word *and* the task. There is no monitored spin, because +/// the edge waited for is a kernel event no store announces, and the park +/// stays on the doorbell even for a word in a noted window — the kernel's +/// word wait does not compose with a join arm. Uncounted in [`stats`], like +/// the star hub's housekeeping wait: the threads that use it are not the +/// ones a bench measures. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) fn wait_or_end( + word: &AtomicU32, + expected: u32, + timeout: Option, + task: kore_abi::session::SlotRef, + report: kore_abi::session::SlotRef, +) -> WaitOutcome { + let Some(me) = current_thread() else { + return WaitOutcome::Unsupported; + }; + // Arm, register, check — `wait`'s order and for its reasons. + if !imp::arm() { + return WaitOutcome::Unsupported; + } + let Some(slot) = claim_wait(me, address_of(word)) else { + return WaitOutcome::Unsupported; + }; + let _slot = WaitSlot { thread: me, slot }; + if word.load(Ordering::SeqCst) != expected { + return WaitOutcome::Retry; + } + imp::park_or_end(timeout, task, report) +} + +/// How long a park that could not raise its join arm rides the doorbell +/// alone. A join arm the kernel did not install answers `WouldBlock` at +/// once, with nothing parked; riding the doorbell for this long instead +/// keeps a caller that loops on [`wait_or_end`] from trapping in a ring. +/// The joiner's own pacing for an unestablished park +/// (`thread::join_park`, two milliseconds), so the two degrade alike. +#[cfg_attr( + not(target_os = "none"), + expect(dead_code, reason = "the join arm is a metal-only trap") +)] +const END_SLICE: Duration = Duration::from_millis(2); + /// Whether this thread can [`wait`] at all — a doorbell to park on and an /// identity in the thread band. /// @@ -1182,15 +1406,115 @@ mod imp { let Some(rt) = crate::entry::thread_context() else { return false; }; - let Ok(peer) = u16::try_from(index) else { - return false; - }; - let Some(send) = rt.bootstrap().handle(role::DOORBELL, PEER_BASE + peer) else { + kick_with(&rt, index) + } + + /// [`kick`] against a context the caller already resolved — the sweep + /// kicks many peers off one boot-page parse. + fn kick_with(rt: &crate::entry::Rt, index: usize) -> bool { + let Some(send) = send_for(rt, index) else { return false; }; verbs::post_notice(send, COOKIE_FUTEX, index as u64) } + /// The send this thread holds into band `index`'s doorbell: the one its + /// founding stamped at `DOORBELL@(PEER_BASE + index)`, else the one it + /// adopted since ([`super::adopt_peer`]). + fn send_for(rt: &crate::entry::Rt, index: usize) -> Option { + let peer = u16::try_from(index).ok()?; + rt.bootstrap() + .handle(role::DOORBELL, PEER_BASE + peer) + .or_else(|| crate::entry::geometry::late_kick(index)) + } + + pub(super) fn reaches(index: usize) -> bool { + crate::entry::thread_context().is_some_and(|rt| send_for(&rt, index).is_some()) + } + + pub(super) fn adopt_peer(index: usize, send: kore_abi::session::SlotRef) -> bool { + crate::entry::geometry::set_late_kick(index, Some(send)) + } + + pub(super) fn forget_peer(index: usize) { + let _ = crate::entry::geometry::set_late_kick(index, None); + } + + pub(super) fn sweep(me: usize) -> usize { + let Some(rt) = crate::entry::thread_context() else { + return 0; + }; + super::sweep_with(me, |thread| kick_with(&rt, thread)) + } + + /// The hub this thread relays through, if it adopted one. + pub(super) fn hub() -> Option<&'static super::Hub> { + let addr = crate::entry::geometry::futex_hub(); + // SAFETY: `adopt_hub` records only the address of a `&'static Hub`, + // and nothing ever clears or rewrites it to another value. + (addr != 0).then(|| unsafe { &*(addr as *const super::Hub) }) + } + + pub(super) fn adopt_hub(hub: &'static super::Hub) -> bool { + crate::entry::geometry::set_futex_hub(hub as *const super::Hub as usize) + } + + /// Park on the standing post with the join arm watching `task` beside + /// it — the composite `WAIT_DELIVERY | WAIT_TASK` (± `WAIT_SLEEP`) + /// shape the kernel resolves first-wins (`SessionFlags::WAIT_TASK`). + pub(super) fn park_or_end( + timeout: Option, + task: kore_abi::session::SlotRef, + report: kore_abi::session::SlotRef, + ) -> WaitOutcome { + let Some(db) = doorbell() else { + return WaitOutcome::Unsupported; + }; + let deadline = match timeout { + Some(d) => match crate::clock::uptime() { + Ok(now) => Some((now + d).as_nanos()), + Err(_) => return WaitOutcome::Unsupported, + }, + None => None, + }; + let mut p = SessionBuilder::new(); + p.park_standing(db.chan, db.land.index() as u16, db.recv_tag); + if let Some(at) = deadline { + p.wait_sleep(at); + } + if p.join(task, report).is_err() { + return WaitOutcome::Unsupported; + } + p.wait_task(); + let Ok(committed) = TrapTransport::new().commit_join_park(&p) else { + return WaitOutcome::Unsupported; + }; + match committed.status() { + // The doorbell won: consume and re-stand, as `park` does. + Status::Ok => { + let _ = verbs::take_notice(db.land); + post(&db); + WaitOutcome::Woken + } + // The task ended (`Exited`), or the post or the task was reaped + // (`Revoked`) — the caller's re-check reads which; a reaped + // doorbell refuses its next arm. An OCC loss parked nothing and + // is retried by the caller's loop. + Status::Exited | Status::Revoked | Status::Conflict => WaitOutcome::Woken, + Status::Deadline => WaitOutcome::TimedOut, + // Nobody won, or the join arm could not be raised and nothing + // parked: ride the doorbell alone, bounded (`END_SLICE`). + Status::WouldBlock => { + let slice = timeout.map_or(super::END_SLICE, |t| t.min(super::END_SLICE)); + match park(Some(slice)) { + WaitOutcome::TimedOut if timeout.is_none() => WaitOutcome::Woken, + outcome => outcome, + } + } + _ => WaitOutcome::Unsupported, + } + } + // The relay slot sits exactly one past the last band peer — the // module-level constant and this mesh's arithmetic must agree. const _: () = assert!(super::RELAY == PEER_BASE + super::MAX_THREADS as u16); @@ -1261,6 +1585,37 @@ mod imp { false } + pub(super) fn reaches(_index: usize) -> bool { + false + } + + pub(super) fn adopt_peer(_index: usize, _send: kore_abi::session::SlotRef) -> bool { + false + } + + pub(super) fn forget_peer(_index: usize) {} + + pub(super) fn sweep(_me: usize) -> usize { + 0 + } + + /// No runtime block off metal: no hub was ever adopted. + pub(super) fn hub() -> Option<&'static super::Hub> { + None + } + + pub(super) fn adopt_hub(_hub: &'static super::Hub) -> bool { + false + } + + pub(super) fn park_or_end( + _timeout: Option, + _task: kore_abi::session::SlotRef, + _report: kore_abi::session::SlotRef, + ) -> super::WaitOutcome { + super::WaitOutcome::Unsupported + } + pub(super) fn relay() -> bool { false } @@ -1621,4 +1976,62 @@ mod tests { assert!(!wake(&word)); assert_eq!(wake_all(&word), 0); } + + #[test] + fn a_relay_is_counted_before_the_hub_word_moves() { + // The hub's lost-relay argument: it reads `relays` after it sees its + // word move, so the count must already have moved by then. Off metal + // the kick refuses, but the two stores are the shared half. + static HUB: Hub = Hub::new(); + let (word, relays) = (HUB.word().load(Ordering::SeqCst), HUB.relays()); + assert!(!HUB.relay(), "off metal no kick reaches the hub"); + assert_eq!(HUB.relays(), relays.wrapping_add(1)); + assert_eq!(HUB.word().load(Ordering::SeqCst), word.wrapping_add(1)); + // A plain signal moves the word and owes no relay. + assert!(!HUB.signal()); + assert_eq!(HUB.relays(), relays.wrapping_add(1)); + assert_eq!(HUB.word().load(Ordering::SeqCst), word.wrapping_add(2)); + } + + #[test] + fn a_hub_word_owns_its_interference_span() { + // The hub thread's monitored spin watches its word's line; every + // client of the hub stores to it. Sharing the line with neighbours + // would end that spin on their traffic. + assert_eq!(core::mem::align_of::(), 128); + } + + #[test] + fn a_sweep_kicks_only_doorbell_waiters_other_than_itself() { + // Rows 8..=11: the hub (8), a doorbell waiter (9), a word-parked + // waiter (10), and a thread waiting on nothing (11). + for row in 8..=11 { + reset_thread(row); + } + let a = AtomicU32::new(0); + let b = AtomicU32::new(0); + claim_wait(8, address_of(&a)).expect("the hub's own wait"); + claim_wait(9, address_of(&a)).expect("a doorbell waiter"); + claim_wait_on_word(10, address_of(&b)).expect("a word waiter"); + let mut kicked = std::vec::Vec::new(); + let n = sweep_with(8, |thread| { + kicked.push(thread); + true + }); + assert_eq!(kicked, [9], "the sweep kicked a thread no kick can wake"); + assert_eq!(n, 1); + // A kick that refuses is not counted as a wake. + assert_eq!(sweep_with(8, |_| false), 0); + for row in 8..=11 { + reset_thread(row); + } + } + + #[test] + fn the_own_kick_is_named_past_every_peer_and_the_relay() { + // No peer arithmetic (`PEER_BASE + band`) and no relay lookup may + // ever resolve a thread's own kick. + assert_eq!(OWN_KICK, RELAY + 1); + assert!(OWN_KICK > PEER_BASE + MAX_THREADS as u16 - 1); + } } diff --git a/native/crates/rt/src/thread.rs b/native/crates/rt/src/thread.rs index 63b8781..1a849a5 100644 --- a/native/crates/rt/src/thread.rs +++ b/native/crates/rt/src/thread.rs @@ -629,16 +629,11 @@ pub(crate) fn heap_endowment() -> Option { Some(WorkerCap::heap(rt.bootstrap().handle(role::HEAP, 0)?)) } -/// Off metal no thread founds another; there is no heap to endow. -#[cfg(not(target_os = "none"))] -pub(crate) fn heap_endowment() -> Option { - None -} - /// The rights of a wake send a thread may pass on: the notice itself /// (`ACCESS`, [`verbs::NOTICE_RIGHTS`](crate::cap::verbs::NOTICE_RIGHTS)) and /// `DERIVE`, so the thread can found children holding the same end. A send /// into a wake channel conveys nothing but the right to kick it. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] pub(crate) const WAKE_SEND_RIGHTS: kore_abi::cap::Rights = kore_abi::cap::Rights::ACCESS.union(kore_abi::cap::Rights::DERIVE); @@ -674,12 +669,68 @@ pub(crate) fn wake_endowment() -> Option { }) } -/// Off metal no thread founds another; there is no wake to inherit. +impl WorkerCap { + /// **Channel space, endowed** — `role::CHANNEL_SPACE@0`: a revocable + /// child of the mint grant `space`, so the thread can mint the channels + /// that wake the threads *it* founds (a `DoorbellWorker` of its own, or + /// the [`thread_controller`](crate::thread_controller) a C program's + /// `pthread_create` starts on its main thread). The parent keeps its + /// grant; revoking it ends the child's. + pub fn channel_space(space: SlotRef) -> Self { + use kore_abi::cap::ResourceKind; + Self { + role: role::CHANNEL_SPACE, + index: 0, + subject: space, + kind: ResourceKind::ChannelSpace as u16 as u8, + rights: crate::cap::CHANNEL_SPACE_RIGHTS, + } + } +} + +/// [`WorkerCap::channel_space`] over this thread's own `role::CHANNEL_SPACE@0` +/// — `None` when it holds no mint grant. +#[cfg(target_os = "none")] +pub fn channel_space_endowment() -> Option { + let rt = crate::entry::thread_context()?; + Some(WorkerCap::channel_space( + rt.bootstrap().handle(role::CHANNEL_SPACE, 0)?, + )) +} + +/// Off metal no thread founds another. #[cfg(not(target_os = "none"))] -pub(crate) fn wake_endowment() -> Option { +pub fn channel_space_endowment() -> Option { None } +/// This thread's **own kick**: a send into its own doorbell it may derive +/// from, as a slot — `DOORBELL@`[`OWN_KICK`](crate::futex::OWN_KICK), or for +/// thread 0 the founder-carved self-kick at `DOORBELL@PEER_BASE`. +/// +/// `None` when it holds none, or one it may not derive from: the authority a +/// thread hands a peer so the peer can wake it must itself be derivable. The +/// arena record decides, as for [`wake_endowment`]. +#[cfg(target_os = "none")] +pub(crate) fn own_kick() -> Option { + use kore_abi::cap::{CapRecord, ResourceKind}; + let rt = crate::entry::thread_context()?; + let boot = rt.bootstrap(); + let send = boot + .handle(role::DOORBELL, crate::futex::OWN_KICK) + .or_else(|| { + (crate::entry::geometry::shard_index() == Some(0)) + .then(|| boot.handle(role::DOORBELL, crate::futex::PEER_BASE)) + .flatten() + })?; + let record_va = crate::entry::geometry::slot_record_va(send)?; + // SAFETY: the slot alias is this thread's kernel-authored, read-only view + // of its own arena record for a slot its boot block names. + let record = unsafe { core::ptr::read_volatile(record_va as *const CapRecord) }; + (record.kind == ResourceKind::ChanSend as u16 && record.rights.contains(WAKE_SEND_RIGHTS)) + .then_some(send) +} + /// Build exactly the capability namespace a spawned worker receives. /// /// This stays target-independent so host tests can pin a subtle shared-space @@ -787,7 +838,9 @@ pub enum WorkerStack { /// doorbell (its recv; the parent keeps the send that kicks it) and a /// **wake** channel the other way (the worker holds the send at /// `DOORBELL@PEER_BASE`; the parent parks on the recv) — plus a heap -/// position over the task's supply so it may allocate. +/// position over the task's supply so it may allocate, and the worker's +/// own kick, a derivable send into its doorbell it can hand a thread it +/// founds. /// /// The wake channel exists because a launched task holds no self-kick: /// the founder carves `DOORBELL@2` for service rows that declare workers, @@ -896,9 +949,9 @@ pub fn kick_parent() -> bool { } /// Spawn `entry` on a worker founded holding its own doorbell, a send -/// into thread 0's doorbell, a heap position, and `extra`. `None` when -/// the task holds no `CHANNEL_SPACE` (its profile lacks `channels`), the -/// mint or a derive refuses, or [`spawn_with`] does. +/// into thread 0's doorbell, its own kick ([`own_kick`]), a heap position, +/// and `extra`. `None` when the task holds no `CHANNEL_SPACE` (its profile +/// lacks `channels`), the mint or a derive refuses, or [`spawn_with`] does. #[cfg(target_os = "none")] pub fn spawn_with_doorbell(entry: fn(), extra: &[WorkerCap]) -> Option { spawn_with_doorbell_on(entry, extra, WorkerStack::Band) @@ -947,7 +1000,8 @@ pub fn spawn_with_doorbell_on( kind: 0, rights: Rights::empty(), }; MAX_WORKER_CAPS]; - if 3 + extra.len() > caps.len() { + const OWN: usize = 4; + if OWN + extra.len() > caps.len() { let _ = pair.send.try_retire(); let _ = pair.root.try_retire(); let _ = wake.send.try_retire(); @@ -972,9 +1026,19 @@ pub fn spawn_with_doorbell_on( kind: ResourceKind::ChanSend as u16 as u8, rights: WAKE_SEND_RIGHTS, }; - caps[2] = heap; - caps[3..3 + extra.len()].copy_from_slice(extra); - let Some(thread) = spawn_entry(ThreadEntry::Plain(entry), &caps[..3 + extra.len()], stack) + // The worker's own kick ([`own_kick`]): a derivable send into its own + // doorbell, so a thread it founds — the controller its `pthread_create` + // starts — can be handed the means to wake it. + caps[2] = WorkerCap { + role: role::DOORBELL, + index: crate::futex::OWN_KICK, + subject: pair.root.slot(), + kind: ResourceKind::ChanSend as u16 as u8, + rights: WAKE_SEND_RIGHTS, + }; + caps[3] = heap; + caps[OWN..OWN + extra.len()].copy_from_slice(extra); + let Some(thread) = spawn_entry(ThreadEntry::Plain(entry), &caps[..OWN + extra.len()], stack) else { let _ = pair.send.try_retire(); let _ = pair.root.try_retire(); @@ -1933,6 +1997,13 @@ pub fn exit(code: ExitCode) -> ! { let Some(rt) = crate::entry::thread_context() else { crate::entry::refuse(); }; + // A thread whose end is awaited announces it first ([`on_exit`]): every + // way a thread ends that runs code — its entry returning, a panic, an + // explicit exit — comes through here. Taken, so a hook that itself ends + // the thread runs once. + if let Some((call, argument)) = crate::entry::geometry::take_exit_hook() { + call(argument); + } let Some(slot) = rt.bootstrap().handle(role::SELF, 0) else { crate::entry::refuse(); }; @@ -1958,6 +2029,22 @@ pub fn exit(_code: ExitCode) -> ! { crate::entry::refuse() } +/// Have this thread's [`exit`] run `call(argument)` first, on this thread, +/// before the exit commits — how the thread that reaps it learns it is +/// ending without polling for it (the +/// [`thread_controller`](crate::thread_controller)'s children). The last hook +/// set stands. `call` runs after the thread's own code, so it must not block, +/// and must not end the thread itself. +/// +/// The kernel's exit commits after the hook returns, so a reaper woken by +/// the hook can still find the thread live for a moment; it parks on the +/// thread's end for that moment (`futex::wait_or_end`), which no store could +/// announce. +#[cfg(target_os = "none")] +pub(crate) fn on_exit(call: fn(usize), argument: usize) { + crate::entry::geometry::set_exit_hook(call, argument); +} + /// One caller-owned report landing reserved for a complete worker-join fold. /// /// The slot is affine for the same reason a [`Thread`] is: once worker @@ -2293,6 +2380,19 @@ fn join_park( } } +/// [`Thread::poll`] through a landing reserved for it: a repeat observer — +/// the [`thread_controller`](crate::thread_controller), between a child's +/// announced end and its reap — probes without spending an arena slot each +/// time, and so without the one refusal that is about the observer rather +/// than the thread. +#[cfg(target_os = "none")] +pub(crate) fn poll_reserved( + thread: &Thread, + scratch: &JoinScratch, +) -> Result { + poll_task_handle_in(&TrapTransport::new(), thread.handle, scratch.report) +} + /// Issue one unflagged join probe. `WouldBlock` is the one positive live-task /// result; unlike the blocking path, an OCC conflict or unrelated park status /// is surfaced so a nonblocking caller controls its own retry policy. diff --git a/native/crates/rt/src/thread_controller.rs b/native/crates/rt/src/thread_controller.rs index 688f0cf..55891d7 100644 --- a/native/crates/rt/src/thread_controller.rs +++ b/native/crates/rt/src/thread_controller.rs @@ -4,49 +4,50 @@ //! records. Only the controller touches native tickets. A detached record is //! reusable only after native exit and successful reclamation. Storage is //! supplied by the process and must outlive every client and child. +//! +//! # Every wait is a futex park +//! +//! Nothing here polls. The controller thread waits on its hub word +//! ([`futex::Hub`]), and everything that needs it — a create request, a +//! child's end, a wake relayed through it — stores, then wakes that word, the +//! order [`futex`] requires. A client waits on its record's event word, which +//! the controller bumps and wakes after every transition a client can wait +//! for. A child announces its end through the hook [`thread::exit`] runs, on +//! every way a thread ends that runs code; the kernel's exit commits a moment +//! later, an edge no store can announce, so the controller reaps through a +//! park whose join arm watches that child beside its word +//! ([`futex::wait_or_end`]). +//! +//! # The controller is its threads' hub +//! +//! The star's rule (`res/SPMD.md` §3) for threads that come and go. The +//! starter mints the controller's doorbell, keeps a send into it (an adopted +//! peer), and founds it holding that doorbell, its own kick, the starter's +//! kick, the host's wake, the heap and a channel-space grant. For each child +//! the controller mints a doorbell, keeps the send (an adopted peer, withdrawn +//! before the child's band can be reused), and founds the child holding it, a +//! send into the controller, the starter's kick, the host's wake and the heap. +//! Children and the starter relay through the controller's hub, so a wake +//! between two threads neither can kick directly still arrives: the controller +//! sweeps ([`futex::sweep`]). +//! +//! # Degrade, never wedge +//! +//! A starter with no channel space founds a controller without a doorbell: +//! neither it nor its children can park, and each of their waits takes the +//! deadline park every thread can ([`pace::cede`]). A client the controller +//! cannot kick waits in bounded slices ([`UNREACHED_SLICE`]). A thread that +//! cannot wake a parking controller is refused a create, which would +//! otherwise wait on a request nothing announced. +//! +//! [`thread::exit`]: crate::thread::exit +//! [`pace::cede`]: crate::pace::cede use core::cell::UnsafeCell; -use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; - -// Each owner derives the process heap into its children using its own local -// slot. No creator's slot number is ever forwarded across the controller seam. -use super::thread::{WorkerCap, heap_endowment, wake_endowment}; - -/// What a managed thread is founded holding, re-derived by each founder from -/// its own holdings: the heap position its callback allocates through, and a -/// send into the wake channel its founder kicks (`thread::wake_endowment`), so -/// a managed thread reaches the same host thread 0 a `DoorbellWorker` does — -/// the host a language runtime keeps there to serve the process and end it. -/// Either is absent when the founder holds none. -struct Endowment { - caps: [WorkerCap; 2], - len: usize, -} +use core::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use core::time::Duration; -impl Endowment { - fn inherited() -> Self { - let vacant = WorkerCap { - role: 0, - index: 0, - subject: kore_abi::session::SlotRef::from_wire(0), - kind: 0, - rights: kore_abi::cap::Rights::empty(), - }; - let mut endowment = Self { - caps: [vacant; 2], - len: 0, - }; - for cap in [heap_endowment(), wake_endowment()].into_iter().flatten() { - endowment.caps[endowment.len] = cap; - endowment.len += 1; - } - endowment - } - - fn as_slice(&self) -> &[WorkerCap] { - &self.caps[..self.len] - } -} +use super::futex::{self, Hub, WaitOutcome}; const FREE: u64 = 0; const BUILD: u64 = 1; @@ -62,6 +63,37 @@ const RETIRED: u64 = 15; const MASK: u64 = 15; const COUNT: usize = super::thread::MAX_THREADS as usize - 2; +/// Lifecycle of the controller thread itself, in [`Controller::started`]: +/// not started, being founded, or started — parking on its doorbell, or +/// without one, taking the deadline park. +const IDLE: u32 = 0; +const STARTING: u32 = 1; +const PARKING: u32 = 2; +#[cfg_attr( + not(target_os = "none"), + expect(dead_code, reason = "only a metal founding starts") +)] +const POLLING: u32 = 3; + +/// How long one park lasts for a client the controller cannot kick — a +/// thread it holds no send into, such as an initial thread launched without +/// a self-kick. Nothing can end such a park early, so its slice is the +/// latency it pays, and this is the cadence those waits paced themselves +/// at before the controller parked (`pace::CEDE_NS`): a client the wake +/// fabric cannot reach is no worse off than it was. Shorter would wake an +/// unreachable joiner more often for nothing; longer would delay it. +const UNREACHED_SLICE: Duration = Duration::from_nanos(crate::pace::CEDE_NS); + +/// How long the controller rides its doorbell after a reap the kernel did +/// not complete — a refused lifecycle observation or reclamation, which no +/// later event is bound to retry. The joiner's own pacing for a join that +/// cannot establish its park (`thread::join_park`, two milliseconds). +#[cfg_attr( + not(target_os = "none"), + expect(dead_code, reason = "reaping is metal-only") +)] +const REAP_RETRY: Duration = Duration::from_millis(2); + /// An identifier within one controller, never a capability-slot number. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Id { @@ -85,14 +117,30 @@ pub enum Error { struct Start { call: fn(usize) -> usize, argument: usize, + /// Read by the controller thread, after the reap. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] retire: Option, + /// The controller this record belongs to, for the child's hooks. + home: usize, } fn empty(_: usize) -> usize { 0 } +/// One managed thread's shared record, on its own interference span: its +/// creator, its joiner, the child and the controller each write it, and a +/// waiting client's monitored spin sleeps on its event word's line, which a +/// neighbour's traffic would end for nothing. The span is the one the +/// barrier isolates against (`barrier::Line`). +#[repr(align(128))] struct Record { state: AtomicU64, + /// Bumped by the controller after each transition a client can wait + /// for; clients park on it. An event count: the state is the truth. + event: AtomicU32, + /// Set by the child's exit hook: it has stopped running its own code + /// and is committing its end. + ending: AtomicU32, start: UnsafeCell, value: AtomicUsize, identity: AtomicUsize, @@ -105,10 +153,13 @@ impl Record { const fn new() -> Self { Self { state: AtomicU64::new(FREE), + event: AtomicU32::new(0), + ending: AtomicU32::new(0), start: UnsafeCell::new(Start { call: empty, argument: 0, retire: None, + home: 0, }), value: AtomicUsize::new(0), identity: AtomicUsize::new(0), @@ -121,13 +172,34 @@ impl Record { Ordering::Release, ); } + /// Publish a transition a client may be parked on: the state was + /// stored; now the event moves, then its waiters are woken. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + fn announce(&self) { + self.event.fetch_add(1, Ordering::SeqCst); + futex::wake_all(&self.event); + } } /// Named process-owned storage. Start once during application startup and /// share its static reference with workers. This consumes one native thread /// for the process lifetime. No native ticket escapes its owner. pub struct Controller { - started: AtomicUsize, + /// [`IDLE`], [`STARTING`], [`PARKING`] or [`POLLING`]; clients park on + /// it while the controller is being founded. + started: AtomicU32, + /// What the controller thread waits on, and what its threads relay + /// through. + hub: Hub, + /// The controller thread's band, plus one (`0` until it is founded). + band: AtomicU32, + /// The band of the thread that started the controller, plus one. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + starter: AtomicU32, + /// The bands whose doorbells the controller can kick, a bit each: the + /// clients its wakes reach. A child's bit is set once it is founded and + /// cleared before its band can be reused. + reach: AtomicU32, records: [Record; COUNT], } impl Default for Controller { @@ -135,35 +207,50 @@ impl Default for Controller { Self::new() } } +// The reach bitmap has one bit per band. +const _: () = assert!(super::thread::MAX_THREADS <= u32::BITS); + impl Controller { pub const fn new() -> Self { Self { - started: AtomicUsize::new(0), + started: AtomicU32::new(IDLE), + hub: Hub::new(), + band: AtomicU32::new(0), + starter: AtomicU32::new(0), + reach: AtomicU32::new(0), records: [const { Record::new() }; COUNT], } } /// Start the owner. Concurrent/repeated calls do not create another owner. + /// + /// The calling thread becomes the controller's starter: it mints the + /// controller's wake channel from its channel-space grant and holds that + /// channel's root, as a `DoorbellWorker`'s parent holds its worker's, so + /// start from a thread that outlives the controller's use — the initial + /// thread, or a program's main. pub fn start(&'static self) -> Result<(), Error> { self.started - .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) + .compare_exchange(IDLE, STARTING, Ordering::AcqRel, Ordering::Acquire) .map_err(|_| Error::Busy)?; - let endowment = Endowment::inherited(); - match super::thread::spawn_with_arg( - controller_entry, - self as *const Self as usize, - endowment.as_slice(), - ) { - Some(ticket) => { - // This owner deliberately lives until process teardown. Managed - // children never use native detach: their tickets are retained. - ticket.detach(); - self.started.store(2, Ordering::Release); - Ok(()) - } - None => { - self.started.store(0, Ordering::Release); - Err(Error::Unavailable) + let mode = self.found(); + self.started.store(mode.unwrap_or(IDLE), Ordering::Release); + futex::wake_all(&self.started); + mode.map(|_| ()).ok_or(Error::Unavailable) + } + + /// How the started controller waits, waiting out its founding; `None` + /// when it is not started. + fn mode(&self) -> Option { + loop { + match self.started.load(Ordering::Acquire) { + IDLE => return None, + STARTING => { + // The starter wakes every thread it can when it is done; + // a thread it cannot reach rides its slice. + park(&self.started, STARTING, Some(UNREACHED_SLICE)); + } + mode => return Some(mode), } } } @@ -179,13 +266,19 @@ impl Controller { /// runs once on the controller after native reclamation, including when the /// callback exits without returning. On error custody remains with caller. /// The cleanup must not make blocking calls back into this controller. + /// + /// Call it from a thread that can wake the controller — its starter, or a + /// thread it created. A parking controller sleeps until something wakes + /// it, so any other thread is refused [`Error::Unavailable`] rather than + /// left waiting on a request nothing announced. pub fn create_with_cleanup( &'static self, call: fn(usize) -> usize, argument: usize, retire: Option, ) -> Result { - if self.started.load(Ordering::Acquire) != 2 { + let mode = self.mode().ok_or(Error::Unavailable)?; + if mode == PARKING && !self.wakes_controller() { return Err(Error::Unavailable); } for (slot, record) in self.records.iter().enumerate() { @@ -206,31 +299,54 @@ impl Controller { call, argument, retire, + home: self as *const Self as usize, }; } record.identity.store(0, Ordering::Relaxed); record.value.store(0, Ordering::Relaxed); record.returned.store(0, Ordering::Relaxed); + record.ending.store(0, Ordering::Relaxed); record.state.store(old | REQUEST, Ordering::Release); - loop { - let state = record.state.load(Ordering::Acquire); - if state == old | FAILED { - record.release(old); - return Err(Error::Unavailable); - } - if state != old | REQUEST { - return Ok(Id { - controller: self as *const Self as usize, - slot, - generation: old, - }); - } - pause(); + self.hub.signal(); + let state = self.settle(record, |state| state != old | REQUEST); + if state == old | FAILED { + record.release(old); + return Err(Error::Unavailable); } + return Ok(Id { + controller: self as *const Self as usize, + slot, + generation: old, + }); } Err(Error::Full) } + /// Whether this thread holds a send into the controller's doorbell. + fn wakes_controller(&self) -> bool { + self.band + .load(Ordering::Acquire) + .checked_sub(1) + .is_some_and(futex::reaches) + } + + /// Park until `record`'s state satisfies `done`, and return that state. + fn settle(&self, record: &Record, done: impl Fn(u64) -> bool) -> u64 { + loop { + // The event count before the state: a transition published after + // this load moves the count, which the park re-reads once it is + // registered (`futex`'s order), so none is missed. + let seen = record.event.load(Ordering::SeqCst); + let state = record.state.load(Ordering::Acquire); + if done(state) { + return state; + } + let reached = futex::current_thread() + .is_some_and(|band| self.reach.load(Ordering::SeqCst) & (1 << band) != 0); + park(&record.event, seen, (!reached).then_some(UNREACHED_SLICE)); + } + } + fn record(&self, id: Id) -> Result<&Record, Error> { if id.controller != self as *const Self as usize { return Err(Error::Invalid); @@ -299,9 +415,7 @@ impl Controller { continue; } if next == JOIN { - while record.state.load(Ordering::Acquire) != id.generation | JOIN_READY { - pause(); - } + self.settle(record, |state| state == id.generation | JOIN_READY); } // Acquire completion before reading the value it publishes. The owner // is a different thread from the callback on weakly ordered ISAs. @@ -350,96 +464,511 @@ impl Controller { } } +/// One wait on `word`, still holding `expected`: the futex park, or — on a +/// thread that cannot park at all — the deadline park every thread can take, +/// which a spin hint stands in for only off metal. +fn park(word: &AtomicU32, expected: u32, slice: Option) { + if futex::wait(word, expected, slice) == WaitOutcome::Unsupported { + pause(); + } +} + fn pause() { if !crate::pace::cede() { core::hint::spin_loop(); } } +/// The managed thread's entry: announce its end however it comes, relay +/// through the controller, run the callback. fn child_entry(argument: usize) { // SAFETY: controller storage is static; record cannot be reused until join. let record = unsafe { &*(argument as *const Record) }; #[cfg(target_os = "none")] - record.identity.store( - crate::entry::geometry::thread_owner_identity().unwrap_or(0), - Ordering::Release, - ); + { + crate::thread::on_exit(child_ending, argument); + record.identity.store( + crate::entry::geometry::thread_owner_identity().unwrap_or(0), + Ordering::Release, + ); + } // Acquire the request publication (or the owner's subsequent state) // before reading callback storage. It remains immutable through native join. let _ = record.state.load(Ordering::Acquire); let start = unsafe { *record.start.get() }; + // SAFETY: `home` names the static controller whose record this is. + let _ = futex::adopt_hub(&unsafe { &*(start.home as *const Controller) }.hub); record .value .store((start.call)(start.argument), Ordering::Release); record.returned.store(1, Ordering::Release); } -fn controller_entry(argument: usize) { - // SAFETY: start requires static storage. - let controller = unsafe { &*(argument as *const Controller) }; - let endowment = Endowment::inherited(); - let mut tickets: [Option; COUNT] = [const { None }; COUNT]; - loop { - for (record, ticket) in controller.records.iter().zip(tickets.iter_mut()) { - let state = record.state.load(Ordering::Acquire); - let generation = state & !MASK; - if state & MASK == REQUEST { - *ticket = super::thread::spawn_with_arg( - child_entry, - record as *const Record as usize, - endowment.as_slice(), - ); - record.state.store( - generation | if ticket.is_some() { RUNNING } else { FAILED }, - Ordering::Release, - ); +/// The child's exit hook: say the thread is ending, then wake the +/// controller to reap it. +#[cfg(target_os = "none")] +fn child_ending(argument: usize) { + // SAFETY: as in `child_entry`; the record outlives its child. + let record = unsafe { &*(argument as *const Record) }; + record.ending.store(1, Ordering::SeqCst); + // SAFETY: `start` is immutable until the controller has reaped this + // thread, which it cannot do before this hook returns. + let start = unsafe { *record.start.get() }; + let controller = unsafe { &*(start.home as *const Controller) }; + controller.hub.signal(); +} + +#[cfg(target_os = "none")] +mod owner { + //! The controller thread and its founding. + + use core::sync::atomic::Ordering; + + use kore_abi::cap::{ResourceKind, Rights}; + use kore_abi::session::SlotRef; + use kore_abi::task::role; + + use super::{ + COMPLETE, COUNT, Controller, DETACHED, JOIN, JOIN_READY, MASK, PARKING, POLLING, + REAP_RETRY, RUNNING, Record, child_entry, + }; + use crate::cap::{ + CHANNEL_ENDPOINT_RIGHTS, ChanSendToken, ChannelRoot, ChannelSpaceToken, verbs, + }; + use crate::futex::{self, PEER_BASE, WaitOutcome}; + use crate::thread::{self, JoinScratch, Thread, WAKE_SEND_RIGHTS, WorkerCap}; + + /// The most capabilities either founding here endows: the controller's + /// doorbell, its own kick, the heap, the host's wake, a channel-space + /// grant and the starter's kick. + const MAX_ENDOW: usize = 6; + + /// A founding's endowment, filled in order. + struct Endowment { + caps: [WorkerCap; MAX_ENDOW], + len: usize, + } + + impl Endowment { + fn new() -> Self { + let vacant = WorkerCap { + role: 0, + index: 0, + subject: SlotRef::from_wire(0), + kind: 0, + rights: Rights::empty(), + }; + Self { + caps: [vacant; MAX_ENDOW], + len: 0, } - let Some(child) = ticket.as_ref() else { - continue; + } + + /// Append `cap`, if any; `true` when it is now in the endowment. + fn push(&mut self, cap: Option) -> bool { + let (Some(cap), Some(at)) = (cap, self.caps.get_mut(self.len)) else { + return false; }; - if !matches!( - child.poll(), - Ok(crate::cap::TaskPollOutcome::Exited(_) | crate::cap::TaskPollOutcome::Reaped) - ) { - continue; + *at = cap; + self.len += 1; + true + } + + fn as_slice(&self) -> &[WorkerCap] { + &self.caps[..self.len] + } + } + + /// A send at `DOORBELL@index`, derived from `subject` with `rights`. + fn send(index: u16, subject: SlotRef, rights: Rights) -> WorkerCap { + WorkerCap { + role: role::DOORBELL, + index, + subject, + kind: ResourceKind::ChanSend as u16 as u8, + rights, + } + } + + /// A thread's own doorbell (`DOORBELL@0`), derived from its channel's root. + fn doorbell(root: &ChannelRoot) -> WorkerCap { + WorkerCap { + role: role::DOORBELL, + index: 0, + subject: root.slot(), + kind: ResourceKind::ChanRecv as u16 as u8, + rights: CHANNEL_ENDPOINT_RIGHTS, + } + } + + /// One thread's wake channel: its root and the send its founder keeps. + /// The founder's own recv is surplus — the thread's recv is a fresh + /// derive of the root at its founding (the star's rule). + struct Wires { + root: ChannelRoot, + kick: ChanSendToken, + } + + impl Wires { + fn mint(space: &ChannelSpaceToken) -> Option { + let pair = verbs::mint_channel(space).ok()?; + let _ = pair.recv.try_retire(); + Some(Self { + root: pair.root, + kick: pair.send, + }) + } + + fn retire(self) { + let _ = self.kick.try_retire(); + let _ = self.root.try_retire(); + } + } + + impl Controller { + /// Found the controller thread from the calling thread: the mode it + /// will wait in, or `None` when it could not be founded. + pub(super) fn found(&'static self) -> Option { + let starter = futex::current_thread()? as u32; + self.starter.store(starter + 1, Ordering::Release); + let rt = crate::entry::thread_context()?; + let space = rt + .bootstrap() + .handle(role::CHANNEL_SPACE, 0) + .and_then(verbs::claim_channel_space); + let wires = space.as_ref().and_then(Wires::mint); + let own_kick = thread::own_kick(); + + let mut endowment = Endowment::new(); + if let Some(wires) = &wires { + endowment.push(Some(doorbell(&wires.root))); + // The controller derives every child's send into itself from + // this one, so it keeps DERIVE. + endowment.push(Some(send( + futex::OWN_KICK, + wires.root.slot(), + WAKE_SEND_RIGHTS, + ))); + } + endowment.push(thread::heap_endowment()); + // Peer 0. For a starter that is thread 0 it is that thread's own + // kick, and the starter is reached through it. + let peer_zero = endowment.push(thread::wake_endowment()); + endowment.push(thread::channel_space_endowment()); + // Any other starter is reached at its own band's peer index. + let reached = if starter == 0 { + own_kick.is_some() && peer_zero + } else { + endowment.push( + own_kick.map(|kick| send(PEER_BASE + starter as u16, kick, WAKE_SEND_RIGHTS)), + ) + }; + let Some(ticket) = thread::spawn_with_arg( + super::controller_entry, + self as *const Self as usize, + endowment.as_slice(), + ) else { + if let Some(wires) = wires { + wires.retire(); + } + return None; + }; + let band = ticket.band_index(); + self.band.store(band + 1, Ordering::Release); + if reached { + self.reach.fetch_or(1 << starter, Ordering::SeqCst); + } + // This owner deliberately lives until process teardown. Managed + // children never use native detach: their tickets are retained. + ticket.detach(); + // The claimed grant is a witness, not a reservation: dropping it + // retires nothing (the star's rule, `star::Wires`). + drop(space); + let Some(wires) = wires else { + return Some(POLLING); + }; + // Wake the controller directly, and relay through it. + if !futex::adopt_peer(band, wires.kick.slot()) { + wires.retire(); + return Some(POLLING); + } + let _ = futex::adopt_hub(&self.hub); + // The channel lives as long as the controller does: its root and + // this thread's send stay in this thread's arena for good. + core::mem::forget(wires); + Some(PARKING) + } + } + + /// A managed child the controller founded and will reap. + struct Child { + thread: Thread, + band: u32, + /// Its doorbell's root and the controller's send into it; `None` + /// when the controller has no channel space to mint them. + wires: Option, + } + + /// The controller thread's own state, on its stack for the process life. + pub(super) struct Owner { + controller: &'static Controller, + band: u32, + /// The grant children's doorbells are minted from. + space: Option, + /// This thread's own kick: every child's send into it derives from it. + kick: Option, + /// The starter's band and the send into it, when the starter is not + /// thread 0 (which children reach as peer 0). + starter: Option<(u32, SlotRef)>, + /// One report landing, reused serially by every probe, join arm and + /// reap: the controller never spends an arena slot to observe. + scratch: Option, + children: [Option; COUNT], + /// The hub's relay count at the last sweep. + swept: u32, + } + + /// What one reap attempt came to. + enum Reap { + /// Nothing left to reap. + Done, + /// The child announced its end and has not committed it. + Pending, + /// The kernel refused an observation or the reclamation. + Failed, + } + + impl Owner { + pub(super) fn new(controller: &'static Controller) -> Self { + let band = futex::current_thread().map_or(u32::MAX, |band| band as u32); + let rt = crate::entry::thread_context(); + let boot = rt.as_ref().map(|rt| rt.bootstrap()); + let space = boot + .as_ref() + .and_then(|boot| boot.handle(role::CHANNEL_SPACE, 0)) + .and_then(verbs::claim_channel_space); + // Derivable, or no child is handed a send into this thread. + let kick = thread::own_kick(); + let starter = controller + .starter + .load(Ordering::Acquire) + .checked_sub(1) + .filter(|&starter| starter != 0) + .and_then(|starter| { + let send = boot + .as_ref()? + .handle(role::DOORBELL, PEER_BASE + starter as u16)?; + Some((starter, send)) + }); + Self { + controller, + band, + space, + kick, + starter, + scratch: JoinScratch::reserve(), + children: [const { None }; COUNT], + swept: controller.hub.relays(), } - match super::thread::join(ticket.take().unwrap()) { - Err(error) => *ticket = Some(error.into_thread()), - Ok(_) => { - // Child can no longer access its context; cleanup also runs - // for abrupt native exit and detached completion. - let start = unsafe { *record.start.get() }; - if let Some(retire) = start.retire { - retire(start.argument); + } + + /// Serve the records for the process's life. + pub(super) fn serve(mut self) -> ! { + let controller = self.controller; + loop { + let seen = controller.hub.word().load(Ordering::SeqCst); + let owed = controller.hub.relays(); + if owed != self.swept { + self.swept = owed; + futex::sweep(); + } + let mut watch = None; + let mut failed = false; + for (slot, record) in controller.records.iter().enumerate() { + let state = record.state.load(Ordering::Acquire); + if state & MASK == super::REQUEST { + self.found_child(slot, record, state & !MASK); } - loop { - let old = record.state.load(Ordering::Acquire); - let next = match old & MASK { - RUNNING => COMPLETE, - JOIN => JOIN_READY, - DETACHED => { - record.release(generation); - break; + if record.ending.load(Ordering::Acquire) != 0 { + match self.reap(slot, record) { + Reap::Done => {} + Reap::Pending => { + watch.get_or_insert(slot); } - _ => unreachable!(), - }; - if record - .state - .compare_exchange( - old, - generation | next, - Ordering::Release, - Ordering::Acquire, - ) - .is_ok() - { - break; + Reap::Failed => failed = true, } } } + let word = controller.hub.word(); + let outcome = match (watch, &self.scratch) { + (Some(slot), Some(scratch)) if !failed => match &self.children[slot] { + Some(child) => futex::wait_or_end( + word, + seen, + None, + child.thread.handle(), + scratch.report_slot(), + ), + None => futex::wait_quiet(word, seen, None), + }, + _ => futex::wait_quiet(word, seen, failed.then_some(REAP_RETRY)), + }; + if outcome == WaitOutcome::Unsupported { + super::pause(); + } } } - pause(); + + /// Found the child `record` asks for, then publish the outcome. + fn found_child(&mut self, slot: usize, record: &'static Record, generation: u64) { + let child = self.spawn(record); + let next = if child.is_some() { + RUNNING + } else { + super::FAILED + }; + self.children[slot] = child; + record.state.store(generation | next, Ordering::Release); + record.announce(); + } + + fn spawn(&mut self, record: &'static Record) -> Option { + let wires = self.space.as_ref().and_then(Wires::mint); + let mut endowment = Endowment::new(); + if let Some(wires) = &wires { + endowment.push(Some(doorbell(&wires.root))); + } + // A child neither founds threads nor hands these on: notice + // rights only. + endowment.push( + self.kick + .map(|kick| send(PEER_BASE + self.band as u16, kick, verbs::NOTICE_RIGHTS)), + ); + endowment.push(self.starter.map(|(starter, kick)| { + send(PEER_BASE + starter as u16, kick, verbs::NOTICE_RIGHTS) + })); + endowment.push(thread::heap_endowment()); + endowment.push(thread::wake_endowment()); + let Some(thread) = thread::spawn_with_arg( + child_entry, + record as *const Record as usize, + endowment.as_slice(), + ) else { + if let Some(wires) = wires { + wires.retire(); + } + return None; + }; + let band = thread.band_index(); + if let Some(wires) = &wires + && futex::adopt_peer(band, wires.kick.slot()) + { + self.controller.reach.fetch_or(1 << band, Ordering::SeqCst); + } + Some(Child { + thread, + band, + wires, + }) + } + + /// Reap `slot`'s child if it has ended, then publish its end. + fn reap(&mut self, slot: usize, record: &Record) -> Reap { + if self.scratch.is_none() { + // The landing was not available when this thread started. + self.scratch = JoinScratch::reserve(); + } + let (Some(child), Some(scratch)) = + (self.children[slot].as_ref(), self.scratch.as_mut()) + else { + return if self.children[slot].is_none() { + Reap::Done + } else { + Reap::Failed + }; + }; + match thread::poll_reserved(&child.thread, scratch) { + Ok(crate::cap::TaskPollOutcome::Running) => return Reap::Pending, + Ok(_) => {} + Err(_) => return Reap::Failed, + } + // Withdraw the child's kick before its band can be handed to + // another thread: no wake may present this send, and no client + // at that band may count on this controller reaching it. + futex::forget_peer(child.band); + self.controller + .reach + .fetch_and(!(1 << child.band), Ordering::SeqCst); + let Some(Child { + thread, + band, + wires, + }) = self.children[slot].take() + else { + return Reap::Done; + }; + if let Err(error) = thread::join_reserved(thread, scratch) { + self.children[slot] = Some(Child { + thread: error.into_thread(), + band, + wires, + }); + return Reap::Failed; + } + if let Some(wires) = wires { + wires.retire(); + } + // Child can no longer access its context; cleanup also runs + // for abrupt native exit and detached completion. + let start = unsafe { *record.start.get() }; + if let Some(retire) = start.retire { + retire(start.argument); + } + record.ending.store(0, Ordering::Relaxed); + loop { + let old = record.state.load(Ordering::Acquire); + let generation = old & !MASK; + let next = match old & MASK { + RUNNING => COMPLETE, + JOIN => JOIN_READY, + DETACHED => { + record.release(generation); + break; + } + // Only this thread moves a founded child's record out of + // these three. + _ => break, + }; + if record + .state + .compare_exchange(old, generation | next, Ordering::Release, Ordering::Acquire) + .is_ok() + { + break; + } + } + record.announce(); + Reap::Done + } + } +} + +fn controller_entry(argument: usize) { + // SAFETY: start requires static storage. + let controller = unsafe { &*(argument as *const Controller) }; + #[cfg(target_os = "none")] + owner::Owner::new(controller).serve(); + #[cfg(not(target_os = "none"))] + let _ = controller; +} + +#[cfg(not(target_os = "none"))] +impl Controller { + /// Off metal no thread founds another: the controller never starts. + fn found(&'static self) -> Option { + let _ = child_entry; + let _ = controller_entry; + None } } @@ -492,4 +1021,48 @@ mod tests { ); assert_eq!(controller.join(final_id), Err(Error::Invalid)); } + #[test] + fn a_parked_joiner_is_released_by_the_announced_transition() { + // The joiner's half of the protocol, with a host thread standing in + // for the controller: it moves the record JOIN → JOIN_READY and + // announces it; the joiner returns the value only after that. + let controller = Controller::new(); + let record = &controller.records[0]; + record.state.store(RUNNING, Ordering::Release); + let id = Id { + controller: &controller as *const Controller as usize, + slot: 0, + generation: 0, + }; + std::thread::scope(|scope| { + let joiner = scope.spawn(|| controller.join(id)); + while record.state.load(Ordering::Acquire) != JOIN { + std::thread::yield_now(); + } + record.value.store(5, Ordering::Relaxed); + record.returned.store(1, Ordering::Release); + let before = record.event.load(Ordering::SeqCst); + record.state.store(JOIN_READY, Ordering::Release); + record.announce(); + assert_eq!(record.event.load(Ordering::SeqCst), before.wrapping_add(1)); + assert_eq!(joiner.join().unwrap(), Ok(5)); + }); + assert_eq!(record.state.load(Ordering::Acquire), 16); + } + #[test] + fn an_unstarted_controller_refuses_and_never_waits() { + let controller: &'static Controller = + std::boxed::Box::leak(std::boxed::Box::new(Controller::new())); + assert_eq!(controller.create(|_| 0, 0), Err(Error::Unavailable)); + // Off metal the founding cannot happen; the refusal leaves it idle + // and restartable rather than stuck in its founding state. + assert_eq!(controller.start(), Err(Error::Unavailable)); + assert_eq!(controller.started.load(Ordering::Acquire), IDLE); + assert_eq!(controller.create(|_| 0, 0), Err(Error::Unavailable)); + } + #[test] + fn records_own_their_interference_span() { + assert_eq!(core::mem::align_of::(), 128); + assert_eq!(core::mem::size_of::(), 128); + } } diff --git a/native/provenance.json b/native/provenance.json index c60b681..b582189 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "eab77f8d904352eabe65fedb84a455d6ccee5b7d", + "revision": "0018956a3aca19dc6acea8d28fb8ec9fadefcb09", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", @@ -200,14 +200,14 @@ "crates/rt/src/clock/x86_64.rs": "ca499d4dc51b7ed0eb196b831295b5fdecb09a1bd2df628a82e749e27011994a", "crates/rt/src/core_id.rs": "cadde39801a1d0ed05d70d0634a9e52d58043878737d790c4130183751b622cf", "crates/rt/src/crash.rs": "b6463a490f80124e22fdb5e13652bbaa94461eb1c4aefcffeca2e7dc7e977611", - "crates/rt/src/entry.rs": "ec5a610c79a1d5d7f02316bf0b2e77b8203a4d7a09c20f7312da73a7aa7c807d", + "crates/rt/src/entry.rs": "e76b7b7014a4c0b7859091c93fc4a8fadfd04690307c5021bfdaf76b8dc34b06", "crates/rt/src/event.rs": "709a5500231735ad5bdb9ab9ab2fcee7b8f72f93e1b4741afba72fdc8b7c54f3", "crates/rt/src/executor/mod.rs": "071e3dc66bca448e6dabf96976702600bc77fdc98933d58847662fb2de073cb8", "crates/rt/src/executor/task.rs": "b3017cb97f287b8796c61d245c49e5c1b0ebfabe8c535cc2b3842cdbc854dc29", "crates/rt/src/executor/time.rs": "62f9ca9e5e17b395ec2b19a52fc6ed1bb53fd4acc3554cfca81eaf20e48fc272", "crates/rt/src/exit.rs": "3bf7e8ce6dac63b4ce365bdf2e743a9fd171851f1c5e40f8afbb4548877cef1b", "crates/rt/src/fiber_stack.rs": "48bf4fedba07ea2826615368e22882301ae4e0bbce0d893d0d7754d86bd02ce6", - "crates/rt/src/futex.rs": "9ec11227ff7260d141589fa63fdbb238eed3f52e9731ed4d9c7a14c9523a6709", + "crates/rt/src/futex.rs": "df5622a1bd811d7d97ec20dc44a568dba3a2c995254fc30405a60d41371e7390", "crates/rt/src/heap/mod.rs": "a9c39d3de56187a14d4633a229cbbb6e7ddb2c438d6f9f154b3b2551031b9f81", "crates/rt/src/heap/pool.rs": "67d42c6ffc7c7e176d21e97dd3179a3e4ff331f46dd457ed4d29914b1a404de8", "crates/rt/src/heap/shard.rs": "d97f91b3ecc71e3ec56fde0017f9d98cb99c5fbeeffe6ff79ddd153f5c6690fa", @@ -244,8 +244,8 @@ "crates/rt/src/svc/serve.rs": "a22172638a94e3ab52b88095dff9d78c88a1d693c81c6242af4932881d268706", "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", - "crates/rt/src/thread.rs": "4ee436e8828038c1dcecc196b631c20e4804e787585eaf7c532f85f8353c450d", - "crates/rt/src/thread_controller.rs": "aef46a87fc013df9139f9be508ac21da970d906f2a0e8e3549ea05072f1ce3ae", + "crates/rt/src/thread.rs": "3b6b1087a4c20bb592331ae4c48692d8f93056afc98be9708ae32236f494980e", + "crates/rt/src/thread_controller.rs": "5595f7f37bb77edf8a7fa498fcb5c65395f6227e24a651b62bb13db393ebfc30", "crates/rt/src/tls.rs": "85bd95bafb3de73d249d7a2eb8779b5afc929077e27b4baa5f7c7428063063ab", "crates/rt/src/transport/mod.rs": "d5af890c800dd53abdbea8329383068cf43f0bf294ccb6b46dd7d8592ae5f321", "crates/rt/src/upcall.rs": "c0cbd2a59479dfba006850bb1bc5dd4f20f59317d178af776ba678f0e6404f02", From b9eb3c2ddb1a75d669721e81d97b1010cb5f7d5a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 22:01:38 +0000 Subject: [PATCH 07/18] c, ffi: a C program's threads park on the futex - kore-ffi's `kore_thread_join` waited for the controller to retire its target by ceding a millisecond at a time. It now parks on the target's retirement word, which `retire` stores and then wakes. A joiner whose cancellation is enabled parks one slice at a time, as a cancellable futex wait already does: a cancellation request is a flag no wake carries. Both slices are now one named constant, `CANCEL_SLICE_NS`. Every other joiner, the program's main thread among them, parks until the controller wakes it. - kore-c founds the program's main thread holding a channel-space grant of its own. The first `pthread_create` starts kore-rt's thread controller on the main thread, which mints the controller's doorbell, and through it each child's, from that grant. Without one the controller still runs, on the deadline park. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- crates/c/src/program.rs | 7 ++++++- crates/ffi/src/futex.rs | 3 ++- crates/ffi/src/thread.rs | 24 ++++++++++++++++++------ crates/ffi/src/thread_local.rs | 6 ++++++ docs/provenance.md | 13 ++++++++++--- 5 files changed, 42 insertions(+), 11 deletions(-) diff --git a/crates/c/src/program.rs b/crates/c/src/program.rs index 96b5946..91da94a 100644 --- a/crates/c/src/program.rs +++ b/crates/c/src/program.rs @@ -310,9 +310,14 @@ impl Founded { let rt = entry::thread_context()?; let doorbell = rt.doorbell()?; HOSTED.store(true, Ordering::Release); + // A mint grant of its own: the first `pthread_create` on the main + // thread starts kore-rt's thread controller there, which mints the + // doorbells its threads park on (`thread_controller`). Without one the + // controller still runs, taking the deadline park. + let grant = thread::channel_space_endowment(); match thread::spawn_with_doorbell_on( main_thread, - &[], + grant.as_slice(), WorkerStack::Bytes(crate::api::KORE_MAIN_STACK_BYTES), ) { Some(worker) => Some(Self { diff --git a/crates/ffi/src/futex.rs b/crates/ffi/src/futex.rs index c76621d..6e04454 100644 --- a/crates/ffi/src/futex.rs +++ b/crates/ffi/src/futex.rs @@ -91,7 +91,8 @@ pub unsafe extern "C" fn kore_futex_wait( Ok(time) => time.as_nanos(), Err(_) => return KoreFutexOutcome::Unsupported, }; - let slice = deadline.map_or(1_000_000, |end| end.saturating_sub(now).min(1_000_000)); + let slice = crate::thread_local::CANCEL_SLICE_NS; + let slice = deadline.map_or(slice, |end| end.saturating_sub(now).min(slice)); let outcome = futex::wait(word, expected, Some(Duration::from_nanos(slice))); unsafe { crate::kore_thread_testcancel() }; if outcome != futex::WaitOutcome::TimedOut { diff --git a/crates/ffi/src/thread.rs b/crates/ffi/src/thread.rs index a5a550c..9e05906 100644 --- a/crates/ffi/src/thread.rs +++ b/crates/ffi/src/thread.rs @@ -98,7 +98,11 @@ fn retire(argument: usize) { // SAFETY: called exactly once by native owner after successful reclamation. unsafe { let context = NonNull::new_unchecked(argument as *mut StartContext); - context.as_ref().control.retired.store(1, Ordering::Release); + let control = &context.as_ref().control; + // Store, then wake (the futex's order): a joiner parked on the word + // re-reads it once it is registered, so neither can miss the other. + control.retired.store(1, Ordering::SeqCst); + kore::futex::wake_all(&control.retired); release(context); } } @@ -196,16 +200,24 @@ pub unsafe extern "C" fn kore_thread_join( return KORE_THREAD_EDEADLK; } // Wait without claiming the native join record: cancellation must leave the - // target joinable. Only the controller can retire native capabilities. + // target joinable. Only the controller can retire native capabilities, and + // it wakes `retired` when it does. loop { unsafe { crate::kore_thread_testcancel() }; let control = unsafe { &ticket.as_ref().control }; - if control.retired.load(Ordering::Acquire) != 0 { + if control.retired.load(Ordering::SeqCst) != 0 { break; } - // Independent native workers need not own a doorbell. Use the - // controller's deadline park, which requires no peer wake capability. - if !kore::runtime::pace::cede() { + // A cancellation request is a flag, not a wake: a joiner that can be + // cancelled parks one slice at a time, as a cancellable futex wait + // does. Any other joiner parks until the controller retires the target. + let slice = crate::thread_local::cancellable() + .then(|| core::time::Duration::from_nanos(crate::thread_local::CANCEL_SLICE_NS)); + if kore::futex::wait(&control.retired, 0, slice) == kore::futex::WaitOutcome::Unsupported + // A native worker with no doorbell takes the deadline park, which + // needs no wake capability at all. + && !kore::runtime::pace::cede() + { return KORE_THREAD_EAGAIN; } } diff --git a/crates/ffi/src/thread_local.rs b/crates/ffi/src/thread_local.rs index d0e24c3..64bd577 100644 --- a/crates/ffi/src/thread_local.rs +++ b/crates/ffi/src/thread_local.rs @@ -15,6 +15,12 @@ pub const KORE_THREAD_CANCEL_ENABLE: i32 = 0; pub const KORE_THREAD_CANCEL_DISABLE: i32 = 1; pub const KORE_THREAD_CANCEL_DEFERRED: i32 = 0; pub const KORE_THREAD_CANCEL_ASYNCHRONOUS: i32 = 1; +/// How long one park lasts at a cancellation point that waits: a request is +/// a flag no wake carries, so a cancellable wait re-checks it this often. +/// One millisecond is the cadence the native runtime parks an unreachable +/// waiter at (`kore-rt`'s `pace::CEDE_NS`): a cancellation is acted on within +/// one, and an idle cancellable waiter wakes no more often than that. +pub(crate) const CANCEL_SLICE_NS: u64 = 1_000_000; pub type KoreThreadKey = u64; pub type KoreThreadDestructor = Option; diff --git a/docs/provenance.md b/docs/provenance.md index 347f3f9..737162c 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -321,10 +321,10 @@ Kore-target build; CI's host job runs them. ## A C program's runtime, over Kore #1494's kore-rt -The export records Kore `eab77f8d904352eabe65fedb84a455d6ccee5b7d`, a commit +The export records Kore `0018956a3aca19dc6acea8d28fb8ec9fadefcb09`, a commit on MicroPerceptron/kore#1494's branch, which brings TinyCC to Kore's shell. -Ten of the 23 commits since `7074bda2` reach the closure, which keeps its 26 -packages and 382 files: +Eleven of the 26 commits since `7074bda2` reach the closure, which keeps its +26 packages and 382 files: - `kore-rt` carries what `kore-c`'s host needs: - `startup::barrier`, the process barrier without the exit; @@ -332,6 +332,13 @@ packages and 382 files: - `crash::report`, a panic's report without the thread's end; - also a per-thread C `errno` word, guarded worker stacks sized in bytes, and the verbs and bundle call a shell's private install takes. +- `kore-rt`'s thread controller, which `kore-ffi`'s pthreads run on, parks + on the futex and never polls (Kore `0018956a`). Its threads get their own + doorbells and the controller relays the wakes they cannot deliver to each + other. A thread's end reaches it through the exit hook `thread::exit` + runs, and its reap parks on a join arm. Here `kore-ffi`'s join parks on + its retirement word, which the controller wakes. `kore-c`'s main thread + holds a channel-space grant to mint the controller's doorbells from. - `kore-abi` carries the install request and its replies, and the claimant band sized by driver rows (`MAX_CLAIMANT_FACES`), which the device service uses. From c476f3277f1a6d8a607949d2d1718e9f06489bb4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 03:34:00 +0000 Subject: [PATCH 08/18] native: re-export at Kore 2a0730dd, where threads that found take a founder's arena `tools/sync-native.py` at MicroPerceptron/kore@2a0730dd6ba3369c82f025c9957184f0bb9f8a66. kore-rt founds the threads that found others (doorbell workers such as a C program's `main`, the thread controller and its threads) with the 256-slot founding granule rather than a task's 64. It returns a committed founding's scrubbed result names at once, budgets an SPMD launch before it mints or founds, and clamps the gang to what the arena can hold. Before this, `kore_spmd_dispatch` refused from C `main` and a C program kept at most two pthreads alive. The services' `boot.toml` digests follow Kore's reseal at 76fddb9f. `fmt`, `native-check`, `owners` and `bindings-check` pass on this export. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- native/crates/rt/src/spmd.rs | 227 +++++++++++++++++- native/crates/rt/src/star.rs | 63 ++++- native/crates/rt/src/thread.rs | 270 +++++++++++++++++++--- native/crates/rt/src/thread_controller.rs | 14 +- native/provenance.json | 20 +- native/services/block/boot.toml | 4 +- native/services/dagfs/boot.toml | 4 +- native/services/device/boot.toml | 4 +- native/services/mm/boot.toml | 4 +- native/services/swap/boot.toml | 4 +- 10 files changed, 549 insertions(+), 65 deletions(-) diff --git a/native/crates/rt/src/spmd.rs b/native/crates/rt/src/spmd.rs index 135b2c9..5c8b23a 100644 --- a/native/crates/rt/src/spmd.rs +++ b/native/crates/rt/src/spmd.rs @@ -847,6 +847,7 @@ fn launch( // ticket whose founding never happened refuses its join and // detaches (the band stays spent — visible, never hidden). ── decide(ctx, ABORT); + let mut detached = false; for staged in workers.iter_mut().filter_map(Option::take) { // A refused join here is a transient observation failure on // a worker that is already exiting; one bounded retry, then @@ -855,7 +856,10 @@ fn launch( Ok(_) => {} Err(e) => match crate::thread::join_reserved(e.into_thread(), &mut join_scratch) { Ok(_) => {} - Err(e) => e.into_thread().detach(), + Err(e) => { + detached = true; + e.into_thread().detach() + } }, } } @@ -866,6 +870,15 @@ fn launch( // invariant changes later. let _ = star.finish(); } + // Every worker this launch founded is joined, so none can read the + // gate again: restage it, and the next launch over `ctx` may run. + // A detached worker may still be parked on it, and a restaged gate + // would release it into the next launch's body: `ctx` then stays + // aborted, refusing every launch over it, rather than hand one + // region's job to another's worker. + if !detached { + ctx.restage(); + } return None; } decide(ctx, RUN); @@ -1539,6 +1552,10 @@ fn dispatch_resident( .map(|m| m.fitting_width()) .unwrap_or(1) .clamp(1, DISPATCH_CAP as u32); + // No wider than this thread's arena can found: a resident gang + // holds its founder's names for the task's life, and a thread + // other than thread 0 founds from a fixed arena. + let fit = affordable(fit, false); let founded = (fit >= 2) .then(|| { found_resident::( @@ -1556,7 +1573,15 @@ fn dispatch_resident( // contract says; an executable takes `PROFILE_CHANNELS` // to found the star. Some(region) if !region.star_active() => { - let _ = region.retire(); + // The retire told every worker to end before its first + // join, so each join it could not complete is one still + // to finish: collect them all, as a dispatch collects its + // own. Dropping the region instead would leave the gang + // running, its bands, stacks and slots spent for the + // task's life. + if let Err(refused) = region.retire() { + join_dispatched(refused.spmd); + } gang.state.store(RESIDENT_REFUSED, Relaxed); return ResidentOutcome::Unavailable; } @@ -1764,6 +1789,104 @@ fn block_split(span: u32, workers: u32, worker: u32) -> (u32, u32) { (start, start + len) } +/// What a [`launch`] founds beside its workers: what its arena budget +/// ([`launch_slots`]) depends on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +struct LaunchShape { + /// The star a launch here mints: `Some(founder_kick)` — whether each + /// member's mesh carries the founder's kick — or `None` for a starless + /// launch. + star: Option, + /// Whether the launch founds the star's relay hub: a one-shot region does, + /// a resident one does not. + hub: bool, + /// Whether each worker takes a TLS block. + tls: bool, + /// Whether each compute worker is endowed the process heap. + heap: bool, +} + +/// The arena names a [`launch`] of `width` workers holds at its peak. +/// +/// The launch holds its join landing and the star's wires +/// ([`star::wires_slots`](crate::star::wires_slots)) throughout. Each gang +/// session then holds every one of its workers' founding names at its commit +/// ([`thread::gang_founding_slots`](crate::thread::gang_founding_slots)), on +/// top of what the sessions before it kept ([`RETAINED_SLOTS`] a worker), and +/// the hub's session holds its own on top of all of them. A launch whose peak +/// fits the arena cannot run out of names part way through, where a refusal +/// leaks what it claimed. The sessions are [`launch`]'s own split +/// ([`founding_sessions`]): a starred session carries at most +/// [`MESH_CEILING`](crate::star::MESH_CEILING) workers. +/// +/// [`RETAINED_SLOTS`]: crate::thread::RETAINED_SLOTS +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +fn launch_slots(width: u32, shape: LaunchShape) -> usize { + use crate::thread::{RETAINED_SLOTS, gang_founding_slots}; + let wires = shape + .star + .map_or(0, |_| crate::star::wires_slots(width as usize)); + let mut held = 1 + wires; + let mut peak = held; + let ceiling = match shape.star { + Some(_) => crate::star::MESH_CEILING, + None => crate::session::MAX_SESSION_FOUNDS, + }; + for (start, end) in founding_sessions(width, ceiling as u32) { + let chunk = (end - start) as usize; + let mesh = shape + .star + .map_or(0, |kick| crate::star::mesh_caps(kick, chunk)); + let endowments = chunk * (mesh + usize::from(shape.heap)); + peak = peak.max(held + gang_founding_slots(chunk, shape.tls, endowments)); + held += chunk * (RETAINED_SLOTS + usize::from(shape.tls)); + } + if shape.hub && shape.star.is_some() { + // The hub's endowment: its own recv and a send to every worker. + peak = peak.max(held + gang_founding_slots(1, shape.tls, 1 + width as usize)); + } + peak +} + +/// The widest launch of at most `want` workers whose peak +/// ([`launch_slots`]) fits `available` arena names, or 0 when not one worker +/// fits. +/// +/// Every width is weighed, widest first: the peak is not monotonic in the +/// width, because a wider gang splits into more sessions whose meshes are +/// smaller (seven meshed workers cost one session of seven; eight cost two of +/// four). +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +fn affordable_width(want: u32, shape: LaunchShape, available: usize) -> u32 { + (1..=want) + .rev() + .find(|&width| launch_slots(width, shape) <= available) + .unwrap_or(0) +} + +/// The shape a launch over this thread would take, read before it mints +/// anything. +#[cfg(target_os = "none")] +fn launch_shape(hub: bool) -> LaunchShape { + LaunchShape { + star: crate::star::shape().map(|star| star.founder_kick), + hub, + tls: crate::entry::geometry::tls_template().is_some(), + heap: worker_heap().is_some(), + } +} + +/// [`affordable_width`] on this thread's arena, now. +#[cfg(target_os = "none")] +fn affordable(want: u32, hub: bool) -> u32 { + affordable_width( + want, + launch_shape(hub), + crate::entry::geometry::available_cap_slots(), + ) +} + /// **The parallel-for** — run `body` over `0..tiles`, sized to the /// machine, with no thread configuration at all. /// @@ -1855,6 +1978,12 @@ fn dispatch_erased( ResidentOutcome::Busy => return None, ResidentOutcome::Unavailable => {} } + // The one-shot region, as wide as asked and no wider than this thread's + // arena can found whole; `None` only when not one worker fits. + let width = affordable(width, true); + if width == 0 { + return None; + } let env = &body as *const &(dyn Fn(&'static SpmdCtx, u32) + Sync) as *mut (); let region = launch::( &DISPATCH_CTX, @@ -2319,6 +2448,100 @@ mod tests { assert_eq!(split(12), [6, 6]); } + const fn shape(star: Option, hub: bool, tls: bool) -> LaunchShape { + LaunchShape { + star, + hub, + tls, + heap: true, + } + } + + /// The budget counts what the founding holds, step by step: the join + /// landing and the star's wires, every worker's founding names and mesh + /// at the session's commit, what each session keeps, then the hub's + /// session on top. Three meshed workers founded off thread 0 peak at 46 + /// names: two more than a 64-slot worker's arena had left, the C + /// `main` that `kore_spmd_dispatch` refused on. + #[test] + fn a_launch_budget_counts_what_its_founding_holds() { + // Resident, starred, no founder kick: 1 + 9 wires, then 3 × 7 + // founding names and 3 × (4 mesh + heap) endowments. + assert_eq!(launch_slots(3, shape(Some(false), false, false)), 46); + // The one-shot's hub founds after the gang returned its plan: 10 + // held + 3 × 4 kept + 7 + (1 + 3) endowments = 33, under the gang's + // peak. + assert_eq!(launch_slots(3, shape(Some(false), true, false)), 46); + // Thread 0's star carries its kick: one more mesh cap a worker. + assert_eq!(launch_slots(3, shape(Some(true), false, false)), 49); + // Starless: the landing, then 3 × (7 + heap). + assert_eq!(launch_slots(3, shape(None, true, false)), 25); + // A TLS block is one more founding name and one more kept name. + assert_eq!(launch_slots(3, shape(None, false, true)), 28); + // Two sessions: the second commits over what the first kept. + // 8 meshed workers: 1 + 19, then 4 × 7 + 4 × 6 = 72, keeping 16; + // then 36 + 52 = 88. + assert_eq!(launch_slots(8, shape(Some(false), false, false)), 88); + } + + /// A wider gang can cost less at its peak: past the mesh ceiling it + /// splits into sessions whose meshes are smaller. The clamp weighs every + /// width for that reason. + #[test] + fn a_launch_peak_is_not_monotonic_in_its_width() { + let meshed = crate::star::MESH_CEILING as u32; + let starred = shape(Some(false), false, false); + assert!(launch_slots(meshed + 1, starred) < launch_slots(meshed, starred)); + let between = launch_slots(meshed + 1, starred); + assert_eq!(affordable_width(meshed + 1, starred, between), meshed + 1); + assert!(affordable_width(meshed, starred, between) < meshed); + } + + /// The clamp takes the widest width that fits and refuses only when one + /// worker does not: the 44 names a 64-slot C `main` had left found two of + /// its three workers, and a founder's arena founds every one. + #[test] + fn a_launch_is_clamped_to_its_arena() { + let resident = shape(Some(false), false, false); + assert_eq!(affordable_width(3, resident, 44), 2); + assert_eq!(affordable_width(3, resident, 46), 3); + assert_eq!( + affordable_width(3, resident, launch_slots(1, resident) - 1), + 0 + ); + assert_eq!(affordable_width(0, resident, usize::MAX), 0); + // A founder's dynamic names: the founding granule below the linker + // band, after a doorbell worker's boot names and runtime layout. + let founder = crate::entry::STARTUP_LINK_SLOT_BASE as usize - (3 + 5 + 1 + 10 + 2); + for tls in [false, true] { + for hub in [false, true] { + let widest = crate::thread::MAX_THREADS - 1; + let width = affordable_width(widest, shape(Some(false), hub, tls), founder); + assert!(width >= 16, "tls {tls} hub {hub}: {width}"); + } + } + } + + /// The starless degrade (a refused mint) never needs more than the + /// starred launch its budget was weighed for, at any width, so a launch + /// sized for its star cannot run out of names when the star fails. + #[test] + fn the_starless_degrade_fits_its_starred_budget() { + for width in 1..crate::thread::MAX_THREADS { + for kick in [false, true] { + for tls in [false, true] { + for hub in [false, true] { + assert!( + launch_slots(width, shape(None, hub, tls)) + <= launch_slots(width, shape(Some(kick), hub, tls)), + "width {width} kick {kick} tls {tls} hub {hub}" + ); + } + } + } + } + } + /// A scripted worker: joins yield the scripted outcomes in order. struct MockWorker { outcomes: &'static [Result, crate::cap::TaskWaitError>], diff --git a/native/crates/rt/src/star.rs b/native/crates/rt/src/star.rs index 7be7730..2476ce3 100644 --- a/native/crates/rt/src/star.rs +++ b/native/crates/rt/src/star.rs @@ -110,6 +110,49 @@ pub(crate) struct StarPlan { wires: Wires, } +/// Arena names a star of `workers` holds in its founder for the region's +/// life: each worker's channel root and the parent's send into it (the mint's +/// third name, the parent's recv, is retired at once), and the hub channel's +/// root, send and recv. [`provision`]'s peak is the same count: the last +/// mint's recv is the hub's, kept. +pub(crate) const fn wires_slots(workers: usize) -> usize { + 2 * workers + 3 +} + +/// The star a thread could mint, before it mints one: whether it holds a +/// channel space to mint from, and whether its members would carry the +/// founder's kick. [`provision`] mints exactly this shape, and a launch +/// budgets its arena by it (`spmd::launch_slots`). +#[cfg(target_os = "none")] +#[derive(Clone, Copy, Debug)] +pub(crate) struct StarShape { + /// The founder's own kick rides every member's mesh: only thread 0 + /// holds a send into its own doorbell (`DOORBELL@PEER_BASE`). + pub(crate) founder_kick: bool, +} + +/// This thread's [`StarShape`], or `None` with no `CHANNEL_SPACE` grant. +#[cfg(target_os = "none")] +pub(crate) fn shape() -> Option { + let rt = crate::entry::thread_context()?; + rt.bootstrap() + .handle(kore_abi::task::role::CHANNEL_SPACE, 0)?; + Some(StarShape { + founder_kick: founder_kick(&rt).is_some(), + }) +} + +/// The founder kick a member's mesh carries (see [`StarShape`]). +#[cfg(target_os = "none")] +fn founder_kick(rt: &crate::entry::Rt) -> Option { + if crate::entry::geometry::shard_index() == Some(0) { + rt.bootstrap() + .handle(kore_abi::task::role::DOORBELL, crate::futex::PEER_BASE) + } else { + None + } +} + /// Mint the region's whole forest, all-or-nothing: `workers` channels /// plus the hub's. `None` — no `CHANNEL_SPACE` boot grant, dynamic /// arena slots exhausted, a refused mint — releases everything minted @@ -164,12 +207,7 @@ pub(crate) fn provision(workers: usize) -> Option { // doorbell — a spawned thread's `DOORBELL@PEER_BASE` names thread 0's — // so a region founded elsewhere carries no founder kick and its // launcher rides the park's slice (`res/SPMD.md` §14.2). - let founder_kick = if crate::entry::geometry::shard_index() == Some(0) { - rt.bootstrap() - .handle(kore_abi::task::role::DOORBELL, crate::futex::PEER_BASE) - } else { - None - }; + let founder_kick = founder_kick(&rt); Some(StarPlan { wires: Wires { @@ -186,6 +224,11 @@ pub(crate) fn provision(workers: usize) -> Option { }) } +/// [`StarPlan::mesh_caps`] before a plan exists, for a launch's budget. +pub(crate) const fn mesh_caps(founder_kick: bool, chunk: usize) -> usize { + 2 + founder_kick as usize + chunk.saturating_sub(1) +} + /// Descriptor invocations one meshed worker costs its gang session in a /// chunk of `chunk`: two views, the boot delegate and the Found, plus its /// endowment of `chunk + 3` caps (own recv, hub send, the founder's send, @@ -219,6 +262,14 @@ pub(crate) const MESH_CEILING: usize = { #[cfg(target_os = "none")] impl StarPlan { + /// Capabilities one member of a chunk of `chunk` carries in its mesh: + /// its own recv, the hub's send, the founder's kick when the plan has + /// one, and a send to every chunk sibling — the count + /// [`worker_endow_mesh`](Self::worker_endow_mesh) fills. + pub(crate) const fn mesh_caps(&self, chunk: usize) -> usize { + mesh_caps(self.wires.founder_kick.is_some(), chunk) + } + /// Worker `w`'s endowments — **the intra-chunk kick mesh**: its own /// recv (→ `DOORBELL@0`, which the stamped landing name at /// `DOORBELL@1` completes into a working park), the hub send diff --git a/native/crates/rt/src/thread.rs b/native/crates/rt/src/thread.rs index 1a849a5..3c68103 100644 --- a/native/crates/rt/src/thread.rs +++ b/native/crates/rt/src/thread.rs @@ -415,6 +415,22 @@ fn release_thread_index(n: u32) { } } +/// **How many thread bands are free now**: how many more threads this task +/// could found before its band plan is full. An upper bound — a founding also +/// needs its founder's arena names and, for a strip stack, room in the stack +/// window — and a snapshot, which a concurrent founding or join moves. +#[cfg(target_os = "none")] +pub fn free_bands() -> u32 { + let used = THREAD_SLOTS.load(Ordering::Acquire); + (!used & (((1u64 << MAX_THREADS) - 1) as u32)).count_ones() +} + +/// Off the metal no thread founds another: no band is free. +#[cfg(not(target_os = "none"))] +pub fn free_bands() -> u32 { + 0 +} + /// How a spawned thread enters Rust: the plain `fn()` every service /// worker uses, or an **spmd entry** carrying a region launch's context /// and the index it assigned (`res/SPMD.md` §2) — the CUDA @@ -484,14 +500,15 @@ pub struct Thread { /// stack, runtime block, boot staging page. Revoking them is what /// actually unmaps the band; the parent holds them precisely so that /// a dead thread cannot keep its own stack alive. - provisioned: [SlotRef; 3], + provisioned: [SlotRef; PROVISIONED_REGIONS], tls_region: Option, - /// Founding result slots the kernel already scrubbed (the staged - /// space, the delivered view, the boot delegation, one per - /// endowment): no capability is left in them, but the *userland* slot - /// reservation is, and a task that never returned it would run out of - /// arena long before it ran out of bands. - spent: [SlotRef; 3 + MAX_WORKER_CAPS], + /// Founding result slots whose return waits for [`join`]: the staged + /// space, the delivered view, the boot delegation and one per endowment + /// of a gang founding that did not commit whole, whose outcome only the + /// join learns. A founding that commits returns them at once + /// ([`release_scrubbed`]), so a founder holds per live child only what + /// `join` revokes; this list is then empty. + spent: [SlotRef; SCRUBBED_RESULTS + MAX_WORKER_CAPS], /// Live entries in [`spent`](Self::spent). spent_len: u8, /// The arena strip this stack was carved from — `Some` for SPMD @@ -790,7 +807,12 @@ fn push_worker_thread_blocks(gblock: &mut GeomBlock, local: Window) -> bool { /// view. [`spawn_with`] when the thread must hold capabilities. #[cfg(target_os = "none")] pub fn spawn(entry: fn()) -> Option { - spawn_entry(ThreadEntry::Plain(entry), &[], WorkerStack::Band) + spawn_entry( + ThreadEntry::Plain(entry), + &[], + WorkerStack::Band, + WorkerArena::Task, + ) } // (The per-worker `spawn_worker` shim retired with G5: workers found as a @@ -807,7 +829,12 @@ pub fn spawn(entry: fn()) -> Option { /// into [`exit`]`(0)`. #[cfg(target_os = "none")] pub fn spawn_with(entry: fn(), endow: &[WorkerCap]) -> Option { - spawn_entry(ThreadEntry::Plain(entry), endow, WorkerStack::Band) + spawn_entry( + ThreadEntry::Plain(entry), + endow, + WorkerStack::Band, + WorkerArena::Task, + ) } /// Where a spawned worker's stack lives. @@ -833,6 +860,100 @@ pub enum WorkerStack { Bytes(usize), } +/// How many capability slots a spawned thread's arena holds: the one sizing +/// question a founding leaves to its founder (`FoundReq.arena_slots`, which +/// the kernel accepts in `1..=ARENA_SLOTS`). +/// +/// Every thread's arena holds its boot names, the runtime's serve scratch, +/// report and carve slots, and whatever it reserves while it runs. A thread +/// that **founds** holds more, for as long as each child lives: the child's +/// handle and the regions its stack, runtime block and boot page live in +/// (what [`join`] revokes), and, while a founding is in flight, that +/// founding's whole result plan. A thread that founds a gang, or a thread +/// controller that founds a process's threads, therefore needs an arena that +/// scales with its children, the case the kernel's founding granule is sized +/// for ("founders are the arena's real sizing case", `ARENA_SLOTS`). A thread +/// controller founded with a task's 64 slots held two live threads; with the +/// founding granule it holds over two dozen, within a couple of the band +/// plan's own limit ([`MAX_THREADS`]). +/// +/// Spawned arenas cannot grow: only thread 0 carries the extension window, so +/// the size is chosen once, here. +#[cfg(any(test, target_os = "none"))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum WorkerArena { + /// A thread that founds nothing: [`TASK_ARENA_SLOTS`]. + Task, + /// A thread that founds others: the founding granule, + /// [`ARENA_SLOT_CEILING`](crate::entry::ARENA_SLOT_CEILING). + Founder, +} + +/// A task's arena: one word of the runtime's slot ledger. +/// +/// It holds the widest endowment a worker can take, the serve scratch, the +/// report and carve slots behind it, and the short-lived reservations of a +/// thread that founds nothing (a channel mint, a join landing). The assert +/// below proves the first three fit; a widening of [`MAX_WORKER_CAPS`] or of +/// the serve scratch that broke that would fail the build here, not a +/// founding at run time. +#[cfg(any(test, target_os = "none"))] +pub(crate) const TASK_ARENA_SLOTS: u32 = u64::BITS; + +#[cfg(any(test, target_os = "none"))] +const _: () = { + // The highest boot name a worker can carry, then the runtime layout the + // entry derives after it: serve scratch, the report slot, the carve slot. + let top = child_slot::ENDOW_BASE as usize + MAX_WORKER_CAPS; + let carve = top + crate::entry::SERVE_SCRATCH_SLOTS + 2; + assert!(carve < TASK_ARENA_SLOTS as usize); + assert!(TASK_ARENA_SLOTS <= crate::entry::ARENA_SLOT_CEILING); + // Both sizes fit the per-thread slot alias band: the slot alias of a + // founder's arena is the band's whole width. + assert!(kore_abi::execution::alias_bytes(crate::entry::ARENA_SLOT_CEILING) <= SLOTS_BYTES); +}; + +#[cfg(any(test, target_os = "none"))] +impl WorkerArena { + /// The `FoundReq.arena_slots` this arena is founded with. + pub(crate) const fn slots(self) -> u32 { + match self { + Self::Task => TASK_ARENA_SLOTS, + Self::Founder => crate::entry::ARENA_SLOT_CEILING, + } + } +} + +/// The regions a founder provisions for each thread and holds until +/// [`join`] revokes them: the stack, the runtime block and the boot staging +/// page ([`Thread`]'s `provisioned`; a TLS block is one more, apart). +const PROVISIONED_REGIONS: usize = 3; + +/// The founding results a committed `Found` scrubs from the founder, beside +/// one per endowment: the staged space, the child's view and the boot +/// delegation ([`release_scrubbed`]; [`Thread`]'s `spent`). +const SCRUBBED_RESULTS: usize = 3; + +/// Arena names one thread's founding holds in its founder while it is in +/// flight, beside one per endowment: its provisioned regions, the results the +/// commit scrubs, and the handle. One more with a TLS block. +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +pub(crate) const FOUNDING_SLOTS: usize = PROVISIONED_REGIONS + SCRUBBED_RESULTS + 1; + +/// Arena names a founded thread keeps in its founder until [`join`]: its +/// handle and its provisioned regions (one more with a TLS block). The rest +/// of the founding's plan returns at commit ([`release_scrubbed`]). +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +pub(crate) const RETAINED_SLOTS: usize = 1 + PROVISIONED_REGIONS; + +/// The arena names a gang session founding `workers` threads holds at its +/// peak, just before its founding commits: every worker's claims and every +/// endowment result. +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +pub(crate) const fn gang_founding_slots(workers: usize, tls: bool, endowments: usize) -> usize { + workers * (FOUNDING_SLOTS + tls as usize) + endowments +} + /// A worker that can **park and be kicked**, and kick back: two channels /// minted off the task's `role::CHANNEL_SPACE` — the worker's own /// doorbell (its recv; the parent keeps the send that kicks it) and a @@ -952,6 +1073,14 @@ pub fn kick_parent() -> bool { /// into thread 0's doorbell, its own kick ([`own_kick`]), a heap position, /// and `extra`. `None` when the task holds no `CHANNEL_SPACE` (its profile /// lacks `channels`), the mint or a derive refuses, or [`spawn_with`] does. +/// +/// The worker takes a founder's arena, the founding granule of capability +/// slots rather than a task's 64: a doorbell worker is where a host runs a +/// program's own code (libkore's C `main`, a +/// language engine), and that code founds threads of its own — its first +/// `pthread_create` starts the [thread controller](crate::thread_controller) +/// and an SPMD dispatch founds a gang, each holding the founder's arena for +/// as long as its children live. #[cfg(target_os = "none")] pub fn spawn_with_doorbell(entry: fn(), extra: &[WorkerCap]) -> Option { spawn_with_doorbell_on(entry, extra, WorkerStack::Band) @@ -1038,8 +1167,12 @@ pub fn spawn_with_doorbell_on( }; caps[3] = heap; caps[OWN..OWN + extra.len()].copy_from_slice(extra); - let Some(thread) = spawn_entry(ThreadEntry::Plain(entry), &caps[..OWN + extra.len()], stack) - else { + let Some(thread) = spawn_entry( + ThreadEntry::Plain(entry), + &caps[..OWN + extra.len()], + stack, + WorkerArena::Founder, + ) else { let _ = pair.send.try_retire(); let _ = pair.root.try_retire(); let _ = wake.send.try_retire(); @@ -1067,6 +1200,24 @@ pub fn spawn_with_arg(entry: fn(usize), argument: usize, endow: &[WorkerCap]) -> ThreadEntry::Argument(entry, argument), endow, WorkerStack::Band, + WorkerArena::Task, + ) +} + +/// [`spawn_with_arg`] founding a thread that founds others, with a founder's +/// arena ([`WorkerArena::Founder`]): the thread controller, and the threads +/// it founds, any of which may found a gang or ask for more threads. +#[cfg(target_os = "none")] +pub(crate) fn spawn_founder_with_arg( + entry: fn(usize), + argument: usize, + endow: &[WorkerCap], +) -> Option { + spawn_entry( + ThreadEntry::Argument(entry, argument), + endow, + WorkerStack::Band, + WorkerArena::Founder, ) } #[cfg(not(target_os = "none"))] @@ -1074,11 +1225,34 @@ pub fn spawn_with_arg(_entry: fn(usize), _argument: usize, _endow: &[WorkerCap]) None } +/// Return a committed founding's result names to the slot ledger. +/// +/// A `Found` that commits moves the staged space, the child's view and every +/// endowment out of the founder (the delivery map scrubs each one's founder +/// slot) and clears the founder's delegated `Region` results, the boot page's +/// among them. No capability is left under these names, and none can have +/// been copied: the founder was inside its own trap from apply to scrub. They +/// are free the moment the commit reports `Ok`, and only that proves it: a +/// refused or unreadable commit keeps its names (the founding's +/// leak-on-refusal rule), as a gang founding that did not commit whole keeps +/// them in its tickets for `join`. +#[cfg(target_os = "none")] +fn release_scrubbed(slots: impl IntoIterator) { + for slot in slots { + crate::entry::geometry::release_cap_slot(slot); + } +} + /// The one founding every public spawn shape resolves to — the entry /// shape is the only thing the shapes disagree on, and it rides the /// start record untouched by everything below. #[cfg(target_os = "none")] -fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap], stack: WorkerStack) -> Option { +fn spawn_entry( + entry: ThreadEntry, + endow: &[WorkerCap], + stack: WorkerStack, + arena: WorkerArena, +) -> Option { if endow.len() > MAX_WORKER_CAPS { return None; } @@ -1092,7 +1266,7 @@ fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap], stack: WorkerStack) -> O // ordinary slot exhaustion before spending a band or provisioning memory, // and leave a report slot available to poll/join existing children. // Session A owns three regions (+ TLS); B owns four results + endowments. - let slots_needed = 3 + usize::from(template.is_some()) + 4 + endow.len() + 1; + let slots_needed = FOUNDING_SLOTS + usize::from(template.is_some()) + endow.len() + 1; if crate::entry::geometry::available_cap_slots() < slots_needed { return None; } @@ -1204,7 +1378,7 @@ fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap], stack: WorkerStack) -> O // ── Stamp the boot page (at its staging VA — the delegation moves // it to `boot_va`, which is what the child receives as `arg`). ── - let arena_slots: u32 = 64; + let arena_slots = arena.slots(); if let (Some(t), Some(p)) = (template, tls) { unsafe { t.initialize(p.mapping.base()) }; } @@ -1268,22 +1442,24 @@ fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap], stack: WorkerStack) -> O if committed.status() != kore_abi::session::Status::Ok { return None; } - // The join ticket: what the thread cost, in the order `join` must - // give it back. - let mut spent = [space_res; 3 + MAX_WORKER_CAPS]; - spent[1] = view_res; - spent[2] = boot_res; - for (i, e) in staged[..endow.len()].iter().enumerate() { - spent[3 + i] = e.result; - } + // The founding's result plan is spent and its names come back now: the + // commit moved the staged space, the view and every endowment into the + // child and scrubbed their founder slots, and cleared the boot + // delegation's (`release_scrubbed`). The join ticket keeps only what + // `join` must revoke. + release_scrubbed( + [space_res, view_res, boot_res] + .into_iter() + .chain(staged[..endow.len()].iter().map(|e| e.result)), + ); Some(Thread { owner, index, handle: handle_res, provisioned: [stack_res, tblock_res, stage_res], tls_region, - spent, - spent_len: (3 + endow.len()) as u8, + spent: [handle_res; SCRUBBED_RESULTS + MAX_WORKER_CAPS], + spent_len: 0, strip, }) } @@ -1487,6 +1663,19 @@ pub(crate) fn spawn_gang_workers( if star.is_some() && n > crate::star::MESH_CEILING { return false; } + // Refuse a session this arena cannot finish before claiming anything: + // past here a refusal leaks what it claimed. A planned endowment is the + // star's mesh closed with the heap; an explicit one is as given. + let endowments = if endows.is_empty() { + n * (star.map_or(0, |plan| plan.mesh_caps(n)) + usize::from(heap.is_some())) + } else { + endows.iter().map(|caps| caps.len()).sum() + }; + if crate::entry::geometry::available_cap_slots() + < gang_founding_slots(n, template.is_some(), endowments) + { + return false; + } let Some(self_space) = crate::entry::geometry::self_space_slot() else { return false; }; @@ -1509,7 +1698,8 @@ pub(crate) fn spawn_gang_workers( tls_region: Option, } let mut claims: [Option; MAX_SESSION_FOUNDS] = [const { None }; MAX_SESSION_FOUNDS]; - let arena_slots: u32 = 64; + // A gang worker runs a region's body and founds nothing. + let arena_slots = WorkerArena::Task.slots(); let transport = TrapTransport::new(); // ── Session A′: provision every worker's three pages, one commit. ── let mut a = SessionBuilder::new(); @@ -1779,15 +1969,29 @@ pub(crate) fn spawn_gang_workers( let founded_all = committed.status() == kore_abi::session::Status::Ok; // Mint every ticket now that the founding committed: whichever // workers exist are joinable, and a vacant handle's join refuses in - // the caller's abort arm rather than orphaning anyone. + // the caller's abort arm rather than orphaning anyone. A whole commit + // returns each worker's founding plan now (`release_scrubbed`); a + // partial one leaves it to the tickets, for the join that learns which + // workers exist. for (k, claim) in claims.iter_mut().take(n).enumerate() { let Some(c) = claim.take() else { return false }; - let mut spent = [c.space_res; 3 + MAX_WORKER_CAPS]; - spent[1] = c.view_res; - spent[2] = c.boot_res; - for (i, e) in endow_store[k][..endow_lens[k]].iter().enumerate() { - spent[3 + i] = e.result; - } + let endowed = &endow_store[k][..endow_lens[k]]; + let mut spent = [c.space_res; SCRUBBED_RESULTS + MAX_WORKER_CAPS]; + let spent_len = if founded_all { + release_scrubbed( + [c.space_res, c.view_res, c.boot_res] + .into_iter() + .chain(endowed.iter().map(|e| e.result)), + ); + 0 + } else { + spent[1] = c.view_res; + spent[2] = c.boot_res; + for (i, e) in endowed.iter().enumerate() { + spent[3 + i] = e.result; + } + SCRUBBED_RESULTS + endowed.len() + }; out[k] = Some(Thread { owner, index: c.index, @@ -1795,7 +1999,7 @@ pub(crate) fn spawn_gang_workers( provisioned: [c.stack_res, c.tblock_res, c.stage_res], tls_region: c.tls_region, spent, - spent_len: (3 + endow_lens[k]) as u8, + spent_len: spent_len as u8, strip: Some(c.strip), }); } diff --git a/native/crates/rt/src/thread_controller.rs b/native/crates/rt/src/thread_controller.rs index 55891d7..90e73da 100644 --- a/native/crates/rt/src/thread_controller.rs +++ b/native/crates/rt/src/thread_controller.rs @@ -31,6 +31,12 @@ //! between two threads neither can kick directly still arrives: the controller //! sweeps ([`futex::sweep`]). //! +//! Both the controller and every child take a founder's arena (the +//! founding granule of capability slots): the controller holds each live child's handle, +//! its provisioned regions and the root and send of its doorbell until the +//! reap, a whole process's worth of them, and a child may found an SPMD gang +//! of its own. +//! //! # Degrade, never wedge //! //! A starter with no channel space founds a controller without a doorbell: @@ -663,7 +669,7 @@ mod owner { own_kick.map(|kick| send(PEER_BASE + starter as u16, kick, WAKE_SEND_RIGHTS)), ) }; - let Some(ticket) = thread::spawn_with_arg( + let Some(ticket) = thread::spawn_founder_with_arg( super::controller_entry, self as *const Self as usize, endowment.as_slice(), @@ -838,8 +844,8 @@ mod owner { if let Some(wires) = &wires { endowment.push(Some(doorbell(&wires.root))); } - // A child neither founds threads nor hands these on: notice - // rights only. + // A child hands these on to no one (the threads it asks for are + // founded here, not by it): notice rights only. endowment.push( self.kick .map(|kick| send(PEER_BASE + self.band as u16, kick, verbs::NOTICE_RIGHTS)), @@ -849,7 +855,7 @@ mod owner { })); endowment.push(thread::heap_endowment()); endowment.push(thread::wake_endowment()); - let Some(thread) = thread::spawn_with_arg( + let Some(thread) = thread::spawn_founder_with_arg( child_entry, record as *const Record as usize, endowment.as_slice(), diff --git a/native/provenance.json b/native/provenance.json index b582189..987812a 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "0018956a3aca19dc6acea8d28fb8ec9fadefcb09", + "revision": "2a0730dd6ba3369c82f025c9957184f0bb9f8a66", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", @@ -234,9 +234,9 @@ "crates/rt/src/session/mod.rs": "466c2fbd8af228b3277075ad90a6d81031c1cb0a4c877f75669fff80e15338ea", "crates/rt/src/session/slots.rs": "82594125aa491429ed8c10b1316c19a82a162dc99f6dc97cdaa76398b92558e9", "crates/rt/src/session/tests.rs": "80aa5e5d3b0ace53cbbb7a8e9f5bd8760d20f93e3655021172319e88097961ae", - "crates/rt/src/spmd.rs": "ebaaa608c8177ed80091215b2b00240ae0d689e620f5fd418c860e78034b7e4b", + "crates/rt/src/spmd.rs": "8c2264263b737af8538b10c278055bb1843e9c01ead56553022aa754e222c69e", "crates/rt/src/stack_arena.rs": "b252d45bfe845038ca9c8dc32941c634e448a2150dd49cf130e537048602f77b", - "crates/rt/src/star.rs": "a6cf5ff1cbf01f18bb3605e3597da6921ca611da087a3a30ecd61cd026ce6d30", + "crates/rt/src/star.rs": "91de20ffe5734e7a3c553408b6866087af179b7deddb2d576f05be0ca709ab66", "crates/rt/src/startup.rs": "a005158737d337ea8c270a2860d09762048347e65d2b937734f8b59e3890d324", "crates/rt/src/svc/client.rs": "062b52a0544eb11b9d4f8a938ebe75299bfd6c2ff7f42a1d8d21e8cb487c16d7", "crates/rt/src/svc/mod.rs": "3c62c81946943268c24aee3a0a8e3778442a5475dc7bf3bdb42780aeaaa958e9", @@ -244,8 +244,8 @@ "crates/rt/src/svc/serve.rs": "a22172638a94e3ab52b88095dff9d78c88a1d693c81c6242af4932881d268706", "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", - "crates/rt/src/thread.rs": "3b6b1087a4c20bb592331ae4c48692d8f93056afc98be9708ae32236f494980e", - "crates/rt/src/thread_controller.rs": "5595f7f37bb77edf8a7fa498fcb5c65395f6227e24a651b62bb13db393ebfc30", + "crates/rt/src/thread.rs": "54b067cc76785f16c36cd083e97b56456a654542ad0e57198141eeb2d8524587", + "crates/rt/src/thread_controller.rs": "e3ee7eb23edca916aad3d3bf78d2b87abea3806a59f2b5ced73263238afc5d01", "crates/rt/src/tls.rs": "85bd95bafb3de73d249d7a2eb8779b5afc929077e27b4baa5f7c7428063063ab", "crates/rt/src/transport/mod.rs": "d5af890c800dd53abdbea8329383068cf43f0bf294ccb6b46dd7d8592ae5f321", "crates/rt/src/upcall.rs": "c0cbd2a59479dfba006850bb1bc5dd4f20f59317d178af776ba678f0e6404f02", @@ -295,13 +295,13 @@ "drivers/nvme/src/regs.rs": "d9410b5c64bb70f99e042f7209119c0f0fdb57f859c6a1c07eba90199f1e98c1", "drivers/nvme/src/swap.rs": "f5e1099b83b2cdeb9d288f4d337f1a234d9dfd16c31e9d9303c10f6f14972141", "services/block/Cargo.toml": "2729b36369486a6f5c1c29200eaa4603b25c26352f06f52301b70d9d57260e43", - "services/block/boot.toml": "b39c8ff266d2110b706d887770421a732b47ab99b1e84c805a840e439c71511e", + "services/block/boot.toml": "dafeae8271530e4e21885595b7697e278143197c10cb5a617e92e805fa4a696b", "services/block/src/client.rs": "6aed79a404620f89ffa13078f615e846fdf6f3db5629c5cee5fbccbeaa7636cf", "services/block/src/lib.rs": "cf45960460cf6c9dba72f7a6727b51497275c686166a90be531d675097c275f1", "services/block/src/service.rs": "191e9d887d3f3267d6e2aa96e81996f01321bb2febb85e68581159f6e098b4b5", "services/block/src/wire.rs": "a9d64aef9ec3d58d37ca55b841933fc51b76ab94e18b245278b4519e96c65939", "services/dagfs/Cargo.toml": "d229a126616b1d7638f0f5a6edb4ac37d9d116f32afaa1897ffe484fbb286e5e", - "services/dagfs/boot.toml": "cfd9a5931d61fd8c4f0240b312d434002d9e33f85da3e9c607dbb26eb7d0c240", + "services/dagfs/boot.toml": "6a5c74554acdeb02588105ba7f56802b7694d6da7d127362970ad183e1e7333c", "services/dagfs/examples/fsdemo.rs": "1666a14ef62fd4fc46f7f9efa2eaa5021ed79823f273a77f53758307f884e7d0", "services/dagfs/examples/fswire.rs": "994b97d699796b317741e2c87cbe812cc1776ddcd01173323e72a5f74c8ac267", "services/dagfs/examples/model_volume.rs": "0e3623e962da02b2126062a19d198b9e0a8f6e91896fd9a6f7d13f6f30ddc5f1", @@ -343,7 +343,7 @@ "services/dagfs/tests/provider.rs": "ba4070af34ae034c109e391e97bcc4755c6bd70b8e2fb229aaf9d747fef71e2d", "services/dagfs/tests/query_experiment.rs": "6fe147810474b6754ef31ad0fb95493e0b3c0886fbbafd78deb3729d410e8155", "services/device/Cargo.toml": "8f35c9ddd701b58a538f151e95e93ce618f0b346960491adad6d843287165d1c", - "services/device/boot.toml": "44fc18ed82e7b3e07bcdc0203953dc6ae9f849b762d27b9735f384ab2bf96319", + "services/device/boot.toml": "510b8448648e4431695de9de4ddb78ac33298a0d078185aa9b0fd1d21e829bb5", "services/device/src/client.rs": "71660deb4a9f51edb7a72fbb3edac94170bf4cccf9ce182cbd478ee3daf406eb", "services/device/src/lib.rs": "215c7c7e81d5d51664ac77b72b2edb0bab739d5a54b30bc9ad08f9dd3a3cfea4", "services/device/src/service/mod.rs": "6038ba0b9375e3c6bbb25c49bf0849a5878a402f4a3a691dc9df1494849d252b", @@ -385,7 +385,7 @@ "services/extract/vendor/jsmn/jsmn.h": "babfcb7413354a43dcd539c600a1e00bd2d1579ad0a6f6c7d8b81dccedb16cd0", "services/extract/vendor/jsmn/kore_jsmn.c": "a666060a7af03c819d1faaccc0d6faa19b76bcfe78e4ef61aeab273ad052c95c", "services/mm/Cargo.toml": "25809b6a03916d96b8de64a5988d005c71c49314fafbe0d920c3ec06f2dc5951", - "services/mm/boot.toml": "461ca1cf03071b333bc2d7a11ba3be273fa4bff923e712b194e263f53c915539", + "services/mm/boot.toml": "afb0cdf8a64635992180edf814518c525651949aa4afc559c89d81090c14ab18", "services/mm/src/client.rs": "c6dcc0dc3ed59e0a4cf7532fd306af4c1dc4d4e5ad1edd1d86887def791c5aa6", "services/mm/src/heap.rs": "e55210f3609f2e3e90f30d5497355ddab5801acc60ccbe3f8acbbffff71dbfde", "services/mm/src/lib.rs": "d3820740666ed993e547ee1b3b579231bf439a813d9766860e3b1b57fe815bd6", @@ -397,7 +397,7 @@ "services/output/src/service/mod.rs": "d80c3beb50979b3e1e500abdfa3c0b9d38e354c24e2e34aea443732abbead6f1", "services/output/src/wire.rs": "3752e994eb18714518facae1b78a9ee2b34f99aad8687653f0b89dc8fdeca621", "services/swap/Cargo.toml": "9c0af01403a6ad5a7f4ff69a02b18c8b1b2624a684814d6dc1ac07968c5bbe53", - "services/swap/boot.toml": "1638a88d04cfec0a5a16c7e61f7a43e2284991e63ca253b84ca38a84dc7d2c23", + "services/swap/boot.toml": "323531c62ec0b063cea376e941218719352c87c1b91e579e560fe55dc1bcbc42", "services/swap/src/client.rs": "04f735d059b491765d5ba63a4f5f640332735bf336e62dcbae86c17647583e2c", "services/swap/src/extent.rs": "d6705f77951df1c42cda51fc4526ba6262cab6af4a0545405ce06d2d2e1db7b4", "services/swap/src/lib.rs": "883684df54a87ae66aa7895ea19a872cabc8ab7fd4aded4ea65b7e3dd51fe0cf", diff --git a/native/services/block/boot.toml b/native/services/block/boot.toml index 1b02999..2c43aaf 100644 --- a/native/services/block/boot.toml +++ b/native/services/block/boot.toml @@ -12,5 +12,5 @@ chain = true endowment = "kore_svc_block::BlockEndowment" protocol = "kore_svc_block::BlockService" endows = ["DMA_POOL@1"] -digest_x86_64 = "blake3:c55dc025414bbcd3a511a3072095c412aedfee76d69f22bedeb51859bc30ec6b" -digest_aarch64 = "blake3:807f59729471b0b98d09a5b8c8e1d1931986475835c116aaee136fbe5843fdc4" +digest_x86_64 = "blake3:a22f3a5b08f35a278ebf663d557e0b1ec490c8e732cdee0fa68151e349ddbc57" +digest_aarch64 = "blake3:2855f4168dcfe5b78dca4c508bc6f3120af84412c74498f2cf6475b01508c8e2" diff --git a/native/services/dagfs/boot.toml b/native/services/dagfs/boot.toml index 2ba36ca..a153d4e 100644 --- a/native/services/dagfs/boot.toml +++ b/native/services/dagfs/boot.toml @@ -47,5 +47,5 @@ exec_client = true # `dagfs_heap_supply_holds_every_session` holds this row to it; the rest is # room for the proofs open handles carry, allocated per open at their depth. heap_supply_bytes = 0x00D0_0000 -digest_x86_64 = "blake3:88925d674abc8a20dd326c89e6ef95e68f0ac98624520a8cc29213d402cee329" -digest_aarch64 = "blake3:6ce4008a3a308dc0fce3bde1d0c4ee18e650b321c6c683854c5bd55e3606e0da" +digest_x86_64 = "blake3:453a43f78c11f4fedf1715a141e4627a4d9858ab975fc8ad03c5d44b8814b7d9" +digest_aarch64 = "blake3:9521f0dae0fede5b0061ca144be8fdb421798faf17fd6cb5376acacb5741777e" diff --git a/native/services/device/boot.toml b/native/services/device/boot.toml index 8e20284..0d1f546 100644 --- a/native/services/device/boot.toml +++ b/native/services/device/boot.toml @@ -47,5 +47,5 @@ protocol = "kore_svc_device::DeviceBroker" # row's own claimant band. instances = 8 endows = ["ROOT_COMPLEX@0", "PLATFORM_BUS@0", "FACE_SUPPLY@0"] -digest_x86_64 = "blake3:7915aee8c49e13907542815f1f4627ab41f656933a8067e5b247f03b55b0bae2" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:a30ae8ec2936590328807fc1522c906d6ab5afb473437547821899f5df7db4e8" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:78d5b044c7b3ad3467484859549f8404cffc61ece27dd793cdd21c8c2c0a930a" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:7006c3d2336f8e984751fce7983384668fb0201e565dea03c8cbfd4c43992b3f" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/mm/boot.toml b/native/services/mm/boot.toml index 51a97ec..0071ac5 100644 --- a/native/services/mm/boot.toml +++ b/native/services/mm/boot.toml @@ -54,5 +54,5 @@ supply_quota_bytes = 27917287424 # 26 GiB endows = ["MEM_SUPPLY@0", "MEM_REGION@0", "SYSTEM_OBSERVE@0"] # review mirror; asserted === PROFILE.roles shell_client = true exec_client = true -digest_x86_64 = "blake3:f46d34fd54f1190518157c9b634cf9b5191ec174271dc5f8007bbf8bad824657" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:b871832aa6a5ca4b4d6fee6beed6e0b1625d96bc019c996da1b02eb28c0bec58" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:d1ce3128c73f02293e323f87d79850b8558dfd82d882b65169767febb7baf6e4" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:a03d13d5caf95b15a82902ed087c816af46f74aa305ca47d01681b039e9d55fe" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/swap/boot.toml b/native/services/swap/boot.toml index 77f463e..80722a4 100644 --- a/native/services/swap/boot.toml +++ b/native/services/swap/boot.toml @@ -9,5 +9,5 @@ image = "kore-svc-swap" endowment = "kore_svc_swap::ZramEndowment" protocol = "kore_svc_swap::ZramSwap" endows = ["MEM_SUPPLY@0"] -digest_x86_64 = "blake3:9a81892b71b57bc7cea7b2797749293c18ea2fd9cf231bf11d5742d9602c7e7e" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:7cd848899239fa79c8038af480783b1ccf1772488d2b31d72656c6bd587ef06d" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:ffa0c5ffe03770b1810bf23a9c546d1e92ce897708af3a5d0fc928c6dace51cd" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:f71c92926d0ca9faf5a1704659f96931bbf4aed4e1546bb6b1433d7d6d7057e4" # res/aarch64/; make boot-digests-aarch64 From 5be44eba9b9bafd21be3f738aa793b5e225ba9c2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 04:58:23 +0000 Subject: [PATCH 09/18] native: re-export at Kore 6c57f344, where a founding's staged space waits for the reap `tools/sync-native.py` at MicroPerceptron/kore@6c57f344311306868f68e13a37f87abf0ce04e02. The export at 2a0730dd returned a thread founding's staged space to the slot ledger at commit. The Found consumes that space but leaves the founder's record of it binding until the reap, so the name was not free. Kore's production boot gate caught it: a later receive could not land on the name, and the first shell never reported ready. kore-rt now keeps the staged space in the join ticket and instead returns the boot staging page at commit, which the founding's delegation consumes. A founded thread still keeps four names until join, so the launch budgets and the controller's admission are those of 2a0730dd. `fmt`, `native-check`, `owners` and `bindings-check` pass on this export. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- native/crates/rt/src/thread.rs | 182 +++++++++++++--------- native/crates/rt/src/thread_controller.rs | 6 +- native/provenance.json | 6 +- 3 files changed, 114 insertions(+), 80 deletions(-) diff --git a/native/crates/rt/src/thread.rs b/native/crates/rt/src/thread.rs index 3c68103..7064582 100644 --- a/native/crates/rt/src/thread.rs +++ b/native/crates/rt/src/thread.rs @@ -496,19 +496,21 @@ pub struct Thread { /// The parent-arena slot holding the child's task handle: the join /// subject, and (revoked) the reap. handle: SlotRef, - /// The parent-held `Region`s the provisioning session produced — - /// stack, runtime block, boot staging page. Revoking them is what - /// actually unmaps the band; the parent holds them precisely so that - /// a dead thread cannot keep its own stack alive. - provisioned: [SlotRef; PROVISIONED_REGIONS], + /// The parent-held `Region`s the provisioning session produced that + /// outlive the founding: the stack and the runtime block. Revoking them + /// is what actually unmaps the band; the parent holds them precisely so + /// that a dead thread cannot keep its own stack alive. The third, the + /// boot staging page, is the founding's delegation source, which its + /// commit consumes ([`HELD_REGIONS`]). + regions: [SlotRef; HELD_REGIONS], tls_region: Option, - /// Founding result slots whose return waits for [`join`]: the staged - /// space, the delivered view, the boot delegation and one per endowment - /// of a gang founding that did not commit whole, whose outcome only the - /// join learns. A founding that commits returns them at once - /// ([`release_scrubbed`]), so a founder holds per live child only what - /// `join` revokes; this list is then empty. - spent: [SlotRef; SCRUBBED_RESULTS + MAX_WORKER_CAPS], + /// Founding names whose return waits for [`join`]: always the staged + /// space, whose founder record the `Found` leaves standing until the reap + /// ([`STAGED_SPACE`]); and, for a gang founding that did not commit whole, + /// whose outcome only the join learns, what a whole commit returns at + /// once ([`release_committed`]: [`RELEASED_AT_COMMIT`] names and one per + /// endowment). + spent: [SlotRef; SPENT_CAPACITY], /// Live entries in [`spent`](Self::spent). spent_len: u8, /// The arena strip this stack was carved from — `Some` for SPMD @@ -924,27 +926,56 @@ impl WorkerArena { } } -/// The regions a founder provisions for each thread and holds until -/// [`join`] revokes them: the stack, the runtime block and the boot staging -/// page ([`Thread`]'s `provisioned`; a TLS block is one more, apart). +/// The regions a founder provisions for each thread: the stack, the runtime +/// block and the boot staging page (a TLS block is one more, apart). const PROVISIONED_REGIONS: usize = 3; +/// The provisioned regions a founder holds until [`join`] revokes them: the +/// stack and the runtime block ([`Thread`]'s `regions`). The boot staging +/// page is the founding's delegation source: the commit moves its page into +/// the child and retires its founder record, so its name is free from then +/// on, a dead record any later landing or result replaces. +const HELD_REGIONS: usize = 2; + +/// The staged space a thread's `Found` consumes, whose founder name stays +/// spent until [`join`]'s reap. The commit moves the space into the child but +/// scrubs from the founder only what it delivers and delegates, so the +/// founder's record of the space stands — an `AddrSpace` with every right, +/// bound to nothing the founder holds — until the reap retires it. A name +/// returned before that is not vacant: the next post landing there is refused +/// as occupied, and the next result written there overwrites a live record. +const STAGED_SPACE: usize = 1; + /// The founding results a committed `Found` scrubs from the founder, beside -/// one per endowment: the staged space, the child's view and the boot -/// delegation ([`release_scrubbed`]; [`Thread`]'s `spent`). -const SCRUBBED_RESULTS: usize = 3; +/// one per endowment: the child's delivered view and the boot delegation's. +const SCRUBBED_RESULTS: usize = 2; + +/// The founding names a committed `Found` leaves without a live record, +/// beside one per endowment: the consumed staging page and the scrubbed +/// results ([`release_committed`]). +const RELEASED_AT_COMMIT: usize = PROVISIONED_REGIONS - HELD_REGIONS + SCRUBBED_RESULTS; + +/// [`Thread`]'s `spent`: the staged space, and what a gang founding that did +/// not commit whole could not return. +const SPENT_CAPACITY: usize = STAGED_SPACE + RELEASED_AT_COMMIT + MAX_WORKER_CAPS; /// Arena names one thread's founding holds in its founder while it is in -/// flight, beside one per endowment: its provisioned regions, the results the -/// commit scrubs, and the handle. One more with a TLS block. +/// flight, beside one per endowment: its provisioned regions, the staged +/// space, the results the commit scrubs, and the handle. One more with a TLS +/// block. #[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] -pub(crate) const FOUNDING_SLOTS: usize = PROVISIONED_REGIONS + SCRUBBED_RESULTS + 1; +pub(crate) const FOUNDING_SLOTS: usize = PROVISIONED_REGIONS + STAGED_SPACE + SCRUBBED_RESULTS + 1; /// Arena names a founded thread keeps in its founder until [`join`]: its -/// handle and its provisioned regions (one more with a TLS block). The rest -/// of the founding's plan returns at commit ([`release_scrubbed`]). +/// handle, its held regions and its staged space (one more with a TLS block). +/// The rest of the founding's plan returns at commit ([`release_committed`]). #[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] -pub(crate) const RETAINED_SLOTS: usize = 1 + PROVISIONED_REGIONS; +pub(crate) const RETAINED_SLOTS: usize = 1 + HELD_REGIONS + STAGED_SPACE; + +// Every founding name is kept until the reap or returned at commit, never +// both and never neither: the launch budgets ([`crate::spmd`]) count the two +// halves apart. +const _: () = assert!(FOUNDING_SLOTS == RETAINED_SLOTS + RELEASED_AT_COMMIT); /// The arena names a gang session founding `workers` threads holds at its /// peak, just before its founding commits: every worker's claims and every @@ -1225,19 +1256,22 @@ pub fn spawn_with_arg(_entry: fn(usize), _argument: usize, _endow: &[WorkerCap]) None } -/// Return a committed founding's result names to the slot ledger. +/// Return to the slot ledger the founding names a committed `Found` left +/// without a live record. /// -/// A `Found` that commits moves the staged space, the child's view and every -/// endowment out of the founder (the delivery map scrubs each one's founder -/// slot) and clears the founder's delegated `Region` results, the boot page's -/// among them. No capability is left under these names, and none can have -/// been copied: the founder was inside its own trap from apply to scrub. They -/// are free the moment the commit reports `Ok`, and only that proves it: a -/// refused or unreadable commit keeps its names (the founding's -/// leak-on-refusal rule), as a gang founding that did not commit whole keeps -/// them in its tickets for `join`. +/// A `Found` that commits delivers the child's view and every endowment out +/// of the founder (the delivery map scrubs each one's founder slot), clears +/// the founder's delegated `Region` results, the boot page's among them, and +/// retires the tag of the staging page its delegation consumed. No live +/// record is left under these names, and none can have been copied: the +/// founder was inside its own trap from apply to scrub. They are free the +/// moment the commit reports `Ok`, and only that proves it: a refused or +/// unreadable commit keeps its names (the founding's leak-on-refusal rule), +/// as a gang founding that did not commit whole keeps them in its tickets for +/// `join`. The staged space is not among them: the commit consumes it without +/// scrubbing its founder record ([`STAGED_SPACE`]). #[cfg(target_os = "none")] -fn release_scrubbed(slots: impl IntoIterator) { +fn release_committed(slots: impl IntoIterator) { for slot in slots { crate::entry::geometry::release_cap_slot(slot); } @@ -1442,13 +1476,14 @@ fn spawn_entry( if committed.status() != kore_abi::session::Status::Ok { return None; } - // The founding's result plan is spent and its names come back now: the - // commit moved the staged space, the view and every endowment into the - // child and scrubbed their founder slots, and cleared the boot - // delegation's (`release_scrubbed`). The join ticket keeps only what - // `join` must revoke. - release_scrubbed( - [space_res, view_res, boot_res] + // The founding's plan is spent, and the names it left without a live + // record come back now: the commit moved the view and every endowment + // into the child, scrubbing their founder slots, cleared the boot + // delegation's, and consumed the staging page (`release_committed`). The + // staged space's record stands until the reap ([`STAGED_SPACE`]), so its + // name rides the join ticket beside what `join` revokes. + release_committed( + [stage_res, view_res, boot_res] .into_iter() .chain(staged[..endow.len()].iter().map(|e| e.result)), ); @@ -1456,10 +1491,10 @@ fn spawn_entry( owner, index, handle: handle_res, - provisioned: [stack_res, tblock_res, stage_res], + regions: [stack_res, tblock_res], tls_region, - spent: [handle_res; SCRUBBED_RESULTS + MAX_WORKER_CAPS], - spent_len: 0, + spent: [space_res; SPENT_CAPACITY], + spent_len: STAGED_SPACE as u8, strip, }) } @@ -1970,33 +2005,31 @@ pub(crate) fn spawn_gang_workers( // Mint every ticket now that the founding committed: whichever // workers exist are joinable, and a vacant handle's join refuses in // the caller's abort arm rather than orphaning anyone. A whole commit - // returns each worker's founding plan now (`release_scrubbed`); a - // partial one leaves it to the tickets, for the join that learns which - // workers exist. + // returns what it left of each worker's founding without a live record + // now (`release_committed`); a partial one leaves those names to the + // tickets, for the join that learns which workers exist. Either way the + // staged space's name waits for the reap ([`STAGED_SPACE`]). for (k, claim) in claims.iter_mut().take(n).enumerate() { let Some(c) = claim.take() else { return false }; let endowed = &endow_store[k][..endow_lens[k]]; - let mut spent = [c.space_res; SCRUBBED_RESULTS + MAX_WORKER_CAPS]; + let committed = [c.stage_res, c.view_res, c.boot_res] + .into_iter() + .chain(endowed.iter().map(|e| e.result)); + let mut spent = [c.space_res; SPENT_CAPACITY]; let spent_len = if founded_all { - release_scrubbed( - [c.space_res, c.view_res, c.boot_res] - .into_iter() - .chain(endowed.iter().map(|e| e.result)), - ); - 0 + release_committed(committed); + STAGED_SPACE } else { - spent[1] = c.view_res; - spent[2] = c.boot_res; - for (i, e) in endowed.iter().enumerate() { - spent[3 + i] = e.result; + for (name, slot) in spent[STAGED_SPACE..].iter_mut().zip(committed) { + *name = slot; } - SCRUBBED_RESULTS + endowed.len() + STAGED_SPACE + RELEASED_AT_COMMIT + endowed.len() }; out[k] = Some(Thread { owner, index: c.index, handle: c.handle_res, - provisioned: [c.stack_res, c.tblock_res, c.stage_res], + regions: [c.stack_res, c.tblock_res], tls_region: c.tls_region, spent, spent_len: spent_len as u8, @@ -2305,8 +2338,9 @@ impl Drop for JoinScratch { /// 2. **the parent-held regions** — stack, runtime block, boot staging. /// These are the parent's precisely so a dead thread cannot keep its /// own stack alive; -/// 3. **the arena slots** — the founding results the kernel scrubbed, plus -/// the handle; +/// 3. **the arena slots** — the held regions, the staged space, whose +/// founder record the reap has just retired, whatever a partial gang +/// commit left in the ticket, and the handle; /// 4. **the band index — last.** Every window in that band is now /// unmapped, so the next [`spawn`] can be handed it. Released any /// earlier and a new thread would map its stack over a corpse's: the @@ -2375,21 +2409,21 @@ pub(crate) fn join_reserved_at( }); } // The stack and runtime-block regions are parent-held and die here. - // The STAGE region is deliberately NOT presented: the founding's + // The STAGE region is deliberately not among them: the founding's // delegate consumed it (the boot page re-rooted under the child, its // parent record retired with the move), so revoking it refuses the // WHOLE session as `Revoked` — which un-revokes the handle too and // silently leaves `torn_down` false. That exact shape leaked every // joined thread's strip and band index since #407 (bounded, and - // invisible until the G7 retirement witness demanded a real reap); - // the slot itself is still released below, corpse and all. - for slot in [ - Some(thread.provisioned[0]), - Some(thread.provisioned[1]), - thread.tls_region, - ] - .into_iter() - .flatten() + // invisible until the G7 retirement witness demanded a real reap). + // Its name came back when the founding committed, or rides `spent` + // from a gang founding that did not commit whole. + for slot in thread + .regions + .into_iter() + .map(Some) + .chain([thread.tls_region]) + .flatten() { if b.revoke(slot).is_err() { return Err(ThreadJoinError { @@ -2416,7 +2450,7 @@ pub(crate) fn join_reserved_at( if let Some(slot) = thread.tls_region { crate::entry::geometry::release_cap_slot(slot); } - for slot in thread.provisioned { + for slot in thread.regions { crate::entry::geometry::release_cap_slot(slot); } for slot in &thread.spent[..thread.spent_len as usize] { diff --git a/native/crates/rt/src/thread_controller.rs b/native/crates/rt/src/thread_controller.rs index 90e73da..764447a 100644 --- a/native/crates/rt/src/thread_controller.rs +++ b/native/crates/rt/src/thread_controller.rs @@ -33,9 +33,9 @@ //! //! Both the controller and every child take a founder's arena (the //! founding granule of capability slots): the controller holds each live child's handle, -//! its provisioned regions and the root and send of its doorbell until the -//! reap, a whole process's worth of them, and a child may found an SPMD gang -//! of its own. +//! its stack and runtime-block regions, its staged space and the root and send +//! of its doorbell until the reap, a whole process's worth of them, and a +//! child may found an SPMD gang of its own. //! //! # Degrade, never wedge //! diff --git a/native/provenance.json b/native/provenance.json index 987812a..10a549b 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "2a0730dd6ba3369c82f025c9957184f0bb9f8a66", + "revision": "6c57f344311306868f68e13a37f87abf0ce04e02", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", @@ -244,8 +244,8 @@ "crates/rt/src/svc/serve.rs": "a22172638a94e3ab52b88095dff9d78c88a1d693c81c6242af4932881d268706", "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", - "crates/rt/src/thread.rs": "54b067cc76785f16c36cd083e97b56456a654542ad0e57198141eeb2d8524587", - "crates/rt/src/thread_controller.rs": "e3ee7eb23edca916aad3d3bf78d2b87abea3806a59f2b5ced73263238afc5d01", + "crates/rt/src/thread.rs": "06d0eefa92c46fe31392db8b10e891e008e95c3ee4162f9f4034f7dec7d9b35c", + "crates/rt/src/thread_controller.rs": "6ce535ded212d30b4838d15108cf9e9625b4f9e9fd83e0fecde51be833b1326d", "crates/rt/src/tls.rs": "85bd95bafb3de73d249d7a2eb8779b5afc929077e27b4baa5f7c7428063063ab", "crates/rt/src/transport/mod.rs": "d5af890c800dd53abdbea8329383068cf43f0bf294ccb6b46dd7d8592ae5f321", "crates/rt/src/upcall.rs": "c0cbd2a59479dfba006850bb1bc5dd4f20f59317d178af776ba678f0e6404f02", From 13b0a91ef8d84d45d21e88f0dcc4c75c29984a34 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 06:31:29 +0000 Subject: [PATCH 10/18] native: re-export at Kore 2d4998ba, with Kore's C library `tools/sync-native.py` at MicroPerceptron/kore@2d4998baa8be794191fdacaf63dffca295c2a109. kore-c-shim joins the owner paths, so the export now carries Kore's C library: the `printf` family over every conversion C17 names, floats included and exact; the standard streams behind an installable sink whose lock one call holds across its writes; ``, ``'s parsers and `qsort`; and `errno`. Its allocator, `abort` and `` classes each sit behind their own feature, so an image that brings its own allocator and `abort` (as `libkore.so` does) builds without them. kore-rt and the services are those of 6c57f344; their `boot.toml` digests follow Kore's reseal at e3d59c09. `fmt`, `native-check`, `owners` and `bindings-check` pass on this export. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- Cargo.toml | 2 +- native/Cargo.lock | 24 + native/Cargo.toml | 1 + native/crates/c-shim/Cargo.toml | 44 + native/crates/c-shim/src/alloc_symbols.rs | 107 +++ native/crates/c-shim/src/errno.rs | 90 ++ native/crates/c-shim/src/format.rs | 724 +++++++++++++++ native/crates/c-shim/src/format/float.rs | 866 ++++++++++++++++++ .../crates/c-shim/src/format/float/tests.rs | 620 +++++++++++++ native/crates/c-shim/src/format_symbols.rs | 342 +++++++ native/crates/c-shim/src/lib.rs | 83 ++ native/crates/c-shim/src/long_double.rs | 341 +++++++ native/crates/c-shim/src/number.rs | 474 ++++++++++ native/crates/c-shim/src/sort.rs | 115 +++ native/crates/c-shim/src/stdio.rs | 210 +++++ native/crates/c-shim/src/stdlib_symbols.rs | 240 +++++ native/crates/c-shim/src/string_symbols.rs | 158 ++++ .../c-shim/src/va_long_double/aarch64.rs | 47 + .../crates/c-shim/src/va_long_double/mod.rs | 54 ++ .../c-shim/src/va_long_double/riscv64.rs | 24 + .../c-shim/src/va_long_double/x86_64.rs | 32 + native/crates/c-shim/src/wide_symbols.rs | 77 ++ native/provenance.json | 34 +- native/services/block/boot.toml | 2 +- native/services/dagfs/boot.toml | 2 +- native/services/device/boot.toml | 2 +- native/services/mm/boot.toml | 2 +- native/services/swap/boot.toml | 4 +- 28 files changed, 4707 insertions(+), 14 deletions(-) create mode 100644 native/crates/c-shim/Cargo.toml create mode 100644 native/crates/c-shim/src/alloc_symbols.rs create mode 100644 native/crates/c-shim/src/errno.rs create mode 100644 native/crates/c-shim/src/format.rs create mode 100644 native/crates/c-shim/src/format/float.rs create mode 100644 native/crates/c-shim/src/format/float/tests.rs create mode 100644 native/crates/c-shim/src/format_symbols.rs create mode 100644 native/crates/c-shim/src/lib.rs create mode 100644 native/crates/c-shim/src/long_double.rs create mode 100644 native/crates/c-shim/src/number.rs create mode 100644 native/crates/c-shim/src/sort.rs create mode 100644 native/crates/c-shim/src/stdio.rs create mode 100644 native/crates/c-shim/src/stdlib_symbols.rs create mode 100644 native/crates/c-shim/src/string_symbols.rs create mode 100644 native/crates/c-shim/src/va_long_double/aarch64.rs create mode 100644 native/crates/c-shim/src/va_long_double/mod.rs create mode 100644 native/crates/c-shim/src/va_long_double/riscv64.rs create mode 100644 native/crates/c-shim/src/va_long_double/x86_64.rs create mode 100644 native/crates/c-shim/src/wide_symbols.rs diff --git a/Cargo.toml b/Cargo.toml index ebc60ab..3739aa4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,4 +35,4 @@ kore-driver = { path = "native/crates/driver", optional = true } # root: the offline volume procedure (and the filesystem service closure it # drives) that host programs such as vKore consume; it never enters a Kore # executable's graph. -owner-paths = { kore-start = "crates/start", kore-abi = "crates/abi", kore-app = "crates/app", kore-rt = "crates/rt", kore-dagfs = "crates/dagfs", kore-driver = "crates/driver", kore-late = "crates/late", kore-vol = "tools/kore-vol" } +owner-paths = { kore-start = "crates/start", kore-abi = "crates/abi", kore-app = "crates/app", kore-rt = "crates/rt", kore-c-shim = "crates/c-shim", kore-dagfs = "crates/dagfs", kore-driver = "crates/driver", kore-late = "crates/late", kore-vol = "tools/kore-vol" } diff --git a/native/Cargo.lock b/native/Cargo.lock index 8eef346..22b245d 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -247,6 +247,14 @@ dependencies = [ "kore-window", ] +[[package]] +name = "kore-c-shim" +version = "0.1.0" +dependencies = [ + "kore-rt", + "rustc_apfloat", +] + [[package]] name = "kore-collections" version = "0.1.0" @@ -583,6 +591,16 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + [[package]] name = "rustix" version = "1.1.6" @@ -622,6 +640,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "smallvec" +version = "1.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149309498a5f7334b12ddae0cad8aba81ce785fe91755068867e017cf98fee46" + [[package]] name = "stable_deref_trait" version = "1.2.1" diff --git a/native/Cargo.toml b/native/Cargo.toml index 20a77a3..3164724 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -3,6 +3,7 @@ resolver = "3" members = [ "crates/abi", "crates/app", + "crates/c-shim", "crates/collections", "crates/dagfs", "crates/driver", diff --git a/native/crates/c-shim/Cargo.toml b/native/crates/c-shim/Cargo.toml new file mode 100644 index 0000000..1dde6da --- /dev/null +++ b/native/crates/c-shim/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "kore-c-shim" +version = "0.1.0" +edition = "2024" +publish = false +description = "The C symbols a vendored C library imports on a kore user target" + +[features] +default = ["runtime", "malloc", "abort", "wide-classes"] +# The process half: the `` writes over the standard sink the process +# installs, `errno` and the parsers that set it. An executable takes it; a +# shared library leaves it out and imports those symbols from the executable +# it links into, so one process never holds two heaps. +runtime = ["dep:kore-rt"] +# `malloc`, `calloc`, `realloc` and `free` over the task's global heap. A C +# runtime that exports its own allocator under those names (libkore.so) +# leaves this out, so one process never holds two `malloc`s whose blocks +# the other's `free` would misread. +malloc = ["runtime"] +# `abort`, ending the calling task with the aborted status. A C runtime that +# ends a whole program its own way (libkore.so's host) leaves this out and +# exports its own. +abort = ["runtime"] +# The wide character classes (`iswalnum` … `iswxdigit`, `towlower`, +# `towupper`) a vendored C library calls: tree-sitter's scanners. Pure, so a +# shared library may carry its own. A C runtime whose headers declare no +# `` (libkore.so) leaves them out, so it exports exactly the ABI +# its headers declare. +wide-classes = [] +# `strtold` and `ldexpl` as byte-level helpers a consumer's C layer wraps +# (`long_double`): C's `long double` has no Rust spelling, so the C side owns +# the type and this crate the arithmetic. Off unless a consumer compiles C +# that converts `long double` (TCC does: its float literals). +long-double = ["runtime", "dep:rustc_apfloat"] + +[dependencies] +kore-rt = { path = "../rt", optional = true } +# LLVM's APFloat (the code clang folds float literals with): correctly +# rounded decimal conversion and scaling for x87 double-extended and IEEE +# quad. `no_std`, `forbid(unsafe_code)`; it allocates only to convert decimal. +rustc_apfloat = { version = "=0.2.3", optional = true } + +[dev-dependencies] +rustc_apfloat = "=0.2.3" diff --git a/native/crates/c-shim/src/alloc_symbols.rs b/native/crates/c-shim/src/alloc_symbols.rs new file mode 100644 index 0000000..6e97947 --- /dev/null +++ b/native/crates/c-shim/src/alloc_symbols.rs @@ -0,0 +1,107 @@ +//! `malloc`, `calloc`, `realloc` and `free` over the task's global heap. +//! +//! C frees without a size and Rust deallocates with one, so every block +//! carries its payload size in a header just below the pointer C sees. The +//! header is one `max_align_t` wide, so the payload keeps C's alignment +//! guarantee. Overflow and exhaustion return null, as C expects; a size of +//! zero returns null too, which `free` and `realloc` accept. + +extern crate alloc; + +use core::alloc::Layout; +use core::ffi::c_void; +use core::ptr; + +/// C's `max_align_t` on every kore target: the payload alignment `malloc` +/// promises, and the header's width. +const ALIGN: usize = 16; + +fn layout(payload: usize) -> Option { + Layout::from_size_align(payload.checked_add(ALIGN)?, ALIGN).ok() +} + +/// The payload size recorded below `payload`. +/// +/// # Safety +/// `payload` came from this module and is live. +unsafe fn payload_size(payload: *mut c_void) -> usize { + // SAFETY: the caller's contract; the header sits `ALIGN` bytes below. + unsafe { payload.cast::().sub(ALIGN).cast::().read() } +} + +/// Write `size` into the block's header and return its payload. +/// +/// # Safety +/// `block` is a live allocation of `layout(size)`. +unsafe fn finish(block: *mut u8, size: usize) -> *mut c_void { + if block.is_null() { + return ptr::null_mut(); + } + // SAFETY: the block is at least `ALIGN` bytes and `ALIGN`-aligned. + unsafe { + block.cast::().write(size); + block.add(ALIGN).cast() + } +} + +#[unsafe(no_mangle)] +extern "C" fn malloc(size: usize) -> *mut c_void { + if size == 0 { + return ptr::null_mut(); + } + let Some(layout) = layout(size) else { + return ptr::null_mut(); + }; + // SAFETY: the layout is non-zero; the header is written before return. + unsafe { finish(alloc::alloc::alloc(layout), size) } +} + +#[unsafe(no_mangle)] +extern "C" fn calloc(count: usize, size: usize) -> *mut c_void { + let Some(total) = count.checked_mul(size).filter(|&total| total != 0) else { + return ptr::null_mut(); + }; + let Some(layout) = layout(total) else { + return ptr::null_mut(); + }; + // SAFETY: as `malloc`; the zeroed allocation covers the payload. + unsafe { finish(alloc::alloc::alloc_zeroed(layout), total) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn realloc(payload: *mut c_void, size: usize) -> *mut c_void { + if payload.is_null() { + return malloc(size); + } + if size == 0 { + // SAFETY: the caller's pointer came from this module. + unsafe { free(payload) }; + return ptr::null_mut(); + } + // SAFETY: a non-null pointer C passes back came from this module. + let old = unsafe { payload_size(payload) }; + let (Some(old_layout), Some(_)) = (layout(old), layout(size)) else { + return ptr::null_mut(); + }; + // SAFETY: the block and its layout are the live allocation; the new size + // fits a layout (checked above). + unsafe { + let block = payload.cast::().sub(ALIGN); + finish(alloc::alloc::realloc(block, old_layout, size + ALIGN), size) + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn free(payload: *mut c_void) { + if payload.is_null() { + return; + } + // SAFETY: a non-null pointer C passes back came from this module, so + // its header names the layout it was allocated with. + unsafe { + let size = payload_size(payload); + if let Some(layout) = layout(size) { + alloc::alloc::dealloc(payload.cast::().sub(ALIGN), layout); + } + } +} diff --git a/native/crates/c-shim/src/errno.rs b/native/crates/c-shim/src/errno.rs new file mode 100644 index 0000000..249fc5c --- /dev/null +++ b/native/crates/c-shim/src/errno.rs @@ -0,0 +1,90 @@ +//! `errno`: one per thread, as C requires, and the codes this crate and its +//! consumers set. +//! +//! The word is the calling thread's runtime-block slot +//! ([`kore_rt::thread::c_errno`]), not an ELF thread-local: a TLS segment in +//! every image that carries C would buy each of them a TCB migration, a +//! template every spawn must place, and `kore-late`'s refusal of a host with +//! TLS — all for one `int`. +//! +//! The numbers are musl's `` — the headers every kore C build +//! compiles against (`zig cc -target -linux-musl`), which use the +//! same values on x86_64, aarch64 and riscv64. A consumer's C code compares +//! `errno` against those headers' macros, so these must agree with them. + +use core::ffi::{CStr, c_int}; + +pub const EPERM: c_int = 1; +pub const ENOENT: c_int = 2; +pub const EIO: c_int = 5; +pub const EBADF: c_int = 9; +pub const EAGAIN: c_int = 11; +pub const ENOMEM: c_int = 12; +pub const EACCES: c_int = 13; +pub const EBUSY: c_int = 16; +pub const EEXIST: c_int = 17; +pub const ENOTDIR: c_int = 20; +pub const EISDIR: c_int = 21; +pub const EINVAL: c_int = 22; +pub const ENFILE: c_int = 23; +pub const EMFILE: c_int = 24; +pub const EFBIG: c_int = 27; +pub const ENOSPC: c_int = 28; +pub const ESPIPE: c_int = 29; +pub const EROFS: c_int = 30; +pub const ERANGE: c_int = 34; +pub const ENAMETOOLONG: c_int = 36; +pub const ENOSYS: c_int = 38; +pub const ENOTEMPTY: c_int = 39; +pub const ELOOP: c_int = 40; +pub const EOVERFLOW: c_int = 75; + +/// Set this thread's `errno`. +pub fn set(code: c_int) { + // SAFETY: the calling thread's own word, live for its life and touched + // by no other thread (`kore_rt::thread::c_errno`'s contract). + unsafe { kore_rt::thread::c_errno().write(code) }; +} + +/// This thread's `errno`. +pub fn get() -> c_int { + // SAFETY: as `set`. + unsafe { kore_rt::thread::c_errno().read() } +} + +#[unsafe(no_mangle)] +extern "C" fn __errno_location() -> *mut c_int { + kore_rt::thread::c_errno() +} + +/// The message `strerror` answers for `code`. +pub fn describe(code: c_int) -> &'static CStr { + match code { + 0 => c"No error information", + EPERM => c"Operation not permitted", + ENOENT => c"No such file or directory", + EIO => c"I/O error", + EBADF => c"Bad file descriptor", + EAGAIN => c"Resource temporarily unavailable", + ENOMEM => c"Out of memory", + EACCES => c"Permission denied", + EBUSY => c"Resource busy", + EEXIST => c"File exists", + ENOTDIR => c"Not a directory", + EISDIR => c"Is a directory", + EINVAL => c"Invalid argument", + ENFILE => c"Too many open files in system", + EMFILE => c"No file descriptors available", + EFBIG => c"File too large", + ENOSPC => c"No space left on device", + ESPIPE => c"Invalid seek", + EROFS => c"Read-only file system", + ERANGE => c"Result not representable", + ENAMETOOLONG => c"Filename too long", + ENOSYS => c"Function not implemented", + ENOTEMPTY => c"Directory not empty", + ELOOP => c"Symbolic link loop", + EOVERFLOW => c"Value too large for data type", + _ => c"Unknown error", + } +} diff --git a/native/crates/c-shim/src/format.rs b/native/crates/c-shim/src/format.rs new file mode 100644 index 0000000..0702d76 --- /dev/null +++ b/native/crates/c-shim/src/format.rs @@ -0,0 +1,724 @@ +//! The `printf` conversion engine behind every formatted write in this crate. +//! +//! One engine serves `snprintf` into a caller's buffer and `fprintf` into a +//! stream, so the two cannot disagree about what a conversion means. It is +//! generic over where arguments come from ([`Args`]: a C `va_list` on the +//! target, a scripted list in host tests) and where bytes go ([`Sink`]), and +//! it is pure: the host suite drives exactly the code the target runs. +//! +//! ## What it formats +//! +//! ISO C's integer, character, string and pointer conversions in full — +//! `d i o u x X c s p %`, the `- + space # 0` flags, field width and +//! precision (both also as `*`), and the `hh h l ll j z t` length modifiers. +//! A precision bounds a `%s` read, so `%.*s` over a counted, unterminated +//! string never reads past its count. `%p` is spelled `0x` and the address +//! in lowercase hexadecimal (the spelling is implementation-defined; this is +//! the one this crate has always used). +//! +//! The float conversions `e E f F g G a A` too, of a `double` and (with `L`) +//! of a `long double` in either of kore's formats, every flag, width and +//! precision honoured and every digit exact ([`float`]): the C programs +//! libkore.so runs print through this engine. +//! +//! ## What it deliberately does not +//! +//! - **`%n`** stores nothing: its pointer is consumed and ignored. A format +//! that writes through an argument is the classic format-string attack, +//! and no consumer needs one. +//! - **Wide characters**: an ASCII `%lc` is written as its byte and anything +//! else as `?`; `%ls` consumes its pointer and is spelled back. +//! +//! An unknown conversion is spelled back and consumes nothing — its type +//! cannot be known, and guessing would misalign every argument after it. + +use core::ffi::c_int; +use core::marker::PhantomData; + +mod float; + +pub use float::Float; + +/// The C type a length modifier names. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Length { + /// `hh`: `char`, promoted to `int` in a variadic call. + Char, + /// `h`: `short`, promoted to `int`. + Short, + /// No modifier: `int`. + Int, + /// `l`: `long`. + Long, + /// `ll` (and BSD's `q`): `long long`. + LongLong, + /// `j`: `intmax_t`. + Max, + /// `z`: `size_t`. + Size, + /// `t`: `ptrdiff_t`. + Diff, + /// `L`: `long double` beside a float conversion; `long long` beside an + /// integer one, as glibc reads it. + LongDouble, +} + +/// Where each conversion's argument comes from. +/// +/// Every method consumes exactly one argument of the C type it names, as a +/// variadic callee must. +pub trait Args { + /// A signed integer of `length`'s type, sign-extended. + fn signed(&mut self, length: Length) -> i64; + /// An unsigned integer of `length`'s type, zero-extended. + fn unsigned(&mut self, length: Length) -> u64; + /// An `int`: a `%c` argument, or a `*` width or precision. + fn int(&mut self) -> c_int; + /// A pointer, as an address: `%p`, `%n`, `%ls`, and `%s`'s string. + fn pointer(&mut self) -> usize; + /// A `double` (a `float` arrives promoted to one). + fn double(&mut self) -> f64; + /// A `long double`, read in the C ABI's own format. Rust has no type for + /// it, so the source decodes it. + fn long_double(&mut self) -> Float; +} + +/// Where formatted bytes go. The engine counts them; a sink only stores. +pub trait Sink { + /// Accept the next run of output bytes. + fn put(&mut self, bytes: &[u8]); +} + +/// Format `format` into `sink`, drawing arguments from `args`, and return +/// how many bytes the complete output has — the count `snprintf` returns +/// even when its buffer is shorter. +/// +/// # Safety +/// `format` is NUL-terminated, and its conversions match the arguments +/// `args` yields. +pub unsafe fn format(sink: &mut impl Sink, format: *const u8, args: &mut impl Args) -> usize { + let format = Format { + base: format, + _bytes: PhantomData, + }; + let mut out = Out { sink, count: 0 }; + let mut at = 0usize; + loop { + // The literal run up to the next conversion, written whole. + let start = at; + // SAFETY (every `format.at` / `format.slice` below): reads stop at + // the terminator, and `at` only advances past bytes already read as + // non-NUL. + while !matches!(unsafe { format.at(at) }, 0 | b'%') { + at += 1; + } + out.put(unsafe { format.slice(start, at) }); + if unsafe { format.at(at) } == 0 { + return out.count; + } + let spec_start = at; + at += 1; + + let mut spec = Spec::default(); + loop { + match unsafe { format.at(at) } { + b'-' => spec.left = true, + b'+' => spec.plus = true, + b' ' => spec.space = true, + b'#' => spec.alternate = true, + b'0' => spec.zero = true, + _ => break, + } + at += 1; + } + if unsafe { format.at(at) } == b'*' { + at += 1; + let width = args.int(); + // A negative `*` width is a `-` flag and a positive width. + spec.left |= width < 0; + spec.width = (width.unsigned_abs() as usize).min(MAX_FIELD); + } else { + spec.width = unsafe { format.decimal(&mut at) }; + } + if unsafe { format.at(at) } == b'.' { + at += 1; + if unsafe { format.at(at) } == b'*' { + at += 1; + // A negative precision is taken as if it were omitted. + spec.precision = usize::try_from(args.int()).ok(); + } else { + spec.precision = Some(unsafe { format.decimal(&mut at) }); + } + } + let (length, skip) = match unsafe { [format.at(at), format.at(at + 1)] } { + [b'h', b'h'] => (Length::Char, 2), + [b'h', _] => (Length::Short, 1), + [b'l', b'l'] => (Length::LongLong, 2), + [b'l', _] => (Length::Long, 1), + [b'q', _] => (Length::LongLong, 1), + [b'j', _] => (Length::Max, 1), + [b'z', _] => (Length::Size, 1), + [b't', _] => (Length::Diff, 1), + [b'L', _] => (Length::LongDouble, 1), + _ => (Length::Int, 0), + }; + at += skip; + let conversion = unsafe { format.at(at) }; + if conversion == 0 { + // A specification cut off by the terminator: spell it back. + out.put(unsafe { format.slice(spec_start, at) }); + return out.count; + } + at += 1; + // The specification's own bytes, for the conversions spelled back. + let spelled = unsafe { format.slice(spec_start, at) }; + + match conversion { + b'%' => out.put(b"%"), + b'd' | b'i' => { + let value = args.signed(length); + integer( + &mut out, + spec, + value.unsigned_abs(), + value < 0, + Radix::Signed, + ); + } + b'u' => integer(&mut out, spec, args.unsigned(length), false, Radix::Decimal), + b'o' => integer(&mut out, spec, args.unsigned(length), false, Radix::Octal), + b'x' => integer(&mut out, spec, args.unsigned(length), false, Radix::Hex), + b'X' => integer( + &mut out, + spec, + args.unsigned(length), + false, + Radix::HexUpper, + ), + b'c' => { + let value = args.int(); + let byte = if length == Length::Long { + // `wint_t`: only an ASCII code point has a byte here. + u8::try_from(value) + .ok() + .filter(u8::is_ascii) + .unwrap_or(b'?') + } else { + value as u8 + }; + text(&mut out, spec, &[byte]); + } + b's' if length == Length::Long => { + args.pointer(); + out.put(spelled); + } + b's' => { + let address = args.pointer() as *const u8; + if address.is_null() { + text(&mut out, spec, b"(null)"); + } else { + // SAFETY: C promises a string readable to its NUL, or to + // the precision when one bounds the read. + let bytes = unsafe { c_string(address, spec.precision) }; + text(&mut out, spec, bytes); + } + } + b'p' => { + let mut digits = [0u8; 22]; + let digits = render(args.pointer() as u64, Radix::Hex, &mut digits); + let mut buffer = [0u8; 18]; + buffer[..2].copy_from_slice(b"0x"); + buffer[2..2 + digits.len()].copy_from_slice(digits); + text(&mut out, spec, &buffer[..2 + digits.len()]); + } + b'n' => { + args.pointer(); + } + b'e' | b'E' | b'f' | b'F' | b'g' | b'G' | b'a' | b'A' => { + if length == Length::LongDouble { + let value = args.long_double(); + float::convert::<{ float::LONG_DOUBLE_LIMBS }, _>( + &mut out, spec, conversion, value, + ); + } else { + let value = Float::from_f64(args.double()); + float::convert::<{ float::DOUBLE_LIMBS }, _>(&mut out, spec, conversion, value); + } + } + _ => out.put(spelled), + } + } +} + +/// A NUL-terminated C format string. +struct Format<'f> { + base: *const u8, + _bytes: PhantomData<&'f [u8]>, +} + +impl<'f> Format<'f> { + /// # Safety + /// `at` lies at or before the terminator. + unsafe fn at(&self, at: usize) -> u8 { + // SAFETY: the caller's contract. + unsafe { *self.base.add(at) } + } + + /// The bytes in `start..end`. + /// + /// # Safety + /// The range was read and lies before the terminator. + unsafe fn slice(&self, start: usize, end: usize) -> &'f [u8] { + // SAFETY: the caller's contract. + unsafe { core::slice::from_raw_parts(self.base.add(start), end - start) } + } + + /// A run of decimal digits as a field width or precision, saturating at + /// [`MAX_FIELD`]. + /// + /// # Safety + /// `at` lies at or before the terminator. + unsafe fn decimal(&self, at: &mut usize) -> usize { + let mut value = 0usize; + // SAFETY: the caller's contract; a digit is never the terminator. + while let digit @ b'0'..=b'9' = unsafe { self.at(*at) } { + value = value + .saturating_mul(10) + .saturating_add(usize::from(digit - b'0')) + .min(MAX_FIELD); + *at += 1; + } + value + } +} + +/// A `*` or decimal width or precision is an `int`; this is the largest. +const MAX_FIELD: usize = c_int::MAX as usize; + +/// Output padding comes from these, a run at a time. +const SPACES: [u8; 32] = [b' '; 32]; +const ZEROS: [u8; 32] = [b'0'; 32]; + +/// The engine's counting front: every byte a sink sees passes through here. +struct Out<'s, S: Sink> { + sink: &'s mut S, + count: usize, +} + +impl Out<'_, S> { + fn put(&mut self, bytes: &[u8]) { + if !bytes.is_empty() { + self.sink.put(bytes); + self.count = self.count.saturating_add(bytes.len()); + } + } + + fn pad(&mut self, fill: &[u8; 32], mut count: usize) { + while count > 0 { + let run = count.min(fill.len()); + self.put(&fill[..run]); + count -= run; + } + } +} + +/// One parsed conversion specification. +#[derive(Clone, Copy, Debug, Default)] +struct Spec { + left: bool, + plus: bool, + space: bool, + alternate: bool, + zero: bool, + width: usize, + precision: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Radix { + Octal, + /// `%d`/`%i`: decimal, and the only radix the `+` and space flags mark. + Signed, + /// `%u`. + Decimal, + Hex, + HexUpper, +} + +/// The bytes of the C string at `address`, stopping at its NUL or, when +/// `limit` is given, after `limit` bytes — whichever comes first. +/// +/// # Safety +/// `address` is readable up to its NUL or `limit`. +unsafe fn c_string<'s>(address: *const u8, limit: Option) -> &'s [u8] { + let limit = limit.unwrap_or(usize::MAX); + let mut len = 0usize; + // SAFETY: the caller's contract bounds every read. + while len < limit && unsafe { *address.add(len) } != 0 { + len += 1; + } + unsafe { core::slice::from_raw_parts(address, len) } +} + +/// `value`'s digits, written into the tail of `buffer` (22 bytes hold a u64 +/// in octal). +fn render(mut value: u64, radix: Radix, buffer: &mut [u8; 22]) -> &[u8] { + let (base, digits): (u64, &[u8; 16]) = match radix { + Radix::Octal => (8, b"0123456789abcdef"), + Radix::Signed | Radix::Decimal => (10, b"0123456789abcdef"), + Radix::Hex => (16, b"0123456789abcdef"), + Radix::HexUpper => (16, b"0123456789ABCDEF"), + }; + let mut at = buffer.len(); + loop { + at -= 1; + buffer[at] = digits[(value % base) as usize]; + value /= base; + if value == 0 { + break; + } + } + &buffer[at..] +} + +/// An integer conversion: sign or prefix, the precision's leading zeros, +/// digits, and the field's padding, placed as ISO C 7.21.6.1 places them. +fn integer(out: &mut Out<'_, impl Sink>, spec: Spec, magnitude: u64, negative: bool, radix: Radix) { + let mut buffer = [0u8; 22]; + let mut digits = render(magnitude, radix, &mut buffer); + // A zero precision prints no digits for a zero value. + if magnitude == 0 && spec.precision == Some(0) { + digits = &[]; + } + let prefix: &[u8] = match radix { + _ if negative => b"-", + Radix::Signed if spec.plus => b"+", + Radix::Signed if spec.space => b" ", + Radix::Hex if spec.alternate && magnitude != 0 => b"0x", + Radix::HexUpper if spec.alternate && magnitude != 0 => b"0X", + _ => b"", + }; + let mut leading = spec + .precision + .map_or(0, |precision| precision.saturating_sub(digits.len())); + // `#` with `o` raises the precision just enough for a leading zero. + if spec.alternate && radix == Radix::Octal && leading == 0 && digits.first() != Some(&b'0') { + leading = 1; + } + let fill = spec + .width + .saturating_sub(prefix.len() + leading + digits.len()); + // `0` pads between the prefix and the digits, unless a precision was + // given or the field is left-justified. + let zero_fill = spec.zero && !spec.left && spec.precision.is_none(); + if !spec.left && !zero_fill { + out.pad(&SPACES, fill); + } + out.put(prefix); + if zero_fill { + out.pad(&ZEROS, fill); + } + out.pad(&ZEROS, leading); + out.put(digits); + if spec.left { + out.pad(&SPACES, fill); + } +} + +/// A text conversion (`%c`, `%s`, `%p`): the bytes and the field's padding. +/// The `0` flag means nothing here, as in C. +fn text(out: &mut Out<'_, impl Sink>, spec: Spec, bytes: &[u8]) { + let fill = spec.width.saturating_sub(bytes.len()); + if !spec.left { + out.pad(&SPACES, fill); + } + out.put(bytes); + if spec.left { + out.pad(&SPACES, fill); + } +} + +#[cfg(test)] +mod tests { + use super::*; + extern crate std; + use std::{collections::VecDeque, string::String, vec::Vec}; + + /// One scripted argument. + #[derive(Clone, Copy, Debug)] + enum Arg { + Int(i64), + Uint(u64), + Ptr(usize), + Double(f64), + LongDouble(Float), + } + + /// A NUL-terminated (or, for `%.*s`, counted) string argument. + fn text(bytes: &'static [u8]) -> Arg { + Arg::Ptr(bytes.as_ptr() as usize) + } + + struct Script { + args: VecDeque, + /// Each consumption, so a test can prove alignment. + taken: Vec<&'static str>, + } + + impl Script { + fn take(&mut self, kind: &'static str) -> Arg { + self.taken.push(kind); + self.args + .pop_front() + .expect("the format consumed more arguments than given") + } + } + + impl Args for Script { + fn signed(&mut self, length: Length) -> i64 { + let Arg::Int(value) = self.take("signed") else { + panic!("a signed conversion met another type") + }; + // The truncation a C callee applies to a promoted argument. + match length { + Length::Char => value as i8 as i64, + Length::Short => value as i16 as i64, + Length::Int => value as i32 as i64, + _ => value, + } + } + fn unsigned(&mut self, length: Length) -> u64 { + let value = match self.take("unsigned") { + Arg::Uint(value) => value, + Arg::Int(value) => value as u64, + other => panic!("an unsigned conversion met {other:?}"), + }; + match length { + Length::Char => value as u8 as u64, + Length::Short => value as u16 as u64, + Length::Int => value as u32 as u64, + _ => value, + } + } + fn int(&mut self) -> c_int { + let Arg::Int(value) = self.take("int") else { + panic!("an int met another type") + }; + value as c_int + } + fn pointer(&mut self) -> usize { + let Arg::Ptr(value) = self.take("pointer") else { + panic!("a pointer met another type") + }; + value + } + fn double(&mut self) -> f64 { + let Arg::Double(value) = self.take("double") else { + panic!("a double met another type") + }; + value + } + fn long_double(&mut self) -> Float { + let Arg::LongDouble(value) = self.take("long double") else { + panic!("a long double met another type") + }; + value + } + } + + struct Collect(Vec); + + impl Sink for Collect { + fn put(&mut self, bytes: &[u8]) { + self.0.extend_from_slice(bytes); + } + } + + /// Format a NUL-terminated `format`; return the text, count and script. + fn run(format: &[u8], args: &[Arg]) -> (String, usize, Vec<&'static str>) { + assert_eq!(format.last(), Some(&0), "test formats are NUL-terminated"); + let mut script = Script { + args: args.iter().copied().collect(), + taken: Vec::new(), + }; + let mut sink = Collect(Vec::new()); + // SAFETY: the format is NUL-terminated and the script matches it. + let count = unsafe { super::format(&mut sink, format.as_ptr(), &mut script) }; + assert!( + script.args.is_empty(), + "the format left arguments unconsumed" + ); + (String::from_utf8(sink.0).unwrap(), count, script.taken) + } + + fn check(format: &[u8], args: &[Arg], expected: &str) { + let (out, count, _) = run(format, args); + assert_eq!( + out, + expected, + "format {:?}", + String::from_utf8_lossy(format) + ); + assert_eq!(count, expected.len()); + } + + // Integer, character and string expectations match the host C library's + // printf for the same format and arguments. + + #[test] + fn literal_text_and_percent() { + check(b"plain\0", &[], "plain"); + check(b"100%% sure\0", &[], "100% sure"); + check(b"\0", &[], ""); + } + + #[test] + fn signed_decimal_with_flags_width_and_precision() { + check(b"%d\0", &[Arg::Int(42)], "42"); + check(b"%d\0", &[Arg::Int(-42)], "-42"); + check(b"%i\0", &[Arg::Int(0)], "0"); + check(b"%5d|\0", &[Arg::Int(42)], " 42|"); + check(b"%-5d|\0", &[Arg::Int(42)], "42 |"); + check(b"%05d\0", &[Arg::Int(-42)], "-0042"); + check(b"%+d %+d\0", &[Arg::Int(7), Arg::Int(-7)], "+7 -7"); + check(b"% d % d\0", &[Arg::Int(7), Arg::Int(-7)], " 7 -7"); + check(b"%.3d\0", &[Arg::Int(7)], "007"); + check(b"%6.3d|\0", &[Arg::Int(-7)], " -007|"); + check(b"%06.3d|\0", &[Arg::Int(7)], " 007|"); + check(b"%.0d|\0", &[Arg::Int(0)], "|"); + check(b"%-2d|\0", &[Arg::Int(3)], "3 |"); + check(b"%d\0", &[Arg::Int(i32::MIN as i64)], "-2147483648"); + check(b"%lld\0", &[Arg::Int(i64::MIN)], "-9223372036854775808"); + check(b"%ld\0", &[Arg::Int(1 << 40)], "1099511627776"); + check(b"%hhd\0", &[Arg::Int(300)], "44"); + check(b"%hd\0", &[Arg::Int(70000)], "4464"); + } + + #[test] + fn unsigned_octal_and_hexadecimal() { + check(b"%u\0", &[Arg::Uint(4294967295)], "4294967295"); + check( + b"%x %X\0", + &[Arg::Uint(0xbeef), Arg::Uint(0xbeef)], + "beef BEEF", + ); + check(b"%08x\0", &[Arg::Uint(0x1a)], "0000001a"); + check(b"%#x %#X\0", &[Arg::Uint(255), Arg::Uint(255)], "0xff 0XFF"); + check(b"%#x\0", &[Arg::Uint(0)], "0"); + check(b"%#010x\0", &[Arg::Uint(255)], "0x000000ff"); + check( + b"%o %#o %#o\0", + &[Arg::Uint(8), Arg::Uint(8), Arg::Uint(0)], + "10 010 0", + ); + check(b"%#.0o\0", &[Arg::Uint(0)], "0"); + check(b"%.8x\0", &[Arg::Uint(0x1234)], "00001234"); + check(b"%3X|\0", &[Arg::Uint(0xa)], " A|"); + check(b"%8.2X|\0", &[Arg::Uint(0x5)], " 05|"); + check(b"%lx\0", &[Arg::Uint(0xdead_beef_cafe)], "deadbeefcafe"); + check(b"%08llx\0", &[Arg::Uint(0x12)], "00000012"); + check(b"%llu\0", &[Arg::Uint(u64::MAX)], "18446744073709551615"); + check( + b"%zu %zx\0", + &[Arg::Uint(4096), Arg::Uint(4096)], + "4096 1000", + ); + check(b"%hhx\0", &[Arg::Uint(0x1ff)], "ff"); + // `+` and space belong to signed conversions only. + check(b"%+u % u\0", &[Arg::Uint(5), Arg::Uint(5)], "5 5"); + } + + #[test] + fn characters_strings_and_pointers() { + check(b"[%c]\0", &[Arg::Int(b'A' as i64)], "[A]"); + check( + b"[%3c][%-3c]\0", + &[Arg::Int(b'x' as i64), Arg::Int(b'y' as i64)], + "[ x][y ]", + ); + check(b"%s\0", &[text(b"hello\0")], "hello"); + check( + b"[%8s][%-8s]\0", + &[text(b"ab\0"), text(b"cd\0")], + "[ ab][cd ]", + ); + check(b"%.2s\0", &[text(b"hello\0")], "he"); + check(b"%s\0", &[Arg::Ptr(0)], "(null)"); + check( + b"%lc %lc\0", + &[Arg::Int(b'q' as i64), Arg::Int(0x263a)], + "q ?", + ); + // `%p` is this crate's own spelling, not the host's. + check(b"%p\0", &[Arg::Ptr(0x1000)], "0x1000"); + check(b"%p\0", &[Arg::Ptr(0)], "0x0"); + check(b"[%10p]\0", &[Arg::Ptr(0xabc)], "[ 0xabc]"); + } + + #[test] + fn a_precision_bounds_the_read_of_an_unterminated_string() { + // No NUL in these bytes: `%.*s` must stop at its count. + check(b"[%.*s]\0", &[Arg::Int(3), text(b"abcd")], "[abc]"); + check(b"[%.4s]\0", &[text(b"abcd")], "[abcd]"); + } + + #[test] + fn star_width_and_precision() { + check(b"[%*d]\0", &[Arg::Int(5), Arg::Int(42)], "[ 42]"); + check(b"[%*d]\0", &[Arg::Int(-5), Arg::Int(42)], "[42 ]"); + check(b"[%*s]\0", &[Arg::Int(4), text(b"x\0")], "[ x]"); + check(b"[%.*s]\0", &[Arg::Int(2), text(b"xyz\0")], "[xy]"); + // A negative precision is as if it were absent. + check(b"[%.*s]\0", &[Arg::Int(-1), text(b"xyz\0")], "[xyz]"); + check(b"[%.*d]\0", &[Arg::Int(4), Arg::Int(7)], "[0007]"); + } + + #[test] + fn floats_are_formatted_and_the_arguments_after_them_line_up() { + let (out, _, taken) = run(b"%0.3f s, %u lines\0", &[Arg::Double(1.5), Arg::Uint(9)]); + assert_eq!(out, "1.500 s, 9 lines"); + assert_eq!(taken, ["double", "unsigned"]); + // `L` reads a long double, in the source's own format. + let x87_two_and_a_half = Float::from_x87(0x4000_a000_0000_0000_0000); + let (out, _, taken) = run( + b"a %Lf then %d\0", + &[Arg::LongDouble(x87_two_and_a_half), Arg::Int(7)], + ); + assert_eq!(out, "a 2.500000 then 7"); + assert_eq!(taken, ["long double", "signed"]); + check( + b"[%8.2e|%-8g|%+a]\0", + &[Arg::Double(-0.0), Arg::Double(1e-5), Arg::Double(0.5)], + "[-0.00e+00|1e-05 |+0x1p-1]", + ); + } + + #[test] + fn n_stores_nothing_and_unknown_conversions_consume_nothing() { + let (out, _, taken) = run(b"ab%ncd\0", &[Arg::Ptr(0x10)]); + assert_eq!(out, "abcd"); + assert_eq!(taken, ["pointer"]); + let (out, _, taken) = run(b"(%rip) %d\0", &[Arg::Int(1)]); + assert_eq!(out, "(%rip) 1"); + assert_eq!(taken, ["signed"]); + } + + #[test] + fn a_specification_cut_off_by_the_terminator_is_spelled_back() { + check(b"tail %-05\0", &[], "tail %-05"); + check(b"tail %\0", &[], "tail %"); + } + + #[test] + fn a_field_width_saturates_at_int_max() { + let digits = b"99999999999999999999x\0"; + let format = Format { + base: digits.as_ptr(), + _bytes: PhantomData, + }; + let mut at = 0; + // SAFETY: NUL-terminated; the scan stops at the `x`. + assert_eq!(unsafe { format.decimal(&mut at) }, MAX_FIELD); + assert_eq!(at, 20); + } +} diff --git a/native/crates/c-shim/src/format/float.rs b/native/crates/c-shim/src/format/float.rs new file mode 100644 index 0000000..2766af5 --- /dev/null +++ b/native/crates/c-shim/src/format/float.rs @@ -0,0 +1,866 @@ +//! The float conversions of the `printf` engine ([`super::format`]): `%e %E +//! %f %F %g %G %a %A`, of a `double` and of a `long double` in either format +//! kore's C ABIs give it — x87 double-extended on x86_64, IEEE binary128 on +//! aarch64 and riscv64. +//! +//! ## Every digit is the value's +//! +//! A finite float is `mantissa × 2^exponent` with an integer mantissa, so its +//! decimal expansion is finite, and a conversion prints that expansion +//! rounded once, to nearest with ties to even, at the last digit it keeps: +//! the correctly rounded conversion C asks for in the default rounding mode +//! (C11 7.21.6.1p13, F.5), and what glibc and musl print. A [`Decimal`] holds +//! the expansion in base-10⁹ limbs, built from the mantissa by doubling 29 +//! bits or halving 9 bits at a time (musl's scheme, kept in integers), so no +//! digit is estimated. +//! +//! Halving moves digits only toward the low end: a limb below the last digit +//! a conversion can keep never changes one above it. Such limbs are folded +//! into one sticky bit as they appear, which is all a round to nearest asks +//! of them, so a conversion stores at most the digits it prints and two limbs +//! more, and never more than the expansion has: a `%.1000000f` of a `double` +//! costs what its 1074 binary places give. Doubling carries upward, so an +//! integer part is built whole: at most 4933 digits, for a `long double`. +//! +//! ## Spellings +//! +//! An infinity is `inf` and a NaN `nan`, upper case under `E F G A`, each +//! signed as any value is: a NaN's sign bit is its sign, as glibc and musl +//! print it. `%a` normalizes every nonzero value to a leading `1`, subnormals +//! included, as musl does (glibc spells a subnormal `double` with a leading +//! `0`, and an x87 value with its explicit integer bit); with a precision it +//! rounds to nearest-even, which can carry the leading digit to `2`. + +use core::cmp::Ordering; + +use super::{Out, SPACES, Sink, Spec, ZEROS}; + +/// A float as a conversion reads it: its sign bit, and what it holds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Float { + negative: bool, + class: Class, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Class { + /// `mantissa × 2^exponent`; a zero mantissa is a zero of that sign. + Finite { + mantissa: u128, + exponent: i32, + }, + Infinite, + Nan, +} + +/// IEEE binary64: 52 stored fraction bits under an 11-bit exponent, the sign +/// on top. +const F64_FRACTION: u32 = 52; +const F64_BIAS: i32 = 1023; +const F64_EXPONENT_MASK: u64 = 0x7ff; +const F64_SIGN: u32 = u64::BITS - 1; + +/// x87 double-extended: a 64-bit significand whose integer bit is stored, +/// under a 15-bit exponent and the sign, in the low 80 bits. +const X87_SIGNIFICAND: u32 = u64::BITS; +const X87_FRACTION: u32 = X87_SIGNIFICAND - 1; +const X87_BIAS: i32 = 16383; +const X87_SIGN: u32 = X87_SIGNIFICAND + 15; + +/// IEEE binary128: 112 stored fraction bits under a 15-bit exponent, the +/// sign on top. +const QUAD_FRACTION: u32 = 112; +const QUAD_BIAS: i32 = 16383; +const QUAD_SIGN: u32 = u128::BITS - 1; + +/// The 15-bit exponent field both `long double` formats share. +const WIDE_EXPONENT_MASK: u128 = 0x7fff; + +impl Float { + /// A `double`. + pub fn from_f64(value: f64) -> Self { + let bits = value.to_bits(); + let biased = (bits >> F64_FRACTION) & F64_EXPONENT_MASK; + let fraction = u128::from(bits & ((1 << F64_FRACTION) - 1)); + let class = match biased { + F64_EXPONENT_MASK if fraction == 0 => Class::Infinite, + F64_EXPONENT_MASK => Class::Nan, + // Subnormal: the least exponent, and no implicit bit. + 0 => Class::Finite { + mantissa: fraction, + exponent: 1 - F64_BIAS - F64_FRACTION as i32, + }, + _ => Class::Finite { + mantissa: fraction | 1 << F64_FRACTION, + exponent: biased as i32 - F64_BIAS - F64_FRACTION as i32, + }, + }; + Self { + negative: bits >> F64_SIGN != 0, + class, + } + } + + /// An x87 double-extended `long double`: its 80 bits, low-aligned (any + /// padding above them is ignored). + /// + /// The encodings no x87 since the 387 produces are read as musl reads + /// them: an unnormal (a nonzero exponent and a clear integer bit) is a + /// NaN, a pseudo-infinity (the top exponent, a clear integer bit and a + /// zero fraction) an infinity, and a pseudo-denormal (a zero exponent and + /// a set integer bit) the value its bits spell. + pub fn from_x87(bits: u128) -> Self { + let negative = (bits >> X87_SIGN) & 1 != 0; + let biased = (bits >> X87_SIGNIFICAND) & WIDE_EXPONENT_MASK; + let significand = bits as u64; + let integer = significand >> X87_FRACTION != 0; + let class = match biased { + WIDE_EXPONENT_MASK if significand << 1 == 0 => Class::Infinite, + WIDE_EXPONENT_MASK => Class::Nan, + 0 => Class::Finite { + mantissa: u128::from(significand), + exponent: 1 - X87_BIAS - X87_FRACTION as i32, + }, + _ if !integer => Class::Nan, + _ => Class::Finite { + mantissa: u128::from(significand), + exponent: biased as i32 - X87_BIAS - X87_FRACTION as i32, + }, + }; + Self { negative, class } + } + + /// An IEEE binary128 `long double`. + pub fn from_quad(bits: u128) -> Self { + let biased = (bits >> QUAD_FRACTION) & WIDE_EXPONENT_MASK; + let fraction = bits & ((1 << QUAD_FRACTION) - 1); + let class = match biased { + WIDE_EXPONENT_MASK if fraction == 0 => Class::Infinite, + WIDE_EXPONENT_MASK => Class::Nan, + 0 => Class::Finite { + mantissa: fraction, + exponent: 1 - QUAD_BIAS - QUAD_FRACTION as i32, + }, + _ => Class::Finite { + mantissa: fraction | 1 << QUAD_FRACTION, + exponent: biased as i32 - QUAD_BIAS - QUAD_FRACTION as i32, + }, + }; + Self { + negative: bits >> QUAD_SIGN != 0, + class, + } + } +} + +/// The precision `e`, `f` and `g` take when none is given (C11 7.21.6.1p8). +const DEFAULT_PRECISION: i64 = 6; + +/// One base-10⁹ limb's radix. +const BASE: u32 = 1_000_000_000; + +/// Decimal digits per limb. +const LIMB_DIGITS: i64 = 9; + +/// `POW10[k]` is `10^k`, up to a limb's radix. +const POW10: [u32; 10] = [ + 1, + 10, + 100, + 1_000, + 10_000, + 100_000, + 1_000_000, + 10_000_000, + 100_000_000, + 1_000_000_000, +]; + +/// Bits one doubling pass shifts in: a limb below 10⁹ times 2²⁹, plus the +/// carry below 2²⁹ + 1, stays under 2⁶³, and the carry out fits one limb. +const DOUBLE_STEP: u32 = 29; + +/// Bits one halving pass shifts out. 10⁹ = 2⁹·5⁹, so a limb's remainder +/// below 2⁹ passes to the next limb as exactly `remainder × 10⁹ / 2⁹`, and +/// a pass adds at most one limb below the last. +const HALVE_STEP: u32 = 9; + +/// Limbs kept free in front of an expansion: rounding at the digit just +/// above it materializes one limb, and a carry out of the top one more. +const HEADROOM: usize = 2; + +/// Decimal digits an integer of `bits` bits can have: `⌊bits·log₁₀2⌋ + 1`, +/// by 30103/100000, which exceeds log₁₀2. +const fn decimal_digits(bits: u32) -> usize { + bits as usize * 30103 / 100_000 + 1 +} + +/// Limbs a mantissa occupies before it is scaled: any `u128`. +const MANTISSA_LIMBS: usize = decimal_digits(u128::BITS).div_ceil(LIMB_DIGITS as usize); + +/// The limbs a format's widest expansion needs: its largest integer part +/// (`mantissa_bits + max_exponent` bits, which doubling builds whole), or its +/// longest fraction (one digit per binary place below the point, and the +/// window's one limb more) beside the integer part a fraction can have; and +/// the rounding headroom. +const fn capacity(mantissa_bits: u32, max_exponent: i32, min_exponent: i32) -> usize { + let whole = decimal_digits(mantissa_bits + max_exponent as u32).div_ceil(LIMB_DIGITS as usize); + let fraction = (min_exponent.unsigned_abs() as usize).div_ceil(LIMB_DIGITS as usize) + + 1 + + decimal_digits(mantissa_bits).div_ceil(LIMB_DIGITS as usize); + (if whole > fraction { whole } else { fraction }) + HEADROOM +} + +const fn max(a: usize, b: usize) -> usize { + if a > b { a } else { b } +} + +/// A `double`'s expansion: 125 limbs. +pub(super) const DOUBLE_LIMBS: usize = capacity( + F64_FRACTION + 1, + (2 * F64_BIAS) - F64_BIAS - F64_FRACTION as i32, + 1 - F64_BIAS - F64_FRACTION as i32, +); + +/// A `long double`'s expansion, in either format: 1840 limbs. +pub(super) const LONG_DOUBLE_LIMBS: usize = max( + capacity( + X87_FRACTION + 1, + (2 * X87_BIAS) - X87_BIAS - X87_FRACTION as i32, + 1 - X87_BIAS - X87_FRACTION as i32, + ), + capacity( + QUAD_FRACTION + 1, + (2 * QUAD_BIAS) - QUAD_BIAS - QUAD_FRACTION as i32, + 1 - QUAD_BIAS - QUAD_FRACTION as i32, + ), +); + +/// The decimal expansion of a finite nonzero value, most significant limb +/// first: `limbs[start..start + len]`, the first of which is `10^(9·top)`'s. +/// Digits below the last stored limb are zero, or — when `sticky` is set — +/// include a nonzero one the window let go. +struct Decimal { + limbs: [u32; N], + start: usize, + len: usize, + top: i32, + sticky: bool, +} + +impl Decimal { + /// `mantissa × 2^exponent` (`mantissa` nonzero), exact in every limb whose + /// base-10⁹ exponent is at least `cut`. + fn new(mantissa: u128, exponent: i32, cut: i64) -> Self { + let mut low_first = [0u32; MANTISSA_LIMBS]; + let mut count = 0; + let mut rest = mantissa; + while rest != 0 { + low_first[count] = (rest % u128::from(BASE)) as u32; + rest /= u128::from(BASE); + count += 1; + } + // Doubling grows toward the front and halving toward the back. + let start = if exponent > 0 { N - count } else { HEADROOM }; + let mut decimal = Self { + limbs: [0; N], + start, + len: count, + top: count as i32 - 1, + sticky: false, + }; + for (slot, &limb) in decimal.limbs[start..start + count] + .iter_mut() + .zip(low_first[..count].iter().rev()) + { + *slot = limb; + } + match exponent.cmp(&0) { + Ordering::Greater => decimal.double(exponent.unsigned_abs()), + Ordering::Less => decimal.halve(exponent.unsigned_abs(), cut), + Ordering::Equal => {} + } + decimal + } + + fn double(&mut self, mut shift: u32) { + while shift > 0 { + let step = shift.min(DOUBLE_STEP); + let mut carry = 0u64; + for limb in self.limbs[self.start..self.start + self.len] + .iter_mut() + .rev() + { + let wide = (u64::from(*limb) << step) + carry; + *limb = (wide % u64::from(BASE)) as u32; + carry = wide / u64::from(BASE); + } + if carry != 0 { + self.start -= 1; + self.len += 1; + self.top += 1; + self.limbs[self.start] = carry as u32; + } + shift -= step; + } + } + + fn halve(&mut self, mut shift: u32, cut: i64) { + self.fold_below(cut); + while shift > 0 && self.len > 0 { + let step = shift.min(HALVE_STEP); + let mask = (1u32 << step) - 1; + let share = BASE >> step; + let mut carry = 0u32; + for limb in &mut self.limbs[self.start..self.start + self.len] { + let rest = *limb & mask; + *limb = (*limb >> step) + carry; + carry = share * rest; + } + // Only the leading limb can empty in a pass: the one after it + // gains the leading limb's whole remainder. + if self.limbs[self.start] == 0 { + self.start += 1; + self.len -= 1; + self.top -= 1; + } + if carry != 0 { + if i64::from(self.top) - self.len as i64 >= cut { + self.limbs[self.start + self.len] = carry; + self.len += 1; + } else { + self.sticky = true; + } + } + shift -= step; + } + } + + /// Fold every limb below the base-10⁹ exponent `cut` into `sticky`. + fn fold_below(&mut self, cut: i64) { + let keep = (i64::from(self.top) - cut + 1).clamp(0, self.len as i64) as usize; + let end = self.start + self.len; + if self.limbs[self.start + keep..end] + .iter() + .any(|&limb| limb != 0) + { + self.sticky = true; + } + self.len = keep; + } + + fn is_zero(&self) -> bool { + self.len == 0 + } + + /// The position of the leading digit: `p` for the digit worth `10^p`. + /// The expansion is nonzero. + fn leading(&self) -> i64 { + let first = self.limbs[self.start]; + let digits = POW10 + .iter() + .position(|&power| first < power) + .unwrap_or(POW10.len()); + LIMB_DIGITS * i64::from(self.top) + digits as i64 - 1 + } + + /// The digit at `position`. + #[cfg(test)] + fn digit(&self, position: i64) -> u32 { + let back = i64::from(self.top) - position.div_euclid(LIMB_DIGITS); + if !(0..self.len as i64).contains(&back) { + return 0; + } + let limb = self.limbs[self.start + back as usize]; + limb / POW10[position.rem_euclid(LIMB_DIGITS) as usize] % 10 + } + + /// Keep the digits at `position` and above, rounded to nearest, ties to + /// even, on everything below. + fn round(&mut self, position: i64) { + let below_everything = self.is_zero() || position - 1 > self.leading(); + let limb = position.div_euclid(LIMB_DIGITS); + let lowest = i64::from(self.top) - self.len as i64 + 1; + if below_everything { + // Under a tenth of the last kept digit's unit: zero. An empty + // expansion here had every digit folded below its window, which + // lies under the digit after `position`. + self.len = 0; + self.sticky = false; + return; + } + if limb < lowest { + // Every stored digit is kept, and what lies below them is under a + // tenth of the last kept digit's unit (the window reaches below + // `position`'s limb whenever anything was folded). + self.sticky = false; + return; + } + if limb > i64::from(self.top) { + // `position - 1` is the top digit of the leading limb, so the + // kept digit is the first of the limb above: materialize it. + self.start -= 1; + self.len += 1; + self.top += 1; + self.limbs[self.start] = 0; + } + let at = self.start + (i64::from(self.top) - limb) as usize; + let end = self.start + self.len; + let k = position.rem_euclid(LIMB_DIGITS) as usize; + let (round_digit, rest_nonzero) = if k > 0 { + let under = self.limbs[at] % POW10[k]; + ( + under / POW10[k - 1], + under % POW10[k - 1] != 0 || self.limbs[at + 1..end].iter().any(|&l| l != 0), + ) + } else if at + 1 < end { + let next = self.limbs[at + 1]; + let top_digit = POW10[LIMB_DIGITS as usize - 1]; + ( + next / top_digit, + next % top_digit != 0 || self.limbs[at + 2..end].iter().any(|&l| l != 0), + ) + } else { + (0, false) + }; + let rest_nonzero = rest_nonzero || self.sticky; + self.limbs[at] -= self.limbs[at] % POW10[k]; + self.len = at + 1 - self.start; + self.sticky = false; + let odd = self.limbs[at] / POW10[k] % 2 == 1; + if round_digit > 5 || (round_digit == 5 && (rest_nonzero || odd)) { + let mut carry_at = at; + self.limbs[carry_at] += POW10[k]; + while self.limbs[carry_at] >= BASE { + self.limbs[carry_at] -= BASE; + if carry_at == self.start { + self.start -= 1; + self.len += 1; + self.top += 1; + self.limbs[self.start] = 0; + } + carry_at -= 1; + self.limbs[carry_at] += 1; + } + } + while self.len > 0 && self.limbs[self.start] == 0 { + self.start += 1; + self.len -= 1; + self.top -= 1; + } + } + + /// The position of the lowest nonzero stored digit. + fn lowest(&self) -> Option { + let stored = &self.limbs[self.start..self.start + self.len]; + let back = stored.iter().rposition(|&limb| limb != 0)?; + let limb = stored[back]; + let zeros = POW10[1..] + .iter() + .take_while(|&&power| limb % power == 0) + .count(); + Some(LIMB_DIGITS * (i64::from(self.top) - back as i64) + zeros as i64) + } + + /// Write the digits at positions `from` down to `to`, inclusive. + fn write(&self, out: &mut Out<'_, S>, from: i64, to: i64) { + if from < to { + return; + } + let mut position = from; + let (high, low) = if self.is_zero() { + (i64::MIN, i64::MAX) + } else { + ( + LIMB_DIGITS * i64::from(self.top) + LIMB_DIGITS - 1, + LIMB_DIGITS * (i64::from(self.top) - self.len as i64 + 1), + ) + }; + if position > high { + let zeros = position - high.max(to - 1); + out.pad(&ZEROS, zeros as usize); + position -= zeros; + } + while position >= to && position >= low { + let limb = position.div_euclid(LIMB_DIGITS); + let at = self.start + (i64::from(self.top) - limb) as usize; + let mut text = [b'0'; LIMB_DIGITS as usize]; + let mut value = self.limbs[at]; + for byte in text.iter_mut().rev() { + *byte = b'0' + (value % 10) as u8; + value /= 10; + } + let hi = position.rem_euclid(LIMB_DIGITS) as usize; + let lo = if to.div_euclid(LIMB_DIGITS) == limb { + to.rem_euclid(LIMB_DIGITS) as usize + } else { + 0 + }; + let last = LIMB_DIGITS as usize - 1; + out.put(&text[last - hi..=last - lo]); + position -= (hi - lo + 1) as i64; + } + if position >= to { + out.pad(&ZEROS, (position - to + 1) as usize); + } + } +} + +/// Write the digits of `digits` (all zeros for a zero) from `from` down to +/// `to`. +fn write_digits( + out: &mut Out<'_, S>, + digits: Option<&Decimal>, + from: i64, + to: i64, +) { + match digits { + Some(digits) => digits.write(out, from, to), + None if from >= to => out.pad(&ZEROS, (from - to + 1) as usize), + None => {} + } +} + +/// The lowest base-10⁹ limb a rounding at `position` must read exactly: the +/// one holding the digit after it, and one more. Nothing below the +/// expansion's last digit is ever needed: a fraction of `-exponent` binary +/// places has exactly that many decimal ones. +fn window(position: i64, exponent: i32) -> i64 { + let cut = (position - 1).div_euclid(LIMB_DIGITS) - 1; + cut.max(i64::from(exponent).div_euclid(LIMB_DIGITS) - 1) +} + +/// log₁₀2 = 0.30102999566398…, truncated to eleven places: below the truth, +/// by less than 10⁻¹¹. +const LOG10_2_NUMERATOR: i64 = 30_102_999_566; +const LOG10_2_DENOMINATOR: i64 = 100_000_000_000; + +/// A position at or below that of `mantissa × 2^exponent`'s leading digit. +/// The value is at least `2^power` for `power = bits − 1 + exponent`, whose +/// leading digit sits at `⌊power·log₁₀2⌋`; the truncated logarithm moves +/// that floor by at most one over every exponent a `long double` has, and +/// one digit of slack absorbs it. +fn leading_floor(mantissa: u128, exponent: i32) -> i64 { + let power = i64::from(u128::BITS - mantissa.leading_zeros()) - 1 + i64::from(exponent); + (power * LOG10_2_NUMERATOR).div_euclid(LOG10_2_DENOMINATOR) - 1 +} + +/// `spec`'s precision for `e`, `f` and `g`. +fn precision(spec: Spec) -> i64 { + spec.precision + .map_or(DEFAULT_PRECISION, |precision| precision as i64) +} + +/// Format `value` by `conversion` (`e E f F g G a A`) under `spec`, its +/// expansion held in at most `N` limbs. Never inlined into the engine: a +/// `long double`'s limbs are 7 KiB of stack, which only its conversions pay. +#[inline(never)] +pub(super) fn convert( + out: &mut Out<'_, S>, + spec: Spec, + conversion: u8, + value: Float, +) { + let upper = conversion.is_ascii_uppercase(); + let sign: &[u8] = if value.negative { + b"-" + } else if spec.plus { + b"+" + } else if spec.space { + b" " + } else { + b"" + }; + let (mantissa, exponent) = match value.class { + Class::Infinite => return special(out, spec, sign, if upper { b"INF" } else { b"inf" }), + Class::Nan => return special(out, spec, sign, if upper { b"NAN" } else { b"nan" }), + Class::Finite { mantissa, exponent } => (mantissa, exponent), + }; + match conversion.to_ascii_lowercase() { + b'a' => hexadecimal(out, spec, sign, upper, mantissa, exponent), + b'f' => fixed::(out, spec, sign, mantissa, exponent), + b'e' => scientific::(out, spec, sign, upper, mantissa, exponent), + _ => general::(out, spec, sign, upper, mantissa, exponent), + } +} + +/// Lay out a number of `body` bytes after `sign` and `prefix` in `spec`'s +/// field: spaces before it, or after it under `-`, or zeros between the +/// prefix and the body under `0` (which `-` overrides). +fn field( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + prefix: &[u8], + body: usize, + write: impl FnOnce(&mut Out<'_, S>), +) { + let fill = spec.width.saturating_sub(sign.len() + prefix.len() + body); + let zeros = spec.zero && !spec.left; + if !spec.left && !zeros { + out.pad(&SPACES, fill); + } + out.put(sign); + out.put(prefix); + if zeros { + out.pad(&ZEROS, fill); + } + write(out); + if spec.left { + out.pad(&SPACES, fill); + } +} + +/// An infinity or a NaN: a word, which the `0` flag never pads. +fn special(out: &mut Out<'_, S>, spec: Spec, sign: &[u8], word: &[u8; 3]) { + let spec = Spec { + zero: false, + ..spec + }; + field(out, spec, sign, b"", word.len(), |out| out.put(word)); +} + +/// `%f`: the digits down to the precision's last. +fn fixed( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + mantissa: u128, + exponent: i32, +) { + let precision = precision(spec); + let digits = (mantissa != 0).then(|| { + let mut digits = Decimal::::new(mantissa, exponent, window(-precision, exponent)); + digits.round(-precision); + digits + }); + fixed_layout(out, spec, sign, digits.as_ref(), precision); +} + +/// `%e`: one digit, the precision's after the point, and the exponent. +fn scientific( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + upper: bool, + mantissa: u128, + exponent: i32, +) { + let precision = precision(spec); + let (digits, leading) = significant::(mantissa, exponent, precision); + exponent_layout(out, spec, sign, upper, digits.as_ref(), leading, precision); +} + +/// `%g`: `%e`'s rounding to the precision's significant digits, laid out as +/// `%f` when the exponent is at least −4 and below the precision, without +/// trailing fraction zeros unless `#` keeps them (C11 7.21.6.1p8). +fn general( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + upper: bool, + mantissa: u128, + exponent: i32, +) { + let significant_digits = match spec.precision { + Some(0) => 1, + Some(precision) => precision as i64, + None => DEFAULT_PRECISION, + }; + let (digits, leading) = significant::(mantissa, exponent, significant_digits - 1); + let lowest = digits.as_ref().and_then(Decimal::lowest); + if significant_digits > leading && leading >= -4 { + let precision = significant_digits - 1 - leading; + let fraction = if spec.alternate { + precision + } else { + lowest.map_or(0, |low| (-low).clamp(0, precision)) + }; + fixed_layout(out, spec, sign, digits.as_ref(), fraction); + } else { + let precision = significant_digits - 1; + let fraction = if spec.alternate { + precision + } else { + lowest.map_or(0, |low| (leading - low).clamp(0, precision)) + }; + exponent_layout(out, spec, sign, upper, digits.as_ref(), leading, fraction); + } +} + +/// The expansion rounded to `precision` digits after its leading one, and +/// that leading digit's position (`0` for a zero). The window reaches from a +/// bound below the leading digit, so the expansion is never empty. +fn significant( + mantissa: u128, + exponent: i32, + precision: i64, +) -> (Option>, i64) { + if mantissa == 0 { + return (None, 0); + } + let floor = leading_floor(mantissa, exponent); + let mut digits = Decimal::::new(mantissa, exponent, window(floor - precision, exponent)); + let leading = digits.leading(); + digits.round(leading - precision); + // A carry can open a new leading digit: 9.96 at one place is 1.0e1. + let leading = digits.leading(); + (Some(digits), leading) +} + +/// `[-]d…d[.d…d]`: the integer digits (one `0` when there are none), then the +/// point and `fraction` digits when there are any or `#` asks for the point. +fn fixed_layout( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + digits: Option<&Decimal>, + fraction: i64, +) { + let leading = digits + .filter(|digits| !digits.is_zero()) + .map(Decimal::leading); + let whole = leading.map_or(1, |leading| leading.max(0) + 1); + let point = fraction > 0 || spec.alternate; + let body = whole as usize + usize::from(point) + fraction as usize; + field(out, spec, sign, b"", body, |out| { + write_digits(out, digits, whole - 1, 0); + if point { + out.put(b"."); + } + write_digits(out, digits, -1, -fraction); + }); +} + +/// `[-]d[.d…d]e±dd`: the leading digit, then the point and `fraction` digits +/// when there are any or `#` asks for the point, then the exponent in at +/// least two digits. +fn exponent_layout( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + upper: bool, + digits: Option<&Decimal>, + leading: i64, + fraction: i64, +) { + let point = fraction > 0 || spec.alternate; + let mut text = [0u8; EXPONENT_TEXT]; + let power = exponent_text(&mut text, if upper { b'E' } else { b'e' }, leading, 2); + let body = 1 + usize::from(point) + fraction as usize + power.len(); + field(out, spec, sign, b"", body, |out| { + write_digits(out, digits, leading, leading); + if point { + out.put(b"."); + } + write_digits(out, digits, leading - 1, leading - fraction); + out.put(power); + }); +} + +/// The letter, the sign and up to six digits: a binary exponent of a +/// `long double` subnormal (−16494) is the widest. +const EXPONENT_TEXT: usize = 8; + +/// `letter`, a sign, and `value`'s decimal digits, at least `min_digits`. +fn exponent_text( + text: &mut [u8; EXPONENT_TEXT], + letter: u8, + value: i64, + min_digits: usize, +) -> &[u8] { + let mut digits = [b'0'; EXPONENT_TEXT - 2]; + let mut at = digits.len(); + let mut rest = value.unsigned_abs(); + while rest != 0 || digits.len() - at < min_digits { + at -= 1; + digits[at] = b'0' + (rest % 10) as u8; + rest /= 10; + } + let count = digits.len() - at; + text[0] = letter; + text[1] = if value < 0 { b'-' } else { b'+' }; + text[2..2 + count].copy_from_slice(&digits[at..]); + &text[..2 + count] +} + +/// `%a`: `0x1.h…hp±d`, the value normalized to a leading `1` and every +/// fraction digit it has — or, with a precision, that many, rounded to +/// nearest-even. +fn hexadecimal( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + upper: bool, + mantissa: u128, + exponent: i32, +) { + let alphabet: &[u8; 16] = if upper { + b"0123456789ABCDEF" + } else { + b"0123456789abcdef" + }; + let prefix: &[u8] = if upper { b"0X" } else { b"0x" }; + let (mut lead, mut fraction, mut count, power) = if mantissa == 0 { + (0u8, 0u128, 0usize, 0i64) + } else { + // The bits below the leading one, padded out to whole hex digits, + // without the trailing zero digits a precision did not ask for. + let tail = u128::BITS - 1 - mantissa.leading_zeros(); + let digits = tail.div_ceil(4); + let mut fraction = (mantissa ^ (1 << tail)) << (4 * digits - tail); + let mut count = digits as usize; + while count > 0 && fraction & 0xf == 0 { + fraction >>= 4; + count -= 1; + } + (1u8, fraction, count, i64::from(exponent) + i64::from(tail)) + }; + let mut zeros = 0usize; + if let Some(precision) = spec.precision { + if precision < count { + let dropped = 4 * (count - precision) as u32; + let half = 1u128 << (dropped - 1); + let rest = fraction & ((1u128 << dropped) - 1); + fraction >>= dropped; + let odd = if precision == 0 { + lead & 1 == 1 + } else { + fraction & 1 == 1 + }; + if rest > half || (rest == half && odd) { + fraction += 1; + if precision == 0 || fraction >> (4 * precision) != 0 { + lead += 1; + fraction = 0; + } + } + count = precision; + } else { + zeros = precision - count; + } + } + let point = count + zeros > 0 || spec.alternate; + let mut text = [0u8; EXPONENT_TEXT]; + let power = exponent_text(&mut text, if upper { b'P' } else { b'p' }, power, 1); + let body = 1 + usize::from(point) + count + zeros + power.len(); + field(out, spec, sign, prefix, body, |out| { + out.put(&[alphabet[usize::from(lead)]]); + if point { + out.put(b"."); + } + let mut digits = [0u8; HEX_FRACTION_DIGITS]; + for (i, digit) in digits[..count].iter_mut().enumerate() { + *digit = alphabet[((fraction >> (4 * (count - 1 - i))) & 0xf) as usize]; + } + out.put(&digits[..count]); + out.pad(&ZEROS, zeros); + out.put(power); + }); +} + +/// Hex digits below the leading one any mantissa can have: a `u128`'s. +const HEX_FRACTION_DIGITS: usize = (u128::BITS as usize - 1).div_ceil(4); + +#[cfg(test)] +mod tests; diff --git a/native/crates/c-shim/src/format/float/tests.rs b/native/crates/c-shim/src/format/float/tests.rs new file mode 100644 index 0000000..db777af --- /dev/null +++ b/native/crates/c-shim/src/format/float/tests.rs @@ -0,0 +1,620 @@ +//! The float conversions against the host C library's own. glibc's +//! `snprintf` is the reference for `double`, through every flag, width and +//! precision. For the two `long double` formats it is glibc's `strfroml` +//! (x87, on an x86_64 host) and `strfromf128` (binary128), reached through +//! trampolines: Rust has no x87 type to pass, nor a binary128 one that +//! crosses into C. Where musl and glibc disagree — `%a` of a subnormal, of an +//! x87 value, and x87's invalid encodings — the cases are musl's spellings, +//! written out. + +extern crate std; + +use core::ffi::{c_char, c_int}; +use std::ffi::CString; +use std::string::String; +use std::vec::Vec; + +use super::super::{Args, Length, Sink, format}; +use super::{DOUBLE_LIMBS, Decimal, Float, LONG_DOUBLE_LIMBS, window}; + +/// The one float argument a test format converts. +enum Value { + Double(f64), + Long(Float), +} + +struct One(Option); + +impl Args for One { + fn signed(&mut self, _: Length) -> i64 { + unreachable!("a float test formats one float") + } + fn unsigned(&mut self, _: Length) -> u64 { + unreachable!("a float test formats one float") + } + fn int(&mut self) -> c_int { + unreachable!("a float test formats one float") + } + fn pointer(&mut self) -> usize { + unreachable!("a float test formats one float") + } + fn double(&mut self) -> f64 { + match self.0.take() { + Some(Value::Double(value)) => value, + _ => panic!("the format read a double it was not given"), + } + } + fn long_double(&mut self) -> Float { + match self.0.take() { + Some(Value::Long(value)) => value, + _ => panic!("the format read a long double it was not given"), + } + } +} + +struct Collect(Vec); + +impl Sink for Collect { + fn put(&mut self, bytes: &[u8]) { + self.0.extend_from_slice(bytes); + } +} + +fn ours(spec: &str, value: Value) -> String { + let spec = CString::new(spec).unwrap(); + let mut args = One(Some(value)); + let mut sink = Collect(Vec::new()); + // SAFETY: NUL-terminated, and the one conversion matches the argument. + let count = unsafe { format(&mut sink, spec.as_ptr().cast(), &mut args) }; + assert!(args.0.is_none(), "the format did not read its argument"); + assert_eq!(count, sink.0.len(), "the count is the output's length"); + String::from_utf8(sink.0).unwrap() +} + +/// Room for the widest `%f` of a `long double` with a long precision. +const HOST_BUFFER: usize = 1 << 14; + +unsafe extern "C" { + fn snprintf(buffer: *mut c_char, size: usize, format: *const c_char, ...) -> c_int; +} + +fn host_double(spec: &str, value: f64) -> String { + let spec = CString::new(spec).unwrap(); + let mut buffer = std::vec![0u8; HOST_BUFFER]; + // SAFETY: a buffer of its stated size and a matching double. + let written = unsafe { + snprintf( + buffer.as_mut_ptr().cast(), + buffer.len(), + spec.as_ptr(), + value, + ) + }; + let written = usize::try_from(written).unwrap(); + assert!(written < buffer.len()); + String::from_utf8(buffer[..written].to_vec()).unwrap() +} + +/// xorshift64*: fixed seeds, so a failure reproduces. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) + } +} + +/// Values the conversions turn on: zeros and ones, exact ties at every +/// place, the powers of ten either side of a rounding carry, the subnormal +/// and normal edges, the largest finite value, and the non-finite ones. +fn edge_doubles() -> Vec { + let mut values = std::vec![ + 0.0, + -0.0, + 1.0, + -1.0, + 0.5, + 1.5, + 2.5, + 3.5, + -2.5, + 0.125, + 0.375, + 0.625, + 0.05, + 0.15, + 0.25, + 0.35, + 9.5, + 10.5, + 1e-7, + 1e-5, + 1e-4, + 0.000099999995, + 9.9999995, + 99999.95, + 999999.5, + 123456.0, + 1234567.0, + 1e15, + 1e16, + 1e17, + 1e21, + 1e22, + 1e23, + 1e100, + 9007199254740992.0, + 9007199254740994.0, + 1.0 / 3.0, + 2.0 / 3.0, + 0.1, + 0.2, + 0.3, + f64::EPSILON, + f64::MIN_POSITIVE, + f64::from_bits(1), + f64::from_bits(3), + f64::from_bits(0x000f_ffff_ffff_ffff), + f64::MAX, + -f64::MAX, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + -f64::NAN, + ]; + for exponent in -30..30 { + values.push(10f64.powi(exponent)); + values.push(5.0 * 10f64.powi(exponent)); + } + values +} + +/// Random bit patterns (every exponent, so mostly huge and tiny values) +/// and random values of everyday size. +fn random_doubles(count: usize) -> Vec { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let mut values = Vec::with_capacity(2 * count); + for _ in 0..count { + values.push(f64::from_bits(rng.next())); + let everyday = (rng.next() & 0x800f_ffff_ffff_ffff) | ((1023 - 40 + rng.next() % 80) << 52); + values.push(f64::from_bits(everyday)); + } + values +} + +const DOUBLE_SPECS: &[&str] = &[ + "%e", + "%.0e", + "%.1e", + "%.3e", + "%.16e", + "%.17e", + "%.40e", + "%E", + "%#.0e", + "%+e", + "% .2e", + "[%15.3e]", + "[%-15.3e]", + "%015.3e", + "%f", + "%.0f", + "%.1f", + "%.3f", + "%.20f", + "%.1100f", + "%F", + "%#.0f", + "%+08.2f", + "[%-12.4f]", + "%g", + "%.0g", + "%.1g", + "%.2g", + "%.10g", + "%.17g", + "%.40g", + "%G", + "%+g", + "%012g", + "[%-12g]", + "%a", + "%.0a", + "%.1a", + "%.3a", + "%.13a", + "%.20a", + "%A", + "%#.0a", + "%+a", + "%020a", + "[%-20a]", +]; + +/// `%a` is one conversion musl and glibc spell differently for a `double`: +/// glibc keeps a subnormal's leading `0`. +fn hex_differs(spec: &str, value: f64) -> bool { + spec.contains(['a', 'A']) && value != 0.0 && value.abs() < f64::MIN_POSITIVE +} + +/// `%#g` is the other: when the rounding carries the exponent up to the +/// precision, glibc drops the trailing zeros `#` keeps (musl keeps them, as +/// C 7.21.6.1p8 says). The reference is built from glibc's `%e` and `%#f` or +/// `%#e`, as that paragraph defines `%g`. +const ALTERNATE_GENERAL: &[(&str, Option)] = + &[("%#g", None), ("%#.3g", Some(3)), ("%#.0g", Some(0))]; + +fn host_alternate_general(precision: Option, value: f64) -> String { + if !value.is_finite() { + return host_double("%#g", value); + } + let significant = match precision { + Some(0) => 1, + Some(precision) => precision, + None => 6, + }; + let scientific = host_double(&std::format!("%.{}e", significant - 1), value); + let exponent: i64 = scientific.rsplit_once('e').unwrap().1.parse().unwrap(); + if significant > exponent && exponent >= -4 { + host_double(&std::format!("%#.{}f", significant - 1 - exponent), value) + } else { + host_double(&std::format!("%#.{}e", significant - 1), value) + } +} + +#[test] +fn doubles_print_as_the_host_prints_them() { + let mut values = edge_doubles(); + values.extend(random_doubles(1500)); + for &value in &values { + for &spec in DOUBLE_SPECS { + if hex_differs(spec, value) { + continue; + } + assert_eq!( + ours(spec, Value::Double(value)), + host_double(spec, value), + "{spec} of {value:e} ({:#018x})", + value.to_bits() + ); + } + for &(spec, precision) in ALTERNATE_GENERAL { + assert_eq!( + ours(spec, Value::Double(value)), + host_alternate_general(precision, value), + "{spec} of {value:e} ({:#018x})", + value.to_bits() + ); + } + } +} + +#[test] +fn an_alternate_general_keeps_its_zeros_through_a_carry_as_musl_does() { + let cases: &[(&str, f64, &str)] = &[ + ("%#g", 999999.5, "1.00000e+06"), + ("%#.3g", 999.5, "1.00e+03"), + ("%#.2g", 99.5, "1.0e+02"), + ("%#.1g", 9.5, "1.e+01"), + ("%#.0g", 9.5, "1.e+01"), + ("%#.6g", 0.0009999995, "0.00100000"), + ]; + for &(spec, value, expected) in cases { + assert_eq!( + ours(spec, Value::Double(value)), + expected, + "{spec} of {value:e}" + ); + } +} + +#[test] +fn a_subnormal_double_in_hex_is_normalized_as_musl_spells_it() { + let cases: &[(&str, u64, &str)] = &[ + ("%a", 1, "0x1p-1074"), + ("%a", 3, "0x1.8p-1073"), + ("%.0a", 3, "0x2p-1073"), + ("%a", 0x000f_ffff_ffff_ffff, "0x1.ffffffffffffep-1023"), + ("%.1A", 0x0008_0000_0000_0001, "0X1.0P-1023"), + ]; + for &(spec, bits, expected) in cases { + assert_eq!( + ours(spec, Value::Double(f64::from_bits(bits))), + expected, + "{spec} of {bits:#x}" + ); + } +} + +#[test] +fn every_format_holds_its_widest_expansion() { + // The widest integer part, built whole by doubling. + let max = Decimal::::new((1 << 53) - 1, 971, 0); + assert_eq!(max.leading(), 308); + // The longest fraction a window can reach, whatever the precision. + let least = Decimal::::new(1, -1074, window(-1_000_000, -1074)); + assert_eq!(least.leading(), -324); + assert_eq!( + (-339..=-324) + .rev() + .map(|p| least.digit(p)) + .collect::>(), + [4, 9, 4, 0, 6, 5, 6, 4, 5, 8, 4, 1, 2, 4, 6, 5] + ); + assert!(!least.sticky, "nothing of the expansion was let go"); + assert_eq!(least.lowest(), Some(-1074)); + let quad_max = Decimal::::new((1 << 113) - 1, 16271, 0); + assert_eq!(quad_max.leading(), 4932); + let quad_least = Decimal::::new(1, -16494, window(-1_000_000, -16494)); + assert_eq!(quad_least.leading(), -4966); + assert_eq!(quad_least.lowest(), Some(-16494)); + let x87_least = Decimal::::new(1, -16445, window(-1_000_000, -16445)); + assert_eq!(x87_least.lowest(), Some(-16445)); +} + +#[test] +fn a_huge_precision_prints_its_zeros_without_storing_them() { + let text = ours("%.100000f", Value::Double(0.5)); + assert_eq!(text.len(), 2 + 100_000); + assert!(text.starts_with("0.5000")); + assert!(text[3..].bytes().all(|b| b == b'0')); +} + +/// The `(precision, conversion)` pairs `strfrom*` accepts, which name no +/// flag or width: those are the engine's own, which the `double` sweep has +/// covered. +const WIDE_SPECS: &[(Option, char)] = &[ + (None, 'e'), + (Some(0), 'e'), + (Some(1), 'e'), + (Some(5), 'e'), + (Some(18), 'e'), + (Some(25), 'e'), + (Some(36), 'e'), + (Some(60), 'E'), + (None, 'f'), + (Some(0), 'f'), + (Some(3), 'f'), + (Some(30), 'f'), + (Some(60), 'F'), + (None, 'g'), + (Some(0), 'g'), + (Some(1), 'g'), + (Some(5), 'g'), + (Some(18), 'g'), + (Some(21), 'g'), + (Some(36), 'G'), + (Some(50), 'g'), +]; + +fn wide_spec(precision: Option, conversion: char, long: bool) -> String { + let length = if long { "L" } else { "" }; + match precision { + Some(precision) => std::format!("%.{precision}{length}{conversion}"), + None => std::format!("%{length}{conversion}"), + } +} + +/// Random `long double` encodings of `format`: every exponent, every +/// everyday one, and the subnormals; each a valid encoding. +fn random_wide(rng: &mut Rng, x87: bool) -> u128 { + let sign = u128::from(rng.next() >> 63) << if x87 { 79 } else { 127 }; + let roll = rng.next() % 8; + let biased: u128 = match roll { + 0 => 0, + 1..=3 => u128::from(16383 - 200 + rng.next() % 400), + _ => u128::from(1 + rng.next() % 0x7ffd), + }; + let fraction = (u128::from(rng.next()) << 64) | u128::from(rng.next()); + if x87 { + let integer = u128::from(biased != 0) << 63; + sign | biased << 64 | integer | (fraction & ((1 << 63) - 1)) + } else { + sign | biased << 112 | (fraction & ((1 << 112) - 1)) + } +} + +#[cfg(target_arch = "x86_64")] +mod x86_64_host { + use super::*; + + unsafe extern "C" { + fn strfroml(); + fn strfromf128(); + } + + /// glibc's `strfroml(buffer, size, format, value)`: the x87 value rides + /// the stack, as System V passes every `long double`. + fn host_x87(spec: &str, bits: u128) -> String { + let spec = CString::new(spec).unwrap(); + let mut buffer = std::vec![0u8; HOST_BUFFER]; + let value = bits.to_le_bytes(); + let written: i32; + // SAFETY: the stack stays 16-aligned for the call (`asm!` enters + // aligned and the frame is sixteen bytes), the arguments are live, + // and every caller-saved register is declared clobbered. + unsafe { + core::arch::asm!( + "sub rsp, 16", + "mov rax, qword ptr [r10]", + "mov qword ptr [rsp], rax", + "mov rax, qword ptr [r10 + 8]", + "mov qword ptr [rsp + 8], rax", + "call {f}", + "add rsp, 16", + f = sym strfroml, + in("r10") value.as_ptr(), + in("rdi") buffer.as_mut_ptr(), + in("rsi") buffer.len(), + in("rdx") spec.as_ptr(), + lateout("eax") written, + clobber_abi("C"), + ); + } + let written = usize::try_from(written).unwrap(); + assert!(written < buffer.len()); + String::from_utf8(buffer[..written].to_vec()).unwrap() + } + + /// glibc's `strfromf128(buffer, size, format, value)`: `_Float128` + /// rides `xmm0`. + fn host_quad(spec: &str, bits: u128) -> String { + let spec = CString::new(spec).unwrap(); + let mut buffer = std::vec![0u8; HOST_BUFFER]; + let value = bits.to_le_bytes(); + let written: i32; + // SAFETY: as `host_x87`; the value is loaded into `xmm0` from live + // bytes, and `xmm0` is caller-saved. + unsafe { + core::arch::asm!( + "movdqu xmm0, xmmword ptr [r10]", + "call {f}", + f = sym strfromf128, + in("r10") value.as_ptr(), + in("rdi") buffer.as_mut_ptr(), + in("rsi") buffer.len(), + in("rdx") spec.as_ptr(), + lateout("eax") written, + clobber_abi("C"), + ); + } + let written = usize::try_from(written).unwrap(); + assert!(written < buffer.len()); + String::from_utf8(buffer[..written].to_vec()).unwrap() + } + + fn edge_wide(x87: bool) -> Vec { + if x87 { + std::vec![ + 0, + 1 << 79, + 0x3fff_8000_0000_0000_0000, + 0x3ffb_cccc_cccc_cccc_cccd, + 0x4000_a000_0000_0000_0000, + 0x7ffe_ffff_ffff_ffff_ffff, + 0x0001_8000_0000_0000_0000, + 0x0000_0000_0000_0000_0001, + 0x0000_7fff_ffff_ffff_ffff, + 0x7fff_8000_0000_0000_0000, + 0xffff_8000_0000_0000_0000, + 0x7fff_c000_0000_0000_0000, + ] + } else { + std::vec![ + 0, + 1 << 127, + 0x3fff_0000_0000_0000_0000_0000_0000_0000, + 0x3ffb_9999_9999_9999_9999_9999_9999_999a, + 0x7ffe_ffff_ffff_ffff_ffff_ffff_ffff_ffff, + 0x0001_0000_0000_0000_0000_0000_0000_0000, + 0x0000_0000_0000_0000_0000_0000_0000_0001, + 0x7fff_0000_0000_0000_0000_0000_0000_0000, + 0xffff_0000_0000_0000_0000_0000_0000_0000, + 0x7fff_8000_0000_0000_0000_0000_0000_0000, + ] + } + } + + fn sweep(x87: bool) { + let mut rng = Rng(if x87 { + 0x0123_4567_89ab_cdef + } else { + 0xfedc_ba98_7654_3210 + }); + let mut values = edge_wide(x87); + values.extend((0..400).map(|_| random_wide(&mut rng, x87))); + for &bits in &values { + let value = if x87 { + Float::from_x87(bits) + } else { + Float::from_quad(bits) + }; + for &(precision, conversion) in WIDE_SPECS { + let host = if x87 { + host_x87(&wide_spec(precision, conversion, false), bits) + } else { + host_quad(&wide_spec(precision, conversion, false), bits) + }; + let spec = wide_spec(precision, conversion, true); + assert_eq!( + ours(&spec, Value::Long(value)), + host, + "{spec} of {bits:#034x}" + ); + } + // glibc and musl agree on `%a` of a normal binary128 value only. + let subnormal = (bits >> 112) & 0x7fff == 0 && bits << 1 != 0; + if !x87 && !subnormal { + for spec in ["%a", "%.0a", "%.5a", "%.28a", "%.40A"] { + let long = spec.replace('a', "La").replace('A', "LA"); + assert_eq!( + ours(&long, Value::Long(value)), + host_quad(spec, bits), + "{long} of {bits:#034x}" + ); + } + } + } + } + + #[test] + fn x87_long_doubles_print_as_the_host_prints_them() { + sweep(true); + } + + #[test] + fn binary128_long_doubles_print_as_the_host_prints_them() { + sweep(false); + } +} + +#[test] +fn an_x87_value_in_hex_is_normalized_as_musl_spells_it() { + let cases: &[(u128, &str)] = &[ + (0x3fff_8000_0000_0000_0000, "0x1p+0"), + (0x3ffb_cccc_cccc_cccc_cccd, "0x1.999999999999999ap-4"), + (0x7ffe_ffff_ffff_ffff_ffff, "0x1.fffffffffffffffep+16383"), + (0x0000_0000_0000_0000_0001, "0x1p-16445"), + (0xbfff_c000_0000_0000_0000, "-0x1.8p+0"), + ]; + for &(bits, expected) in cases { + assert_eq!( + ours("%La", Value::Long(Float::from_x87(bits))), + expected, + "{bits:#x}" + ); + } +} + +#[test] +fn x87_encodings_no_fpu_produces_are_read_as_musl_reads_them() { + let spell = |bits: u128| ours("%Lg", Value::Long(Float::from_x87(bits))); + // An unnormal: a clear integer bit under a nonzero exponent. + assert_eq!(spell(0x3fff_4000_0000_0000_0000), "nan"); + // A pseudo-NaN and a pseudo-infinity: the top exponent, integer bit clear. + assert_eq!(spell(0x7fff_4000_0000_0000_0000), "nan"); + assert_eq!(spell(0xffff_0000_0000_0000_0000), "-inf"); + // A pseudo-denormal is the value its bits spell: the least normal. + let least_normal = ours( + "%.30Le", + Value::Long(Float::from_x87(0x0001_8000_0000_0000_0000)), + ); + assert_eq!( + ours( + "%.30Le", + Value::Long(Float::from_x87(0x0000_8000_0000_0000_0000)) + ), + least_normal + ); + // The padding above the 80 bits is not the value's. + assert_eq!( + ours( + "%Lg", + Value::Long(Float::from_x87(0xdead_0000_3fff_8000_0000_0000_0000)) + ), + "1" + ); +} diff --git a/native/crates/c-shim/src/format_symbols.rs b/native/crates/c-shim/src/format_symbols.rs new file mode 100644 index 0000000..da703e9 --- /dev/null +++ b/native/crates/c-shim/src/format_symbols.rs @@ -0,0 +1,342 @@ +//! The `` writes: the `printf` family over [`crate::format`], and +//! the character and block writes over [`crate::stdio`]'s streams. +//! +//! `snprintf` returns what ISO says — the length the complete output would +//! have had — writing at most `n - 1` bytes and a NUL. A count past `INT_MAX` +//! returns `-1`, C's `EOVERFLOW` case. The stream writes return `EOF` when +//! their stream's write fails, which on `stdout`/`stderr` means no +//! [`StandardSink`](crate::stdio::StandardSink) was installed. + +use core::ffi::{VaList, c_char, c_int, c_long, c_longlong, c_uint, c_ulong, c_ulonglong, c_void}; + +use crate::format::{Args, Length, Sink, format}; +use crate::stdio::{self, File, stdout}; + +const EOF: c_int = -1; + +/// A C `va_list` as the engine's argument source. +struct VaArgs<'a, 'f>(&'a mut VaList<'f>); + +impl Args for VaArgs<'_, '_> { + fn signed(&mut self, length: Length) -> i64 { + // SAFETY (each `next_arg`): the format's conversion names this C + // type, the variadic caller's half of the contract. + unsafe { + match length { + // `char` and `short` arrive promoted to `int`. + Length::Char => i64::from(self.0.next_arg::() as i8), + Length::Short => i64::from(self.0.next_arg::() as i16), + Length::Int => i64::from(self.0.next_arg::()), + Length::Long => self.0.next_arg::(), + // glibc reads `%Ld` as `long long`; so does this. + Length::LongLong | Length::Max | Length::LongDouble => { + self.0.next_arg::() + } + Length::Size | Length::Diff => self.0.next_arg::() as i64, + } + } + } + + fn unsigned(&mut self, length: Length) -> u64 { + // SAFETY: as `signed`. + unsafe { + match length { + Length::Char => u64::from(self.0.next_arg::() as u8), + Length::Short => u64::from(self.0.next_arg::() as u16), + Length::Int => u64::from(self.0.next_arg::()), + Length::Long => self.0.next_arg::(), + Length::LongLong | Length::Max | Length::LongDouble => { + self.0.next_arg::() + } + Length::Size | Length::Diff => self.0.next_arg::() as u64, + } + } + } + + fn int(&mut self) -> c_int { + // SAFETY: as `signed`. + unsafe { self.0.next_arg::() } + } + + fn pointer(&mut self) -> usize { + // SAFETY: as `signed`. + unsafe { self.0.next_arg::<*const c_void>() as usize } + } + + fn double(&mut self) -> f64 { + // SAFETY: as `signed`. + unsafe { self.0.next_arg::() } + } + + fn long_double(&mut self) -> crate::format::Float { + // SAFETY: as `signed`; the conversion's `L` names a `long double`. + unsafe { crate::va_long_double::next(self.0) } + } +} + +/// A C buffer that keeps counting past its end, as `snprintf` must. +struct Bounded { + target: *mut u8, + /// Bytes that fit before the terminator. + room: usize, + stored: usize, +} + +impl Sink for Bounded { + fn put(&mut self, bytes: &[u8]) { + let take = bytes.len().min(self.room - self.stored); + // SAFETY: `stored + take <= room`, inside the caller's buffer. + unsafe { + core::ptr::copy_nonoverlapping(bytes.as_ptr(), self.target.add(self.stored), take) + }; + self.stored += take; + } +} + +/// `vsnprintf` with an explicit room: `None` is `vsprintf`'s unbounded +/// buffer, whose size the C caller vouches for. +/// +/// # Safety +/// C's `vsnprintf` contract. +unsafe fn bounded( + target: *mut c_char, + len: Option, + format_: *const c_char, + args: &mut VaList, +) -> c_int { + let room = match len { + Some(0) => 0, + Some(len) => len - 1, + None => usize::MAX, + }; + let mut out = Bounded { + target: target.cast(), + room, + stored: 0, + }; + // SAFETY: the caller's contract. + let count = unsafe { format(&mut out, format_.cast(), &mut VaArgs(args)) }; + if len != Some(0) { + // SAFETY: `stored <= room`, which left one byte for the NUL. + unsafe { *out.target.add(out.stored) = 0 }; + } + c_int::try_from(count).unwrap_or(-1) +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn vsnprintf( + target: *mut c_char, + len: usize, + format: *const c_char, + mut args: VaList, +) -> c_int { + // SAFETY: the caller's contract is vsnprintf's. + unsafe { bounded(target, Some(len), format, &mut args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn snprintf( + target: *mut c_char, + len: usize, + format: *const c_char, + args: ... +) -> c_int { + // SAFETY: the caller's contract is vsnprintf's. + unsafe { vsnprintf(target, len, format, args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn vsprintf( + target: *mut c_char, + format: *const c_char, + mut args: VaList, +) -> c_int { + // SAFETY: the caller vouches that the buffer holds the output. + unsafe { bounded(target, None, format, &mut args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn sprintf(target: *mut c_char, format: *const c_char, args: ...) -> c_int { + // SAFETY: as `vsprintf`. + unsafe { vsprintf(target, format, args) } +} + +/// A stream write, batched: the engine's runs collect here and reach the +/// stream a buffer at a time. +struct Streamed { + file: *mut File, + buffer: [u8; 256], + held: usize, + failed: bool, +} + +impl Streamed { + fn drain(&mut self) { + // SAFETY: the stream is live for the whole call. + if self.held > 0 && !unsafe { stdio::write(self.file, &self.buffer[..self.held]) } { + self.failed = true; + } + self.held = 0; + } +} + +impl Sink for Streamed { + fn put(&mut self, bytes: &[u8]) { + if self.held + bytes.len() > self.buffer.len() { + self.drain(); + } + if bytes.len() > self.buffer.len() { + // A run bigger than the buffer goes straight through. + // SAFETY: as `drain`. + if !unsafe { stdio::write(self.file, bytes) } { + self.failed = true; + } + return; + } + self.buffer[self.held..self.held + bytes.len()].copy_from_slice(bytes); + self.held += bytes.len(); + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn vfprintf(file: *mut File, format_: *const c_char, mut args: VaList) -> c_int { + // One call, one lock: every batch reaches the stream before another + // thread's call does. + // SAFETY: C's vfprintf contract: a live stream and a matching format. + unsafe { + stdio::locked(file, || { + let mut out = Streamed { + file, + buffer: [0; 256], + held: 0, + failed: false, + }; + let count = format(&mut out, format_.cast(), &mut VaArgs(&mut args)); + out.drain(); + if out.failed { + return EOF; + } + c_int::try_from(count).unwrap_or(-1) + }) + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fprintf(file: *mut File, format: *const c_char, args: ...) -> c_int { + // SAFETY: as `vfprintf`. + unsafe { vfprintf(file, format, args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn vprintf(format: *const c_char, args: VaList) -> c_int { + // SAFETY: as `vfprintf`, on `stdout`. + unsafe { vfprintf(stdout.file(), format, args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn printf(format: *const c_char, args: ...) -> c_int { + // SAFETY: as `vfprintf`, on `stdout`. + unsafe { vfprintf(stdout.file(), format, args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fputc(byte: c_int, file: *mut File) -> c_int { + let byte = byte as u8; + // SAFETY: C's contract: a live stream. + if unsafe { stdio::write(file, &[byte]) } { + c_int::from(byte) + } else { + EOF + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn putc(byte: c_int, file: *mut File) -> c_int { + // SAFETY: as `fputc`. + unsafe { fputc(byte, file) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn putchar(byte: c_int) -> c_int { + // SAFETY: `stdout` is live. + unsafe { fputc(byte, stdout.file()) } +} + +/// The bytes of a C string, without its NUL. +/// +/// # Safety +/// `text` is NUL-terminated. +unsafe fn c_str<'a>(text: *const c_char) -> &'a [u8] { + // SAFETY: the caller's contract. + unsafe { core::ffi::CStr::from_ptr(text) }.to_bytes() +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fputs(text: *const c_char, file: *mut File) -> c_int { + // SAFETY: C's contract: a NUL-terminated string and a live stream. + if unsafe { stdio::write(file, c_str(text)) } { + 0 + } else { + EOF + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn puts(text: *const c_char) -> c_int { + let file = stdout.file(); + // The line and its newline under one lock, as one call. + // SAFETY: as `fputs`, on `stdout`. + let written = unsafe { + stdio::locked(file, || { + stdio::write(file, c_str(text)) && stdio::write(file, b"\n") + }) + }; + if written { 0 } else { EOF } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fwrite( + source: *const c_void, + size: usize, + count: usize, + file: *mut File, +) -> usize { + let Some(total) = size.checked_mul(count) else { + return 0; + }; + if total == 0 { + return 0; + } + // SAFETY: C promises `size * count` readable bytes and a live stream. + let bytes = unsafe { core::slice::from_raw_parts(source.cast::(), total) }; + if unsafe { stdio::write(file, bytes) } { + count + } else { + 0 + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fflush(file: *mut File) -> c_int { + if file.is_null() { + // "Every output stream": the standard pair holds nothing back, and a + // consumer's streams flush on their own close. + return 0; + } + // SAFETY: C's contract: a live stream. + if unsafe { stdio::flush(file) } { + 0 + } else { + EOF + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fclose(file: *mut File) -> c_int { + // SAFETY: C's contract: a live stream, not used again. + if unsafe { stdio::close(file) } { + 0 + } else { + EOF + } +} diff --git a/native/crates/c-shim/src/lib.rs b/native/crates/c-shim/src/lib.rs new file mode 100644 index 0000000..68bb647 --- /dev/null +++ b/native/crates/c-shim/src/lib.rs @@ -0,0 +1,83 @@ +//! The C symbols a vendored C library imports on a kore user target. +//! +//! A C library built by `tools/kore-cc` is freestanding: Zig's musl triple +//! supplies ISO C headers and nothing links musl's runtime. What such a +//! library still calls is defined here, over the task's own heap and Rust's +//! Unicode tables, so each consumer does not grow its own copy: +//! +//! - allocation (`malloc`, `calloc`, `realloc`, `free`) and `abort`, each +//! behind its own feature for a C runtime that exports its own; +//! - the `` routines beyond `compiler_builtins`' `memcpy`, +//! `memmove`, `memset`, `memcmp` and `strlen` (its `mem` feature, which +//! every kore user build enables); +//! - with `wide-classes` (a default), the wide character classes; +//! - the `` writes: the `printf` family over one conversion engine +//! ([`format`]), every float conversion exact for `double` and both +//! `long double` formats, whose `va_arg` each ABI's own rule reads +//! (`va_long_double`); and `stdout`/`stderr` plus consumer-built streams +//! ([`stdio`]); +//! - the `` parsers (`strtol` … `strtod`), `qsort`, `strerror`, +//! and a per-thread `errno`; +//! - with `long-double`, the byte-level halves of `strtold` and `ldexpl` +//! ([`long_double`]) a consumer's C layer wraps, since only C can spell +//! the type. +//! +//! The set began as tree-sitter's (its `wasm-stdlib/imports.txt`) and grew +//! with TCC, the second C consumer (`res/TCC.md` §2.2), and with the C +//! programs libkore.so runs, the third: their `printf` is this crate's. +//! +//! Linking this crate is what supplies the symbols: a consumer names it +//! (`use kore_c_shim as _;`) so the linker keeps it. On the host the crate's +//! symbols are absent — the host's libc serves host tests, and a second +//! `malloc` there would replace the process allocator — while the pure +//! engines behind them ([`format`], [`number`], [`sort`], [`long_double`]) +//! are compiled for the host suite, so it tests the code the target runs. +//! +//! The string routines and wide character classes are pure, so a shared +//! library may carry its own (`libkore_grammars.so` takes the classes alone). Everything that touches the process — the +//! standard streams, `errno` (and with it the parsers that set it), and with +//! the default `malloc` and `abort` features the heap's C names and `abort` — +//! sits behind the default `runtime` feature, and a shared library built +//! without it imports those from the executable it links into +//! (`libkore_grammars.so` does), so the heap stays the task's one heap. A C +//! runtime that is itself the process's C library (libkore.so) takes +//! `runtime` alone and exports its own allocator and `abort`. +#![no_std] +#![cfg_attr(all(target_os = "none", feature = "runtime"), feature(c_variadic))] + +#[cfg(test)] +extern crate std; + +#[cfg(any(test, all(target_os = "none", feature = "runtime")))] +pub mod format; +#[cfg(any(test, all(target_os = "none", feature = "long-double")))] +pub mod long_double; +#[cfg(any(test, all(target_os = "none", feature = "runtime")))] +pub mod number; +#[cfg(any(test, all(target_os = "none", feature = "runtime")))] +pub mod sort; + +#[cfg(all(target_os = "none", feature = "malloc"))] +mod alloc_symbols; +#[cfg(all(target_os = "none", feature = "runtime"))] +pub mod errno; +#[cfg(all(target_os = "none", feature = "runtime"))] +mod format_symbols; +#[cfg(all(target_os = "none", feature = "runtime"))] +pub mod stdio; +#[cfg(all(target_os = "none", feature = "runtime"))] +mod stdlib_symbols; +#[cfg(target_os = "none")] +mod string_symbols; +#[cfg(all(target_os = "none", feature = "runtime"))] +mod va_long_double; +#[cfg(all(target_os = "none", feature = "wide-classes"))] +mod wide_symbols; + +/// The C `abort`: the library hit a state it cannot leave. The task ends +/// with the aborted status, which its controller reports. +#[cfg(all(target_os = "none", feature = "abort"))] +#[unsafe(no_mangle)] +extern "C" fn abort() -> ! { + kore_rt::thread::exit(kore_rt::ExitCode::ABORTED) +} diff --git a/native/crates/c-shim/src/long_double.rs b/native/crates/c-shim/src/long_double.rs new file mode 100644 index 0000000..70601b1 --- /dev/null +++ b/native/crates/c-shim/src/long_double.rs @@ -0,0 +1,341 @@ +//! `long double`'s two conversions, `strtold` and `ldexpl`, as bytes. +//! +//! The type is the C compiler's, never Rust's: x87 double-extended (a 64-bit +//! significand with an explicit integer bit) on x86_64, IEEE binary128 on +//! aarch64 and riscv64. Rust has no type for the first and no ABI promise +//! that its `f128` crosses `extern "C"` as C's `long double` for the second, +//! so the consumer's C layer owns the type: it passes its value's bytes and +//! `LDBL_MANT_DIG`, and this side computes on them. Nothing here names an +//! architecture — the format is the C side's word, checked. +//! +//! The arithmetic is LLVM's APFloat ([`rustc_apfloat`]), the code clang +//! folds `long double` literals with: decimal conversion is correctly +//! rounded for any length of input, and scaling rounds once, subnormals and +//! overflow included. The subject grammar is [`crate::number`]'s, so +//! `strtold` reads exactly the sequence `strtod` does. + +use rustc_apfloat::ieee::{Quad, X87DoubleExtended}; +use rustc_apfloat::{Float, Round, Status}; + +use crate::number::{FloatSubject, float_subject}; + +/// `sizeof(long double)` on every kore C ABI: x87's ten bytes padded to +/// sixteen, or binary128's sixteen. The C side asserts its own type agrees. +pub const BYTES: usize = 16; + +/// A `long double` format, by the `LDBL_MANT_DIG` that names it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Format { + /// x87 double-extended (`LDBL_MANT_DIG == 64`). + X87, + /// IEEE binary128 (`LDBL_MANT_DIG == 113`). + Quad, +} + +impl Format { + /// The format `LDBL_MANT_DIG` names, or `None` for one kore has no ABI + /// with. + pub const fn from_mant_dig(mant_dig: i32) -> Option { + match mant_dig { + 64 => Some(Self::X87), + 113 => Some(Self::Quad), + _ => None, + } + } +} + +/// A `long double` value as its little-endian bytes: the significant bits +/// low (x87's ten, binary128's sixteen), any padding zero. +pub const fn to_bytes(bits: u128) -> [u8; BYTES] { + bits.to_le_bytes() +} + +/// The bits of a `long double`'s bytes. x87's six padding bytes are +/// whatever the C side's storage held, so they are cleared here. +pub const fn from_bytes(format: Format, bytes: [u8; BYTES]) -> u128 { + let bits = u128::from_le_bytes(bytes); + match format { + Format::X87 => bits & ((1u128 << 80) - 1), + Format::Quad => bits, + } +} + +/// What a `strtold` parse found. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Parsed { + /// The value, as [`to_bytes`] lays it out. + pub bits: u128, + /// Overflow to infinity, or a nonzero number rounded to zero: `ERANGE`. + pub range: bool, + /// Bytes consumed; zero when there was no number. + pub end: usize, +} + +/// Parse a `long double` from the start of `text`. +pub fn parse(format: Format, text: &[u8]) -> Parsed { + match format { + Format::X87 => parse_as::(text), + Format::Quad => parse_as::(text), + } +} + +fn parse_as(text: &[u8]) -> Parsed { + let (value, range, end) = match float_subject(text) { + FloatSubject::None => (F::ZERO, false, 0), + FloatSubject::Special { nan, negative, end } => { + let value = if nan { F::NAN } else { F::INFINITY }; + (if negative { -value } else { value }, false, end) + } + FloatSubject::Decimal { + negative, + digits, + nonzero, + end, + } => { + // The subject is ASCII digits, a `.` and an exponent — always + // UTF-8, and always inside APFloat's decimal grammar (a subject + // has at least one digit, and an exponent only with digits). + let converted = core::str::from_utf8(digits) + .ok() + .and_then(|digits| F::from_str_r(digits, Round::NearestTiesToEven).ok()); + let Some(converted) = converted else { + // Unreachable by the grammar; refuse the conversion rather + // than invent a value. + return Parsed { + bits: F::ZERO.to_bits(), + range: false, + end: 0, + }; + }; + let value = if negative { + -converted.value + } else { + converted.value + }; + let range = + converted.status.intersects(Status::OVERFLOW) || (value.is_zero() && nonzero); + (value, range, end) + } + }; + Parsed { + bits: value.to_bits(), + range, + end, + } +} + +/// `ldexpl`: `bits × 2^exponent`, rounded once to nearest-even. +pub fn scale(format: Format, bits: u128, exponent: i32) -> u128 { + match format { + Format::X87 => scale_as::(bits, exponent), + Format::Quad => scale_as::(bits, exponent), + } +} + +fn scale_as(bits: u128, exponent: i32) -> u128 { + F::from_bits(bits) + .scalbn_r(exponent, Round::NearestTiesToEven) + .to_bits() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + use std::vec::Vec; + + // The host's own C library is the reference: on an x86_64 host its + // `long double` is x87 and glibc's `strtold` rounds correctly, and + // `strtof128` is binary128. + unsafe extern "C" { + fn strtold(text: *const core::ffi::c_char, end: *mut *mut core::ffi::c_char) -> X87Bytes; + fn ldexpl(value: X87Bytes, exponent: i32) -> X87Bytes; + } + + /// `long double` as the host ABI moves it: Rust has no x87 type, so + /// these tests read the host's answer through its bytes. + #[repr(C, align(16))] + #[derive(Clone, Copy)] + struct X87Bytes([u8; 16]); + + fn host(text: &str) -> (u128, usize) { + let c = CString::new(text).unwrap(); + let mut end = core::ptr::null_mut(); + // SAFETY: a NUL-terminated string and a valid end pointer. + let value = unsafe { host_strtold(c.as_ptr(), &mut end) }; + let consumed = end as usize - c.as_ptr() as usize; + (from_bytes(Format::X87, value), consumed) + } + + /// glibc's `strtold` returns in `st(0)`; this trampoline stores it. + /// The destination rides `r12`, callee-saved, so the call keeps it. + unsafe fn host_strtold( + text: *const core::ffi::c_char, + end: *mut *mut core::ffi::c_char, + ) -> [u8; 16] { + let mut out = [0u8; 16]; + // SAFETY: the caller's pointers; `fstp` stores the ten bytes and + // leaves the x87 stack empty, as the ABI requires. + unsafe { + core::arch::asm!( + "call {f}", + "fstp tbyte ptr [r12]", + f = sym strtold, + in("r12") out.as_mut_ptr(), + in("rdi") text, + in("rsi") end, + clobber_abi("C"), + ); + } + out + } + + fn ours(text: &str) -> (u128, usize) { + let parsed = parse(Format::X87, text.as_bytes()); + (parsed.bits, parsed.end) + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn x87_parses_as_the_host_does() { + let mut corpus: Vec = [ + "0", + "1", + "0.1", + "1e4932", + "1e4933", + "1e-4951", + "3.6e-4951", + "3.7e-4951", + "1.18973149535723176502e4932", + "1.18973149535723176508e4932", + "3.36210314311209350626e-4932", + "6.95e-4950", + "0.30000000000000000001", + "123456789012345678901234567890", + " -2.5e3x", + ".5", + "5.", + "1e+", + "inf", + "-INFINITY", + "nan", + "18446744073709551615", + "18446744073709551616.5", + "9.999999999999999999999e999", + "2.2250738585072011e-308", + "1.00000000000000000005421010862427522170037264004349708557128906250", + "1.0000000000000000000542101086242752217003726400434970855712890625", + "1.00000000000000000016263032587282567", + "4.9406564584124654e-324", + ] + .iter() + .map(|s| std::string::String::from(*s)) + .collect(); + // A deterministic spread of digit strings and exponents. + let mut seed = 0x9e37_79b9_7f4a_7c15u64; + for _ in 0..4000 { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + let digits = 1 + (seed % 40) as usize; + let mut s = std::string::String::new(); + let mut x = seed; + for i in 0..digits { + if i == digits / 2 && seed & 1 == 0 { + s.push('.'); + } + s.push((b'0' + (x % 10) as u8) as char); + x = x / 10 ^ x.rotate_left(29); + } + let exp = ((seed >> 20) % 9900) as i64 - 4950; + s.push_str(&std::format!("e{exp}")); + corpus.push(s); + } + for text in &corpus { + assert_eq!(ours(text), host(text), "{text}"); + } + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn x87_scales_as_the_host_does() { + let values = [ + "1", + "0.1", + "3.0e-4950", + "1.18973149535723176502e4932", + "-7.25", + "inf", + "0", + ]; + for value in values { + let (bits, _) = host(value); + for exponent in [ + -20000, -16445, -16444, -16383, -100, -1, 0, 1, 100, 16383, 16384, 20000, + ] { + let x = X87Bytes(to_bytes(bits)); + // SAFETY: glibc's `ldexpl` on an x87 value passed in memory. + let expected = unsafe { host_ldexpl(x, exponent) }; + assert_eq!( + scale(Format::X87, bits, exponent), + from_bytes(Format::X87, expected), + "{value} {exponent}" + ); + } + } + } + + /// `ldexpl` takes its `long double` on the stack and returns in `st(0)`. + unsafe fn host_ldexpl(value: X87Bytes, exponent: i32) -> [u8; 16] { + let mut out = [0u8; 16]; + // SAFETY: SysV passes an x87 argument in memory, in order, aligned + // 16 (an asm block without `nostack` enters call-aligned); the + // destination rides callee-saved `r12` across the call. + unsafe { + core::arch::asm!( + "sub rsp, 16", + "movdqu xmm0, [{value}]", + "movdqu [rsp], xmm0", + "call {f}", + "add rsp, 16", + "fstp tbyte ptr [r12]", + f = sym ldexpl, + value = in(reg) value.0.as_ptr(), + in("r12") out.as_mut_ptr(), + in("edi") exponent, + clobber_abi("C"), + ); + } + out + } + + #[test] + fn quad_parses_round_numbers_exactly() { + // binary128 of 1.0: biased exponent 0x3fff, zero fraction. + assert_eq!(parse(Format::Quad, b"1").bits, 0x3fff_u128 << 112); + assert_eq!(parse(Format::Quad, b"-2").bits, (0xc000_u128) << 112); + // 0.1 is the nearest binary128, rounded up in its last bit. + assert_eq!( + parse(Format::Quad, b"0.1").bits, + 0x3ffb_9999_9999_9999_9999_9999_9999_999a + ); + let huge = parse(Format::Quad, b"1e5000"); + assert!(huge.range && huge.bits == 0x7fff_u128 << 112); + let tiny = parse(Format::Quad, b"1e-5000"); + assert!(tiny.range && tiny.bits == 0); + assert_eq!( + scale(Format::Quad, parse(Format::Quad, b"1").bits, 3), + parse(Format::Quad, b"8").bits + ); + } + + #[test] + fn formats_are_named_by_mant_dig() { + assert_eq!(Format::from_mant_dig(64), Some(Format::X87)); + assert_eq!(Format::from_mant_dig(113), Some(Format::Quad)); + assert_eq!(Format::from_mant_dig(53), None); + // x87's padding never leaks into the value. + assert_eq!(from_bytes(Format::X87, [0xff; 16]), (1u128 << 80) - 1); + } +} diff --git a/native/crates/c-shim/src/number.rs b/native/crates/c-shim/src/number.rs new file mode 100644 index 0000000..85e3f8f --- /dev/null +++ b/native/crates/c-shim/src/number.rs @@ -0,0 +1,474 @@ +//! The parsing behind `strtol`, `strtoul`, `strtod` and their siblings: ISO C +//! 7.22.1's grammar over the bytes before a string's NUL, pure so the host +//! suite drives the code the target runs. +//! +//! Integers follow the standard exactly: leading C-locale whitespace, an +//! optional sign, base `0` choosing 16 for `0x`, 8 for a leading `0` and 10 +//! otherwise, overflow reported rather than wrapped, and an end offset that +//! is zero when no digit was found. +//! +//! Decimal floats are rounded correctly — the digits go to core's parser, +//! which rounds any length of input exactly — and `inf`, `infinity` and +//! `nan[(chars)]` are recognized in any case. **Hexadecimal floats are not +//! parsed**: `0x1p3` reads as `0` and ends at the `x`, which is C89's +//! reading. No consumer needs more; TCC, the one that compiles C, parses +//! hexadecimal float literals itself. +//! +//! The subject grammar is public ([`float_subject`]) so a parser for a type +//! core has no parser for — TCC's `long double` — reads exactly the sequence +//! `strtod` reads. + +/// ISO C locale whitespace (`isspace`): space, `\t \n \v \f \r`. +pub const fn c_space(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r') +} + +/// What an integer parse found. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Integer { + /// The value's magnitude, saturated at `u64::MAX` when `overflow`. + pub magnitude: u64, + /// A `-` preceded the digits. + pub negative: bool, + /// The magnitude did not fit 64 bits. + pub overflow: bool, + /// Bytes consumed through the last digit; zero when there was no number. + pub end: usize, +} + +/// Parse an integer in `base` (0 or 2..=36) from the start of `text`. +/// `None` for any other base — C's `EINVAL`, with no conversion. +pub fn integer(text: &[u8], base: u32) -> Option { + if base == 1 || base > 36 { + return None; + } + let mut at = 0; + while text.get(at).copied().is_some_and(c_space) { + at += 1; + } + let negative = text.get(at) == Some(&b'-'); + if matches!(text.get(at), Some(b'+' | b'-')) { + at += 1; + } + let digit = |byte: u8, base: u32| char::from(byte).to_digit(base); + let hex_prefix = matches!(text.get(at..at + 2), Some([b'0', b'x' | b'X'])) + && text + .get(at + 2) + .is_some_and(|&byte| digit(byte, 16).is_some()); + let base = match base { + 0 if hex_prefix => 16, + 0 if text.get(at) == Some(&b'0') => 8, + 0 => 10, + base => base, + }; + // `0x` is skipped only when a hexadecimal digit follows it; otherwise + // the `0` is the number and parsing ends at the `x`. + if base == 16 && hex_prefix { + at += 2; + } + let first = at; + let mut magnitude = 0u64; + let mut overflow = false; + while let Some(value) = text.get(at).and_then(|&byte| digit(byte, base)) { + match magnitude + .checked_mul(u64::from(base)) + .and_then(|scaled| scaled.checked_add(u64::from(value))) + { + Some(next) => magnitude = next, + None => { + overflow = true; + magnitude = u64::MAX; + } + } + at += 1; + } + let end = if at == first { 0 } else { at }; + Some(Integer { + magnitude, + negative, + overflow, + end, + }) +} + +impl Integer { + /// The value as a signed integer no wider than `bits`, and whether it + /// was clamped (C's `ERANGE`). + pub fn signed(self, bits: u32) -> (i64, bool) { + let max = (1u64 << (bits - 1)) - 1; + if self.negative { + if self.overflow || self.magnitude > max + 1 { + (-(max as i64) - 1, true) + } else { + (0i64.wrapping_sub(self.magnitude as i64), false) + } + } else if self.overflow || self.magnitude > max { + (max as i64, true) + } else { + (self.magnitude as i64, false) + } + } + + /// The value as an unsigned integer no wider than `bits`, and whether it + /// was clamped. A `-` negates in the unsigned type, as C says. + pub fn unsigned(self, bits: u32) -> (u64, bool) { + let max = if bits == 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + }; + if self.overflow || self.magnitude > max { + (max, true) + } else if self.negative { + (self.magnitude.wrapping_neg() & max, false) + } else { + (self.magnitude, false) + } + } +} + +/// What a float parse found. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Float { + pub value: T, + /// Overflow to infinity, or a nonzero number rounded to zero: `ERANGE`. + pub range: bool, + /// Bytes consumed; zero when there was no number. + pub end: usize, +} + +/// The floating subject sequence at the start of `text` (ISO C 7.22.1.3), +/// leading whitespace skipped: what `strtod` and its siblings convert. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FloatSubject<'t> { + /// No subject: nothing is converted, and the end pointer is the input. + None, + /// `inf`, `infinity` or `nan[(n-char-sequence)]`, in any case. + Special { + nan: bool, + negative: bool, + end: usize, + }, + /// A decimal number. `digits` is the sign-free subject — digits, an + /// optional `.`, an optional exponent — in the grammar a + /// correctly-rounding parser takes; `nonzero` says a mantissa digit was + /// not `0`, so a zero result is an underflow rather than a zero. + Decimal { + negative: bool, + digits: &'t [u8], + nonzero: bool, + end: usize, + }, +} + +/// Scan the floating subject at the start of `text`. +pub fn float_subject(text: &[u8]) -> FloatSubject<'_> { + if let Some((nan, negative, end)) = special(text) { + return FloatSubject::Special { nan, negative, end }; + } + match decimal_subject(text) { + Some((negative, digits, nonzero, end)) => FloatSubject::Decimal { + negative, + digits, + nonzero, + end, + }, + None => FloatSubject::None, + } +} + +/// The decimal subject sequence at the start of `text`: its sign, the slice +/// core's parser takes, whether any mantissa digit was nonzero, and the end. +fn decimal_subject(text: &[u8]) -> Option<(bool, &[u8], bool, usize)> { + let mut at = 0; + while text.get(at).copied().is_some_and(c_space) { + at += 1; + } + let negative = text.get(at) == Some(&b'-'); + if matches!(text.get(at), Some(b'+' | b'-')) { + at += 1; + } + let start = at; + let mut digits = 0usize; + let mut nonzero = false; + while let Some(&byte) = text.get(at).filter(|byte| byte.is_ascii_digit()) { + nonzero |= byte != b'0'; + digits += 1; + at += 1; + } + if text.get(at) == Some(&b'.') { + at += 1; + while let Some(&byte) = text.get(at).filter(|byte| byte.is_ascii_digit()) { + nonzero |= byte != b'0'; + digits += 1; + at += 1; + } + } + if digits == 0 { + return None; + } + // An exponent counts only when at least one digit follows it. + if matches!(text.get(at), Some(b'e' | b'E')) { + let mut exponent = at + 1; + if matches!(text.get(exponent), Some(b'+' | b'-')) { + exponent += 1; + } + if text.get(exponent).is_some_and(u8::is_ascii_digit) { + at = exponent; + while text.get(at).is_some_and(u8::is_ascii_digit) { + at += 1; + } + } + } + Some((negative, &text[start..at], nonzero, at)) +} + +/// `inf`, `infinity` or `nan[(n-char-sequence)]`, case-insensitively, at the +/// start of `text`: `(is_nan, negative, end)`. +fn special(text: &[u8]) -> Option<(bool, bool, usize)> { + let mut at = 0; + while text.get(at).copied().is_some_and(c_space) { + at += 1; + } + let negative = text.get(at) == Some(&b'-'); + if matches!(text.get(at), Some(b'+' | b'-')) { + at += 1; + } + let word = |at: usize, word: &[u8]| { + text.get(at..at + word.len()) + .is_some_and(|got| got.eq_ignore_ascii_case(word)) + }; + if word(at, b"infinity") { + return Some((false, negative, at + 8)); + } + if word(at, b"inf") { + return Some((false, negative, at + 3)); + } + if word(at, b"nan") { + let mut end = at + 3; + // `nan(` followed by alphanumerics and `_`, then `)`. + if text.get(end) == Some(&b'(') { + let mut close = end + 1; + while text + .get(close) + .is_some_and(|&byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + close += 1; + } + if text.get(close) == Some(&b')') { + end = close + 1; + } + } + return Some((true, negative, end)); + } + None +} + +macro_rules! float_parser { + ($name:ident, $ty:ty) => { + /// Parse a float from the start of `text`. + pub fn $name(text: &[u8]) -> Float<$ty> { + let (negative, subject, nonzero, end) = match float_subject(text) { + FloatSubject::Special { nan, negative, end } => { + let value = if nan { <$ty>::NAN } else { <$ty>::INFINITY }; + return Float { + value: if negative { -value } else { value }, + range: false, + end, + }; + } + FloatSubject::None => { + return Float { + value: 0.0, + range: false, + end: 0, + }; + } + FloatSubject::Decimal { + negative, + digits, + nonzero, + end, + } => (negative, digits, nonzero, end), + }; + // The subject is ASCII digits, `.` and an exponent: always UTF-8, + // and always in core's grammar (a lone `.` was refused above). + let value: $ty = core::str::from_utf8(subject) + .ok() + .and_then(|digits| digits.parse().ok()) + .unwrap_or(0.0); + let range = value.is_infinite() || (value == 0.0 && nonzero); + Float { + value: if negative { -value } else { value }, + range, + end, + } + } + }; +} + +float_parser!(double, f64); +float_parser!(single, f32); + +#[cfg(test)] +mod tests { + use super::*; + + fn int(text: &str, base: u32) -> Integer { + integer(text.as_bytes(), base).expect("a valid base") + } + + #[test] + fn integers_follow_the_c_grammar() { + assert_eq!( + int(" 42abc", 10), + Integer { + magnitude: 42, + negative: false, + overflow: false, + end: 4 + } + ); + assert_eq!(int("-17", 10).signed(64), (-17, false)); + assert_eq!(int("+9", 0).signed(64), (9, false)); + assert_eq!(int("0x1F", 0).magnitude, 31); + assert_eq!(int("0x1F", 16).magnitude, 31); + assert_eq!(int("017", 0).magnitude, 15); + assert_eq!(int("z", 36).magnitude, 35); + // `0x` without a hex digit after it: the `0` is the number. + assert_eq!( + int("0xg", 0), + Integer { + magnitude: 0, + negative: false, + overflow: false, + end: 1 + } + ); + assert_eq!(int("0x", 16).end, 1); + // No digits: no conversion, end zero. + assert_eq!(int(" -", 10).end, 0); + assert_eq!(int("", 10).end, 0); + assert!(integer(b"1", 1).is_none()); + assert!(integer(b"1", 37).is_none()); + } + + #[test] + fn integer_overflow_clamps_and_reports() { + assert_eq!(int("9223372036854775807", 10).signed(64), (i64::MAX, false)); + assert_eq!(int("9223372036854775808", 10).signed(64), (i64::MAX, true)); + assert_eq!( + int("-9223372036854775808", 10).signed(64), + (i64::MIN, false) + ); + assert_eq!(int("-9223372036854775809", 10).signed(64), (i64::MIN, true)); + assert_eq!( + int("99999999999999999999999", 10).signed(64), + (i64::MAX, true) + ); + assert_eq!(int("2147483648", 10).signed(32), (i32::MAX as i64, true)); + assert_eq!( + int("18446744073709551615", 10).unsigned(64), + (u64::MAX, false) + ); + assert_eq!( + int("18446744073709551616", 10).unsigned(64), + (u64::MAX, true) + ); + // A minus sign negates in the unsigned type. + assert_eq!(int("-1", 10).unsigned(64), (u64::MAX, false)); + assert_eq!(int("-1", 10).unsigned(32), (u32::MAX as u64, false)); + // Overflowed digits are still consumed. + assert_eq!(int("99999999999999999999999x", 10).end, 23); + } + + #[test] + fn decimal_floats_round_correctly() { + assert_eq!( + double(b"1.5"), + Float { + value: 1.5, + range: false, + end: 3 + } + ); + assert_eq!(double(b" -2.5e3x").value, -2500.0); + assert_eq!(double(b" -2.5e3x").end, 8); + assert_eq!(double(b".5").value, 0.5); + assert_eq!(double(b"5.").value, 5.0); + assert_eq!(double(b"5.").end, 2); + // An exponent with no digits is not part of the number. + assert_eq!(double(b"1e+").end, 1); + assert_eq!(double(b"0.1").value, 0.1); + assert_eq!(single(b"0.1").value, 0.1f32); + assert_eq!(double(b"").end, 0); + assert_eq!(double(b".").end, 0); + assert_eq!(double(b"-").end, 0); + } + + #[test] + fn float_range_and_specials() { + let huge = double(b"1e400"); + assert!(huge.value.is_infinite() && huge.range); + let tiny = double(b"1e-400"); + assert!(tiny.value == 0.0 && tiny.range); + let zero = double(b"0.000"); + assert!(zero.value == 0.0 && !zero.range); + assert!(single(b"1e39").range); + assert_eq!(double(b"INF").value, f64::INFINITY); + assert_eq!(double(b"-Infinity").value, f64::NEG_INFINITY); + assert_eq!(double(b"-Infinity").end, 9); + assert!(double(b"nan(0x1)").value.is_nan()); + assert_eq!(double(b"nan(abc)").end, 8); + assert_eq!(double(b"nan(").end, 3); + // Hexadecimal floats read as C89 does: the `0`, ending at the `x`. + assert_eq!( + double(b"0x1p3"), + Float { + value: 0.0, + range: false, + end: 1 + } + ); + } + + #[test] + fn the_subject_is_the_parsers_own() { + assert_eq!( + float_subject(b" -1.25e3x"), + FloatSubject::Decimal { + negative: true, + digits: b"1.25e3", + nonzero: true, + end: 8 + } + ); + assert_eq!( + float_subject(b"0.000"), + FloatSubject::Decimal { + negative: false, + digits: b"0.000", + nonzero: false, + end: 5 + } + ); + assert_eq!( + float_subject(b"-INF"), + FloatSubject::Special { + nan: false, + negative: true, + end: 4 + } + ); + assert_eq!( + float_subject(b"nan(7)"), + FloatSubject::Special { + nan: true, + negative: false, + end: 6 + } + ); + assert_eq!(float_subject(b" ."), FloatSubject::None); + assert_eq!(float_subject(b"+"), FloatSubject::None); + } +} diff --git a/native/crates/c-shim/src/sort.rs b/native/crates/c-shim/src/sort.rs new file mode 100644 index 0000000..7b16c60 --- /dev/null +++ b/native/crates/c-shim/src/sort.rs @@ -0,0 +1,115 @@ +//! `qsort`'s algorithm: heapsort over raw elements of a C-supplied size. +//! +//! Heapsort because it is O(n log n) in the worst case, iterative (no +//! recursion to exhaust a 64 KiB worker stack), and in place. C's `qsort` +//! promises no stability, and this keeps none. + +/// Sort `count` elements of `size` bytes at `base` by `before(a, b)`, which +/// answers whether `a` sorts strictly after `b` (a C comparator's `> 0`). +/// +/// # Safety +/// `base` points to `count * size` writable bytes, and `after` reads only +/// the `size` bytes at each pointer it is given. +pub unsafe fn heapsort( + base: *mut u8, + count: usize, + size: usize, + after: &mut impl FnMut(*const u8, *const u8) -> bool, +) { + if count < 2 || size == 0 { + return; + } + let at = |index: usize| -> *mut u8 { + // SAFETY: every index the algorithm forms is below `count`. + unsafe { base.add(index * size) } + }; + // SAFETY (the swaps): distinct indices name disjoint elements. + let sift = + |mut root: usize, end: usize, after: &mut dyn FnMut(*const u8, *const u8) -> bool| { + loop { + let left = 2 * root + 1; + if left >= end { + return; + } + let right = left + 1; + let child = if right < end && after(at(right), at(left)) { + right + } else { + left + }; + if !after(at(child), at(root)) { + return; + } + unsafe { core::ptr::swap_nonoverlapping(at(root), at(child), size) }; + root = child; + } + }; + let mut start = count / 2; + while start > 0 { + start -= 1; + sift(start, count, after); + } + let mut end = count; + while end > 1 { + end -= 1; + unsafe { core::ptr::swap_nonoverlapping(at(0), at(end), size) }; + sift(0, end, after); + } +} + +#[cfg(test)] +mod tests { + use super::*; + extern crate std; + use std::vec::Vec; + + fn sort_u32(values: &mut [u32]) { + let len = values.len(); + // SAFETY: the slice's bytes, elements of four bytes. + unsafe { + heapsort(values.as_mut_ptr().cast(), len, 4, &mut |a, b| { + a.cast::().read_unaligned() > b.cast::().read_unaligned() + }) + }; + } + + #[test] + fn sorts_every_shape() { + for len in 0..40 { + let mut ascending: Vec = (0..len).collect(); + let mut descending: Vec = (0..len).rev().collect(); + let mut repeated: Vec = (0..len).map(|i| i % 3).collect(); + let mut scrambled: Vec = + (0..len).map(|i| i.wrapping_mul(2654435761) % 97).collect(); + for values in [ + &mut ascending, + &mut descending, + &mut repeated, + &mut scrambled, + ] { + let mut expected = values.clone(); + expected.sort_unstable(); + sort_u32(values); + assert_eq!(*values, expected); + } + } + } + + #[test] + fn sorts_wide_elements_by_their_key() { + // 24-byte records keyed on their first word, as C sorts structs. + let mut records: Vec<[u64; 3]> = (0..17u64).map(|i| [(i * 7) % 17, i, !i]).collect(); + let len = records.len(); + // SAFETY: the vector's bytes, elements of 24 bytes. + unsafe { + heapsort(records.as_mut_ptr().cast(), len, 24, &mut |a, b| { + a.cast::().read_unaligned() > b.cast::().read_unaligned() + }) + }; + for (index, record) in records.iter().enumerate() { + assert_eq!(record[0], index as u64); + // Each record moved whole. + assert_eq!(record[2], !record[1]); + } + } +} diff --git a/native/crates/c-shim/src/stdio.rs b/native/crates/c-shim/src/stdio.rs new file mode 100644 index 0000000..f004ee4 --- /dev/null +++ b/native/crates/c-shim/src/stdio.rs @@ -0,0 +1,210 @@ +//! `FILE` streams: the standard pair, and the streams a consumer builds over +//! its own file layer. +//! +//! A [`File`] is a write operation and its state. C only ever holds a +//! `FILE *` to one (musl's `FILE` is incomplete in the headers every kore C +//! build uses), so the layout is this crate's own. +//! +//! - **`stdout` and `stderr`** go to the [`StandardSink`] the process +//! installs once ([`install_standard_sink`]) — its terminal, or a result +//! buffer it reports when the task ends. Until one is installed a write +//! reports `EOF` and goes nowhere, which is what every consumer that never +//! installs one has always seen. +//! - **Any other stream** is built by the consumer that owns a file layer +//! (`fdopen` over its own descriptors) with [`File::new`] and its own +//! [`FileOps`]. This crate never opens a file: kore has no ambient +//! filesystem for it to open one in. + +use core::ffi::c_void; +use core::ptr; +use core::sync::atomic::{AtomicPtr, Ordering}; + +/// A C `FILE`. +#[repr(C)] +pub struct File { + ops: &'static FileOps, + state: *mut c_void, +} + +// SAFETY: the standard streams' state is an inert tag, never dereferenced; +// a consumer that places its own stream in a `static` owns that stream's +// synchronization, as C's own `FILE` locking would. +unsafe impl Sync for File {} + +/// What a stream does with its bytes. +pub struct FileOps { + /// Accept `bytes` whole. `false` is a write error: C's `EOF`. + pub write: unsafe fn(state: *mut c_void, bytes: &[u8]) -> bool, + /// Push anything buffered onward. `false` is an error. + pub flush: unsafe fn(state: *mut c_void) -> bool, + /// `fclose`'s last act: flush and release the stream, the `File` itself + /// included when its owner allocated it. `false` is an error; the stream + /// is closed either way, as C's `fclose` leaves it. + pub close: unsafe fn(file: *mut File) -> bool, +} + +impl File { + /// A stream over `ops`, carrying the consumer's `state`. + pub const fn new(ops: &'static FileOps, state: *mut c_void) -> Self { + Self { ops, state } + } + + /// The consumer's state, for its [`FileOps`]. + pub const fn state(&self) -> *mut c_void { + self.state + } +} + +/// Write `bytes` to the stream at `file`. +/// +/// # Safety +/// `file` is a live `FILE *` this crate or a consumer constructed. +pub(crate) unsafe fn write(file: *mut File, bytes: &[u8]) -> bool { + if bytes.is_empty() { + return true; + } + // SAFETY: the caller's contract; only the fields are read. + unsafe { ((*file).ops.write)((*file).state, bytes) } +} + +/// # Safety +/// As [`write`]. +pub(crate) unsafe fn flush(file: *mut File) -> bool { + // SAFETY: the caller's contract. + unsafe { ((*file).ops.flush)((*file).state) } +} + +/// # Safety +/// As [`write`]; `file` is not used again. +pub(crate) unsafe fn close(file: *mut File) -> bool { + // SAFETY: the caller's contract. + unsafe { ((*file).ops.close)(file) } +} + +/// The two standard output streams. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Standard { + /// `stdout`. + Output, + /// `stderr`. + Error, +} + +/// Where the standard streams' bytes go, and the lock one stream call holds +/// across every write it makes. +/// +/// POSIX has each stdio call lock its stream (`flockfile`), so one call's +/// output reaches the stream together even when the call writes it in +/// pieces, as `vfprintf`'s batches and `puts`' line and newline are. The +/// process decides what the lock is: a sink only one thread writes takes +/// none. +pub struct StandardSink { + /// Write `bytes` to the stream whole. `false` reports a write error. + pub write: fn(Standard, &[u8]) -> bool, + /// Take the standard streams' lock for one call. A sink reached with + /// the lock already held by the calling thread takes it again: the lock + /// is the sink's to make reentrant. + pub lock: fn(), + /// Release one [`lock`](Self::lock). + pub unlock: fn(), +} + +/// The installed sink; null until a process installs one. +static SINK: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +/// Send `stdout` and `stderr` to `sink` from now on. A later install +/// replaces an earlier one; a call in flight finishes on the sink it read. +pub fn install_standard_sink(sink: &'static StandardSink) { + SINK.store(ptr::from_ref(sink).cast_mut(), Ordering::Release); +} + +fn installed() -> Option<&'static StandardSink> { + // SAFETY: only `install_standard_sink` stores here, and it stores a + // `&'static StandardSink`. + unsafe { SINK.load(Ordering::Acquire).as_ref() } +} + +/// Run one stream call on `file` holding its lock: around every write it +/// makes, the standard streams' sink's lock; nothing for a consumer's +/// stream, whose owner synchronizes it. +/// +/// # Safety +/// `file` is a live `FILE *` this crate or a consumer constructed. +pub(crate) unsafe fn locked(file: *mut File, call: impl FnOnce() -> R) -> R { + // SAFETY: the caller's contract. + let standard = ptr::eq(unsafe { (*file).ops }, &STANDARD_OPS); + let sink = if standard { installed() } else { None }; + if let Some(sink) = sink { + (sink.lock)(); + } + let result = call(); + if let Some(sink) = sink { + (sink.unlock)(); + } + result +} + +/// The standard streams' state: an inert tag naming which one. +const OUTPUT_TAG: *mut c_void = ptr::without_provenance_mut(1); +const ERROR_TAG: *mut c_void = ptr::without_provenance_mut(2); + +unsafe fn standard_write(state: *mut c_void, bytes: &[u8]) -> bool { + let Some(sink) = installed() else { + return false; + }; + let stream = if state == OUTPUT_TAG { + Standard::Output + } else { + Standard::Error + }; + (sink.write)(stream, bytes) +} + +unsafe fn standard_flush(_: *mut c_void) -> bool { + // The sink receives every write whole; there is nothing buffered here. + true +} + +unsafe fn standard_close(_: *mut File) -> bool { + // C lets a program close `stdout`; the stream is static, so closing it + // only ends this program's use of it. + true +} + +static STANDARD_OPS: FileOps = FileOps { + write: standard_write, + flush: standard_flush, + close: standard_close, +}; + +static STDOUT: File = File::new(&STANDARD_OPS, OUTPUT_TAG); +static STDERR: File = File::new(&STANDARD_OPS, ERROR_TAG); + +/// A `FILE *` value: musl declares `extern FILE *const stdout`, so the +/// symbol holds a pointer to the stream, not the stream. +#[repr(transparent)] +pub struct Stream(*mut File); + +// SAFETY: a pointer to an immutable static; nothing writes through it. +unsafe impl Sync for Stream {} + +impl Stream { + pub(crate) fn file(&self) -> *mut File { + self.0 + } +} + +/// The `FILE *` of a standard stream — what a consumer's `fdopen` answers +/// for descriptor 1 or 2. +pub fn standard_stream(which: Standard) -> *mut File { + match which { + Standard::Output => stdout.file(), + Standard::Error => stderr.file(), + } +} + +#[unsafe(no_mangle)] +pub static stdout: Stream = Stream(&STDOUT as *const File as *mut File); + +#[unsafe(no_mangle)] +pub static stderr: Stream = Stream(&STDERR as *const File as *mut File); diff --git a/native/crates/c-shim/src/stdlib_symbols.rs b/native/crates/c-shim/src/stdlib_symbols.rs new file mode 100644 index 0000000..fa0a508 --- /dev/null +++ b/native/crates/c-shim/src/stdlib_symbols.rs @@ -0,0 +1,240 @@ +//! The `` and `` routines that are pure computation: +//! number parsing, `qsort`, `strerror`. +//! +//! The parsers set `errno` as ISO C requires, so they sit with it behind the +//! `runtime` feature; `qsort` and `strerror` are pure but travel with them. + +use core::ffi::{ + c_char, c_double, c_float, c_int, c_long, c_longlong, c_ulong, c_ulonglong, c_void, +}; + +use crate::errno::{self, EINVAL, ERANGE}; +use crate::number; + +/// The bytes before `text`'s NUL. +/// +/// # Safety +/// `text` is NUL-terminated. +unsafe fn bytes<'a>(text: *const c_char) -> &'a [u8] { + // SAFETY: the caller's contract. + unsafe { core::ffi::CStr::from_ptr(text) }.to_bytes() +} + +/// Store `text + end` through `end_out`, as the parsers' `endptr`. +/// +/// # Safety +/// `end_out` is null or writable; `end` lies within `text`'s string. +unsafe fn report_end(text: *const c_char, end: usize, end_out: *mut *mut c_char) { + if !end_out.is_null() { + // SAFETY: the caller's contract. + unsafe { *end_out = text.add(end).cast_mut() }; + } +} + +/// The shared front of the integer parsers: parse, report the end, and set +/// `EINVAL` for a base C does not define. +/// +/// # Safety +/// C's `strtol` contract. +unsafe fn parse( + text: *const c_char, + end_out: *mut *mut c_char, + base: c_int, +) -> Option { + // SAFETY: the caller's contract. + let subject = unsafe { bytes(text) }; + let parsed = u32::try_from(base) + .ok() + .and_then(|base| number::integer(subject, base)); + let end = parsed.map_or(0, |parsed| parsed.end); + // SAFETY: the end lies within the string. + unsafe { report_end(text, end, end_out) }; + if parsed.is_none() { + errno::set(EINVAL); + } + parsed +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtoll( + text: *const c_char, + end: *mut *mut c_char, + base: c_int, +) -> c_longlong { + // SAFETY: C's contract. + let Some(parsed) = (unsafe { parse(text, end, base) }) else { + return 0; + }; + let (value, clamped) = parsed.signed(c_longlong::BITS); + if clamped { + errno::set(ERANGE); + } + value +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtol(text: *const c_char, end: *mut *mut c_char, base: c_int) -> c_long { + // SAFETY: C's contract. + let Some(parsed) = (unsafe { parse(text, end, base) }) else { + return 0; + }; + let (value, clamped) = parsed.signed(c_long::BITS); + if clamped { + errno::set(ERANGE); + } + value as c_long +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtoull( + text: *const c_char, + end: *mut *mut c_char, + base: c_int, +) -> c_ulonglong { + // SAFETY: C's contract. + let Some(parsed) = (unsafe { parse(text, end, base) }) else { + return 0; + }; + let (value, clamped) = parsed.unsigned(c_ulonglong::BITS); + if clamped { + errno::set(ERANGE); + } + value +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtoul(text: *const c_char, end: *mut *mut c_char, base: c_int) -> c_ulong { + // SAFETY: C's contract. + let Some(parsed) = (unsafe { parse(text, end, base) }) else { + return 0; + }; + let (value, clamped) = parsed.unsigned(c_ulong::BITS); + if clamped { + errno::set(ERANGE); + } + value as c_ulong +} + +/// `atoi`/`atol`: `strtol` in base 10, as glibc defines them — out-of-range +/// input is undefined in C; here it is `strtol`'s clamp, truncated. +#[unsafe(no_mangle)] +unsafe extern "C" fn atoi(text: *const c_char) -> c_int { + // SAFETY: C's contract. + unsafe { strtol(text, core::ptr::null_mut(), 10) as c_int } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn atol(text: *const c_char) -> c_long { + // SAFETY: C's contract. + unsafe { strtol(text, core::ptr::null_mut(), 10) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtod(text: *const c_char, end: *mut *mut c_char) -> c_double { + // SAFETY: C's contract. + let parsed = number::double(unsafe { bytes(text) }); + unsafe { report_end(text, parsed.end, end) }; + if parsed.range { + errno::set(ERANGE); + } + parsed.value +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtof(text: *const c_char, end: *mut *mut c_char) -> c_float { + // SAFETY: C's contract. + let parsed = number::single(unsafe { bytes(text) }); + unsafe { report_end(text, parsed.end, end) }; + if parsed.range { + errno::set(ERANGE); + } + parsed.value +} + +/// A C comparator. +type Compare = unsafe extern "C" fn(*const c_void, *const c_void) -> c_int; + +#[unsafe(no_mangle)] +unsafe extern "C" fn qsort(base: *mut c_void, count: usize, size: usize, compare: Compare) { + // SAFETY: C's contract: `count * size` bytes at `base`, and a comparator + // that reads only the elements it is handed. + unsafe { + crate::sort::heapsort(base.cast(), count, size, &mut |a, b| { + compare(a.cast(), b.cast()) > 0 + }) + }; +} + +#[unsafe(no_mangle)] +extern "C" fn strerror(code: c_int) -> *mut c_char { + errno::describe(code).as_ptr().cast_mut() +} + +/// `strtold`'s byte-level half (`long_double`): parse `text` as the +/// `long double` `mant_dig` names, store its [`long_double::BYTES`] bytes at +/// `out`, and the end at `*end`. Answers the `errno` the C wrapper sets — +/// `ERANGE` as `strtod` would, `EINVAL` for a format or buffer this crate +/// has no ABI for (the value is then `+0` and nothing is consumed) — or `0`. +/// +/// # Safety +/// C's `strtold` contract for `text` and `end`; `out` is writable for +/// `out_len` bytes. +#[cfg(feature = "long-double")] +#[unsafe(no_mangle)] +unsafe extern "C" fn __kore_strtold_bytes( + text: *const c_char, + end: *mut *mut c_char, + mant_dig: c_int, + out: *mut u8, + out_len: usize, +) -> c_int { + use crate::long_double; + let Some(format) = + long_double::Format::from_mant_dig(mant_dig).filter(|_| out_len == long_double::BYTES) + else { + // SAFETY: C's contract. + unsafe { report_end(text, 0, end) }; + if !out.is_null() { + // SAFETY: `out_len` writable bytes. + unsafe { core::ptr::write_bytes(out, 0, out_len) }; + } + return EINVAL; + }; + // SAFETY: C's contract. + let parsed = long_double::parse(format, unsafe { bytes(text) }); + unsafe { + report_end(text, parsed.end, end); + out.cast::<[u8; long_double::BYTES]>() + .write_unaligned(long_double::to_bytes(parsed.bits)); + } + if parsed.range { ERANGE } else { 0 } +} + +/// `ldexpl`'s byte-level half: replace the `long double` at `value` with +/// itself times `2^exponent`, rounded once. `EINVAL` (value untouched) for a +/// format or buffer this crate has no ABI for, else `0`. +/// +/// # Safety +/// `value` is readable and writable for `len` bytes. +#[cfg(feature = "long-double")] +#[unsafe(no_mangle)] +unsafe extern "C" fn __kore_ldexpl_bytes( + value: *mut u8, + len: usize, + mant_dig: c_int, + exponent: c_int, +) -> c_int { + use crate::long_double; + let Some(format) = + long_double::Format::from_mant_dig(mant_dig).filter(|_| len == long_double::BYTES) + else { + return EINVAL; + }; + let slot = value.cast::<[u8; long_double::BYTES]>(); + // SAFETY: the caller's `len` bytes. + let bits = long_double::from_bytes(format, unsafe { slot.read_unaligned() }); + let scaled = long_double::scale(format, bits, exponent); + // SAFETY: as above. + unsafe { slot.write_unaligned(long_double::to_bytes(scaled)) }; + 0 +} diff --git a/native/crates/c-shim/src/string_symbols.rs b/native/crates/c-shim/src/string_symbols.rs new file mode 100644 index 0000000..92206b4 --- /dev/null +++ b/native/crates/c-shim/src/string_symbols.rs @@ -0,0 +1,158 @@ +//! The ISO string routines a C library imports beyond `compiler_builtins`' +//! `memcpy`/`memmove`/`memset`/`memcmp`/`strlen`. + +use core::ffi::{CStr, c_char, c_int, c_void}; + +/// The bytes of a C string, without its NUL. +/// +/// # Safety +/// `text` is NUL-terminated. +unsafe fn bytes<'a>(text: *const c_char) -> &'a [u8] { + // SAFETY: the caller's contract. + unsafe { CStr::from_ptr(text) }.to_bytes() +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strchr(text: *const c_char, value: c_int) -> *mut c_char { + let needle = value as u8; + // SAFETY: C promises a NUL-terminated string. + let body = unsafe { bytes(text) }; + // The terminator itself is part of the searchable string: `strchr(s, 0)` + // answers the NUL's address. + let found = if needle == 0 { + Some(body.len()) + } else { + body.iter().position(|&byte| byte == needle) + }; + match found { + // SAFETY: the offset lies within the string or on its NUL. + Some(offset) => unsafe { text.add(offset) }.cast_mut(), + None => core::ptr::null_mut(), + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strrchr(text: *const c_char, value: c_int) -> *mut c_char { + let needle = value as u8; + // SAFETY: C promises a NUL-terminated string. + let body = unsafe { bytes(text) }; + let found = if needle == 0 { + Some(body.len()) + } else { + body.iter().rposition(|&byte| byte == needle) + }; + match found { + // SAFETY: as `strchr`. + Some(offset) => unsafe { text.add(offset) }.cast_mut(), + None => core::ptr::null_mut(), + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strcpy(target: *mut c_char, source: *const c_char) -> *mut c_char { + // SAFETY: C promises `source` NUL-terminated and `target` large enough + // for it and its NUL, not overlapping. + unsafe { + let len = bytes(source).len(); + core::ptr::copy_nonoverlapping(source, target, len + 1); + } + target +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strstr(haystack: *const c_char, needle: *const c_char) -> *mut c_char { + // SAFETY: C promises both NUL-terminated. + let (hay, pin) = unsafe { (bytes(haystack), bytes(needle)) }; + let Some((&first, rest)) = pin.split_first() else { + return haystack.cast_mut(); + }; + if pin.len() > hay.len() { + return core::ptr::null_mut(); + } + // Scan for the first byte, then compare the rest in place. + let last_start = hay.len() - pin.len(); + let mut at = 0; + while let Some(skip) = hay[at..=last_start].iter().position(|&byte| byte == first) { + let start = at + skip; + if &hay[start + 1..start + pin.len()] == rest { + // SAFETY: `start` lies within the haystack. + return unsafe { haystack.add(start) }.cast_mut(); + } + at = start + 1; + if at > last_start { + break; + } + } + core::ptr::null_mut() +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn memchr(source: *const c_void, value: c_int, len: usize) -> *mut c_void { + let bytes = source.cast::(); + let needle = value as u8; + for index in 0..len { + // SAFETY: C promises `len` readable bytes at `source`. + if unsafe { *bytes.add(index) } == needle { + return unsafe { bytes.add(index) }.cast_mut().cast(); + } + } + core::ptr::null_mut() +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strcmp(left: *const c_char, right: *const c_char) -> c_int { + // SAFETY: both are NUL-terminated, so the bounded compare stops at a NUL. + unsafe { strncmp(left, right, usize::MAX) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strncmp(left: *const c_char, right: *const c_char, len: usize) -> c_int { + for index in 0..len { + // SAFETY: C promises both strings readable up to their NUL or `len`. + let (a, b) = unsafe { (*left.add(index) as u8, *right.add(index) as u8) }; + if a != b { + return c_int::from(a) - c_int::from(b); + } + if a == 0 { + break; + } + } + 0 +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strncpy(target: *mut c_char, source: *const c_char, len: usize) -> *mut c_char { + let mut index = 0; + // SAFETY: C promises `len` writable bytes at `target` and `source` + // readable up to its NUL or `len`; the tail is NUL-padded, as ISO says. + unsafe { + while index < len && *source.add(index) != 0 { + *target.add(index) = *source.add(index); + index += 1; + } + while index < len { + *target.add(index) = 0; + index += 1; + } + } + target +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strncat(target: *mut c_char, source: *const c_char, len: usize) -> *mut c_char { + // SAFETY: C promises `target` NUL-terminated with room for at most `len` + // more bytes and a NUL, and `source` readable up to its NUL or `len`. + unsafe { + let mut end = 0; + while *target.add(end) != 0 { + end += 1; + } + let mut index = 0; + while index < len && *source.add(index) != 0 { + *target.add(end + index) = *source.add(index); + index += 1; + } + *target.add(end + index) = 0; + } + target +} diff --git a/native/crates/c-shim/src/va_long_double/aarch64.rs b/native/crates/c-shim/src/va_long_double/aarch64.rs new file mode 100644 index 0000000..3aaf12a --- /dev/null +++ b/native/crates/c-shim/src/va_long_double/aarch64.rs @@ -0,0 +1,47 @@ +//! AAPCS64: a `long double` is a quad-precision floating-point type, passed in +//! the next SIMD and floating-point register while any remains, its save slot +//! in the register save area, and on the stack aligned to sixteen after that +//! (AAPCS64 §B.4, `va_arg` for a floating-point type). + +use super::{BYTES, Float, VaList, align_up, read}; + +/// `va_list` as AAPCS64 declares it (`__va_list`). +#[repr(C)] +struct List { + stack: *mut u8, + gr_top: *mut u8, + vr_top: *mut u8, + gr_offs: i32, + vr_offs: i32, +} + +const _: () = assert!(core::mem::size_of::>() == core::mem::size_of::()); + +/// One vector register's save slot. +const VECTOR_SLOT: i32 = 16; + +/// The next argument, a binary128 `long double`. +/// +/// # Safety +/// The next argument `list` holds is a `long double`. +pub(crate) unsafe fn next(list: &mut VaList<'_>) -> Float { + // SAFETY: `VaList` is the platform's `va_list` on this target (the size + // is asserted), and both areas are the caller's. + let list = unsafe { &mut *(list as *mut VaList<'_>).cast::() }; + let offset = list.vr_offs; + let at = if offset < 0 && offset + VECTOR_SLOT <= 0 { + list.vr_offs = offset + VECTOR_SLOT; + list.vr_top.wrapping_offset(offset as isize) + } else { + if offset < 0 { + // The registers ran out under this argument: the standard still + // counts it against them before it reads the stack. + list.vr_offs = offset + VECTOR_SLOT; + } + let at = align_up(list.stack); + list.stack = at.wrapping_add(BYTES); + at + }; + // SAFETY: the caller's contract puts the argument here. + Float::from_quad(unsafe { read(at) }) +} diff --git a/native/crates/c-shim/src/va_long_double/mod.rs b/native/crates/c-shim/src/va_long_double/mod.rs new file mode 100644 index 0000000..5f6922a --- /dev/null +++ b/native/crates/c-shim/src/va_long_double/mod.rs @@ -0,0 +1,54 @@ +//! `va_arg(ap, long double)`, which Rust's [`VaList`] cannot spell: Rust has +//! no type for x87's 80-bit format and no `VaArgSafe` binary128, so each ABI's +//! own rule reads the argument's bytes and the `printf` engine decodes them +//! ([`Float`]). +//! +//! [`VaList`] is laid out as the platform's `va_list`, the struct or pointer +//! its procedure-call standard defines, and the rule for a 16-byte, +//! 16-aligned argument comes from that same standard. One file per ABI, +//! selected here; the engine never names one. + +use core::ffi::VaList; + +use crate::format::Float; + +#[cfg(target_arch = "aarch64")] +mod aarch64; +#[cfg(target_arch = "riscv64")] +mod riscv64; +#[cfg(target_arch = "x86_64")] +mod x86_64; + +#[cfg(target_arch = "aarch64")] +pub(crate) use aarch64::next; +#[cfg(target_arch = "riscv64")] +pub(crate) use riscv64::next; +#[cfg(target_arch = "x86_64")] +pub(crate) use x86_64::next; + +#[cfg(not(any( + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "x86_64" +)))] +compile_error!("kore's C ABIs are x86_64, aarch64 and riscv64: name this one's long double rule"); + +/// Every kore C ABI's `long double`: sixteen bytes, aligned to sixteen (x87's +/// ten padded out, or binary128's own). +const BYTES: usize = 16; +const ALIGN: usize = 16; + +/// The 16-aligned address at or above `at`. +fn align_up(at: *mut u8) -> *mut u8 { + at.wrapping_add(at.addr().wrapping_neg() & (ALIGN - 1)) +} + +/// The `long double` stored at `at`, as its bits. +/// +/// # Safety +/// `at` is the address of a `long double` argument the caller passed. +unsafe fn read(at: *const u8) -> u128 { + // SAFETY: the caller's contract; unaligned for no reason but uniformity, + // since every rule aligns its slot. + u128::from_le_bytes(unsafe { at.cast::<[u8; BYTES]>().read_unaligned() }) +} diff --git a/native/crates/c-shim/src/va_long_double/riscv64.rs b/native/crates/c-shim/src/va_long_double/riscv64.rs new file mode 100644 index 0000000..d2b872f --- /dev/null +++ b/native/crates/c-shim/src/va_long_double/riscv64.rs @@ -0,0 +1,24 @@ +//! RISC-V LP64D: a `long double` is a 2×XLEN scalar with 2×XLEN alignment, so a +//! variadic one takes an aligned (even, odd) pair of argument registers, or +//! the stack aligned to sixteen (psABI, "Integer Calling Convention" and +//! "va_list, va_start, and va_arg"). The prologue spills the argument +//! registers just below the stack arguments, a pair to a 16-aligned slot, so +//! one pointer walks both and `va_arg` aligns it. + +use super::{BYTES, Float, VaList, align_up, read}; + +const _: () = assert!(core::mem::size_of::>() == core::mem::size_of::<*mut u8>()); + +/// The next argument, a binary128 `long double`. +/// +/// # Safety +/// The next argument `list` holds is a `long double`. +pub(crate) unsafe fn next(list: &mut VaList<'_>) -> Float { + // SAFETY: `VaList` is the platform's `va_list`, one pointer, on this + // target (the size is asserted). + let cursor = unsafe { &mut *(list as *mut VaList<'_>).cast::<*mut u8>() }; + let at = align_up(*cursor); + *cursor = at.wrapping_add(BYTES); + // SAFETY: the caller's contract puts the argument here. + Float::from_quad(unsafe { read(at) }) +} diff --git a/native/crates/c-shim/src/va_long_double/x86_64.rs b/native/crates/c-shim/src/va_long_double/x86_64.rs new file mode 100644 index 0000000..79de2cf --- /dev/null +++ b/native/crates/c-shim/src/va_long_double/x86_64.rs @@ -0,0 +1,32 @@ +//! System V AMD64: a `long double` is class X87, which the caller always +//! passes in memory, in the overflow area, aligned to sixteen (psABI §3.5.7, +//! the `va_arg` algorithm's step for a type that is not passed in +//! registers). + +use super::{BYTES, Float, VaList, align_up, read}; + +/// `va_list`'s one element, as the psABI declares it (`__va_list_tag`). +#[repr(C)] +struct Tag { + gp_offset: u32, + fp_offset: u32, + overflow_arg_area: *mut u8, + reg_save_area: *mut u8, +} + +const _: () = assert!(core::mem::size_of::>() == core::mem::size_of::()); + +/// The next argument, an x87 `long double`. +/// +/// # Safety +/// The next argument `list` holds is a `long double`. +pub(crate) unsafe fn next(list: &mut VaList<'_>) -> Float { + // SAFETY: `VaList` is the platform's `va_list` element on this target + // (the size is asserted), and the overflow area is the caller's frame. + let tag = unsafe { &mut *(list as *mut VaList<'_>).cast::() }; + let at = align_up(tag.overflow_arg_area); + // SAFETY: the caller's contract puts the argument here. + let bits = unsafe { read(at) }; + tag.overflow_arg_area = at.wrapping_add(BYTES); + Float::from_x87(bits) +} diff --git a/native/crates/c-shim/src/wide_symbols.rs b/native/crates/c-shim/src/wide_symbols.rs new file mode 100644 index 0000000..34a3191 --- /dev/null +++ b/native/crates/c-shim/src/wide_symbols.rs @@ -0,0 +1,77 @@ +//! The `` classes and case maps, answered from Rust's Unicode +//! tables: a grammar's scanner asks whether a code point is a letter or a +//! space, and the answer should not depend on a locale kore does not have. + +use core::ffi::c_int; + +/// C's `wint_t`: an unsigned 32-bit code point (or `WEOF`). +type WInt = u32; + +fn class(code: WInt, test: impl Fn(char) -> bool) -> c_int { + c_int::from(char::from_u32(code).is_some_and(test)) +} + +#[unsafe(no_mangle)] +extern "C" fn iswalnum(code: WInt) -> c_int { + class(code, char::is_alphanumeric) +} + +#[unsafe(no_mangle)] +extern "C" fn iswalpha(code: WInt) -> c_int { + class(code, char::is_alphabetic) +} + +#[unsafe(no_mangle)] +extern "C" fn iswblank(code: WInt) -> c_int { + class(code, |c| c == ' ' || c == '\t' || (c.is_whitespace() && !matches!(c, '\n'..='\r' | '\u{85}' | '\u{2028}' | '\u{2029}'))) +} + +#[unsafe(no_mangle)] +extern "C" fn iswdigit(code: WInt) -> c_int { + class(code, |c| c.is_ascii_digit()) +} + +#[unsafe(no_mangle)] +extern "C" fn iswlower(code: WInt) -> c_int { + class(code, char::is_lowercase) +} + +#[unsafe(no_mangle)] +extern "C" fn iswupper(code: WInt) -> c_int { + class(code, char::is_uppercase) +} + +#[unsafe(no_mangle)] +extern "C" fn iswspace(code: WInt) -> c_int { + class(code, char::is_whitespace) +} + +#[unsafe(no_mangle)] +extern "C" fn iswxdigit(code: WInt) -> c_int { + class(code, |c| c.is_ascii_hexdigit()) +} + +/// Printable, not a space, not a letter or digit: ISO's definition. +#[unsafe(no_mangle)] +extern "C" fn iswpunct(code: WInt) -> c_int { + class(code, |c| !c.is_whitespace() && !c.is_alphanumeric() && !c.is_control()) +} + +/// The one-code-point case maps; a mapping that expands to several code +/// points (`ß` → `SS`) leaves the code point as it is, as C's does. +fn single(mut mapped: impl Iterator, code: WInt) -> WInt { + match (mapped.next(), mapped.next()) { + (Some(c), None) => c as WInt, + _ => code, + } +} + +#[unsafe(no_mangle)] +extern "C" fn towlower(code: WInt) -> WInt { + char::from_u32(code).map_or(code, |c| single(c.to_lowercase(), code)) +} + +#[unsafe(no_mangle)] +extern "C" fn towupper(code: WInt) -> WInt { + char::from_u32(code).map_or(code, |c| single(c.to_uppercase(), code)) +} diff --git a/native/provenance.json b/native/provenance.json index 10a549b..c084f4f 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,9 +1,10 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "6c57f344311306868f68e13a37f87abf0ce04e02", + "revision": "2d4998baa8be794191fdacaf63dffca295c2a109", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", + "crates/c-shim": "kore-c-shim", "crates/collections": "kore-collections", "crates/dagfs": "kore-dagfs", "crates/driver": "kore-driver", @@ -29,7 +30,7 @@ "services/swap": "kore-svc-swap", "tools/kore-vol": "kore-vol" }, - "workspace_sha256": "17edafdee9c32e592cda34f7db82a7075ab803793d04119a25b42b11a89784e3", + "workspace_sha256": "da01593a48d1a10e9995fa7af159b5d8675f3bba89cabb1104ac370934643fde", "files": { "LICENSE": "c9b2b12d3bad9080090798a442b4b2a77c48312ae75089f041281b7c8ad21738", "crates/abi/Cargo.toml": "f9a4811b4b6d06947a47a763354cad3bc739a5ea387194a4dcf22409351f3c7b", @@ -68,6 +69,25 @@ "crates/abi/src/uring.rs": "e98d9db77fbe00d5547b582700a1104d2c5b2546bbe57742dfff8362c32dcc2e", "crates/app/Cargo.toml": "fb793b25eec583da7b5e404b19bd656a6b8bcee2a86a20b0da399be2210a9398", "crates/app/src/lib.rs": "5e7e8d791236f278ee85a02a3c37f41412d879cc41d67f6f6d3d701811973d7d", + "crates/c-shim/Cargo.toml": "85a4ad7f806ec588631dda39ef312513282217f3b0ddcdeeb3b5cbf6c1223685", + "crates/c-shim/src/alloc_symbols.rs": "e42b67a45271e7af676425cc915655e95153ee1f198864f5f527541e9c7092d8", + "crates/c-shim/src/errno.rs": "522a18b3561753cec41464a13ea978c5ea25b57cebde35363f4088ac074b1f38", + "crates/c-shim/src/format.rs": "d49d93e313f1eb0997deb40cd68942e8987152292c42feaf37546a883d21f4a9", + "crates/c-shim/src/format/float.rs": "eca54620399362cb35d09c842b7f3e40bef6af497c1a5b808df7d9e55c91b413", + "crates/c-shim/src/format/float/tests.rs": "8d579334d89afedcc9945c231a713e36b0baf5e99c5eb8a72d134100068dce9c", + "crates/c-shim/src/format_symbols.rs": "638807fa2f1e33095bfd295707d7ad7b63147d9cdd3cb66a132b49b167682a2f", + "crates/c-shim/src/lib.rs": "d52774104d1c0a3761f2e2825d74154b4e26050ee34bd0dcc93e43226ae41ee1", + "crates/c-shim/src/long_double.rs": "6d4434d610e973ff777f72b728a96537539c98bf076033c906d56e606796ab7b", + "crates/c-shim/src/number.rs": "e1fd500132cafdc70289f713f4ec04ae4e1bc9bc44884ebabbc8591fd6bc96ac", + "crates/c-shim/src/sort.rs": "4afb36cbb5c334d05712780bdee4d11ec30ab6be899f86e9828728f415d4f9db", + "crates/c-shim/src/stdio.rs": "d737a7b844aba0dbb852b6388ccc67c2342205992aaf609a9ac5353b97eea7d2", + "crates/c-shim/src/stdlib_symbols.rs": "5bd8645c999dcbe6e3722971d7ff57c654b74257aae48d9e76aba152f6906b72", + "crates/c-shim/src/string_symbols.rs": "cdcb976d8706218f0478de58993744e213d58a5012c0f25e999d96597c8928c0", + "crates/c-shim/src/va_long_double/aarch64.rs": "b954e8316eae7a30cce9823dcebecc7e8a258eb76a0b82bdc82cb03bd7b90c45", + "crates/c-shim/src/va_long_double/mod.rs": "96bb94ca1c36e0d1933e5abc5740c01a4b208b1d69b3102b0562109898ae35d0", + "crates/c-shim/src/va_long_double/riscv64.rs": "f170be5efa22626436e3b37749d828c20f90b75515a236f7bcf61c725b58673b", + "crates/c-shim/src/va_long_double/x86_64.rs": "49719570f536ac6aa665d7c2e6a5dc2d84ba4601f278f95bb9253fdfb0563f02", + "crates/c-shim/src/wide_symbols.rs": "494d37f1325b5a563c75f51c9ced46f6b338a4eb648a630bcdcd6d09a5d1d4a7", "crates/collections/Cargo.toml": "0b570a215ecd3d18b86b0f1b554e95de6636520c513eefd6f02d16cc2f37ebd6", "crates/collections/src/bitmat.rs": "35c1b7e572de0fa4e3dfc8c1c2cb8b7c5a031a812fb9ed7d4c86466b2a8399ec", "crates/collections/src/bitset.rs": "941e147ef204a6f3100c7d06652c6b9c3efb796ca7bdb738660bf6975a7ed815", @@ -295,13 +315,13 @@ "drivers/nvme/src/regs.rs": "d9410b5c64bb70f99e042f7209119c0f0fdb57f859c6a1c07eba90199f1e98c1", "drivers/nvme/src/swap.rs": "f5e1099b83b2cdeb9d288f4d337f1a234d9dfd16c31e9d9303c10f6f14972141", "services/block/Cargo.toml": "2729b36369486a6f5c1c29200eaa4603b25c26352f06f52301b70d9d57260e43", - "services/block/boot.toml": "dafeae8271530e4e21885595b7697e278143197c10cb5a617e92e805fa4a696b", + "services/block/boot.toml": "00aa7a50eb6cea63d8da291572b941ef44136a57682715a73a811d0610055c6a", "services/block/src/client.rs": "6aed79a404620f89ffa13078f615e846fdf6f3db5629c5cee5fbccbeaa7636cf", "services/block/src/lib.rs": "cf45960460cf6c9dba72f7a6727b51497275c686166a90be531d675097c275f1", "services/block/src/service.rs": "191e9d887d3f3267d6e2aa96e81996f01321bb2febb85e68581159f6e098b4b5", "services/block/src/wire.rs": "a9d64aef9ec3d58d37ca55b841933fc51b76ab94e18b245278b4519e96c65939", "services/dagfs/Cargo.toml": "d229a126616b1d7638f0f5a6edb4ac37d9d116f32afaa1897ffe484fbb286e5e", - "services/dagfs/boot.toml": "6a5c74554acdeb02588105ba7f56802b7694d6da7d127362970ad183e1e7333c", + "services/dagfs/boot.toml": "25a46002f036013e4e1dd87927324713a4c184a3c73b5e40435aedb4615c0586", "services/dagfs/examples/fsdemo.rs": "1666a14ef62fd4fc46f7f9efa2eaa5021ed79823f273a77f53758307f884e7d0", "services/dagfs/examples/fswire.rs": "994b97d699796b317741e2c87cbe812cc1776ddcd01173323e72a5f74c8ac267", "services/dagfs/examples/model_volume.rs": "0e3623e962da02b2126062a19d198b9e0a8f6e91896fd9a6f7d13f6f30ddc5f1", @@ -343,7 +363,7 @@ "services/dagfs/tests/provider.rs": "ba4070af34ae034c109e391e97bcc4755c6bd70b8e2fb229aaf9d747fef71e2d", "services/dagfs/tests/query_experiment.rs": "6fe147810474b6754ef31ad0fb95493e0b3c0886fbbafd78deb3729d410e8155", "services/device/Cargo.toml": "8f35c9ddd701b58a538f151e95e93ce618f0b346960491adad6d843287165d1c", - "services/device/boot.toml": "510b8448648e4431695de9de4ddb78ac33298a0d078185aa9b0fd1d21e829bb5", + "services/device/boot.toml": "3d5eaf0c17d8a4a976405c9d3182319a0be1d36993842b95cf3be6ce64cc721a", "services/device/src/client.rs": "71660deb4a9f51edb7a72fbb3edac94170bf4cccf9ce182cbd478ee3daf406eb", "services/device/src/lib.rs": "215c7c7e81d5d51664ac77b72b2edb0bab739d5a54b30bc9ad08f9dd3a3cfea4", "services/device/src/service/mod.rs": "6038ba0b9375e3c6bbb25c49bf0849a5878a402f4a3a691dc9df1494849d252b", @@ -385,7 +405,7 @@ "services/extract/vendor/jsmn/jsmn.h": "babfcb7413354a43dcd539c600a1e00bd2d1579ad0a6f6c7d8b81dccedb16cd0", "services/extract/vendor/jsmn/kore_jsmn.c": "a666060a7af03c819d1faaccc0d6faa19b76bcfe78e4ef61aeab273ad052c95c", "services/mm/Cargo.toml": "25809b6a03916d96b8de64a5988d005c71c49314fafbe0d920c3ec06f2dc5951", - "services/mm/boot.toml": "afb0cdf8a64635992180edf814518c525651949aa4afc559c89d81090c14ab18", + "services/mm/boot.toml": "294b4639deec93e2ef5188522da1d3f5ef0428e8a37e21616a697a2b6182fd24", "services/mm/src/client.rs": "c6dcc0dc3ed59e0a4cf7532fd306af4c1dc4d4e5ad1edd1d86887def791c5aa6", "services/mm/src/heap.rs": "e55210f3609f2e3e90f30d5497355ddab5801acc60ccbe3f8acbbffff71dbfde", "services/mm/src/lib.rs": "d3820740666ed993e547ee1b3b579231bf439a813d9766860e3b1b57fe815bd6", @@ -397,7 +417,7 @@ "services/output/src/service/mod.rs": "d80c3beb50979b3e1e500abdfa3c0b9d38e354c24e2e34aea443732abbead6f1", "services/output/src/wire.rs": "3752e994eb18714518facae1b78a9ee2b34f99aad8687653f0b89dc8fdeca621", "services/swap/Cargo.toml": "9c0af01403a6ad5a7f4ff69a02b18c8b1b2624a684814d6dc1ac07968c5bbe53", - "services/swap/boot.toml": "323531c62ec0b063cea376e941218719352c87c1b91e579e560fe55dc1bcbc42", + "services/swap/boot.toml": "4b6d4873ce4124a5d8ffa6bf93d15fdce0be213520e909ac9baeceec1ab2c120", "services/swap/src/client.rs": "04f735d059b491765d5ba63a4f5f640332735bf336e62dcbae86c17647583e2c", "services/swap/src/extent.rs": "d6705f77951df1c42cda51fc4526ba6262cab6af4a0545405ce06d2d2e1db7b4", "services/swap/src/lib.rs": "883684df54a87ae66aa7895ea19a872cabc8ab7fd4aded4ea65b7e3dd51fe0cf", diff --git a/native/services/block/boot.toml b/native/services/block/boot.toml index 2c43aaf..b085cfb 100644 --- a/native/services/block/boot.toml +++ b/native/services/block/boot.toml @@ -13,4 +13,4 @@ endowment = "kore_svc_block::BlockEndowment" protocol = "kore_svc_block::BlockService" endows = ["DMA_POOL@1"] digest_x86_64 = "blake3:a22f3a5b08f35a278ebf663d557e0b1ec490c8e732cdee0fa68151e349ddbc57" -digest_aarch64 = "blake3:2855f4168dcfe5b78dca4c508bc6f3120af84412c74498f2cf6475b01508c8e2" +digest_aarch64 = "blake3:93e621b55fbacf93391e4a32212cacfed71ede1797fb63935e2e971da15ab824" diff --git a/native/services/dagfs/boot.toml b/native/services/dagfs/boot.toml index a153d4e..ffb941d 100644 --- a/native/services/dagfs/boot.toml +++ b/native/services/dagfs/boot.toml @@ -48,4 +48,4 @@ exec_client = true # room for the proofs open handles carry, allocated per open at their depth. heap_supply_bytes = 0x00D0_0000 digest_x86_64 = "blake3:453a43f78c11f4fedf1715a141e4627a4d9858ab975fc8ad03c5d44b8814b7d9" -digest_aarch64 = "blake3:9521f0dae0fede5b0061ca144be8fdb421798faf17fd6cb5376acacb5741777e" +digest_aarch64 = "blake3:44ed7f47afd777ec49dc5b1193947e3d9cbf3ad3534f2b6edd162056abab35fe" diff --git a/native/services/device/boot.toml b/native/services/device/boot.toml index 0d1f546..021ef5d 100644 --- a/native/services/device/boot.toml +++ b/native/services/device/boot.toml @@ -48,4 +48,4 @@ protocol = "kore_svc_device::DeviceBroker" instances = 8 endows = ["ROOT_COMPLEX@0", "PLATFORM_BUS@0", "FACE_SUPPLY@0"] digest_x86_64 = "blake3:78d5b044c7b3ad3467484859549f8404cffc61ece27dd793cdd21c8c2c0a930a" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:7006c3d2336f8e984751fce7983384668fb0201e565dea03c8cbfd4c43992b3f" # res/aarch64/; make boot-digests-aarch64 +digest_aarch64 = "blake3:5632f4509b69316c1cac719cd0ddbf5a4ac0239db6fb42488ca6ec9811de4382" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/mm/boot.toml b/native/services/mm/boot.toml index 0071ac5..eee0bbd 100644 --- a/native/services/mm/boot.toml +++ b/native/services/mm/boot.toml @@ -55,4 +55,4 @@ endows = ["MEM_SUPPLY@0", "MEM_REGION@0", "SYSTEM_OBSERVE@0"] # review mirro shell_client = true exec_client = true digest_x86_64 = "blake3:d1ce3128c73f02293e323f87d79850b8558dfd82d882b65169767febb7baf6e4" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:a03d13d5caf95b15a82902ed087c816af46f74aa305ca47d01681b039e9d55fe" # res/aarch64/; make boot-digests-aarch64 +digest_aarch64 = "blake3:901b68a3acc55e7424ee98c90758fa17d5675f50292a1bc8e277591b2f0f5803" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/swap/boot.toml b/native/services/swap/boot.toml index 80722a4..3c9cab3 100644 --- a/native/services/swap/boot.toml +++ b/native/services/swap/boot.toml @@ -9,5 +9,5 @@ image = "kore-svc-swap" endowment = "kore_svc_swap::ZramEndowment" protocol = "kore_svc_swap::ZramSwap" endows = ["MEM_SUPPLY@0"] -digest_x86_64 = "blake3:ffa0c5ffe03770b1810bf23a9c546d1e92ce897708af3a5d0fc928c6dace51cd" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:f71c92926d0ca9faf5a1704659f96931bbf4aed4e1546bb6b1433d7d6d7057e4" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:dddc0488ba218b48a691a1e10e5995e962296056daf64ef0408ab15ecbd3a67f" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:b18a607ce1dec8f172effb4ae331e3be712ee99c6d3951dd33e907c5491e5cba" # res/aarch64/; make boot-digests-aarch64 From 0460f5c2a3f61aa386f28ed0776ad5d99742be7b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 06:19:18 +0000 Subject: [PATCH 11/18] c: stdio for C programs, from every thread, through the host A C program's `printf`, `puts`, `fwrite` and `fprintf(stderr, ...)` now reach the terminal it was run from. libkore.so links Kore's C library (kore-c-shim without its allocator and `abort`, which libkore exports itself) and installs its standard sink. The launcher's stream is bound to thread 0, the host, so every thread's bytes reach it through one in-process ring: - A writer holds the streams' lock across its whole C call, as POSIX's `flockfile` does, so one call's output is never split by another thread's. It kicks the host only if the host armed itself before parking, so a loop of `printf` costs no trap per call. The host holds no send into a pthread's doorbell, so every wait for room is bounded by the runtime's slice. - The host drains a pass at a time (what was queued when the pass began) and waits for the launcher's capacity when its records are full, so a slow terminal holds the program back rather than losing its output. Between passes it serves the program's end and its filesystem sessions, so a program that never stops printing still ends when it asks to. - The host's own writes (a filesystem step's callbacks run on it) cannot wait for the host. They are kept in order beside the ring, each stamped with the ring position it follows. - The end drains what was queued and closes the stream before the process ends, so the launcher shows the output before the exit status. An end committed on the host itself relays at once what the launcher has room for, and a write after the end reports `EOF`. `abort` ends the whole program with the aborted status, from any thread. cbindgen leaves it out of `kore/program.h`, since C declares it in ``, and `tools/check-program.py` holds libkore.so's exports to the declared ABI plus the C library's names. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- Cargo.lock | 8 + crates/c/Cargo.toml | 5 + crates/c/cbindgen.toml | 7 +- crates/c/src/lib.rs | 5 + crates/c/src/program.rs | 145 +++++- crates/c/src/stdio.rs | 979 ++++++++++++++++++++++++++++++++++++++++ docs/languages.md | 44 +- tools/check-program.py | 20 +- 8 files changed, 1188 insertions(+), 25 deletions(-) create mode 100644 crates/c/src/stdio.rs diff --git a/Cargo.lock b/Cargo.lock index 3967315..6a0168b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -806,10 +806,18 @@ dependencies = [ name = "kore-c" version = "0.1.0-alpha.1" dependencies = [ + "kore-c-shim", "kore-ffi", "kore-std", ] +[[package]] +name = "kore-c-shim" +version = "0.1.0" +dependencies = [ + "kore-rt", +] + [[package]] name = "kore-collections" version = "0.1.0" diff --git a/crates/c/Cargo.toml b/crates/c/Cargo.toml index 274ab56..f9434dc 100644 --- a/crates/c/Cargo.toml +++ b/crates/c/Cargo.toml @@ -23,3 +23,8 @@ kore-std = { package = "kore-std", path = "../..", features = ["heap", "fs"] } # The C ABI itself, with the standard C allocation and thread names a C # program calls (`malloc`, `pthread_create`, ...) exported as real symbols. kore-ffi = { path = "../ffi", features = ["c-stdlib"] } +# Kore's C library: the `printf` family and the standard streams, `` +# and ``'s parsers, `errno`. Its process half without its own +# `malloc` and `abort`: kore-ffi's allocator is this image's `malloc`, and the +# host ends a whole program. +kore-c-shim = { path = "../../native/crates/c-shim", default-features = false, features = ["runtime"] } diff --git a/crates/c/cbindgen.toml b/crates/c/cbindgen.toml index 511b74b..5f64a0d 100644 --- a/crates/c/cbindgen.toml +++ b/crates/c/cbindgen.toml @@ -12,9 +12,10 @@ header = "/* libkore.so: what an independent C program on Kore adds to kore.h. L parse_deps = false [export] # The C library's half of the program lifecycle (its start and exit call -# these), not an application API: the C library declares them itself. -# `KoreFsStep` is kore.h's typedef, which this header includes. -exclude = ["__kore_program_start", "__kore_program_exit", "KoreFsStep"] +# these), not an application API: the C library declares them itself, as +# its declares `abort`. `KoreFsStep` is kore.h's typedef, which +# this header includes. +exclude = ["__kore_program_start", "__kore_program_exit", "abort", "KoreFsStep"] [enum] prefix_with_name = true rename_variants = "ScreamingSnakeCase" diff --git a/crates/c/src/lib.rs b/crates/c/src/lib.rs index 36a22b0..07cd23a 100644 --- a/crates/c/src/lib.rs +++ b/crates/c/src/lib.rs @@ -38,12 +38,17 @@ // The C ABI's exports are this image's reason to exist; naming the crate keeps // them in the link. use kore_ffi as _; +// Kore's C library, exported whole from this image. +#[cfg(target_os = "none")] +use kore_c_shim as _; pub mod api; #[cfg(any(test, target_os = "none"))] mod exit; #[cfg(target_os = "none")] mod program; +#[cfg(any(test, target_os = "none"))] +mod stdio; /// The process heap over the founder's `role::HEAP` supply, grown on demand. /// The program's allocations (`malloc`, `kore_alloc_*`) and this library's diff --git a/crates/c/src/program.rs b/crates/c/src/program.rs index 91da94a..f8e2ca8 100644 --- a/crates/c/src/program.rs +++ b/crates/c/src/program.rs @@ -76,6 +76,9 @@ pub type End = unsafe extern "C" fn(c_int) -> !; /// The program, as [`__kore_program_start`] received it, for the main thread. static PROGRAM: AtomicUsize = AtomicUsize::new(0); +/// The C library's exit, as [`__kore_program_start`] received it, for an +/// [`abort`] on thread 0. +static END: AtomicUsize = AtomicUsize::new(0); /// Set by the first [`__kore_program_start`]; a second never founds a host. static STARTED: AtomicBool = AtomicBool::new(false); /// The host serves: the main thread exists and thread 0 parks for it. Set @@ -97,6 +100,16 @@ enum Role { #[thread_local] static ROLE: Cell = Cell::new(Role::Other); +/// Whether the calling thread is the host: thread 0 holding the program. +pub(crate) fn on_host() -> bool { + ROLE.get() == Role::Host +} + +/// Whether the calling thread is the main thread, the one the host can kick. +pub(crate) fn on_main() -> bool { + ROLE.get() == Role::Main +} + // ── The end ───────────────────────────────────────────────────────────── /// The program's end, once asked for (see [`ExitWord`]). @@ -175,6 +188,7 @@ pub unsafe extern "C" fn __kore_program_start(program: Option, end: Opt } ROLE.set(Role::Host); PROGRAM.store(program as usize, Ordering::Release); + END.store(end as usize, Ordering::Release); match Founded::found() { Some(host) => host.serve(end), None => unsafe { program() }, @@ -194,8 +208,13 @@ pub extern "C" fn __kore_program_exit(status: c_int, finalize: c_int) { let finalize = finalize != 0; match ROLE.get() { // Thread 0 itself: the program when there is no host, or a callback - // or finalizer the host is running. The barrier is this thread's. - Role::Host => startup::barrier(finalize), + // or finalizer the host is running. It cannot wait for its own loop, + // so it relays what the launcher has room for now; the barrier is + // this thread's. + Role::Host => { + crate::stdio::flush_now(); + startup::barrier(finalize) + } role if HOSTED.load(Ordering::Acquire) => { let _ = EXIT.post(Exit { status, @@ -213,6 +232,33 @@ pub extern "C" fn __kore_program_exit(status: c_int, finalize: c_int) { } } +/// `abort` (C11 7.22.4.1): the program's abnormal end. It ends with the +/// aborted status, 134 — what a shell reports for a program `SIGABRT` ended — +/// and, like `_Exit`, runs no `atexit` handler or finalizer. No stream +/// buffers, so what the program wrote before has already left it: the host's +/// loop relays all of it before the end when another thread aborts, and a +/// callback's `abort` on the host itself relays what the launcher has room +/// for at once (`stdio::flush_now`). +/// +/// The C library's `_Exit` lives in the program, so this takes `_Exit`'s +/// path itself: any other thread hands the status to the host and ends, and +/// thread 0, the barrier raised, commits the end the C library handed over. +#[unsafe(no_mangle)] +pub extern "C" fn abort() -> ! { + let status = ExitCode::ABORTED.get(); + __kore_program_exit(status, 0); + match END.load(Ordering::Acquire) { + // A program the C library never started: thread 0 ends the task. + 0 => thread::exit(ExitCode::ABORTED), + end => { + // SAFETY: `__kore_program_start` stored an `End`, which never + // returns; every other thread is stopped. + let end: End = unsafe { core::mem::transmute(end) }; + unsafe { end(status) } + } + } +} + /// The main thread's half of [`kore_fs_run`](crate::api::kore_fs_run): post /// the session to the host and wait for its answer. /// @@ -264,7 +310,10 @@ pub(crate) unsafe fn session(step: KoreFsStep, user: *mut c_void) -> (KoreStatus pub(crate) fn panicked(info: &core::panic::PanicInfo<'_>) -> ! { crash::report(info); match ROLE.get() { - Role::Host => startup::barrier(false), + Role::Host => { + crate::stdio::flush_now(); + startup::barrier(false) + } role if HOSTED.load(Ordering::Acquire) => { let _ = EXIT.post(Exit { status: ExitCode::PANIC.get(), @@ -297,10 +346,12 @@ fn fatal() -> ! { struct Fatal; /// Thread 0 with its program founded: the runtime context, the executor's -/// doorbell, and the main thread. +/// doorbell, the launcher's stream when the launch carried one, and the main +/// thread. struct Founded { rt: Rt, doorbell: executor::Doorbell, + output: Option<&'static crate::stdio::Endpoint>, worker: DoorbellWorker, } @@ -309,6 +360,9 @@ impl Founded { fn found() -> Option { let rt = entry::thread_context()?; let doorbell = rt.doorbell()?; + // The C streams reach the launcher through this thread: map its + // stream and install the sink before any thread can write. + let output = crate::stdio::install(&rt); HOSTED.store(true, Ordering::Release); // A mint grant of its own: the first `pthread_create` on the main // thread starts kore-rt's thread controller there, which mints the @@ -323,10 +377,16 @@ impl Founded { Some(worker) => Some(Self { rt, doorbell, + output, worker, }), None => { HOSTED.store(false, Ordering::Release); + // No host will drain the stream: writes report `EOF`, as + // they do for a launch that carried none. + if output.is_some() { + crate::stdio::abandon(); + } None } } @@ -337,12 +397,25 @@ impl Founded { let Self { rt, doorbell, + output, worker, } = self; let transport = TrapTransport::new(); let exit = { let mut executor = Executor::new(entry::transport(), doorbell); - executor.run(host(&rt, &transport, &worker)) + executor.run(async { + let exit = host(&rt, &transport, &worker, output).await; + // Whatever ended the program, what it wrote reaches the + // launcher, and then the stream's close, before the process + // ends: the launcher shows all of it before the status. + if let Some(output) = output { + crate::stdio::finish(output, &|| { + let _ = worker.kick(); + }) + .await; + } + exit + }) }; let exit = exit.unwrap_or(Exit { status: ExitCode::INTERNAL.get(), @@ -375,13 +448,30 @@ fn main_thread() { /// The host's loop: answer each session the main thread posts, until any /// thread asks for the end. -async fn host(rt: &Rt, transport: &TrapTransport, worker: &DoorbellWorker) -> Result { +async fn host( + rt: &Rt, + transport: &TrapTransport, + worker: &DoorbellWorker, + output: Option<&'static crate::stdio::Endpoint>, +) -> Result { let mut wake = NoticeRx::new(transport, worker.wake_recv()); let mut filesystem = Filesystem::new(rt, transport); + // The one writer the host can kick, woken when its room is made. + let wake_main = || { + let _ = worker.kick(); + }; loop { if let Some(exit) = EXIT.posted() { return Ok(exit); } + if let Some(output) = output { + // One bounded pass: the end and the next session are served + // between passes, however fast the program writes. + crate::stdio::drain(output, &wake_main).await; + if let Some(exit) = EXIT.posted() { + return Ok(exit); + } + } if SESSION.state.load(Ordering::Acquire) == POSTED { // SAFETY: POSTED hands the cell to the host until it stores SERVED. let (step, user) = { @@ -396,7 +486,7 @@ async fn host(rt: &Rt, transport: &TrapTransport, worker: &DoorbellWorker) -> Re // the abandoned session's owner is quarantined, as cancellation // leaves it. let session = pin!(filesystem.run(step, user)); - let ended = pin!(until_exit(&mut wake)); + let ended = pin!(until_exit(&mut wake, output, &wake_main)); let outcome = race(session, ended).await; let Some((status, error)) = outcome else { continue; @@ -411,7 +501,10 @@ async fn host(rt: &Rt, transport: &TrapTransport, worker: &DoorbellWorker) -> Re let _ = worker.kick(); continue; } - park(&mut wake, pending).await?; + // Armed before the recheck `park` makes, so a writer that published + // meanwhile either kicks or is seen. + crate::stdio::arm(); + park(&mut wake, || pending() || crate::stdio::pending()).await?; } } @@ -433,6 +526,10 @@ async fn race( .await } +/// How soon a receive refused for want of a standing post looks again: the +/// tree's figure for a rotating notice receiver (the WASI host's handoff). +const WOULD_BLOCK_RETRY_NS: u64 = 100_000; + /// Park until a kick, a spurious wake, or `ready` holds, having stood the /// wake post before re-reading what `ready` reads. async fn park( @@ -452,21 +549,39 @@ async fn park( let _ = executor::next_wake_on(post).await; } Err(RecvError::Refused(Status::WouldBlock)) => { - let _ = executor::next_wake().await; + // No post stands to wake on: look again shortly, as the + // tree's other receivers do, rather than wait for a wake + // nothing armed. + let _ = executor::next_wake_for(WOULD_BLOCK_RETRY_NS).await; } Err(_) => return Err(Fatal), } } } -/// Resolve once an end is posted, parking on the wake channel meanwhile: the -/// session being served is not a reason to wake, only an end is. On a broken -/// channel it never resolves, since the host can no longer hear an end; the -/// session it races then runs to its own completion. -async fn until_exit(wake: &mut NoticeRx<'_, TrapTransport>) { +/// Resolve once an end is posted, parking on the wake channel meanwhile and +/// draining what the program writes: the session being served is not a +/// reason to wake, only an end or output is. On a broken channel it never +/// resolves, since the host can no longer hear an end; the session it races +/// then runs to its own completion. Dropped when the session ends first, +/// which the drain survives (`stdio::drain`). Each pass yields, so the +/// session it races is polled however fast the program writes. +async fn until_exit( + wake: &mut NoticeRx<'_, TrapTransport>, + output: Option<&'static crate::stdio::Endpoint>, + wake_main: &dyn Fn(), +) { let ended = || EXIT.posted().is_some(); while !ended() { - if park(wake, ended).await.is_err() { + if let Some(output) = output { + crate::stdio::drain(output, wake_main).await; + executor::yield_now().await; + } + crate::stdio::arm(); + if park(wake, || ended() || crate::stdio::pending()) + .await + .is_err() + { core::future::pending::<()>().await; } } diff --git a/crates/c/src/stdio.rs b/crates/c/src/stdio.rs new file mode 100644 index 0000000..1f724f6 --- /dev/null +++ b/crates/c/src/stdio.rs @@ -0,0 +1,979 @@ +//! A C program's `stdout` and `stderr`. +//! +//! The launcher's stream (the launch's `PROFILE_INTERACTIVE` window) is +//! mapped on thread 0, the host, and is bound to it: only thread 0 may publish +//! into it. Every thread writes the C streams, so their bytes reach the +//! stream through the host, by one in-process ring: +//! +//! - **A writer** holds the streams' lock for its whole C call: kore-c-shim +//! takes it around each stream call, as POSIX's `flockfile` does, so one +//! call's output is never split by another thread's. Under the lock the +//! writer is the ring's one producer. It waits for room for each piece, +//! copies it in, and kicks the host only if the host armed itself before +//! parking, so a loop of `printf` costs no trap per call. The host can +//! kick the main thread but holds no send into a pthread's doorbell, so +//! every wait for room is bounded by the runtime's slice: a writer the +//! host cannot reach re-reads the ring each slice, as the thread +//! controller's unreached clients do. +//! - **The host** drains the ring a pass at a time — what was queued when +//! the pass began — into the stream's output records, as many as the +//! launcher has room for, and waits for their capacity when they are full: +//! a slow terminal holds the program back rather than losing its output. +//! Between passes it serves the program's end and its filesystem sessions, +//! so a program that never stops printing still ends when it asks to. +//! - **The host's own writes** (a filesystem step's callbacks run on it) +//! cannot wait for the host. They are kept in order beside the ring, each +//! stamped with the ring position it follows, and drained in that order. +//! The host never takes the streams' lock, which a writer waiting on the +//! host may hold. +//! - **The end** drains what was queued and closes the stream before the +//! process ends, so the launcher shows the program's output before its exit +//! status. An end committed on the host itself (a callback's `exit` or +//! `abort`, a panic there) cannot wait, and relays at once what the +//! launcher has room for. A write after the end, or once the stream is +//! gone, reports `EOF`. +//! +//! The ring and its record codec are plain data over atomics ([`Ring`]), so +//! the host suite exercises the code the target runs; the endpoint, the lock's +//! parks and the kicks are the metal half (`metal`). + +use core::cell::UnsafeCell; +#[cfg(target_os = "none")] +use core::sync::atomic::AtomicBool; +use core::sync::atomic::{AtomicU32, Ordering}; + +/// Which C stream a record carries. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Stream { + Out, + Err, +} + +impl Stream { + const fn tag(self) -> u8 { + match self { + Self::Out => 1, + Self::Err => 2, + } + } + + const fn from_tag(tag: u8) -> Option { + match tag { + 1 => Some(Self::Out), + 2 => Some(Self::Err), + _ => None, + } + } +} + +/// Bytes of one record's header: its stream's tag and its length. +const HEADER: usize = 2; +/// The most payload one record carries: its length is one byte. +pub(crate) const RECORD_PAYLOAD: usize = u8::MAX as usize; + +/// The ring's bytes ([`RING`]). A power of two, so a position is an index by +/// a mask, and far below `u32`'s range, so the free-running positions' +/// difference never wraps past a lap. +const RING_BYTES: usize = 1 << 14; + +/// The ring bytes `len` payload bytes take, in whole records. +pub(crate) const fn framed(len: usize) -> usize { + len + HEADER * len.div_ceil(RECORD_PAYLOAD) +} + +/// The largest piece the ring keeps whole: half of it, so a writer waiting +/// for that much room never waits for more room than the host can make +/// while another piece is queued ahead of it. A longer write goes in pieces +/// of this size, under the one lock its call holds. +pub(crate) const WHOLE_WRITE: usize = + RING_BYTES / 2 - HEADER * (RING_BYTES / 2).div_ceil(RECORD_PAYLOAD); + +const _: () = assert!(RING_BYTES.is_power_of_two()); +const _: () = assert!(framed(WHOLE_WRITE) <= RING_BYTES / 2); + +/// A cursor on a cache line of its own: the writer's `tail` and the host's +/// `head` move on different cores, and must not share a line. +#[repr(align(64))] +struct Cursor(AtomicU32); + +/// One ring of framed records over free-running `u32` positions: `tail` is +/// written only by the lock holder, `head` only by the host. +pub(crate) struct Ring { + bytes: UnsafeCell<[u8; N]>, + /// The first byte the host has not consumed. + head: Cursor, + /// One past the last byte a writer published. + tail: Cursor, +} + +// SAFETY: the bytes between `head` and `tail` belong to the host, the rest to +// the lock holder; each side publishes its cursor after its last access and +// reads the other's with Acquire before its first. +unsafe impl Sync for Ring {} + +impl Ring { + pub(crate) const fn new() -> Self { + Self { + bytes: UnsafeCell::new([0; N]), + head: Cursor(AtomicU32::new(0)), + tail: Cursor(AtomicU32::new(0)), + } + } + + /// The host's cursor, for a writer to wait on. + pub(crate) fn head(&self) -> &AtomicU32 { + &self.head.0 + } + + /// Bytes published and not consumed, at most: `head` is read first, and + /// `tail` never falls behind any `head` read before it. + pub(crate) fn queued(&self) -> usize { + let head = self.head.0.load(Ordering::Acquire); + self.tail.0.load(Ordering::Acquire).wrapping_sub(head) as usize + } + + /// Free bytes: exact under the lock, where `tail` is this thread's; + /// conservative outside it, where the host and another writer may both + /// have moved between the two reads. + pub(crate) fn room(&self) -> usize { + N.saturating_sub(self.queued()) + } + + /// Room for `bytes` more, framed. + pub(crate) fn fits(&self, bytes: usize) -> bool { + self.room() >= framed(bytes) + } + + /// The position the next record will be published at. + pub(crate) fn tail(&self) -> u32 { + self.tail.0.load(Ordering::Acquire) + } + + /// Publish `payload` as records of `stream`, whole. + /// + /// # Safety + /// The caller is the ring's one producer (it holds the streams' lock) and + /// saw [`fits`](Self::fits) for `payload` under that lock. + pub(crate) unsafe fn publish(&self, stream: Stream, payload: &[u8]) { + let mut at = self.tail.0.load(Ordering::Relaxed); + for record in payload.chunks(RECORD_PAYLOAD) { + // SAFETY: the caller's contract: `[tail, tail + framed)` is free, + // so the host reads none of these bytes until the store below. + unsafe { + self.put(at, stream.tag()); + self.put(at.wrapping_add(1), record.len() as u8); + for (i, byte) in record.iter().enumerate() { + self.put(at.wrapping_add((HEADER + i) as u32), *byte); + } + } + at = at.wrapping_add((HEADER + record.len()) as u32); + } + // SeqCst, not Release: the host's park recheck reads `tail` after + // arming, and the writer reads the armed flag after this store — the + // Dekker pair that makes a kick unnecessary unless the host armed. + self.tail.0.store(at, Ordering::SeqCst); + } + + /// The record at the host's cursor, if any: its stream and its payload, + /// copied into `out`. The host is the one consumer. + pub(crate) fn peek(&self, out: &mut [u8; RECORD_PAYLOAD]) -> Option<(Stream, usize)> { + let head = self.head.0.load(Ordering::Relaxed); + if self.tail.0.load(Ordering::Acquire) == head { + return None; + } + // SAFETY: `[head, tail)` holds whole records the Acquire above made + // visible, and only this consumer advances `head`. + unsafe { + let stream = Stream::from_tag(self.get(head))?; + let len = self.get(head.wrapping_add(1)) as usize; + for (i, byte) in out.iter_mut().take(len).enumerate() { + *byte = self.get(head.wrapping_add((HEADER + i) as u32)); + } + Some((stream, len)) + } + } + + /// Consume the record [`peek`](Self::peek) returned, `len` payload bytes. + pub(crate) fn consume(&self, len: usize) { + let head = self.head.0.load(Ordering::Relaxed); + self.head + .0 + .store(head.wrapping_add((HEADER + len) as u32), Ordering::Release); + } + + /// Drop every record published so far, as the one consumer. + pub(crate) fn discard(&self) { + let tail = self.tail.0.load(Ordering::Acquire); + self.head.0.store(tail, Ordering::Release); + } + + unsafe fn put(&self, at: u32, byte: u8) { + // SAFETY: the caller owns byte `at` (see `publish`). + unsafe { (*self.bytes.get())[at as usize & (N - 1)] = byte } + } + + unsafe fn get(&self, at: u32) -> u8 { + // SAFETY: the caller owns byte `at` (see `peek`). + unsafe { (*self.bytes.get())[at as usize & (N - 1)] } + } +} + +/// The program's one ring. +#[cfg(target_os = "none")] +pub(crate) static RING: Ring = Ring::new(); + +/// The host is parked, or about to, and a writer must kick it (see +/// [`Ring::publish`]). +#[cfg(target_os = "none")] +static ARMED: AtomicBool = AtomicBool::new(false); + +/// The room the main thread waits for, framed, or 0: the one writer the host +/// can kick, kicked once that much is free rather than at every record. +#[cfg(target_os = "none")] +static MAIN_NEEDS: AtomicU32 = AtomicU32::new(0); + +/// The launcher's stream is gone, or there is no host to drain the ring: +/// writes fail rather than wait for a drain that will not come. +#[cfg(target_os = "none")] +static LOST: AtomicBool = AtomicBool::new(false); + +/// The program has ended: nothing written from here is relayed. +#[cfg(target_os = "none")] +static FINISHED: AtomicBool = AtomicBool::new(false); + +/// Writes report `EOF` from here on. +#[cfg(target_os = "none")] +fn closed() -> bool { + LOST.load(Ordering::Acquire) || FINISHED.load(Ordering::Acquire) +} + +/// Whether the host has anything to drain. +#[cfg(target_os = "none")] +pub(crate) fn pending() -> bool { + !closed() && (RING.queued() != 0 || metal::host_pending()) +} + +/// A write the host made while it could not drain: its bytes, and the ring +/// position it follows. +#[cfg(target_os = "none")] +pub(crate) struct Held { + pub(crate) after: u32, + pub(crate) stream: Stream, + pub(crate) bytes: alloc::vec::Vec, +} + +/// Whether a held write is the next item to drain: it belongs at ring +/// position `held_after`, so it goes once the host's cursor reaches that +/// point, after every record published before it and before every one +/// published after. A position the cursor has already passed (a write made +/// during a call whose first records were already relayed) goes at once. +/// Positions are free-running, and a stamp never strays half the `u32` +/// range from the cursor. +pub(crate) fn held_is_next(held_after: u32, head: u32) -> bool { + head.wrapping_sub(held_after) as i32 >= 0 +} + +extern crate alloc; + +#[cfg(target_os = "none")] +pub(crate) use metal::{Endpoint, abandon, arm, drain, finish, flush_now, install}; + +#[cfg(target_os = "none")] +mod metal { + use super::*; + use alloc::collections::VecDeque; + use alloc::vec::Vec; + use core::cell::{Cell, RefCell}; + use core::mem::MaybeUninit; + use core::sync::atomic::AtomicU64; + use core::time::Duration; + use kore_c_shim::stdio::{Standard, StandardSink, install_standard_sink}; + use kore_std::abi::process::{PROCESS_OUTPUT_CHUNK_BYTES, ProcessOutputEvent}; + use kore_std::process::{ + MappedProcessStdioChild, ProcessStdioParkStep, ProcessStdioWaitFor, ProcessStdioWake, + }; + use kore_std::runtime::futex::{self, WaitOutcome}; + use kore_std::runtime::{executor, pace, thread}; + + /// How long a writer waits before it re-reads what it waits for: the + /// runtime's slice for a waiter its waker cannot reach (the thread + /// controller's `UNREACHED_SLICE`). The host kicks the main thread + /// sooner; a pthread rides the slice. + const SLICE: Duration = Duration::from_nanos(pace::CEDE_NS); + + /// The host's half: the launcher's stream and the writes the host held. + /// Thread 0 alone touches it (the stream is bound to that thread), and + /// never across an await. + pub(crate) struct Endpoint { + stdio: RefCell, + held: RefCell>, + /// Bytes of the next item (the front held write, or the ring's head + /// record) already published. A drain can be dropped between two + /// records it publishes (it races a filesystem session), so it + /// consumes an item only once all of it is out, and resumes past + /// what it already sent rather than send it twice. + sent: Cell, + } + + /// Thread 0's endpoint, in static storage: the stream is mapped before + /// the main thread is founded, and a launch without a heap still has + /// one. + struct EndpointCell(UnsafeCell>); + // SAFETY: written once by the host before any other thread exists, and + // read only on thread 0 (`crate::program::on_host`) once `MAPPED` is set. + unsafe impl Sync for EndpointCell {} + static ENDPOINT: EndpointCell = EndpointCell(UnsafeCell::new(MaybeUninit::uninit())); + static MAPPED: AtomicBool = AtomicBool::new(false); + + fn host() -> Option<&'static Endpoint> { + if !crate::program::on_host() || !MAPPED.load(Ordering::Acquire) { + return None; + } + // SAFETY: on thread 0, after `install` wrote it (see `EndpointCell`). + Some(unsafe { (*ENDPOINT.0.get()).assume_init_ref() }) + } + + pub(crate) fn host_pending() -> bool { + host().is_some_and(|endpoint| !endpoint.held.borrow().is_empty()) + } + + /// The C streams' sink and their lock. + static SINK: StandardSink = StandardSink { + write: sink, + lock: lock_streams, + unlock: unlock_streams, + }; + + /// Map the launcher's stream and send the C streams to it. `None` when + /// the launch carried no stream: `stdout` then reports `EOF`, as it does + /// for every program that has none. + pub(crate) fn install(rt: &kore_std::runtime::entry::Rt) -> Option<&'static Endpoint> { + let stdio = kore_std::process::map_stdio(rt)?; + // SAFETY: thread 0, before any other thread exists, and only once + // (`__kore_program_start` founds one host). + let endpoint = unsafe { + (*ENDPOINT.0.get()).write(Endpoint { + stdio: RefCell::new(stdio), + held: RefCell::new(VecDeque::new()), + sent: Cell::new(0), + }) + }; + MAPPED.store(true, Ordering::Release); + install_standard_sink(&SINK); + Some(endpoint) + } + + /// The host could not be founded after the stream was mapped: no one + /// will drain it, so every write reports `EOF`. + pub(crate) fn abandon() { + LOST.store(true, Ordering::SeqCst); + } + + // ── The streams' lock ─────────────────────────────────────────────── + + /// The classic three-state futex mutex, reentrant per thread: a stream + /// call holds it across all its writes, and the sink each write reaches + /// takes it again. + static LOCK: AtomicU32 = AtomicU32::new(FREE); + const FREE: u32 = 0; + const HELD: u32 = 1; + const CONTENDED: u32 = 2; + + /// How many times this thread holds [`LOCK`]. + #[thread_local] + static DEPTH: Cell = Cell::new(0); + + /// The call holding [`LOCK`], if any: [`IN_CALL`] beside the ring's + /// `tail` when it began. The host's own writes made meanwhile are + /// stamped there, so they go before the call rather than between its + /// pieces. + static CALL: AtomicU64 = AtomicU64::new(0); + const IN_CALL: u64 = 1 << u32::BITS; + + /// Take the streams' lock. The host never does: a writer that holds it + /// may be waiting for the host to drain. + fn lock_streams() { + if crate::program::on_host() { + return; + } + let depth = DEPTH.get(); + if depth == 0 { + acquire(); + // The holder is the ring's one producer: `tail` is exact here. + CALL.store(IN_CALL | u64::from(RING.tail()), Ordering::SeqCst); + } + DEPTH.set(depth + 1); + } + + fn unlock_streams() { + if crate::program::on_host() { + return; + } + match DEPTH.get() { + // Released more often than taken: the caller's mistake, and not + // a reason to release another thread's hold. + 0 => {} + 1 => { + DEPTH.set(0); + CALL.store(0, Ordering::SeqCst); + release(); + } + depth => DEPTH.set(depth - 1), + } + } + + fn acquire() { + if LOCK + .compare_exchange(FREE, HELD, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + return; + } + while LOCK.swap(CONTENDED, Ordering::Acquire) != FREE { + // Bounded: the holder's wake may not reach this thread. + if futex::wait(&LOCK, CONTENDED, Some(SLICE)) == WaitOutcome::Unsupported { + // No doorbell to park on: the deadline park any thread can. + let _ = pace::cede(); + } + } + } + + fn release() { + // SeqCst, as the futex contract asks of the store before a wake. + if LOCK.swap(FREE, Ordering::SeqCst) == CONTENDED { + futex::wake(&LOCK); + } + } + + // ── Writers ───────────────────────────────────────────────────────── + + fn sink(standard: Standard, bytes: &[u8]) -> bool { + let stream = match standard { + Standard::Output => Stream::Out, + Standard::Error => Stream::Err, + }; + if bytes.is_empty() { + return true; + } + if closed() { + return false; + } + if let Some(endpoint) = host() { + return hold(endpoint, stream, bytes); + } + // The call's lock, again: this thread is the ring's one producer for + // every piece. + lock_streams(); + let written = bytes.chunks(WHOLE_WRITE).all(|piece| write(stream, piece)); + unlock_streams(); + written + } + + /// One piece, under the streams' lock: wait for its room, then copy it + /// in. + fn write(stream: Stream, piece: &[u8]) -> bool { + loop { + if closed() { + return false; + } + // Read before the room check: a wait on this value returns at + // once if the host has moved since. + let head = RING.head().load(Ordering::SeqCst); + if RING.fits(piece.len()) { + // SAFETY: the lock makes this thread the one producer, and + // the room was checked under it. + unsafe { RING.publish(stream, piece) }; + if ARMED.swap(false, Ordering::SeqCst) { + let _ = thread::kick_parent(); + } + return true; + } + // The ring is full of records the host has not drained. A host + // parked on its wake (armed) is kicked; one that is not is + // draining already, or waiting for the launcher's room, and + // moves its cursor when it has drained. + let main = crate::program::on_main(); + if main { + MAIN_NEEDS.store(framed(piece.len()) as u32, Ordering::SeqCst); + } + if ARMED.swap(false, Ordering::SeqCst) { + let _ = thread::kick_parent(); + } + let outcome = futex::wait(RING.head(), head, Some(SLICE)); + if main { + MAIN_NEEDS.store(0, Ordering::SeqCst); + } + if outcome == WaitOutcome::Unsupported { + let _ = pace::cede(); + } + } + } + + /// The host's own write: kept, stamped, for its next drain. Stamped at + /// the start of a call in progress, so it does not split that call. + fn hold(endpoint: &Endpoint, stream: Stream, bytes: &[u8]) -> bool { + let mut copy = Vec::new(); + if copy.try_reserve_exact(bytes.len()).is_err() { + return false; + } + copy.extend_from_slice(bytes); + let mut held = endpoint.held.borrow_mut(); + if held.try_reserve(1).is_err() { + return false; + } + let call = CALL.load(Ordering::SeqCst); + let after = if call & IN_CALL != 0 { + call as u32 + } else { + RING.tail() + }; + held.push_back(Held { + after, + stream, + bytes: copy, + }); + true + } + + // ── The host ──────────────────────────────────────────────────────── + + /// How one item went out. + enum Pumped { + /// All of it. + Done, + /// The launcher has no room for the next record: wait, or stop. + Full, + /// The launcher's stream is gone. + Lost, + } + + /// Publish what is left of one item, `len` bytes that `fill` copies out + /// from an offset, as stream records while the launcher has room, past + /// what [`Endpoint::sent`] says is out. Never waits. + fn pump( + endpoint: &Endpoint, + stream: Stream, + len: usize, + fill: &dyn Fn(usize, &mut [u8]) -> usize, + ) -> Pumped { + let mut stdio = endpoint.stdio.borrow_mut(); + while endpoint.sent.get() < len { + if stdio.output_capacity_left() == 0 { + return Pumped::Full; + } + let from = endpoint.sent.get(); + let mut chunk = [0u8; PROCESS_OUTPUT_CHUNK_BYTES]; + let want = (len - from).min(PROCESS_OUTPUT_CHUNK_BYTES); + let filled = fill(from, &mut chunk[..want]); + if filled == 0 { + // The item vanished under the drain: nothing can resume it. + return Pumped::Lost; + } + let event = match stream { + Stream::Out => ProcessOutputEvent::stdout(&chunk[..filled]), + Stream::Err => ProcessOutputEvent::stderr(&chunk[..filled]), + }; + let Ok(event) = event else { + return Pumped::Lost; + }; + match stdio.send_output_now(event) { + Ok(ProcessStdioWake::Unreached(_)) | Err(_) => return Pumped::Lost, + Ok(_) => endpoint.sent.set(from + filled), + } + } + endpoint.sent.set(0); + Pumped::Done + } + + /// [`pump`] one item whole, waiting for the launcher's room between its + /// records. `false` once the stream is gone. Dropped at a wait, it has + /// published whole records only, and `sent` says how far. + async fn send( + endpoint: &Endpoint, + stream: Stream, + len: usize, + fill: &dyn Fn(usize, &mut [u8]) -> usize, + ) -> bool { + loop { + match pump(endpoint, stream, len, fill) { + Pumped::Done => return true, + Pumped::Lost => return false, + Pumped::Full => { + if !room(endpoint).await { + return false; + } + } + } + } + } + + /// Copy the front held write's bytes from `from` into `out`. + fn held_bytes(endpoint: &Endpoint, from: usize, out: &mut [u8]) -> usize { + let held = endpoint.held.borrow(); + let Some(front) = held.front() else { + return 0; + }; + let Some(bytes) = front.bytes.get(from..from + out.len()) else { + return 0; + }; + out.copy_from_slice(bytes); + out.len() + } + + /// The next item due, if any: the front held write when the host's + /// cursor reached its stamp, else the ring's head record if it lies + /// before `end`. + enum Next { + Held(Stream, usize), + Record(Stream, usize), + None, + } + + fn next(endpoint: &Endpoint, end: u32, record: &mut [u8; RECORD_PAYLOAD]) -> Next { + let head = RING.head().load(Ordering::Acquire); + if let Some(front) = endpoint.held.borrow().front() + && held_is_next(front.after, head) + { + return Next::Held(front.stream, front.bytes.len()); + } + if head == end { + return Next::None; + } + match RING.peek(record) { + Some((stream, len)) => Next::Record(stream, len), + None => Next::None, + } + } + + /// A record went out: consume it, and wake the main thread once the + /// room it waits for is free. Fenced, so the cursor's store is ordered + /// before the read of what main needs, the other half of the writer's + /// pair. + fn consumed(len: usize, wake_main: &dyn Fn()) { + RING.consume(len); + core::sync::atomic::fence(Ordering::SeqCst); + let needs = MAIN_NEEDS.load(Ordering::SeqCst) as usize; + if needs != 0 && RING.room() >= needs { + wake_main(); + } + } + + /// The launcher's stream is gone: drop everything queued, and let every + /// writer see it. + fn lose(endpoint: &Endpoint, wake_main: &dyn Fn()) { + LOST.store(true, Ordering::SeqCst); + endpoint.held.borrow_mut().clear(); + endpoint.sent.set(0); + RING.discard(); + core::sync::atomic::fence(Ordering::SeqCst); + if MAIN_NEEDS.load(Ordering::SeqCst) != 0 { + wake_main(); + } + } + + /// How long a rotating notice receiver's handoff is given before the park + /// is prepared again: the WASI host's figure. + const HANDOFF_NS: u64 = 100_000; + + /// Wait until the launcher has room for one more record. `false` once its + /// stream is gone. The endpoint is borrowed to prepare and settle each + /// park, never across the wait (the WASI host's shape). + async fn room(endpoint: &Endpoint) -> bool { + loop { + let step = { + let mut stdio = endpoint.stdio.borrow_mut(); + // The drain snapshot comes before the capacity check, so room + // the launcher makes between them is seen by the park's + // recheck. + let drain = stdio.begin_drain(); + if stdio.output_capacity_left() > 0 { + return true; + } + match stdio.prepare_park(&drain, ProcessStdioWaitFor::Capacity) { + Ok(step) => step, + Err(_) => return false, + } + }; + match step { + ProcessStdioParkStep::Changed => {} + ProcessStdioParkStep::Handoff => { + let _ = executor::next_wake_for(HANDOFF_NS).await; + } + ProcessStdioParkStep::Park(ticket) => { + let reason = executor::next_wake_on(ticket.post()).await; + if endpoint + .stdio + .borrow_mut() + .settle_park(ticket, reason) + .is_err() + { + return false; + } + } + } + } + } + + /// One pass: what the ring held when it began, and the host's writes + /// held by then, in order. Bounded on both, so the host serves the + /// program between passes: a filesystem step's callbacks hold writes + /// while a pass waits for the launcher's room, and those wait for the + /// next pass. Cancellation-safe at every wait: an item leaves its queue + /// only once all of it is published. + pub(crate) async fn drain(endpoint: &Endpoint, wake_main: &dyn Fn()) { + let end = RING.tail(); + let mut held_left = endpoint.held.borrow().len(); + let mut record = [0u8; RECORD_PAYLOAD]; + loop { + if LOST.load(Ordering::Acquire) { + lose(endpoint, wake_main); + return; + } + let sent = match next(endpoint, end, &mut record) { + // A write held after the pass began: the next pass's, and so + // is everything after it. + Next::Held(..) if held_left == 0 => return, + Next::Held(stream, len) => { + let fill = |from: usize, out: &mut [u8]| held_bytes(endpoint, from, out); + let sent = send(endpoint, stream, len, &fill).await; + if sent { + endpoint.held.borrow_mut().pop_front(); + held_left -= 1; + } + sent + } + Next::Record(stream, len) => { + let bytes = record; + let fill = |from: usize, out: &mut [u8]| { + out.copy_from_slice(&bytes[from..from + out.len()]); + out.len() + }; + let sent = send(endpoint, stream, len, &fill).await; + if sent { + consumed(len, wake_main); + } + sent + } + Next::None => return, + }; + if !sent { + lose(endpoint, wake_main); + return; + } + } + } + + /// The host arms itself before it rechecks what it would park for: a + /// writer that published before this reads it and kicks, and one that + /// publishes after is seen by the recheck (the Dekker pair with + /// [`Ring::publish`]'s `SeqCst` store and the writer's swap). + pub(crate) fn arm() { + ARMED.store(true, Ordering::SeqCst); + core::sync::atomic::fence(Ordering::SeqCst); + } + + /// The program's end on the host's loop: drain what was queued and close + /// the stream. A write from here on reports `EOF`. + pub(crate) async fn finish(endpoint: &Endpoint, wake_main: &dyn Fn()) { + FINISHED.store(true, Ordering::SeqCst); + drain(endpoint, wake_main).await; + if LOST.load(Ordering::Acquire) || !room(endpoint).await { + return; + } + let _ = endpoint + .stdio + .borrow_mut() + .send_output_now(ProcessOutputEvent::closed()); + } + + /// The program's end on the host itself, which cannot wait (a callback's + /// `exit` or `abort`, a panic there): relay at once what the launcher has + /// room for, and close the stream if all of it went. A write from here on + /// reports `EOF`. + pub(crate) fn flush_now() { + let Some(endpoint) = host() else { + return; + }; + if closed() { + return; + } + FINISHED.store(true, Ordering::SeqCst); + // A borrow still held (a panic inside the drain itself) leaves the + // stream as it is rather than panic again. + if endpoint.stdio.try_borrow_mut().is_err() || endpoint.held.try_borrow_mut().is_err() { + return; + } + let end = RING.tail(); + let mut record = [0u8; RECORD_PAYLOAD]; + loop { + let pumped = match next(endpoint, end, &mut record) { + Next::Held(stream, len) => { + let fill = |from: usize, out: &mut [u8]| held_bytes(endpoint, from, out); + let pumped = pump(endpoint, stream, len, &fill); + if matches!(pumped, Pumped::Done) { + endpoint.held.borrow_mut().pop_front(); + } + pumped + } + Next::Record(stream, len) => { + let bytes = record; + let fill = |from: usize, out: &mut [u8]| { + out.copy_from_slice(&bytes[from..from + out.len()]); + out.len() + }; + let pumped = pump(endpoint, stream, len, &fill); + if matches!(pumped, Pumped::Done) { + RING.consume(len); + } + pumped + } + Next::None => break, + }; + if !matches!(pumped, Pumped::Done) { + return; + } + } + let mut stdio = endpoint.stdio.borrow_mut(); + if stdio.output_capacity_left() > 0 { + let _ = stdio.send_output_now(ProcessOutputEvent::closed()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn drain(ring: &Ring) -> std::vec::Vec<(Stream, std::vec::Vec)> { + let mut out = std::vec::Vec::new(); + let mut record = [0u8; RECORD_PAYLOAD]; + while let Some((stream, len)) = ring.peek(&mut record) { + out.push((stream, record[..len].to_vec())); + ring.consume(len); + } + out + } + + /// A write longer than a record is framed into records of at most a + /// record's payload, in order, and the ring accounts every header. + #[test] + fn a_write_is_framed_into_records_and_read_back_in_order() { + let ring = Ring::<1024>::new(); + let bytes: std::vec::Vec = (0..600u32).map(|i| i as u8).collect(); + assert!(ring.fits(bytes.len())); + unsafe { ring.publish(Stream::Out, &bytes) }; + assert_eq!(ring.queued(), framed(600)); + assert_eq!(framed(600), 600 + 3 * HEADER); + unsafe { ring.publish(Stream::Err, b"oops") }; + let records = drain(&ring); + let joined: std::vec::Vec = records + .iter() + .filter(|(stream, _)| *stream == Stream::Out) + .flat_map(|(_, bytes)| bytes.iter().copied()) + .collect(); + assert_eq!(joined, bytes); + assert_eq!(records.len(), 4); + assert_eq!(records[3], (Stream::Err, b"oops".to_vec())); + assert!( + records + .iter() + .all(|(_, bytes)| bytes.len() <= RECORD_PAYLOAD) + ); + assert_eq!(ring.queued(), 0); + } + + /// The positions run free across many laps and past `u32::MAX`; every + /// record still reads back whole, and room is exactly what is unconsumed. + #[test] + fn the_ring_wraps_its_bytes_and_its_positions() { + let ring = Ring::<64>::new(); + ring.head.0.store(u32::MAX - 20, Ordering::Relaxed); + ring.tail.0.store(u32::MAX - 20, Ordering::Relaxed); + for round in 0..200u32 { + let payload: std::vec::Vec = + (0..(round % 40) + 1).map(|i| (i + round) as u8).collect(); + assert!(ring.fits(payload.len()), "round {round}"); + unsafe { ring.publish(Stream::Out, &payload) }; + assert!(!ring.fits(64), "a full lap never fits"); + let records = drain(&ring); + let joined: std::vec::Vec = records.into_iter().flat_map(|(_, b)| b).collect(); + assert_eq!(joined, payload, "round {round}"); + } + } + + /// Room is the framed size: a write fits only while its records and + /// their headers do. + #[test] + fn room_counts_headers() { + let ring = Ring::<16>::new(); + assert!(ring.fits(14)); + assert!(!ring.fits(15)); + unsafe { ring.publish(Stream::Out, &[7; 10]) }; + assert!(ring.fits(2)); + assert!(!ring.fits(3)); + } + + /// A cursor read out of order — the host's moved past a `tail` read + /// before it — never reads as room the ring does not have. + #[test] + fn a_stale_read_never_finds_room_that_is_not_there() { + let ring = Ring::<16>::new(); + // A reader that saw `tail` = 0, then a `head` the host has advanced + // to 10, would compute a wrapped, enormous backlog. + ring.tail.0.store(0, Ordering::Relaxed); + ring.head.0.store(10, Ordering::Relaxed); + assert!(!ring.fits(1)); + // The real order, head first, is exact. + ring.tail.0.store(14, Ordering::Relaxed); + assert_eq!(ring.queued(), 4); + assert!(ring.fits(10)); + } + + /// Discarding consumes every published record at once. + #[test] + fn discard_drops_everything_published() { + let ring = Ring::<64>::new(); + unsafe { ring.publish(Stream::Out, b"lost") }; + unsafe { ring.publish(Stream::Err, b"too") }; + ring.discard(); + assert_eq!(ring.queued(), 0); + assert!(drain(&ring).is_empty()); + } + + /// The largest piece kept whole leaves room for another queued ahead of + /// it, whatever was framed in between. + #[test] + fn a_whole_write_fits_half_the_ring() { + assert!(framed(WHOLE_WRITE) <= RING_BYTES / 2); + assert!(framed(WHOLE_WRITE + RECORD_PAYLOAD) > RING_BYTES / 2); + } + + /// The host's held writes go after the ring records that preceded them + /// and before the ones that followed. + #[test] + fn a_held_write_drains_after_what_preceded_it() { + let ring = Ring::<256>::new(); + unsafe { ring.publish(Stream::Out, b"a") }; + let stamp = ring.tail(); + unsafe { ring.publish(Stream::Out, b"b") }; + let mut record = [0u8; RECORD_PAYLOAD]; + assert!(!held_is_next(stamp, ring.head().load(Ordering::Relaxed))); + let (_, len) = ring.peek(&mut record).unwrap(); + assert_eq!(&record[..len], b"a"); + ring.consume(len); + assert!(held_is_next(stamp, ring.head().load(Ordering::Relaxed))); + } + + /// A write stamped where a call began goes at once if the cursor already + /// passed that point, and never if the point is still ahead, across the + /// positions' wrap. + #[test] + fn a_held_write_behind_the_cursor_goes_at_once() { + for base in [0u32, u32::MAX - 3] { + let head = base.wrapping_add(10); + assert!(held_is_next(base, head), "passed at {base}"); + assert!(held_is_next(head, head), "reached at {base}"); + assert!(!held_is_next(head.wrapping_add(1), head), "ahead at {base}"); + } + } +} diff --git a/docs/languages.md b/docs/languages.md index 353a92f..33cc6eb 100644 --- a/docs/languages.md +++ b/docs/languages.md @@ -72,11 +72,18 @@ one-line `cdylib` named `kore` over it, linked with `--soname=libkore.so --no-undefined` (Kore's `user/runtime/libkore`). It ships the library beside the headers, which `kore-c` publishes to that crate's build script as `DEP_KORE_INCLUDE`, so they come from the same revision. A program links it with -`-lkore` and gets: +`-lkore` — Kore's TCC links it into every program, its `-lc` reading a script +that joins `libkore.so` to the program's own `libc.a` — and gets: - every `kore.h` function, and the C names `kore/stdlib.h` and `kore/pthread.h` alias as real symbols (`c-stdlib`), over the process heap `kore-c` installs on the launch's `HEAP` supply; +- the rest of the C library the program's headers declare (Kore's hosted + ``, ``, `` and ``, beside TCC): + `kore-c-shim`'s `printf` family, every conversion C has but the wide ones and + every float digit exact, `stdout` and `stderr`, the `strto*` parsers, + `qsort`, `strerror`, the string routines and a per-thread `errno`, and + `kore-c`'s own `abort`; - a host on thread 0, entered from the C library's start; - `kore_fs_run(step, user, &error)` (`kore/program.h`), which runs a `KoreFsStep` program as the program's filesystem session. @@ -92,9 +99,28 @@ one word. Every park is a doorbell park; nothing spins. at the directory the program was launched in, and runs `kore-ffi`'s `Session` on its executor. The callbacks run there, as on any native executor, while the caller waits: they must not depend on its - thread-local state. A thread that cannot park (any but the main thread) + thread-local state, nor wait for another program thread — while one runs + the host serves nothing else, the streams included. A thread that cannot park (any but the main thread) is refused, as is a launch with no face. A quarantined session can never run again. +- **The streams.** `stdout` and `stderr` reach the launch's interactive + stream (`PROFILE_INTERACTIVE`), the terminal the program was run from, + which only thread 0 may publish into. Any thread's write goes whole into + one in-process ring under a futex lock, which a stream call holds across + all its writes (POSIX's `flockfile`): the holder is the ring's one + producer, so one call's output is never split by another thread's, and it + kicks the host only if the host armed itself before parking. Every wait + for room is bounded by the runtime's slice: the host kicks the main thread + when its room is free, and a pthread, which the host cannot reach, + re-reads the ring each slice. The host drains the + ring into the stream's records as the launcher has room, parks on the + stream's capacity when it has none — a slow terminal holds the program + back rather than losing its output — and wakes the writers waiting for + space, a pass at a time, serving the program's end and its sessions + between passes. The host's own writes (a filesystem step's callbacks) are + kept in order beside the ring, before any call another thread has in + progress. A launch without the stream has no sink, and a + write to either stream returns `EOF`. - **The end.** The C library's `exit` and `_Exit` call `__kore_program_exit(status, finalize)` after their `atexit` handlers, on whatever thread called them. The first request wins and reaches the host, @@ -110,6 +136,13 @@ one word. Every park is a doorbell park; nothing spins. - An end requested mid-session does not wait for the session. - A panic in the library ends the program the same way, its record published first. + - `abort` takes `_Exit`'s path with the aborted status, 134, what a shell + reports for a program `SIGABRT` ended: no `atexit` handler runs. + - An end the host's loop serves drains the ring and closes the stream + before the status, so the launcher shows all of the program's output + first. An end committed on the host itself (a callback's `exit` or + `abort`, a panic there) cannot wait: it relays what the launcher has room + for, and closes the stream if that was all of it. - **Without a host.** A launch that cannot carry one (no channel space or heap) runs the program on thread 0, as its C library alone would. Then the filesystem session is refused, and only thread 0's exit ends the program. @@ -125,10 +158,13 @@ and checks it: - its soname, nothing needed or undefined; - static initial-exec TLS only; - exactly the declared ABI exported: `kore.h`, `kore/program.h`, the - `c-stdlib` names and the two lifecycle entry points. + `c-stdlib` names, the two lifecycle entry points, and the C library names + Kore's hosted headers declare (`C_LIBRARY`). Kore's `make keytest-tcc-x86` compiles a C program against these headers at -Kore's shell, links `libkore.so`, and runs it through every surface above +Kore's shell, links `libkore.so`, and runs it through every surface above. It +also compiles Kore's `hello.c`, written against the standard headers alone, +reads every line it prints off the terminal, and ends a program by `abort` (Kore `res/TCC.md` §10). That is boot evidence; this repository's checks are compiler and link evidence. Task preparation (`kore_task_*`) still needs a Rust host to admit its batch: the independent host does not admit one. diff --git a/tools/check-program.py b/tools/check-program.py index 7fda49e..e52550c 100755 --- a/tools/check-program.py +++ b/tools/check-program.py @@ -12,8 +12,9 @@ - the soname consumers record, nothing it needs, nothing left undefined; - static initial-exec TLS only (Kore's startup linker resolves no dynamic TLS); - exactly the declared ABI exported: every function kore.h and kore/program.h - declare, the C names kore/stdlib.h and kore/pthread.h alias, and the C - library's two lifecycle entry points; no other symbol. + declare, the C names kore/stdlib.h and kore/pthread.h alias, the C + library's two lifecycle entry points, and the C library names Kore's + hosted headers declare and libkore.so implements; no other symbol. """ import argparse import json @@ -36,6 +37,19 @@ '__cxa_thread_atexit', '__cxa_thread_atexit_impl'} # The C library's start and exit call these when a program links libkore. LIFECYCLE = {'__kore_program_start', '__kore_program_exit'} +# The rest of the C library a program's headers declare (Kore's TCC +# `include/`: , , , ), which libkore.so +# implements: kore-c-shim's printf family, streams, parsers, string routines +# and per-thread errno, and kore-c's own abort. The program's libc.a holds the +# rest: exit, atexit and the routines a compiler assumes. +C_LIBRARY = {'printf', 'fprintf', 'sprintf', 'snprintf', 'vprintf', 'vfprintf', 'vsprintf', + 'vsnprintf', 'fputc', 'putc', 'putchar', 'fputs', 'puts', 'fwrite', 'fflush', + 'fclose', 'stdout', 'stderr', + 'strtol', 'strtoll', 'strtoul', 'strtoull', 'atoi', 'atol', 'strtod', 'strtof', + 'qsort', 'strerror', 'abort', + 'memchr', 'strchr', 'strrchr', 'strcmp', 'strncmp', 'strcpy', 'strncpy', 'strncat', + 'strstr', + '__errno_location'} DYNAMIC_TLS = re.compile(r'TLSDESC|DTPMOD|DTPOFF|TLSGD|TLSLD|TLS_DTP') @@ -123,6 +137,6 @@ def check(arch, expected): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument('--arch', choices=ARCHES) args = parser.parse_args() - expected = declared() | C_STDLIB | LIFECYCLE + expected = declared() | C_STDLIB | LIFECYCLE | C_LIBRARY for arch in [args.arch] if args.arch else ARCHES: check(arch, expected) From 23a0ca0eab40fbe09a55cccb98651074be2b951d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 10:09:07 +0000 Subject: [PATCH 12/18] native: re-export at Kore 10c7b400, with the link window's free span `tools/sync-native.py` at MicroPerceptron/kore@10c7b400aa96c7533be05054fb76e43850408b23. kore-rt names the free part of a task's link window once (`Rt::link_free_span`), which a C program's `mmap` places in, and kore-c-shim's `errno` gains `ENXIO`, `EFAULT` and `ENOTSUP`. kore-ld joins the owner paths: its `SharedPlacer` is the one cursor a window's mappings take their addresses from, and kore-c is about to place with it. The export's packages are unchanged, since kore-ld was already in kore-late's closure. `fmt`, `native-check`, `owners` and `bindings-check` pass on this export. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- Cargo.toml | 2 +- native/crates/c-shim/src/errno.rs | 6 +++ native/crates/rt/src/entry.rs | 64 +++++++++++++++++++++++++++++++ native/provenance.json | 6 +-- 4 files changed, 74 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3739aa4..e6ec140 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,4 +35,4 @@ kore-driver = { path = "native/crates/driver", optional = true } # root: the offline volume procedure (and the filesystem service closure it # drives) that host programs such as vKore consume; it never enters a Kore # executable's graph. -owner-paths = { kore-start = "crates/start", kore-abi = "crates/abi", kore-app = "crates/app", kore-rt = "crates/rt", kore-c-shim = "crates/c-shim", kore-dagfs = "crates/dagfs", kore-driver = "crates/driver", kore-late = "crates/late", kore-vol = "tools/kore-vol" } +owner-paths = { kore-start = "crates/start", kore-abi = "crates/abi", kore-app = "crates/app", kore-rt = "crates/rt", kore-c-shim = "crates/c-shim", kore-dagfs = "crates/dagfs", kore-driver = "crates/driver", kore-late = "crates/late", kore-ld = "crates/ld", kore-vol = "tools/kore-vol" } diff --git a/native/crates/c-shim/src/errno.rs b/native/crates/c-shim/src/errno.rs index 249fc5c..7a5fbf8 100644 --- a/native/crates/c-shim/src/errno.rs +++ b/native/crates/c-shim/src/errno.rs @@ -17,10 +17,12 @@ use core::ffi::{CStr, c_int}; pub const EPERM: c_int = 1; pub const ENOENT: c_int = 2; pub const EIO: c_int = 5; +pub const ENXIO: c_int = 6; pub const EBADF: c_int = 9; pub const EAGAIN: c_int = 11; pub const ENOMEM: c_int = 12; pub const EACCES: c_int = 13; +pub const EFAULT: c_int = 14; pub const EBUSY: c_int = 16; pub const EEXIST: c_int = 17; pub const ENOTDIR: c_int = 20; @@ -38,6 +40,7 @@ pub const ENOSYS: c_int = 38; pub const ENOTEMPTY: c_int = 39; pub const ELOOP: c_int = 40; pub const EOVERFLOW: c_int = 75; +pub const ENOTSUP: c_int = 95; /// Set this thread's `errno`. pub fn set(code: c_int) { @@ -64,10 +67,12 @@ pub fn describe(code: c_int) -> &'static CStr { EPERM => c"Operation not permitted", ENOENT => c"No such file or directory", EIO => c"I/O error", + ENXIO => c"No such device or address", EBADF => c"Bad file descriptor", EAGAIN => c"Resource temporarily unavailable", ENOMEM => c"Out of memory", EACCES => c"Permission denied", + EFAULT => c"Bad address", EBUSY => c"Resource busy", EEXIST => c"File exists", ENOTDIR => c"Not a directory", @@ -85,6 +90,7 @@ pub fn describe(code: c_int) -> &'static CStr { ENOTEMPTY => c"Directory not empty", ELOOP => c"Symbolic link loop", EOVERFLOW => c"Value too large for data type", + ENOTSUP => c"Not supported", _ => c"Unknown error", } } diff --git a/native/crates/rt/src/entry.rs b/native/crates/rt/src/entry.rs index babb71e..00b9b80 100644 --- a/native/crates/rt/src/entry.rs +++ b/native/crates/rt/src/entry.rs @@ -176,6 +176,28 @@ const PROCESS_STDIO_ATTACHMENT: usize = 1 << 1; #[cfg(any(test, target_os = "none"))] const PROCESS_STDIO_HOST_ATTACHMENT: usize = 1 << 2; +/// The free part of a task's link window ([`Rt::link_free_span`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LinkSpan { + /// The task's whole link window. + pub base: u64, + /// Its end below the module page. + pub limit: u64, + /// The first free address: above everything the startup linker placed. + pub free: u64, +} + +/// Why a task has no free link span ([`Rt::link_free_span`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LinkSpanError { + /// The task has no `geom::LINK` window. + NoWindow, + /// The window has no module page. + NoModulePage, + /// Nothing free above what the startup linker placed. + Full, +} + /// Atomically spend one runtime attachment name. /// /// This is deliberately a bit in the process-root @@ -255,6 +277,48 @@ impl Rt { None } + /// **The free part of this task's link window** (`geom::LINK`): where a + /// program may place mappings of its own after startup — a Wasmtime + /// engine's guest mappings, a C program's `mmap`. + /// + /// The floor is the larger of the module page's two marks: the recorded + /// segments' top ([`ModulePage::high_water`]) and how far the startup + /// placement reached ([`ModulePage::placed_high_water`]: the copied + /// template views and the TLS block, which no record names). Neither + /// alone is the floor. The limit is the module page itself, the + /// window's last page. + /// + /// This names a span and claims nothing. **One placer per window**: a + /// program places everything it maps there from one cursor + /// (`kore_ld::place::SharedPlacer`), and one that also links native + /// modules late (`kore-late`) partitions the span between the two + /// itself, since two cursors over one window hand out each other's + /// addresses. + /// + /// [`ModulePage::high_water`]: kore_abi::image::ModulePage::high_water + /// [`ModulePage::placed_high_water`]: kore_abi::image::ModulePage::placed_high_water + pub fn link_free_span(&self) -> Result { + let (base, len) = self + .geom() + .and_then(|geom| geom.window(kore_abi::task::geom::LINK)) + .ok_or(LinkSpanError::NoWindow)?; + let limit = + kore_abi::image::module_page_va(base, len).ok_or(LinkSpanError::NoModulePage)?; + let page = self.modules(); + let free = page + .and_then(|page| page.high_water()) + .unwrap_or(base) + .max( + page.and_then(|page| page.placed_high_water(limit)) + .unwrap_or(base), + ) + .max(base); + if free >= limit { + return Err(LinkSpanError::Full); + } + Ok(LinkSpan { base, limit, free }) + } + /// The `.kore.geom` window table, if the founder stamped it — where the /// task learns its founder-mapped `(va, len)` windows (scratch, slots, …) /// instead of hardcoded constants (`res/USERLAND.md` §4). `None` for a diff --git a/native/provenance.json b/native/provenance.json index c084f4f..bbbd22f 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "2d4998baa8be794191fdacaf63dffca295c2a109", + "revision": "10c7b400aa96c7533be05054fb76e43850408b23", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", @@ -71,7 +71,7 @@ "crates/app/src/lib.rs": "5e7e8d791236f278ee85a02a3c37f41412d879cc41d67f6f6d3d701811973d7d", "crates/c-shim/Cargo.toml": "85a4ad7f806ec588631dda39ef312513282217f3b0ddcdeeb3b5cbf6c1223685", "crates/c-shim/src/alloc_symbols.rs": "e42b67a45271e7af676425cc915655e95153ee1f198864f5f527541e9c7092d8", - "crates/c-shim/src/errno.rs": "522a18b3561753cec41464a13ea978c5ea25b57cebde35363f4088ac074b1f38", + "crates/c-shim/src/errno.rs": "3fe0a17c84d6678f193e63e2db3fad89fef843cdd3bec348e064412b1d730352", "crates/c-shim/src/format.rs": "d49d93e313f1eb0997deb40cd68942e8987152292c42feaf37546a883d21f4a9", "crates/c-shim/src/format/float.rs": "eca54620399362cb35d09c842b7f3e40bef6af497c1a5b808df7d9e55c91b413", "crates/c-shim/src/format/float/tests.rs": "8d579334d89afedcc9945c231a713e36b0baf5e99c5eb8a72d134100068dce9c", @@ -220,7 +220,7 @@ "crates/rt/src/clock/x86_64.rs": "ca499d4dc51b7ed0eb196b831295b5fdecb09a1bd2df628a82e749e27011994a", "crates/rt/src/core_id.rs": "cadde39801a1d0ed05d70d0634a9e52d58043878737d790c4130183751b622cf", "crates/rt/src/crash.rs": "b6463a490f80124e22fdb5e13652bbaa94461eb1c4aefcffeca2e7dc7e977611", - "crates/rt/src/entry.rs": "e76b7b7014a4c0b7859091c93fc4a8fadfd04690307c5021bfdaf76b8dc34b06", + "crates/rt/src/entry.rs": "499fd1ac8c1161ceb1241f093f167d7fcb8a5ef0405e5b43dc8e7293942058f3", "crates/rt/src/event.rs": "709a5500231735ad5bdb9ab9ab2fcee7b8f72f93e1b4741afba72fdc8b7c54f3", "crates/rt/src/executor/mod.rs": "071e3dc66bca448e6dabf96976702600bc77fdc98933d58847662fb2de073cb8", "crates/rt/src/executor/task.rs": "b3017cb97f287b8796c61d245c49e5c1b0ebfabe8c535cc2b3842cdbc854dc29", From c876e4386b81f21863a6838682f61f737d4bb8fc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 10:09:27 +0000 Subject: [PATCH 13/18] c: descriptors over the capabilities a C program holds, and mmap `open`, `close`, `mmap` and `munmap` are C's spelling for using a capability the program was founded holding. They add no authority and no policy: int fd = open("/kore/memory", O_RDWR); char *region = mmap(NULL, 1 << 20, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - The path is not a file. `open` looks it up in one static table whose rows are a name and the bootstrap coordinate (role, index) it stands for, and succeeds only for a capability the calling thread holds, with access its rights cover. The first row is `/kore/memory`, the memory supply the allocator grows on. - A descriptor stores the coordinate, never the capability. Capabilities are per thread on Kore, so each use resolves the coordinate in the calling thread's boot block: numbers are the process's, and no authority crosses a thread through one. `close` forgets the number. - `mmap` on the supply answers fresh, zeroed memory, as mapping `/dev/zero` does elsewhere, by the sequence a Wasmtime engine's guest memory takes: a lease from the supply's top (clear of the allocator), an address from the free part of the link window, the whole lease mapped there. The caller never picks the address, and nothing a program wrote is executable. - Thread 0 carves and maps for whichever thread asked, over one request cell, as it serves a filesystem session. The share of the supply a thread is founded with carries no `REVOKE`, and a region that can be unmapped again is a carve its holder may revoke; the supply's root is thread 0's. A mapping is therefore the process's, and any thread may `munmap` it. - `munmap` removes the mapping and parks the region. The supply's and the window's cursors only advance, so the program keeps the lease and its address, and the next `mmap` of that length takes it, zeroed again. One size mapped and unmapped in a loop uses one region. - `kore_fd_describe` (`kore/program.h`) reads a descriptor's kind, what its capability's rights permit, and a supply's length. The streams' futex lock moves to `lock.rs`, which the cell's callers share. The table and the request rules are plain data, tested on the host. The POSIX names are exported on Kore alone, and `tools/check-program.py` holds libkore.so's exports to the declared ABI with them. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- Cargo.lock | 1 + crates/c/Cargo.toml | 3 + crates/c/cbindgen.toml | 4 +- crates/c/src/api.rs | 115 ++++++ crates/c/src/fd.rs | 405 +++++++++++++++++++ crates/c/src/lib.rs | 13 +- crates/c/src/lock.rs | 54 +++ crates/c/src/mmap.rs | 684 ++++++++++++++++++++++++++++++++ crates/c/src/posix.rs | 94 +++++ crates/c/src/program.rs | 34 +- crates/c/src/stdio.rs | 38 +- crates/c/tests/program.c | 16 +- docs/languages.md | 66 +++ extern/c/include/kore/program.h | 58 +++ tools/check-program.py | 9 +- 15 files changed, 1551 insertions(+), 43 deletions(-) create mode 100644 crates/c/src/fd.rs create mode 100644 crates/c/src/lock.rs create mode 100644 crates/c/src/mmap.rs create mode 100644 crates/c/src/posix.rs diff --git a/Cargo.lock b/Cargo.lock index 6a0168b..beb52d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -808,6 +808,7 @@ version = "0.1.0-alpha.1" dependencies = [ "kore-c-shim", "kore-ffi", + "kore-ld", "kore-std", ] diff --git a/crates/c/Cargo.toml b/crates/c/Cargo.toml index f9434dc..2188bbc 100644 --- a/crates/c/Cargo.toml +++ b/crates/c/Cargo.toml @@ -28,3 +28,6 @@ kore-ffi = { path = "../ffi", features = ["c-stdlib"] } # `malloc` and `abort`: kore-ffi's allocator is this image's `malloc`, and the # host ends a whole program. kore-c-shim = { path = "../../native/crates/c-shim", default-features = false, features = ["runtime"] } +# The link window's placer: one cursor every `mmap` of the process takes its +# address from, as a Wasmtime engine's guest mappings do. +kore-ld = { path = "../../native/crates/ld", default-features = false } diff --git a/crates/c/cbindgen.toml b/crates/c/cbindgen.toml index 5f64a0d..e241550 100644 --- a/crates/c/cbindgen.toml +++ b/crates/c/cbindgen.toml @@ -15,7 +15,9 @@ parse_deps = false # these), not an application API: the C library declares them itself, as # its declares `abort`. `KoreFsStep` is kore.h's typedef, which # this header includes. -exclude = ["__kore_program_start", "__kore_program_exit", "abort", "KoreFsStep"] +# `open`, `close`, `mmap` and `munmap` are POSIX's, declared by , +# and . +exclude = ["__kore_program_start", "__kore_program_exit", "abort", "open", "close", "mmap", "munmap", "KoreFsStep"] [enum] prefix_with_name = true rename_variants = "ScreamingSnakeCase" diff --git a/crates/c/src/api.rs b/crates/c/src/api.rs index 8fa4fc8..aa16bb3 100644 --- a/crates/c/src/api.rs +++ b/crates/c/src/api.rs @@ -68,6 +68,90 @@ pub unsafe extern "C" fn kore_fs_run( status } +/// [`KoreFdInfo::kind`]: the memory supply the program's allocator grows +/// on. `mmap` answers a fresh, zeroed region of it. +pub const KORE_FD_MEMORY: u32 = 1; +/// [`KoreFdInfo::access`]: the capability may be read. +pub const KORE_FD_READ: u32 = 1 << 0; +/// [`KoreFdInfo::access`]: the capability may be written. +pub const KORE_FD_WRITE: u32 = 1 << 1; + +/// What a descriptor names, as the calling thread holds it +/// ([`kore_fd_describe`]). +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct KoreFdInfo { + /// What kind of capability it is: `KORE_FD_MEMORY`. + pub kind: u32, + /// What the capability's own rights permit this thread, as + /// `KORE_FD_READ | KORE_FD_WRITE` bits: the rights it was founded with, + /// which the access `open` asked for can only narrow. + pub access: u32, + /// For `KORE_FD_MEMORY`, the supply's whole length in bytes: what the + /// allocator and every mapping share, not what is still free. + pub length: u64, +} + +/// Describe the capability behind `fd`, a descriptor `open` answered +/// (``): `open` finds a capability this program was founded holding +/// by its name, and this reads what was found. Nothing is granted or changed. +/// +/// Capabilities are held per thread, so the answer is the calling thread's: +/// the same descriptor may describe a capability on the thread that opened +/// it and be refused on one founded without it. +/// +/// Returns `KORE_STATUS_OK` with `*out` written. +/// `KORE_STATUS_INVALID_ARGUMENT`: `out` is null. `KORE_STATUS_STALE_HANDLE`: +/// `fd` is not an open descriptor. `KORE_STATUS_REFUSED`: the calling thread +/// holds no such capability, or this is not a Kore program. A refusal leaves +/// `*out` unchanged. +/// +/// # Safety +/// `out` is null or valid for one aligned write. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn kore_fd_describe( + fd: core::ffi::c_int, + out: *mut KoreFdInfo, +) -> KoreStatus { + if out.is_null() { + return KoreStatus::InvalidArgument; + } + let Ok((row, _)) = crate::fd::TABLE.get(fd) else { + return KoreStatus::StaleHandle; + }; + let Some(info) = describe(&crate::fd::NAMES[row]) else { + return KoreStatus::Refused; + }; + // SAFETY: the caller's contract: valid for one aligned write. + unsafe { out.write(info) }; + KoreStatus::Ok +} + +/// What the calling thread holds at `name`. +#[cfg(target_os = "none")] +fn describe(name: &crate::fd::Name) -> Option { + use crate::fd::Kind; + let held = crate::fd::holding(name)?; + let (kind, length) = match name.kind { + Kind::Memory => ( + KORE_FD_MEMORY, + kore_std::runtime::cap::verbs::lease_geometry(held.slot)?.1, + ), + }; + Some(KoreFdInfo { + kind, + access: if held.readable { KORE_FD_READ } else { 0 } + | if held.writable { KORE_FD_WRITE } else { 0 }, + length, + }) +} + +/// Off Kore no thread holds a capability. +#[cfg(not(target_os = "none"))] +fn describe(_name: &crate::fd::Name) -> Option { + None +} + #[cfg(test)] mod tests { use super::*; @@ -102,4 +186,35 @@ mod tests { unsafe { kore_fs_run(Some(never), core::ptr::null_mut(), core::ptr::null_mut()) }; assert_eq!(status, KoreStatus::Refused); } + + #[test] + fn describing_refuses_by_cause_and_leaves_the_answer_alone() { + let untouched = KoreFdInfo { + kind: 7, + access: 7, + length: 7, + }; + let mut info = untouched; + // SAFETY: null is permitted; `info` is valid. + unsafe { + assert_eq!( + kore_fd_describe(3, core::ptr::null_mut()), + KoreStatus::InvalidArgument + ); + // The streams' numbers and an unopened one are not descriptors. + for fd in [0, 1, 2, -1, 63, 1 << 20] { + assert_eq!(kore_fd_describe(fd, &mut info), KoreStatus::StaleHandle); + } + } + // An open descriptor, off Kore: no thread holds a capability. + let access = crate::fd::Access::of(crate::fd::O_RDWR).unwrap(); + let fd = crate::fd::TABLE.claim(0, access).unwrap(); + // SAFETY: `info` is valid. + assert_eq!( + unsafe { kore_fd_describe(fd, &mut info) }, + KoreStatus::Refused + ); + crate::fd::TABLE.release(fd).unwrap(); + assert_eq!(info, untouched); + } } diff --git a/crates/c/src/fd.rs b/crates/c/src/fd.rs new file mode 100644 index 0000000..91d271f --- /dev/null +++ b/crates/c/src/fd.rs @@ -0,0 +1,405 @@ +//! Descriptors: C's spelling for "a capability I hold". +//! +//! A program's authority arrives as capabilities at fixed places: its boot +//! block names each by a role and an index, and the capability itself sits in +//! the thread's arena. `open` is the readable way to say which one is meant. +//! It grants nothing and reaches no filesystem: +//! +//! - **A name is a row of one static table** ([`NAMES`]): a path-shaped +//! string, the bootstrap coordinate it stands for, and what kind of +//! capability lives there. A path that is no row is `ENOENT`. +//! - **A descriptor stores the row, never the capability.** Capabilities are +//! held per thread on Kore, and a thread is a task: `main` runs on one, a +//! `pthread` on another, and each has its own boot block. So every use of a +//! descriptor resolves the row's coordinate in the *calling* thread's boot +//! block. Descriptor numbers are the process's, as C expects, and no +//! authority crosses a thread through one: a thread that was not founded +//! holding the capability gets `ENXIO`, whoever opened the descriptor. +//! - **`close` forgets the number.** The capability was the program's before +//! `open` and is after `close`; nothing is revoked. +//! +//! Descriptors 0, 1 and 2 are the standard streams' numbers and are never +//! handed out; the streams themselves are ``'s. +//! +//! The table is plain data over atomics, so the host suite exercises the code +//! the target runs. Finding a row's capability in the boot block is the metal +//! half ([`holding`]). + +use core::ffi::c_int; +use core::sync::atomic::{AtomicU32, Ordering}; + +use kore_std::abi::task::role; + +/// `errno` values this module and [`crate::mmap`] report: musl's numbers, +/// which kore-c-shim's `errno` and the hosted `` share. +pub(crate) mod errno { + use core::ffi::c_int; + pub(crate) const ENOENT: c_int = 2; + pub(crate) const ENXIO: c_int = 6; + pub(crate) const EBADF: c_int = 9; + pub(crate) const EAGAIN: c_int = 11; + pub(crate) const ENOMEM: c_int = 12; + pub(crate) const EACCES: c_int = 13; + pub(crate) const EFAULT: c_int = 14; + pub(crate) const EINVAL: c_int = 22; + pub(crate) const EMFILE: c_int = 24; + pub(crate) const ENOTSUP: c_int = 95; +} + +/// What kind of capability a name stands for: which operations its +/// descriptor has. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kind { + /// The memory supply the program's allocator grows on. `mmap` takes a + /// region off it, through the runtime's own reservation so the two never + /// meet ([`crate::mmap`]). + Memory, +} + +/// One row: a name and the bootstrap coordinate it stands for. +#[derive(Debug)] +pub(crate) struct Name { + /// The path `open` matches, byte for byte. + pub(crate) path: &'static [u8], + /// The boot block role of the capability. + pub(crate) role: u16, + /// The instance within that role. + pub(crate) index: u16, + /// What lives there. + pub(crate) kind: Kind, +} + +/// **The names.** Each is a coordinate every thread that holds the +/// capability finds it under, so a row is true on whichever thread asks. +/// +/// `/kore/memory` is `role::HEAP@0`: the launch's memory supply, which +/// kore-rt endows every thread that may allocate with a share of +/// (`WorkerCap::heap`). +pub(crate) static NAMES: [Name; 1] = [Name { + path: b"/kore/memory", + role: role::HEAP, + index: 0, + kind: Kind::Memory, +}]; + +/// The row `path` names. +pub(crate) fn resolve(path: &[u8]) -> Option { + NAMES.iter().position(|name| name.path == path) +} + +/// What an `open` asked to do with the capability. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Access { + pub(crate) read: bool, + pub(crate) write: bool, +} + +/// ``'s access modes, musl's values. They are the only flags `open` +/// takes: every other flag describes a file this namespace does not have. +pub(crate) const O_RDONLY: c_int = 0; +pub(crate) const O_WRONLY: c_int = 1; +pub(crate) const O_RDWR: c_int = 2; + +impl Access { + /// The access `flags` ask for, or `EINVAL` for any flag but an access mode. + pub(crate) fn of(flags: c_int) -> Result { + match flags { + O_RDONLY => Ok(Self { + read: true, + write: false, + }), + O_WRONLY => Ok(Self { + read: false, + write: true, + }), + O_RDWR => Ok(Self { + read: true, + write: true, + }), + _ => Err(errno::EINVAL), + } + } + + /// Whether a capability that can be read (`ACCESS`) and, if `writable`, + /// written (`MODIFY`) permits this access. + pub(crate) fn within(self, readable: bool, writable: bool) -> bool { + (!self.read || readable) && (!self.write || writable) + } + + const fn bits(self) -> u32 { + self.read as u32 | (self.write as u32) << 1 + } + + const fn from_bits(bits: u32) -> Self { + Self { + read: bits & 1 != 0, + write: bits & 2 != 0, + } + } +} + +/// The first number `open` hands out: 0, 1 and 2 are the standard streams'. +pub(crate) const FIRST: c_int = 3; + +/// How many descriptors a process may hold open. A descriptor is one word +/// naming a row, and the rows are few, so this bounds a program that forgets +/// to `close` rather than one that needs many: sixty-four entries are one +/// 256-byte table, scanned whole only by an `open` that finds it full. +pub(crate) const CAPACITY: usize = 64; + +/// A descriptor table: one word per descriptor, zero when free. +/// +/// A live word is [`LIVE`], the access in two bits, and the row's index. +/// Claiming and releasing are one compare-and-swap each, so threads opening +/// and closing at once never share a lock, and a descriptor is never handed +/// to two of them. +pub(crate) struct Table { + words: [AtomicU32; N], +} + +const LIVE: u32 = 1 << 31; +const ACCESS_SHIFT: u32 = 16; +const ROW_MASK: u32 = (1 << ACCESS_SHIFT) - 1; + +// Every row's index fits its field. +const _: () = assert!(NAMES.len() <= ROW_MASK as usize + 1); + +impl Table { + pub(crate) const fn new() -> Self { + Self { + words: [const { AtomicU32::new(0) }; N], + } + } + + /// Open a descriptor for `row` with `access`: the lowest free number, as + /// POSIX has it, or `EMFILE` when every one is taken. + pub(crate) fn claim(&self, row: usize, access: Access) -> Result { + let word = LIVE | access.bits() << ACCESS_SHIFT | row as u32; + for (at, slot) in self.words.iter().enumerate() { + if slot + .compare_exchange(0, word, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + return Ok(FIRST + at as c_int); + } + } + Err(errno::EMFILE) + } + + /// The row and access behind `fd`, or `EBADF`. + pub(crate) fn get(&self, fd: c_int) -> Result<(usize, Access), c_int> { + let word = self.slot(fd)?.load(Ordering::Acquire); + if word & LIVE == 0 { + return Err(errno::EBADF); + } + Ok(( + (word & ROW_MASK) as usize, + Access::from_bits(word >> ACCESS_SHIFT), + )) + } + + /// Close `fd`, or `EBADF` when it was not open. + pub(crate) fn release(&self, fd: c_int) -> Result<(), c_int> { + match self.slot(fd)?.swap(0, Ordering::AcqRel) { + 0 => Err(errno::EBADF), + _ => Ok(()), + } + } + + fn slot(&self, fd: c_int) -> Result<&AtomicU32, c_int> { + fd.checked_sub(FIRST) + .and_then(|at| usize::try_from(at).ok()) + .and_then(|at| self.words.get(at)) + .ok_or(errno::EBADF) + } +} + +/// The process's descriptors. +pub(crate) static TABLE: Table = Table::new(); + +/// What the calling thread holds at a row's coordinate. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Holding { + /// The capability's arena slot, a name this thread alone may use. + pub(crate) slot: kore_std::abi::session::SlotRef, + /// It carries `ACCESS`. + pub(crate) readable: bool, + /// It carries `MODIFY`. + pub(crate) writable: bool, +} + +/// **The capability `name` stands for, as the calling thread holds it**: +/// its slot from this thread's boot block, and the rights its record +/// carries. `None` when the thread was not founded holding one of that kind +/// there. The record is the kernel's, read through the arena's read-only +/// alias, and every later effect binds the slot again in the kernel, so this +/// is a description and never a grant. +#[cfg(target_os = "none")] +pub(crate) fn holding(name: &Name) -> Option { + use kore_std::abi::cap::{ResourceKind, Rights}; + use kore_std::runtime::cap::verbs; + let slot = kore_std::runtime::entry::thread_context()? + .bootstrap() + .handle(name.role, name.index)?; + let kind = match name.kind { + Kind::Memory => ResourceKind::MemLease, + }; + let rights = verbs::capability_rights(slot, kind)?; + Some(Holding { + slot, + readable: rights.contains(Rights::ACCESS), + writable: rights.contains(Rights::MODIFY), + }) +} + +/// `open`: a descriptor for the capability `path` names, with `flags`' +/// access. +/// +/// `ENOENT`: no such name. `EINVAL`: a flag other than an access mode. +/// `ENXIO`: the calling thread holds no such capability. `EACCES`: it holds +/// one whose rights do not cover the access. `EMFILE`: no descriptor is free. +#[cfg(target_os = "none")] +pub(crate) fn open(path: &[u8], flags: c_int) -> Result { + let row = resolve(path).ok_or(errno::ENOENT)?; + let access = Access::of(flags)?; + let held = holding(&NAMES[row]).ok_or(errno::ENXIO)?; + if !access.within(held.readable, held.writable) { + return Err(errno::EACCES); + } + TABLE.claim(row, access) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_path_is_a_row_or_nothing() { + assert_eq!(resolve(b"/kore/memory"), Some(0)); + for path in [ + &b""[..], + b"/", + b"/kore", + b"/kore/memory/", + b"/kore/memory\0", + b"kore/memory", + b"/KORE/MEMORY", + b"/dev/zero", + ] { + assert_eq!(resolve(path), None, "{:?}", core::str::from_utf8(path)); + } + assert_eq!((NAMES[0].role, NAMES[0].index), (role::HEAP, 0)); + } + + #[test] + fn only_an_access_mode_is_a_flag() { + assert_eq!( + Access::of(O_RDONLY), + Ok(Access { + read: true, + write: false + }) + ); + assert_eq!( + Access::of(O_WRONLY), + Ok(Access { + read: false, + write: true + }) + ); + assert_eq!( + Access::of(O_RDWR), + Ok(Access { + read: true, + write: true + }) + ); + // The access modes are not bits: 3 is no mode, and neither is a mode + // with O_CREAT (0o100), O_TRUNC (0o1000) or O_CLOEXEC (0o2000000). + for flags in [3, O_RDWR | 0o100, O_WRONLY | 0o1000, 0o2000000, -1] { + assert_eq!(Access::of(flags), Err(errno::EINVAL), "{flags:#o}"); + } + } + + #[test] + fn access_is_held_to_the_capabilitys_rights() { + let read = Access::of(O_RDONLY).unwrap(); + let write = Access::of(O_WRONLY).unwrap(); + let both = Access::of(O_RDWR).unwrap(); + // (readable, writable) of the capability. + assert!(read.within(true, false) && read.within(true, true)); + assert!(!read.within(false, true)); + assert!(write.within(false, true) && !write.within(true, false)); + assert!(both.within(true, true)); + assert!(!both.within(true, false) && !both.within(false, true)); + } + + #[test] + fn the_lowest_free_number_is_handed_out_from_three() { + let table = Table::<4>::new(); + let both = Access::of(O_RDWR).unwrap(); + let read = Access::of(O_RDONLY).unwrap(); + assert_eq!(table.claim(0, both), Ok(3)); + assert_eq!(table.claim(0, read), Ok(4)); + assert_eq!(table.claim(0, both), Ok(5)); + assert_eq!(table.get(4), Ok((0, read))); + assert_eq!(table.release(4), Ok(())); + assert_eq!(table.get(4), Err(errno::EBADF)); + // The hole is filled before a higher number is used. + assert_eq!(table.claim(0, both), Ok(4)); + assert_eq!(table.claim(0, both), Ok(6)); + assert_eq!(table.claim(0, both), Err(errno::EMFILE)); + assert_eq!(table.get(6), Ok((0, both))); + } + + #[test] + fn a_number_that_is_not_open_is_ebadf() { + let table = Table::<4>::new(); + // The streams' numbers, a negative, one past the table, and a free one. + for fd in [0, 1, 2, -1, c_int::MIN, 7, c_int::MAX, 3] { + assert_eq!(table.get(fd), Err(errno::EBADF), "{fd}"); + assert_eq!(table.release(fd), Err(errno::EBADF), "{fd}"); + } + let fd = table.claim(0, Access::of(O_RDONLY).unwrap()).unwrap(); + assert_eq!(table.release(fd), Ok(())); + // A second close is of a descriptor that is no longer open. + assert_eq!(table.release(fd), Err(errno::EBADF)); + } + + #[test] + fn the_row_and_access_survive_the_word() { + let table = Table::<2>::new(); + let row = ROW_MASK as usize; + for flags in [O_RDONLY, O_WRONLY, O_RDWR] { + let access = Access::of(flags).unwrap(); + let fd = table.claim(row, access).unwrap(); + assert_eq!(table.get(fd), Ok((row, access))); + table.release(fd).unwrap(); + } + } + + #[test] + fn threads_never_share_a_descriptor() { + use std::collections::HashSet; + use std::sync::Arc; + let table = Arc::new(Table::<64>::new()); + let access = Access::of(O_RDWR).unwrap(); + let claims: Vec<_> = (0..8) + .map(|_| { + let table = Arc::clone(&table); + std::thread::spawn(move || { + (0..8) + .map(|_| table.claim(0, access).unwrap()) + .collect::>() + }) + }) + .collect(); + let all: Vec = claims + .into_iter() + .flat_map(|thread| thread.join().unwrap()) + .collect(); + assert_eq!(all.len(), 64); + assert_eq!(all.iter().copied().collect::>().len(), 64); + assert_eq!(table.claim(0, access), Err(errno::EMFILE)); + } +} diff --git a/crates/c/src/lib.rs b/crates/c/src/lib.rs index 07cd23a..1231e32 100644 --- a/crates/c/src/lib.rs +++ b/crates/c/src/lib.rs @@ -32,7 +32,7 @@ //! `kore-std` with its own entry, and two runtimes in one process would grow one //! heap supply behind two cursors. #![cfg_attr(target_os = "none", no_std)] -#![cfg_attr(target_os = "none", feature(thread_local))] +#![cfg_attr(target_os = "none", feature(thread_local, c_variadic))] #![deny(unsafe_op_in_unsafe_fn)] // The C ABI's exports are this image's reason to exist; naming the crate keeps @@ -45,6 +45,17 @@ use kore_c_shim as _; pub mod api; #[cfg(any(test, target_os = "none"))] mod exit; +// Off Kore only `kore_fd_describe` and the host suite read these two: the +// names, the verbs and most refusals are the Kore program's alone. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +mod fd; +#[cfg(target_os = "none")] +mod lock; +#[cfg(any(test, target_os = "none"))] +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +mod mmap; +#[cfg(target_os = "none")] +mod posix; #[cfg(target_os = "none")] mod program; #[cfg(any(test, target_os = "none"))] diff --git a/crates/c/src/lock.rs b/crates/c/src/lock.rs new file mode 100644 index 0000000..b54a2f1 --- /dev/null +++ b/crates/c/src/lock.rs @@ -0,0 +1,54 @@ +//! The futex mutex this runtime's threads take turns with: the streams' +//! lock, and the callers of thread 0's memory cell. +//! +//! The classic three states. A waiter's park is bounded by the runtime's +//! slice, because a holder's wake may not reach it: a thread holds a send +//! into a peer's doorbell only where the thread controller relays one. + +use core::sync::atomic::{AtomicU32, Ordering}; +use core::time::Duration; + +use kore_std::runtime::futex::{self, WaitOutcome}; +use kore_std::runtime::pace; + +const FREE: u32 = 0; +const HELD: u32 = 1; +const CONTENDED: u32 = 2; + +/// How long a waiter parks before it tries again: the runtime's slice for a +/// waiter its waker cannot reach (the thread controller's `UNREACHED_SLICE`). +const SLICE: Duration = Duration::from_nanos(pace::CEDE_NS); + +/// A mutex over one futex word. Not reentrant: a holder that needs to be +/// counts its own depth ([`crate::stdio`]). +pub(crate) struct Lock(AtomicU32); + +impl Lock { + pub(crate) const fn new() -> Self { + Self(AtomicU32::new(FREE)) + } + + pub(crate) fn acquire(&self) { + if self + .0 + .compare_exchange(FREE, HELD, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + return; + } + while self.0.swap(CONTENDED, Ordering::Acquire) != FREE { + // Bounded: the holder's wake may not reach this thread. + if futex::wait(&self.0, CONTENDED, Some(SLICE)) == WaitOutcome::Unsupported { + // No doorbell to park on: the deadline park any thread can. + let _ = pace::cede(); + } + } + } + + pub(crate) fn release(&self) { + // SeqCst, as the futex contract asks of the store before a wake. + if self.0.swap(FREE, Ordering::SeqCst) == CONTENDED { + futex::wake(&self.0); + } + } +} diff --git a/crates/c/src/mmap.rs b/crates/c/src/mmap.rs new file mode 100644 index 0000000..cebe56a --- /dev/null +++ b/crates/c/src/mmap.rs @@ -0,0 +1,684 @@ +//! `mmap` and `munmap` over a memory capability. +//! +//! `mmap(NULL, len, prot, flags, fd, 0)` on a descriptor for the program's +//! memory supply (`/kore/memory`, [`crate::fd`]) answers `len` bytes of fresh, +//! zeroed memory, as mapping `/dev/zero` does on a POSIX system. Underneath +//! it is three of kore-rt's verbs, the sequence a Wasmtime engine's guest +//! memory already takes (`kore-wasm-platform`): +//! +//! 1. **Carve.** `heap::reserve_from_top` takes a lease off the top of the +//! supply the allocator grows from the bottom of. The two share cursors, so +//! a region and `malloc`'s extents never meet, which a raw carve at an +//! offset of the same capability could not promise. +//! 2. **Place.** The address comes from the free part of the task's link +//! window (`Rt::link_free_span`), from one process-wide cursor. The caller +//! never picks it: a hint is ignored and `MAP_FIXED` is refused. +//! 3. **Map.** The whole lease is mapped there. +//! +//! Thread 0 does all three, for whichever thread asked. Every thread that +//! may allocate is founded holding a share of the supply, and `open` and +//! `mmap` hold the caller's access to that share's rights. But a share +//! carries no `REVOKE` (the founder keeps it, to tear the share down), and +//! a region that can be unmapped again is a carve its holder may revoke. The +//! supply's root is thread 0's, so the caller posts its request there and +//! waits, as a filesystem session does. A mapping is therefore the +//! process's: any thread may unmap it. +//! +//! # What `munmap` returns +//! +//! The supply's cursors and the window's cursor only advance: bytes carved +//! and addresses placed are spent for the life of the program. So `munmap` +//! removes the mapping, and a stale access faults, but it *parks* the region: +//! its lease and its address stay with the program, and the next `mmap` of the +//! same length takes a parked region, zeroed again, before anything new is +//! carved. A program that maps and unmaps one size in a loop therefore uses +//! one region. One that maps ever-different sizes spends supply each time, +//! and ends with `ENOMEM`: the table holds [`REGIONS`] regions, and when all +//! are taken a parked one of another length is given up for good. +//! +//! The request rules and the region table are plain data, so the host suite +//! exercises the code the target runs; the verbs are the metal half. + +use core::ffi::c_int; + +use crate::fd::errno; + +/// The mapping granule: kore's user targets pin a 4 KiB page +/// (`max-page-size=4096`), which is also the granule a supply is carved in +/// (`kore_rt::mem_source::ALIGN`), so a region is exactly its pages. +pub(crate) const PAGE: usize = 4096; + +/// ``'s protections, musl's values. +pub(crate) const PROT_READ: c_int = 1; +pub(crate) const PROT_WRITE: c_int = 2; +/// ``'s flags, musl's values. +pub(crate) const MAP_SHARED: c_int = 0x01; +pub(crate) const MAP_PRIVATE: c_int = 0x02; +pub(crate) const MAP_FIXED: c_int = 0x10; + +/// A request `mmap` can serve. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Request { + /// Whether the mapping is writable. + pub(crate) write: bool, + /// The length asked for, in whole pages' bytes. + pub(crate) asked: usize, +} + +/// The whole pages `len` bytes take, or `None` when that overflows. +pub(crate) fn pages(len: usize) -> Option { + len.checked_add(PAGE - 1).map(|bytes| bytes & !(PAGE - 1)) +} + +impl Request { + /// What `mmap`'s arguments ask for, or the `errno` that refuses them. + /// + /// `EINVAL`: no length, an offset (a supply has no positions a caller can + /// name), neither or both of `MAP_SHARED` and `MAP_PRIVATE`, or a flag + /// with no meaning here. `ENOTSUP`: `MAP_FIXED` (the kernel's side picks + /// every address), and any protection but read or read-write: memory a + /// program wrote is never executable, and a mapping nothing can touch is + /// not one this API reserves. `ENOMEM`: a length that does not round to + /// pages. + /// + /// `MAP_SHARED` and `MAP_PRIVATE` are both accepted and mean the same: + /// the region is fresh and this program's alone, so there is no other + /// mapper for a store to reach or to be kept from. `PROT_WRITE` alone is + /// read-write, which POSIX permits and every Kore ISA's page tables imply. + pub(crate) fn of(len: usize, prot: c_int, flags: c_int, offset: i64) -> Result { + if len == 0 || offset != 0 { + return Err(errno::EINVAL); + } + if flags & MAP_FIXED != 0 { + return Err(errno::ENOTSUP); + } + if flags != MAP_SHARED && flags != MAP_PRIVATE { + return Err(errno::EINVAL); + } + let write = match prot { + PROT_READ => false, + p if p == PROT_WRITE || p == PROT_READ | PROT_WRITE => true, + _ => return Err(errno::ENOTSUP), + }; + let asked = pages(len).ok_or(errno::ENOMEM)?; + Ok(Self { write, asked }) + } +} + +/// How many regions a program may hold, mapped and parked together. A region +/// is a lease and a view in thread 0's arena and a row here; sixteen is the +/// table a Wasmtime engine keeps for the same purpose +/// (`kore_wasm_platform::MAX_MAPPINGS`), small enough to scan, and a program +/// that needs more distinct regions than that at once is better served by +/// one larger mapping it divides itself. +pub(crate) const REGIONS: usize = 16; + +/// One region carved for the program: mapped, or parked for the next `mmap` +/// of its length. `L` is its lease and `V` its view, capabilities dropped +/// with the row. +pub(crate) enum Region { + Vacant, + /// Mapped at `base` for a request of `asked` bytes. + Mapped { + lease: L, + view: V, + base: usize, + asked: usize, + }, + /// Unmapped, keeping its lease and the address it had. + Parked { + lease: L, + base: usize, + /// The lease's own length, which a map must name exactly. + len: usize, + asked: usize, + }, +} + +/// A program's regions. +pub(crate) struct Regions { + rows: [Region; N], +} + +impl Regions { + pub(crate) const fn new() -> Self { + Self { + rows: [const { Region::Vacant }; N], + } + } + + /// A row nothing occupies. + pub(crate) fn vacant(&self) -> Option { + self.rows + .iter() + .position(|row| matches!(row, Region::Vacant)) + } + + /// A parked region that served a request of exactly `asked` bytes. + pub(crate) fn parked(&self, asked: usize) -> Option { + self.rows + .iter() + .position(|row| matches!(row, Region::Parked { asked: was, .. } if *was == asked)) + } + + /// The region mapped at exactly `base`. + pub(crate) fn mapped(&self, base: usize) -> Option { + self.rows + .iter() + .position(|row| matches!(row, Region::Mapped { base: at, .. } if *at == base)) + } + + /// Any parked region: what a full table gives up for a new one. + pub(crate) fn any_parked(&self) -> Option { + self.rows + .iter() + .position(|row| matches!(row, Region::Parked { .. })) + } + + /// Take row `at`, leaving it vacant. + pub(crate) fn take(&mut self, at: usize) -> Region { + core::mem::replace(&mut self.rows[at], Region::Vacant) + } + + /// Fill row `at`. + pub(crate) fn put(&mut self, at: usize, region: Region) { + self.rows[at] = region; + } +} + +#[cfg(target_os = "none")] +pub(crate) use metal::{arm, map, posted, serve, unmap}; + +#[cfg(target_os = "none")] +mod metal { + use core::cell::UnsafeCell; + use core::ffi::c_int; + use core::sync::atomic::{AtomicU32, Ordering}; + use core::time::Duration; + + use kore_ld::place::SharedPlacer; + use kore_std::runtime::cap::{MappedLeaseView, RetireOutcome, StashedLease, verbs}; + use kore_std::runtime::entry::Rt; + use kore_std::runtime::futex::{self, WaitOutcome}; + use kore_std::runtime::{pace, thread}; + + use super::{PAGE, REGIONS, Region, Regions, Request}; + use crate::fd::{Holding, errno}; + use crate::lock::Lock; + + /// Where regions land: the free part of the task's link window, armed + /// once by the host before any other thread exists. Unarmed (a program + /// with no host, or no link window), every `mmap` is `ENOMEM`. + static WINDOW: SharedPlacer = SharedPlacer::unarmed(); + + /// The process's regions, which thread 0 alone touches: the leases and + /// views in it are capabilities of its arena. + struct HostOnly(UnsafeCell); + // SAFETY: only thread 0 forms a reference ([`perform`]'s contract). + unsafe impl Sync for HostOnly {} + static REGIONS_HELD: HostOnly> = + HostOnly(UnsafeCell::new(Regions::new())); + + /// Publish the window regions are placed in: `rt`'s free link span + /// (thread 0's, which is the process's). A task without one leaves the + /// placer unarmed. + pub(crate) fn arm(rt: &Rt) { + if let Ok(span) = rt.link_free_span() { + let _ = WINDOW.arm(span.free, span.limit - span.free); + } + } + + /// Map `lease` whole at `base`, zeroed, with the request's access. + /// + /// The lease is mapped writable to be zeroed whatever was asked: a parked + /// region holds what its last user left, and a fresh one is zeroed too + /// rather than trusted to be. A read-only request is then mapped again + /// without write. + fn map_zeroed( + lease: &StashedLease, + base: usize, + len: usize, + write: bool, + ) -> Option { + let slot = lease.slot()?; + let mut view = verbs::map_owned_lease_at_rw(slot, base as u64, len as u64)?; + // SAFETY: `view` maps `len` writable bytes at its pointer, which + // nothing else has been handed yet. + unsafe { view.as_mut_ptr().write_bytes(0, len) }; + if write { + return Some(view); + } + match view.try_retire_outcome() { + RetireOutcome::Retired | RetireOutcome::AlreadyDead => {} + // Still mapped writable: refusing here would strand the view, so + // the caller gets nothing and the row keeps the lease. + RetireOutcome::Retryable => return None, + } + verbs::map_owned_lease_at(slot, base as u64, len as u64) + } + + /// One request of thread 0. + #[derive(Clone, Copy)] + enum Op { + Map(Request), + Unmap { base: usize, len: usize }, + } + + /// Carry `op` out. Thread 0 only: the supply's root, with the `REVOKE` + /// an unmappable carve needs, is a capability of its arena. + fn perform(op: Op) -> Result { + // SAFETY: thread 0 is the one caller, and it is not re-entered: no + // verb below calls back into this module. + let regions = unsafe { &mut *REGIONS_HELD.0.get() }; + match op { + Op::Map(request) => map_here(regions, request), + Op::Unmap { base, len } => unmap_here(regions, base, len).map(|()| 0), + } + } + + fn map_here( + regions: &mut Regions, + request: Request, + ) -> Result { + let (row, lease, base, len) = match regions.parked(request.asked) { + Some(row) => match regions.take(row) { + Region::Parked { + lease, base, len, .. + } => (row, lease, base, len), + // `parked` answered this row. + _ => return Err(errno::ENOMEM), + }, + None => { + // The row first: a carve with nowhere to be recorded would + // spend supply for nothing. A full table gives up a parked + // region of another length, whose lease ends with its row. + let row = match regions.vacant() { + Some(row) => row, + None => { + let row = regions.any_parked().ok_or(errno::ENOMEM)?; + drop(regions.take(row)); + row + } + }; + let lease = kore_std::runtime::heap::reserve_from_top(request.asked as u64) + .ok_or(errno::ENOMEM)?; + let len = lease + .slot() + .and_then(verbs::lease_geometry) + .and_then(|(_, len)| usize::try_from(len).ok()) + .ok_or(errno::ENOMEM)?; + let base = WINDOW + .choose(len as u64, PAGE as u64) + .ok() + .and_then(|base| usize::try_from(base).ok()) + .ok_or(errno::ENOMEM)?; + (row, lease, base, len) + } + }; + match map_zeroed(&lease, base, len, request.write) { + Some(view) => { + regions.put( + row, + Region::Mapped { + lease, + view, + base, + asked: request.asked, + }, + ); + Ok(base) + } + None => { + regions.put( + row, + Region::Parked { + lease, + base, + len, + asked: request.asked, + }, + ); + Err(errno::ENOMEM) + } + } + } + + fn unmap_here( + regions: &mut Regions, + base: usize, + len: usize, + ) -> Result<(), c_int> { + let asked = super::pages(len).filter(|&bytes| bytes != 0); + let row = regions.mapped(base).ok_or(errno::EINVAL)?; + let Region::Mapped { + lease, + mut view, + base, + asked: was, + } = regions.take(row) + else { + // `mapped` answered this row. + return Err(errno::EINVAL); + }; + if asked != Some(was) { + regions.put( + row, + Region::Mapped { + lease, + view, + base, + asked: was, + }, + ); + return Err(errno::EINVAL); + } + let len = view.len(); + match view.try_retire_outcome() { + RetireOutcome::Retired | RetireOutcome::AlreadyDead => { + regions.put( + row, + Region::Parked { + lease, + base, + len, + asked: was, + }, + ); + Ok(()) + } + RetireOutcome::Retryable => { + regions.put( + row, + Region::Mapped { + lease, + view, + base, + asked: was, + }, + ); + Err(errno::EAGAIN) + } + } + } + + // ── Reaching thread 0 ─────────────────────────────────────────────── + + // The cell's states. A caller owns it at IDLE (under [`CALLERS`]) and at + // SERVED, thread 0 at POSTED. + const IDLE: u32 = 0; + const POSTED: u32 = 1; + const SERVED: u32 = 2; + + /// One request at a time: the caller holding [`CALLERS`] writes `op` and + /// posts, thread 0 writes `answer` and serves. + struct Call { + state: AtomicU32, + op: UnsafeCell, + answer: UnsafeCell>, + } + // SAFETY: `op` and `answer` are touched only by the side `state` hands + // the cell to. + unsafe impl Sync for Call {} + static CALL: Call = Call { + state: AtomicU32::new(IDLE), + op: UnsafeCell::new(Op::Unmap { base: 0, len: 0 }), + answer: UnsafeCell::new(Ok(0)), + }; + + /// Callers take turns at the cell. + static CALLERS: Lock = Lock::new(); + + /// How long a caller waits before it re-reads the cell: thread 0 kicks + /// the main thread when it has answered, and holds no send into a + /// pthread's doorbell, which rides this slice (the streams' rule). + const SLICE: Duration = Duration::from_nanos(pace::CEDE_NS); + + /// Whether a request waits for thread 0. + pub(crate) fn posted() -> bool { + CALL.state.load(Ordering::Acquire) == POSTED + } + + /// Thread 0: answer the posted request, if there is one. `true` when one + /// was answered, and its caller should be woken. + pub(crate) fn serve() -> bool { + if !posted() { + return false; + } + // SAFETY: POSTED hands the cell to thread 0 until it stores SERVED. + let answer = perform(unsafe { *CALL.op.get() }); + // SAFETY: as above. + unsafe { *CALL.answer.get() = answer }; + // SeqCst, as the futex contract asks of the store before a wake. + CALL.state.store(SERVED, Ordering::SeqCst); + futex::wake(&CALL.state); + true + } + + /// Have thread 0 carry `op` out, and wait for its answer. + /// + /// Thread 0 itself (a filesystem callback, a program with no host) does + /// it in place. Any other thread of a program without a host has no one + /// to ask: `ENOMEM`. + fn ask(op: Op) -> Result { + if crate::program::on_host() { + return perform(op); + } + if !crate::program::hosted() { + return Err(errno::ENOMEM); + } + CALLERS.acquire(); + // SAFETY: IDLE under the lock hands the cell to this caller. + unsafe { *CALL.op.get() = op }; + CALL.state.store(POSTED, Ordering::SeqCst); + // Refused only when no post stood, and then thread 0 re-reads the + // cell before it parks. + let _ = thread::kick_parent(); + loop { + match CALL.state.load(Ordering::Acquire) { + SERVED => break, + state => { + if futex::wait(&CALL.state, state, Some(SLICE)) == WaitOutcome::Unsupported { + // No doorbell to park on: the deadline park any + // thread can. + let _ = pace::cede(); + } + } + } + } + // SAFETY: SERVED hands the cell back to this caller. + let answer = unsafe { *CALL.answer.get() }; + CALL.state.store(IDLE, Ordering::Release); + CALLERS.release(); + answer + } + + /// `mmap` on a memory supply the calling thread holds as `held`. + /// + /// The caller's own share is what `open` and this call hold the access + /// to. The region is carved by thread 0, which holds the supply's root: + /// a share a thread is founded with carries no `REVOKE`, and a carve + /// that can be unmapped again needs it. + /// + /// `EACCES`: a writable mapping of a capability without `MODIFY`. + /// `ENOMEM`: every row of the table is a live mapping, the supply or the + /// window has no room, the kernel refused the map, or the program has no + /// host to carve for this thread. A region whose map is refused is + /// parked, so its supply bytes and address serve the next request of its + /// length. + pub(crate) fn map(held: Holding, request: Request) -> Result<*mut u8, c_int> { + if request.write && !held.writable { + return Err(errno::EACCES); + } + ask(Op::Map(request)).map(|base| base as *mut u8) + } + + /// `munmap`: remove the mapping at `base`, which `len` must name whole, + /// and park its region. + /// + /// `EINVAL`: the program has no mapping at exactly `base`, or `len` is + /// not that mapping's length. `EAGAIN`: the kernel did not confirm the + /// unmap; the mapping stands and the call may be made again. + pub(crate) fn unmap(base: usize, len: usize) -> Result<(), c_int> { + ask(Op::Unmap { base, len }).map(|_| ()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const RW: c_int = PROT_READ | PROT_WRITE; + + #[test] + fn a_length_is_whole_pages() { + assert_eq!(pages(1), Some(PAGE)); + assert_eq!(pages(PAGE), Some(PAGE)); + assert_eq!(pages(PAGE + 1), Some(2 * PAGE)); + assert_eq!(pages(usize::MAX), None); + assert_eq!(pages(usize::MAX - PAGE + 2), None); + assert_eq!(pages(usize::MAX - PAGE + 1), Some(usize::MAX - PAGE + 1)); + } + + #[test] + fn a_readable_region_of_the_supply_is_served() { + assert_eq!( + Request::of(1 << 20, RW, MAP_PRIVATE, 0), + Ok(Request { + write: true, + asked: 1 << 20 + }) + ); + assert_eq!( + Request::of(1, PROT_READ, MAP_SHARED, 0), + Ok(Request { + write: false, + asked: PAGE + }) + ); + // Write alone is read-write. + assert_eq!( + Request::of(PAGE, PROT_WRITE, MAP_SHARED, 0), + Ok(Request { + write: true, + asked: PAGE + }) + ); + } + + #[test] + fn what_has_no_meaning_here_is_refused_by_name() { + // No length, and an offset into a supply. + assert_eq!(Request::of(0, RW, MAP_PRIVATE, 0), Err(errno::EINVAL)); + assert_eq!( + Request::of(PAGE, RW, MAP_PRIVATE, PAGE as i64), + Err(errno::EINVAL) + ); + assert_eq!(Request::of(PAGE, RW, MAP_PRIVATE, -1), Err(errno::EINVAL)); + // Neither, both, or a stranger: MAP_ANONYMOUS is 0x20. + assert_eq!(Request::of(PAGE, RW, 0, 0), Err(errno::EINVAL)); + assert_eq!( + Request::of(PAGE, RW, MAP_SHARED | MAP_PRIVATE, 0), + Err(errno::EINVAL) + ); + assert_eq!( + Request::of(PAGE, RW, MAP_PRIVATE | 0x20, 0), + Err(errno::EINVAL) + ); + // The caller never picks the address. + assert_eq!( + Request::of(PAGE, RW, MAP_PRIVATE | MAP_FIXED, 0), + Err(errno::ENOTSUP) + ); + // No access, executable (4), and bits past the three. + for prot in [0, 4, PROT_READ | 4, RW | 4, 8, -1] { + assert_eq!( + Request::of(PAGE, prot, MAP_PRIVATE, 0), + Err(errno::ENOTSUP), + "{prot}" + ); + } + assert_eq!( + Request::of(usize::MAX, RW, MAP_PRIVATE, 0), + Err(errno::ENOMEM) + ); + } + + type Rows = Regions; + + fn mapped(lease: u32, base: usize, asked: usize) -> Region { + Region::Mapped { + lease, + view: lease, + base, + asked, + } + } + + fn park(rows: &mut Rows, base: usize) { + let row = rows.mapped(base).unwrap(); + let Region::Mapped { + lease, base, asked, .. + } = rows.take(row) + else { + unreachable!() + }; + rows.put( + row, + Region::Parked { + lease, + base, + len: asked, + asked, + }, + ); + } + + #[test] + fn an_unmapped_region_serves_the_next_request_of_its_length() { + let mut rows = Rows::new(); + assert_eq!(rows.vacant(), Some(0)); + rows.put(0, mapped(7, 0x1000, PAGE)); + rows.put(1, mapped(8, 0x9000, 2 * PAGE)); + assert_eq!(rows.vacant(), Some(2)); + assert_eq!((rows.parked(PAGE), rows.any_parked()), (None, None)); + park(&mut rows, 0x1000); + // Parked, it is no longer a mapping, and only its own length takes it. + assert_eq!(rows.mapped(0x1000), None); + assert_eq!(rows.parked(2 * PAGE), None); + assert_eq!(rows.parked(PAGE), Some(0)); + let Region::Parked { lease, base, .. } = rows.take(0) else { + panic!("the parked row") + }; + // The same lease at the same address. + assert_eq!((lease, base), (7, 0x1000)); + assert_eq!(rows.vacant(), Some(0)); + } + + #[test] + fn a_mapping_is_found_by_its_base_alone() { + let mut rows = Rows::new(); + rows.put(1, mapped(1, 0x4000, 4 * PAGE)); + assert_eq!(rows.mapped(0x4000), Some(1)); + // An address inside it is not its base. + assert_eq!(rows.mapped(0x5000), None); + assert_eq!(rows.mapped(0), None); + } + + #[test] + fn a_full_table_has_no_vacant_row() { + let mut rows = Rows::new(); + for at in 0..3 { + rows.put(at, mapped(at as u32, 0x1000 * (at + 1), PAGE)); + } + assert_eq!(rows.vacant(), None); + park(&mut rows, 0x2000); + // A parked row is still occupied: it holds a lease. It is what a + // request of another length takes the place of. + assert_eq!(rows.vacant(), None); + assert_eq!(rows.parked(PAGE), Some(1)); + assert_eq!(rows.parked(2 * PAGE), None); + assert_eq!(rows.any_parked(), Some(1)); + assert!(matches!(rows.take(1), Region::Parked { lease: 1, .. })); + assert_eq!((rows.vacant(), rows.any_parked()), (Some(1), None)); + } +} diff --git a/crates/c/src/posix.rs b/crates/c/src/posix.rs new file mode 100644 index 0000000..49da7bc --- /dev/null +++ b/crates/c/src/posix.rs @@ -0,0 +1,94 @@ +//! The POSIX names over [`crate::fd`] and [`crate::mmap`]: `open`, `close`, +//! `mmap` and `munmap`, as ``, `` and `` +//! declare them. Each sets `errno` and answers `-1` (`MAP_FAILED` for `mmap`) +//! when it refuses. +//! +//! Exported on Kore alone, like the rest of the C library: on a host these +//! names are the host libc's. + +use core::ffi::{CStr, c_char, c_int, c_void}; + +use kore_c_shim::errno::set as set_errno; + +use crate::fd::{self, Kind, NAMES, TABLE, errno}; +use crate::mmap::{self, Request}; + +/// `mmap`'s refusal: `MAP_FAILED`, `(void *)-1`. +const MAP_FAILED: *mut c_void = usize::MAX as *mut c_void; + +fn refused(code: c_int) -> c_int { + set_errno(code); + -1 +} + +/// `open(path, flags)`: a descriptor for the capability `path` names +/// ([`fd::open`]). The mode a variadic `open` may carry belongs to `O_CREAT`, +/// which no name here takes, so it is never read. +/// +/// # Safety +/// `path` is null or a NUL-terminated string. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn open(path: *const c_char, flags: c_int, _mode: ...) -> c_int { + if path.is_null() { + return refused(errno::EFAULT); + } + // SAFETY: the caller's contract: a NUL-terminated string. + let path = unsafe { CStr::from_ptr(path) }.to_bytes(); + fd::open(path, flags).unwrap_or_else(refused) +} + +/// `close(fd)`: forget the descriptor. The capability it named stays the +/// program's, and mappings made through it stand. +#[unsafe(no_mangle)] +pub extern "C" fn close(fd: c_int) -> c_int { + match TABLE.release(fd) { + Ok(()) => 0, + Err(code) => refused(code), + } +} + +/// `mmap(addr, len, prot, flags, fd, offset)` ([`mmap`]'s module docs). +/// `addr` is a hint POSIX lets an implementation ignore, and it is. +/// +/// `EBADF`: `fd` is not open. `EACCES`: it was not opened for reading, or a +/// writable mapping was asked of one not opened for writing. `ENXIO`: the +/// calling thread does not hold the capability. Otherwise as +/// [`Request::of`] and [`mmap::map`] refuse. +#[unsafe(no_mangle)] +pub extern "C" fn mmap( + _addr: *mut c_void, + len: usize, + prot: c_int, + flags: c_int, + fd: c_int, + offset: i64, +) -> *mut c_void { + let mapped = TABLE.get(fd).and_then(|(row, access)| { + let request = Request::of(len, prot, flags, offset)?; + if !access.read || (request.write && !access.write) { + return Err(errno::EACCES); + } + let name = &NAMES[row]; + let held = fd::holding(name).ok_or(errno::ENXIO)?; + match name.kind { + Kind::Memory => mmap::map(held, request), + } + }); + match mapped { + Ok(base) => base.cast(), + Err(code) => { + set_errno(code); + MAP_FAILED + } + } +} + +/// `munmap(addr, len)`: unmap the program's mapping at `addr`, whole +/// ([`mmap::unmap`]). +#[unsafe(no_mangle)] +pub extern "C" fn munmap(addr: *mut c_void, len: usize) -> c_int { + match mmap::unmap(addr.addr(), len) { + Ok(()) => 0, + Err(code) => refused(code), + } +} diff --git a/crates/c/src/program.rs b/crates/c/src/program.rs index f8e2ca8..cc57c56 100644 --- a/crates/c/src/program.rs +++ b/crates/c/src/program.rs @@ -105,6 +105,11 @@ pub(crate) fn on_host() -> bool { ROLE.get() == Role::Host } +/// Whether a host serves: thread 0 answers what other threads post. +pub(crate) fn hosted() -> bool { + HOSTED.load(Ordering::Acquire) +} + /// Whether the calling thread is the main thread, the one the host can kick. pub(crate) fn on_main() -> bool { ROLE.get() == Role::Main @@ -160,9 +165,11 @@ impl Request { } } -/// Anything the host must answer: an end, or a session. +/// Anything the host must answer: an end, a session, or a mapping. fn pending() -> bool { - EXIT.posted().is_some() || SESSION.state.load(Ordering::Acquire) == POSTED + EXIT.posted().is_some() + || SESSION.state.load(Ordering::Acquire) == POSTED + || crate::mmap::posted() } // ── Entry points the C library calls ──────────────────────────────────── @@ -363,6 +370,9 @@ impl Founded { // The C streams reach the launcher through this thread: map its // stream and install the sink before any thread can write. let output = crate::stdio::install(&rt); + // Where `mmap` places: this task's free link span, published before + // any thread that could map exists. + crate::mmap::arm(&rt); HOSTED.store(true, Ordering::Release); // A mint grant of its own: the first `pthread_create` on the main // thread starts kore-rt's thread controller there, which mints the @@ -472,6 +482,11 @@ async fn host( return Ok(exit); } } + // A thread's `mmap` or `munmap`: thread 0 holds the supply's root. + if crate::mmap::serve() { + wake_main(); + continue; + } if SESSION.state.load(Ordering::Acquire) == POSTED { // SAFETY: POSTED hands the cell to the host until it stores SERVED. let (step, user) = { @@ -577,10 +592,19 @@ async fn until_exit( crate::stdio::drain(output, wake_main).await; executor::yield_now().await; } + // Another thread's mapping does not wait for the session to end, + // and the session is polled between two of them. + if crate::mmap::serve() { + wake_main(); + executor::yield_now().await; + continue; + } crate::stdio::arm(); - if park(wake, || ended() || crate::stdio::pending()) - .await - .is_err() + if park(wake, || { + ended() || crate::stdio::pending() || crate::mmap::posted() + }) + .await + .is_err() { core::future::pending::<()>().await; } diff --git a/crates/c/src/stdio.rs b/crates/c/src/stdio.rs index 1f724f6..7b2bcbf 100644 --- a/crates/c/src/stdio.rs +++ b/crates/c/src/stdio.rs @@ -371,13 +371,10 @@ mod metal { // ── The streams' lock ─────────────────────────────────────────────── - /// The classic three-state futex mutex, reentrant per thread: a stream - /// call holds it across all its writes, and the sink each write reaches - /// takes it again. - static LOCK: AtomicU32 = AtomicU32::new(FREE); - const FREE: u32 = 0; - const HELD: u32 = 1; - const CONTENDED: u32 = 2; + /// The streams' lock, reentrant per thread ([`DEPTH`]): a stream call + /// holds it across all its writes, and the sink each write reaches takes + /// it again. + static LOCK: crate::lock::Lock = crate::lock::Lock::new(); /// How many times this thread holds [`LOCK`]. #[thread_local] @@ -398,7 +395,7 @@ mod metal { } let depth = DEPTH.get(); if depth == 0 { - acquire(); + LOCK.acquire(); // The holder is the ring's one producer: `tail` is exact here. CALL.store(IN_CALL | u64::from(RING.tail()), Ordering::SeqCst); } @@ -416,35 +413,12 @@ mod metal { 1 => { DEPTH.set(0); CALL.store(0, Ordering::SeqCst); - release(); + LOCK.release(); } depth => DEPTH.set(depth - 1), } } - fn acquire() { - if LOCK - .compare_exchange(FREE, HELD, Ordering::Acquire, Ordering::Relaxed) - .is_ok() - { - return; - } - while LOCK.swap(CONTENDED, Ordering::Acquire) != FREE { - // Bounded: the holder's wake may not reach this thread. - if futex::wait(&LOCK, CONTENDED, Some(SLICE)) == WaitOutcome::Unsupported { - // No doorbell to park on: the deadline park any thread can. - let _ = pace::cede(); - } - } - } - - fn release() { - // SeqCst, as the futex contract asks of the store before a wake. - if LOCK.swap(FREE, Ordering::SeqCst) == CONTENDED { - futex::wake(&LOCK); - } - } - // ── Writers ───────────────────────────────────────────────────────── fn sink(standard: Standard, bytes: &[u8]) -> bool { diff --git a/crates/c/tests/program.c b/crates/c/tests/program.c index 7009957..fda382c 100644 --- a/crates/c/tests/program.c +++ b/crates/c/tests/program.c @@ -1,5 +1,6 @@ /* kore/program.h as an independent C program uses it: a session whose - * callbacks are kore.h's own KoreFsStep, run through kore_fs_run. Compiled + * callbacks are kore.h's own KoreFsStep, run through kore_fs_run, and a + * descriptor described through kore_fd_describe. Compiled * freestanding for every Kore ISA by tools/check-bindings.py. */ #include @@ -28,3 +29,16 @@ int program_session(void) KoreStatus status = run(open_root, &reader, &error); return status == KORE_STATUS_OK && error == KORE_FS_ERROR_NONE && reader.done; } + +/* A descriptor's description: the exact entry type, its fields and the + * constants its fields take. */ +static KoreStatus (*const describe)(int, KoreFdInfo *) = kore_fd_describe; + +int program_memory(int fd) +{ + KoreFdInfo info = {0}; + if (describe(fd, &info) != KORE_STATUS_OK) + return 0; + return info.kind == KORE_FD_MEMORY && (info.access & (KORE_FD_READ | KORE_FD_WRITE)) != 0 && + info.length != 0; +} diff --git a/docs/languages.md b/docs/languages.md index 33cc6eb..42d764b 100644 --- a/docs/languages.md +++ b/docs/languages.md @@ -84,6 +84,9 @@ that joins `libkore.so` to the program's own `libc.a` — and gets: every float digit exact, `stdout` and `stderr`, the `strto*` parsers, `qsort`, `strerror`, the string routines and a per-thread `errno`, and `kore-c`'s own `abort`; +- descriptors over the capabilities it holds (``, ``, + ``: `open`, `close`, `mmap`, `munmap`), and + `kore_fd_describe` (`kore/program.h`), described below; - a host on thread 0, entered from the C library's start; - `kore_fs_run(step, user, &error)` (`kore/program.h`), which runs a `KoreFsStep` program as the program's filesystem session. @@ -147,6 +150,69 @@ one word. Every park is a doorbell park; nothing spins. heap) runs the program on thread 0, as its C library alone would. Then the filesystem session is refused, and only thread 0's exit ends the program. +### Descriptors: C's spelling for a held capability + +A program's authority is the capabilities it was founded holding. Each sits +at a fixed place: the boot block names it by a role and an index, and the +capability is a slot of the thread's arena. `open` is the readable way to say +which one is meant, and nothing more: + +```c +int fd = open("/kore/memory", O_RDWR); +char *region = mmap(NULL, 1 << 20, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); +... +munmap(region, 1 << 20); +close(fd); +``` + +- **The path is not a file.** `open` looks the string up in one static table + (`crates/c/src/fd.rs`) whose rows are a name and the bootstrap coordinate it + stands for. No filesystem is asked, and no name outside the table exists: + `ENOENT`. The first row is `/kore/memory`, `role::HEAP@0`, the memory supply + the allocator grows on. +- **It grants nothing.** `open` succeeds only for a capability the calling + thread holds (`ENXIO` otherwise), with access its rights cover (`EACCES`). + The only flags are the access modes. `close` forgets the number; the + capability stays the program's, and mappings made through it stand. +- **A descriptor stores the coordinate, never the capability.** Capabilities + are per thread on Kore, and `main` and every `pthread` have their own boot + block. Each use resolves the coordinate in the *calling* thread's. Numbers + are the process's, as C expects, from 3 (0 to 2 are the streams'), and no + authority crosses a thread through one. +- **`kore_fd_describe(fd, &info)`** reads what the calling thread holds + there: its kind (`KORE_FD_MEMORY`), what its rights permit, and for a + memory supply its whole length. + +`mmap` on the memory supply answers fresh, zeroed memory, as mapping +`/dev/zero` does elsewhere. It is three of kore-rt's verbs, the sequence a +Wasmtime engine's guest memory takes: a lease carved from the supply's top, +clear of the allocator, which grows from its bottom; an address from the +free part of the task's link window (`Rt::link_free_span`); the whole lease +mapped there. The rules follow from that: + +- **The caller never picks the address.** A hint is ignored, and `MAP_FIXED` + is `ENOTSUP`. +- **Read, or read and write.** `PROT_EXEC` is `ENOTSUP`: memory a program + wrote is never executable. `MAP_SHARED` and `MAP_PRIVATE` mean the same, + since the region is this program's alone. There is no offset, and no + `MAP_ANONYMOUS`: memory comes from a capability, so a descriptor names it. +- **Thread 0 maps, for whichever thread asked.** Every thread that may + allocate is founded holding a share of the supply, and `open` and `mmap` + hold the caller's access to that share's rights. A share carries no + `REVOKE`, though, and a region that can be unmapped again is a carve its + holder may revoke, so the caller posts its request to thread 0, which + holds the supply's root, and waits as a filesystem session does. A mapping + is therefore the process's: any thread may `munmap` it, whole, by its base + and its length. +- **`munmap` parks the region.** The supply's cursors and the window's only + advance, so carved bytes and placed addresses never return to them. The + mapping is removed and a stale access faults, but the program keeps the + lease and its address, and the next `mmap` of the same length takes it, + zeroed again, before anything new is carved. One size mapped and unmapped + in a loop uses one region. Ever-different sizes spend supply each time and + end with `ENOMEM`. A program holds sixteen regions, mapped and parked + together. + The C library's half is two weak references. Its start calls `__kore_program_start(program, end)` when the program links libkore, and its exit calls `__kore_program_exit`. A program without libkore is unchanged. On diff --git a/extern/c/include/kore/program.h b/extern/c/include/kore/program.h index ce10f4b..b8e3f05 100644 --- a/extern/c/include/kore/program.h +++ b/extern/c/include/kore/program.h @@ -22,6 +22,44 @@ */ #define KORE_MAIN_STACK_BYTES (1 << 20) +/** + * [`KoreFdInfo::kind`]: the memory supply the program's allocator grows + * on. `mmap` answers a fresh, zeroed region of it. + */ +#define KORE_FD_MEMORY 1 + +/** + * [`KoreFdInfo::access`]: the capability may be read. + */ +#define KORE_FD_READ (1 << 0) + +/** + * [`KoreFdInfo::access`]: the capability may be written. + */ +#define KORE_FD_WRITE (1 << 1) + +/** + * What a descriptor names, as the calling thread holds it + * ([`kore_fd_describe`]). + */ +typedef struct KoreFdInfo { + /** + * What kind of capability it is: `KORE_FD_MEMORY`. + */ + uint32_t kind; + /** + * What the capability's own rights permit this thread, as + * `KORE_FD_READ | KORE_FD_WRITE` bits: the rights it was founded with, + * which the access `open` asked for can only narrow. + */ + uint32_t access; + /** + * For `KORE_FD_MEMORY`, the supply's whole length in bytes: what the + * allocator and every mapping share, not what is still free. + */ + uint64_t length; +} KoreFdInfo; + #ifdef __cplusplus extern "C" { #endif // __cplusplus @@ -51,6 +89,26 @@ extern "C" { */ KoreStatus kore_fs_run(KoreFsStep step, void *user, KoreFsError *error); +/** + * Describe the capability behind `fd`, a descriptor `open` answered + * (``): `open` finds a capability this program was founded holding + * by its name, and this reads what was found. Nothing is granted or changed. + * + * Capabilities are held per thread, so the answer is the calling thread's: + * the same descriptor may describe a capability on the thread that opened + * it and be refused on one founded without it. + * + * Returns `KORE_STATUS_OK` with `*out` written. + * `KORE_STATUS_INVALID_ARGUMENT`: `out` is null. `KORE_STATUS_STALE_HANDLE`: + * `fd` is not an open descriptor. `KORE_STATUS_REFUSED`: the calling thread + * holds no such capability, or this is not a Kore program. A refusal leaves + * `*out` unchanged. + * + * # Safety + * `out` is null or valid for one aligned write. + */ +KoreStatus kore_fd_describe(int fd, struct KoreFdInfo *out); + #ifdef __cplusplus } // extern "C" #endif // __cplusplus diff --git a/tools/check-program.py b/tools/check-program.py index e52550c..03a09d7 100755 --- a/tools/check-program.py +++ b/tools/check-program.py @@ -40,8 +40,10 @@ # The rest of the C library a program's headers declare (Kore's TCC # `include/`: , , , ), which libkore.so # implements: kore-c-shim's printf family, streams, parsers, string routines -# and per-thread errno, and kore-c's own abort. The program's libc.a holds the -# rest: exit, atexit and the routines a compiler assumes. +# and per-thread errno, and kore-c's own abort; and , and +# 's open, close, mmap and munmap over the capabilities the +# program holds. The program's libc.a holds the rest: exit, atexit and the +# routines a compiler assumes. C_LIBRARY = {'printf', 'fprintf', 'sprintf', 'snprintf', 'vprintf', 'vfprintf', 'vsprintf', 'vsnprintf', 'fputc', 'putc', 'putchar', 'fputs', 'puts', 'fwrite', 'fflush', 'fclose', 'stdout', 'stderr', @@ -49,7 +51,8 @@ 'qsort', 'strerror', 'abort', 'memchr', 'strchr', 'strrchr', 'strcmp', 'strncmp', 'strcpy', 'strncpy', 'strncat', 'strstr', - '__errno_location'} + '__errno_location', + 'open', 'close', 'mmap', 'munmap'} DYNAMIC_TLS = re.compile(r'TLSDESC|DTPMOD|DTPOFF|TLSGD|TLSLD|TLS_DTP') From 98186bf9bb9b5f455e9336f8bf4dc1144afe330e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 13:11:44 +0000 Subject: [PATCH 14/18] lock: rustix 1.1.6 RUSTSEC-2026-0486: rustix 1.0.0 to 1.1.5 marked unwritten bytes as initialized when an extended-attribute function was given a zero-length buffer. The advisory reached the database after this branch's last green run, and `cargo audit --deny warnings` now refuses 1.1.5, which the lock already held. 1.1.6 is the patched release. Only the host crates' graph (wasmtime's) holds rustix; the native lock already had 1.1.6. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index beb52d2..7e4ec99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1331,9 +1331,9 @@ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustix" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ "bitflags", "errno", From 42893612c5ec250c7bfc02ff21a7da3222cac33e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 02:00:14 +0000 Subject: [PATCH 15/18] native: re-export at Kore dfece644, where the process page carries the command name `tools/sync-native.py` at MicroPerceptron/kore@dfece6447cd331375a28b2bb290691cb8d5ab060, which is Kore's branch after its merge of main. The export gains: - `ProcessIo::command` (page ABI 14): the name a launcher resolved a program by, beside the argument tail it always carried. kore-c is about to hand both to a C program's `main`. - the device inventory records (`kore_abi::device::state`) and the observations behind them, and the command snapshot's second ABI. - main's own changes to the exported crates: the filesystem's watch plane, the seat clipboard's window verbs, and the wakes a service owes the peers it publishes events to. `fmt`, `test`, `check`, `docs`, `owners`, `bindings-check`, `miri`, the native type composition and each ISA's target checks pass on this export with the change that follows it. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- native/crates/abi/src/command.rs | 93 +- native/crates/abi/src/device.rs | 2 + native/crates/abi/src/device/state.rs | 922 ++++++++++++++++++ native/crates/abi/src/process.rs | 97 +- native/crates/abi/src/session.rs | 17 + native/crates/abi/src/svc.rs | 163 +++- native/crates/abi/src/task.rs | 8 +- native/crates/dagfs/src/fs.rs | 77 +- native/crates/dagfs/src/lib.rs | 2 + native/crates/dagfs/src/protocol.rs | 27 + native/crates/dagfs/src/text.rs | 36 +- native/crates/dagfs/src/watch.rs | 251 +++++ native/crates/dagfs/src/wire.rs | 107 ++ native/crates/ld/tests/built_images.rs | 1 + native/crates/rt/src/process.rs | 7 + native/crates/rt/src/service/mod.rs | 15 + native/crates/rt/src/svc/serve.rs | 144 +++ native/crates/window/src/client.rs | 114 +++ native/crates/window/src/clipboard.rs | 107 ++ native/crates/window/src/control.rs | 168 +++- native/crates/window/src/keysplit.rs | 315 ++++++ native/crates/window/src/lib.rs | 3 + native/crates/window/src/relay.rs | 40 +- native/crates/window/src/shortcut.rs | 283 ++++++ native/crates/window/src/wire.rs | 238 ++++- native/provenance.json | 72 +- native/services/block/boot.toml | 4 +- native/services/dagfs/boot.toml | 4 +- native/services/dagfs/src/durable.rs | 120 ++- native/services/dagfs/src/engine.rs | 27 + native/services/dagfs/src/lib.rs | 4 + native/services/dagfs/src/namespace.rs | 413 +++++++- native/services/dagfs/src/router.rs | 17 +- native/services/dagfs/src/service/contract.rs | 37 + native/services/dagfs/src/service/mod.rs | 388 +++++++- native/services/dagfs/src/watches.rs | 310 ++++++ native/services/device/boot.toml | 4 +- native/services/mm/boot.toml | 4 +- native/services/swap/boot.toml | 4 +- 39 files changed, 4550 insertions(+), 95 deletions(-) create mode 100644 native/crates/abi/src/device/state.rs create mode 100644 native/crates/dagfs/src/watch.rs create mode 100644 native/crates/window/src/clipboard.rs create mode 100644 native/crates/window/src/keysplit.rs create mode 100644 native/crates/window/src/shortcut.rs create mode 100644 native/services/dagfs/src/watches.rs diff --git a/native/crates/abi/src/command.rs b/native/crates/abi/src/command.rs index d62b18c..f34eca2 100644 --- a/native/crates/abi/src/command.rs +++ b/native/crates/abi/src/command.rs @@ -7,14 +7,22 @@ use crate::image::{ImageName, NAME_BYTES}; -/// Snapshot wire version. -pub const COMMAND_SNAPSHOT_ABI: u8 = 1; +/// Snapshot wire version. 2: an entry may carry no usage. A launcher +/// carries the usage of the rows its command line names and only the name +/// and summary of the rest ([`CommandSnapshotBuilder::push_listed`]), so the +/// record grows with the catalogue's names and summaries rather than with +/// every row's usage, and has room for the programs a task installs. A `1` +/// reader refuses an empty usage. +pub const COMMAND_SNAPSHOT_ABI: u8 = 2; /// The largest task-local command set accepted by the decoder. Raised /// from 32 when the desktop's `editor` and `power-ui` rows joined the two /// native filesystem verbs, and from 40 when Files and the documents editor -/// became two rows; the byte budget (`PROCESS_CONTEXT_BYTES`) is the other -/// bound, proven by exec's host test over the real catalogue. -pub const MAX_COMMAND_SNAPSHOT_ENTRIES: usize = 48; +/// became two rows; and from 48 for the programs a task installs, which +/// its launcher lists beside the catalogue (47 rows and a shell's share of +/// the installed pool no longer fit 48). The byte budget +/// (`PROCESS_CONTEXT_BYTES`) is the other bound; exec's host test proves +/// both over the real catalogue with that share installed. +pub const MAX_COMMAND_SNAPSHOT_ENTRIES: usize = 64; /// The launcher holds a power-control face and therefore handles these two /// privileged controls in-process. pub const COMMAND_SNAPSHOT_SYSTEM_POWER: u8 = 1 << 0; @@ -38,6 +46,8 @@ pub enum CommandSnapshotError { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct CommandDescription<'a> { pub local_name: &'a str, + /// The command's usage, or empty when the launcher carried none for + /// this row. pub usage: &'a str, pub summary: &'a str, } @@ -67,13 +77,35 @@ impl<'a> CommandSnapshotBuilder<'a> { }) } - /// Append one local namespace row. The executable capability paired with - /// this metadata deliberately is not accepted by this API. + /// Append one local namespace row with its usage. The executable + /// capability paired with this metadata deliberately is not accepted by + /// this API. pub fn push( &mut self, local_name: &ImageName, usage: &[u8], summary: &[u8], + ) -> Result<(), CommandSnapshotError> { + validate_text(usage, false)?; + self.append(local_name, usage, summary) + } + + /// Append one local namespace row without its usage: a row the reader + /// lists but was not asked to describe. + pub fn push_listed( + &mut self, + local_name: &ImageName, + summary: &[u8], + ) -> Result<(), CommandSnapshotError> { + self.append(local_name, &[], summary) + } + + /// `usage` is valid text, or empty for a row that carries none. + fn append( + &mut self, + local_name: &ImageName, + usage: &[u8], + summary: &[u8], ) -> Result<(), CommandSnapshotError> { if usize::from(self.count) >= MAX_COMMAND_SNAPSHOT_ENTRIES { return Err(CommandSnapshotError::TooManyEntries); @@ -82,7 +114,6 @@ impl<'a> CommandSnapshotBuilder<'a> { .as_str() .ok_or(CommandSnapshotError::InvalidText)?; validate_text(name.as_bytes(), true)?; - validate_text(usage, false)?; validate_text(summary, false)?; if name.len() > u8::MAX as usize || usage.len() > u8::MAX as usize @@ -260,7 +291,10 @@ fn decode_entry( cursor += usage_len; let summary = &bytes[cursor..cursor + summary_len]; validate_text(name, true)?; - validate_text(usage, false)?; + // A row the launcher only listed carries no usage. + if !usage.is_empty() { + validate_text(usage, false)?; + } validate_text(summary, false)?; Ok(( CommandDescription { @@ -318,6 +352,47 @@ mod tests { assert!(snapshot.find(b"touch").is_none()); } + #[test] + fn a_listed_row_carries_its_name_and_summary_and_no_usage() { + let mut bytes = [0u8; 256]; + let mut builder = CommandSnapshotBuilder::new(&mut bytes, 0).unwrap(); + builder + .push_listed(&image("hello"), b"dynamic-linker witness") + .unwrap(); + builder + .push(&image("ls"), b"ls [-l] [path]", b"list directory entries") + .unwrap(); + let listed_only = 4 + "hello".len() + "dynamic-linker witness".len(); + let len = builder.finish(); + let snapshot = CommandSnapshot::parse(&bytes[..len]).unwrap(); + assert_eq!( + snapshot.find(b"hello"), + Some(CommandDescription { + local_name: "hello", + usage: "", + summary: "dynamic-linker witness", + }) + ); + assert_eq!(snapshot.find(b"ls").unwrap().usage, "ls [-l] [path]"); + // The usage a listed row leaves out is exactly what it saves. + assert_eq!( + len, + 8 + listed_only + 4 + "ls".len() + "ls [-l] [path]".len() + 22 + ); + // A row that is described carries a usage: `push` takes no empty one, + // and a summary is never optional. + let mut bytes = [0u8; 64]; + let mut builder = CommandSnapshotBuilder::new(&mut bytes, 0).unwrap(); + assert_eq!( + builder.push(&image("one"), b"", b"first"), + Err(CommandSnapshotError::InvalidText) + ); + assert_eq!( + builder.push_listed(&image("one"), b""), + Err(CommandSnapshotError::InvalidText) + ); + } + #[test] fn duplicate_and_malformed_rows_fail_closed() { let mut bytes = [0u8; 128]; diff --git a/native/crates/abi/src/device.rs b/native/crates/abi/src/device.rs index d8483f1..463489e 100644 --- a/native/crates/abi/src/device.rs +++ b/native/crates/abi/src/device.rs @@ -21,8 +21,10 @@ mod census; mod i2c_admission; +mod state; pub use census::*; pub use i2c_admission::*; +pub use state::*; /// An I/O virtual address — a device-domain address translated by the /// IOMMU. **Never** a physical address: userland holds only this, so it can diff --git a/native/crates/abi/src/device/state.rs b/native/crates/abi/src/device/state.rs new file mode 100644 index 0000000..5dab25c --- /dev/null +++ b/native/crates/abi/src/device/state.rs @@ -0,0 +1,922 @@ +//! The read-only **live state** of the PCI functions the census +//! ([`super::census`]) lists, and the shape of one function's BARs. +//! +//! The census is boot-frozen identity. This is what changes after boot, and +//! what a driver writer asks before touching a device: whether a driver +//! claimed the function, which founded row drives its DMA, which BARs exist, +//! which of them a driver may map and which are mapped now, and how the +//! function can interrupt. A BAR is its index, its size and the length a +//! driver may map; **no address crosses** (res/DEVICE_SERVICE.md §1.1), so +//! nothing here says where a BAR lives. +//! +//! Descriptions only, as a census row is: a state grants no inventory, +//! claim, mapping or route, and reading one changes nothing. It is a +//! snapshot. A claim can land or a route can be released the moment after +//! the kernel answers, so no consumer may treat it as permission. +//! +//! The path is the census's: the kernel answers one +//! [`SystemMetric::DeviceState`](crate::session::SystemMetric::DeviceState) +//! or [`SystemMetric::DeviceBar`](crate::session::SystemMetric::DeviceBar) +//! observation on the `SystemObserve` capability the monitor holds; the +//! monitor samples them on each request (never cached, since they are live) +//! and vends [`DeviceStatePage`]s and one [`DeviceBarTable`]; exec relays +//! them to a launcher; the launcher stamps a [`DeviceInventory`] into the +//! process-context prefix of an executable whose bootstrap row declares it. + +use zerocopy::byteorder::{LittleEndian, U16, U64}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; + +use super::census::{DeviceCensus, DeviceCensusPage, MAX_DEVICE_CENSUS_ROWS}; + +/// BARs a PCI function has (PCI Local Bus Specification 3.0 §6.2.5: six +/// Base Address Registers in a type 0 header). +pub const DEVICE_BARS: usize = 6; +/// Every BAR's bit, for the three per-BAR bitmaps of a [`DeviceState`]. +pub const DEVICE_BAR_MASK: u8 = (1 << DEVICE_BARS) - 1; +/// `DeviceState::driver` when no founded row drives the function's DMA. +pub const DEVICE_ROW_NONE: u8 = 0xFF; +/// States per vended page: 8 header bytes + 128 = 136 bytes, the census +/// page's size, so a page rides the monitor and exec reply wires as one. +pub const DEVICE_STATE_PAGE_ROWS: usize = 16; +/// Pages the states of a full census span. +pub const DEVICE_STATE_PAGES: usize = MAX_DEVICE_CENSUS_ROWS.div_ceil(DEVICE_STATE_PAGE_ROWS); +/// Layout version of [`DeviceInventory`]. A zeroed record reads 0 and is +/// *unavailable*. +pub const DEVICE_INVENTORY_ABI: u16 = 1; +/// [`SystemMetric::DeviceBar`](crate::session::SystemMetric::DeviceBar)'s +/// operand is `index | bar << DEVICE_BAR_OPERAND_SHIFT`: the catalog index +/// in the low 13 bits (every index a census can hold fits, asserted below), +/// the BAR number in the top three. +pub const DEVICE_BAR_OPERAND_SHIFT: u32 = 13; + +/// The operand naming BAR `bar` of the function at catalog `index`; `None` +/// when either is out of the operand's range. +pub const fn device_bar_operand(index: u16, bar: u8) -> Option { + if index >> DEVICE_BAR_OPERAND_SHIFT != 0 || bar as usize >= DEVICE_BARS { + return None; + } + Some(index | (bar as u16) << DEVICE_BAR_OPERAND_SHIFT) +} + +/// The `(index, bar)` a [`device_bar_operand`] names; `None` for a BAR +/// number no function has. +pub const fn device_bar_of(operand: u16) -> Option<(u16, u8)> { + let bar = (operand >> DEVICE_BAR_OPERAND_SHIFT) as u8; + if bar as usize >= DEVICE_BARS { + return None; + } + Some((operand & ((1 << DEVICE_BAR_OPERAND_SHIFT) - 1), bar)) +} + +/// One function's live state. 8 bytes, frozen: one observation's value +/// word, byte for byte. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceState { + /// [`Self::PRESENT`] and the bits below it. + pub flags: u8, + /// The founded row (the founding record's index) whose task activated + /// DMA on this function, or [`DEVICE_ROW_NONE`]. The kernel's own + /// shutdown ledger, so it names who drives the device rather than who + /// claimed it: a function a row claimed and delegated reads the row it + /// was delegated to, and a driver that never activates DMA reads none. + pub driver: u8, + /// Bit `n`: BAR `n` exists as memory the kernel resolved. An I/O BAR is + /// not one: no driver may reach port space through a BAR. + pub bars_present: u8, + /// Bit `n`: a driver may map a prefix of BAR `n`. + pub bars_mappable: u8, + /// Bit `n`: BAR `n` is mapped by a driver now. + pub bars_mapped: u8, + /// Vectors the function's MSI capability can deliver (1, 2, 4, 8, 16 or + /// 32); 0 when it has none. + pub msi_vectors: u8, + /// Entries in the function's MSI-X table; 0 when it has none. + pub msix_entries: U16, +} + +impl DeviceState { + /// A populated state; clear marks the zero state. + pub const PRESENT: u8 = 1 << 0; + /// A driver holds the function: its claim is committed. + pub const CLAIMED: u8 = 1 << 1; + /// An interrupt route is established on the function now. + pub const ROUTED: u8 = 1 << 2; + /// A driver can establish a remapped MSI-family route (MSI or MSI-X), + /// the only kind userland may hold. + pub const ROUTE_MSI: u8 = 1 << 3; + /// A driver can establish a remapped MSI-X route. + pub const ROUTE_MSIX: u8 = 1 << 4; + /// The function has a legacy interrupt pin; [`Self::pin`] says which. + pub const HAS_PIN: u8 = 1 << 5; + /// Bits 6 and 7: the legacy pin, 0 for INTA through 3 for INTD. + pub const PIN_SHIFT: u32 = 6; + + pub const EMPTY: Self = Self { + flags: 0, + driver: 0, + bars_present: 0, + bars_mappable: 0, + bars_mapped: 0, + msi_vectors: 0, + msix_entries: U16::ZERO, + }; + + pub const fn is_present(&self) -> bool { + self.flags & Self::PRESENT != 0 + } + + pub const fn claimed(&self) -> bool { + self.flags & Self::CLAIMED != 0 + } + + /// The legacy interrupt pin, 0 for INTA through 3 for INTD. + pub const fn pin(&self) -> Option { + if self.flags & Self::HAS_PIN == 0 { + return None; + } + Some(self.flags >> Self::PIN_SHIFT) + } + + /// The founded row driving the function's DMA. + pub const fn driver(&self) -> Option { + if self.driver == DEVICE_ROW_NONE { + None + } else { + Some(self.driver) + } + } + + pub const fn validate(&self) -> bool { + if !self.is_present() { + return self.flags == 0 + && self.driver == 0 + && self.bars_present == 0 + && self.bars_mappable == 0 + && self.bars_mapped == 0 + && self.msi_vectors == 0 + && self.msix_entries.get() == 0; + } + // A pin without its flag would be a second spelling of "no pin". + (self.flags & Self::HAS_PIN != 0 || self.flags >> Self::PIN_SHIFT == 0) + && self.bars_present & !DEVICE_BAR_MASK == 0 + // Only a BAR that exists can be mapped, and only one a driver + // may map is. + && self.bars_mappable & !self.bars_present == 0 + && self.bars_mapped & !self.bars_mappable == 0 + && (self.msi_vectors == 0 + || (self.msi_vectors.is_power_of_two() && self.msi_vectors <= 32)) + && self.msix_entries.get() <= super::DeviceRecord::MAX_MSIX_TABLE_ENTRIES + } + + /// The state as one `SystemObserve` report, `(value, aux)`, with the + /// function's catalog `index` echoed in `aux[63:48]` so a reader can + /// prove the report answers the function it asked for. + pub const fn to_observation(&self, index: u16) -> (u64, u64) { + let value = u64::from_le_bytes([ + self.flags, + self.driver, + self.bars_present, + self.bars_mappable, + self.bars_mapped, + self.msi_vectors, + self.msix_entries.get().to_le_bytes()[0], + self.msix_entries.get().to_le_bytes()[1], + ]); + (value, (index as u64) << 48) + } + + /// Inverse of [`to_observation`](Self::to_observation); `None` when the + /// echoed index is not `index`, the rest of `aux` is not zero, or the + /// state is not a valid populated one. + pub const fn from_observation(index: u16, value: u64, aux: u64) -> Option { + if aux != (index as u64) << 48 { + return None; + } + let bytes = value.to_le_bytes(); + let state = Self { + flags: bytes[0], + driver: bytes[1], + bars_present: bytes[2], + bars_mappable: bytes[3], + bars_mapped: bytes[4], + msi_vectors: bytes[5], + msix_entries: U16::new(u16::from_le_bytes([bytes[6], bytes[7]])), + }; + if state.is_present() && state.validate() { + Some(state) + } else { + None + } + } +} + +/// One vended page of states, row for row beside the census's rows. +/// 136 bytes, frozen. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceStatePage { + /// Page ordinal, `0..DEVICE_STATE_PAGES`. + pub page: u8, + /// Populated states in this page, `0..=DEVICE_STATE_PAGE_ROWS`. + pub rows_valid: u8, + /// States the whole table holds: the census's `count`. + pub count: U16, + pub reserved: [u8; 4], + pub states: [DeviceState; DEVICE_STATE_PAGE_ROWS], +} + +impl DeviceStatePage { + pub const EMPTY: Self = Self { + page: 0, + rows_valid: 0, + count: U16::ZERO, + reserved: [0; 4], + states: [DeviceState::EMPTY; DEVICE_STATE_PAGE_ROWS], + }; + + /// Page `page` of a table of `count` states, empty: the assembler's + /// start. `None` past the table's last page, though a table of no states + /// still has page 0. + pub const fn new(page: u8, count: u16) -> Option { + let count = if count as usize > MAX_DEVICE_CENSUS_ROWS { + MAX_DEVICE_CENSUS_ROWS + } else { + count as usize + }; + let pages = count.div_ceil(DEVICE_STATE_PAGE_ROWS); + let last = if pages == 0 { 0 } else { pages - 1 }; + if page as usize > last { + return None; + } + Some(Self { + page, + rows_valid: 0, + count: U16::new(count as u16), + reserved: [0; 4], + states: [DeviceState::EMPTY; DEVICE_STATE_PAGE_ROWS], + }) + } + + /// The catalog index of this page's first state. + pub const fn first(&self) -> usize { + self.page as usize * DEVICE_STATE_PAGE_ROWS + } + + /// States this page holds once complete. + pub const fn expected(&self) -> usize { + let left = (self.count.get() as usize).saturating_sub(self.first()); + if left > DEVICE_STATE_PAGE_ROWS { + DEVICE_STATE_PAGE_ROWS + } else { + left + } + } + + /// Append the next state, from one kernel observation of the function at + /// the catalog index this page reaches next; `false` when the page is + /// full or the report does not decode. + pub fn push(&mut self, value: u64, aux: u64) -> bool { + let at = self.rows_valid as usize; + if at >= self.expected() { + return false; + } + let Some(state) = DeviceState::from_observation((self.first() + at) as u16, value, aux) + else { + return false; + }; + self.states[at] = state; + self.rows_valid += 1; + true + } + + pub fn validate(&self) -> bool { + let count = self.count.get() as usize; + if self.page as usize >= DEVICE_STATE_PAGES + || count > MAX_DEVICE_CENSUS_ROWS + || self.reserved != [0; 4] + || self.rows_valid as usize != self.expected() + { + return false; + } + let valid = self.rows_valid as usize; + self.states + .iter() + .enumerate() + .all(|(i, state)| state.validate() && state.is_present() == (i < valid)) + } +} + +/// One BAR's shape. 16 bytes, frozen. Lengths only: where the BAR lives is +/// not here and never will be. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceBar { + /// The BAR's whole size in bytes; 0 for a BAR the function has not. + pub size: U64, + /// Bytes from the BAR's start a driver may map: the whole BAR, or the + /// part before the page its MSI-X table or pending bits begin in, which + /// stay the kernel's. 0 when a driver may map none of it. + pub mappable: U64, +} + +impl DeviceBar { + pub const EMPTY: Self = Self { + size: U64::ZERO, + mappable: U64::ZERO, + }; + + /// Bits of `aux` the mappable length takes; the BAR number is echoed + /// above them. + const AUX_LENGTH_BITS: u32 = 56; + + pub const fn is_present(&self) -> bool { + self.size.get() != 0 + } + + pub const fn validate(&self) -> bool { + self.mappable.get() <= self.size.get() + } + + /// The BAR as one `SystemObserve` report, `(value, aux)`: its size, and + /// its mappable length with the BAR number echoed in `aux[58:56]`. + /// `None` when a length does not leave the echo room, which no BAR of a + /// machine with a 56-bit physical address space or less can reach. + pub const fn to_observation(&self, bar: u8) -> Option<(u64, u64)> { + if self.mappable.get() >> Self::AUX_LENGTH_BITS != 0 || bar as usize >= DEVICE_BARS { + return None; + } + Some(( + self.size.get(), + self.mappable.get() | (bar as u64) << Self::AUX_LENGTH_BITS, + )) + } + + /// Inverse of [`to_observation`](Self::to_observation); `None` when the + /// echoed BAR is not `bar` or more can be mapped than the BAR holds. + pub const fn from_observation(bar: u8, value: u64, aux: u64) -> Option { + if aux >> Self::AUX_LENGTH_BITS != bar as u64 { + return None; + } + let shape = Self { + size: U64::new(value), + mappable: U64::new(aux & ((1 << Self::AUX_LENGTH_BITS) - 1)), + }; + if shape.validate() { Some(shape) } else { None } + } +} + +/// The BARs of one function. 104 bytes, frozen. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceBarTable { + /// The function's catalog index: its row in the census. + pub index: U16, + /// 1 for a populated table; 0 marks the zero table. + pub present: u8, + pub reserved: [u8; 5], + pub bars: [DeviceBar; DEVICE_BARS], +} + +impl DeviceBarTable { + pub const EMPTY: Self = Self { + index: U16::ZERO, + present: 0, + reserved: [0; 5], + bars: [DeviceBar::EMPTY; DEVICE_BARS], + }; + + /// The table of the function at catalog `index`, every BAR absent: the + /// assembler's start. + pub const fn new(index: u16) -> Self { + Self { + index: U16::new(index), + present: 1, + reserved: [0; 5], + bars: [DeviceBar::EMPTY; DEVICE_BARS], + } + } + + pub const fn is_present(&self) -> bool { + self.present == 1 + } + + /// Set BAR `bar` from one kernel observation; `false` when the report + /// does not decode. + pub fn set_bar(&mut self, bar: u8, value: u64, aux: u64) -> bool { + let (Some(slot), Some(shape)) = ( + self.bars.get_mut(bar as usize), + DeviceBar::from_observation(bar, value, aux), + ) else { + return false; + }; + *slot = shape; + true + } + + pub fn validate(&self) -> bool { + match self.present { + 0 => *self == Self::EMPTY, + 1 => self.reserved == [0; 5] && self.bars.iter().all(DeviceBar::validate), + _ => false, + } + } +} + +/// What a device tool is launched with: the census, each of its rows' live +/// state, and the BARs of the one function the command line named, if it +/// named one the census holds. 3008 bytes, inside the process-context +/// prefix (asserted below). +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceInventory { + /// [`DEVICE_INVENTORY_ABI`]; 0 = unavailable. + pub abi: U16, + pub reserved: [u8; 6], + pub census: DeviceCensus, + /// `states[i]` is the live state of `census.rows[i]`, for every row the + /// census holds; the zero state past them. + pub states: [DeviceState; MAX_DEVICE_CENSUS_ROWS], + /// The named function's BARs, or the zero table. + pub bars: DeviceBarTable, +} + +impl DeviceInventory { + pub const EMPTY: Self = Self { + abi: U16::ZERO, + reserved: [0; 6], + census: DeviceCensus::EMPTY, + states: [DeviceState::EMPTY; MAX_DEVICE_CENSUS_ROWS], + bars: DeviceBarTable::EMPTY, + }; + + /// An available inventory over `census`, with no state and no BARs yet. + pub const fn new(census: DeviceCensus) -> Self { + Self { + abi: U16::new(DEVICE_INVENTORY_ABI), + reserved: [0; 6], + census, + states: [DeviceState::EMPTY; MAX_DEVICE_CENSUS_ROWS], + bars: DeviceBarTable::EMPTY, + } + } + + pub const fn available(&self) -> bool { + self.abi.get() == DEVICE_INVENTORY_ABI + } + + /// Fold one vended page of states in; `false`, changing nothing, when + /// the page does not validate or is not a page of this census's table. + pub fn insert_states(&mut self, page: &DeviceStatePage) -> bool { + if !self.available() || !page.validate() || page.count != self.census.count { + return false; + } + let first = page.first(); + let valid = page.rows_valid as usize; + self.states[first..first + valid].copy_from_slice(&page.states[..valid]); + true + } + + /// Whether every census row has its state. + pub fn complete(&self) -> bool { + let count = self.census.rows().len(); + self.available() && self.states[..count].iter().all(DeviceState::is_present) + } + + pub fn validate(&self) -> bool { + if self.abi.get() == 0 { + return *self == Self::EMPTY; + } + let count = self.census.rows().len(); + self.abi.get() == DEVICE_INVENTORY_ABI + && self.reserved == [0; 6] + && self.census.available() + && self.census.validate() + && self + .states + .iter() + .enumerate() + .all(|(i, state)| state.validate() && (i < count || !state.is_present())) + && self.bars.validate() + && (!self.bars.is_present() || (self.bars.index.get() as usize) < count) + } +} + +/// Bytes of the device payload a monitor or exec reply carries. +pub const DEVICE_PAGE_BYTES: usize = 136; + +/// The device payload slot of the monitor's and exec's reply wires: one +/// census page, one page of states, or one function's BARs, as the reply's +/// tag says, and zero on every other reply. Bytes, so the three records +/// share the slot without one posing as another: each accessor decodes and +/// validates its own. +#[repr(transparent)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DevicePage(pub [u8; DEVICE_PAGE_BYTES]); + +impl DevicePage { + pub const EMPTY: Self = Self([0; DEVICE_PAGE_BYTES]); + + pub fn of_census(page: &DeviceCensusPage) -> Self { + let mut out = Self::EMPTY; + out.0.copy_from_slice(page.as_bytes()); + out + } + + /// The census page this slot carries, validated. + pub fn census(&self) -> Option { + DeviceCensusPage::read_from_bytes(&self.0) + .ok() + .filter(DeviceCensusPage::validate) + } + + pub fn of_states(page: &DeviceStatePage) -> Self { + let mut out = Self::EMPTY; + out.0.copy_from_slice(page.as_bytes()); + out + } + + /// The page of states this slot carries, validated. + pub fn states(&self) -> Option { + DeviceStatePage::read_from_bytes(&self.0) + .ok() + .filter(DeviceStatePage::validate) + } + + pub fn of_bars(table: &DeviceBarTable) -> Self { + let mut out = Self::EMPTY; + out.0[..core::mem::size_of::()].copy_from_slice(table.as_bytes()); + out + } + + /// The BAR table this slot carries: a populated, valid one, with the + /// slot's remaining bytes zero. + pub fn bars(&self) -> Option { + let (table, rest) = DeviceBarTable::read_from_prefix(&self.0).ok()?; + (table.is_present() && table.validate() && rest.iter().all(|byte| *byte == 0)) + .then_some(table) + } +} + +/// A PCI function's address as a person writes it: `ssss:bb:dd.f`, or +/// `bb:dd.f` in segment 0, every field hexadecimal but the function, which +/// is one digit `0` to `7`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PciAddress { + pub segment: u16, + /// `bus << 8 | device << 3 | function`, a census row's `locator`. + pub locator: u16, +} + +impl PciAddress { + /// Bytes of the longest spelling, `ssss:bb:dd.f`. + pub const MAX_TEXT: usize = 12; + + /// Parse `text`, the whole of it. `None` unless it is exactly an + /// address: a launcher looks for one among a command line's words, so a + /// word that merely resembles one must not name a device. + pub fn parse(text: &[u8]) -> Option { + let (rest, function) = text.split_last_chunk::<2>()?; + let [b'.', function @ b'0'..=b'7'] = *function else { + return None; + }; + let mut fields = rest.rsplitn(3, |byte| *byte == b':'); + let device = hex_field(fields.next()?, 2)?; + let bus = hex_field(fields.next()?, 2)?; + let segment = match fields.next() { + Some(field) => hex_field(field, 4)?, + None => 0, + }; + if device > 0x1F { + return None; + } + Some(Self { + segment, + locator: bus << 8 | device << 3 | (function - b'0') as u16, + }) + } + + /// The census row this address names. + pub fn row_in(&self, census: &DeviceCensus) -> Option { + census + .rows() + .iter() + .position(|row| row.segment.get() == self.segment && row.locator.get() == self.locator) + .map(|index| index as u16) + } +} + +/// `field` as exactly `digits` hexadecimal digits. +fn hex_field(field: &[u8], digits: usize) -> Option { + if field.len() != digits { + return None; + } + field.iter().try_fold(0u16, |value, byte| { + let digit = (*byte as char).to_digit(16)?; + Some(value << 4 | digit as u16) + }) +} + +const _: () = assert!(core::mem::size_of::() == 8); +const _: () = assert!(core::mem::size_of::() == 8 + 8 * DEVICE_STATE_PAGE_ROWS); +const _: () = assert!( + core::mem::size_of::() == core::mem::size_of::() +); +const _: () = assert!(core::mem::size_of::() == DEVICE_PAGE_BYTES); +const _: () = assert!(core::mem::size_of::() == DEVICE_PAGE_BYTES); +const _: () = assert!(core::mem::size_of::() <= DEVICE_PAGE_BYTES); +const _: () = assert!(core::mem::size_of::() == 16); +const _: () = assert!(core::mem::size_of::() == 8 + 16 * DEVICE_BARS); +const _: () = assert!(core::mem::size_of::() == 3008); +const _: () = + assert!(core::mem::size_of::() <= crate::process::PROCESS_CONTEXT_BYTES); +const _: () = assert!(DEVICE_STATE_PAGES <= u8::MAX as usize); +// Every index a census row can carry fits the BAR operand beside the BAR. +const _: () = assert!(MAX_DEVICE_CENSUS_ROWS <= 1 << DEVICE_BAR_OPERAND_SHIFT); +const _: () = assert!(DEVICE_BARS <= 1 << (u16::BITS - DEVICE_BAR_OPERAND_SHIFT)); + +#[cfg(test)] +mod tests { + use super::*; + use crate::device::DeviceCensusRow; + + fn state(flags: u8) -> DeviceState { + DeviceState { + flags: DeviceState::PRESENT | flags, + driver: DEVICE_ROW_NONE, + bars_present: 0b01_0001, + bars_mappable: 0b00_0001, + bars_mapped: 0, + msi_vectors: 1, + msix_entries: U16::new(0), + } + } + + fn census(count: u16) -> DeviceCensus { + let mut census = DeviceCensus::new(count, 1); + for index in 0..count { + census.rows[index as usize] = DeviceCensusRow { + locator: U16::new(index << 3), + present: 1, + ..DeviceCensusRow::EMPTY + }; + } + census + } + + #[test] + fn a_state_round_trips_one_observation() { + let mut state = state(DeviceState::CLAIMED | DeviceState::ROUTE_MSI | DeviceState::HAS_PIN); + state.flags |= 2 << DeviceState::PIN_SHIFT; + state.driver = 7; + state.msix_entries = U16::new(0); + let (value, aux) = state.to_observation(41); + assert_eq!(DeviceState::from_observation(41, value, aux), Some(state)); + assert_eq!(state.pin(), Some(2)); + assert_eq!(state.driver(), Some(7)); + assert!(state.claimed()); + // Another function's report, or one with stray auxiliary bits, is + // not this function's state. + assert_eq!(DeviceState::from_observation(40, value, aux), None); + assert_eq!(DeviceState::from_observation(41, value, aux | 1), None); + } + + #[test] + fn an_impossible_state_is_refused() { + let ok = state(0); + assert!(ok.validate()); + // The zero state is the empty row, never a kernel answer. + assert!(DeviceState::EMPTY.validate()); + assert_eq!(DeviceState::from_observation(0, 0, 0), None); + for broken in [ + DeviceState { + bars_present: 0x40, + ..ok + }, + DeviceState { + bars_mappable: 0b10, + ..ok + }, + DeviceState { + bars_mapped: 0b01_0000, + ..ok + }, + DeviceState { + msi_vectors: 3, + ..ok + }, + DeviceState { + msi_vectors: 64, + ..ok + }, + DeviceState { + flags: ok.flags | 1 << DeviceState::PIN_SHIFT, + ..ok + }, + DeviceState { + msix_entries: U16::new(crate::device::DeviceRecord::MAX_MSIX_TABLE_ENTRIES + 1), + ..ok + }, + ] { + assert!(!broken.validate(), "{broken:?}"); + } + } + + #[test] + fn a_page_takes_exactly_its_rows_in_order() { + // 20 states: page 0 holds 16, page 1 the last 4, and there is no + // page 2. + let mut first = DeviceStatePage::new(0, 20).unwrap(); + for index in 0..16u16 { + let (value, aux) = state(0).to_observation(index); + assert!(first.push(value, aux)); + } + let (value, aux) = state(0).to_observation(16); + assert!(!first.push(value, aux), "a full page takes no more"); + assert!(first.validate()); + + let mut second = DeviceStatePage::new(1, 20).unwrap(); + // The next state is index 16's; index 17's report is out of order. + let (value, aux) = state(0).to_observation(17); + assert!(!second.push(value, aux)); + assert!(!second.validate(), "a short page is not a page"); + for index in 16..20u16 { + let (value, aux) = state(0).to_observation(index); + assert!(second.push(value, aux)); + } + assert!(second.validate()); + assert!(DeviceStatePage::new(2, 20).is_none()); + // A machine with no function still has an empty page 0. + assert!(DeviceStatePage::new(0, 0).unwrap().validate()); + assert!(DeviceStatePage::new(1, 0).is_none()); + } + + #[test] + fn a_bar_round_trips_and_never_maps_more_than_it_holds() { + let bar = DeviceBar { + size: U64::new(0x10_0000), + mappable: U64::new(0x8000), + }; + let (value, aux) = bar.to_observation(4).unwrap(); + assert_eq!(DeviceBar::from_observation(4, value, aux), Some(bar)); + assert_eq!(DeviceBar::from_observation(3, value, aux), None); + // An absent BAR is the zero shape, and still answers for its number. + let (value, aux) = DeviceBar::EMPTY.to_observation(5).unwrap(); + assert_eq!( + DeviceBar::from_observation(5, value, aux), + Some(DeviceBar::EMPTY) + ); + assert_eq!(DeviceBar::from_observation(0, 0x1000, 0x2000), None); + assert!(bar.to_observation(6).is_none()); + let wide = DeviceBar { + size: U64::new(u64::MAX), + mappable: U64::new(1 << 56), + }; + assert!(wide.to_observation(0).is_none()); + } + + #[test] + fn a_bar_operand_names_a_function_and_a_bar() { + let operand = device_bar_operand(119, 5).unwrap(); + assert_eq!(device_bar_of(operand), Some((119, 5))); + assert_eq!(device_bar_operand(0, 0), Some(0)); + assert!(device_bar_operand(1 << DEVICE_BAR_OPERAND_SHIFT, 0).is_none()); + assert!(device_bar_operand(0, 6).is_none()); + assert!(device_bar_of(7 << DEVICE_BAR_OPERAND_SHIFT).is_none()); + } + + #[test] + fn an_inventory_holds_a_state_per_census_row() { + let mut inventory = DeviceInventory::new(census(20)); + assert!(inventory.validate()); + assert!(!inventory.complete()); + for page in 0..2u8 { + let mut states = DeviceStatePage::new(page, 20).unwrap(); + for index in states.first()..states.first() + states.expected() { + let (value, aux) = state(0).to_observation(index as u16); + assert!(states.push(value, aux)); + } + assert!(inventory.insert_states(&states)); + } + assert!(inventory.complete()); + assert!(inventory.validate()); + // A page of another census's table is refused whole. + let other = DeviceStatePage::new(0, 21).unwrap(); + assert!(!inventory.insert_states(&other)); + + let mut bars = DeviceBarTable::new(19); + let shape = DeviceBar { + size: U64::new(0x4000), + mappable: U64::new(0x4000), + }; + let (value, aux) = shape.to_observation(0).unwrap(); + assert!(bars.set_bar(0, value, aux)); + assert!(!bars.set_bar(1, value, aux), "the echo names BAR 0"); + inventory.bars = bars; + assert!(inventory.validate()); + // BARs of a function the census does not hold are not this + // inventory's. + inventory.bars.index = U16::new(20); + assert!(!inventory.validate()); + assert!(DeviceInventory::EMPTY.validate()); + assert!(!DeviceInventory::EMPTY.available()); + } + + #[test] + fn a_page_slot_decodes_only_the_record_it_carries() { + let mut states = DeviceStatePage::new(0, 2).unwrap(); + for index in 0..2u16 { + let (value, aux) = state(0).to_observation(index); + assert!(states.push(value, aux)); + } + let slot = DevicePage::of_states(&states); + assert_eq!(slot.states(), Some(states)); + + let mut table = DeviceBarTable::new(3); + let shape = DeviceBar { + size: U64::new(0x1000), + mappable: U64::new(0x1000), + }; + let (value, aux) = shape.to_observation(2).unwrap(); + assert!(table.set_bar(2, value, aux)); + let slot = DevicePage::of_bars(&table); + assert_eq!(slot.bars(), Some(table)); + // Bytes past the table are not part of it. + let mut stray = slot; + stray.0[DEVICE_PAGE_BYTES - 1] = 1; + assert_eq!(stray.bars(), None); + + let page = census(3).page(0).unwrap(); + assert_eq!(DevicePage::of_census(&page).census(), Some(page)); + // The zero slot is no record at all. + assert_eq!(DevicePage::EMPTY.bars(), None); + assert_eq!( + DevicePage::EMPTY.census().map(|page| page.rows_valid), + Some(0) + ); + } + + #[test] + fn an_address_is_exactly_an_address() { + let address = PciAddress::parse(b"0000:00:1f.3").unwrap(); + assert_eq!((address.segment, address.locator), (0, 0x1F << 3 | 3)); + assert_eq!(PciAddress::parse(b"00:1f.3"), Some(address)); + assert_eq!( + PciAddress::parse(b"0001:ab:02.0"), + Some(PciAddress { + segment: 1, + locator: 0xAB << 8 | 2 << 3 + }) + ); + for not in [ + &b""[..], + b"1f.3", + b"00:1f", + b"00:1f.8", + b"00:20.0", + b"0:1f.3", + b"000:00:1f.3", + b"00:1f.3x", + b"x0:1f.3", + b"0000:00:1f:3", + b"00000:00:1f.3", + b"8086:a828", + ] { + assert_eq!( + PciAddress::parse(not), + None, + "{:?}", + core::str::from_utf8(not) + ); + } + let census = census(4); + assert_eq!( + PciAddress { + segment: 0, + locator: 3 << 3 + } + .row_in(&census), + Some(3) + ); + assert_eq!( + PciAddress { + segment: 1, + locator: 3 << 3 + } + .row_in(&census), + None + ); + } +} diff --git a/native/crates/abi/src/process.rs b/native/crates/abi/src/process.rs index 48b0841..b1e72eb 100644 --- a/native/crates/abi/src/process.rs +++ b/native/crates/abi/src/process.rs @@ -565,8 +565,11 @@ pub const PROCESS_OUTPUT_BYTES: usize = PROCESS_IO_BYTES - HEADER_BYTES - PROCES /// would take that word for zero-or-corrupt. Each change shifts an offset or /// a meaning and therefore burns a version. 13: the system snapshot's /// scheduler-timer arm census, paid from the argument buffer together with -/// the output bytes its status row needs, so no report lost a row. -pub const PROCESS_IO_ABI: u16 = 13; +/// the output bytes its status row needs, so no report lost a row. 14: the +/// second reserved word gains a meaning, the length of the command name the +/// launcher resolved the program by, which rides the argument buffer after +/// the tail ([`ProcessIo::command`]): a `13` reader requires that word zero. +pub const PROCESS_IO_ABI: u16 = 14; /// One launch-time snapshot of exec's bounded process table. /// @@ -717,9 +720,12 @@ pub struct ProcessIo { /// holds the frame service's diagnostic face. net: crate::monitor::NetSnapshot, // `reserved[0]` was intentionally available for an additive ABI field. - // It now carries the immutable launcher-context live length; the remaining - // words stay reserved and must be zero. + // It now carries the immutable launcher-context live length, and + // `reserved[1]` the command name's length; the remaining words stay + // reserved and must be zero. reserved: [U32; 4], + /// The argument tail, `arg_len` bytes, then the command name, + /// `reserved[1]` bytes; zero after them. args: [u8; PROCESS_ARG_BYTES], /// Alternating payload: launcher context before START, child output after /// execution. The task lifecycle is the exclusive-ownership boundary. @@ -776,8 +782,10 @@ impl ProcessIo { && self.flags.get() == 0 && self.telemetry.validate() && self.filesystem.validate() - && self.reserved[1..].iter().all(|v| v.get() == 0) + && self.reserved[2..].iter().all(|v| v.get() == 0) && usize::from(self.arg_len.get()) <= PROCESS_ARG_BYTES + && usize::try_from(self.reserved[1].get()) + .is_ok_and(|name| name <= PROCESS_ARG_BYTES - usize::from(self.arg_len.get())) && usize::try_from(self.reserved[0].get()).is_ok_and(|len| len <= PROCESS_CONTEXT_BYTES) && usize::from(self.output_len.get()) <= PROCESS_OUTPUT_BYTES && (self.reserved[0].get() == 0 || self.output_len.get() == 0) @@ -791,6 +799,36 @@ impl ProcessIo { Ok(&self.args[..usize::from(self.arg_len.get())]) } + /// The name the launcher resolved this program by: the first word of the + /// command line it was run from, which the tail in [`Self::args`] does + /// not hold. Empty when the launcher gave none. A label: it selects + /// nothing and proves nothing about the image. + pub fn command(&self) -> Result<&[u8], ProcessIoError> { + if !self.validate() { + return Err(ProcessIoError::Invalid); + } + let tail = usize::from(self.arg_len.get()); + Ok(&self.args[tail..tail + self.reserved[1].get() as usize]) + } + + /// Publish the command name before the child starts. It shares the + /// argument buffer with the tail, so a name the buffer has no room for + /// after the tail is refused whole and the page is unchanged. + pub fn write_command(&mut self, name: &[u8]) -> Result<(), ProcessIoError> { + if !self.validate() { + return Err(ProcessIoError::Invalid); + } + let tail = usize::from(self.arg_len.get()); + let end = tail + .checked_add(name.len()) + .filter(|end| *end <= PROCESS_ARG_BYTES) + .ok_or(ProcessIoError::ArgumentsTooLong)?; + self.args[tail..].fill(0); + self.args[tail..end].copy_from_slice(name); + self.reserved[1].set(name.len() as u32); + Ok(()) + } + /// Immutable launcher context for this invocation. It is data, never an /// authority selector; a program must still hold every capability needed /// for the operation it describes. @@ -1248,6 +1286,55 @@ mod tests { assert_eq!(page.output(), Err(ProcessIoError::Invalid)); } + #[test] + fn the_command_name_rides_after_the_tail_and_leaves_it_alone() { + let mut page = ProcessIo::new_zeroed(); + page.initialize_with_context( + b"-v 'two words'", + b"context", + crate::monitor::TelemetrySnapshot::EMPTY, + ) + .unwrap(); + // A launcher that names nothing leaves the name empty. + assert_eq!(page.command().unwrap(), b""); + page.write_command(b"answer").unwrap(); + assert_eq!(page.command().unwrap(), b"answer"); + assert_eq!(page.args().unwrap(), b"-v 'two words'"); + assert_eq!(page.context().unwrap(), b"context"); + // A second name replaces the first whole, a shorter one included. + page.write_command(b"a").unwrap(); + assert_eq!(page.command().unwrap(), b"a"); + // The buffer is the tail's and the name's together: a name that + // does not fit after the tail is refused and nothing changes. + let long = [b'n'; PROCESS_ARG_BYTES]; + assert_eq!( + page.write_command(&long), + Err(ProcessIoError::ArgumentsTooLong) + ); + assert_eq!(page.command().unwrap(), b"a"); + assert_eq!(page.args().unwrap(), b"-v 'two words'"); + let fits = &long[..PROCESS_ARG_BYTES - page.args().unwrap().len()]; + page.write_command(fits).unwrap(); + assert_eq!(page.command().unwrap(), fits); + // The next launch starts with no name. + page.initialize(b"", crate::monitor::TelemetrySnapshot::EMPTY) + .unwrap(); + assert_eq!(page.command().unwrap(), b""); + } + + #[test] + fn a_command_name_past_the_buffer_fails_closed() { + let mut page = ProcessIo::new_zeroed(); + page.initialize(b"tail", crate::monitor::TelemetrySnapshot::EMPTY) + .unwrap(); + page.reserved[1].set((PROCESS_ARG_BYTES - 3) as u32); + assert_eq!(page.command(), Err(ProcessIoError::Invalid)); + assert_eq!(page.args(), Err(ProcessIoError::Invalid)); + page.reserved[1].set(0); + page.reserved[2].set(1); + assert_eq!(page.args(), Err(ProcessIoError::Invalid)); + } + #[test] fn malformed_context_length_fails_closed() { let mut page = ProcessIo::new_zeroed(); diff --git a/native/crates/abi/src/session.rs b/native/crates/abi/src/session.rs index 8d05539..c010a5a 100644 --- a/native/crates/abi/src/session.rs +++ b/native/crates/abi/src/session.rs @@ -3145,6 +3145,21 @@ pub enum SystemMetric { IommuUnit = 35, /// Indexed capabilities and scope counts, Unit::features() observation. IommuUnitFeatures = 36, + /// One PCI function's live state; operand = its catalog index, as for + /// [`DeviceRow`](Self::DeviceRow). The pair is + /// [`DeviceState::to_observation`](crate::device::DeviceState::to_observation): + /// whether a driver claimed it, the founded row driving its DMA, which + /// BARs exist, may be mapped and are mapped, and how it can interrupt. + /// A snapshot, never permission; no address. + DeviceState = 37, + /// One BAR's shape; operand = + /// [`device_bar_operand`](crate::device::device_bar_operand) of the + /// function's catalog index and the BAR number. The pair is + /// [`DeviceBar::to_observation`](crate::device::DeviceBar::to_observation): + /// the BAR's size and the length a driver may map. Lengths only: where + /// the BAR lives never crosses. A BAR the function has not answers the + /// zero shape. + DeviceBar = 38, } /// Records the kernel retains for [`SystemMetric::IommuFaultRecord`]. @@ -3516,6 +3531,8 @@ impl SystemObserveReq { matches!( metric, SystemMetric::DeviceRow + | SystemMetric::DeviceState + | SystemMetric::DeviceBar | SystemMetric::ProcessorCapture | SystemMetric::ProcessorIdentity | SystemMetric::ProcessorTopology diff --git a/native/crates/abi/src/svc.rs b/native/crates/abi/src/svc.rs index 437e013..469b2d9 100644 --- a/native/crates/abi/src/svc.rs +++ b/native/crates/abi/src/svc.rs @@ -205,6 +205,13 @@ impl MonitorReqWire { /// records. The monitor assembles the census from its own /// `SystemObserve` capability; the request carries no authority. pub const OP_DEVICE_CENSUS: u8 = 2; + /// One page of the census functions' live state + /// (`crate::device::state`): `reserved[0]` is the page ordinal. Sampled + /// when asked, never cached; the request carries no authority. + pub const OP_DEVICE_STATES: u8 = 5; + /// One function's BAR shapes: `reserved[0..2]` is its catalog index, + /// little-endian. Lengths only. + pub const OP_DEVICE_BARS: u8 = 6; } /// Monitor reply: a complete bounded snapshot or a closed refusal. The @@ -222,9 +229,11 @@ pub struct MonitorRepWire { /// Tagged observation payload: ProcessorPage for processor replies, or /// iommu_status::Snapshot encoded by its wire-page methods for IOMMU replies. pub processor: crate::processor_status::ProcessorPage, - /// [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS) only; the zero page - /// on every other tag (the codec refuses a stray one). - pub census: crate::device::DeviceCensusPage, + /// The device record of [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS), + /// [`TAG_DEVICE_STATES`](Self::TAG_DEVICE_STATES) or + /// [`TAG_DEVICE_BARS`](Self::TAG_DEVICE_BARS); the zero slot on every + /// other tag (the codec refuses a stray one). + pub device: crate::device::DevicePage, } impl MonitorRepWire { @@ -233,6 +242,8 @@ impl MonitorRepWire { pub const TAG_SNAPSHOT: u8 = 1; pub const TAG_REFUSED: u8 = 2; pub const TAG_DEVICE_CENSUS: u8 = 3; + pub const TAG_DEVICE_STATES: u8 = 6; + pub const TAG_DEVICE_BARS: u8 = 7; pub const REFUSE_DENIED: u8 = 1; } @@ -1527,6 +1538,15 @@ impl TaskReqWire { /// makes executable from its own lease. The generation binds no global /// name: only the capability's holder reaches it. pub const OP_INSTALL_FILE: u8 = 26; + /// One page of the census functions' live state + /// (`crate::device::state`), relayed from the monitor as + /// [`OP_DEVICE_CENSUS`](Self::OP_DEVICE_CENSUS)'s pages are, but never + /// cached: `slot` is the page ordinal. Bind-inert data, no grants, + /// answered on any launcher pair. + pub const OP_DEVICE_STATES: u8 = 27; + /// One function's BAR shapes, relayed from the monitor: `slot` is the + /// function's catalog index. Lengths only; no grants. + pub const OP_DEVICE_BARS: u8 = 28; /// Every retired `TaskReqWire` discriminant, in one definition the /// decoder cites (the [`derive_kind_known`](crate::session::derive_kind_known) @@ -1700,9 +1720,11 @@ pub struct TaskRepWire { /// Tagged observation payload: ProcessorPage for processor replies, or /// iommu_status::Snapshot encoded by its wire-page methods for IOMMU replies. pub processor: crate::processor_status::ProcessorPage, - /// [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS)'s page; the zero - /// page on every other tag. - pub census: crate::device::DeviceCensusPage, + /// The device record of [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS), + /// [`TAG_DEVICE_STATES`](Self::TAG_DEVICE_STATES) or + /// [`TAG_DEVICE_BARS`](Self::TAG_DEVICE_BARS); the zero slot on every + /// other tag. + pub device: crate::device::DevicePage, /// Only `REFUSE_LEDGER_FULL` with `index == 1` may carry this bounded /// failure snapshot. Every other reply requires the all-zero form. pub cleanup: CleanupRowDiagnosticWire, @@ -1845,6 +1867,10 @@ impl TaskRepWire { /// which a caller may retry with; nothing was registered, and the supply /// was retired, so its memory is the caller's again. pub const TAG_INSTALL_SUPPLY_SHORT: u8 = 27; + /// Reply to [`TaskReqWire::OP_DEVICE_STATES`]: `device` is the page. + pub const TAG_DEVICE_STATES: u8 = 28; + /// Reply to [`TaskReqWire::OP_DEVICE_BARS`]: `device` is the table. + pub const TAG_DEVICE_BARS: u8 = 29; /// The unit [`TAG_INSTALL_SUPPLY_SHORT`](Self::TAG_INSTALL_SUPPLY_SHORT) /// counts in: mm grants a base-class supply in whole 4 KiB pages, so a /// size the caller can ask for is a multiple of it. @@ -3939,6 +3965,16 @@ impl InReqWire { /// `InputStreamRequest::Reprint` — echo the line being edited, cursor /// in place, then apply every key queued since `Hold`. pub const OP_REPRINT: u8 = 6; + /// `InputStreamRequest::Insert` — type text into the **held** line at + /// its cursor, as if typed, without echo (the `Reprint` shows it): + /// `reserved0` = the byte count (1 to [`INSERT_BYTES`](Self::INSERT_BYTES)), + /// `reserved2..reserved4` = the bytes, little-endian, the words past the + /// count zero. Printable ASCII only: the line takes bytes up to the first + /// it cannot hold — any other byte, or one past a full line — so no + /// insert can ever complete a line. + pub const OP_INSERT: u8 = 7; + /// The most one `Insert` carries: its three data words. + pub const INSERT_BYTES: usize = 24; } /// The stream-input-service reply wire form. The legacy stream counters are @@ -4031,6 +4067,9 @@ impl InRepWire { /// `InputStreamResponse::Reprinted` — the line is echoed again and the /// held keys applied. pub const TAG_REPRINTED: u8 = 7; + /// `InputStreamResponse::Inserted` — `produced` = how many of the + /// insert's bytes the line took (zero to its count). + pub const TAG_INSERTED: u8 = 8; /// `InputStreamRefusal::Busy` — a queue is already attached (the v1 /// single-peer bound, `res/USERLAND.md` §5.9). @@ -4040,6 +4079,9 @@ impl InRepWire { /// `InputStreamRefusal::BadQueue` — the delivered slot was absent or /// not a well-formed queue window. pub const REFUSE_BAD_QUEUE: u8 = 3; + /// `InputStreamRefusal::NotHeld` — an insert into a line that is not + /// held, or not being edited (a completed line still in flight). + pub const REFUSE_NOT_HELD: u8 = 4; } /// One data-plane chunk — the `ring::spsc` record the backend @@ -4476,6 +4518,32 @@ impl SurReqWire { /// compositor snapshots the name and ends the pick. No subject /// ([`SurRepWire::TAG_PICK_ANSWERED`]). pub const OP_PICKED: u8 = 13; + /// `SurfaceRequest::CopyChunk` — one piece of the text this window puts + /// on its seat's clipboard (`res/CLIPBOARD.md`), written at the head of + /// its pair lease's request plane (`kore_window::menu`'s page): `a` = + /// the piece's offset in the text `| len << 32` (1 to the plane's size), + /// `b` = the whole text's byte length (1 to + /// `kore_window::clipboard::CLIPBOARD_BYTES`). The piece at offset zero + /// begins a copy, and only the seat's focused window may begin one, on + /// a gesture of the user's it has not spent; later pieces continue it + /// in order, and the piece that reaches the length ends it. The + /// compositor copies each piece out before replying, so the plane may be + /// reused once answered. No subject ([`SurRepWire::TAG_COPIED`]). + pub const OP_COPY_CHUNK: u8 = 14; + /// `SurfaceRequest::Paste` — read the seat's clipboard into this + /// window's request plane: `a` = the offset to read from. Offset zero + /// begins a read, and only the seat's focused window may begin one, on + /// a gesture of the user's it has not spent; later offsets continue it + /// until it reaches the end or the clipboard changes. No subject + /// ([`SurRepWire::TAG_CLIPBOARD`], [`SurRepWire::TAG_CLIPBOARD_EMPTY`]). + pub const OP_PASTE: u8 = 15; + /// `SurfaceRequest::PasteToShell` — the seat's terminal window asks for + /// the clipboard to be typed into its shell's line: queued as an inert + /// `ShellAction::Paste` for its controller, which reads the text and + /// edits the line itself. Only the seat's focused terminal window may + /// ask, on a gesture it has not spent. No words, no subject + /// ([`SurRepWire::TAG_PASTE_QUEUED`], [`SurRepWire::TAG_CLIPBOARD_EMPTY`]). + pub const OP_PASTE_TO_SHELL: u8 = 16; /// [`OP_SET_CURSOR`](Self::OP_SET_CURSOR) `a`: the standard arrow. pub const CURSOR_DEFAULT: u64 = 0; @@ -4597,6 +4665,20 @@ impl SurRepWire { /// `SurfaceResponse::PickAnswered` — `a` echoes the token answered; the /// pick has ended and the plane may be reused. pub const TAG_PICK_ANSWERED: u8 = 14; + /// `SurfaceResponse::Copied` — `a` = the text copied so far (the + /// piece's offset plus its length, echoed); when it is the whole + /// length, the text is the seat's clipboard. + pub const TAG_COPIED: u8 = 15; + /// `SurfaceResponse::Clipboard` — `a` = the offset read from (echoed) + /// `| len << 32`, the bytes now at the head of the request plane (1 to + /// the plane's size), `b` = the clipboard text's whole byte length. + pub const TAG_CLIPBOARD: u8 = 16; + /// `SurfaceResponse::ClipboardEmpty` — the seat's clipboard holds no + /// text. No words. + pub const TAG_CLIPBOARD_EMPTY: u8 = 17; + /// `SurfaceResponse::PasteQueued` — `a` = the byte length of the text + /// the controller was asked to type. + pub const TAG_PASTE_QUEUED: u8 = 18; /// A v1 vocabulary gate: an unknown op, a non-XRGB8888 format, a /// non-CPU reach, a non-LINEAR modifier, a non-empty flag set. @@ -4613,6 +4695,10 @@ impl SurRepWire { /// A capability transfer or teardown did not settle. A retiring surface /// keeps its backing reserved until revocation succeeds; retry is safe. pub const REFUSE_BUSY: u8 = 6; + /// The user did not ask: a clipboard verb from a window that is not its + /// seat's focused window, or with no unspent gesture of the user's + /// behind it. + pub const REFUSE_DENIED: u8 = 7; } /// One damage rectangle of a [`CommitWire`] record — pixel-addressed, @@ -4956,7 +5042,17 @@ impl FsReqWire { /// route no longer validates. The view must belong to the caller's task. /// `window_offset` is return-post correlation. pub const OP_MATCH_VIEW: u8 = 65; - pub const OP_MAX: u8 = Self::OP_MATCH_VIEW; + /// Arm watch slot `kind` on the directory `handle`, tagged `length` + /// (non-zero, the client's own name for this arming): from now on the + /// pair's watch plane — the event channel after the rings in this pair's + /// own window — counts every change to what a listing of that directory + /// shows. Re-arming a slot replaces its watch. Observation, not + /// authority: the handle's route is revalidated before every count, and + /// a route that stops proving ends the watch. Answered `Done`. + pub const OP_WATCH: u8 = 66; + /// Disarm watch slot `kind`. Answered `Done`, armed or not. + pub const OP_UNWATCH: u8 = 67; + pub const OP_MAX: u8 = Self::OP_UNWATCH; /// [`OP_STOR_MAINTAIN`](Self::OP_STOR_MAINTAIN): report only. pub const MAINTAIN_STATUS: u8 = 0; @@ -5119,6 +5215,59 @@ impl FsDirEntryWire { pub const BYTES: usize = 48; } +/// Watch slots one filesystem pair may arm ([`FsReqWire::OP_WATCH`]). +/// +/// Sized by the client that follows the most folders at once: the Files +/// explorer watches the folder it shows and the sidebar's live folders (its +/// root and the expanded folders whose lines are on screen), fifteen of them +/// at most. Every armed watch holds a handle, so this stays under half the +/// service's per-session handle bound (32, `kore_dagfs::bounds`) — a client +/// watching every slot keeps the handles a listing needs. The plane's +/// `ended` word is one bit per slot, so 32 is the layout's own ceiling. +pub const FS_WATCH_SLOTS: usize = 16; +const _: () = assert!(FS_WATCH_SLOTS <= u32::BITS as usize); + +/// The level state of a filesystem pair's **watch plane**: the event channel +/// (`res/EVENT.md`) the service formats after the request/reply rings in the +/// pair's own window and republishes whole whenever a watched directory +/// changes. Slot `k` belongs to the watch [`FsReqWire::OP_WATCH`] armed with +/// slot `k`. 136 bytes. +/// +/// Counts, not contents: a change is "what a listing of this directory, or +/// of one of its child folders, shows may differ" — an entry made, removed +/// or renamed in it, a child's length, revision or folder count moved, or an +/// entry made or removed in a child folder (its line's count and whether it +/// holds folders) — and the client lists again through its own handle, whose +/// route the service revalidates. Nothing here names an entry, so the plane +/// carries no observation the handle could not make itself. +#[repr(C)] +#[derive( + Clone, Copy, Debug, Default, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, +)] +pub struct FsWatchState { + /// Per slot: the tag the slot was armed with; zero while disarmed. A + /// client reads a slot's counts only under the tag it armed, so a count + /// left by the slot's previous watch is never mistaken for its own. + pub tag: [u32; FS_WATCH_SLOTS], + /// Per slot: changes seen since the slot was armed, wrapping. Coalesced: + /// several changes between two service passes advance it once. + pub changes: [u32; FS_WATCH_SLOTS], + /// Bit `k`: slot `k`'s watch ended — its handle closed, its route stopped + /// proving, or its directory was reclaimed. Cleared by re-arming. + pub ended: u32, + /// Reserved; zero. + pub reserved: u32, +} + +/// The watch plane's record ring: one slot of a reserved edge kind, never +/// pushed. A later per-entry change record lands in it without moving the +/// plane's state or its offset. +pub type FsWatchRecord = u64; +/// Records the watch plane's ring holds. +pub const FS_WATCH_RECORDS: u32 = 1; + +const _: () = assert!(core::mem::size_of::() == 136); + /// Maximum bytes in one view-relative search path. /// /// The filesystem admits thirty-two 32-byte components diff --git a/native/crates/abi/src/task.rs b/native/crates/abi/src/task.rs index 6958fcf..3cd22f4 100644 --- a/native/crates/abi/src/task.rs +++ b/native/crates/abi/src/task.rs @@ -465,19 +465,19 @@ pub const INTERACTIVE_CHILD_LOADER_FACES: usize = DESKTOP_BOOTSTRAP_WINDOW_SLOTS + 2 * DESKTOP_CONTROLLER_CAPACITY; /// Live immutable image-generation peers the boot catalogue can hold at once: -/// the 46-row bootstrap catalogue — the desktop's `files`, its documents +/// the 47-row bootstrap catalogue — the desktop's `files`, its documents /// `editor`, `photos`, `settings` and `power-ui`, the screenshot writer `capture`, beside /// the native filesystem verbs `append`, `sync` and `pack`, the two /// WebAssembly rows, `wasm-witness` and `wasm`, `jsapp`, the JS app host, /// `inference` / `model-data-witness`, `shell`, the guest shell `term` -/// opens, and `tcc`, the C compiler — plus the founded shell, the graphical -/// terminal, and three extraction parsers. +/// opens, `tcc`, the C compiler, and `ioctl`, the device tool — plus the +/// founded shell, the graphical terminal, and three extraction parsers. /// Exec asserts this covers its catalogue (`kore-svc-exec`), the one place /// that sees both. Named /// separately from the interactive reservation below because the two grow for /// unrelated reasons: this one when a row joins the catalogue, that one when /// the desktop admits another window or controller. -pub const BOOTSTRAP_IMAGE_PEERS: usize = 51; +pub const BOOTSTRAP_IMAGE_PEERS: usize = 52; /// Private image generations installed at runtime that can live at once, /// across every session: exec's `InstallFile` rows, each a file a task diff --git a/native/crates/dagfs/src/fs.rs b/native/crates/dagfs/src/fs.rs index 55b4b8d..3dd38e4 100644 --- a/native/crates/dagfs/src/fs.rs +++ b/native/crates/dagfs/src/fs.rs @@ -106,6 +106,10 @@ pub struct Fs<'w, 't> { pub(crate) ring_files: u64, /// Why the last whole-file read did or did not take that ring. pub(crate) ring_note: &'static str, + /// This pair's watch plane — the event channel the service formats + /// after the rings in this pair's own window (`crate::watch`) — or + /// `None` on a pair formatted without one. + pub(crate) watch: Option>, } impl<'w, 't> Fs<'w, 't> { @@ -128,11 +132,13 @@ impl<'w, 't> Fs<'w, 't> { // window only through the SPSC protocol. let window: &'static mut [u8] = unsafe { core::slice::from_raw_parts_mut(window.cast::().as_ptr(), window.len()) }; - // An *attach*, not an init: the filesystem row's own `init_window` - // formatted this ring at founding, and `client_rings` fail-closes - // on an unformatted window rather than reading a zeroed page as an - // empty ring. - let rings = client_rings::(window, REQ_CAP, kore_rt::svc::REP_CAP).ok()?; + // An *attach*, not an init: the filesystem row's own formatter + // laid out this window at founding — the rings, then the watch + // plane — and both attaches fail closed on an unformatted window + // rather than reading a zeroed page as an empty ring. + let (pair, tail) = window.split_at_mut(crate::watch::PLANE_AT.min(window.len())); + let rings = client_rings::(pair, REQ_CAP, kore_rt::svc::REP_CAP).ok()?; + let watch = crate::watch::sink(tail).ok(); let kick = rt .bootstrap() .handle(role::SVC_SEND, process_face::REQUEST_KICK)?; @@ -223,9 +229,70 @@ impl<'w, 't> Fs<'w, 't> { ring, ring_files: 0, ring_note: "unused", + watch, }) } + /// This pair's watch plane, if the pair carries one. + pub fn watch_plane(&self) -> Option<&crate::watch::WatchSink<'static>> { + self.watch.as_ref() + } + + /// Arm watch `slot` on the directory `dir`, tagged `tag` (a non-zero + /// name of the client's own for this arming): the plane counts every + /// change to what listing `dir` shows from now on. The view stays the + /// caller's — closing it ends the watch. `Unsupported` on a pair without + /// a plane. + pub async fn watch(&mut self, dir: &crate::View, slot: u8, tag: u32) -> Result<(), Refusal> { + if self.watch.is_none() { + return Err(Refusal::Fs(crate::FsError::Unsupported)); + } + self.checked(&FsRequest::Watch { + handle: dir.handle(), + slot, + tag, + }) + .await + .map(|_| ()) + } + + /// Disarm watch `slot`. + pub async fn unwatch(&mut self, slot: u8) -> Result<(), Refusal> { + self.checked(&FsRequest::Unwatch { slot }).await.map(|_| ()) + } + + /// The plane's wake word, read before a caller drains its sources: a + /// park published against it ([`Self::arm_watch_park`]) is lossless. + /// Zero, and never moving, on a pair without a plane. + pub fn watch_seq(&self) -> u32 { + self.watch.as_ref().map_or(0, crate::watch::WatchSink::seq) + } + + /// The plane's state, `None` on a pair without one (or mid-publish, + /// which the caller treats as nothing new and reads again next pass — + /// the publish ends by moving the wake word). + pub fn watch_state(&self) -> Option { + self.watch.as_ref().and_then(crate::watch::WatchSink::state) + } + + /// Publish this task's intent to wait on the plane, for a caller about to + /// enter one executor wait that every source shares (the multiplexed + /// park, `res/FILES_DESKTOP.md`): the service's next batch then owes + /// this task a notice on its doorbell. `false` when the plane already + /// moved past `seen` — drain again instead of waiting. Always `true` on + /// a pair without a plane, which has nothing to wait for. + pub fn arm_watch_park(&self, seen: u32) -> bool { + self.watch.as_ref().is_none_or(|sink| sink.park(seen)) + } + + /// Withdraw the intent [`Self::arm_watch_park`] published, however the + /// wait ended, so a departed waiter is never charged a notice. + pub fn withdraw_watch_park(&self) { + if let Some(sink) = self.watch.as_ref() { + sink.unpark(); + } + } + /// The bulk data window, read side. Bytes a verb landed here are valid /// until the next request that uses the window. pub fn window(&self) -> &[u8] { diff --git a/native/crates/dagfs/src/lib.rs b/native/crates/dagfs/src/lib.rs index 4800948..0597035 100644 --- a/native/crates/dagfs/src/lib.rs +++ b/native/crates/dagfs/src/lib.rs @@ -67,6 +67,8 @@ mod view; #[cfg(feature = "runtime")] mod walk; #[cfg(feature = "runtime")] +pub mod watch; +#[cfg(feature = "runtime")] pub mod wire; #[cfg(feature = "runtime")] diff --git a/native/crates/dagfs/src/protocol.rs b/native/crates/dagfs/src/protocol.rs index abfbe60..3125092 100644 --- a/native/crates/dagfs/src/protocol.rs +++ b/native/crates/dagfs/src/protocol.rs @@ -193,6 +193,22 @@ pub enum FsRequest { Sync { handle: Handle, }, + /// Arm watch `slot` (below [`FS_WATCH_SLOTS`](kore_abi::svc::FS_WATCH_SLOTS)) + /// on the directory `handle` names, tagged `tag` (non-zero): the pair's + /// watch plane ([`crate::watch`]) counts every change to what a listing + /// of that directory shows, under `tag`, until the slot is re-armed or + /// disarmed or the watch ends. The handle stays the client's; its route + /// is revalidated before every count, and one that stops proving ends the + /// watch rather than reporting through a route the client lost. + Watch { + handle: Handle, + slot: u8, + tag: u32, + }, + /// Disarm watch `slot`, armed or not. + Unwatch { + slot: u8, + }, /// Hold this session's whole-tree commits until the window closes. /// /// The durable contract serializes session zero's entire tree after every @@ -631,6 +647,17 @@ impl SvcProtocol for DagfsProtocol { /// execution pipeline and made boot topology scale with the command set. const SERVE_PAIRS: usize = 2; + /// The rings, then the watch plane ([`crate::watch`]) in the same window: + /// every filesystem pair carries its change channel, so a client never + /// needs a capability beyond the pair to learn that a directory changed. + fn init_pair_window( + window: &mut [u8], + req_cap: u32, + rep_cap: u32, + ) -> Result<(), kore_ring::spsc::RingError> { + crate::watch::init_pair_window(window, req_cap, rep_cap) + } + fn reply_matches(request: &FsReqWire, response: &FsResponse, grants: u8) -> bool { if grants != 0 { return false; diff --git a/native/crates/dagfs/src/text.rs b/native/crates/dagfs/src/text.rs index 49ac01e..3af8d2f 100644 --- a/native/crates/dagfs/src/text.rs +++ b/native/crates/dagfs/src/text.rs @@ -1,6 +1,7 @@ //! The two text helpers every command needs and none should re-spell: a -//! bounded whitespace splitter over one command line, and a bounded line -//! buffer for composing an answer out of pieces. +//! bounded whitespace splitter over one command line (with `"quoted"` words, +//! so a name with spaces can be named), and a bounded line buffer for +//! composing an answer out of pieces. /// A whitespace splitter over one completed command line. /// @@ -18,13 +19,27 @@ impl<'a> Args<'a> { Self { rest: line } } - /// The next space-separated word, or `None` at the end of the line. + /// The next word, or `None` at the end of the line. A word is a run of + /// non-spaces, or — opened by `"` — everything up to the next `"`, + /// spaces included and quotes excluded, so `inspect "Untitled Folder"` + /// names the folder Files makes. A quote left open runs to the end of + /// the line. Borrowed either way: a quoted word is a span of the line. pub fn next(&mut self) -> Option<&'a [u8]> { let start = self.skip_spaces(); if start >= self.rest.len() { self.rest = &self.rest[self.rest.len()..]; return None; } + if self.rest[start] == b'"' { + let open = start + 1; + let close = self.rest[open..] + .iter() + .position(|&byte| byte == b'"') + .map_or(self.rest.len(), |at| open + at); + let word = &self.rest[open..close]; + self.rest = &self.rest[(close + 1).min(self.rest.len())..]; + return Some(word); + } let mut end = start; while end < self.rest.len() && self.rest[end] != b' ' { end += 1; @@ -128,6 +143,21 @@ mod tests { assert_eq!(args.rest(), b""); } + #[test] + fn a_quoted_word_keeps_its_spaces_and_drops_its_quotes() { + let mut args = Args::new(b"remove \"Untitled Folder\" next \"\" \"open to the end"); + assert_eq!(args.next(), Some(&b"remove"[..])); + assert_eq!(args.next(), Some(&b"Untitled Folder"[..])); + assert_eq!(args.next(), Some(&b"next"[..])); + assert_eq!(args.next(), Some(&b""[..]), "an empty quote is an empty word"); + assert_eq!(args.next(), Some(&b"open to the end"[..])); + assert_eq!(args.next(), None); + // A quote inside a word is part of it: only a word's first byte opens. + let mut args = Args::new(b"a\"b c"); + assert_eq!(args.next(), Some(&b"a\"b"[..])); + assert_eq!(args.next(), Some(&b"c"[..])); + } + #[test] fn a_line_buffer_truncates_rather_than_grows() { let mut line = LineBuf::<8>::new(); diff --git a/native/crates/dagfs/src/watch.rs b/native/crates/dagfs/src/watch.rs new file mode 100644 index 0000000..1f9b68a --- /dev/null +++ b/native/crates/dagfs/src/watch.rs @@ -0,0 +1,251 @@ +//! The **watch plane**: a filesystem pair's change channel (`res/EVENT.md`). +//! +//! A client that shows a directory — the Files explorer — needs to learn when +//! what it shows went stale without asking. The plane is the +//! service→client direction of one pair, as the window pair's event channel is +//! the compositor→client one: an event channel formatted after the +//! request/reply rings in the pair's **own** window, so it rides the lease +//! both sides already hold and needs no capability of its own. +//! +//! - **State is level.** [`FsWatchState`] holds, per slot, the tag the slot +//! was armed with and a wrapping count of changes since; the service +//! republishes it whole. Several changes between two service passes advance +//! a count once — a client lists again, it does not replay. +//! - **The wake is conditional.** The service commits after the request (or +//! teardown) that changed a watched directory and owes the client a notice +//! only if it had parked on the plane; a client that is running pays the +//! service nothing. +//! - **No authority, no names.** A watch is armed on a directory handle the +//! client holds; the service revalidates that handle's route before every +//! count and ends the watch the moment it stops proving. The plane says +//! *that* a listing may differ, never *what* it holds — the client lists +//! through its own handle. +//! +//! The record ring is reserved ([`FS_WATCH_RECORDS`], never pushed) so a later +//! per-entry change record lands without moving the state or the offset. + +use kore_abi::svc::{FS_WATCH_RECORDS, FS_WATCH_SLOTS, FsWatchRecord, FsWatchState}; +use kore_ring::event::{EventSink, EventSource}; +use kore_ring::spsc::RingError; +use kore_rt::svc::{REP_CAP, REQ_CAP, SvcProtocol}; + +use crate::DagfsProtocol; + +/// The producer half: the service's. +pub type WatchSource<'a> = EventSource<'a, FsWatchState, FsWatchRecord>; +/// The consumer half: the client's. +pub type WatchSink<'a> = EventSink<'a, FsWatchState, FsWatchRecord>; + +/// Where the plane begins in a pair window: right after the canonical +/// request/reply rings (both 64-byte rounded, so the plane is line-aligned). +pub const PLANE_AT: usize = kore_rt::svc::window_bytes::( + ::REQUEST_CAPACITY, + ::RESPONSE_CAPACITY, +); + +/// Bytes the plane occupies. +pub const PLANE_BYTES: usize = + kore_ring::event::region_bytes::(FS_WATCH_RECORDS); + +/// The smallest pair window that carries the plane. +pub const WINDOW_BYTES: usize = PLANE_AT + PLANE_BYTES; + +// Whoever sizes a filesystem pair window asserts it holds `WINDOW_BYTES` +// against its own constant (exec's `FS_CONTROL_BYTES`); the formatter refuses +// a smaller window at founding. Line alignment is the layout's own invariant. +const _: () = assert!(PLANE_AT % 64 == 0); +const _: () = assert!( + ::REQUEST_CAPACITY == REQ_CAP + && ::RESPONSE_CAPACITY == REP_CAP +); + +/// Format a whole filesystem pair window — the rings, then the plane — before +/// either side maps it. The protocol's [`SvcProtocol::init_pair_window`]. +pub fn init_pair_window(window: &mut [u8], req_cap: u32, rep_cap: u32) -> Result<(), RingError> { + kore_rt::svc::init_window::(window, req_cap, rep_cap)?; + let at = kore_rt::svc::window_bytes::(req_cap, rep_cap); + let plane = window.get_mut(at..).ok_or(RingError::TooSmall)?; + kore_ring::event::init::(plane, FS_WATCH_RECORDS) +} + +/// The service's half over its mapping of the pair window's tail (`aux` is +/// what follows the rings, as `attach_peer_aux` hands it over). +pub fn source(aux: &mut [u8]) -> Result, RingError> { + kore_ring::event::source(aux.get_mut(..PLANE_BYTES).ok_or(RingError::TooSmall)?) +} + +/// The client's half over its mapping of the same tail. +pub fn sink(aux: &mut [u8]) -> Result, RingError> { + kore_ring::event::sink(aux.get_mut(..PLANE_BYTES).ok_or(RingError::TooSmall)?) +} + +/// What one slot's watch did since a [`Cursor`] last looked. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Seen { + /// Nothing the cursor has not seen. + Same, + /// The directory changed: list it again. + Changed, + /// The watch ended (its route stopped proving, or the directory went): + /// what was shown cannot be refreshed through this watch. + Ended, +} + +/// A client's view of one slot: the tag it armed and the last count it acted +/// on. A slot is read only under the cursor's own tag, so the counts the +/// slot carried for an earlier arming — or for nobody — never read as news. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Cursor { + slot: u8, + tag: u32, + seen: u32, + ended: bool, +} + +impl Cursor { + /// A cursor for `slot` armed with `tag`; `None` for a slot past the plane + /// or the zero tag (the plane's "disarmed"). + pub const fn new(slot: u8, tag: u32) -> Option { + if slot as usize >= FS_WATCH_SLOTS || tag == 0 { + return None; + } + Some(Self { + slot, + tag, + seen: 0, + ended: false, + }) + } + + pub const fn slot(&self) -> u8 { + self.slot + } + + pub const fn tag(&self) -> u32 { + self.tag + } + + /// Compare `state` with what this cursor last acted on, and take it as + /// seen. A state still showing another tag in the slot (the arming has + /// not landed, or the slot was re-armed by someone else's request on + /// this pair) is [`Seen::Same`]: nothing of this watch is in it yet. + pub fn observe(&mut self, state: &FsWatchState) -> Seen { + let slot = usize::from(self.slot); + if state.tag[slot] != self.tag { + return Seen::Same; + } + if state.ended & (1 << slot) != 0 { + if self.ended { + return Seen::Same; + } + self.ended = true; + return Seen::Ended; + } + let changes = state.changes[slot]; + if changes == self.seen { + return Seen::Same; + } + self.seen = changes; + Seen::Changed + } +} + +/// The plane's state as a producer edits it — the service's one batch verb +/// over a slot. Kept beside the layout so both halves read one definition. +pub mod state { + use super::{FS_WATCH_SLOTS, FsWatchState}; + + /// Arm `slot` under `tag`: no changes yet, not ended. + pub fn arm(state: &mut FsWatchState, slot: usize, tag: u32) { + if slot < FS_WATCH_SLOTS { + state.tag[slot] = tag; + state.changes[slot] = 0; + state.ended &= !(1 << slot); + } + } + + /// Disarm `slot`. + pub fn disarm(state: &mut FsWatchState, slot: usize) { + if slot < FS_WATCH_SLOTS { + state.tag[slot] = 0; + state.changes[slot] = 0; + state.ended &= !(1 << slot); + } + } + + /// One more change under `slot`'s arming. + pub fn changed(state: &mut FsWatchState, slot: usize) { + if slot < FS_WATCH_SLOTS { + state.changes[slot] = state.changes[slot].wrapping_add(1); + } + } + + /// `slot`'s watch ended; its tag stays, so the client that armed it + /// learns that it did. + pub fn end(state: &mut FsWatchState, slot: usize) { + if slot < FS_WATCH_SLOTS { + state.ended |= 1 << slot; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::vec; + + #[repr(align(64))] + struct Window([u8; WINDOW_BYTES]); + + #[test] + fn the_plane_follows_the_rings_and_both_halves_attach() { + let mut window = Window([0; WINDOW_BYTES]); + init_pair_window(&mut window.0, REQ_CAP, REP_CAP).unwrap(); + // The rings are exactly where every existing client expects them. + let (pair, aux) = window.0.split_at_mut(PLANE_AT); + assert!(kore_rt::svc::client_rings::(pair, REQ_CAP, REP_CAP).is_ok()); + let base = aux.as_mut_ptr(); + // SAFETY: the two halves alias the plane only through the channel + // protocol, as the cross-space form does over two mappings. + let alias = unsafe { core::slice::from_raw_parts_mut(base, PLANE_BYTES) }; + let mut source = source(aux).unwrap(); + let sink = sink(alias).unwrap(); + assert_eq!(sink.state(), Some(FsWatchState::default())); + source.state(|s| state::arm(s, 1, 7)); + assert!(!source.commit(), "nobody parked: no notice owed"); + let seen = sink.seq(); + assert!(sink.park(seen)); + source.state(|s| state::changed(s, 1)); + assert!(source.commit(), "a parked client is owed its notice"); + assert_eq!(sink.state().unwrap().changes[1], 1); + } + + #[test] + fn a_window_too_small_for_the_plane_is_refused() { + let mut short = vec![0u8; PLANE_AT + 64 + 64]; + let at = short.as_ptr().align_offset(64); + let window = &mut short[at..at + PLANE_AT]; + assert_eq!(init_pair_window(window, REQ_CAP, REP_CAP), Err(RingError::TooSmall)); + } + + #[test] + fn a_cursor_reads_only_its_own_arming() { + let mut cursor = Cursor::new(2, 9).unwrap(); + let mut s = FsWatchState::default(); + // The slot's previous watch left counts: not this cursor's news. + state::arm(&mut s, 2, 8); + state::changed(&mut s, 2); + assert_eq!(cursor.observe(&s), Seen::Same); + state::arm(&mut s, 2, 9); + assert_eq!(cursor.observe(&s), Seen::Same); + state::changed(&mut s, 2); + state::changed(&mut s, 2); + assert_eq!(cursor.observe(&s), Seen::Changed, "two changes coalesce"); + assert_eq!(cursor.observe(&s), Seen::Same); + state::end(&mut s, 2); + assert_eq!(cursor.observe(&s), Seen::Ended); + assert_eq!(cursor.observe(&s), Seen::Same, "an ending is reported once"); + assert!(Cursor::new(FS_WATCH_SLOTS as u8, 1).is_none()); + assert!(Cursor::new(0, 0).is_none()); + } +} diff --git a/native/crates/dagfs/src/wire.rs b/native/crates/dagfs/src/wire.rs index 638461e..4cdae6f 100644 --- a/native/crates/dagfs/src/wire.rs +++ b/native/crates/dagfs/src/wire.rs @@ -23,6 +23,8 @@ //! | `OpenAt` | parent | — | — | — | `name` | //! | `OpenGrant` | — | grant id | — | — | — | //! | `Close`/`Stat`/`Sync` | object | — | — | — | — | +//! | `Watch` | directory | — | — | tag (slot in `kind`) | — | +//! | `Unwatch` | — | — | — | — (slot in `kind`) | — | //! | `List` | directory | — | — | cursor | — | //! | `Parents` | object | — | — | cursor | — | //! | `Locate` | anchor | stable identity | — | — | — | @@ -448,6 +450,16 @@ pub fn encode_req(req: &FsRequest) -> FsReqWire { w.op = FsReqWire::OP_SYNC; w.handle = U64::new(handle.to_bits()); } + FsRequest::Watch { handle, slot, tag } => { + w.op = FsReqWire::OP_WATCH; + w.handle = U64::new(handle.to_bits()); + w.kind = slot; + w.length = U32::new(tag); + } + FsRequest::Unwatch { slot } => { + w.op = FsReqWire::OP_UNWATCH; + w.kind = slot; + } FsRequest::DeferCommits { hold } => { w.op = FsReqWire::OP_DEFER_COMMITS; w.kind = u8::from(hold); @@ -713,6 +725,8 @@ pub fn decode_req(w: &FsReqWire) -> Option { | FsReqWire::OP_DEFER_COMMITS | FsReqWire::OP_SET_SYSTEM_SEED | FsReqWire::OP_STOR_MAINTAIN + | FsReqWire::OP_WATCH + | FsReqWire::OP_UNWATCH ) && w.kind != 0 { return None; @@ -914,6 +928,35 @@ pub fn decode_req(w: &FsReqWire) -> Option { } FsReqWire::OP_REVOKE => FsRequest::Revoke { grant }, FsReqWire::OP_SYNC => FsRequest::Sync { handle }, + FsReqWire::OP_WATCH => { + // `kind` is the slot and `length` the tag; nothing else rides. A + // slot past the plane or a zero tag (the plane's "disarmed") is a + // framing mismatch, not a request to guess at. + if usize::from(w.kind) >= kore_abi::svc::FS_WATCH_SLOTS + || w.length.get() == 0 + || w.window_offset.get() != 0 + || w.handle2.get() != 0 + || w.offset.get() != 0 + || w.expected_revision.get() != 0 + || !names_are_empty(w) + { + return None; + } + FsRequest::Watch { + handle, + slot: w.kind, + tag: w.length.get(), + } + } + FsReqWire::OP_UNWATCH => { + if usize::from(w.kind) >= kore_abi::svc::FS_WATCH_SLOTS + || !control_payload_is_empty(w) + || w.window_offset.get() != 0 + { + return None; + } + FsRequest::Unwatch { slot: w.kind } + } FsReqWire::OP_DEFER_COMMITS => { // `kind` is the whole payload: a deferral names no object, and a // value outside the two-state vocabulary is a framing mismatch. @@ -1737,6 +1780,8 @@ pub fn decode_rep_for(request: &FsRequest, w: &FsRepWire) -> Option<(FsResponse, | FsRequest::DetachEdge { .. } | FsRequest::Revoke { .. } | FsRequest::Sync { .. } + | FsRequest::Watch { .. } + | FsRequest::Unwatch { .. } | FsRequest::BeginReplace { .. } | FsRequest::WriteReplacement { .. } | FsRequest::AbortReplace { .. } @@ -1876,6 +1921,8 @@ pub fn reply_matches(request: &FsRequest, response: &FsResponse) -> bool { | FsRequest::DetachEdge { .. } | FsRequest::Revoke { .. } | FsRequest::Sync { .. } + | FsRequest::Watch { .. } + | FsRequest::Unwatch { .. } | FsRequest::BeginReplace { .. } | FsRequest::WriteReplacement { .. } | FsRequest::AbortReplace { .. } @@ -2020,6 +2067,66 @@ mod tests { assert_eq!(decode_req(&stray), None); } + /// A watch carries its directory, its slot and its tag and nothing else; + /// an unwatch carries its slot alone. Both answer `Done`, and a slot past + /// the plane, the zero tag, or any stray operand is refused. + #[test] + fn watch_verbs_round_trip_and_refuse_stray_operands() { + let slots = kore_abi::svc::FS_WATCH_SLOTS as u8; + for (slot, tag) in [(0, 1), (slots - 1, u32::MAX)] { + let watch = FsRequest::Watch { + handle: Handle::from_bits(0x1_0000_0003), + slot, + tag, + }; + let unwatch = FsRequest::Unwatch { slot }; + for request in [watch, unwatch] { + assert!(!request.needs_window()); + assert_eq!(decode_req(&encode_req(&request)), Some(request)); + assert!(reply_matches(&request, &FsResponse::Done)); + assert_eq!( + decode_rep_for(&request, &encode_rep(&FsResponse::Done, 0)), + Some((FsResponse::Done, 0)) + ); + } + } + let wire = encode_req(&FsRequest::Watch { + handle: Handle::from_bits(9), + slot: 1, + tag: 5, + }); + let strays: [fn(&mut FsReqWire); 8] = [ + |w| w.kind = kore_abi::svc::FS_WATCH_SLOTS as u8, + |w| w.length = U32::new(0), + |w| w.window_offset = U32::new(4), + |w| w.handle2 = U64::new(1), + |w| w.offset = U64::new(1), + |w| w.expected_revision = U64::new(1), + |w| w.rights = 1, + |w| { + w.name[0] = b'x'; + w.name_len = 1; + }, + ]; + for stray in strays { + let mut bad = wire; + stray(&mut bad); + assert_eq!(decode_req(&bad), None); + } + let wire = encode_req(&FsRequest::Unwatch { slot: 2 }); + let strays: [fn(&mut FsReqWire); 4] = [ + |w| w.kind = kore_abi::svc::FS_WATCH_SLOTS as u8, + |w| w.handle = U64::new(1), + |w| w.length = U32::new(1), + |w| w.window_offset = U32::new(1), + ]; + for stray in strays { + let mut bad = wire; + stray(&mut bad); + assert_eq!(decode_req(&bad), None); + } + } + /// Every maintenance action survives the wire; only a forced refresh /// carries operands, and an operand on any other action — or a range /// past `u32` — is refused. diff --git a/native/crates/ld/tests/built_images.rs b/native/crates/ld/tests/built_images.rs index f196515..0bf03c1 100644 --- a/native/crates/ld/tests/built_images.rs +++ b/native/crates/ld/tests/built_images.rs @@ -339,6 +339,7 @@ fn check_variant(root: &Path, variant: &str, required: bool) -> us ("sleep", ImageKind::Executable), ("status", ImageKind::NeedsLibrary("libpci_names.so")), ("lspci", ImageKind::NeedsLibrary("libpci_names.so")), + ("ioctl", ImageKind::NeedsLibrary("libpci_names.so")), ("help", ImageKind::Executable), ("window", ImageKind::Executable), ("clear", ImageKind::Executable), diff --git a/native/crates/rt/src/process.rs b/native/crates/rt/src/process.rs index 4417d58..03733ae 100644 --- a/native/crates/rt/src/process.rs +++ b/native/crates/rt/src/process.rs @@ -53,6 +53,13 @@ pub fn initialize_with_context( page.initialize_with_context(args, context, TelemetrySnapshot::EMPTY) } +/// Publish the name the launcher resolved the program by +/// ([`ProcessIo::command`]), after the page is initialized and before START. +pub fn write_command(window: &mut [u8], name: &[u8]) -> Result<(), ProcessIoError> { + let (page, _) = ProcessIo::mut_from_prefix(window).map_err(|_| ProcessIoError::Invalid)?; + page.write_command(name) +} + /// Publish the launcher's filesystem snapshot, same window as the telemetry /// snapshot and the same lifecycle point: after founding, before START. pub fn write_filesystem( diff --git a/native/crates/rt/src/service/mod.rs b/native/crates/rt/src/service/mod.rs index bf11006..a4cc04a 100644 --- a/native/crates/rt/src/service/mod.rs +++ b/native/crates/rt/src/service/mod.rs @@ -413,6 +413,21 @@ pub trait SystemService: crate::svc::SvcProtocol + Sized { /// retryable domain custody still closes in `peer_closed` afterward. fn peer_detaching(&mut self, _peer: crate::svc::PeerId) {} + /// Recyclable peers this service owes one wake: it published a batch into + /// an event channel it keeps in their pair's tail (`attach_peer_aux`, + /// `res/EVENT.md`) and the batch's commit found the consumer parked. + /// + /// [`ServeHost`](crate::svc::ServeHost) asks after every unit of work — + /// a request, a teardown, an idle recheck — and sends each peer named in + /// `owed[..n]` one notice through the endpoint it owns, so a service never + /// copies an affine endpoint out of its peer bundle to wake a client. Name + /// each peer once per owed wake and clear it as it is handed over; a full + /// batch is asked again. The default owes nothing: the request/reply shape, + /// whose only wake is the reply's. + fn take_owed_wakes(&mut self, _owed: &mut [Option]) -> usize { + 0 + } + /// Observe completion of an admission requested through /// [`Serve::admit_peer`]. The host, not the wire request, chooses the /// generational identity and reports only the authenticated result. diff --git a/native/crates/rt/src/svc/serve.rs b/native/crates/rt/src/svc/serve.rs index 1b79832..985f785 100644 --- a/native/crates/rt/src/svc/serve.rs +++ b/native/crates/rt/src/svc/serve.rs @@ -2358,6 +2358,39 @@ impl<'w, 't, T: Transport, S: SystemService> ServeHost<'w, 't, T, S> { // work: a pair a serve step just minted is attached before // the next wake, and a retired one stops now. self.settle_minted(); + self.deliver_owed_wakes().await; + } + } + + /// Send each peer the service owes a wake its one notice + /// ([`SystemService::take_owed_wakes`]), through the endpoint this host + /// owns. A service names a peer once per owed wake, so the batches end + /// within the peer table; the bound only keeps a service that kept + /// answering full batches from holding the loop. + async fn deliver_owed_wakes(&mut self) { + // Peers asked for per call: a stack array, so the ask allocates + // nothing; its size only trades calls for stack, and the founder + // roster's whole recyclable band fits one batch. + const BATCH: usize = 8; + for _ in 0..=self.pairs.len().div_ceil(BATCH) { + let mut owed = [None; BATCH]; + let count = self.svc.take_owed_wakes(&mut owed).min(BATCH); + for peer in owed[..count].iter().flatten() { + // A peer retired since its batch was published has no one + // left to wake; `notify_peer` refuses it by generation. + let _ = self + .notify_peer( + *peer, + Notice { + cookie: crate::event::COOKIE_EVENT, + payload: 0, + }, + ) + .await; + } + if count < BATCH { + break; + } } } @@ -2406,6 +2439,7 @@ impl<'w, 't, T: Transport, S: SystemService> ServeHost<'w, 't, T, S> { // The handshake drains even on an idle pass, so a host test can // attach a freshly minted pair without a request in flight. self.settle_minted(); + self.deliver_owed_wakes().await; served } } @@ -2820,6 +2854,116 @@ mod tests { ); } + /// A service that publishes into its peers' auxiliary planes and owes + /// some of them a wake (`SystemService::take_owed_wakes`). + struct Owing { + /// Peers still owed, or `None` for a service that answers full + /// batches forever. + owed: Option, + asks: u32, + } + + impl SystemService for Owing { + const NAME: &'static str = "owing-test"; + type Endowment = NoCaps; + + fn bind(_: NoCaps) -> Self { + Self { owed: Some(0), asks: 0 } + } + + fn serve(&mut self, _: u64, sv: &mut Serve<'_, Reply>) { + sv.reply(Reply::Granted); + } + + fn take_owed_wakes(&mut self, owed: &mut [Option]) -> usize { + self.asks += 1; + let count = match self.owed.as_mut() { + Some(left) => { + let count = (*left).min(owed.len()); + *left -= count; + count + } + None => owed.len(), + }; + for peer in &mut owed[..count] { + *peer = PeerId::from_arrival(0x102); + } + count + } + } + + impl SvcProtocol for Owing { + type Request = u64; + type Response = Reply; + type ReqWire = u64; + type RepWire = u64; + + const REPLY_GRANTS: usize = 0; + + fn encode_req(req: &u64) -> u64 { + *req + } + + fn decode_req(wire: &u64) -> Option { + Some(*wire) + } + + fn encode_rep(rep: &Reply, grants: u8) -> u64 { + DropAware::encode_rep(rep, grants) + } + + fn decode_rep(wire: &u64) -> Option<(Reply, u8)> { + DropAware::decode_rep(wire) + } + } + + /// The host asks for owed wakes after every pass — a served request and + /// an idle one alike — asks again while batches come back full, and stops + /// within the peer table even for a service that never stops owing. (A + /// notice to a peer that is not a live recyclable generation is refused + /// by generation, which is what a peer retired since its batch gets.) + #[test] + fn the_host_drains_owed_wakes_after_every_pass_and_bounds_the_batches() { + let window = { + let bytes = window_bytes::(REQ_CAP, REP_CAP); + let layout = std::alloc::Layout::from_size_align(bytes, 64).expect("window layout"); + // SAFETY: as `window()` above — one leaked aligned allocation. + unsafe { + let ptr = std::alloc::alloc_zeroed(layout); + assert!(!ptr.is_null()); + core::slice::from_raw_parts_mut(ptr, bytes) + } + }; + let (mut client, rings) = split_window::(window, REQ_CAP, REP_CAP).expect("rings"); + let endpoints = ServeEndpoints { + kick: SlotRef::from_wire(60), + grant_tx: [None; MAX_GRANTS], + scratch: [None; MAX_GRANTS], + return_post: None, + }; + let transport = OkTransport::new(); + let mut host = ServeHost::new(Owing::bind(NoCaps), &transport, rings, endpoints); + + client.req.push(1).expect("request fits"); + assert!(drive(host.serve_next())); + assert_eq!(host.service().asks, 1, "a pass that owes nothing asks once"); + + // Every peer the table holds owed at once: full batches, then the + // short ask that ends them. + let peers = host.pairs.len(); + host.service_mut().owed = Some(peers); + assert!(!drive(host.serve_next())); + assert_eq!(host.service().asks, 1 + peers as u32 / 8 + 1); + assert_eq!(host.service().owed, Some(0)); + + // A service that never stops owing is bounded by the peer table. + host.service_mut().owed = None; + host.service_mut().asks = 0; + assert!(!drive(host.serve_next())); + let bound = host.pairs.len().div_ceil(8) as u32 + 1; + assert_eq!(host.service().asks, bound); + } + /// Walking return posts are host-owned reply-and-wait state: the first /// stands before readiness, and every successor is acknowledged before /// `serve_next` returns with its reply visible. diff --git a/native/crates/window/src/client.rs b/native/crates/window/src/client.rs index bd6892b..2fc2bd0 100644 --- a/native/crates/window/src/client.rs +++ b/native/crates/window/src/client.rs @@ -53,6 +53,9 @@ pub enum ClientError { EventMalformed, /// A menu the encoder refused before anything reached the compositor. Menu(crate::menu::MenuError), + /// A clipboard text past [`crate::clipboard::CLIPBOARD_BYTES`], or one + /// longer than its reader can hold: refused whole, never cut short. + TooLong, } impl ClientError { @@ -946,6 +949,101 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { } } + /// Put `text` on this window's seat's clipboard (`res/CLIPBOARD.md`), a + /// request plane at a time. Only the seat's focused window may, on a key + /// press, button press or menu choice it was just delivered and has not + /// spent ([`SurfaceRefusal::Denied`] otherwise). An empty text is no + /// copy; one past the clipboard's bound is refused before anything is + /// sent, and one that is not UTF-8 the compositor refuses whole. + /// `OutOfMemory` leaves the clipboard and the gesture as they were, so a + /// retry may succeed. + pub async fn copy(&mut self, text: &[u8]) -> Result<(), ClientError> { + let total = u32::try_from(text.len()) + .ok() + .filter(|&total| total as usize <= crate::clipboard::CLIPBOARD_BYTES) + .ok_or(ClientError::TooLong)?; + if total == 0 { + return Err(ClientError::Protocol); + } + let mut offset = 0_u32; + for piece in text.chunks(crate::clipboard::CHUNK_BYTES.min(self.menu.len())) { + self.menu[..piece.len()].copy_from_slice(piece); + let len = piece.len() as u16; + match self + .surface_request(SurfaceRequest::CopyChunk { offset, len, total }) + .await? + { + SurfaceResponse::Copied { through } if through == offset + u32::from(len) => { + offset = through; + } + _ => return Err(ClientError::Protocol), + } + } + Ok(()) + } + + /// Read this window's seat's clipboard (`res/CLIPBOARD.md`): `begin` + /// learns the text's length before any of it, and may refuse it + /// (`false`: longer than the caller can hold, [`ClientError::TooLong`]); + /// `piece` then takes the text in order, a request plane at a time, and + /// answers whether it wants more (`false` ends the read there: a reader + /// that needs only the text's start reads no more of it). Answers the + /// length, zero when the clipboard holds no text. Gated as [`Self::copy`] + /// is; a copy landing while the read is under way refuses it + /// ([`SurfaceRefusal::Unknown`]) rather than mixing two texts. + pub async fn paste( + &mut self, + begin: impl FnOnce(u32) -> bool, + mut piece: impl FnMut(&[u8]) -> bool, + ) -> Result { + let mut begin = Some(begin); + let mut offset = 0_u32; + let mut buffer = [0_u8; PASTE_STAGE_BYTES]; + loop { + let (at, len, total) = match self.surface_request(SurfaceRequest::Paste { offset }).await? { + SurfaceResponse::Clipboard { offset: at, len, total } => (at, len, total), + SurfaceResponse::ClipboardEmpty if offset == 0 => return Ok(0), + _ => return Err(ClientError::Protocol), + }; + if at != offset || usize::from(len) > self.menu.len() { + return Err(ClientError::Protocol); + } + if let Some(begin) = begin.take() + && !begin(total) + { + return Err(ClientError::TooLong); + } + // The compositor wrote the plane before it answered; read it as + // the shared memory it is, after the reply's acquire. + let plane = shared_plane(self.menu); + for chunk in plane[..usize::from(len)].chunks(buffer.len()) { + for (byte, shared) in buffer.iter_mut().zip(chunk) { + *byte = shared.load(core::sync::atomic::Ordering::Relaxed); + } + if !piece(&buffer[..chunk.len()]) { + return Ok(total); + } + } + offset += u32::from(len); + if offset == total { + return Ok(total); + } + } + } + + /// Ask for the clipboard to be typed into this terminal window's shell + /// line (`res/CLIPBOARD.md` §Shell): queued for its controller, which + /// takes what the line can hold. Gated as [`Self::copy`] is, and only the + /// seat's terminal window may ask. Answers the text's length, zero when + /// the clipboard holds no text. + pub async fn paste_to_shell(&mut self) -> Result { + match self.surface_request(SurfaceRequest::PasteToShell).await? { + SurfaceResponse::PasteQueued { total } => Ok(total), + SurfaceResponse::ClipboardEmpty => Ok(0), + _ => Err(ClientError::Protocol), + } + } + async fn read_configuration(&mut self) -> Result { let reply = self .control @@ -1392,6 +1490,22 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { } } +/// A paste's piece reaches its reader through this much stack at a time: +/// the plane is read atomically, a byte at a time, and handed on as a slice. +/// Small enough for any task's stack, large enough that a 4 KiB piece is +/// sixteen calls. +const PASTE_STAGE_BYTES: usize = 256; + +/// The request plane as the memory it is: shared with the compositor, which +/// writes it while it answers a paste. +fn shared_plane(plane: &[u8]) -> &[core::sync::atomic::AtomicU8] { + // SAFETY: `AtomicU8` has `u8`'s size and alignment, and every access the + // returned view makes is atomic, so the compositor's writes race with none + // of this process's plain accesses: the plane's plain writers run only + // between requests, never while this view is live. + unsafe { &*(core::ptr::from_ref(plane) as *const [core::sync::atomic::AtomicU8]) } +} + fn valid_configuration(previous: Option, next: Configure) -> bool { next.window != 0 && next.serial != 0 diff --git a/native/crates/window/src/clipboard.rs b/native/crates/window/src/clipboard.rs new file mode 100644 index 0000000..eca5220 --- /dev/null +++ b/native/crates/window/src/clipboard.rs @@ -0,0 +1,107 @@ +//! The seat's clipboard (`res/CLIPBOARD.md`): one text the compositor holds +//! for each seat, which every window of that seat can put text on and read +//! text from, and which no window can touch without the user asking. +//! +//! The text crosses a window's own pair lease, never a lent view: a piece at +//! a time through the request plane the window already maps +//! ([`crate::menu`]'s page), copied out by the compositor before it answers a +//! copy and copied in before it answers a paste. A lent view per copy or +//! paste would spend the window's map cursor, which no unmap gives back, and +//! carry a capability and a revocation for a few hundred bytes of text. +//! +//! **Gesture-gated.** Only the seat's focused window may begin a copy or a +//! paste, and only on a key press, button press or menu choice the +//! compositor delivered to it moments before — each such gesture begins one +//! copy or one paste. A window in the background can neither overwrite the +//! clipboard nor read it. + +/// The most the clipboard holds, in bytes: a copy past it is refused, never +/// cut short. Equal to `kore_edit::CLIPBOARD_BYTES`, the bound every text +/// field already copies within, so any selection a field can copy fits. +pub const CLIPBOARD_BYTES: usize = 512 << 10; + +/// The most one piece of a copy or a paste carries: the request plane whole. +pub const CHUNK_BYTES: usize = crate::menu::MENU_PLANE_BYTES; + +const _: () = assert!(CLIPBOARD_BYTES <= u32::MAX as usize && CHUNK_BYTES <= u16::MAX as usize); + +/// What a paste into a shell's line takes from the text (`res/CLIPBOARD.md` +/// §Shell): the printable ASCII before the first line break, and never a line +/// break, so a paste can never run a command the user has not read. Why the +/// rest was held back is told to the user, never silently dropped. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ShellStop { + /// The whole text was taken. + Whole, + /// The text goes on past a line break. + LineBreak, + /// A character the line cannot hold: a control byte, or a character + /// outside ASCII. + Unholdable, + /// The line is full. + Full, +} + +impl ShellStop { + /// What the user is told about the text held back: nothing when the + /// whole text was typed. + pub const fn describe(self) -> Option<&'static str> { + match self { + Self::Whole => None, + Self::LineBreak => Some("typed the first line; the other lines were held back"), + Self::Unholdable => { + Some("stopped at a tab, control or non-ASCII character the line cannot hold") + } + Self::Full => Some("the line is full; the rest was held back"), + } + } +} + +/// Scan `bytes`, the text from where the last scan stopped, for what a shell +/// paste may take: the length of the run it takes, and why it stops there. +/// `last` says `bytes` reaches the text's end; `room` bounds the run. `None` +/// means `bytes` ran out before the text did: scan the next piece with the +/// room that is left, even none, to learn what follows. +pub fn shell_run(bytes: &[u8], last: bool, room: usize) -> (usize, Option) { + let limit = bytes.len().min(room); + let taken = bytes[..limit] + .iter() + .position(|&byte| !(b' '..=b'~').contains(&byte)) + .unwrap_or(limit); + let stop = match bytes.get(taken) { + Some(b'\r' | b'\n') => Some(ShellStop::LineBreak), + Some(_) if taken < room => Some(ShellStop::Unholdable), + Some(_) => Some(ShellStop::Full), + None if last => Some(ShellStop::Whole), + None => None, + }; + (taken, stop) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A shell paste takes printable ASCII up to the first line break, the + /// first character the line cannot hold, or the line's room. + #[test] + fn a_shell_run_stops_before_what_the_line_cannot_take() { + assert_eq!(shell_run(b"ls -la", true, 255), (6, Some(ShellStop::Whole))); + assert_eq!(shell_run(b"ls -la", false, 255), (6, None)); + assert_eq!(shell_run(b"echo hi\nrm -rf /", true, 255), (7, Some(ShellStop::LineBreak))); + assert_eq!(shell_run(b"a\r\nb", true, 255), (1, Some(ShellStop::LineBreak))); + assert_eq!(shell_run(b"tab\there", true, 255), (3, Some(ShellStop::Unholdable))); + assert_eq!(shell_run("caf\u{e9}".as_bytes(), true, 255), (3, Some(ShellStop::Unholdable))); + assert_eq!(shell_run(b"abcdef", true, 4), (4, Some(ShellStop::Full))); + // A run that fills the room exactly at the text's end took it whole; + // short of the end, what follows decides. + assert_eq!(shell_run(b"abcd", true, 4), (4, Some(ShellStop::Whole))); + assert_eq!(shell_run(b"abcd", false, 4), (4, None)); + assert_eq!(shell_run(b"\nmore", true, 0), (0, Some(ShellStop::LineBreak))); + assert_eq!(shell_run(b"more", true, 0), (0, Some(ShellStop::Full))); + assert_eq!(shell_run(b"", true, 4), (0, Some(ShellStop::Whole))); + // A line break at the room's edge is still a line break: the line + // is not full, the text has another line. + assert_eq!(shell_run(b"abcd\n", true, 4), (4, Some(ShellStop::LineBreak))); + } +} diff --git a/native/crates/window/src/control.rs b/native/crates/window/src/control.rs index 8aec8e2..21ccca6 100644 --- a/native/crates/window/src/control.rs +++ b/native/crates/window/src/control.rs @@ -40,10 +40,17 @@ const SET_LAYOUT_HINTS: u8 = 0x4e; const PREPARE_CAPTURE: u8 = 0x4f; const REQUEST_CAPTURE: u8 = 0x50; const RELEASE_CAPTURE: u8 = 0x51; +const READ_CLIPBOARD: u8 = 0x54; const ACCEPTED: u8 = 0x40; const ADMITTED: u8 = 0x41; const REFUSED: u8 = 0x42; const SHELL_ACTION: u8 = 0x4d; +const CLIPBOARD_TEXT: u8 = 0x54; + +/// Bytes of clipboard text one [`DisplayRequest::ReadClipboard`] answer +/// carries inline: the reply record's four data words. A shell paste takes at +/// most one line of the shell's line editor, a few of these. +pub const CLIPBOARD_TEXT_BYTES: usize = 32; /// Bound on one window's published tab strip. pub const MAX_TABS: usize = 8; @@ -310,6 +317,12 @@ pub enum ShellAction { /// with `token` (never zero), by which it claims the pick. Data only; /// the compositor brokers the answer. Pick { token: u64, types: u64 }, + /// The user pasted into this controller's terminal window + /// (`res/CLIPBOARD.md`): read the clipboard with + /// [`DisplayRequest::ReadClipboard`] under `ticket` (never zero) and type + /// what the line can hold. The ticket reads the text the user pasted and + /// no other: a copy made since refuses it. + Paste { ticket: u64 }, } impl ShellAction { @@ -361,6 +374,7 @@ impl ShellAction { refusal: CaptureRefusal::from_code(b as u8)?, })), 6 if a != 0 && b != 0 => Some(Some(Self::Pick { token: a, types: b })), + 7 if a != 0 && b == 0 => Some(Some(Self::Paste { ticket: a })), _ => None, } } @@ -397,6 +411,7 @@ impl ShellAction { (5, mode.code() as u64, refusal.code() as u64) } Some(Self::Pick { token, types }) => (6, token, types), + Some(Self::Paste { ticket }) => (7, ticket, 0), } } } @@ -668,8 +683,65 @@ pub enum DisplayRequest { ReleaseCapture { ticket: u32, }, + /// Controller verb: read the clipboard text the user pasted into this + /// controller's terminal, from `offset`, under the ticket its + /// [`ShellAction::Paste`] carried. + ReadClipboard { ticket: u64, offset: u32 }, +} + +/// Clipboard text read for a controller: bytes `offset..offset + len` of a +/// `total`-byte text, the first `len` of `bytes` (the rest zero). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ClipboardText { + pub offset: u32, + pub len: u8, + pub total: u32, + pub bytes: [u8; CLIPBOARD_TEXT_BYTES], +} + +impl ClipboardText { + /// The text this answer carries. + pub fn text(&self) -> &[u8] { + &self.bytes[..usize::from(self.len)] + } + + /// The answer's word: `offset | total << 24 | len << 48`. + const fn word(&self) -> u64 { + self.offset as u64 | (self.total as u64) << 24 | (self.len as u64) << 48 + } + + fn from_words(word: u64, data: [u64; 4]) -> Option { + let (offset, total, len) = ((word & 0xff_ffff) as u32, (word >> 24 & 0xff_ffff) as u32, (word >> 48) as u16); + let mut bytes = [0; CLIPBOARD_TEXT_BYTES]; + for (word, chunk) in data.into_iter().zip(bytes.chunks_exact_mut(8)) { + chunk.copy_from_slice(&word.to_le_bytes()); + } + let len = u8::try_from(len).ok()?; + (len != 0 + && usize::from(len) <= CLIPBOARD_TEXT_BYTES + && total != 0 + && total as usize <= crate::clipboard::CLIPBOARD_BYTES + && u64::from(offset) + u64::from(len) <= u64::from(total) + && bytes[usize::from(len)..].iter().all(|&byte| byte == 0)) + .then_some(Self { + offset, + len, + total, + bytes, + }) + } + + fn data(&self) -> [u64; 4] { + core::array::from_fn(|word| { + u64::from_le_bytes(self.bytes[word * 8..word * 8 + 8].try_into().unwrap()) + }) + } } +const _: () = assert!( + crate::clipboard::CLIPBOARD_BYTES < 1 << 24 && CLIPBOARD_TEXT_BYTES == 4 * 8 +); + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DisplayResponse { Desktop(DesktopState), @@ -689,6 +761,7 @@ pub enum DisplayResponse { Configured(Configure), ControlRefused(ControlRefusal), ShellAction(Option), + ClipboardText(ClipboardText), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -848,6 +921,11 @@ pub fn encode_req(request: &DisplayRequest) -> SurReqWire { w.op = RELEASE_CAPTURE; w.subject = u64::from(ticket).into(); } + DisplayRequest::ReadClipboard { ticket, offset } => { + w.op = READ_CLIPBOARD; + w.subject = ticket.into(); + w.a = u64::from(offset).into(); + } } w } @@ -981,6 +1059,16 @@ pub fn decode_req(w: &SurReqWire) -> Option { .ok() .filter(|ticket| *ticket != 0)?, }, + READ_CLIPBOARD => { + let offset = u32::try_from(w.a.get()).ok()?; + if w.subject.get() == 0 || offset as usize >= crate::clipboard::CLIPBOARD_BYTES { + return None; + } + DisplayRequest::ReadClipboard { + ticket: w.subject.get(), + offset, + } + } op if op & OUTPUT != 0 => { DisplayRequest::Output(kore_svc_output::wire::decode_req(&OutReqWire { op: op & !OUTPUT, @@ -1053,6 +1141,15 @@ pub fn encode_rep(response: &DisplayResponse, grants: u8) -> SurRepWire { w.a = a.into(); w.b = b.into(); } + DisplayResponse::ClipboardText(text) => { + w.tag = CLIPBOARD_TEXT; + w.id = text.word().into(); + let [a, b, c, d] = text.data(); + w.a = a.into(); + w.b = b.into(); + w.c = c.into(); + w.d = d.into(); + } } w } @@ -1119,6 +1216,9 @@ pub fn decode_rep(w: &SurRepWire) -> Option<(DisplayResponse, u8)> { SHELL_ACTION => { DisplayResponse::ShellAction(ShellAction::from_wire(w.id.get(), w.a.get(), w.b.get())?) } + CLIPBOARD_TEXT if w.id.get() >> 56 == 0 => DisplayResponse::ClipboardText( + ClipboardText::from_words(w.id.get(), [w.a.get(), w.b.get(), w.c.get(), w.d.get()])?, + ), tag if tag & OUTPUT != 0 => { DisplayResponse::Output(kore_svc_output::wire::decode_rep(&OutRepWire { tag: tag & !OUTPUT, @@ -1193,6 +1293,10 @@ pub fn reply_matches(w: &SurReqWire, response: &DisplayResponse, grants: u8) -> DisplayResponse::Accepted, ) => true, (DisplayRequest::TakeShellAction, DisplayResponse::ShellAction(_)) => true, + ( + DisplayRequest::ReadClipboard { offset, .. }, + DisplayResponse::ClipboardText(text), + ) => text.offset == offset, (DisplayRequest::TakeAdmission, DisplayResponse::Admitted { .. }) => true, ( DisplayRequest::CancelAdmission, @@ -1220,7 +1324,8 @@ pub fn reply_matches(w: &SurReqWire, response: &DisplayResponse, grants: u8) -> | DisplayRequest::TakeShellAction | DisplayRequest::PrepareCapture { .. } | DisplayRequest::RequestCapture { .. } - | DisplayRequest::ReleaseCapture { .. }, + | DisplayRequest::ReleaseCapture { .. } + | DisplayRequest::ReadClipboard { .. }, DisplayResponse::ControlRefused(_), ) => true, (DisplayRequest::CancelAdmission, DisplayResponse::ControlRefused(_)) => true, @@ -1819,6 +1924,67 @@ mod tests { assert_eq!(decode_rep(&wire), None); } + /// A shell paste names its ticket; a clipboard read answers its own + /// offset with a piece inside its text, zero past its length, and + /// nothing else; and an unticketed or out-of-bound read refuses. + #[test] + fn a_shell_paste_reads_its_ticketed_text_and_fails_closed() { + let paste = DisplayResponse::ShellAction(Some(ShellAction::Paste { ticket: 0xfeed })); + assert_eq!(decode_rep(&encode_rep(&paste, 0)), Some((paste, 0))); + let mut bad = encode_rep(&paste, 0); + bad.a = 0.into(); + assert_eq!(decode_rep(&bad), None, "no ticket"); + bad.a = 1.into(); + bad.b = 1.into(); + assert_eq!(decode_rep(&bad), None, "a stray word"); + + let read = DisplayRequest::ReadClipboard { + ticket: 0xfeed, + offset: 32, + }; + let wire = encode_req(&read); + assert_eq!(decode_req(&wire), Some(read)); + for (ticket, offset) in [(0, 0), (1, crate::clipboard::CLIPBOARD_BYTES as u32)] { + assert_eq!( + decode_req(&encode_req(&DisplayRequest::ReadClipboard { ticket, offset })), + None, + "ticket {ticket} offset {offset}" + ); + } + let mut bytes = [0; CLIPBOARD_TEXT_BYTES]; + bytes[..5].copy_from_slice(b"ls -l"); + let text = ClipboardText { + offset: 32, + len: 5, + total: 37, + bytes, + }; + let answer = DisplayResponse::ClipboardText(text); + let reply = encode_rep(&answer, 0); + assert_eq!(decode_rep(&reply), Some((answer, 0))); + assert_eq!(text.text(), b"ls -l"); + assert!(reply_matches(&wire, &answer, 0)); + assert!(!reply_matches(&wire, &answer, 1)); + assert!(reply_matches(&wire, &DisplayResponse::ControlRefused(ControlRefusal::Stale), 0)); + let elsewhere = encode_req(&DisplayRequest::ReadClipboard { + ticket: 0xfeed, + offset: 0, + }); + assert!(!reply_matches(&elsewhere, &answer, 0)); + let mut tail = reply; + tail.d = (tail.d.get() | 1 << 56).into(); + assert_eq!(decode_rep(&tail), None, "a live byte past the piece"); + for (offset, len, total) in [(0u32, 0u16, 5u32), (0, 33, 64), (33, 5, 37), (0, 1, 0)] { + let mut bad = reply; + bad.id = (u64::from(offset) | u64::from(total) << 24 | u64::from(len) << 48).into(); + bad.a = 0.into(); + bad.b = 0.into(); + bad.c = 0.into(); + bad.d = 0.into(); + assert_eq!(decode_rep(&bad), None, "{offset} {len} {total}"); + } + } + #[test] fn the_ninth_dock_row_round_trips_publication_and_launch() { let request = DisplayRequest::PublishDockItem { diff --git a/native/crates/window/src/keysplit.rs b/native/crates/window/src/keysplit.rs new file mode 100644 index 0000000..2b3fc08 --- /dev/null +++ b/native/crates/window/src/keysplit.rs @@ -0,0 +1,315 @@ +//! The keys a terminal window keeps from its shell (`res/KEYBOARD.md` §8). +//! +//! A terminal's keys are its shell's line, except the chords that act on the +//! window itself: its [`TerminalKeys`] — by default Command+C to copy the +//! selection, Command+V to paste and Command+A to select all, as on a Mac, so +//! Control stays the shell's (Control+C interrupts), and Shift with Page Up, +//! Page Down, Home or End to scroll the window's history. With Command held +//! every key is the window's, bound or not: Command never types. +//! +//! A key belongs to whoever it was pressed for, until it is released: Command +//! let go while C is still held does not hand the shell a fresh `c`, and a +//! letter held down before Command was pressed stays the shell's. So the +//! split is a state machine over complete snapshots, one per source, run by +//! whatever stands between the keyboard and the line: input-stream for the +//! founded terminal, the terminal app for a program it hosts. + +use kore_abi::keyboard::{modifier, usage}; + +use crate::shortcut::{Chord, Keymap, Mods}; + +/// Keys one snapshot lists. +pub const SNAPSHOT_KEYS: usize = 6; + +/// What a terminal window does with a key it keeps. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TerminalAction { + /// Put the selection on the clipboard. + Copy, + /// Type the clipboard into the line. + Paste, + /// Select every row held. + SelectAll, + /// Scroll history a page back, a page forward, to its oldest row, or + /// back to the live grid. + PageUp, + PageDown, + Top, + Bottom, +} + +/// A terminal window's keys: the shortcuts it acts on, and whether Command +/// keeps every key from the line. Both sides of the split read one table — +/// input-stream, which takes the keys out of the founded shell's line, and +/// the window, which acts on them — so a keymap the user edits changes both +/// at once. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TerminalKeys { + pub keymap: Keymap, + /// With Command held, every key is the window's, bound or not. + pub command_reserved: bool, +} + +impl TerminalKeys { + pub const DEFAULT: Self = Self { + keymap: Keymap::new() + .with(Chord::new(usage::C, Mods::COMMAND), TerminalAction::Copy) + .with(Chord::new(usage::V, Mods::COMMAND), TerminalAction::Paste) + .with(Chord::new(usage::A, Mods::COMMAND), TerminalAction::SelectAll) + .with(Chord::new(usage::PAGE_UP, Mods::SHIFT), TerminalAction::PageUp) + .with(Chord::new(usage::PAGE_DOWN, Mods::SHIFT), TerminalAction::PageDown) + .with(Chord::new(usage::HOME, Mods::SHIFT), TerminalAction::Top) + .with(Chord::new(usage::END, Mods::SHIFT), TerminalAction::Bottom), + command_reserved: true, + }; + + /// Whether `key`, pressed with the report bitmap `modifiers` held, is the + /// window's rather than the line's. + pub fn owns(&self, key: u8, modifiers: u8) -> bool { + if key == 0 || modifier::bit(key).is_some() { + return false; + } + self.keymap.action(key, modifiers).is_some() + || (self.command_reserved && Mods::of(modifiers).contains(Mods::COMMAND)) + } +} + +impl Default for TerminalKeys { + fn default() -> Self { + Self::DEFAULT + } +} + +/// Sources a split tells apart at once: input-stream's own bound on the +/// keyboards one terminal reads (its `PrevTable`). A further source shares +/// the last place, which forgets its holder's keys as it changes hands — +/// degraded the way every per-source table there is, never dropped. +const SOURCES: usize = 4; + +/// What one source holds, by whom it was pressed for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Held { + source: u8, + window: [u8; SNAPSHOT_KEYS], + shell: [u8; SNAPSHOT_KEYS], + /// The window's view as last handed out: the modifiers and its keys. + shown: (u8, [u8; SNAPSHOT_KEYS]), +} + +impl Held { + /// A source as the window first knows it: nothing held, no modifiers — + /// the state a fresh window's keyboard starts in, so plain typing never + /// reaches the window at all. + const FRESH: Self = Self { + source: 0, + window: [0; SNAPSHOT_KEYS], + shell: [0; SNAPSHOT_KEYS], + shown: (0, [0; SNAPSHOT_KEYS]), + }; +} + +/// One snapshot, split. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Split { + /// The keys the shell's line reads: the snapshot without the window's. + pub shell: [u8; SNAPSHOT_KEYS], + /// The keys the window reads, with the snapshot's modifiers, when what + /// the window sees changed: a key of its own pressed or released, or a + /// modifier, which it needs for a Shift-click. + pub window: Option<[u8; SNAPSHOT_KEYS]>, +} + +/// The keys of `keys` that `side` holds, in their order, packed to the front +/// as a keyboard reports them. +fn pack(keys: [u8; SNAPSHOT_KEYS], side: &[u8; SNAPSHOT_KEYS]) -> [u8; SNAPSHOT_KEYS] { + let mut packed = [0; SNAPSHOT_KEYS]; + for (slot, key) in packed + .iter_mut() + .zip(keys.into_iter().filter(|key| *key != 0 && side.contains(key))) + { + *slot = key; + } + packed +} + +/// The split's state, per source, under one window's keys. +pub struct KeySplit { + keys: TerminalKeys, + held: [Held; SOURCES], + used: usize, +} + +impl KeySplit { + /// A split under `keys`: [`TerminalKeys::DEFAULT`] until the user's + /// keymap names others. + pub const fn new(keys: TerminalKeys) -> Self { + Self { + keys, + held: [Held::FRESH; SOURCES], + used: 0, + } + } + + /// `source`'s place in `held`, of which `used` are taken. + fn place<'h>(held: &'h mut [Held; SOURCES], used: &mut usize, source: u8) -> &'h mut Held { + let index = match held[..*used].iter().position(|held| held.source == source) { + Some(index) => index, + None => { + let index = (*used).min(SOURCES - 1); + *used = (*used + 1).min(SOURCES); + held[index] = Held { source, ..Held::FRESH }; + index + } + }; + &mut held[index] + } + + /// Split `source`'s complete snapshot: `modifiers` held, `keys` down. + pub fn split(&mut self, source: u8, modifiers: u8, keys: [u8; SNAPSHOT_KEYS]) -> Split { + let keys_owned = &self.keys; + let held = Self::place(&mut self.held, &mut self.used, source); + // A key no longer down is released, whoever held it. + for owned in held.window.iter_mut().chain(held.shell.iter_mut()) { + if *owned != 0 && !keys.contains(owned) { + *owned = 0; + } + } + // A key newly down is the owner's its press made it. + for &key in keys.iter().filter(|&&key| key != 0) { + if held.window.contains(&key) || held.shell.contains(&key) { + continue; + } + let side = if keys_owned.owns(key, modifiers) { + &mut held.window + } else { + &mut held.shell + }; + // The two sides partition one snapshot's keys, so a free place is + // always there. + if let Some(free) = side.iter_mut().find(|slot| **slot == 0) { + *free = key; + } + } + let shell = pack(keys, &held.shell); + let window = pack(keys, &held.window); + let view = (modifiers, window); + let changed = held.shown != view; + held.shown = view; + Split { + shell, + window: changed.then_some(window), + } + } + + /// A route boundary: whatever was held is forgotten, as the window + /// forgets it on the focus change that made the boundary. + pub fn reset(&mut self) { + *self = Self::new(self.keys); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use kore_abi::keyboard::modifier::{GUI, LEFT_GUI, LEFT_SHIFT}; + + const C: u8 = 0x06; + const V: u8 = 0x19; + const A: u8 = 0x04; + + fn keys_of(down: &[u8]) -> [u8; SNAPSHOT_KEYS] { + keys(down) + } + + fn keys(down: &[u8]) -> [u8; SNAPSHOT_KEYS] { + let mut keys = [0; SNAPSHOT_KEYS]; + keys[..down.len()].copy_from_slice(down); + keys + } + + /// Command chords and Shift with the scrolling keys are the window's; + /// everything else, Control chords included, is the shell's. + #[test] + fn the_window_keeps_command_chords_and_shifted_scrolling() { + let keys = TerminalKeys::DEFAULT; + assert!(keys.owns(C, LEFT_GUI)); + assert!(keys.owns(V, GUI)); + assert!(keys.owns(usage::PAGE_UP, LEFT_SHIFT)); + assert!(keys.owns(usage::END, LEFT_SHIFT | LEFT_GUI), "Command keeps every key"); + assert!(!keys.owns(usage::PAGE_UP, 0)); + assert!(!keys.owns(C, modifier::LEFT_CTRL)); + assert!(!keys.owns(A, LEFT_SHIFT)); + assert!(!keys.owns(usage::LEFT_GUI, LEFT_GUI), "a modifier's own usage is state"); + assert!(!keys.owns(0, LEFT_GUI)); + assert_eq!(keys.keymap.action(V, modifier::RIGHT_GUI), Some(TerminalAction::Paste)); + assert_eq!(keys.keymap.action(usage::HOME, LEFT_SHIFT), Some(TerminalAction::Top)); + } + + /// The split follows the keymap it was given: a user's binding takes its + /// key from the line, and without Command reserved, unbound Command keys + /// stay the line's. + #[test] + fn the_split_reads_the_keymap_it_was_given() { + let mut keys = TerminalKeys::DEFAULT; + keys.command_reserved = false; + keys.keymap + .bind(Chord::new(usage::INSERT, Mods::SHIFT).unwrap(), TerminalAction::Paste) + .unwrap(); + assert!(keys.owns(usage::INSERT, LEFT_SHIFT)); + assert!(!keys.owns(usage::Z, LEFT_GUI)); + let mut split = KeySplit::new(keys); + let paste = split.split(0, LEFT_SHIFT, keys_of(&[usage::INSERT])); + assert_eq!((paste.shell, paste.window), (keys_of(&[]), Some(keys_of(&[usage::INSERT])))); + } + + /// A key stays with whoever it was pressed for until it is released, and + /// the window hears its own keys and every modifier change, nothing else. + #[test] + fn a_key_belongs_to_whoever_it_was_pressed_for() { + let mut split = KeySplit::new(TerminalKeys::DEFAULT); + // `a` held, then Command: the `a` stays the shell's, and no Command+A. + let first = split.split(1, 0, keys(&[A])); + assert_eq!((first.shell, first.window), (keys(&[A]), None), "plain typing is the line's alone"); + let chord = split.split(1, LEFT_GUI, keys(&[A])); + assert_eq!((chord.shell, chord.window), (keys(&[A]), Some(keys(&[])))); + // Command+C, with `a` still down. + let copy = split.split(1, LEFT_GUI, keys(&[A, C])); + assert_eq!((copy.shell, copy.window), (keys(&[A]), Some(keys(&[C])))); + // Command let go while C is held: still the window's, no `c` typed. + let late = split.split(1, 0, keys(&[A, C])); + assert_eq!((late.shell, late.window), (keys(&[A]), Some(keys(&[C])))); + // Released: C pressed again now is the shell's. + let up = split.split(1, 0, keys(&[A])); + assert_eq!((up.shell, up.window), (keys(&[A]), Some(keys(&[])))); + let again = split.split(1, 0, keys(&[A, C])); + assert_eq!((again.shell, again.window), (keys(&[A, C]), None), "nothing changed for the window"); + // Another source is its own. + let other = split.split(2, LEFT_SHIFT, keys(&[usage::PAGE_UP])); + assert_eq!((other.shell, other.window), (keys(&[]), Some(keys(&[usage::PAGE_UP])))); + assert_eq!(split.split(1, 0, keys(&[A, C])).window, None); + } + + /// A source past the table's bound takes the last place, forgetting the + /// keys its holder had there rather than reading them as its own. + #[test] + fn a_source_past_the_bound_shares_the_last_place() { + let mut split = KeySplit::new(TerminalKeys::DEFAULT); + for source in 0..SOURCES as u8 { + let _ = split.split(source, LEFT_GUI, keys(&[C])); + } + let stranger = split.split(9, 0, keys(&[C])); + assert_eq!((stranger.shell, stranger.window), (keys(&[C]), None)); + let first = split.split(0, 0, keys(&[C])); + assert_eq!((first.shell, first.window), (keys(&[]), Some(keys(&[C]))), "the first sources keep theirs"); + } + + /// A boundary forgets every key, and the window hears the next state. + #[test] + fn a_reset_forgets_the_holders() { + let mut split = KeySplit::new(TerminalKeys::DEFAULT); + let _ = split.split(0, LEFT_GUI, keys(&[V])); + split.reset(); + let after = split.split(0, 0, keys(&[V])); + assert_eq!((after.shell, after.window), (keys(&[V]), None)); + } +} diff --git a/native/crates/window/src/lib.rs b/native/crates/window/src/lib.rs index b0e622d..98bc056 100644 --- a/native/crates/window/src/lib.rs +++ b/native/crates/window/src/lib.rs @@ -6,6 +6,7 @@ pub mod appearance; pub mod capture; #[cfg(feature = "runtime")] pub mod channel; +pub mod clipboard; #[cfg(feature = "runtime")] pub mod client; pub mod command; @@ -21,6 +22,8 @@ pub mod configure; pub mod control; #[cfg(feature = "runtime")] pub mod event; +pub mod keysplit; +pub mod shortcut; mod face; #[cfg(feature = "runtime")] pub mod relay; diff --git a/native/crates/window/src/relay.rs b/native/crates/window/src/relay.rs index 6ecf091..e6fe9a8 100644 --- a/native/crates/window/src/relay.rs +++ b/native/crates/window/src/relay.rs @@ -148,7 +148,25 @@ impl RelayProducer<'_> { /// Loss is explicit, bounded and nonblocking. While a Reset is pending, /// no snapshot is published, even if the consumer has freed ring slots. pub fn push(&mut self, focus: Focus, report: InputReport) -> bool { - if focus.target != FocusTarget::Window || focus.encode().is_none() || !keyboard(report) { + if focus.target != FocusTarget::Window { + return false; + } + self.publish(focus, report) + } + + /// On the terminal route, the keys its window keeps from the shell + /// ([`crate::keysplit`]): the same snapshot ring and loss rule, tagged + /// with the terminal route's epoch, so the compositor hands them to the + /// terminal window and to nothing a later route focuses. + pub fn push_window_keys(&mut self, focus: Focus, report: InputReport) -> bool { + if focus.target != FocusTarget::Terminal { + return false; + } + self.publish(focus, report) + } + + fn publish(&mut self, focus: Focus, report: InputReport) -> bool { + if focus.encode().is_none() || !keyboard(report) { return false; } if self.cell.reset_epoch.load(Ordering::Acquire) == 0 @@ -546,6 +564,7 @@ mod tests { malformed.data[0] = 1; assert!(!input.reports.push(focus, malformed)); assert!(!input.reports.push(Focus { epoch: 0, ..focus }, report)); + assert!(!input.reports.push_window_keys(focus, report), "a window route's keys are pushed whole"); assert_eq!(input.reports.dropped(), 0); assert!(!input.reports.needs_wake()); assert!(!compositor.reports.pending()); @@ -553,6 +572,25 @@ mod tests { assert_eq!(compositor.echo.pop(), None); } + /// The terminal route's window keys ride the ring under its own epoch. + #[test] + fn a_terminal_routes_window_keys_ride_under_its_epoch() { + let mut page = Page([0; 4096]); + let (mut input, mut compositor) = split(&mut page.0).unwrap(); + let chord = InputReport::keyboard(1, kore_abi::keyboard::modifier::LEFT_GUI, [6, 0, 0, 0, 0, 0]); + assert!(input.reports.push_window_keys(Focus::INITIAL, chord)); + assert_eq!( + compositor.reports.poll(), + Some(RelayEvent::Snapshot { + epoch: Focus::INITIAL.epoch, + report: chord + }) + ); + let mut malformed = chord; + malformed.data[0] = 1; + assert!(!input.reports.push_window_keys(Focus::INITIAL, malformed)); + } + /// The seat's lock state rides the snapshot to the window unchanged — a /// client resolves Caps Lock exactly as the terminal discipline does. #[test] diff --git a/native/crates/window/src/shortcut.rs b/native/crates/window/src/shortcut.rs new file mode 100644 index 0000000..a7535cc --- /dev/null +++ b/native/crates/window/src/shortcut.rs @@ -0,0 +1,283 @@ +//! Shortcuts as data (`res/KEYBOARD.md` §8): a [`Chord`] — one key and the +//! modifiers held with it — bound to an action in a [`Keymap`]. +//! +//! A window's shortcuts are a table it reads, never a match over usages: the +//! same table answers what a key does, what its menu says the shortcut is, +//! and — for a terminal — which keys leave the shell's line at all +//! ([`crate::keysplit`]). A keymap the user edits (Settings ▸ Keyboard & +//! Pointer) replaces a table; nothing that reads one changes. +//! +//! Modifiers are side-agnostic: either Shift satisfies a chord's Shift. A +//! chord matches its modifiers exactly, so Command+Shift+C is not Command+C. +//! Locks never take part: Caps Lock changes what a letter types, not which +//! shortcut its key is. + +use core::fmt; + +use kore_abi::keyboard::{modifier, usage}; + +/// The modifiers a chord needs, left and right alike. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct Mods(u8); + +impl Mods { + pub const NONE: Self = Self(0); + pub const CTRL: Self = Self(1); + pub const SHIFT: Self = Self(1 << 1); + pub const ALT: Self = Self(1 << 2); + /// The GUI key: Command on a Mac keyboard, the Super (Windows) key on a + /// PC's. + pub const COMMAND: Self = Self(1 << 3); + + /// The modifiers a report's bitmap holds. + pub const fn of(modifiers: u8) -> Self { + const fn held(modifiers: u8, mask: u8, bit: Mods) -> u8 { + if modifiers & mask != 0 { bit.0 } else { 0 } + } + Self( + held(modifiers, modifier::CTRL, Self::CTRL) + | held(modifiers, modifier::SHIFT, Self::SHIFT) + | held(modifiers, modifier::ALT, Self::ALT) + | held(modifiers, modifier::GUI, Self::COMMAND), + ) + } + + pub const fn with(self, other: Self) -> Self { + Self(self.0 | other.0) + } + + pub const fn contains(self, other: Self) -> bool { + self.0 & other.0 == other.0 + } +} + +/// One key and the modifiers held with it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Chord { + key: u8, + mods: Mods, +} + +impl Chord { + /// `key` (a Keyboard/Keypad-page usage) with `mods` held. `None` for no + /// key or a modifier's own usage: a modifier is held with a chord, never + /// its key. + pub const fn new(key: u8, mods: Mods) -> Option { + if key == 0 || modifier::bit(key).is_some() { + return None; + } + Some(Self { key, mods }) + } + + pub const fn key(self) -> u8 { + self.key + } + + pub const fn mods(self) -> Mods { + self.mods + } + + /// Whether `key`, pressed with the report bitmap `modifiers` held, is + /// this chord. + pub const fn matches(self, key: u8, modifiers: u8) -> bool { + self.key == key && Mods::of(modifiers).0 == self.mods.0 + } + + /// The chord as a menu or the settings page spells it: `Cmd+C`, + /// `Shift+Page Up`. + pub fn write_label(self, out: &mut impl fmt::Write) -> fmt::Result { + for (bit, name) in [ + (Mods::CTRL, "Ctrl"), + (Mods::ALT, "Alt"), + (Mods::SHIFT, "Shift"), + (Mods::COMMAND, "Cmd"), + ] { + if self.mods.contains(bit) { + write!(out, "{name}+")?; + } + } + match self.key { + usage::A..=usage::Z => out.write_char(char::from(b'A' + (self.key - usage::A))), + usage::DIGIT_1..=usage::DIGIT_9 => out.write_char(char::from(b'1' + (self.key - usage::DIGIT_1))), + usage::DIGIT_0 => out.write_char('0'), + usage::F1..=usage::F12 => write!(out, "F{}", self.key - usage::F1 + 1), + key => out.write_str(match key { + usage::ENTER => "Enter", + usage::ESCAPE => "Esc", + usage::BACKSPACE => "Backspace", + usage::TAB => "Tab", + usage::SPACE => "Space", + usage::INSERT => "Insert", + usage::HOME => "Home", + usage::PAGE_UP => "Page Up", + usage::DELETE => "Delete", + usage::END => "End", + usage::PAGE_DOWN => "Page Down", + usage::RIGHT => "Right", + usage::LEFT => "Left", + usage::DOWN => "Down", + usage::UP => "Up", + _ => "Key", + }), + } + } +} + +/// The most bindings one keymap holds: a window's shortcuts, with room for a +/// user's own. +pub const MAX_BINDINGS: usize = 16; + +/// A keymap is full. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct KeymapFull; + +/// Chords bound to a window's actions: one action per chord, any number of +/// chords per action. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Keymap { + bindings: [Option<(Chord, A)>; MAX_BINDINGS], +} + +impl Default for Keymap { + fn default() -> Self { + Self::new() + } +} + +impl Keymap { + pub const fn new() -> Self { + Self { + bindings: [None; MAX_BINDINGS], + } + } + + /// This keymap with `chord` bound to `action`, for a table built as a + /// constant; a chord already bound or no room left is a build error. + pub const fn with(mut self, chord: Option, action: A) -> Self { + let Some(chord) = chord else { + panic!("a binding names a key"); + }; + let mut free = None; + let mut index = 0; + while index < MAX_BINDINGS { + match self.bindings[index] { + Some((bound, _)) if bound.key == chord.key && bound.mods.0 == chord.mods.0 => { + panic!("a chord is bound once"); + } + None if free.is_none() => free = Some(index), + _ => {} + } + index += 1; + } + let Some(free) = free else { + panic!("the keymap is full"); + }; + self.bindings[free] = Some((chord, action)); + self + } +} + +impl Keymap { + /// Bind `chord` to `action`, replacing what it was bound to. + pub fn bind(&mut self, chord: Chord, action: A) -> Result<(), KeymapFull> { + if let Some(bound) = self.bindings.iter_mut().flatten().find(|(bound, _)| *bound == chord) { + bound.1 = action; + return Ok(()); + } + let free = self.bindings.iter_mut().find(|slot| slot.is_none()).ok_or(KeymapFull)?; + *free = Some((chord, action)); + Ok(()) + } + + /// Leave `chord` unbound. + pub fn unbind(&mut self, chord: Chord) { + for slot in &mut self.bindings { + if slot.is_some_and(|(bound, _)| bound == chord) { + *slot = None; + } + } + } + + /// What `key` pressed with the report bitmap `modifiers` held does. + pub fn action(&self, key: u8, modifiers: u8) -> Option { + self.bindings + .iter() + .flatten() + .find(|(chord, _)| chord.matches(key, modifiers)) + .map(|&(_, action)| action) + } + + /// The chord a menu shows for `action`: the first bound to it. + pub fn chord(&self, action: A) -> Option { + self.bindings + .iter() + .flatten() + .find(|(_, bound)| *bound == action) + .map(|&(chord, _)| chord) + } +} + +#[cfg(test)] +mod tests { + extern crate std; + use super::*; + use std::string::String; + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum Act { + Copy, + Paste, + } + + const MAP: Keymap = Keymap::new() + .with(Chord::new(usage::C, Mods::COMMAND), Act::Copy) + .with(Chord::new(usage::V, Mods::COMMAND), Act::Paste) + .with(Chord::new(usage::INSERT, Mods::SHIFT), Act::Paste); + + /// A chord matches its modifiers exactly, either side's, and no lock. + #[test] + fn a_chord_matches_its_modifiers_exactly_either_side() { + assert_eq!(MAP.action(usage::C, modifier::LEFT_GUI), Some(Act::Copy)); + assert_eq!(MAP.action(usage::C, modifier::RIGHT_GUI), Some(Act::Copy)); + assert_eq!(MAP.action(usage::C, modifier::LEFT_GUI | modifier::LEFT_SHIFT), None); + assert_eq!(MAP.action(usage::C, modifier::LEFT_CTRL), None); + assert_eq!(MAP.action(usage::INSERT, modifier::RIGHT_SHIFT), Some(Act::Paste)); + assert_eq!(Chord::new(usage::LEFT_SHIFT, Mods::NONE), None, "a modifier is never a chord's key"); + assert_eq!(Chord::new(0, Mods::COMMAND), None); + } + + /// Bindings change as data: rebinding replaces, unbinding removes, and a + /// menu reads the chord an action is bound to. + #[test] + fn a_keymap_is_rebound_as_data() { + let mut map = MAP; + assert_eq!(map.chord(Act::Paste), Chord::new(usage::V, Mods::COMMAND)); + let ctrl_c = Chord::new(usage::C, Mods::CTRL.with(Mods::SHIFT)).unwrap(); + map.bind(ctrl_c, Act::Copy).unwrap(); + assert_eq!(map.action(usage::C, modifier::LEFT_CTRL | modifier::LEFT_SHIFT), Some(Act::Copy)); + map.bind(ctrl_c, Act::Paste).unwrap(); + assert_eq!(map.action(usage::C, modifier::LEFT_CTRL | modifier::LEFT_SHIFT), Some(Act::Paste)); + map.unbind(Chord::new(usage::V, Mods::COMMAND).unwrap()); + assert_eq!(map.action(usage::V, modifier::LEFT_GUI), None); + assert_eq!(map.chord(Act::Paste), Chord::new(usage::INSERT, Mods::SHIFT)); + let mut full = Keymap::::new(); + for key in 0..MAX_BINDINGS as u8 { + full.bind(Chord::new(usage::A + key, Mods::COMMAND).unwrap(), Act::Copy).unwrap(); + } + assert_eq!(full.bind(Chord::new(usage::F1, Mods::NONE).unwrap(), Act::Copy), Err(KeymapFull)); + } + + /// A chord spells itself as menus show it. + #[test] + fn a_chord_labels_itself() { + let label = |chord: Option| { + let mut text = String::new(); + chord.unwrap().write_label(&mut text).unwrap(); + text + }; + assert_eq!(label(Chord::new(usage::C, Mods::COMMAND)), "Cmd+C"); + assert_eq!(label(Chord::new(usage::PAGE_UP, Mods::SHIFT)), "Shift+Page Up"); + assert_eq!(label(Chord::new(usage::DIGIT_0, Mods::CTRL.with(Mods::ALT))), "Ctrl+Alt+0"); + assert_eq!(label(Chord::new(usage::F1 + 11, Mods::NONE)), "F12"); + } +} diff --git a/native/crates/window/src/wire.rs b/native/crates/window/src/wire.rs index 3581b10..0432959 100644 --- a/native/crates/window/src/wire.rs +++ b/native/crates/window/src/wire.rs @@ -16,6 +16,7 @@ use kore_abi::session::DISPLAY_FORMAT_XRGB8888; use kore_abi::svc::{CommitWire, MAX_COMMIT_RECTS, SurRectWire, SurRepWire, SurReqWire}; use crate::capture::{CaptureFrame, CaptureMode}; +use crate::clipboard::{CHUNK_BYTES, CLIPBOARD_BYTES}; use crate::cursor::Cursor; use crate::document::DocumentKind; use crate::menu::MENU_PLANE_BYTES; @@ -210,6 +211,21 @@ pub enum SurfaceRequest { /// stable identity, or zero if the user cancelled; its name is the /// first `len` bytes of this window's menu plane (none when cancelled). Picked { token: u64, object: u64, len: u8 }, + /// Put text on this window's seat's clipboard ([`crate::clipboard`]), a + /// piece at a time: bytes `offset..offset + len` of a `total`-byte text + /// are at the head of this window's request plane. Offset zero begins the + /// copy (focused window, unspent gesture); the piece that reaches `total` + /// ends it, and the text then replaces the clipboard's. + CopyChunk { offset: u32, len: u16, total: u32 }, + /// Read the seat's clipboard from `offset` into this window's request + /// plane. Offset zero begins the read (focused window, unspent gesture); + /// later offsets continue it until it reaches the end. + Paste { offset: u32 }, + /// The seat's terminal window asks for the clipboard to be typed into its + /// shell's line: queued as an inert + /// [`ShellAction::Paste`](crate::control::ShellAction::Paste) for its + /// controller (focused window, unspent gesture). + PasteToShell, } /// The surface-class reply vocabulary. @@ -263,6 +279,17 @@ pub enum SurfaceResponse { /// The answer is delivered and the pick has ended; the plane may be /// reused. The token, echoed. PickAnswered { token: u64 }, + /// The copy has taken the text through `through`; when that is the whole + /// text, the text is the seat's clipboard. + Copied { through: u32 }, + /// Bytes `offset..offset + len` of the clipboard's `total`-byte text are + /// at the head of the request plane. + Clipboard { offset: u32, len: u16, total: u32 }, + /// The seat's clipboard holds no text. + ClipboardEmpty, + /// The shell paste is queued for this window's controller; the text is + /// `total` bytes. + PasteQueued { total: u32 }, /// The refusal, echoing the subject the verb named (zero if none). Refused { refusal: SurfaceRefusal, @@ -288,6 +315,9 @@ pub enum SurfaceRefusal { Unknown, /// Authority delivery or revocation has not settled; retry is safe. Busy, + /// The user did not ask: a clipboard verb from a window that is not its + /// seat's focused window, or with no unspent gesture behind it. + Denied, } impl SurfaceRefusal { @@ -299,6 +329,7 @@ impl SurfaceRefusal { Self::OutOfMemory => SurRepWire::REFUSE_OUT_OF_MEMORY, Self::Unknown => SurRepWire::REFUSE_UNKNOWN, Self::Busy => SurRepWire::REFUSE_BUSY, + Self::Denied => SurRepWire::REFUSE_DENIED, } } @@ -310,6 +341,7 @@ impl SurfaceRefusal { SurRepWire::REFUSE_OUT_OF_MEMORY => Self::OutOfMemory, SurRepWire::REFUSE_UNKNOWN => Self::Unknown, SurRepWire::REFUSE_BUSY => Self::Busy, + SurRepWire::REFUSE_DENIED => Self::Denied, _ => return None, }) } @@ -457,10 +489,49 @@ pub fn encode_req(req: &SurfaceRequest) -> SurReqWire { w.b = U64::new(object); w.c = U64::new(u64::from(len)); } + SurfaceRequest::CopyChunk { offset, len, total } => { + w.op = SurReqWire::OP_COPY_CHUNK; + w.a = U64::new(piece_word(offset, len)); + w.b = U64::new(u64::from(total)); + } + SurfaceRequest::Paste { offset } => { + w.op = SurReqWire::OP_PASTE; + w.a = U64::new(u64::from(offset)); + } + SurfaceRequest::PasteToShell => { + w.op = SurReqWire::OP_PASTE_TO_SHELL; + } } w } +/// The piece word a copy and a paste share: `offset | len << 32`. +const fn piece_word(offset: u32, len: u16) -> u64 { + offset as u64 | (len as u64) << 32 +} + +/// A clipboard text's length: 1 to [`CLIPBOARD_BYTES`]. +fn text_length(word: u64) -> Option { + u32::try_from(word) + .ok() + .filter(|&len| len != 0 && len as usize <= CLIPBOARD_BYTES) +} + +/// A piece of a clipboard text, lifted: `offset..offset + len` of `total` +/// bytes, `len` 1 to [`CHUNK_BYTES`] and `total` 1 to [`CLIPBOARD_BYTES`]. +fn piece(word: u64, total: u64) -> Option<(u32, u16, u32)> { + let (offset, len) = (word as u32, (word >> 32) as u16); + let total = u32::try_from(total).ok()?; + let fits = u64::from(offset) + u64::from(len) <= u64::from(total); + (word >> 48 == 0 + && len != 0 + && usize::from(len) <= CHUNK_BYTES + && total != 0 + && total as usize <= CLIPBOARD_BYTES + && fits) + .then_some((offset, len, total)) +} + /// Structural lift; stray bits are the re-encode comparison's to catch. fn raw_decode_req(w: &SurReqWire) -> Option { Some(match w.op { @@ -532,6 +603,18 @@ fn raw_decode_req(w: &SurReqWire) -> Option { } SurfaceRequest::Picked { token, object, len } } + SurReqWire::OP_COPY_CHUNK => { + let (offset, len, total) = piece(w.a.get(), w.b.get())?; + SurfaceRequest::CopyChunk { offset, len, total } + } + SurReqWire::OP_PASTE => { + let offset = u32::try_from(w.a.get()).ok()?; + if offset as usize >= CLIPBOARD_BYTES { + return None; + } + SurfaceRequest::Paste { offset } + } + SurReqWire::OP_PASTE_TO_SHELL => SurfaceRequest::PasteToShell, _ => return None, }) } @@ -674,6 +757,22 @@ pub fn encode_rep(rep: &SurfaceResponse, grants: u8) -> SurRepWire { w.tag = SurRepWire::TAG_PICK_ANSWERED; w.a = U64::new(token); } + SurfaceResponse::Copied { through } => { + w.tag = SurRepWire::TAG_COPIED; + w.a = U64::new(u64::from(through)); + } + SurfaceResponse::Clipboard { offset, len, total } => { + w.tag = SurRepWire::TAG_CLIPBOARD; + w.a = U64::new(piece_word(offset, len)); + w.b = U64::new(u64::from(total)); + } + SurfaceResponse::ClipboardEmpty => { + w.tag = SurRepWire::TAG_CLIPBOARD_EMPTY; + } + SurfaceResponse::PasteQueued { total } => { + w.tag = SurRepWire::TAG_PASTE_QUEUED; + w.a = U64::new(u64::from(total)); + } SurfaceResponse::Refused { refusal, subject } => { w.tag = SurRepWire::TAG_REFUSED; w.refusal = refusal.code(); @@ -751,6 +850,17 @@ fn raw_decode_rep(w: &SurRepWire) -> Option { }, SurRepWire::TAG_PICK_CLAIMED => SurfaceResponse::PickClaimed { token: w.a.get() }, SurRepWire::TAG_PICK_ANSWERED => SurfaceResponse::PickAnswered { token: w.a.get() }, + SurRepWire::TAG_COPIED => SurfaceResponse::Copied { + through: text_length(w.a.get())?, + }, + SurRepWire::TAG_CLIPBOARD => { + let (offset, len, total) = piece(w.a.get(), w.b.get())?; + SurfaceResponse::Clipboard { offset, len, total } + } + SurRepWire::TAG_CLIPBOARD_EMPTY => SurfaceResponse::ClipboardEmpty, + SurRepWire::TAG_PASTE_QUEUED => SurfaceResponse::PasteQueued { + total: text_length(w.a.get())?, + }, SurRepWire::TAG_REFUSED => SurfaceResponse::Refused { refusal: SurfaceRefusal::from_code(w.refusal)?, subject: w.id.get(), @@ -867,6 +977,20 @@ pub fn reply_matches(request: &SurReqWire, response: &SurfaceResponse, grants: u | (SurfaceRequest::Picked { token, .. }, SurfaceResponse::PickAnswered { token: got }) => { token == got } + // The copy echoes how far it has taken the text: exactly to the end + // of the piece just sent. + (SurfaceRequest::CopyChunk { offset, len, .. }, SurfaceResponse::Copied { through }) => { + u64::from(through) == u64::from(offset) + u64::from(len) + } + // A read answers from the offset asked, and an empty clipboard + // answers only the read's first piece: a read under way that finds + // the clipboard changed is refused instead. + (SurfaceRequest::Paste { offset }, SurfaceResponse::Clipboard { offset: got, .. }) => { + offset == got + } + (SurfaceRequest::Paste { offset }, SurfaceResponse::ClipboardEmpty) => offset == 0, + (SurfaceRequest::PasteToShell, SurfaceResponse::PasteQueued { .. }) => true, + (SurfaceRequest::PasteToShell, SurfaceResponse::ClipboardEmpty) => true, _ => false, } } @@ -1167,7 +1291,7 @@ mod tests { } } - fn every_request() -> [SurfaceRequest; 14] { + fn every_request() -> [SurfaceRequest; 19] { [ SurfaceRequest::Query, SurfaceRequest::CreateSurface { @@ -1217,10 +1341,25 @@ mod tests { object: u64::MAX, len: PICK_NAME_BYTES as u8, }, + SurfaceRequest::CopyChunk { + offset: 0, + len: 1, + total: 1, + }, + SurfaceRequest::CopyChunk { + offset: (CLIPBOARD_BYTES - CHUNK_BYTES) as u32, + len: CHUNK_BYTES as u16, + total: CLIPBOARD_BYTES as u32, + }, + SurfaceRequest::Paste { offset: 0 }, + SurfaceRequest::Paste { + offset: CLIPBOARD_BYTES as u32 - 1, + }, + SurfaceRequest::PasteToShell, ] } - fn every_response() -> [SurfaceResponse; 13] { + fn every_response() -> [SurfaceResponse; 19] { [ SurfaceResponse::Info { width: 1920, @@ -1260,9 +1399,104 @@ mod tests { SurfaceResponse::PickQueued { serial: 3 }, SurfaceResponse::PickClaimed { token: 0x7071_ce4e }, SurfaceResponse::PickAnswered { token: 0x7071_ce4e }, + SurfaceResponse::Copied { through: 1 }, + SurfaceResponse::Copied { + through: CLIPBOARD_BYTES as u32, + }, + SurfaceResponse::Clipboard { + offset: 4096, + len: 17, + total: 4113, + }, + SurfaceResponse::ClipboardEmpty, + SurfaceResponse::PasteQueued { total: 9 }, + SurfaceResponse::Refused { + refusal: SurfaceRefusal::Denied, + subject: 0, + }, ] } + /// A clipboard piece names a span inside its text, the text inside the + /// clipboard's bound and the piece inside the plane; nothing crosses + /// with a grant; and each reply answers only the verb and offset asked. + #[test] + fn clipboard_pieces_stay_inside_their_text_and_answer_their_own_offset() { + let copy = SurfaceRequest::CopyChunk { + offset: 4096, + len: 100, + total: 4196, + }; + let wire = encode_req(©); + assert_eq!(SurfaceWire::request_grants(&wire), 0); + assert_eq!(decode_req(&wire), Some(copy)); + assert!(reply_matches(&wire, &SurfaceResponse::Copied { through: 4196 }, 0)); + assert!(!reply_matches(&wire, &SurfaceResponse::Copied { through: 4096 }, 0)); + assert!(!reply_matches(&wire, &SurfaceResponse::Copied { through: 4196 }, 1)); + for (offset, len, total) in [ + (0, 0, 1), + (0, 1, 0), + (1, 1, 1), + (0, CHUNK_BYTES as u16 + 1, CLIPBOARD_BYTES as u32), + (0, 1, CLIPBOARD_BYTES as u32 + 1), + (u32::MAX, 1, CLIPBOARD_BYTES as u32), + ] { + let wire = encode_req(&SurfaceRequest::CopyChunk { offset, len, total }); + assert_eq!(decode_req(&wire), None, "{offset} {len} {total}"); + } + let mut stray = encode_req(©); + stray.a = (stray.a.get() | 1 << 50).into(); + assert_eq!(decode_req(&stray), None); + + let read = encode_req(&SurfaceRequest::Paste { offset: 0 }); + let piece = SurfaceResponse::Clipboard { + offset: 0, + len: 12, + total: 12, + }; + assert!(reply_matches(&read, &piece, 0)); + assert!(reply_matches(&read, &SurfaceResponse::ClipboardEmpty, 0)); + let later = encode_req(&SurfaceRequest::Paste { offset: 12 }); + assert!(!reply_matches(&later, &piece, 0)); + assert!( + !reply_matches(&later, &SurfaceResponse::ClipboardEmpty, 0), + "a read under way never learns the clipboard emptied: it is refused" + ); + let past = encode_req(&SurfaceRequest::Paste { + offset: CLIPBOARD_BYTES as u32, + }); + assert_eq!(decode_req(&past), None); + for piece in [ + (0, 13, 12), + (0, 0, 12), + (0, CHUNK_BYTES as u16 + 1, CLIPBOARD_BYTES as u32), + ] { + let mut w = encode_rep(&SurfaceResponse::ClipboardEmpty, 0); + w.tag = SurRepWire::TAG_CLIPBOARD; + w.a = piece_word(piece.0, piece.1).into(); + w.b = u64::from(piece.2).into(); + assert_eq!(decode_rep(&w), None, "{piece:?}"); + } + + let shell = encode_req(&SurfaceRequest::PasteToShell); + assert!(reply_matches(&shell, &SurfaceResponse::PasteQueued { total: 5 }, 0)); + assert!(reply_matches(&shell, &SurfaceResponse::ClipboardEmpty, 0)); + assert!(!reply_matches(&shell, &SurfaceResponse::Copied { through: 5 }, 0)); + let denied = SurfaceResponse::Refused { + refusal: SurfaceRefusal::Denied, + subject: 0, + }; + for request in [copy, SurfaceRequest::Paste { offset: 0 }, SurfaceRequest::PasteToShell] { + assert!(reply_matches(&encode_req(&request), &denied, 0)); + } + for zero in [ + SurfaceResponse::Copied { through: 0 }, + SurfaceResponse::PasteQueued { total: 0 }, + ] { + assert_eq!(decode_rep(&encode_rep(&zero, 0)), None); + } + } + /// A pick is answered only by its own serial's echo, a claim and an /// answer only by their token's, none carries a grant, and a pick of no /// types, a zero serial or token, or an answer whose name contradicts diff --git a/native/provenance.json b/native/provenance.json index bbbd22f..6832942 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "10c7b400aa96c7533be05054fb76e43850408b23", + "revision": "dfece6447cd331375a28b2bb290691cb8d5ab060", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", @@ -36,11 +36,12 @@ "crates/abi/Cargo.toml": "f9a4811b4b6d06947a47a763354cad3bc739a5ea387194a4dcf22409351f3c7b", "crates/abi/src/cap.rs": "41ba3318d372ae447a47f1ef05932204057de17c7df3dd6b9d9a3dff4a935b3e", "crates/abi/src/clock.rs": "cce1c2839cba2377d3cc8696d60812f9195f071f422bd97bb4530ea0368972fc", - "crates/abi/src/command.rs": "206b8b150688cff68345c9237b78d1170e92f01c8323951e2a3e54b5fb808b97", + "crates/abi/src/command.rs": "e994b1527c989cc106281aa127df9a54dc03e1c85f3b15c49ba70ad5e5d65721", "crates/abi/src/crash.rs": "f3c9e5a954e06743f6993b0a5eeb6937d2199c50af5fe95b6be446393ccf999b", - "crates/abi/src/device.rs": "241909f43812f1bd13a4dc325ed3f43c387d80e65193eb15aa276a4c3f7946f5", + "crates/abi/src/device.rs": "06e8891917210870b64fa2ec3d2018067233ad333cf9d1bd13b3ce848fca97ed", "crates/abi/src/device/census.rs": "6fb54ecaade94b116f41e045b098dc5926e8e1c75eeea33870dd4be01bd1ce23", "crates/abi/src/device/i2c_admission.rs": "1ea2fc04cdb0b15186d2f1d80bde53187860e2197263e225f330965d96e40876", + "crates/abi/src/device/state.rs": "8caf4b751b4cae54f3d78626440bd847a39f1138a4dbc6055d6e8a1a1576ac32", "crates/abi/src/execution.rs": "106d679a3f993c3f4e7ad4eb625d40f4399f5b9b50947cb0336ba44a1d6b51af", "crates/abi/src/founding.rs": "801b8e9771763ff9223c3537aabb5653713d1883d1df62ff673f43982aa3e802", "crates/abi/src/guest/aarch64/mod.rs": "1fd24be3fa6cbc344e9873a0e3e75fabd1b919326e477ec4ea4cf148c01b66f8", @@ -60,12 +61,12 @@ "crates/abi/src/monitor.rs": "8f01b9165d493f78127c3e2d37da401e3c969777b69535612ab2bcbd27eb7871", "crates/abi/src/pkg.rs": "da294e9e37fef2decda03dc5601a2a1131e499378bb14853845c2783b8e1a85a", "crates/abi/src/pointer.rs": "9971a292c1d00b275d8030e4f11fdd71206eaf12671cda8775d220b106ebcbf8", - "crates/abi/src/process.rs": "1dc29781e8d5f500e1e44e8ac36ca466aacf7f14a74b0f730af84336a4319cce", + "crates/abi/src/process.rs": "a5deca9b1d379949b98809a3fcb2bb596e2c06ab18cd27b826151374e6d8a634", "crates/abi/src/processor_status.rs": "c963e48f9b07046ab6c82d4cc2735ee2cc6c94561509a4e6e15927c837c0b0c6", - "crates/abi/src/session.rs": "8679f287e28556b1caff70126e98cc1b99d76dd9489271f8cc0973990bbd46a1", + "crates/abi/src/session.rs": "ba8e90dc3d1167ac81b52c6b4d6369a7a112b9417342f6697793a7c1673e96b3", "crates/abi/src/stor.rs": "eabad4e5588fd343472660e15cccf32cfcc8d8f4c25eb18dffbd1598b6007db2", - "crates/abi/src/svc.rs": "12749268a416577521bf1762cabe0cece8b4adae16572beb13ea3f8228539be3", - "crates/abi/src/task.rs": "1e403147de46734c9f217be6d5fc8edf4dd826d49385a7d4628d0773cd2a0acf", + "crates/abi/src/svc.rs": "ea59db02200cdaa8f6b8f242b43ed9f4d5da4e95c0154749812ca4d22c3fbbd0", + "crates/abi/src/task.rs": "a65caa221d4cd7bcf67755fa1bab4391a1cf8cac80985cc750616ed86e50e276", "crates/abi/src/uring.rs": "e98d9db77fbe00d5547b582700a1104d2c5b2546bbe57742dfff8362c32dcc2e", "crates/app/Cargo.toml": "fb793b25eec583da7b5e404b19bd656a6b8bcee2a86a20b0da399be2210a9398", "crates/app/src/lib.rs": "5e7e8d791236f278ee85a02a3c37f41412d879cc41d67f6f6d3d701811973d7d", @@ -108,23 +109,24 @@ "crates/dagfs/src/entry.rs": "ae077f13df5f0949fbc7c867b5f026aa494843ae605dde4fda845c2a531c33a3", "crates/dagfs/src/error.rs": "f8b93e779e3cc7966db4eaee3098fac4d3615d89441a6fc7ed34061eb6cbaf88", "crates/dagfs/src/face.rs": "b22d6f071397395a8ffda7487f5b876a03fc39590e4a1adefea622d5ae737bc9", - "crates/dagfs/src/fs.rs": "0e0f98c347c6e71363e7f022445e1dc3ccfe6921d2e9ad94f092fcc0296ce1b3", + "crates/dagfs/src/fs.rs": "de64b3e50009141d1be13e6cab3f510ab05cd4ea3d3814642ba2a3b39d388549", "crates/dagfs/src/graph.rs": "833a50a1e6a722ac0d5cefb34b101b9553a9ddd59a5c3d7af0fb723141a063c3", "crates/dagfs/src/ids.rs": "a6cd3127adbbfb3e02c7467f5736e2cabe6b53b9047d98f1cee2ba5de2c78227", "crates/dagfs/src/index.rs": "cd20892b18859bcdf19f63152f994003556bc048b0b0de3816bce250e99d9b81", "crates/dagfs/src/io.rs": "a91034153e6f82d32c9e5b5c8ce21f8b53f35c96adaeb2d507b56b07c75ea577", - "crates/dagfs/src/lib.rs": "aee6c0dd3bec2d44667691a110623a4526a50ff1af3133869b2c7b110e578132", + "crates/dagfs/src/lib.rs": "9d01b5ebc80ba67c7698670e3e9da5c2386d7ba2b4cbe16e3ed2ad4538470bd9", "crates/dagfs/src/name.rs": "a59b8464b96e3a4bb65bd1056e3db5ea80bc3a99da528624ee77757cff4efaf9", "crates/dagfs/src/object.rs": "7c6821771008a5f83b45d8fa53fd9aa00bba405f7a26abec2274962adc1cc00c", "crates/dagfs/src/path.rs": "db0bef7330be961a483e7582df14647d043f2a8fe722eb6320ece488d2fc243d", - "crates/dagfs/src/protocol.rs": "e9a35a3a2d28b1b09e44dc45200323fa258c3ef240d5494afd0b07ea19e17ece", + "crates/dagfs/src/protocol.rs": "de2613cacd587fe64bf613ff5d6de53ed8ccb403d5278bccadc7666e484512f4", "crates/dagfs/src/refusal.rs": "71d612406e9a86509056e6a755e3b3530396232007790588868e8756a5797fe8", "crates/dagfs/src/search.rs": "cfc79463a6367d2fc62a9598ca3fe20d6771c3d9aad07709192da93338a9e4ab", "crates/dagfs/src/stor.rs": "06a270d01e2f3a2dcdb41f803926d863d5fca6229aa14e0cc84337c651acdddb", - "crates/dagfs/src/text.rs": "878ee0abc41bfd30ece0f0a7c6be5935487fb945248773d3eebda8066f8f6032", + "crates/dagfs/src/text.rs": "56517a357cc933ff0f485a8a250d2177baa567d95521b107cc38b558101e7119", "crates/dagfs/src/view.rs": "24d661589ce1bb05e79112cda5ce1027166c7794adc3bcbcfdb3d63e663dfd69", "crates/dagfs/src/walk.rs": "3903476532c8f0933a0db3bf7cae9075c6db8dc06953b479709497329ed7e6a4", - "crates/dagfs/src/wire.rs": "3b5da5293e34e8a6c8b6133228e32b5477c6fef8550c58755ec174b7734feaee", + "crates/dagfs/src/watch.rs": "5070799a20aa65f1c074ec6f605973841547bc378731ea667ee48aac28ab995f", + "crates/dagfs/src/wire.rs": "1777e39074f4da19def29359ce4fe7d079377e4b6911489e37ab540959a2b802", "crates/driver/Cargo.toml": "15154ecb6517360f34580dba34799de5a38d546d8be8a64a626562c59e25e158", "crates/driver/src/accel.rs": "c2355916f1f6647d4bcf8414806c3bb9da45799e458133da70e31190316febfb", "crates/driver/src/block/image.rs": "5bb433b99cafc1a51812469ef8ea637dcc84ad2d2b8a79a3385fb8972884ffb8", @@ -188,7 +190,7 @@ "crates/ld/src/store.rs": "ba5a432a2a3e34e64cdd8f02dccc2c69bab5719b6a4223393c72beca0c7a2b18", "crates/ld/src/stub.rs": "b670715026a889f5baaf233a35d3d24c1891223a0e57665782ee326d90e49f5e", "crates/ld/tests/apply_engine.rs": "e01f5f805f34d614717ceb297275d65936c1b48989df90079fcd2f8ec83b6b1c", - "crates/ld/tests/built_images.rs": "7e6716758f34fab028ac67ce2f016b7e8d7ea93966dd2d300ed4ca254872c055", + "crates/ld/tests/built_images.rs": "400d8f9f4c3a5e62efdbbb790a9eeb3575a6ee3ecfb1aa1ba36c0f8e1fd441fc", "crates/ld/tests/check_gate.rs": "0ead133777f18f929d5e9b0353a289a5b4fa9e2af929155f525524096ee5578c", "crates/ld/tests/common/mod.rs": "1e4bdfa37511848d9e9458d9a550a1ff2410154411c6d026e9106035044ea438", "crates/ld/tests/late_layout.rs": "9bfc460d8036d69c1acdc52c430ee475e25062519cdc7dd30f235b882414a339", @@ -241,14 +243,14 @@ "crates/rt/src/mem_source.rs": "41cbfb3200c222c2d2d0094b19445a9cf7a016e80f20241f8f5cf22872c5ce56", "crates/rt/src/pace.rs": "81217909cee01fa148df9e953e6281d7abf328eed7c6257aac8fbcebc824710c", "crates/rt/src/pinned.rs": "9f86a46572178965c3a111363eb20431fc19916eb9e06c2da50f8f918f3eccf8", - "crates/rt/src/process.rs": "b9286ee4cda4417019e607f258aced1e71fa85d677690099b9602267d27f86a6", + "crates/rt/src/process.rs": "a1de9c1d05a18817adc252fb0c15c82077fbf08659c249670a301978ca039974", "crates/rt/src/process/interrupt.rs": "553bb4aa5678b4efa8d7537c96df0369d8fb810d0b3abb398b71dc7edb3db174", "crates/rt/src/process/words.rs": "cccc85ee65665f1a313c79be8aced6e408899a73986e0d8341ae3734ca79ce31", "crates/rt/src/random.rs": "b6d2738f97a9717d3d9481c49003e6c00df545faf52564189ad1cec508d16af6", "crates/rt/src/reserve.rs": "b2a9cb91873cc404555818d66740e54d4517d7676ad1c1efb620cfd3e43fcdc0", "crates/rt/src/room/mod.rs": "c6f56dda5064ebe75ee85ef122d82c81a507f3fe498b6f9a8463a977a0566c04", "crates/rt/src/room/x86_64.rs": "122e283074fa754a49c73beb733a92660a5d8c53701b5223297ded797c61fc70", - "crates/rt/src/service/mod.rs": "b59017ca341af1f2267dfc1b06d1f11a6e505255b8c327bccee1dfe5d04356fe", + "crates/rt/src/service/mod.rs": "2f31433c345c77bb773036ec1ac259d0a81e74df3b8b24fba0769cc31d50cf8b", "crates/rt/src/session/builder.rs": "4bea448f727d93bca9f956d7d4b5dda6982390ea616d06fb42b0804281e803b1", "crates/rt/src/session/flows.rs": "7205afbbe1d859868b313a24dccff8902e6ba3c1c8c45031e9531f6286259cae", "crates/rt/src/session/mod.rs": "466c2fbd8af228b3277075ad90a6d81031c1cb0a4c877f75669fff80e15338ea", @@ -261,7 +263,7 @@ "crates/rt/src/svc/client.rs": "062b52a0544eb11b9d4f8a938ebe75299bfd6c2ff7f42a1d8d21e8cb487c16d7", "crates/rt/src/svc/mod.rs": "3c62c81946943268c24aee3a0a8e3778442a5475dc7bf3bdb42780aeaaa958e9", "crates/rt/src/svc/recyclable.rs": "9e5af8ad316bdd89e001fef7fbb6914b5a936dcdfbab2122740e03c1cf0ee7ec", - "crates/rt/src/svc/serve.rs": "a22172638a94e3ab52b88095dff9d78c88a1d693c81c6242af4932881d268706", + "crates/rt/src/svc/serve.rs": "01ead15cbff6004fe7d13990c373cb6d50c659526942d1b1cf883f05ec72c8d6", "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", "crates/rt/src/thread.rs": "06d0eefa92c46fe31392db8b10e891e008e95c3ee4162f9f4034f7dec7d9b35c", @@ -288,21 +290,24 @@ "crates/window/src/appearance.rs": "b22e65286b94215bc4265a09ef280a5118f368ac45120311b870a56f7cc40a81", "crates/window/src/capture.rs": "d793f70fff35dfb9b13547f016b5abf8d851a23e53cfe28c1c24ad9949e6ef44", "crates/window/src/channel.rs": "6c96578badb21ddf6f3eb587e65b78af275bd8bcc0e03fd359c88eba6d91138f", - "crates/window/src/client.rs": "2e82a529684f4413cbb1d1475f5d6f108a8eb64f961aa0caec441045c4862d35", + "crates/window/src/client.rs": "485d4bc67d4178c38d1ea53ea7b1c979ea0cde6662df888163f6b2b61a0d82c7", + "crates/window/src/clipboard.rs": "c10c8d7d198b3562900e789077d16761db9cdb69e6425c02bb4a97991a44e55e", "crates/window/src/command.rs": "3ad4867ce62f3a430a68e4d105991229938e131e8cfde8afbbb0f522f99355d3", "crates/window/src/configure.rs": "5731662e919827aa6c9b189dce1d4ae06ebc7f7940c57fafd8662fc33bc46154", - "crates/window/src/control.rs": "0ccad3310bc1abc5efa44ff26d929ee7f7e54c6b83d1a3467c8dca5b3b3f8ca3", + "crates/window/src/control.rs": "3ab2400d32503f75b3419310049252167035179e76f8c838a89094d17946c9a5", "crates/window/src/cursor.rs": "2c320f4f19af4a5f9cd0d4059d1452661ac8cea23f0f6a44e0833cccd4d5f4bd", "crates/window/src/desktop.rs": "2aca94383249fac9f770c4edae35f634ad89388c463b17e4484125190757532b", "crates/window/src/document.rs": "1c4fb5a4bd3ce170a8a79ea97e7efd9aa013b1d2678b5ca1dde06ea1508ddaca", "crates/window/src/event.rs": "cf6565ea882e6c77672f31f0b354b03c57f25591a558893bb9466070b3f503a4", "crates/window/src/face.rs": "1078d8664e6da529c9d4fd43a7e3df77f287d0061f418ed21859d289e86765a0", - "crates/window/src/lib.rs": "310eeb5e3fdafe4e881008d311d0de4346bc1f7a78d417cba2c1e59e354670d6", + "crates/window/src/keysplit.rs": "044360a955b04a5229cba55956e9307a04d7d78facf2dfcf9903a43e3012e3fb", + "crates/window/src/lib.rs": "8b61e7f5e21e981069fc7409d2a7b5fb7add9c3a05aa5db29aa5685cbd5dc479", "crates/window/src/menu.rs": "8baf547a2dd58f83eef635e16e8658a011c917b60125b5d041c885b9eb69db1e", "crates/window/src/notify.rs": "8215605e26509fd615a983eab746ddcec18a156543349f52ddff6f301bdfe569", "crates/window/src/pick.rs": "d861daa5137f7513bc142737da8bd104f84c423e9c26af4f48ef19a3afe1b5aa", - "crates/window/src/relay.rs": "8f56568bcf643825906c74180ee13a14acb8299f6fb08f968919370e82669348", - "crates/window/src/wire.rs": "572daa34bc6233540779c542cd47533ec9c294123562de88ea6a5c2fb54572bf", + "crates/window/src/relay.rs": "bd96d827e9074dfdaddc4bb5a0ea5f924b4c63e5fd2bbffba033ab12f5c6b10b", + "crates/window/src/shortcut.rs": "ce72430d30a58833ddb4bc5c95ff12802b09719bd99383296a52a4d55af70e97", + "crates/window/src/wire.rs": "43789b5b72b003ae74b05b79ae4d1c25a3d015a3798dcddd31c664fe6cbb421f", "drivers/nvme/Cargo.toml": "21acf81448d0cc403eb10ada1c407baf7ff8cea4778ccdeb6c4f0d3508a66fbd", "drivers/nvme/src/block.rs": "4df5122b9c8f28ae878bffb6eeacfdba94569f73182c147a555f345a4d877723", "drivers/nvme/src/commands.rs": "a4cdbfb2b7822143beb7f351f4bba7b4643da1bffa80ffe443f065e47545c097", @@ -315,13 +320,13 @@ "drivers/nvme/src/regs.rs": "d9410b5c64bb70f99e042f7209119c0f0fdb57f859c6a1c07eba90199f1e98c1", "drivers/nvme/src/swap.rs": "f5e1099b83b2cdeb9d288f4d337f1a234d9dfd16c31e9d9303c10f6f14972141", "services/block/Cargo.toml": "2729b36369486a6f5c1c29200eaa4603b25c26352f06f52301b70d9d57260e43", - "services/block/boot.toml": "00aa7a50eb6cea63d8da291572b941ef44136a57682715a73a811d0610055c6a", + "services/block/boot.toml": "99615c152634c9eb62532eb87c917aa3da99fc26d31a1409c4d269043ceace00", "services/block/src/client.rs": "6aed79a404620f89ffa13078f615e846fdf6f3db5629c5cee5fbccbeaa7636cf", "services/block/src/lib.rs": "cf45960460cf6c9dba72f7a6727b51497275c686166a90be531d675097c275f1", "services/block/src/service.rs": "191e9d887d3f3267d6e2aa96e81996f01321bb2febb85e68581159f6e098b4b5", "services/block/src/wire.rs": "a9d64aef9ec3d58d37ca55b841933fc51b76ab94e18b245278b4519e96c65939", "services/dagfs/Cargo.toml": "d229a126616b1d7638f0f5a6edb4ac37d9d116f32afaa1897ffe484fbb286e5e", - "services/dagfs/boot.toml": "25a46002f036013e4e1dd87927324713a4c184a3c73b5e40435aedb4615c0586", + "services/dagfs/boot.toml": "58eda46ef6e6f45bba43df5b72bd7b1c5f45fb039957f9ddedb5d333f10b6b4d", "services/dagfs/examples/fsdemo.rs": "1666a14ef62fd4fc46f7f9efa2eaa5021ed79823f273a77f53758307f884e7d0", "services/dagfs/examples/fswire.rs": "994b97d699796b317741e2c87cbe812cc1776ddcd01173323e72a5f74c8ac267", "services/dagfs/examples/model_volume.rs": "0e3623e962da02b2126062a19d198b9e0a8f6e91896fd9a6f7d13f6f30ddc5f1", @@ -330,12 +335,12 @@ "services/dagfs/src/bounds.rs": "c825fcab0f30777e772c21fcf68cd092b10cf57a741eac22747de87ec3742e87", "services/dagfs/src/client.rs": "3532931af9484f548f7ff60bfd58f56fe392d8a770779f766f4e0edb0580d7d4", "services/dagfs/src/digests.rs": "88a256596ab314714ef523a2bea44dd8d5265436b284534d25d4538535454f3a", - "services/dagfs/src/durable.rs": "93df70108d12fd28134dbaf2434a8e067df2a415bbcd0cdb969a84b86403495d", - "services/dagfs/src/engine.rs": "3c52e78d1acf1bdd89d4bc9a77d95e97c139cebdb2009c2cd25921caa634365e", + "services/dagfs/src/durable.rs": "3caf0dc7bd6c7c13cde2c1a83f1db6d70169efb92f448b54ace80f07b6057456", + "services/dagfs/src/engine.rs": "e7cfda6e4f05a532cfc5bf15854b80254f64f40d25a6e7910bee38bfc0ba52c8", "services/dagfs/src/error.rs": "0a0cac522b38baa34221adda4ccca010fd8119c5aaf1f7b6015ab7a5f602ac33", "services/dagfs/src/extraction.rs": "502c7ba6cc8a37016a6c4a43f46664c3f27fe4c01d2cac21f8eb2cc2591a531d", - "services/dagfs/src/lib.rs": "9e4c0e957c81a9504e4217e0c16a83a0dad48fe35ead5a5785e7d139105d6cc8", - "services/dagfs/src/namespace.rs": "638a66619320f580de82b082f5520f35d47d8ab9b39b21ce413365dbf89dacaa", + "services/dagfs/src/lib.rs": "34140702b546448b1b3e326b0112f1be23e1beeb13af836b7f5377b529bb4e76", + "services/dagfs/src/namespace.rs": "72714f185853fbc2e0b67ab59bca00bf4665896adfc120e42fe5d5e3cdb40173", "services/dagfs/src/object.rs": "ce3468a2a536069eddd3598e8593c7eddc8e5107bd96f6e939af0133eb02fb91", "services/dagfs/src/parent_index.rs": "a46346a378ff2e9dd2ffe2d8a9514d8c777a7d0170ff3b90a9345def3b452a02", "services/dagfs/src/path.rs": "98ddd0026e43f6a4957b896bfa0c33ec217b016229662ebc0402283d1e8b0518", @@ -345,11 +350,12 @@ "services/dagfs/src/provider/mod.rs": "1e40047a77fc1f5140626bc25c9d57e681a38302d8b7a30c9e1c01c140cb3db8", "services/dagfs/src/provider/ram.rs": "fa3175f2d98854f2dc99f41cec1eb47ae7e4a560f9307de5f2c3778f95b7f7f1", "services/dagfs/src/reach.rs": "0b72f192ef37320d0a9a6883b7bdba2cd1a291f76dc343f24c52059d588e94ac", - "services/dagfs/src/router.rs": "07e9ceb72b8af7ab3ec944212df2009fe89d276b9179013d32baaa1e70f7f8d4", + "services/dagfs/src/router.rs": "a598a55029a034a95c14bcf242c1bcb86f3d80759dd797122d4f026af5bc0e26", "services/dagfs/src/search.rs": "98e3807b60e039082e2f16cd975d4637520642b380ea5ee3b7a25a4b843de448", - "services/dagfs/src/service/contract.rs": "32219b508a02f9136416618d2e2ab2db1f3bda46f18f8654683caf43e7eb4a66", - "services/dagfs/src/service/mod.rs": "c815acf69647b61a3b907b19b46e1456894d19237d7f19206892f336c9b1b3e0", + "services/dagfs/src/service/contract.rs": "f978333c7da9cb7797e08b9c71690e09c5b1a58b7eedc3a48256671c7f7577d2", + "services/dagfs/src/service/mod.rs": "3403eaf8628cf0e0bc0ce6ff8d2d9ba7a80ea64d550e2822f680757a6d2672e2", "services/dagfs/src/slot.rs": "e8f8751c09a4e6006bcaccd2d189cbb6762da1be6c4ea56bd331fbdc7f5cf827", + "services/dagfs/src/watches.rs": "9ae8277cd4ea1eadbbc8cc992b53edb6e476f247cf208e43ea6e13af3702b74c", "services/dagfs/src/window.rs": "8be4bd5c8f1e51cfa5744801e12a9406e7dafb351d0340a625ca8b8bddeb3364", "services/dagfs/src/wire.rs": "3c85fd1cdf0e35bb3a85cef10093f1b0d08626a9198fe219ff68fe09ba64595b", "services/dagfs/tests/authority.rs": "1432f45aef2bdffd63b83a2e93481673ba3412b14d4dc2abc1ef91f105172641", @@ -363,7 +369,7 @@ "services/dagfs/tests/provider.rs": "ba4070af34ae034c109e391e97bcc4755c6bd70b8e2fb229aaf9d747fef71e2d", "services/dagfs/tests/query_experiment.rs": "6fe147810474b6754ef31ad0fb95493e0b3c0886fbbafd78deb3729d410e8155", "services/device/Cargo.toml": "8f35c9ddd701b58a538f151e95e93ce618f0b346960491adad6d843287165d1c", - "services/device/boot.toml": "3d5eaf0c17d8a4a976405c9d3182319a0be1d36993842b95cf3be6ce64cc721a", + "services/device/boot.toml": "4a0c84d22eb06e749e5d0803d51b659e3f8af81c52acb14491f58cec4246e752", "services/device/src/client.rs": "71660deb4a9f51edb7a72fbb3edac94170bf4cccf9ce182cbd478ee3daf406eb", "services/device/src/lib.rs": "215c7c7e81d5d51664ac77b72b2edb0bab739d5a54b30bc9ad08f9dd3a3cfea4", "services/device/src/service/mod.rs": "6038ba0b9375e3c6bbb25c49bf0849a5878a402f4a3a691dc9df1494849d252b", @@ -405,7 +411,7 @@ "services/extract/vendor/jsmn/jsmn.h": "babfcb7413354a43dcd539c600a1e00bd2d1579ad0a6f6c7d8b81dccedb16cd0", "services/extract/vendor/jsmn/kore_jsmn.c": "a666060a7af03c819d1faaccc0d6faa19b76bcfe78e4ef61aeab273ad052c95c", "services/mm/Cargo.toml": "25809b6a03916d96b8de64a5988d005c71c49314fafbe0d920c3ec06f2dc5951", - "services/mm/boot.toml": "294b4639deec93e2ef5188522da1d3f5ef0428e8a37e21616a697a2b6182fd24", + "services/mm/boot.toml": "8753f22f227455e5fff90674c7295a933b1d035ee6f3c27b7474ca9b99e149c4", "services/mm/src/client.rs": "c6dcc0dc3ed59e0a4cf7532fd306af4c1dc4d4e5ad1edd1d86887def791c5aa6", "services/mm/src/heap.rs": "e55210f3609f2e3e90f30d5497355ddab5801acc60ccbe3f8acbbffff71dbfde", "services/mm/src/lib.rs": "d3820740666ed993e547ee1b3b579231bf439a813d9766860e3b1b57fe815bd6", @@ -417,7 +423,7 @@ "services/output/src/service/mod.rs": "d80c3beb50979b3e1e500abdfa3c0b9d38e354c24e2e34aea443732abbead6f1", "services/output/src/wire.rs": "3752e994eb18714518facae1b78a9ee2b34f99aad8687653f0b89dc8fdeca621", "services/swap/Cargo.toml": "9c0af01403a6ad5a7f4ff69a02b18c8b1b2624a684814d6dc1ac07968c5bbe53", - "services/swap/boot.toml": "4b6d4873ce4124a5d8ffa6bf93d15fdce0be213520e909ac9baeceec1ab2c120", + "services/swap/boot.toml": "bba1d75a7f4418ffa066453fa0b753eebc5ae9d7e69ac5dca2b2a0ed694479eb", "services/swap/src/client.rs": "04f735d059b491765d5ba63a4f5f640332735bf336e62dcbae86c17647583e2c", "services/swap/src/extent.rs": "d6705f77951df1c42cda51fc4526ba6262cab6af4a0545405ce06d2d2e1db7b4", "services/swap/src/lib.rs": "883684df54a87ae66aa7895ea19a872cabc8ab7fd4aded4ea65b7e3dd51fe0cf", diff --git a/native/services/block/boot.toml b/native/services/block/boot.toml index b085cfb..65fdc1b 100644 --- a/native/services/block/boot.toml +++ b/native/services/block/boot.toml @@ -12,5 +12,5 @@ chain = true endowment = "kore_svc_block::BlockEndowment" protocol = "kore_svc_block::BlockService" endows = ["DMA_POOL@1"] -digest_x86_64 = "blake3:a22f3a5b08f35a278ebf663d557e0b1ec490c8e732cdee0fa68151e349ddbc57" -digest_aarch64 = "blake3:93e621b55fbacf93391e4a32212cacfed71ede1797fb63935e2e971da15ab824" +digest_x86_64 = "blake3:d9f8829568372dbcdccc3028bd4b302850c814eb4d44233290c5bdb165c67cef" +digest_aarch64 = "blake3:d004acf08fb53d2d42656f0df24bf54318e3eda87ec79a6778c2e186b61ade79" diff --git a/native/services/dagfs/boot.toml b/native/services/dagfs/boot.toml index ffb941d..37866c1 100644 --- a/native/services/dagfs/boot.toml +++ b/native/services/dagfs/boot.toml @@ -47,5 +47,5 @@ exec_client = true # `dagfs_heap_supply_holds_every_session` holds this row to it; the rest is # room for the proofs open handles carry, allocated per open at their depth. heap_supply_bytes = 0x00D0_0000 -digest_x86_64 = "blake3:453a43f78c11f4fedf1715a141e4627a4d9858ab975fc8ad03c5d44b8814b7d9" -digest_aarch64 = "blake3:44ed7f47afd777ec49dc5b1193947e3d9cbf3ad3534f2b6edd162056abab35fe" +digest_x86_64 = "blake3:dc2fe387db38e271eac4f5d93ac4b5d815060a82cee8cdc81c16bd363d6e9eed" +digest_aarch64 = "blake3:d27bf4cc470afee6a1b5c7c80127dc7e0f8366569b56b74b09c75719e45a3e23" diff --git a/native/services/dagfs/src/durable.rs b/native/services/dagfs/src/durable.rs index adf8b89..864a3bd 100644 --- a/native/services/dagfs/src/durable.rs +++ b/native/services/dagfs/src/durable.rs @@ -79,7 +79,8 @@ use kore_svc_block::{BlockClient, BlockRefusal, BlockService, ProvisionRequest, use crate::persistence::{PersistentFs, VolumeProbe, bootstrap_data_start}; use crate::{ - FsEndowment, FsError, FsRequest, FsResponse, FsService, ProviderError, RamService, SessionId, + FsEndowment, FsError, FsRequest, FsResponse, FsResult, FsService, Handle, ObjectId, + ProviderError, RamService, SessionId, persistence::{CommitStage, PersistenceError, PersistenceState, commit_engine}, provider::BlockProvider, router::{self, RoutedRequest, Store}, @@ -1248,6 +1249,65 @@ impl MountedEngine { } } + fn attach_watch_plane(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + match self { + Self::Durable { fs, .. } => fs.attach_watch_plane(peer, aux), + Self::Volatile(fs) => fs.attach_watch_plane(peer, aux), + } + } + + fn detach_watch_plane(&mut self, peer: PeerId) { + match self { + Self::Durable { fs, .. } => fs.detach_watch_plane(peer), + Self::Volatile(fs) => fs.detach_watch_plane(peer), + } + } + + fn watch_target(&self, session: SessionId, handle: Handle) -> FsResult<(ObjectId, u64)> { + match self { + Self::Durable { fs, .. } => fs.watch_target(session, handle), + Self::Volatile(fs) => fs.watch_target(session, handle), + } + } + + fn watch_epoch(&self, session: SessionId, handle: Handle, dir: ObjectId) -> FsResult { + match self { + Self::Durable { fs, .. } => fs.watch_epoch(session, handle, dir), + Self::Volatile(fs) => fs.watch_epoch(session, handle, dir), + } + } + + #[allow(clippy::too_many_arguments)] + fn arm_watch( + &mut self, + via: u16, + session: SessionId, + slot: u8, + tag: u32, + store: Store, + handle: Handle, + target: FsResult<(ObjectId, u64)>, + ) -> FsResponse { + match self { + Self::Durable { fs, .. } => fs.arm_watch(via, session, slot, tag, store, handle, target), + Self::Volatile(fs) => fs.arm_watch(via, session, slot, tag, store, handle, target), + } + } + + fn settle_watches(&mut self, system: impl FnMut(SessionId, Handle, ObjectId) -> FsResult) { + match self { + Self::Durable { fs, .. } => fs.settle_watches(system), + Self::Volatile(fs) => fs.settle_watches(system), + } + } + + fn owed_watch_wakes(&mut self, owed: &mut [Option]) -> usize { + match self { + Self::Durable { fs, .. } => fs.owed_watch_wakes(owed), + Self::Volatile(fs) => fs.owed_watch_wakes(owed), + } + } + async fn serve_request_async( &mut self, request: FsRequest, @@ -1417,6 +1477,16 @@ impl MountedFs { self.system.is_some() } + /// Publish what every armed watch saw: the primary frontend holds the + /// planes, and each watch is read through the engine of its own store. + fn settle_watches(&mut self) { + let system = self.system.as_ref(); + self.user.settle_watches(|session, handle, dir| match system { + Some(system) => system.watch_epoch(session, handle, dir), + None => Err(FsError::AccessLost), + }); + } + fn durable(&self) -> bool { self.user.durable() || self.system.as_ref().is_some_and(MountedEngine::durable) } @@ -1711,6 +1781,24 @@ impl MountedFs { } FsResponse::Done } + // A watch on the user volume is the primary frontend's own verb. + RoutedRequest::Watch(Store::User, request) => { + self.user.serve_request_async(request, sv).await + } + // One on the system volume is resolved by the system engine and + // recorded in the pair's own plane: the client holds one plane. + RoutedRequest::Watch(Store::System, FsRequest::Watch { handle, slot, tag }) => { + let Some(system) = self.system.as_ref() else { + return FsResponse::Refused(FsError::InvalidHandle); + }; + let Some(session) = system.session_for_via(sv.via()) else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let target = system.watch_target(session, handle); + self.user + .arm_watch(sv.via(), session, slot, tag, Store::System, handle, target) + } + RoutedRequest::Watch(Store::System, _) => FsResponse::Refused(FsError::Unsupported), RoutedRequest::Both(_) => FsResponse::Refused(FsError::Unsupported), // `DurableService::serve_async` answers the ring verbs before the // engines are consulted; reaching here means this row has none. @@ -3319,6 +3407,28 @@ impl SystemService for DurableService { self.advance_stor(); } + /// A recyclable peer's pair carries its watch plane after the rings; the + /// primary frontend keeps it, whichever volume a watch names. + fn attach_peer_aux(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + self.fs + .as_mut() + .is_some_and(|fs| fs.user.attach_watch_plane(peer, aux)) + } + + fn peer_detaching(&mut self, peer: PeerId) { + if let Some(fs) = self.fs.as_mut() { + fs.user.detach_watch_plane(peer); + } + } + + fn take_owed_wakes(&mut self, owed: &mut [Option]) -> usize { + let Some(fs) = self.fs.as_mut() else { + return 0; + }; + fs.settle_watches(); + fs.user.owed_watch_wakes(owed) + } + fn peer_closed(&mut self, via: u16, reason: PeerClose) -> bool { // A departed session's ring goes first: its targets are client // memory mapped into the controller, and a command still in flight @@ -3481,6 +3591,14 @@ impl SvcProtocol for DurableService { const REPLY_GRANTS: usize = ::REPLY_GRANTS; const SERVE_PAIRS: usize = ::SERVE_PAIRS; + fn init_pair_window( + window: &mut [u8], + req_cap: u32, + rep_cap: u32, + ) -> Result<(), kore_ring::spsc::RingError> { + ::init_pair_window(window, req_cap, rep_cap) + } + fn reply_matches(request: &Self::ReqWire, response: &FsResponse, grants: u8) -> bool { ::reply_matches(request, response, grants) } diff --git a/native/services/dagfs/src/engine.rs b/native/services/dagfs/src/engine.rs index dd2adb0..66a635d 100644 --- a/native/services/dagfs/src/engine.rs +++ b/native/services/dagfs/src/engine.rs @@ -762,6 +762,33 @@ impl Engine

{ }) } + /// What a watch on `handle` follows: the directory it names, and that + /// directory's change epoch now. Watching is observing what a listing + /// shows, so it takes exactly a listing's authority — `READ` on a live + /// route to a directory. + pub(crate) fn watch_target(&self, session: SessionId, handle: Handle) -> FsResult<(ObjectId, u64)> { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + if !self.ns.kind(route.object)?.is_dir() { + return Err(FsError::NotDirectory); + } + Ok((route.object, self.ns.change_epoch(route.object)?)) + } + + /// The change epoch of the directory a watch armed on `handle` follows, + /// read only through a route that still proves to that same directory: + /// a retired edge, a closed handle or a reclaimed directory is + /// `AccessLost`, and the watch ends instead of counting for a client + /// that could no longer list what it counts. + pub(crate) fn watch_epoch(&self, session: SessionId, handle: Handle, dir: ObjectId) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + if route.object != dir { + return Err(FsError::AccessLost); + } + self.ns.change_epoch(dir) + } + /// One page of a directory's entries. pub fn list(&self, session: SessionId, handle: Handle, cursor: u32) -> FsResult { let route = self.route(session, handle)?; diff --git a/native/services/dagfs/src/lib.rs b/native/services/dagfs/src/lib.rs index 0373d1c..ecc50b5 100644 --- a/native/services/dagfs/src/lib.rs +++ b/native/services/dagfs/src/lib.rs @@ -158,6 +158,7 @@ mod router; pub mod search; pub mod service; pub mod slot; +mod watches; pub mod window; pub mod wire; @@ -185,6 +186,9 @@ pub use kore_dagfs::DagfsProtocol; /// The storage-ring client (`res/STORAGE_TRANSPORT.md` §4), for rows that /// read through the filesystem's transport — exec's image loads. pub use kore_dagfs::stor; +/// The watch plane every filesystem pair carries after its rings +/// (`kore_dagfs::watch`), for the founders that format pair windows. +pub use kore_dagfs::watch; pub use namespace::{EdgeId, ListEntry, Name}; pub use object::{ANY_REVISION, ObjectId, ObjectKind, ObjectSubtype, Revision, StableObjectId}; pub use provider::{ diff --git a/native/services/dagfs/src/namespace.rs b/native/services/dagfs/src/namespace.rs index b5ac08d..8adf1db 100644 --- a/native/services/dagfs/src/namespace.rs +++ b/native/services/dagfs/src/namespace.rs @@ -33,7 +33,7 @@ //! while exact edge generations govern route validity. use alloc::{boxed::Box, vec::Vec}; -use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering, fence}; use crate::{ ServiceAlloc, @@ -312,13 +312,29 @@ struct ObjectSnapshotView { /// a metadata commit. A reader never spins behind a preempted writer: it /// returns typed [`FsError::Busy`] and lets the bounded request retry policy /// decide when to revisit the snapshot. +/// +/// `epoch` and `owed` ride the same line but outside the seqlock: each is one +/// word with its own atomic meaning. +/// +/// - `epoch` is a directory's **change epoch** — advanced after every change +/// to what a listing of it shows: an edge made, removed or renamed in it, +/// a child's length, revision or incoming count moved, or an entry made or +/// removed in a child folder (that folder's line: its count, and whether +/// it holds folders). It is never reset, so it stays monotone across slot +/// reuse; the generation is what tells occupants apart. Watches compare it +/// ([`Namespace::change_epoch`]). +/// - `owed` marks a change to what the object's parents show of it — a +/// file's content, a folder's entries — that could not be passed on +/// because another commit held the object's parent row; the holder settles +/// it on release ([`HeldParents`]). #[repr(align(64))] struct ObjectSnapshot { sequence: AtomicU64, generation: AtomicU32, - _pad: AtomicU32, + owed: AtomicU32, len: AtomicU64, dir: AtomicU64, + epoch: AtomicU64, } impl ObjectSnapshot { @@ -326,9 +342,10 @@ impl ObjectSnapshot { Self { sequence: AtomicU64::new(0), generation: AtomicU32::new(0), - _pad: AtomicU32::new(0), + owed: AtomicU32::new(0), len: AtomicU64::new(0), dir: AtomicU64::new(0), + epoch: AtomicU64::new(0), } } @@ -380,6 +397,64 @@ impl ObjectSnapshot { dir: (dir != 0).then(|| DirId(SlotKey::from_bits(dir))), }) } + + /// `id`'s change epoch, validated against the slot's occupant: a slot + /// reused by another object answers `AccessLost`, never its successor's + /// count. + fn read_epoch(&self, id: ObjectId) -> FsResult { + let before = self.sequence.load(Ordering::Acquire); + if before & 1 != 0 { + return Err(FsError::Busy); + } + let generation = self.generation.load(Ordering::Relaxed); + // Acquire: a caller that sees the epoch move sees the change behind + // it (`Namespace::touch` advances it with release order). + let epoch = self.epoch.load(Ordering::Acquire); + let after = self.sequence.load(Ordering::Acquire); + if before != after { + return Err(FsError::Busy); + } + if generation != id.key().generation() || generation == 0 { + return Err(FsError::AccessLost); + } + Ok(epoch) + } +} + +/// A claim on one object's parent row that settles, on release, a change +/// owed to the object's parents ([`Namespace::note_shown`]). +/// +/// Every row claim in this module goes through it, so whichever commit holds +/// the row when a change lands is the one that passes it on: the changer +/// marks the change owed and tries the row; a holder, after letting go, +/// looks for the mark. The two sides store before they look, across a +/// sequentially consistent fence each, so at least one of them sees the +/// other — the change is never lost and never waited for. +pub(crate) struct HeldParents<'a, K> { + ns: &'a Namespace, + object: ObjectId, + claim: Option>, +} + +impl<'a, K> core::ops::Deref for HeldParents<'a, K> { + type Target = ParentClaim<'a>; + fn deref(&self) -> &Self::Target { + self.claim.as_ref().expect("the row is held until the guard drops") + } +} + +impl core::ops::DerefMut for HeldParents<'_, K> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.claim.as_mut().expect("the row is held until the guard drops") + } +} + +impl Drop for HeldParents<'_, K> { + fn drop(&mut self) { + drop(self.claim.take()); + fence(Ordering::SeqCst); + self.ns.settle_owed(self.object); + } } /// One vacant snapshot per object slot, heap-backed for the reason recorded on @@ -396,6 +471,87 @@ fn vacant_snapshots() -> Box<[ObjectSnapshot]> { snapshots.into_boxed_slice() } +/// Change epochs and the parent-row guard. Nothing here depends on the +/// provider's key type: it touches only the snapshot lines and the rows. +impl Namespace { + fn snapshot_line(&self, id: ObjectId) -> Option<&ObjectSnapshot> { + usize::try_from(id.key().index()) + .ok() + .and_then(|index| self.snapshots.get(index)) + } + + /// Advance `dir`'s change epoch, after the change it reports is + /// published (release order; [`ObjectSnapshot::read_epoch`] acquires). + fn touch(&self, dir: ObjectId) { + if let Some(line) = self.snapshot_line(dir) { + line.epoch.fetch_add(1, Ordering::Release); + } + } + + /// Advance the epoch of every directory that names the row's object: what + /// each shows of it — its length, revision, or folder count — moved. + fn touch_parents(&self, row: &ParentClaim<'_>) { + for edge in row.iter() { + self.touch(edge.parent); + } + } + + /// Claim `object`'s parent row through the guard that settles a content + /// change owed while it is held. + fn claim_parents(&self, object: ObjectId) -> FsResult> { + let claim = self.parents_of.claim(object)?; + Ok(HeldParents { + ns: self, + object, + claim: Some(claim), + }) + } + + /// What every directory naming `object` shows of it just changed — a + /// file's length and revision, or a folder's entries (its line's count, + /// and whether it holds folders). Marked owed first, so a commit holding + /// the row passes the change on when it lets go ([`HeldParents`]). + fn note_shown(&self, object: ObjectId) { + let Some(line) = self.snapshot_line(object) else { + return; + }; + line.owed.store(1, Ordering::Relaxed); + fence(Ordering::SeqCst); + self.settle_owed(object); + } + + /// Pass an owed change on to `object`'s parents, if the row is free; a + /// row another commit holds is that commit's to settle. + /// + /// Each pass ends by letting the row go and looking again, because a + /// writer that found the row held during the pass left its mark for this + /// side. Every pass that continues consumed such a mark, so the loop does + /// the writers' work and never waits for anything. + fn settle_owed(&self, object: ObjectId) { + let Some(line) = self.snapshot_line(object) else { + return; + }; + while line.owed.load(Ordering::Relaxed) != 0 { + let Ok(row) = self.parents_of.claim(object) else { + return; + }; + if line.owed.swap(0, Ordering::AcqRel) != 0 { + self.touch_parents(&row); + } + drop(row); + fence(Ordering::SeqCst); + } + } + + /// `dir`'s change epoch: equal readings bracket no change to what a + /// listing of it shows. `AccessLost` once the directory is gone. + pub(crate) fn change_epoch(&self, dir: ObjectId) -> FsResult { + self.snapshot_line(dir) + .ok_or(FsError::AccessLost)? + .read_epoch(dir) + } +} + impl Namespace { pub(crate) fn new() -> Self { // Bootstrap-scale and fixed after startup, exactly like @@ -454,6 +610,7 @@ impl Namespace { debug_assert_eq!(object.id(), id.key()); object.commit_len(len); self.snapshots[id.key().index() as usize].publish(id, len, object.dir()); + self.note_shown(id); object.revision() } @@ -1066,7 +1223,7 @@ impl Namespace { // for the edge, before publishing it, so contention cannot leave a // visible name without its matching count or reverse entry. let mut child_object = self.object_mut(child)?; - let mut parents = self.parents_of.claim(child)?; + let mut parents = self.claim_parents(child)?; parents.reserve()?; let entry = entries .insert(DirEntry { name: *name, child }) @@ -1075,6 +1232,12 @@ impl Namespace { child_object.incoming_inc(); let edge = EdgeId { parent, entry }; parents.add(edge); + // `parent` gained an entry, and every other folder holding the child + // shows one more folder count: the row names them all. `parent`'s own + // folders see its line change (one more entry, perhaps a first + // folder). + self.touch_parents(&parents); + self.note_shown(parent); Ok(edge) } @@ -1093,12 +1256,15 @@ impl Namespace { .ok_or(FsError::NotFound)?; let edge = EdgeId { parent, entry }; let mut child_object = self.object_mut(child)?; - let mut parents = self.parents_of.claim(child)?; + let mut parents = self.claim_parents(child)?; entries .try_remove(edge.entry) .map_err(|error| fs_access(error, FsError::AccessLost))?; child_object.incoming_dec(); parents.remove(edge); + self.touch(parent); + self.touch_parents(&parents); + self.note_shown(parent); Ok((edge, child)) } @@ -1130,7 +1296,7 @@ impl Namespace { return Err(FsError::NotEmpty); } let mut child_object = self.object_mut(child)?; - let mut parents = self.parents_of.claim(child)?; + let mut parents = self.claim_parents(child)?; claims .get_mut(parent)? .try_remove(entry) @@ -1138,6 +1304,9 @@ impl Namespace { child_object.incoming_dec(); let edge = EdgeId { parent, entry }; parents.remove(edge); + self.touch(parent); + self.touch_parents(&parents); + self.note_shown(parent); Ok((edge, child)) } @@ -1166,6 +1335,9 @@ impl Namespace { .try_write(edge.entry) .map_err(|error| fs_access(error, FsError::AccessLost))?; entry.name = *new; + // Only `parent` spells the edge: the child's other folders show it + // under their own names, unchanged. + self.touch(parent); Ok(edge) } @@ -1243,7 +1415,7 @@ impl Namespace { // The child's parent row moves with the edge; it is claimed on its // own, so a file mid-transaction can still be moved. - let mut parents = self.parents_of.claim(child)?; + let mut parents = self.claim_parents(child)?; // Commit: retire the source edge, then create the destination edge. claims @@ -1261,6 +1433,16 @@ impl Namespace { // is deliberately not touched here; the parent row swaps in place. let dst_edge = EdgeId { parent: dst, entry }; parents.replace(src_edge, dst_edge); + // The source lost an entry; the row now names the destination and + // every other folder, whose "located in" for the child moved. + self.touch(src); + self.touch_parents(&parents); + // Across folders, each one's line moved (one entry fewer, one more); + // within one folder only a spelling changed. + if src != dst { + self.note_shown(src); + self.note_shown(dst); + } Ok((src_edge, dst_edge)) } @@ -1360,7 +1542,7 @@ impl Namespace { /// `object`'s incoming edges, copied out of its parent row so no claim /// is held while they are walked. fn incoming_edges(&self, object: ObjectId) -> FsResult> { - let row = self.parents_of.claim(object)?; + let row = self.claim_parents(object)?; let mut edges = BoundedVec::try_new_in(row.len(), crate::service_alloc()) .map_err(|_| FsError::OutOfObjects)?; for edge in row.iter() { @@ -1486,7 +1668,7 @@ impl Namespace { child_ids.as_mut_slice().sort_unstable_by_key(|id| id.key()); let mut children = heapless::Vec::< - (ObjectId, ObjectWrite<'_, K>, ParentClaim<'_>), + (ObjectId, ObjectWrite<'_, K>, HeldParents<'_, K>), MAX_DIR_ENTRIES, >::new(); for child in child_ids { @@ -1494,14 +1676,14 @@ impl Namespace { .objects .try_write(child.key()) .map_err(|error| fs_access(error, FsError::AccessLost))?; - let parents = self.parents_of.claim(child)?; + let parents = self.claim_parents(child)?; children .push((child, claimed, parents)) .map_err(|_| FsError::OutOfObjects)?; } // The reclaimed object names nothing once it is gone; its own row // is emptied with it (it is already empty: nothing names it). - let mut own_parents = self.parents_of.claim(id)?; + let mut own_parents = self.claim_parents(id)?; // All-or-nothing commit: decrement once per removed edge, and drop // its reverse entry, then record only the child generations that @@ -1521,12 +1703,14 @@ impl Namespace { } own_parents.clear(); let mut became_reclaimable = heapless::Vec::new(); - for (id, child, _) in children.iter() { + for (id, child, parents) in children.iter() { if child.is_reclaimable() { became_reclaimable .push(*id) .map_err(|_| FsError::OutOfObjects)?; } + // A child that keeps a folder elsewhere shows one fewer there. + self.touch_parents(parents); } if let Some(entries) = directory.take() { @@ -2743,6 +2927,211 @@ mod tests { assert_eq!(ns.leaked_placements(), 2); } + // ── change epochs ─────────────────────────────────────────────────────── + + /// Whether `dir`'s change epoch moved since `before`, and the new reading. + fn moved(ns: &Ns, dir: ObjectId, before: &mut u64) -> bool { + let now = ns.change_epoch(dir).unwrap(); + let moved = now != *before; + *before = now; + moved + } + + #[test] + fn every_change_a_listing_or_a_folder_line_shows_moves_its_epoch_and_nothing_else_does() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + let b = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + ns.link(root, &name("b"), b).unwrap(); + let file = ns.create_file_object().unwrap(); + let (mut at_a, mut at_b, mut at_root) = ( + ns.change_epoch(a).unwrap(), + ns.change_epoch(b).unwrap(), + ns.change_epoch(root).unwrap(), + ); + + // An entry made in `a`: `a` shows it, and `root` shows `a`'s line + // with one more entry. `b` shows nothing new. + ns.link(a, &name("f"), file).unwrap(); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + assert!(!moved(&ns, b, &mut at_b)); + + // A second folder for the file: `b` gains an entry (and its line in + // `root` moves), and `a` shows one more folder count for it. + ns.link(b, &name("g"), file).unwrap(); + assert!(moved(&ns, b, &mut at_b)); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + + // A rename is one folder's spelling only: its line's count holds. + ns.rename(b, &name("g"), &name("h")).unwrap(); + assert!(moved(&ns, b, &mut at_b)); + assert!(!moved(&ns, a, &mut at_a)); + assert!(!moved(&ns, root, &mut at_root)); + + // A content commit reaches every folder naming the file, and no + // folder line: the folders' entries are unchanged. + { + let mut object = ns.object_mut(file).unwrap(); + ns.commit_file_len(file, &mut object, 12); + } + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, b, &mut at_b)); + assert!(!moved(&ns, root, &mut at_root)); + + // A move: the source loses an entry, the destination gains one, the + // file's other folder sees its location move, and both folders' + // lines move in `root`. + let c = ns.create_dir_object().unwrap(); + ns.link(root, &name("c"), c).unwrap(); + assert!(moved(&ns, root, &mut at_root)); + let mut at_c = ns.change_epoch(c).unwrap(); + ns.relocate(b, &name("h"), c, &name("h")).unwrap(); + assert!(moved(&ns, b, &mut at_b)); + assert!(moved(&ns, c, &mut at_c)); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + + // A move within one folder is a spelling: its line holds. + ns.relocate(c, &name("h"), c, &name("i")).unwrap(); + assert!(moved(&ns, c, &mut at_c)); + assert!(!moved(&ns, root, &mut at_root)); + + // Removing a name: the folder loses it, the other shows one fewer, + // and the folder's line moves. + ns.detach(c, &name("i")).unwrap(); + assert!(moved(&ns, c, &mut at_c)); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + ns.unlink_empty(a, &name("f")).unwrap(); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + + // A refused change moves nothing. + assert_eq!(ns.rename(a, &name("absent"), &name("x")).unwrap_err(), FsError::NotFound); + assert_eq!(ns.detach(a, &name("absent")).unwrap_err(), FsError::NotFound); + assert!(!moved(&ns, a, &mut at_a)); + assert!(!moved(&ns, root, &mut at_root)); + } + + #[test] + fn a_folder_line_change_meeting_its_held_parent_row_is_passed_on_by_the_holder() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + let file = ns.create_file_object().unwrap(); + let mut at_root = ns.change_epoch(root).unwrap(); + + // A commit holds `a`'s own row (a move of `a`, a parents walk) while + // an entry lands in `a`: the linker cannot reach `a`'s folders. + let held = ns.claim_parents(a).unwrap(); + ns.link(a, &name("f"), file).unwrap(); + assert!(!moved(&ns, root, &mut at_root), "the linker left the change owed"); + drop(held); + assert!(moved(&ns, root, &mut at_root), "the holder passed it on"); + drop(ns.claim_parents(a).unwrap()); + assert!(!moved(&ns, root, &mut at_root), "owed once, settled once"); + } + + #[test] + fn a_reclaimed_directory_answers_access_lost_and_its_children_s_other_folders_move() { + let (mut ns, root) = ns_with_root(); + let doomed = ns.create_dir_object().unwrap(); + let kept = ns.create_dir_object().unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("doomed"), doomed).unwrap(); + ns.link(root, &name("kept"), kept).unwrap(); + ns.link(doomed, &name("f"), file).unwrap(); + ns.link(kept, &name("f"), file).unwrap(); + let mut at_kept = ns.change_epoch(kept).unwrap(); + + let (_, child) = ns.detach(root, &name("doomed")).unwrap(); + let mut placements: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + let mut objects: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + assert_eq!(ns.reclaim(child, &mut placements, &mut objects), 1); + assert_eq!(ns.change_epoch(doomed), Err(FsError::AccessLost)); + assert!(moved(&ns, kept, &mut at_kept), "the file is in one folder fewer"); + + // A slot reused by another directory never answers for the old one. + let successor = ns.create_dir_object().unwrap(); + assert_eq!(ns.change_epoch(doomed), Err(FsError::AccessLost)); + assert!(ns.change_epoch(successor).is_ok()); + } + + #[test] + fn a_content_change_meeting_a_held_parent_row_is_passed_on_by_the_holder() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(a, &name("f"), file).unwrap(); + let mut at_a = ns.change_epoch(a).unwrap(); + + // A commit holds the file's row (a move, a parents walk) while the + // file's bytes commit: the writer cannot touch the parents itself. + let held = ns.claim_parents(file).unwrap(); + { + let mut object = ns.object_mut(file).unwrap(); + ns.commit_file_len(file, &mut object, 3); + } + assert!(!moved(&ns, a, &mut at_a), "the writer left the change owed"); + drop(held); + assert!(moved(&ns, a, &mut at_a), "the holder passed it on"); + // Owed once, settled once. + drop(ns.claim_parents(file).unwrap()); + assert!(!moved(&ns, a, &mut at_a)); + } + + #[test] + fn concurrent_writers_and_row_holders_never_lose_a_content_change() { + use std::sync::Arc; + use std::sync::atomic::AtomicBool; + + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(a, &name("f"), file).unwrap(); + let ns = Arc::new(ns); + let stop = Arc::new(AtomicBool::new(false)); + // A holder thread keeps taking and letting go of the row, as parents + // walks and moves do. + let holder = { + let (ns, stop) = (Arc::clone(&ns), Arc::clone(&stop)); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + if let Ok(row) = ns.claim_parents(file) { + core::hint::spin_loop(); + drop(row); + } + } + }) + }; + for len in 0..2_000u64 { + let before = ns.change_epoch(a).unwrap(); + { + let mut object = ns.object_mut(file).unwrap(); + ns.commit_file_len(file, &mut object, len); + } + // Either side settles it; once both have let go, the change is in. + let mut settled = false; + for _ in 0..1_000_000 { + if ns.change_epoch(a).unwrap() != before { + settled = true; + break; + } + std::thread::yield_now(); + } + assert!(settled, "write {len} was never passed on"); + } + stop.store(true, Ordering::Relaxed); + holder.join().unwrap(); + } + /// Write `value` as decimal ASCII into `buf`, returning the byte count. /// Test-local so name generation needs no formatting machinery. fn write_index(buf: &mut [u8; 8], value: usize) -> usize { diff --git a/native/services/dagfs/src/router.rs b/native/services/dagfs/src/router.rs index 8540f77..89c7e10 100644 --- a/native/services/dagfs/src/router.rs +++ b/native/services/dagfs/src/router.rs @@ -58,6 +58,10 @@ pub(crate) enum RoutedRequest { /// A storage-ring verb: neither engine's; served by the row that owns /// the controller (`res/STORAGE_TRANSPORT.md` §4). Stor(FsRequest), + /// Arm a watch on a directory of one store. The request's handle is + /// native to that store; the plane it publishes into is the pair's one, + /// which the primary frontend holds. + Watch(Store, FsRequest), } #[inline] @@ -435,6 +439,13 @@ pub(crate) fn route(request: FsRequest, has_system: bool) -> Result { + let (store, handle) = split_handle(handle); + RoutedRequest::Watch(store, Watch { handle, slot, tag }) + } + // Disarming names no handle: the pair's one plane, in the primary + // frontend, holds every slot, whichever store the slot watched. + Unwatch { .. } => RoutedRequest::One(Store::User, request), LendVersion { handle, window_offset, @@ -541,7 +552,11 @@ pub(crate) fn route(request: FsRequest, has_system: bool) -> Result Option<&'static (dyn core::alloc::Allocator + Sync)> { Some(crate::service_alloc()) } + + /// A recyclable peer's pair carries its watch plane after the rings. + fn attach_peer_aux(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + self.attach_watch_plane(peer, aux) + } + + fn peer_detaching(&mut self, peer: PeerId) { + self.detach_watch_plane(peer); + } + + /// Publish what every armed watch saw since the last unit of work. A lone + /// engine has no system volume, so nothing can be watched on one. + fn take_owed_wakes(&mut self, owed: &mut [Option]) -> usize { + self.settle_watches(|_, _, _| Err(FsError::AccessLost)); + self.owed_watch_wakes(owed) + } } impl FsService

{ @@ -429,6 +445,19 @@ impl FsService

{ Err(error) => FsResponse::Refused(error), }; } + FsRequest::Watch { handle, slot, tag } => { + let target = self.watch_target(session, handle); + self.arm_watch( + sv.via(), + session, + slot, + tag, + crate::router::Store::User, + handle, + target, + ) + } + FsRequest::Unwatch { slot } => self.disarm_watch(sv.via(), session, slot), FsRequest::MatchView { handle, announced } => { let Some(mut input) = sv.returned_input(announced) else { return FsResponse::Refused(FsError::AccessDenied); @@ -501,6 +530,14 @@ impl SvcProtocol for RamService { const REPLY_GRANTS: usize = ::REPLY_GRANTS; const SERVE_PAIRS: usize = ::SERVE_PAIRS; + fn init_pair_window( + window: &mut [u8], + req_cap: u32, + rep_cap: u32, + ) -> Result<(), kore_ring::spsc::RingError> { + DagfsProtocol::init_pair_window(window, req_cap, rep_cap) + } + fn reply_matches(request: &FsReqWire, response: &FsResponse, grants: u8) -> bool { DagfsProtocol::reply_matches(request, response, grants) } diff --git a/native/services/dagfs/src/service/mod.rs b/native/services/dagfs/src/service/mod.rs index 123f0c9..51e04a5 100644 --- a/native/services/dagfs/src/service/mod.rs +++ b/native/services/dagfs/src/service/mod.rs @@ -65,7 +65,7 @@ use kore_rt::{ use kore_svc_extract::schema::InputIdentity; use crate::{ - ServiceAlloc, + ObjectId, ServiceAlloc, authority::{GrantId, Handle, Retired, Rights, SessionId}, block_on, bounds::{FIRST_TASK_VIEW_SESSION, MAX_SESSIONS}, @@ -369,6 +369,9 @@ pub struct FsService { /// immediately before delegating a status request; the default zeros /// honestly describe the bare volatile posture every other harness runs. backend: FsBackendStatus, + /// Which slots' watch planes have a watch armed or owe a wake + /// ([`crate::watches`]); the planes live in the slots' own records. + watches: crate::watches::WatchIndex, } /// What the mounted backend reports about itself for `SystemStatus` — set @@ -484,6 +487,11 @@ struct SessionSlot { /// The session's one private provider placement. The service already /// serializes dispatch; no additional lock or namespace object. replacement: Option>, + /// The recyclable peer's watch plane and its armed watches + /// ([`crate::watches`]), adopted from the pair window's tail at admission + /// and dropped before the host unmaps it. Only the primary frontend's + /// slots carry one: a secondary engine's watches are recorded here too. + watch: Option, } impl SessionSlot { @@ -495,6 +503,7 @@ impl SessionSlot { task_view: None, deferring: false, replacement: None, + watch: None, } } } @@ -601,9 +610,151 @@ impl FsService

{ pending_reclaims: Vec::new_in(crate::service_alloc()), pending_aborts: Vec::new_in(crate::service_alloc()), backend: FsBackendStatus::default(), + watches: crate::watches::WatchIndex::new(), } } + /// Adopt `peer`'s watch plane from its pair window's tail into the slot's + /// record, claiming the record as admission does. Refused for a fixed + /// pair, a heap that cannot hold the record, or a slot whose previous + /// plane never detached — a host/service desync admission must not paper + /// over. A tail without a formatted plane admits the peer with none: the + /// plane is what watches publish into, not what the pair needs to serve, + /// so such a client's `Watch` answers `Unsupported`. + pub(crate) fn attach_watch_plane(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + let slot = peer.slot(); + if slot == 0 { + return false; + } + let Ok(state) = self.slots.claim(slot) else { + return false; + }; + if state.watch.is_some() { + return false; + } + state.watch = crate::watches::Plane::adopt(peer, aux); + true + } + + /// Drop `peer`'s watch plane before its window is unmapped. A late call + /// for an earlier occupant is inert. + pub(crate) fn detach_watch_plane(&mut self, peer: PeerId) { + let slot = peer.slot(); + if let Some(state) = self.slots.get_mut(slot) + && state.watch.as_ref().is_some_and(|plane| plane.peer() == peer) + { + state.watch = None; + self.watches.forget(slot); + } + } + + /// The plane of the pair that delivered a request (`via`) for `session`; + /// `Unsupported` for a fixed pair or a peer admitted without one. + fn watch_plane_mut(&mut self, via: u16, session: SessionId) -> FsResult<&mut crate::watches::Plane> { + self.slots + .get_mut(usize::from(session.index())) + .and_then(|state| state.watch.as_mut()) + .filter(|plane| plane.peer().arrival() == via) + .ok_or(FsError::Unsupported) + } + + /// Arm a watch on a directory of `store`, resolved by that store's engine + /// (`target`), into the plane of the pair that asked. + #[allow(clippy::too_many_arguments)] + pub(crate) fn arm_watch( + &mut self, + via: u16, + session: SessionId, + slot: u8, + tag: u32, + store: crate::router::Store, + handle: Handle, + target: FsResult<(ObjectId, u64)>, + ) -> FsResponse { + let armed = self + .watch_plane_mut(via, session) + .and_then(|plane| { + let owed = plane.arm(slot, tag, store, handle, target?)?; + Ok((plane.armed(), owed)) + }); + match armed { + Ok((armed, owed)) => { + self.watches.note(usize::from(session.index()), armed, owed); + FsResponse::Done + } + Err(error) => FsResponse::Refused(error), + } + } + + pub(crate) fn disarm_watch(&mut self, via: u16, session: SessionId, slot: u8) -> FsResponse { + let disarmed = self.watch_plane_mut(via, session).and_then(|plane| { + let owed = plane.disarm(slot)?; + Ok((plane.armed(), owed)) + }); + match disarmed { + Ok((armed, owed)) => { + self.watches.note(usize::from(session.index()), armed, owed); + FsResponse::Done + } + Err(error) => FsResponse::Refused(error), + } + } + + /// The change epoch a watch on `handle` follows, read through this + /// engine — the secondary half of a dual-router settle. + pub(crate) fn watch_epoch(&self, session: SessionId, handle: Handle, dir: ObjectId) -> FsResult { + self.engine.watch_epoch(session, handle, dir) + } + + /// What a watch on `handle` would follow in this engine. + pub(crate) fn watch_target(&self, session: SessionId, handle: Handle) -> FsResult<(ObjectId, u64)> { + self.engine.watch_target(session, handle) + } + + /// Publish every armed watch whose directory changed. This engine answers + /// for the user store; `system` for the dual router's system volume (a + /// lone engine passes one that refuses, which ends any such watch). + pub(crate) fn settle_watches( + &mut self, + mut system: impl FnMut(SessionId, Handle, ObjectId) -> FsResult, + ) { + if self.watches.idle() { + return; + } + let engine = &self.engine; + for slot in self.watches.armed() { + let Some(plane) = self.slots.get_mut(slot).and_then(|state| state.watch.as_mut()) else { + self.watches.forget(slot); + continue; + }; + let owed = plane.settle(SessionId::new(slot as u16), |store, session, handle, dir| { + match store { + crate::router::Store::User => engine.watch_epoch(session, handle, dir), + crate::router::Store::System => system(session, handle, dir), + } + }); + self.watches.note(slot, plane.armed(), owed); + } + } + + /// Peers owed a watch-plane wake, for the serving host: up to + /// `out.len()`, each handed over once. + pub(crate) fn owed_watch_wakes(&mut self, out: &mut [Option]) -> usize { + let slots = &self.slots; + let mut taken = 0; + self.watches.take_owed(|slot| { + if taken == out.len() { + return false; + } + if let Some(plane) = slots.get(slot).and_then(|state| state.watch.as_ref()) { + out[taken] = Some(plane.peer()); + taken += 1; + } + true + }); + taken + } + /// Replace the backend posture snapshot folded into `SystemStatus` /// answers. The mount owner calls this immediately before delegating a /// status request, so the record reads as one moment. @@ -1140,6 +1291,7 @@ impl FsService

{ if let Some(state) = self.slots.get_mut(slot) { *state = SessionSlot::vacant(); } + self.watches.forget(slot); true } @@ -2466,6 +2618,13 @@ impl FsService

{ self.engine.sync(session, handle).await?; FsResponse::Done } + // A watch publishes into the asking pair's plane, which only the + // pair-aware layer above names (`serve_request_async`, or the dual + // router); reaching the session-only dispatch is a composition + // that forwarded the verb past its owner. + FsRequest::Watch { .. } | FsRequest::Unwatch { .. } => { + return Err(FsError::Unsupported); + } FsRequest::LendVersion { handle, window_offset, @@ -3659,6 +3818,233 @@ mod recyclable_session_tests { assert_eq!(native.stable_object, ls.stable_object); } + /// A recyclable peer's watch plane as exec formats it after the rings, + /// leaked like the host's mapping of it, with the client's half beside + /// the service's. + fn watch_plane() -> ( + &'static mut [u8], + kore_dagfs::watch::WatchSink<'static>, + ) { + #[repr(align(64))] + struct Plane([u8; kore_dagfs::watch::PLANE_BYTES]); + let plane = &mut alloc::boxed::Box::leak(alloc::boxed::Box::new(Plane( + [0; kore_dagfs::watch::PLANE_BYTES], + ))) + .0; + kore_ring::event::init::( + plane, + kore_abi::svc::FS_WATCH_RECORDS, + ) + .unwrap(); + let (base, len) = (plane.as_mut_ptr(), plane.len()); + // SAFETY: the two halves alias the plane only through the channel + // protocol, as the client's and the service's mappings do on metal. + let client = unsafe { core::slice::from_raw_parts_mut(base, len) }; + (plane, kore_dagfs::watch::sink(client).unwrap()) + } + + /// Two peers of one task view: a watcher holding its root, and a writer + /// that changes it. + fn watched_view() -> (RamService, SessionId, SessionId) { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + service + .register_private_invocation_view( + 0xD0, + &Name::new("terminal-0").unwrap(), + Rights::ALL, + None, + ) + .unwrap(); + let watcher = admit_view(&mut service, 0x106, 0xD0, Rights::ALL); + let writer = admit_view(&mut service, 0x105, 0xD0, Rights::ALL); + (service, watcher, writer) + } + + fn settle(service: &mut RamService, owed: &mut [Option]) -> usize { + ::take_owed_wakes(service, owed) + } + + #[test] + fn a_watch_counts_another_peer_s_change_and_owes_a_parked_client_one_wake() { + use kore_dagfs::watch::{Cursor, Seen}; + let (mut service, watcher, writer) = watched_view(); + let peer = PeerId::from_arrival(0x106).unwrap(); + let (plane, sink) = watch_plane(); + assert!(service.attach_watch_plane(peer, plane)); + let root = service.engine.open_root(watcher, Rights::READ).unwrap(); + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 0, 7, crate::router::Store::User, root.handle, target), + FsResponse::Done + ); + let mut cursor = Cursor::new(0, 7).unwrap(); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Same); + + // Nothing changed: nothing published, nobody owed. + let mut owed = [None; 4]; + assert_eq!(settle(&mut service, &mut owed), 0); + + // The client parks; the other peer makes a file in the shared view. + let seen = sink.seq(); + assert!(sink.park(seen)); + let writer_root = service.engine.open_root(writer, Rights::ALL).unwrap(); + crate::block_on(service.engine.create( + writer, + writer_root.handle, + &Name::new("new.txt").unwrap(), + ObjectKind::File, + )) + .unwrap(); + assert_eq!(settle(&mut service, &mut owed), 1); + assert_eq!(owed[0], Some(peer)); + assert_eq!(settle(&mut service, &mut owed), 0, "one change, one wake"); + assert_ne!(sink.seq(), seen); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Changed); + + // A running client is owed nothing for the next change; it reads it. + let created = crate::block_on(service.engine.create( + writer, + writer_root.handle, + &Name::new("more.txt").unwrap(), + ObjectKind::File, + )) + .unwrap(); + assert_eq!(settle(&mut service, &mut owed), 0); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Changed); + + // A child's bytes changing is a change to what the folder shows. + crate::block_on(service.engine.write_at( + writer, + created.handle, + 0, + b"bytes", + crate::ANY_REVISION, + )) + .unwrap(); + assert_eq!(settle(&mut service, &mut owed), 0); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Changed); + + // Closing the watched handle ends the watch: the plane stops + // counting for a route the client let go of. + service.engine.close(watcher, root.handle).unwrap(); + assert_eq!(settle(&mut service, &mut owed), 0); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Ended); + assert_eq!(service.watches.armed_count(), 0); + } + + /// A plane lives in its serving slot's own record, so a peer far past + /// the host's bootstrap table — the seventh program reading files, and + /// the two-hundredth — watches as the first does. + #[test] + fn a_peer_in_a_high_slot_has_its_own_plane() { + use kore_dagfs::watch::{Cursor, Seen}; + let (mut service, _, writer) = watched_view(); + let high = admit_view(&mut service, 0x1C8, 0xD0, Rights::ALL); + assert_eq!(high, SessionId::new(0xC8)); + let peer = PeerId::from_arrival(0x1C8).unwrap(); + let (plane, sink) = watch_plane(); + assert!(service.attach_watch_plane(peer, plane)); + let root = service.engine.open_root(high, Rights::READ).unwrap(); + let target = service.watch_target(high, root.handle); + assert_eq!( + service.arm_watch(0x1C8, high, 3, 11, crate::router::Store::User, root.handle, target), + FsResponse::Done + ); + let mut cursor = Cursor::new(3, 11).unwrap(); + let writer_root = service.engine.open_root(writer, Rights::ALL).unwrap(); + crate::block_on(service.engine.create( + writer, + writer_root.handle, + &Name::new("far.txt").unwrap(), + ObjectKind::File, + )) + .unwrap(); + let mut owed = [None; 4]; + settle(&mut service, &mut owed); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Changed); + // Closing the peer wipes the slot, plane and index together. + service.detach_watch_plane(peer); + assert!(service.peer_closed(0x1C8, PeerClose::Orderly)); + assert_eq!(service.watches.armed_count(), 0); + } + + #[test] + fn a_watch_takes_a_listing_s_authority_and_a_plane_of_the_asking_pair() { + let (mut service, watcher, _) = watched_view(); + let peer = PeerId::from_arrival(0x106).unwrap(); + let root = service.engine.open_root(watcher, Rights::ALL).unwrap(); + let file = crate::block_on(service.engine.create( + watcher, + root.handle, + &Name::new("f").unwrap(), + ObjectKind::File, + )) + .unwrap(); + let user = crate::router::Store::User; + + // No plane attached yet: nothing to publish into. + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 0, 1, user, root.handle, target), + FsResponse::Refused(FsError::Unsupported) + ); + let (plane, sink) = watch_plane(); + assert!(service.attach_watch_plane(peer, plane)); + // A second plane for the occupied slot is refused. + let (again, _) = watch_plane(); + assert!(!service.attach_watch_plane(peer, again)); + // A tail that carries no plane admits its peer without one. + let bare: &'static mut [u8] = alloc::boxed::Box::leak(alloc::boxed::Box::new([0u8; 64])); + assert!(service.attach_watch_plane(PeerId::from_arrival(0x105).unwrap(), bare)); + // Another generation of the slot names no plane of its own. + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x206, watcher, 0, 1, user, root.handle, target), + FsResponse::Refused(FsError::Unsupported) + ); + // A file is not watched: a listing is what a watch follows. + let target = service.watch_target(watcher, file.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 0, 1, user, file.handle, target), + FsResponse::Refused(FsError::NotDirectory) + ); + // A slot past the plane, or the zero tag, is out of range. + for (slot, tag) in [(kore_abi::svc::FS_WATCH_SLOTS as u8, 1), (0, 0)] { + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, slot, tag, user, root.handle, target), + FsResponse::Refused(FsError::Range) + ); + } + assert_eq!(service.watches.armed_count(), 0); + + // Re-arming a slot replaces its watch; disarming clears its tag. + for tag in [3, 4] { + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 1, tag, user, root.handle, target), + FsResponse::Done + ); + } + assert_eq!(service.watches.armed_count(), 1); + assert_eq!(sink.state().unwrap().tag[1], 4); + assert_eq!(service.disarm_watch(0x106, watcher, 1), FsResponse::Done); + assert_eq!(sink.state().unwrap().tag[1], 0); + assert_eq!(service.watches.armed_count(), 0); + + // Detaching the plane before the window goes drops its watches. + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 2, 9, user, root.handle, target), + FsResponse::Done + ); + service.detach_watch_plane(PeerId::from_arrival(0x206).unwrap()); + assert_eq!(service.watches.armed_count(), 1, "a late detach for another generation is inert"); + service.detach_watch_plane(peer); + assert_eq!(service.watches.armed_count(), 0); + } + /// A dead writer's staged replacement is released on the service's next /// dispatch, never inside the synchronous peer-death hook. A release the /// provider refuses stays pending — custody retained — while the dead diff --git a/native/services/dagfs/src/watches.rs b/native/services/dagfs/src/watches.rs new file mode 100644 index 0000000..c7800aa --- /dev/null +++ b/native/services/dagfs/src/watches.rs @@ -0,0 +1,310 @@ +//! Directory watches: the service half of the watch plane +//! (`kore_dagfs::watch`, `res/DAGFS.md` §4). +//! +//! A session arms up to [`FS_WATCH_SLOTS`] watches, each on a directory +//! handle it holds. After every unit of service work each armed directory's +//! change epoch ([`Namespace::change_epoch`]) is read **through the handle's +//! route** — revalidated as every operation's is — and the plane republished +//! when one moved. Only a client parked on its plane is owed a wake; the +//! serving host sends each one notice (`SystemService::take_owed_wakes`). +//! +//! Nothing here is global. A peer's [`Plane`] lives in its serving slot's own +//! record, beside the slot's window and staged replacement, allocated with it +//! and wiped with it; the service keeps only [`WatchIndex`] — which slots have +//! a watch armed, and which owe a wake — so a settle visits exactly the slots +//! that need it, however large the slot space grows. A watch on the dual +//! router's system volume is recorded in the same plane with the store it +//! lives in, so the one plane a client holds carries both. +//! +//! [`Namespace::change_epoch`]: crate::namespace::Namespace::change_epoch + +use kore_abi::svc::{FS_WATCH_SLOTS, FsWatchState}; +use kore_dagfs::watch::{self, WatchSource, state}; +use kore_rt::svc::PeerId; + +use crate::{ + ObjectId, + authority::{Handle, SessionId}, + bounds::PAIR_SESSIONS, + error::{FsError, FsResult}, + router::Store, +}; + +/// One armed slot. +#[derive(Clone, Copy, Debug)] +struct Watch { + store: Store, + handle: Handle, + dir: ObjectId, + /// The epoch the plane last reported for this watch. + seen: u64, +} + +/// One recyclable peer's watch plane and the watches published into it. +pub(crate) struct Plane { + peer: PeerId, + source: WatchSource<'static>, + slots: [Option; FS_WATCH_SLOTS], + /// The plane's state changed since its last commit. + dirty: bool, +} + +impl Plane { + /// `peer`'s plane over its pair window's tail (`aux`, as the host hands it + /// to `attach_peer_aux`), or `None` for a tail without a formatted plane. + /// Starts from nothing armed, whatever the window last carried. + pub(crate) fn adopt(peer: PeerId, aux: &'static mut [u8]) -> Option { + let mut source = watch::source(aux).ok()?; + if source.replace_state(FsWatchState::default()) { + let _ = source.commit(); + } + Some(Self { + peer, + source, + slots: [None; FS_WATCH_SLOTS], + dirty: false, + }) + } + + pub(crate) const fn peer(&self) -> PeerId { + self.peer + } + + /// Whether any slot is armed. + pub(crate) fn armed(&self) -> bool { + self.slots.iter().any(Option::is_some) + } + + fn edit(&mut self, edit: impl FnOnce(&mut FsWatchState)) { + self.source.state(edit); + self.dirty = true; + } + + /// Publish a batch: `true` when its commit found the client parked, and + /// so owes it one notice. + fn commit(&mut self) -> bool { + core::mem::take(&mut self.dirty) && self.source.commit() + } + + /// Arm `slot` under `tag` on the directory `dir` at epoch `seen` (its + /// baseline: only a later change counts), replacing what the slot held. + /// Answers whether a parked client is owed a wake. + pub(crate) fn arm( + &mut self, + slot: u8, + tag: u32, + store: Store, + handle: Handle, + (dir, seen): (ObjectId, u64), + ) -> FsResult { + let slot = usize::from(slot); + if slot >= FS_WATCH_SLOTS || tag == 0 { + return Err(FsError::Range); + } + self.slots[slot] = Some(Watch { + store, + handle, + dir, + seen, + }); + self.edit(|s| state::arm(s, slot, tag)); + Ok(self.commit()) + } + + /// Disarm `slot`, armed or not. Answers whether a parked client is owed + /// a wake. + pub(crate) fn disarm(&mut self, slot: u8) -> FsResult { + let slot = usize::from(slot); + if slot >= FS_WATCH_SLOTS { + return Err(FsError::Range); + } + self.slots[slot] = None; + self.edit(|s| state::disarm(s, slot)); + Ok(self.commit()) + } + + /// Read every armed watch's epoch through `epoch` — the engine of its + /// store, revalidating its route — and publish what moved. A watch whose + /// route no longer proves ends; one whose directory was mid-publish + /// (`Busy`) counts as changed and keeps its baseline, so the next reading + /// reports the change again rather than losing it. Answers whether a + /// parked client is owed a wake. + pub(crate) fn settle( + &mut self, + session: SessionId, + mut epoch: impl FnMut(Store, SessionId, Handle, ObjectId) -> FsResult, + ) -> bool { + for slot in 0..FS_WATCH_SLOTS { + let Some(watch) = self.slots[slot].as_mut() else { + continue; + }; + match epoch(watch.store, session, watch.handle, watch.dir) { + Ok(now) if now == watch.seen => {} + Ok(now) => { + watch.seen = now; + self.edit(|s| state::changed(s, slot)); + } + Err(FsError::Busy) => self.edit(|s| state::changed(s, slot)), + Err(_) => { + self.slots[slot] = None; + self.edit(|s| state::end(s, slot)); + } + } + } + self.commit() + } +} + +/// Words of a set over every pair session's slot. +const WORDS: usize = PAIR_SESSIONS.div_ceil(64); + +/// A set of pair-session slots: one bit each, scanned word by word. +#[derive(Clone, Copy, Debug, Default)] +struct SlotSet([u64; WORDS]); + +impl SlotSet { + fn insert(&mut self, slot: usize) { + if let Some(word) = self.0.get_mut(slot / 64) { + *word |= 1 << (slot % 64); + } + } + + fn remove(&mut self, slot: usize) { + if let Some(word) = self.0.get_mut(slot / 64) { + *word &= !(1 << (slot % 64)); + } + } + + fn is_empty(&self) -> bool { + self.0.iter().all(|word| *word == 0) + } + + /// The members, lowest first, as of the call. + fn members(&self) -> impl Iterator + use<> { + let words = self.0; + (0..WORDS).flat_map(move |at| { + let mut word = words[at]; + core::iter::from_fn(move || { + (word != 0).then(|| { + let bit = word.trailing_zeros() as usize; + word &= word - 1; + at * 64 + bit + }) + }) + }) + } +} + +/// Which slots' planes have a watch armed, and which owe their client a wake. +/// Derived from the planes and kept beside them: a settle and a wake +/// collection cost the slots they name, and nothing when the sets are empty. +#[derive(Debug, Default)] +pub(crate) struct WatchIndex { + armed: SlotSet, + owed: SlotSet, +} + +impl WatchIndex { + pub(crate) const fn new() -> Self { + Self { + armed: SlotSet([0; WORDS]), + owed: SlotSet([0; WORDS]), + } + } + + /// Record what an edit of `slot`'s plane left: whether a watch is still + /// armed there, and whether its client is owed a wake. + pub(crate) fn note(&mut self, slot: usize, armed: bool, owed: bool) { + if armed { + self.armed.insert(slot); + } else { + self.armed.remove(slot); + } + if owed { + self.owed.insert(slot); + } + } + + /// Forget `slot`: its plane is gone. + pub(crate) fn forget(&mut self, slot: usize) { + self.armed.remove(slot); + self.owed.remove(slot); + } + + /// The slots with a watch armed. + pub(crate) fn armed(&self) -> impl Iterator + use<> { + self.armed.members() + } + + pub(crate) fn idle(&self) -> bool { + self.armed.is_empty() + } + + /// Hand `out` each slot owing a wake, lowest first, taking it off the set; + /// `out` answers `false` when its batch is full, and the rest stay owed. + pub(crate) fn take_owed(&mut self, mut out: impl FnMut(usize) -> bool) { + for slot in self.owed.members() { + if !out(slot) { + return; + } + self.owed.remove(slot); + } + } + + #[cfg(test)] + pub(crate) fn armed_count(&self) -> usize { + self.armed.members().count() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_slot_set_names_its_members_lowest_first_across_words() { + let mut set = SlotSet::default(); + for slot in [PAIR_SESSIONS - 1, 3, 64, 65, 0] { + set.insert(slot); + } + set.insert(PAIR_SESSIONS); // past the space: ignored + let members: std::vec::Vec = set.members().collect(); + assert_eq!(members, [0, 3, 64, 65, PAIR_SESSIONS - 1]); + set.remove(64); + set.remove(0); + assert_eq!(set.members().collect::>(), [3, 65, PAIR_SESSIONS - 1]); + assert!(!set.is_empty()); + for slot in [3, 65, PAIR_SESSIONS - 1] { + set.remove(slot); + } + assert!(set.is_empty()); + } + + #[test] + fn owed_wakes_are_taken_once_and_a_full_batch_keeps_the_rest() { + let mut index = WatchIndex::new(); + for slot in [2, 7, 9] { + index.note(slot, true, true); + } + let mut taken = std::vec::Vec::new(); + index.take_owed(|slot| { + if taken.len() == 2 { + return false; + } + taken.push(slot); + true + }); + assert_eq!(taken, [2, 7]); + taken.clear(); + index.take_owed(|slot| { + taken.push(slot); + true + }); + assert_eq!(taken, [9], "the batch's leftover, and nothing twice"); + index.forget(9); + assert_eq!(index.armed().collect::>(), [2, 7]); + index.note(2, false, false); + index.note(7, false, false); + assert!(index.idle()); + } +} diff --git a/native/services/device/boot.toml b/native/services/device/boot.toml index 021ef5d..96c7f02 100644 --- a/native/services/device/boot.toml +++ b/native/services/device/boot.toml @@ -47,5 +47,5 @@ protocol = "kore_svc_device::DeviceBroker" # row's own claimant band. instances = 8 endows = ["ROOT_COMPLEX@0", "PLATFORM_BUS@0", "FACE_SUPPLY@0"] -digest_x86_64 = "blake3:78d5b044c7b3ad3467484859549f8404cffc61ece27dd793cdd21c8c2c0a930a" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:5632f4509b69316c1cac719cd0ddbf5a4ac0239db6fb42488ca6ec9811de4382" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:5a3ff2252bef814e79727f873c89de62565ab491f8065599ada4db348c829223" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:f00fd8b3aad6308541c51f785e3651dc793b60652a821ffee9aa7d79b49eb615" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/mm/boot.toml b/native/services/mm/boot.toml index eee0bbd..b6412e0 100644 --- a/native/services/mm/boot.toml +++ b/native/services/mm/boot.toml @@ -54,5 +54,5 @@ supply_quota_bytes = 27917287424 # 26 GiB endows = ["MEM_SUPPLY@0", "MEM_REGION@0", "SYSTEM_OBSERVE@0"] # review mirror; asserted === PROFILE.roles shell_client = true exec_client = true -digest_x86_64 = "blake3:d1ce3128c73f02293e323f87d79850b8558dfd82d882b65169767febb7baf6e4" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:901b68a3acc55e7424ee98c90758fa17d5675f50292a1bc8e277591b2f0f5803" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:041a33515a21fff195c4cab320e9c8e64cbf775f04f01b7dbe40fa90be5a31a7" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:e17d59e57476b1db9554bc889640ffd3f02b6e2c98d509d93dea1c7efadb2a2a" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/swap/boot.toml b/native/services/swap/boot.toml index 3c9cab3..0a1028a 100644 --- a/native/services/swap/boot.toml +++ b/native/services/swap/boot.toml @@ -9,5 +9,5 @@ image = "kore-svc-swap" endowment = "kore_svc_swap::ZramEndowment" protocol = "kore_svc_swap::ZramSwap" endows = ["MEM_SUPPLY@0"] -digest_x86_64 = "blake3:dddc0488ba218b48a691a1e10e5995e962296056daf64ef0408ab15ecbd3a67f" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:b18a607ce1dec8f172effb4ae331e3be712ee99c6d3951dd33e907c5491e5cba" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:bc3ff6fbb123b3b44645e959346e320519b9ab5f43b5c999d0685ee9c7fed141" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:6495b930db0d4e4ca6ce41b0d19580ea1d816748358cc9c8bceef150ec0ccc30" # res/aarch64/; make boot-digests-aarch64 From 2ce41f10de75dcd30bf4fcf4f85fb8cd4f9fceb5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 01:39:08 +0000 Subject: [PATCH 16/18] c: a C program's main receives its arguments `main` ran with `argc` zero: nothing read the launcher's page, and the page held only the tail of the command line, never the program's name. Kore's page now carries both. The host reads them once, on thread 0 before the main thread exists (`args::capture`), and splits the tail as every Kore program that wants words does: a POSIX shell's quoting without expansion (`kore_rt::process::words`). The C library's start asks for the vector through a third lifecycle hook, `__kore_program_arguments(&argv)`, and calls `main(argc, argv, envp)`: - `argv[0]` is the command name, or the empty string when the launcher gave none; `argv[argc]` is null; the strings are the program's to change, and live as long as it does. - A line that does not split (an unclosed quote, a lone trailing backslash, a NUL inside a word) is not handed to `main`. The host says why on `stderr` and ends the program with status 5 before it runs, as `wasm` and `tcc` answer the same line. - A launch that carried no page, or no host, runs with `argc` zero. The hook is the C library's, not an application API: cbindgen leaves it out of `kore/program.h`, and `check-program.py` expects it among the lifecycle symbols (130 exported). Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- crates/c/cbindgen.toml | 2 +- crates/c/src/args.rs | 247 ++++++++++++++++++++++++++++++++++++++++ crates/c/src/lib.rs | 5 + crates/c/src/program.rs | 14 +++ crates/c/src/stdio.rs | 9 +- docs/languages.md | 18 ++- tools/check-program.py | 2 +- 7 files changed, 291 insertions(+), 6 deletions(-) create mode 100644 crates/c/src/args.rs diff --git a/crates/c/cbindgen.toml b/crates/c/cbindgen.toml index e241550..0c770e4 100644 --- a/crates/c/cbindgen.toml +++ b/crates/c/cbindgen.toml @@ -17,7 +17,7 @@ parse_deps = false # this header includes. # `open`, `close`, `mmap` and `munmap` are POSIX's, declared by , # and . -exclude = ["__kore_program_start", "__kore_program_exit", "abort", "open", "close", "mmap", "munmap", "KoreFsStep"] +exclude = ["__kore_program_start", "__kore_program_exit", "__kore_program_arguments", "abort", "open", "close", "mmap", "munmap", "KoreFsStep"] [enum] prefix_with_name = true rename_variants = "ScreamingSnakeCase" diff --git a/crates/c/src/args.rs b/crates/c/src/args.rs new file mode 100644 index 0000000..693bf2e --- /dev/null +++ b/crates/c/src/args.rs @@ -0,0 +1,247 @@ +//! **A C program's arguments**: `main`'s `argc` and `argv`. +//! +//! A launcher hands a program its command line on the process page: the name +//! it resolved the program by ([`ProcessIo::command`]) and the opaque tail +//! after it ([`ProcessIo::args`]). C wants a vector. The host reads the page +//! once, on thread 0 before the main thread exists, splits the tail the way +//! every kore program that wants words does ([`words`]: a POSIX shell's +//! quoting, without expansion), and keeps the vector for the life of the +//! program. The C library's start asks for it through +//! [`__kore_program_arguments`] and passes it to `main`. +//! +//! `argv[0]` is the command name, or the empty string when the launcher gave +//! none (C11 §5.1.2.2.1p2). A launch with no page at all has no arguments, +//! not even a name: `argc` is zero, which the same clause permits. The +//! strings are the program's to change, as that clause requires, and +//! `argv[argc]` is null. +//! +//! A tail that does not split, an unclosed quote or a lone trailing +//! backslash, is the launcher's line being malformed, not something `main` +//! could be handed: the host says so on `stderr` and ends the program before +//! it runs, as `wasm` and `tcc` answer the same line. +//! +//! [`ProcessIo::command`]: kore_std::abi::process::ProcessIo::command +//! [`ProcessIo::args`]: kore_std::abi::process::ProcessIo::args +//! [`words`]: kore_std::process::words + +extern crate alloc; + +use alloc::vec::Vec; + +use kore_std::process::words::{self, WordsError}; + +/// An argument vector's storage: every word NUL-terminated, back to back, +/// and where each begins. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Vector { + bytes: Vec, + starts: Vec, +} + +impl Vector { + /// `command`, then the words of `tail`. + /// + /// A NUL inside a word would end the C string early and hand `main` a + /// different argument than the line spelled, so such a line is no + /// vector either. + pub(crate) fn of(command: &[u8], tail: &[u8]) -> Result { + let mut vector = Self { + bytes: Vec::new(), + starts: Vec::new(), + }; + vector.push(command.iter().copied())?; + for word in words::words(tail) { + vector.push(word.map_err(Malformed::Words)?.bytes())?; + } + Ok(vector) + } + + fn push(&mut self, word: impl Iterator) -> Result<(), Malformed> { + self.starts.push(self.bytes.len()); + for byte in word { + if byte == 0 { + return Err(Malformed::Nul { + word: self.starts.len() - 1, + }); + } + self.bytes.push(byte); + } + self.bytes.push(0); + Ok(()) + } + + /// Words held: `argc`. + pub(crate) fn len(&self) -> usize { + self.starts.len() + } + + /// Word `index`, without its terminator. + #[cfg(test)] + fn word(&self, index: usize) -> &[u8] { + let start = self.starts[index]; + let end = self.bytes[start..] + .iter() + .position(|byte| *byte == 0) + .unwrap(); + &self.bytes[start..start + end] + } +} + +/// Why a command line is no argument vector. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Malformed { + /// The tail does not split into words. + Words(WordsError), + /// Word `word` (0 is the command name) holds a NUL byte. + Nul { word: usize }, +} + +impl core::fmt::Display for Malformed { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Words(error) => error.fmt(f), + Self::Nul { word } => write!(f, "argument {word} holds a NUL byte"), + } + } +} + +#[cfg(target_os = "none")] +pub(crate) use metal::capture; + +#[cfg(target_os = "none")] +mod metal { + use super::alloc::boxed::Box; + use super::alloc::vec::Vec; + use super::{Malformed, Vector}; + use core::ffi::{c_char, c_int}; + use core::sync::atomic::{AtomicI32, AtomicPtr, Ordering}; + use kore_std::runtime::entry::Rt; + + /// `argc`, and the vector `argv` points at: `argc` pointers and a null. + /// Published once by [`capture`] on thread 0 before the main thread is + /// founded, whose founding orders these stores before `main`'s loads. + static ARGC: AtomicI32 = AtomicI32::new(0); + static ARGV: AtomicPtr<*mut c_char> = AtomicPtr::new(core::ptr::null_mut()); + + /// Read the launcher's page and keep the program's argument vector. + /// Thread 0, once, before any other thread exists. A launch that carried + /// no page leaves the program without arguments. + pub(crate) fn capture(rt: &Rt) -> Result<(), Malformed> { + let Some(page) = kore_std::process::map(rt) else { + return Ok(()); + }; + // A page that does not validate is no command line: no arguments, + // as for a launch without one. + let (Ok(command), Ok(tail)) = (page.command(), page.args()) else { + return Ok(()); + }; + let Vector { bytes, starts } = Vector::of(command, tail)?; + // The program's for its whole life, and its to change: never freed. + let bytes: &'static mut [u8] = Box::leak(bytes.into_boxed_slice()); + let base = bytes.as_mut_ptr(); + let mut pointers: Vec<*mut c_char> = starts + .iter() + // SAFETY: each start is an offset inside `bytes`. + .map(|start| unsafe { base.add(*start) }.cast::()) + .collect(); + pointers.push(core::ptr::null_mut()); + let argc = c_int::try_from(starts.len()).unwrap_or(c_int::MAX); + let pointers: &'static mut [*mut c_char] = Box::leak(pointers.into_boxed_slice()); + ARGV.store(pointers.as_mut_ptr(), Ordering::Release); + ARGC.store(argc, Ordering::Release); + Ok(()) + } + + /// The program's argument vector: answers `argc` and stores `argv` + /// through `argv`, `argv[argc]` null. The C library's start calls it on + /// the main thread before `main`. A program whose launch carried no + /// command line gets zero and a vector holding only the null. + /// + /// # Safety + /// `argv` is null or names one writable `char **`. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn __kore_program_arguments(argv: *mut *mut *mut c_char) -> c_int { + /// The vector of a program without arguments: its null. + struct Empty(core::cell::UnsafeCell<[*mut c_char; 1]>); + // SAFETY: never written: C may not store through `argv[argc]`'s + // slot, and there is no other. + unsafe impl Sync for Empty {} + static EMPTY: Empty = Empty(core::cell::UnsafeCell::new([core::ptr::null_mut()])); + + let held = ARGV.load(Ordering::Acquire); + let (count, vector) = if held.is_null() { + (0, EMPTY.0.get().cast::<*mut c_char>()) + } else { + (ARGC.load(Ordering::Acquire), held) + }; + if !argv.is_null() { + // SAFETY: the caller's contract. + unsafe { argv.write(vector) }; + } + count + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn vector(command: &[u8], tail: &[u8]) -> Vec> { + let vector = Vector::of(command, tail).unwrap(); + (0..vector.len()) + .map(|index| vector.word(index).to_vec()) + .collect() + } + + #[test] + fn the_command_is_the_first_word_and_the_tail_follows_as_a_shell_splits_it() { + assert_eq!(vector(b"answer", b""), [b"answer".to_vec()]); + assert_eq!( + vector(b"copy", b" a.txt b.txt "), + [b"copy".to_vec(), b"a.txt".to_vec(), b"b.txt".to_vec()] + ); + assert_eq!( + vector(b"say", br#"-c 'two words' "a \"quote\"" back\ slash """#), + [ + b"say".to_vec(), + b"-c".to_vec(), + b"two words".to_vec(), + br#"a "quote""#.to_vec(), + b"back slash".to_vec(), + b"".to_vec(), + ] + ); + // A launcher that names nothing still gives `argv[0]`, empty. + assert_eq!(vector(b"", b"x"), [b"".to_vec(), b"x".to_vec()]); + } + + #[test] + fn every_word_is_its_own_terminated_string() { + let vector = Vector::of(b"p", b"ab c").unwrap(); + assert_eq!(vector.bytes, b"p\0ab\0c\0"); + assert_eq!(vector.starts, [0, 2, 5]); + assert_eq!(vector.len(), 3); + } + + #[test] + fn a_line_that_does_not_split_is_no_vector() { + assert_eq!( + Vector::of(b"p", b"'open"), + Err(Malformed::Words(WordsError::Unterminated { + quote: b'\'', + at: 0 + })) + ); + assert_eq!( + Vector::of(b"p", b"a \\"), + Err(Malformed::Words(WordsError::TrailingBackslash)) + ); + // A NUL would end the C string early: `a\0b` is not the word `a`. + assert_eq!(Vector::of(b"p", b"x a\0b"), Err(Malformed::Nul { word: 2 })); + assert_eq!(Vector::of(b"p\0q", b""), Err(Malformed::Nul { word: 0 })); + let mut text = alloc::string::String::new(); + core::fmt::Write::write_fmt(&mut text, format_args!("{}", Malformed::Nul { word: 2 })) + .unwrap(); + assert_eq!(text, "argument 2 holds a NUL byte"); + } +} diff --git a/crates/c/src/lib.rs b/crates/c/src/lib.rs index 1231e32..11eced7 100644 --- a/crates/c/src/lib.rs +++ b/crates/c/src/lib.rs @@ -43,6 +43,11 @@ use kore_ffi as _; use kore_c_shim as _; pub mod api; +// Off Kore only the host suite reads the vector's storage: the page it is +// read from and the hook that answers it are the Kore program's alone. +#[cfg(any(test, target_os = "none"))] +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +mod args; #[cfg(any(test, target_os = "none"))] mod exit; // Off Kore only `kore_fd_describe` and the host suite read these two: the diff --git a/crates/c/src/program.rs b/crates/c/src/program.rs index cc57c56..657040d 100644 --- a/crates/c/src/program.rs +++ b/crates/c/src/program.rs @@ -48,6 +48,8 @@ //! the C library alone would: allocation, threads, time and random work, a //! filesystem session is refused, and only thread 0's exit ends the program. +extern crate alloc; + use core::cell::{Cell, UnsafeCell}; use core::ffi::{c_int, c_void}; use core::future::poll_fn; @@ -370,6 +372,18 @@ impl Founded { // The C streams reach the launcher through this thread: map its // stream and install the sink before any thread can write. let output = crate::stdio::install(&rt); + // `main`'s arguments, read from the launcher's page while this is + // still the only thread. A line that is no argument vector cannot + // be handed to `main`: say why on `stderr` and end here, before the + // program runs. + if let Err(malformed) = crate::args::capture(&rt) { + crate::stdio::report(alloc::format!("kore: {malformed}\n").as_bytes()); + crate::stdio::flush_now(); + // SAFETY: the C library's exit, which `__kore_program_start` + // stored before it called this. + let end: End = unsafe { core::mem::transmute(END.load(Ordering::Acquire)) }; + unsafe { end(ExitCode::USAGE.get()) } + } // Where `mmap` places: this task's free link span, published before // any thread that could map exists. crate::mmap::arm(&rt); diff --git a/crates/c/src/stdio.rs b/crates/c/src/stdio.rs index 7b2bcbf..7e6f712 100644 --- a/crates/c/src/stdio.rs +++ b/crates/c/src/stdio.rs @@ -276,7 +276,7 @@ pub(crate) fn held_is_next(held_after: u32, head: u32) -> bool { extern crate alloc; #[cfg(target_os = "none")] -pub(crate) use metal::{Endpoint, abandon, arm, drain, finish, flush_now, install}; +pub(crate) use metal::{Endpoint, abandon, arm, drain, finish, flush_now, install, report}; #[cfg(target_os = "none")] mod metal { @@ -333,6 +333,13 @@ mod metal { Some(unsafe { (*ENDPOINT.0.get()).assume_init_ref() }) } + /// A line on `stderr` from the host itself: what it must say when the + /// program cannot run. Kept in order and relayed as any write made on + /// the host is. Nothing is said when the launch carried no stream. + pub(crate) fn report(line: &[u8]) { + let _ = sink(Standard::Error, line); + } + pub(crate) fn host_pending() -> bool { host().is_some_and(|endpoint| !endpoint.held.borrow().is_empty()) } diff --git a/docs/languages.md b/docs/languages.md index 42d764b..0086c3f 100644 --- a/docs/languages.md +++ b/docs/languages.md @@ -213,9 +213,21 @@ mapped there. The rules follow from that: end with `ENOMEM`. A program holds sixteen regions, mapped and parked together. -The C library's half is two weak references. Its start calls -`__kore_program_start(program, end)` when the program links libkore, and its -exit calls `__kore_program_exit`. A program without libkore is unchanged. On +**`main` gets its arguments.** The launcher's page carries the name it +resolved the program by and the rest of the command line. The host reads it +once, on thread 0 before the main thread exists, and splits the rest as every +Kore program that wants words does: a POSIX shell's quoting, without +expansion. `argv[0]` is the name, or the empty string when the launcher gave +none, `argv[argc]` is null, and the strings are the program's to change. A +line that does not split (an unclosed quote, a lone trailing backslash) is +not handed to `main`: the host says why on `stderr` and the program ends with +status 5 before it runs. `envp` is the empty vector, since a Kore task has no +environment. A launch that carried no page runs with `argc` zero. + +The C library's half is three weak references. Its start calls +`__kore_program_start(program, end)` when the program links libkore, asks +`__kore_program_arguments(&argv)` for the vector before it calls `main`, and +its exit calls `__kore_program_exit`. A program without libkore is unchanged. On Kore those are the start files Kore ships with TCC. `tools/check-program.py` (`make cross`, and CI per ISA) links the shared object as a distribution does and checks it: diff --git a/tools/check-program.py b/tools/check-program.py index 03a09d7..c6b0a1b 100755 --- a/tools/check-program.py +++ b/tools/check-program.py @@ -36,7 +36,7 @@ 'pthread_setcanceltype', 'pthread_testcancel', 'pthread_exit', '__cxa_thread_atexit', '__cxa_thread_atexit_impl'} # The C library's start and exit call these when a program links libkore. -LIFECYCLE = {'__kore_program_start', '__kore_program_exit'} +LIFECYCLE = {'__kore_program_start', '__kore_program_exit', '__kore_program_arguments'} # The rest of the C library a program's headers declare (Kore's TCC # `include/`: , , , ), which libkore.so # implements: kore-c-shim's printf family, streams, parsers, string routines From 2da52d63f906c39e268a7b4e13f6dfc9a61e0441 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 10:08:42 +0000 Subject: [PATCH 17/18] native: re-export at Kore cac17a02, with the virtual-device client `tools/sync-native.py` at MicroPerceptron/kore@cac17a02. The export's roots gain `kore-svc-vdev` and `kore-vdev`: the client of Kore's virtual-device service and the register block and memory views a driver shares with a device. `kore-svc-vdev` is built without its `host` feature, so no device model enters this library's graph; its optional model crates are exported as source and compiled by nothing here. `kore-std` re-exports the two as `vdev` behind a feature of that name. kore-rt at this revision lets a thread hand a capability on to the threads it founds (`thread::share_endowment`, `Controller::hand_on`), and names a deadline from a clock every target has (`clock::deadline_base_nanos`). Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- Cargo.lock | 25 + Cargo.toml | 7 +- native/Cargo.lock | 98 +- native/Cargo.toml | 4 + native/crates/abi/src/device.rs | 2 + native/crates/abi/src/device/class.rs | 153 ++ native/crates/abi/src/device/state.rs | 15 +- native/crates/abi/src/image.rs | 15 +- native/crates/abi/src/svc.rs | 82 + native/crates/abi/src/task.rs | 7 +- native/crates/ld/tests/built_images.rs | 1 + native/crates/ld/tests/shipping_list.rs | 6 +- native/crates/rt/src/clock/mod.rs | 18 + native/crates/rt/src/svc/mod.rs | 13 + native/crates/rt/src/thread.rs | 87 +- native/crates/rt/src/thread_controller.rs | 196 ++- native/crates/vdev-accel/Cargo.toml | 32 + native/crates/vdev-accel/src/device.rs | 226 +++ native/crates/vdev-accel/src/device/tests.rs | 372 +++++ native/crates/vdev-accel/src/fp8.rs | 138 ++ native/crates/vdev-accel/src/fp8/tables.rs | 225 +++ native/crates/vdev-accel/src/fp8/tests.rs | 286 ++++ native/crates/vdev-accel/src/lib.rs | 25 + native/crates/vdev-accel/tests/session.rs | 438 +++++ native/crates/vdev/Cargo.toml | 21 + native/crates/vdev/src/device.rs | 69 + native/crates/vdev/src/driver.rs | 624 +++++++ native/crates/vdev/src/driver/tests.rs | 392 +++++ native/crates/vdev/src/lib.rs | 43 + native/crates/vdev/src/mem.rs | 294 ++++ native/crates/vdev/src/regs.rs | 119 ++ native/crates/vdev/src/ring.rs | 630 +++++++ native/crates/vdev/src/ring/tests.rs | 683 ++++++++ native/crates/vdev/src/transport.rs | 460 ++++++ native/crates/vdev/src/transport/tests.rs | 1070 ++++++++++++ native/provenance.json | 73 +- native/services/accel-spmd/Cargo.toml | 18 + native/services/accel-spmd/src/backend.rs | 1563 ++++++++++++++++++ native/services/accel-spmd/src/kernel.rs | 362 ++++ native/services/accel-spmd/src/lib.rs | 86 + native/services/block/boot.toml | 4 +- native/services/dagfs/boot.toml | 4 +- native/services/device/boot.toml | 4 +- native/services/mm/boot.toml | 4 +- native/services/swap/boot.toml | 4 +- native/services/vdev/Cargo.toml | 29 + native/services/vdev/boot.toml | 30 + native/services/vdev/src/client.rs | 439 +++++ native/services/vdev/src/face.rs | 29 + native/services/vdev/src/host.rs | 259 +++ native/services/vdev/src/host/tests.rs | 397 +++++ native/services/vdev/src/lib.rs | 53 + native/services/vdev/src/service.rs | 389 +++++ native/services/vdev/src/wire.rs | 465 ++++++ src/lib.rs | 11 + 55 files changed, 11030 insertions(+), 69 deletions(-) create mode 100644 native/crates/abi/src/device/class.rs create mode 100644 native/crates/vdev-accel/Cargo.toml create mode 100644 native/crates/vdev-accel/src/device.rs create mode 100644 native/crates/vdev-accel/src/device/tests.rs create mode 100644 native/crates/vdev-accel/src/fp8.rs create mode 100644 native/crates/vdev-accel/src/fp8/tables.rs create mode 100644 native/crates/vdev-accel/src/fp8/tests.rs create mode 100644 native/crates/vdev-accel/src/lib.rs create mode 100644 native/crates/vdev-accel/tests/session.rs create mode 100644 native/crates/vdev/Cargo.toml create mode 100644 native/crates/vdev/src/device.rs create mode 100644 native/crates/vdev/src/driver.rs create mode 100644 native/crates/vdev/src/driver/tests.rs create mode 100644 native/crates/vdev/src/lib.rs create mode 100644 native/crates/vdev/src/mem.rs create mode 100644 native/crates/vdev/src/regs.rs create mode 100644 native/crates/vdev/src/ring.rs create mode 100644 native/crates/vdev/src/ring/tests.rs create mode 100644 native/crates/vdev/src/transport.rs create mode 100644 native/crates/vdev/src/transport/tests.rs create mode 100644 native/services/accel-spmd/Cargo.toml create mode 100644 native/services/accel-spmd/src/backend.rs create mode 100644 native/services/accel-spmd/src/kernel.rs create mode 100644 native/services/accel-spmd/src/lib.rs create mode 100644 native/services/vdev/Cargo.toml create mode 100644 native/services/vdev/boot.toml create mode 100644 native/services/vdev/src/client.rs create mode 100644 native/services/vdev/src/face.rs create mode 100644 native/services/vdev/src/host.rs create mode 100644 native/services/vdev/src/host/tests.rs create mode 100644 native/services/vdev/src/lib.rs create mode 100644 native/services/vdev/src/service.rs create mode 100644 native/services/vdev/src/wire.rs diff --git a/Cargo.lock b/Cargo.lock index 7e4ec99..5959e79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -976,6 +976,8 @@ dependencies = [ "kore-dagfs", "kore-driver", "kore-rt", + "kore-svc-vdev", + "kore-vdev", "zeroize", ] @@ -1028,6 +1030,23 @@ dependencies = [ "kore-rt", ] +[[package]] +name = "kore-svc-vdev" +version = "0.0.0" +dependencies = [ + "kore-abi", + "kore-rt", + "kore-vdev", +] + +[[package]] +name = "kore-vdev" +version = "0.0.1" +dependencies = [ + "heapless", + "virtio-accel-transport", +] + [[package]] name = "kore-vol" version = "0.0.0" @@ -1678,6 +1697,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtio-accel-transport" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7747200a37db419a82c884730d5c31d1d80a6b4d14cc3de312064e4793dc289" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index e6ec140..ed8b435 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ fs = ["dep:kore-dagfs", "kore-dagfs/runtime"] heap = ["kore-rt/heap"] crt0 = ["kore-rt/crt0"] driver = ["dep:kore-driver", "kore-driver/service"] +vdev = ["dep:kore-svc-vdev", "dep:kore-vdev"] # Byte-exact native export; source revision and hashes live in native/provenance.json. [dependencies] @@ -28,6 +29,10 @@ kore-app = { path = "native/crates/app" } kore-rt = { path = "native/crates/rt", default-features = false } kore-dagfs = { path = "native/crates/dagfs", optional = true } kore-driver = { path = "native/crates/driver", optional = true } +# A driver's side of a virtual device: the service's client without the +# service (no `host`), and the register block and memory views the two share. +kore-svc-vdev = { path = "native/services/vdev", default-features = false, optional = true } +kore-vdev = { path = "native/crates/vdev", optional = true } [package.metadata.kore] # Used by tools/local-kore.py to construct one coherent local source override, @@ -35,4 +40,4 @@ kore-driver = { path = "native/crates/driver", optional = true } # root: the offline volume procedure (and the filesystem service closure it # drives) that host programs such as vKore consume; it never enters a Kore # executable's graph. -owner-paths = { kore-start = "crates/start", kore-abi = "crates/abi", kore-app = "crates/app", kore-rt = "crates/rt", kore-c-shim = "crates/c-shim", kore-dagfs = "crates/dagfs", kore-driver = "crates/driver", kore-late = "crates/late", kore-ld = "crates/ld", kore-vol = "tools/kore-vol" } +owner-paths = { kore-start = "crates/start", kore-abi = "crates/abi", kore-app = "crates/app", kore-rt = "crates/rt", kore-c-shim = "crates/c-shim", kore-dagfs = "crates/dagfs", kore-driver = "crates/driver", kore-svc-vdev = "services/vdev", kore-vdev = "crates/vdev", kore-late = "crates/late", kore-ld = "crates/ld", kore-vol = "tools/kore-vol" } diff --git a/native/Cargo.lock b/native/Cargo.lock index 22b245d..fbe8c84 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -381,6 +381,15 @@ dependencies = [ "kore-rt", ] +[[package]] +name = "kore-svc-accel-spmd" +version = "0.0.1" +dependencies = [ + "kore-driver", + "kore-rt", + "virtio-accel-core", +] + [[package]] name = "kore-svc-block" version = "0.0.1" @@ -485,10 +494,48 @@ dependencies = [ "lz4_flex", ] +[[package]] +name = "kore-svc-vdev" +version = "0.0.0" +dependencies = [ + "kore-abi", + "kore-rt", + "kore-vdev", + "kore-vdev-accel", + "virtio-accel-guest", + "virtio-accel-proto", + "virtio-accel-transport", +] + [[package]] name = "kore-uart" version = "0.0.0" +[[package]] +name = "kore-vdev" +version = "0.0.1" +dependencies = [ + "heapless", + "virtio-accel-split-queue", + "virtio-accel-transport", +] + +[[package]] +name = "kore-vdev-accel" +version = "0.0.1" +dependencies = [ + "kore-driver", + "kore-rt", + "kore-svc-accel-spmd", + "kore-vdev", + "virtio-accel-core", + "virtio-accel-device", + "virtio-accel-guest", + "virtio-accel-proto", + "virtio-accel-transport", + "zerocopy", +] + [[package]] name = "kore-vol" version = "0.0.0" @@ -764,6 +811,49 @@ dependencies = [ "virtio-accel-transport", ] +[[package]] +name = "virtio-accel-device" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad6a272d3b25527b3a6ea2de02088fead249efb8bbd8d2a2590f7151eb0ea463" +dependencies = [ + "virtio-accel-core", + "virtio-accel-proto", + "virtio-accel-transport", + "zerocopy", +] + +[[package]] +name = "virtio-accel-guest" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520c262718f4214d42333ed985f3f86e9589d8f3509a1afe69f4606cd120f4da" +dependencies = [ + "bitflags", + "virtio-accel-proto", + "virtio-accel-transport", + "zerocopy", +] + +[[package]] +name = "virtio-accel-proto" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cda1ee8cd8ebd8b100ceae0ccd9de228036d3c1dc50483f37c5f5b9c63c0f0f" +dependencies = [ + "bitflags", + "zerocopy", +] + +[[package]] +name = "virtio-accel-split-queue" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf3ad9ce01297c6bf1f01ffaf9cc4e11bff4d639661af6b5c1136d2819d6291" +dependencies = [ + "virtio-accel-transport", +] + [[package]] name = "virtio-accel-transport" version = "0.4.1" @@ -802,18 +892,18 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "zerocopy" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" dependencies = [ "proc-macro2", "quote", diff --git a/native/Cargo.toml b/native/Cargo.toml index 3164724..8487af6 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -16,8 +16,11 @@ members = [ "crates/scrub", "crates/start", "crates/uart", + "crates/vdev", + "crates/vdev-accel", "crates/window", "drivers/nvme", + "services/accel-spmd", "services/block", "services/dagfs", "services/device", @@ -27,5 +30,6 @@ members = [ "services/mm", "services/output", "services/swap", + "services/vdev", "tools/kore-vol", ] diff --git a/native/crates/abi/src/device.rs b/native/crates/abi/src/device.rs index 463489e..f739f5e 100644 --- a/native/crates/abi/src/device.rs +++ b/native/crates/abi/src/device.rs @@ -20,9 +20,11 @@ //! route) — and teardown is an ordinary `Revoke` of the leaf. mod census; +mod class; mod i2c_admission; mod state; pub use census::*; +pub use class::*; pub use i2c_admission::*; pub use state::*; diff --git a/native/crates/abi/src/device/class.rs b/native/crates/abi/src/device/class.rs new file mode 100644 index 0000000..e43aa63 --- /dev/null +++ b/native/crates/abi/src/device/class.rs @@ -0,0 +1,153 @@ +//! What a PCI function's **class code** says it is to a driver. +//! +//! A function is offered to drivers on positive evidence only. The class +//! code is the one identification every vendor's function carries, so it is +//! the evidence asked first, and it is asked in one direction: a class named +//! here as a peripheral admits a function; no class, and no absence of one, +//! proves a function safe to hand out. Everything this table does not name +//! is [`Platform`](PciClassRole::Platform), the functions a machine keeps +//! for itself: its management engine, flash and SMBus controllers, power +//! and thermal telemetry, shared memory, and whatever a vendor adds next. +//! Those differ on every platform and their classes are mostly the +//! catch-all "other" subclasses, so they are not enumerated: they are what +//! is left. A new or unknown function is therefore kept, never offered. +//! +//! The kernel decides ([`DeviceRecord::DRIVER_ENDPOINT`]); a reader of a +//! census row uses the same table to say *why* a function is withheld. +//! Codes are the PCI-SIG's (PCI Code and ID Assignment Specification, +//! "Class Codes"). +//! +//! [`DeviceRecord::DRIVER_ENDPOINT`]: super::DeviceRecord::DRIVER_ENDPOINT + +/// What a class code alone admits a function as. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PciClassRole { + /// Bus fabric: a bridge of any kind (`06`), or an IOMMU (`08/06`). It + /// carries other functions' traffic and no driver takes it. + Fabric, + /// A display controller (`03`), or the legacy VGA-compatible spelling + /// (`00/01`). A peripheral, but one firmware may still be scanning out + /// of: the kernel also requires proof of its boot ownership. + Display, + /// A device with a standard programming model that a driver is written + /// against, and that a machine keeps running without. + Peripheral, + /// Anything else: a function of the platform itself. Not offered. + Platform, +} + +impl PciClassRole { + /// The role `class`/`subclass`/`progif` admit. + pub const fn of(class: u8, subclass: u8, progif: u8) -> Self { + match (class, subclass) { + (0x06, _) | (0x08, 0x06) => Self::Fabric, + (0x03, _) | (0x00, 0x01) => Self::Display, + // Mass storage, network, multimedia (audio, video, telephony), + // input, wireless and processing accelerators: every subclass + // is a device someone attaches or a driver brings up, including + // the vendor-specific "other" ones (a Wi-Fi radio is `02/80`). + (0x01 | 0x02 | 0x04 | 0x09 | 0x0d | 0x12, _) => Self::Peripheral, + // A UART with a named register model: 16450 through 16950. + // Programming interface 0 ("generic XT") names none, and is + // what a platform stamps on functions that are not UARTs at + // all (a sensor hub, a management engine's serial redirection). + (0x07, 0x00) if matches!(progif, 0x01..=0x06) => Self::Peripheral, + // An SD host controller: the one base-system peripheral that is + // a slot for removable media rather than part of the machine. + (0x08, 0x05) => Self::Peripheral, + // Host controllers of the external peripheral buses: IEEE 1394, + // USB (every interface, USB4 included), Fibre Channel and + // InfiniBand. SMBus (`0c/05`), IPMI (`0c/08`), I3C (`0c/0a`) + // and "other" (`0c/80`: a platform's I2C, SPI and flash + // controllers) are the machine's own buses. + (0x0c, 0x00 | 0x03 | 0x04 | 0x07) => Self::Peripheral, + _ => Self::Platform, + } + } +} + +#[cfg(test)] +mod tests { + use super::PciClassRole::{self, Display, Fabric, Peripheral, Platform}; + + #[test] + fn a_class_admits_a_peripheral_and_everything_unnamed_is_the_platforms() { + for (class, subclass, progif, role, what) in [ + (0x06, 0x00, 0x00, Fabric, "host bridge"), + (0x06, 0x01, 0x00, Fabric, "ISA/LPC bridge"), + (0x06, 0x04, 0x00, Fabric, "PCI-to-PCI bridge"), + (0x08, 0x06, 0x00, Fabric, "IOMMU"), + (0x03, 0x00, 0x00, Display, "VGA controller"), + (0x03, 0x80, 0x00, Display, "display, other"), + (0x00, 0x01, 0x00, Display, "legacy VGA-compatible"), + (0x01, 0x08, 0x02, Peripheral, "NVMe"), + (0x01, 0x06, 0x01, Peripheral, "AHCI"), + (0x02, 0x00, 0x00, Peripheral, "Ethernet"), + (0x02, 0x80, 0x00, Peripheral, "Wi-Fi radio"), + (0x04, 0x03, 0x00, Peripheral, "HD Audio"), + (0x09, 0x00, 0x00, Peripheral, "keyboard"), + (0x0d, 0x11, 0x00, Peripheral, "Bluetooth"), + (0x12, 0x00, 0x00, Peripheral, "neural accelerator"), + (0x07, 0x00, 0x02, Peripheral, "16550 UART"), + (0x08, 0x05, 0x01, Peripheral, "SD host controller"), + (0x0c, 0x03, 0x30, Peripheral, "xHCI"), + (0x0c, 0x03, 0x40, Peripheral, "USB4 host interface"), + // What a machine keeps. The identities are the ones one Lunar + // Lake laptop and one AMD laptop enumerate. + (0x05, 0x00, 0x00, Platform, "PMC shared SRAM"), + (0x07, 0x80, 0x00, Platform, "management engine HECI"), + ( + 0x07, + 0x00, + 0x00, + Platform, + "sensor hub stamped generic serial", + ), + ( + 0x0c, + 0x80, + 0x00, + Platform, + "SPI flash or LPSS I2C controller", + ), + (0x0c, 0x05, 0x00, Platform, "SMBus"), + (0x0c, 0x0a, 0x00, Platform, "I3C"), + (0x11, 0x80, 0x00, Platform, "thermal and power tuning"), + (0x10, 0x80, 0x00, Platform, "platform security processor"), + (0x13, 0x00, 0x00, Platform, "instrumentation"), + (0x08, 0x80, 0x00, Platform, "system peripheral, other"), + (0x00, 0x00, 0x00, Platform, "unclassified"), + (0x00, 0xff, 0x00, Platform, "unclassified, vendor"), + (0xff, 0x00, 0x00, Platform, "unassigned"), + ] { + assert_eq!( + PciClassRole::of(class, subclass, progif), + role, + "{what} ({class:02x}/{subclass:02x}.{progif:02x})" + ); + } + } + + /// The table only ever admits by naming: no class outside the ones it + /// names is a peripheral, whatever its subclass or interface. + #[test] + fn no_unnamed_class_is_ever_a_peripheral() { + for class in 0..=u8::MAX { + if matches!( + class, + 0x01 | 0x02 | 0x04 | 0x07 | 0x08 | 0x09 | 0x0c | 0x0d | 0x12 + ) { + continue; + } + for subclass in 0..=u8::MAX { + for progif in [0x00, 0x01, 0x02, 0x30, 0x80, 0xff] { + assert_ne!( + PciClassRole::of(class, subclass, progif), + Peripheral, + "{class:02x}/{subclass:02x}.{progif:02x}" + ); + } + } + } + } +} diff --git a/native/crates/abi/src/device/state.rs b/native/crates/abi/src/device/state.rs index 5dab25c..dacf681 100644 --- a/native/crates/abi/src/device/state.rs +++ b/native/crates/abi/src/device/state.rs @@ -3,7 +3,7 @@ //! //! The census is boot-frozen identity. This is what changes after boot, and //! what a driver writer asks before touching a device: whether a driver -//! claimed the function, which founded row drives its DMA, which BARs exist, +//! claimed the function, which founded row holds that claim, which BARs exist, //! which of them a driver may map and which are mapped now, and how the //! function can interrupt. A BAR is its index, its size and the length a //! driver may map; **no address crosses** (res/DEVICE_SERVICE.md §1.1), so @@ -33,7 +33,7 @@ use super::census::{DeviceCensus, DeviceCensusPage, MAX_DEVICE_CENSUS_ROWS}; pub const DEVICE_BARS: usize = 6; /// Every BAR's bit, for the three per-BAR bitmaps of a [`DeviceState`]. pub const DEVICE_BAR_MASK: u8 = (1 << DEVICE_BARS) - 1; -/// `DeviceState::driver` when no founded row drives the function's DMA. +/// `DeviceState::driver` when no running founded row holds the function. pub const DEVICE_ROW_NONE: u8 = 0xFF; /// States per vended page: 8 header bytes + 128 = 136 bytes, the census /// page's size, so a page rides the monitor and exec reply wires as one. @@ -77,11 +77,12 @@ pub const fn device_bar_of(operand: u16) -> Option<(u16, u8)> { pub struct DeviceState { /// [`Self::PRESENT`] and the bits below it. pub flags: u8, - /// The founded row (the founding record's index) whose task activated - /// DMA on this function, or [`DEVICE_ROW_NONE`]. The kernel's own - /// shutdown ledger, so it names who drives the device rather than who - /// claimed it: a function a row claimed and delegated reads the row it - /// was delegated to, and a driver that never activates DMA reads none. + /// The founded row (the founding record's index) whose task holds this + /// function's claim, or [`DEVICE_ROW_NONE`]. The kernel records whom a + /// claim's grant is sealed for when the claim commits, and follows that + /// exact grant if it is carried on, so this names the claimant whether + /// or not it does DMA. A claim whose task is not a running founded row + /// (it ended, or it is a program) reads none: the claim still stands. pub driver: u8, /// Bit `n`: BAR `n` exists as memory the kernel resolved. An I/O BAR is /// not one: no driver may reach port space through a BAR. diff --git a/native/crates/abi/src/image.rs b/native/crates/abi/src/image.rs index 38cbe11..12d98a2 100644 --- a/native/crates/abi/src/image.rs +++ b/native/crates/abi/src/image.rs @@ -180,6 +180,18 @@ pub const PROFILE_MODEL_DATA: u32 = 1 << 13; /// facilities every window preset is built on. pub const PROFILE_WINDOW: u32 = 1 << 14; +/// Launch profile: the child receives a **virtual-device face** — its own +/// request ring and register page with the virtual-device service, a send +/// onto that service's doorbell, and a return leg to lend it memory on +/// (`res/VIRTUAL_DEVICE.md`). +/// +/// The face is what lets a program be a driver of a device with no hardware +/// behind it. It is derived per launch and dies with the child, and it +/// carries no authority over the machine: the service it reaches holds +/// none. A launch that asks for it and cannot be given it still launches; +/// the program then finds no device to open. +pub const PROFILE_VIRTUAL_DEVICE: u32 = 1 << 15; + pub const PROFILE_KNOWN: u32 = PROFILE_TELEMETRY | PROFILE_FILESYSTEM | PROFILE_FS_VIEW_BINDER @@ -194,7 +206,8 @@ pub const PROFILE_KNOWN: u32 = PROFILE_TELEMETRY | PROFILE_ENGINE | PROFILE_GPU | PROFILE_MODEL_DATA - | PROFILE_WINDOW; + | PROFILE_WINDOW + | PROFILE_VIRTUAL_DEVICE; /// Validate the profile vocabulary and its dependencies: both filesystem /// auxiliary endpoint kinds are meaningful only beside an actual FS face, an diff --git a/native/crates/abi/src/svc.rs b/native/crates/abi/src/svc.rs index 469b2d9..0c06b51 100644 --- a/native/crates/abi/src/svc.rs +++ b/native/crates/abi/src/svc.rs @@ -4106,6 +4106,88 @@ impl InChunk { pub const CHUNK_DATA: usize = 24; } +// ─── vdev — the virtual-device service (kore-svc-vdev) ────────────── +// +// Kept last, after every other service's records: a record added in the +// middle of this file moves the lines after it, and with them the panic +// locations compiled into every image that uses those lines. + +/// The virtual-device request wire form (`VdevRequest`'s crossing shape). +/// +/// The 8-byte shape of its siblings. Discriminants are **1-based**, so a +/// zeroed record is invalid and never an administration request. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct VdevReqWire { + pub op: u8, + pub reserved: [u8; 7], +} + +impl VdevReqWire { + /// Administration pair: one component of a driver's face; the part, then + /// the announced return landing as a little-endian `u32`. + pub const OP_ADMIT_PEER: u8 = 1; + pub const OP_TAKE_ADMISSION: u8 = 2; + /// Administration pair: the peer's arrival as a little-endian `u16`. + pub const OP_RETIRE_PEER: u8 = 3; + pub const OP_CONFIRM_RETIRED: u8 = 4; + /// A driver ends its own face. + pub const OP_CLOSE_PEER: u8 = 5; + /// A driver asks for a device model by its register-block id, a + /// little-endian `u32`. + pub const OP_OPEN: u8 = 6; + /// A driver lends the memory device addresses are offsets into: the + /// announced return landing as a little-endian `u32`. + pub const OP_LEND: u8 = 7; + /// A driver gives its device up, and takes back the memory it lent. + pub const OP_CLOSE: u8 = 8; +} + +/// The virtual-device reply wire form. The service mints no authority, so +/// `grants` is pinned zero and a non-zero value fails the codec closed. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct VdevRepWire { + pub tag: u8, + pub refusal: u8, + /// Always zero. + pub grants: u8, + pub reserved: [u8; 5], +} + +impl VdevRepWire { + pub const TAG_ACCEPTED: u8 = 1; + pub const TAG_REFUSED: u8 = 2; + /// The admitted peer's arrival as a little-endian `u16`. + pub const TAG_PEER_ADMITTED: u8 = 3; + /// The memory the device now addresses, in 4 KiB pages, as a + /// little-endian `u32`. + pub const TAG_LENT: u8 = 4; + + /// The request is not this face's to make. + pub const REFUSE_DENIED: u8 = 1; + /// The admission result is not published yet. + pub const REFUSE_PENDING: u8 = 2; + /// An earlier admission must be settled first. + pub const REFUSE_CONFLICT: u8 = 3; + /// The service cannot do this now: no mapping geometry, or a model that + /// would not start. + pub const REFUSE_UNAVAILABLE: u8 = 4; + /// No device model has that id. + pub const REFUSE_NO_DEVICE: u8 = 5; + /// The device is not in a state for this: already open, not open, or + /// holding a queue in the memory it has. + pub const REFUSE_BUSY: u8 = 6; + /// Every device the service hosts at once is taken. + pub const REFUSE_LIMIT: u8 = 7; + /// What was lent is not memory the device can address. + pub const REFUSE_MEMORY: u8 = 8; +} + #[cfg(test)] mod tests { use super::*; diff --git a/native/crates/abi/src/task.rs b/native/crates/abi/src/task.rs index 3cd22f4..e4f934d 100644 --- a/native/crates/abi/src/task.rs +++ b/native/crates/abi/src/task.rs @@ -465,19 +465,20 @@ pub const INTERACTIVE_CHILD_LOADER_FACES: usize = DESKTOP_BOOTSTRAP_WINDOW_SLOTS + 2 * DESKTOP_CONTROLLER_CAPACITY; /// Live immutable image-generation peers the boot catalogue can hold at once: -/// the 47-row bootstrap catalogue — the desktop's `files`, its documents +/// the 48-row bootstrap catalogue — the desktop's `files`, its documents /// `editor`, `photos`, `settings` and `power-ui`, the screenshot writer `capture`, beside /// the native filesystem verbs `append`, `sync` and `pack`, the two /// WebAssembly rows, `wasm-witness` and `wasm`, `jsapp`, the JS app host, /// `inference` / `model-data-witness`, `shell`, the guest shell `term` -/// opens, `tcc`, the C compiler, and `ioctl`, the device tool — plus the +/// opens, `tcc`, the C compiler, `ioctl`, the device tool, and `vdev-test`, +/// the virtual accelerator's driver — plus the /// founded shell, the graphical terminal, and three extraction parsers. /// Exec asserts this covers its catalogue (`kore-svc-exec`), the one place /// that sees both. Named /// separately from the interactive reservation below because the two grow for /// unrelated reasons: this one when a row joins the catalogue, that one when /// the desktop admits another window or controller. -pub const BOOTSTRAP_IMAGE_PEERS: usize = 52; +pub const BOOTSTRAP_IMAGE_PEERS: usize = 53; /// Private image generations installed at runtime that can live at once, /// across every session: exec's `InstallFile` rows, each a file a task diff --git a/native/crates/ld/tests/built_images.rs b/native/crates/ld/tests/built_images.rs index 0bf03c1..d428f84 100644 --- a/native/crates/ld/tests/built_images.rs +++ b/native/crates/ld/tests/built_images.rs @@ -340,6 +340,7 @@ fn check_variant(root: &Path, variant: &str, required: bool) -> us ("status", ImageKind::NeedsLibrary("libpci_names.so")), ("lspci", ImageKind::NeedsLibrary("libpci_names.so")), ("ioctl", ImageKind::NeedsLibrary("libpci_names.so")), + ("vdev-test", ImageKind::Executable), ("help", ImageKind::Executable), ("window", ImageKind::Executable), ("clear", ImageKind::Executable), diff --git a/native/crates/ld/tests/shipping_list.rs b/native/crates/ld/tests/shipping_list.rs index 10dbf43..93dc1af 100644 --- a/native/crates/ld/tests/shipping_list.rs +++ b/native/crates/ld/tests/shipping_list.rs @@ -69,9 +69,9 @@ const UNSHIPPED: &[(&str, &str)] = &[ ( "accel-spmd", "backend crate: the CPU-SIMD `Accelerator` impl over kore-rt's SPMD \ - region launch, the third tenant crates/abi/src/svc.rs declares. No boot \ - row yet — its compute step (`SpmdBackend::run_pending`) needs a serve \ - loop to drive it, and that lands with the row", + region launch. It is the backend of the virtual accelerator \ + (crates/vdev-accel), linked into user/services/vdev, whose founded \ + image is kore-svc-vdev; it is not a boot row of its own", ), ( "swap-nvme", diff --git a/native/crates/rt/src/clock/mod.rs b/native/crates/rt/src/clock/mod.rs index a249927..ba1dbc9 100644 --- a/native/crates/rt/src/clock/mod.rs +++ b/native/crates/rt/src/clock/mod.rs @@ -92,6 +92,24 @@ pub(crate) fn monotonic_nanos() -> Option { uptime().ok().map(Uptime::as_nanos) } +/// Monotonic nanoseconds now, for a thread that names a deadline to the +/// kernel: the calibrated counter where the task has one, and the kernel's +/// own stamp where it has none. +/// +/// A target without a calibrated user clock (an emulator with no invariant +/// counter, a thread founded before the clock page) can still park against +/// a deadline, because the kernel keeps the time: a sleep to an instant +/// already past resolves at once and returns the kernel's clock, in whole +/// milliseconds. That costs a trap and is a millisecond coarse, so the +/// answer may be up to a millisecond behind; a deadline made from it is +/// never early by more than that. `None` only where the thread cannot trap +/// at all (a host build). +pub fn deadline_base_nanos() -> Option { + monotonic_nanos().or_else(|| { + crate::cap::verbs::sleep_until(1).map(|stamp| (stamp >> 32).saturating_mul(1_000_000)) + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/native/crates/rt/src/svc/mod.rs b/native/crates/rt/src/svc/mod.rs index db6c47e..41177f8 100644 --- a/native/crates/rt/src/svc/mod.rs +++ b/native/crates/rt/src/svc/mod.rs @@ -498,6 +498,8 @@ const _: () = { assert!(!kernel_reserved(COOKIE_HOMES)); assert!(COOKIE_HOMES != COOKIE_STORE); assert!(!kernel_reserved(COOKIE_SPMD)); + assert!(!kernel_reserved(COOKIE_VDEV)); + assert!(COOKIE_VDEV != COOKIE_SPMD); assert!(!kernel_reserved(COOKIE_LUA)); assert!(!kernel_reserved(COOKIE_LUA_REPL)); assert!(!kernel_reserved(COOKIE_MPY)); @@ -999,6 +1001,17 @@ pub const COOKIE_EXEC_CLEANUP_C: u64 = 0x6B72_5F65_7843_6C43; // "kr_exClC" /// `spmd-witness` prints the detailed line instead). pub const COOKIE_SPMD: u64 = 0x6B72_5F73_706D_6457; // "kr_spmdW" +/// Completion cookie for the boot virtual-device run (`"kr_vdevT"`), posted +/// by INIT beside [`COOKIE_SPMD`]: it launches `vdev-test` through the +/// ordinary generation-bound spawn path, with the face exec mints for the +/// virtual-device profile, and forwards the verdict. Payload 0 is the clean +/// run; 900 no row/capability; 1000+s is a nonzero exit s, where application +/// slots 256..262 are `vdev-test`'s stages (open, identity, lend, bring-up, +/// command, result, close) and 4 a launch that was given no face. Anything +/// else is the launch's spawn rung. Non-fatal, as every boot verdict is: the +/// gates key on it, and a terminal rerun of `vdev-test` prints the stage. +pub const COOKIE_VDEV: u64 = 0x6B72_5F76_6465_7654; // "kr_vdevT" + /// Completion cookie for INIT's production Lua VM validation (`"kr_luaOK"`). /// The same archive generation exposed to each shell is launched through the /// ordinary profile-driven path with `--selftest`; payload zero is success and diff --git a/native/crates/rt/src/thread.rs b/native/crates/rt/src/thread.rs index 7064582..b4d6606 100644 --- a/native/crates/rt/src/thread.rs +++ b/native/crates/rt/src/thread.rs @@ -656,38 +656,85 @@ pub(crate) fn heap_endowment() -> Option { pub(crate) const WAKE_SEND_RIGHTS: kore_abi::cap::Rights = kore_abi::cap::Rights::ACCESS.union(kore_abi::cap::Rights::DERIVE); -/// [`WorkerCap`] re-deriving this thread's own `DOORBELL@PEER_BASE` send for a -/// child, so the child's [`kick_parent`] reaches the same thread this one's -/// does: a [`DoorbellWorker`]'s wake end, or one inherited the same way. +/// [`WorkerCap`] handing a child this thread's own capability at +/// `(role, index)`: a derive of it, of the same `kind`, holding `rights`, +/// which the child finds under the same coordinate. /// -/// `None` when this thread holds no such send, or holds one it may not derive -/// from (a band peer's notice-only end, [`spmd`](crate::spmd)'s mesh): an -/// inheritance this thread cannot make is not offered, so it never fails the -/// spawn it would have rode on. The arena record decides, which is the -/// kernel's own word for the slot's kind and rights. +/// `None` when this thread holds nothing there, something of another kind, +/// or a capability that cannot give `rights` to a child: its own must cover +/// them and `DERIVE`. A share this thread cannot make is not offered, so it +/// never fails the spawn it would have rode on. The arena record decides, +/// which is the kernel's own word for the slot's kind and rights. +/// +/// This is how a capability reaches every thread of a process without any +/// thread inheriting another's holdings: each founder names what it hands on +/// and re-derives it. Ask for `DERIVE` in `rights` when the child founds +/// threads that are to hold it too. +pub fn share_endowment( + role: u16, + index: u16, + kind: kore_abi::cap::ResourceKind, + rights: kore_abi::cap::Rights, +) -> Option { + share_endowment_of_kind(role, index, kind as u16, rights) +} + +/// [`share_endowment`] with the kind as its wire discriminant, for a caller +/// that stored it. #[cfg(target_os = "none")] -pub(crate) fn wake_endowment() -> Option { - use kore_abi::cap::{CapRecord, ResourceKind}; +pub(crate) fn share_endowment_of_kind( + role: u16, + index: u16, + kind: u16, + rights: kore_abi::cap::Rights, +) -> Option { + use kore_abi::cap::{CapRecord, Rights}; let rt = crate::entry::thread_context()?; - let send = rt - .bootstrap() - .handle(role::DOORBELL, crate::futex::PEER_BASE)?; - let record_va = crate::entry::geometry::slot_record_va(send)?; + let subject = rt.bootstrap().handle(role, index)?; + let record_va = crate::entry::geometry::slot_record_va(subject)?; // SAFETY: the slot alias is this thread's kernel-authored, read-only view // of its own arena record for a slot its boot block names. let record = unsafe { core::ptr::read_volatile(record_va as *const CapRecord) }; - if record.kind != ResourceKind::ChanSend as u16 || !record.rights.contains(WAKE_SEND_RIGHTS) { + if record.kind != kind || !record.rights.contains(rights | Rights::DERIVE) { return None; } Some(WorkerCap { - role: role::DOORBELL, - index: crate::futex::PEER_BASE, - subject: send, - kind: ResourceKind::ChanSend as u16 as u8, - rights: WAKE_SEND_RIGHTS, + role, + index, + subject, + kind: u8::try_from(kind).ok()?, + rights, }) } +/// Off metal no thread founds another. +#[cfg(not(target_os = "none"))] +pub(crate) fn share_endowment_of_kind( + _role: u16, + _index: u16, + _kind: u16, + _rights: kore_abi::cap::Rights, +) -> Option { + None +} + +/// [`WorkerCap`] re-deriving this thread's own `DOORBELL@PEER_BASE` send for a +/// child, so the child's [`kick_parent`] reaches the same thread this one's +/// does: a [`DoorbellWorker`]'s wake end, or one inherited the same way. +/// +/// `None` when this thread holds no such send, or holds one it may not derive +/// from (a band peer's notice-only end, [`spmd`](crate::spmd)'s mesh): see +/// [`share_endowment`]. +#[cfg(target_os = "none")] +pub(crate) fn wake_endowment() -> Option { + share_endowment( + role::DOORBELL, + crate::futex::PEER_BASE, + kore_abi::cap::ResourceKind::ChanSend, + WAKE_SEND_RIGHTS, + ) +} + impl WorkerCap { /// **Channel space, endowed** — `role::CHANNEL_SPACE@0`: a revocable /// child of the mint grant `space`, so the thread can mint the channels diff --git a/native/crates/rt/src/thread_controller.rs b/native/crates/rt/src/thread_controller.rs index 764447a..dbee9b4 100644 --- a/native/crates/rt/src/thread_controller.rs +++ b/native/crates/rt/src/thread_controller.rs @@ -187,6 +187,70 @@ impl Record { } } +/// A capability every thread a controller founds is founded holding, beyond +/// the runtime's own: a share of the one its founder holds at the same +/// boot-block coordinate ([`thread::share_endowment`]). +/// +/// Capabilities are per thread, so a program whose threads must all act on +/// one (park on a word of a shared window, ring a peer's doorbell) says so +/// here, once, and each founding hands it on: the starter to the controller, +/// the controller to each thread. A thread whose founder does not hold it is +/// founded without it. +/// +/// [`thread::share_endowment`]: crate::thread::share_endowment +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Handed { + /// The boot-block role every holder finds it under. + pub role: u16, + /// The instance within that role. + pub index: u16, + /// The kind of capability there. + pub kind: kore_abi::cap::ResourceKind, + /// The rights a thread holds it with. The controller holds `DERIVE` as + /// well, which is what lets it hand the share on. + pub rights: kore_abi::cap::Rights, +} + +/// How many capabilities a controller hands on. Each costs a derive at +/// every founding, and what a program's threads share is a face or two, not +/// a namespace: two is a device's interrupt word and its doorbell. +pub const MAX_HANDED: usize = 2; + +impl Handed { + /// Marks a taken entry: a vacant one is zero. + const TAKEN: u64 = 1 << 63; + + fn pack(self) -> Option { + let kind = u8::try_from(self.kind as u16).ok()?; + Some( + Self::TAKEN + | u64::from(self.role) << 32 + | u64::from(self.index) << 16 + | u64::from(kind) << 8 + | u64::from(self.rights.bits()), + ) + } + + /// `(role, index, kind, rights)` of a taken entry. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + fn unpack(word: u64) -> Option<(u16, u16, u16, kore_abi::cap::Rights)> { + (word & Self::TAKEN != 0).then(|| { + ( + (word >> 32) as u16, + (word >> 16) as u16, + u16::from((word >> 8) as u8), + kore_abi::cap::Rights::from_bits_truncate(word as u8), + ) + }) + } + + /// The coordinate alone: two entries for one coordinate would stamp it + /// twice in a thread's boot block. + const fn coordinate(word: u64) -> u64 { + word & (0xffff_ffff << 16) + } +} + /// Named process-owned storage. Start once during application startup and /// share its static reference with workers. This consumes one native thread /// for the process lifetime. No native ticket escapes its owner. @@ -206,6 +270,9 @@ pub struct Controller { /// clients its wakes reach. A child's bit is set once it is founded and /// cleared before its band can be reused. reach: AtomicU32, + /// What every founding here hands on ([`Handed`], packed; zero is + /// vacant). + handed: [AtomicU64; MAX_HANDED], records: [Record; COUNT], } impl Default for Controller { @@ -224,10 +291,39 @@ impl Controller { band: AtomicU32::new(0), starter: AtomicU32::new(0), reach: AtomicU32::new(0), + handed: [const { AtomicU64::new(0) }; MAX_HANDED], records: [const { Record::new() }; COUNT], } } + /// Have every thread this controller founds hold `share`. + /// + /// Call before [`start`](Self::start), from any thread: the controller + /// thread is founded holding what was named by then, and can hand on + /// nothing it was not founded with. `false` when the controller has + /// started, the share's kind has no place in an endowment, another + /// share already names the coordinate, or [`MAX_HANDED`] are taken. + /// Naming the same share again is `true`. + pub fn hand_on(&self, share: Handed) -> bool { + let Some(word) = share.pack() else { + return false; + }; + if self.started.load(Ordering::Acquire) != IDLE { + return false; + } + for entry in &self.handed { + match entry.compare_exchange(0, word, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => return true, + Err(taken) if taken == word => return true, + Err(taken) if Handed::coordinate(taken) == Handed::coordinate(word) => { + return false; + } + Err(_) => {} + } + } + false + } + /// Start the owner. Concurrent/repeated calls do not create another owner. /// /// The calling thread becomes the controller's starter: it mints the @@ -546,8 +642,27 @@ mod owner { /// The most capabilities either founding here endows: the controller's /// doorbell, its own kick, the heap, the host's wake, a channel-space - /// grant and the starter's kick. - const MAX_ENDOW: usize = 6; + /// grant and the starter's kick, and what the program has it hand on. + const MAX_ENDOW: usize = 6 + super::MAX_HANDED; + const _: () = assert!(MAX_ENDOW <= thread::MAX_WORKER_CAPS); + + /// The shares `controller` hands on, as this thread can make them: + /// derives of its own holdings, with `DERIVE` too when `derivable`. + fn handed(controller: &Controller, derivable: bool, endowment: &mut Endowment) { + for entry in &controller.handed { + let Some((role, index, kind, rights)) = + super::Handed::unpack(entry.load(Ordering::Acquire)) + else { + continue; + }; + let rights = if derivable { + rights | Rights::DERIVE + } else { + rights + }; + endowment.push(thread::share_endowment_of_kind(role, index, kind, rights)); + } + } /// A founding's endowment, filled in order. struct Endowment { @@ -661,6 +776,8 @@ mod owner { // kick, and the starter is reached through it. let peer_zero = endowment.push(thread::wake_endowment()); endowment.push(thread::channel_space_endowment()); + // The controller hands these on to every thread it founds. + handed(self, true, &mut endowment); // Any other starter is reached at its own band's peer index. let reached = if starter == 0 { own_kick.is_some() && peer_zero @@ -855,6 +972,7 @@ mod owner { })); endowment.push(thread::heap_endowment()); endowment.push(thread::wake_endowment()); + handed(self.controller, false, &mut endowment); let Some(thread) = thread::spawn_founder_with_arg( child_entry, record as *const Record as usize, @@ -982,6 +1100,80 @@ impl Controller { mod tests { extern crate std; use super::*; + use kore_abi::cap::{ResourceKind, Rights}; + + fn share(role: u16, index: u16, rights: Rights) -> Handed { + Handed { + role, + index, + kind: ResourceKind::MemLease, + rights, + } + } + + #[test] + fn a_handed_share_survives_its_packing() { + let wanted = Handed { + role: 0xbeef, + index: 0x0108, + kind: ResourceKind::ChanSend, + rights: Rights::ACCESS | Rights::DERIVE, + }; + let word = wanted.pack().expect("a channel send packs"); + assert_ne!(word, 0, "a taken entry is never the vacant word"); + assert_eq!( + Handed::unpack(word), + Some(( + wanted.role, + wanted.index, + ResourceKind::ChanSend as u16, + wanted.rights + )) + ); + assert_eq!(Handed::unpack(0), None, "a vacant entry hands nothing on"); + // Role and index zero with no rights is still a taken entry. + let bare = share(0, 0, Rights::empty()).pack().expect("packs"); + assert!(Handed::unpack(bare).is_some()); + } + + #[test] + fn a_controller_hands_on_each_coordinate_once_and_only_so_many() { + let controller = Controller::new(); + let first = share(7, 8, Rights::ACCESS); + assert!(controller.hand_on(first)); + assert!(controller.hand_on(first), "naming the same share again"); + assert!( + !controller.hand_on(share(7, 8, Rights::ACCESS | Rights::MODIFY)), + "one coordinate, two shares: a boot block names a coordinate once" + ); + assert!(controller.hand_on(share(7, 9, Rights::ACCESS))); + assert!( + !controller.hand_on(share(9, 9, Rights::ACCESS)), + "the table is full" + ); + let taken = controller + .handed + .iter() + .filter(|entry| entry.load(Ordering::Acquire) != 0) + .count(); + assert_eq!(taken, MAX_HANDED); + } + + #[test] + fn a_started_controller_takes_no_more_shares() { + let controller = Controller::new(); + controller.started.store(PARKING, Ordering::Release); + assert!( + !controller.hand_on(share(7, 8, Rights::ACCESS)), + "its thread was founded without the share and cannot hand it on" + ); + assert!( + controller + .handed + .iter() + .all(|entry| entry.load(Ordering::Acquire) == 0) + ); + } fn completed(controller: &Controller, generation: u64) -> Id { let record = &controller.records[0]; record.value.store(73, Ordering::Relaxed); diff --git a/native/crates/vdev-accel/Cargo.toml b/native/crates/vdev-accel/Cargo.toml new file mode 100644 index 0000000..f30cdeb --- /dev/null +++ b/native/crates/vdev-accel/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "kore-vdev-accel" +version = "0.0.1" +edition = "2024" +description = "kore's virtual accelerator: a virtio-accel 1.0 device with no hardware behind it, computing six functions of FP8 (E4M3) values on the CPU backend." +publish = false + +[dependencies] +# The CPU backend this device is an instance of: its registered-kernel +# program format, its admission rules and its SPMD launch. The device adds a +# kernel registry and nothing else. +kore-svc-accel-spmd = { path = "../../services/accel-spmd" } +# The device model's contract, and the transport that carries a driver to +# an implementation of it. +kore-vdev = { path = "../vdev" } +# The frozen protocol (MicroPerceptron/virtio-accel 1.0), at the pin every +# accelerator row uses. `-device` is the command engine a device runs: it +# decodes one request frame, drives the backend and writes the response. +# `-proto` is the frame layouts and the configuration a driver reads. +virtio-accel-core = { version = "=0.4.1", default-features = false } +virtio-accel-device = { version = "=0.4.1", default-features = false } +virtio-accel-proto = { version = "=0.4.1", default-features = false } +# The configuration a driver reads is the protocol's `WireConfig`, as bytes. +zerocopy = { version = "0.8", default-features = false, features = ["derive"] } + +[dev-dependencies] +kore-driver = { path = "../driver", features = ["accel"] } +kore-rt = { path = "../rt", features = ["std"] } +# The protocol's reference driver: the session test is that driver, over +# kore's registers and queue, against this device. +virtio-accel-guest = { version = "=0.4.1", default-features = false } +virtio-accel-transport = { version = "=0.4.1", default-features = false } diff --git a/native/crates/vdev-accel/src/device.rs b/native/crates/vdev-accel/src/device.rs new file mode 100644 index 0000000..ba3ca3a --- /dev/null +++ b/native/crates/vdev-accel/src/device.rs @@ -0,0 +1,226 @@ +//! **The device**: the protocol's command engine over the CPU backend. +//! +//! A virtio-accel device has one job per driver request: take a request +//! frame, act on it, and write a response frame. The protocol crate ships the +//! engine that does this for any backend (`virtio_accel_device`'s +//! `CommandProcessor`): it decodes and validates the frame, keeps the object +//! graph (contexts, buffers, programs, queues, events) and refuses a request +//! before the backend sees it. This module is that engine instantiated over +//! [`SpmdBackend`] with the FP8 registry, which is the whole device. +//! +//! Three things a transport needs, and nothing about any transport: +//! +//! - [`AccelDevice::CONFIG`], the configuration a driver reads before it +//! sends a frame: the protocol version and the largest frames and chains +//! this device takes; +//! - [`AccelDevice::command`], one request in and one response out; +//! - [`AccelDevice::drive`], the device's own turn. A submission is admitted +//! and answered at once, as the protocol requires, and its work runs here. +//! On hardware this is the silicon's; for this device it is a call the +//! host makes between commands, so a driver that submits and then polls +//! sees the event complete. +//! +//! Those are the device model's contract in `kore-vdev` +//! ([`VirtualDevice`]), which is how a driver reaches this device: through +//! that crate's register block and queue, with nothing here knowing of +//! either. +//! +//! Two of the protocol's rules that a driver writer meets first: a buffer +//! bound to a submission is the event's until the driver destroys the event, +//! so results are read after `DestroyEvent`, not after the poll that saw it +//! complete; and a reset gives every object a new namespace, so an id a +//! driver kept from before the reset names nothing afterwards. + +use kore_svc_accel_spmd::SpmdBackend; +use kore_vdev::device::{ReadableBytes, Unrecoverable, VirtualDevice, WritableBytes}; +use virtio_accel_core::{ByteSink, ByteSource, TransportByteSink, TransportByteSource}; +use virtio_accel_device::{ + ChainRegion, CommandOutcome, CommandProcessError, CommandProcessor, CommandProcessorInitError, + DeviceHealth, ObjectNamespace, ResetDisposition, ResourcePolicy, +}; +use virtio_accel_proto::{ + BASELINE_COMMAND_QUEUES, Le16, Le32, PROTOCOL_MAJOR, PROTOCOL_MINOR, WireConfig, +}; +use zerocopy::IntoBytes; + +use crate::fp8; + +/// Descriptors one command chain may have. A request is at most a header, a +/// fixed payload and one run of data, and a response a header and one run: +/// sixteen leaves a driver room to split either without making a chain a +/// place to hide unbounded work. +pub const MAX_CHAIN_DESCRIPTORS: u16 = 16; + +/// The largest request or response frame, header included. +/// +/// A buffer transfer rides inside a frame, so this bounds one transfer, not +/// a buffer: a driver moves a larger buffer in pieces. 64 KiB is sixteen +/// pages of a driver's shared memory per frame in flight. +pub const MAX_FRAME_BYTES: u32 = 64 * 1024; + +/// Buffer backing the device will hold for a driver at once: 2 MiB, two +/// million E4M3 values. The backing is the host's heap, so this is what one +/// driver can make the device's host allocate, and a host that serves +/// several drivers from one heap sizes that heap by it. +pub const MAX_BUFFER_BACKING_BYTES: u64 = 2 << 20; + +/// Resident program bytes the device will hold. A program here is a +/// registered kernel's handle, a few bytes; 64 KiB admits every program the +/// backend's own count limit allows many times over. +pub const MAX_PROGRAM_RESIDENT_BYTES: u64 = 64 * 1024; + +/// What a driver reads from the register block's device id: `"accl"`. The +/// protocol assigns itself no Virtio device id, so the name is kore's. +pub const DEVICE_ID: u32 = u32::from_le_bytes(*b"accl"); + +/// The virtual accelerator. +pub struct AccelDevice { + processor: CommandProcessor, + /// The namespace the device's objects live in. Each reset takes the + /// next, which is what makes an id from before a reset name nothing + /// after it. + namespace: ObjectNamespace, +} + +impl AccelDevice { + /// What a driver reads before its first frame. + pub const CONFIG: WireConfig = WireConfig { + protocol_major: Le16::new(PROTOCOL_MAJOR), + protocol_minor: Le16::new(PROTOCOL_MINOR), + command_queue_count: Le16::new(BASELINE_COMMAND_QUEUES), + max_chain_descriptors: Le16::new(MAX_CHAIN_DESCRIPTORS), + max_request_bytes: Le32::new(MAX_FRAME_BYTES), + max_response_bytes: Le32::new(MAX_FRAME_BYTES), + }; + + /// A device in its first state. + pub fn new() -> Result { + const FIRST: ObjectNamespace = ObjectNamespace::new(1).unwrap(); + Self::in_namespace(FIRST) + } + + fn in_namespace(namespace: ObjectNamespace) -> Result { + const POLICY: ResourcePolicy = + ResourcePolicy::new(MAX_BUFFER_BACKING_BYTES, MAX_PROGRAM_RESIDENT_BYTES).unwrap(); + Ok(Self { + processor: CommandProcessor::new( + SpmdBackend::new(fp8::KERNELS), + &Self::CONFIG, + namespace, + POLICY, + )?, + namespace, + }) + } + + /// One command chain: `request` is its device-readable bytes, `response` + /// its device-writable bytes and `regions` their layout in the chain. + /// + /// The outcome says how many response bytes were written, or that the + /// chain cannot carry a response at all. An `Err` is the transport's own + /// memory failing under the engine, not a protocol error: those are + /// responses. + pub fn command( + &mut self, + regions: &[ChainRegion], + request: &dyn ByteSource, + response: &mut dyn ByteSink, + ) -> Result { + self.processor.process(regions, request, response) + } + + /// Run the work submissions admitted, oldest first. Answers whether any + /// ran, which is when an event a driver holds may have become terminal. + pub fn drive(&mut self) -> bool { + let mut ran = false; + // One admitted launch is in flight at a time, so this loop runs it + // and ends: the backend refuses a second admission until then. + while self.processor.accelerator().run_pending() { + ran = true; + } + ran + } + + /// Whether the device still serves, or needs the reset a transport + /// offers its driver. + pub fn health(&self) -> DeviceHealth { + self.processor.health() + } +} + +impl core::fmt::Debug for AccelDevice { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + // The engine's object graph is its own business, and a driver's. + f.debug_struct("AccelDevice") + .field("namespace", &self.namespace.get()) + .field("health", &self.health()) + .finish_non_exhaustive() + } +} + +impl VirtualDevice for AccelDevice { + const DEVICE_ID: u32 = DEVICE_ID; + /// The largest ring there is: four of the longest chains, or thirty-two + /// commands of two descriptors, in flight at once. + const MAX_QUEUE_SIZE: u16 = kore_vdev::ring::MAX_QUEUE_SIZE; + const MAX_CHAIN_DESCRIPTORS: u16 = MAX_CHAIN_DESCRIPTORS; + + fn config(&self) -> &[u8] { + const CONFIG: &WireConfig = &AccelDevice::CONFIG; + CONFIG.as_bytes() + } + + fn command( + &mut self, + regions: &[ChainRegion], + request: &dyn ReadableBytes, + response: &mut dyn WritableBytes, + ) -> u32 { + // The engine's byte ports over the transport's, with no copy + // between: a frame is read from and written to the driver's memory. + let request = TransportByteSource::new(request); + let mut response = TransportByteSink::new(response); + match AccelDevice::command(self, regions, &request, &mut response) { + Ok(CommandOutcome::Response { used, .. }) => used, + // A frame with no usable response, or the chain's memory failing + // under the engine. Either way nothing was written the driver + // may read; if the failure came after the engine had acted, the + // engine has marked itself, and `needs_reset` says so. + Ok(CommandOutcome::Unusable(_)) | Err(_) => 0, + } + } + + fn drive(&mut self) -> bool { + AccelDevice::drive(self) + } + + fn needs_reset(&self) -> bool { + self.health() != DeviceHealth::Running + } + + fn reset(&mut self) -> Result<(), Unrecoverable> { + // The namespace is sixteen bits and never reused: a device reset + // sixty-five thousand times is not brought back, because an id from + // its first life would name an object of its last. + let next = self + .namespace + .get() + .checked_add(1) + .and_then(ObjectNamespace::new) + .ok_or(Unrecoverable)?; + let report = self.processor.reset(next).map_err(|_| Unrecoverable)?; + match report.disposition { + ResetDisposition::BackendReusable => self.namespace = next, + // The backend could not account for something it held. The + // protocol forbids calling it again: it is dropped whole, and a + // new engine over a new backend takes its place. + ResetDisposition::BackendDiscardRequired => { + *self = Self::in_namespace(next).map_err(|_| Unrecoverable)?; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/native/crates/vdev-accel/src/device/tests.rs b/native/crates/vdev-accel/src/device/tests.rs new file mode 100644 index 0000000..0350327 --- /dev/null +++ b/native/crates/vdev-accel/src/device/tests.rs @@ -0,0 +1,372 @@ +use super::*; +use crate::fp8::Op; +use kore_svc_accel_spmd::{ + FORMAT_REGISTERED, IDENTITY_MAGIC, REGISTERED_MANIFEST, REGISTERED_RESIDENT_BYTES, +}; +use std::vec::Vec; +use virtio_accel_device::{SegmentedSink, SegmentedSource}; +use virtio_accel_proto::{ + AllocateBufferRequest, CreateContextRequest, CreateQueueRequest, KnownEventState, KnownOpcode, + Le64, LoadProgramRequest, ObjectPayload, RequestFlags, RequestHeader, ResponseHeader, + StatusCode, SubmitRequest, SubmitResponse, TransferBufferRequest, WireBinding, WireDeviceInfo, + WireEventState, read_exact, +}; +use zerocopy::IntoBytes; + +const HEADER: usize = core::mem::size_of::(); +/// The registered-kernel manifest's length. Spelled through `size_of_val`: +/// in this module an array's `len` is also the byte ports'. +const MANIFEST_BYTES: u64 = core::mem::size_of_val(®ISTERED_MANIFEST) as u64; + +/// The driver's side of one command: a request frame in one readable region +/// and room for the response in one writable region, as a two-descriptor +/// chain carries them. +struct Driver { + device: AccelDevice, + next_request: u64, +} + +impl Driver { + fn new() -> Self { + Self { + device: AccelDevice::new().expect("device"), + next_request: 1, + } + } + + /// Send `opcode` with `payload`; answer the status and response payload. + fn call(&mut self, opcode: KnownOpcode, payload: &[u8], room: usize) -> (StatusCode, Vec) { + let id = self.next_request; + self.next_request += 1; + let header = RequestHeader::new(opcode, RequestFlags::empty(), payload.len() as u32, id); + let mut request = header.as_bytes().to_vec(); + request.extend_from_slice(payload); + let mut response = std::vec![0u8; HEADER + room]; + let regions = [ + ChainRegion::readable(request.len() as u64), + ChainRegion::writable(response.len() as u64), + ]; + let outcome = { + let segments = [request.as_slice()]; + let source = SegmentedSource::new(&segments).unwrap(); + let mut parts = [response.as_mut_slice()]; + let mut sink = SegmentedSink::new(&mut parts).unwrap(); + self.device + .command(®ions, &source, &mut sink) + .expect("the chain's memory holds") + }; + let CommandOutcome::Response { + request_id, + status, + used, + } = outcome + else { + panic!("the chain could not carry a response: {outcome:?}"); + }; + assert_eq!(request_id, id, "a response answers its request"); + let header: ResponseHeader = read_exact(&response[..HEADER]).unwrap(); + assert_eq!(header.request_id.get(), id); + assert_eq!(header.status.get(), status.0); + let payload_bytes = header.payload_bytes.get() as usize; + assert_eq!(used as usize, HEADER + payload_bytes); + (status, response[HEADER..HEADER + payload_bytes].to_vec()) + } + + fn ok(&mut self, opcode: KnownOpcode, payload: &[u8], room: usize) -> Vec { + let (status, payload) = self.call(opcode, payload, room); + assert_eq!(status, StatusCode::OK, "{opcode:?}"); + payload + } + + fn object(&mut self, opcode: KnownOpcode, payload: &[u8]) -> u64 { + let reply = self.ok(opcode, payload, 8); + read_exact::(&reply).unwrap().object_id.get() + } + + fn buffer(&mut self, context: u64, bytes: u64, usage: u32) -> u64 { + self.object( + KnownOpcode::AllocateBuffer, + AllocateBufferRequest { + context_id: Le64::new(context), + bytes: Le64::new(bytes), + alignment: Le64::new(1), + memory_domain: MEMORY_DOMAIN_HOST, + reserved0: [0; 7], + usage: Le32::new(usage), + reserved1: Le32::new(0), + } + .as_bytes(), + ) + } + + fn program(&mut self, context: u64, kernel: u32) -> u64 { + let mut target = [Le32::new(0); 12]; + target[0] = Le32::new(IDENTITY_MAGIC); + target[1] = Le32::new(kernel); + let mut payload = LoadProgramRequest { + context_id: Le64::new(context), + format: Le32::new(FORMAT_REGISTERED), + flags: Le32::new(0), + target, + payload_bytes: Le64::new(MANIFEST_BYTES), + resident_bytes: Le64::new(REGISTERED_RESIDENT_BYTES), + } + .as_bytes() + .to_vec(); + payload.extend_from_slice(®ISTERED_MANIFEST); + self.object(KnownOpcode::LoadProgram, &payload) + } +} + +/// Provider memory a driver reaches by transfer: `MemoryDomain::Host`. +const MEMORY_DOMAIN_HOST: u8 = 1; +const TRANSFER_SOURCE: u32 = 1 << 0; +const TRANSFER_DESTINATION: u32 = 1 << 1; +const PROGRAM_INPUT: u32 = 1 << 2; +const PROGRAM_OUTPUT: u32 = 1 << 3; +const ACCESS_READ: u8 = 1; +const ACCESS_WRITE: u8 = 2; + +fn transfer(buffer: u64, bytes: u64) -> TransferBufferRequest { + TransferBufferRequest { + buffer_id: Le64::new(buffer), + offset: Le64::new(0), + bytes: Le64::new(bytes), + } +} + +fn binding(buffer: u64, slot: u32, access: u8) -> WireBinding { + WireBinding { + buffer_id: Le64::new(buffer), + offset: Le64::new(0), + bytes: Le64::new(256), + slot: Le32::new(slot), + access, + reserved: [0; 3], + } +} + +/// The configuration is one the protocol accepts, and is what the engine +/// was built with. +#[test] +fn the_configuration_is_the_baseline_protocol() { + assert_eq!(AccelDevice::CONFIG.validate(), Ok(())); + assert_eq!(AccelDevice::CONFIG.protocol_major.get(), 1); + assert_eq!(AccelDevice::CONFIG.command_queue_count.get(), 1); +} + +/// A whole driver session in the protocol's own frames: discover the device, +/// make a context, two buffers, a program and a queue, write every E4M3 +/// value, submit, poll, and read the results back. Then again for each +/// function, on the same buffers. +#[test] +fn a_driver_session_computes_each_function_through_protocol_frames() { + let mut driver = Driver::new(); + + let info: WireDeviceInfo = read_exact(&driver.ok( + KnownOpcode::GetDeviceInfo, + &[], + core::mem::size_of::(), + )) + .unwrap(); + assert_eq!(&info.uuid[..13], b"kore-spmd-cpu"); + assert_eq!(info.max_contexts.get(), 1); + assert!(info.max_bindings_per_submission.get() >= 2); + + let context = driver.object( + KnownOpcode::CreateContext, + CreateContextRequest { + flags: Le32::new(0), + reserved: Le32::new(0), + } + .as_bytes(), + ); + let operands = driver.buffer(context, 256, PROGRAM_INPUT | TRANSFER_DESTINATION); + let results = driver.buffer(context, 256, PROGRAM_OUTPUT | TRANSFER_SOURCE); + let queue = driver.object( + KnownOpcode::CreateQueue, + CreateQueueRequest { + context_id: Le64::new(context), + flags: Le32::new(0), + reserved: Le32::new(0), + } + .as_bytes(), + ); + + let every_byte: [u8; 256] = core::array::from_fn(|i| i as u8); + let mut write = transfer(operands, 256).as_bytes().to_vec(); + write.extend_from_slice(&every_byte); + driver.ok(KnownOpcode::WriteBuffer, &write, 0); + + for op in Op::ALL { + let program = driver.program(context, op.kernel_id()); + let mut submit = SubmitRequest { + queue_id: Le64::new(queue), + program_id: Le64::new(program), + binding_count: Le32::new(2), + flags: Le32::new(0), + // Zero: no deadline. + timeout_ns: Le64::new(0), + } + .as_bytes() + .to_vec(); + submit.extend_from_slice(binding(operands, 0, ACCESS_READ).as_bytes()); + submit.extend_from_slice(binding(results, 1, ACCESS_WRITE).as_bytes()); + let event = read_exact::(&driver.ok(KnownOpcode::Submit, &submit, 8)) + .unwrap() + .event_id + .get(); + + // Admitted and answered, not yet run: the work is the device's turn. + let poll = |driver: &mut Driver| { + let reply = driver.ok( + KnownOpcode::PollEvent, + ObjectPayload { + object_id: Le64::new(event), + } + .as_bytes(), + 8, + ); + read_exact::(&reply) + .unwrap() + .known_state() + .unwrap() + }; + assert_eq!(poll(&mut driver), KnownEventState::Pending); + assert!(driver.device.drive(), "the admitted submission runs"); + assert!(!driver.device.drive(), "and nothing is left to run"); + assert_eq!(poll(&mut driver), KnownEventState::Complete); + + // A buffer a submission bound stays the event's until the driver + // destroys the event, terminal or not: the protocol's rule, and the + // order a driver must follow. + let read = transfer(results, 256); + assert_eq!( + driver.call(KnownOpcode::ReadBuffer, read.as_bytes(), 256).0, + StatusCode::BUSY + ); + let object = |id: u64| ObjectPayload { + object_id: Le64::new(id), + }; + driver.ok(KnownOpcode::DestroyEvent, object(event).as_bytes(), 0); + let read = driver.ok(KnownOpcode::ReadBuffer, read.as_bytes(), 256); + assert_eq!(read.as_slice(), op.table(), "{}", op.name()); + driver.ok(KnownOpcode::UnloadProgram, object(program).as_bytes(), 0); + } + assert_eq!(driver.device.health(), DeviceHealth::Running); +} + +/// What a driver gets wrong is answered in a response, and the device goes +/// on serving. +#[test] +fn a_drivers_mistakes_are_answered_and_the_device_keeps_serving() { + let mut driver = Driver::new(); + let context = driver.object( + KnownOpcode::CreateContext, + CreateContextRequest { + flags: Le32::new(0), + reserved: Le32::new(0), + } + .as_bytes(), + ); + + // A program this device does not have: another registry's kernel. + let mut target = [Le32::new(0); 12]; + target[0] = Le32::new(IDENTITY_MAGIC); + target[1] = Le32::new(1); + let mut load = LoadProgramRequest { + context_id: Le64::new(context), + format: Le32::new(FORMAT_REGISTERED), + flags: Le32::new(0), + target, + payload_bytes: Le64::new(MANIFEST_BYTES), + resident_bytes: Le64::new(REGISTERED_RESIDENT_BYTES), + } + .as_bytes() + .to_vec(); + load.extend_from_slice(®ISTERED_MANIFEST); + assert_eq!( + driver.call(KnownOpcode::LoadProgram, &load, 8).0, + StatusCode::INCOMPATIBLE + ); + + // An id that names nothing. + let stranger = ObjectPayload { + object_id: Le64::new(0xdead_beef), + }; + let (status, _) = driver.call(KnownOpcode::PollEvent, stranger.as_bytes(), 8); + assert_ne!(status, StatusCode::OK); + + // A second context, where the device has one. + assert_eq!( + driver + .call( + KnownOpcode::CreateContext, + CreateContextRequest { + flags: Le32::new(0), + reserved: Le32::new(0), + } + .as_bytes(), + 8 + ) + .0, + StatusCode::RESOURCE_LIMIT + ); + + // A frame past the configured limit is refused before it is read. + let oversized = std::vec![0u8; MAX_FRAME_BYTES as usize]; + let (status, _) = driver.call(KnownOpcode::WriteBuffer, &oversized, 0); + assert_ne!(status, StatusCode::OK); + + assert_eq!(driver.device.health(), DeviceHealth::Running); + driver.ok(KnownOpcode::GetDeviceInfo, &[], 128); +} + +/// A reset releases what the driver made and renames every object: an id +/// from before it names nothing after. +#[test] +fn a_reset_releases_everything_and_stales_every_id() { + let mut driver = Driver::new(); + let create = CreateContextRequest { + flags: Le32::new(0), + reserved: Le32::new(0), + }; + let context = driver.object(KnownOpcode::CreateContext, create.as_bytes()); + let buffer = driver.buffer(context, 4096, TRANSFER_SOURCE | TRANSFER_DESTINATION); + + VirtualDevice::reset(&mut driver.device).expect("reset"); + assert_eq!(driver.device.health(), DeviceHealth::Running); + + let (status, _) = driver.call(KnownOpcode::ReadBuffer, transfer(buffer, 16).as_bytes(), 16); + assert_eq!(status, StatusCode::STALE_OBJECT); + // The one context the device has is free again. + let again = driver.object(KnownOpcode::CreateContext, create.as_bytes()); + assert_ne!(again, context); + + // And again: each reset is a new namespace, so the id from between the + // two is as stale as the one from before the first. + VirtualDevice::reset(&mut driver.device).expect("a second reset"); + assert_eq!(driver.device.health(), DeviceHealth::Running); + let object = ObjectPayload { + object_id: Le64::new(again), + }; + let (status, _) = driver.call(KnownOpcode::DestroyContext, object.as_bytes(), 0); + assert_eq!(status, StatusCode::STALE_OBJECT); + let third = driver.object(KnownOpcode::CreateContext, create.as_bytes()); + assert!(third != again && third != context); +} + +/// A namespace is never reused, and there are sixteen bits of them. A +/// device reset past the last is not brought back: its transport then +/// reports it dead, and the driver is given a new device. +#[test] +fn a_device_out_of_namespaces_is_not_brought_back() { + let mut device = AccelDevice::new().expect("device"); + // The first life is namespace one; each reset takes the next. + for reset in 1..u32::from(u16::MAX) { + assert_eq!(VirtualDevice::reset(&mut device), Ok(()), "reset {reset}"); + } + assert_eq!(device.health(), DeviceHealth::Running); + assert_eq!(VirtualDevice::reset(&mut device), Err(Unrecoverable)); + assert_eq!(VirtualDevice::reset(&mut device), Err(Unrecoverable)); +} diff --git a/native/crates/vdev-accel/src/fp8.rs b/native/crates/vdev-accel/src/fp8.rs new file mode 100644 index 0000000..d8bd5f8 --- /dev/null +++ b/native/crates/vdev-accel/src/fp8.rs @@ -0,0 +1,138 @@ +//! **FP8 (E4M3) elementwise kernels**: the programs the virtual accelerator +//! runs. +//! +//! One operand, one result, each an E4M3 byte: OCP 8-bit floating point +//! (`torch.float8_e4m3fn`), a sign, four exponent bits biased by 7 and three +//! mantissa bits, with no infinities, one NaN (`S.1111.111`), a largest +//! magnitude of 448 and subnormals in steps of 2⁻⁹. +//! +//! Each kernel is a table lookup. The tables are generated by +//! `tools/fp8_tables.py`, whose documentation defines every entry: the exact +//! real result rounded to nearest, ties to even; saturating at ±448, since +//! there is no infinity; the canonical NaN `0x7f` for a NaN operand and for a +//! result outside the reals. The generator derives them in exact and +//! 120-digit arithmetic and proves each rounding decision by its margin, so +//! a result does not depend on any machine's math library. That is the point +//! of defining a device by a table: a driver writer needs one answer. +//! +//! A program is named the way every kernel of the CPU backend is: the +//! registered-kernel artifact format, with the kernel's id in the target +//! identity ([`Kernel::identity`]). A submission binds the operand buffer to +//! slot 0, read, and the result buffer to slot 1, written; the result's +//! length is the element count. + +use kore_svc_accel_spmd::{Kernel, TileCtx}; + +mod tables; + +/// One of the device's functions. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Op { + Sin, + Cos, + Tan, + Exp, + Log, + Sqrt, +} + +impl Op { + /// Every function, in kernel-id order. + pub const ALL: [Self; 6] = [ + Self::Sin, + Self::Cos, + Self::Tan, + Self::Exp, + Self::Log, + Self::Sqrt, + ]; + + /// The kernel id a program names this function by. + /// + /// The ids are this device's own: a registry is per backend instance, + /// so they share nothing with another CPU row's kernels. They start at + /// `0x0100` so that a driver which confuses the two registries names a + /// kernel neither has. + pub const fn kernel_id(self) -> u32 { + 0x0100 + self as u32 + } + + /// The table that defines this function: its result for each byte. + pub const fn table(self) -> &'static [u8; 256] { + match self { + Self::Sin => &tables::SIN, + Self::Cos => &tables::COS, + Self::Tan => &tables::TAN, + Self::Exp => &tables::EXP, + Self::Log => &tables::LOG, + Self::Sqrt => &tables::SQRT, + } + } + + /// The name a status line prints. + pub const fn name(self) -> &'static str { + match self { + Self::Sin => "sin.e4m3", + Self::Cos => "cos.e4m3", + Self::Tan => "tan.e4m3", + Self::Exp => "exp.e4m3", + Self::Log => "log.e4m3", + Self::Sqrt => "sqrt.e4m3", + } + } + + /// This function of one value. + #[inline] + pub const fn apply(self, operand: u8) -> u8 { + self.table()[operand as usize] + } +} + +/// Run one span: `out[i] = table[operand[first + i]]`. +/// +/// A byte indexes a 256-byte table, which is the whole computation and has +/// no wider form: the table fits four cache lines and the span is already +/// one worker's share of the grid. +#[inline] +fn map(ctx: TileCtx<'_>, table: &[u8; 256]) { + let first = ctx.tiles.start as usize; + let operands = &ctx.inputs[0][first..first + ctx.out.len()]; + for (result, operand) in ctx.out.iter_mut().zip(operands) { + *result = table[usize::from(*operand)]; + } +} + +macro_rules! kernels { + ($($op:ident => $run:ident),+ $(,)?) => { + $(fn $run(ctx: TileCtx<'_>) { + map(ctx, Op::$op.table()); + })+ + + /// The device's kernel registry: what [`SpmdBackend::new`] is given. + /// + /// [`SpmdBackend::new`]: kore_svc_accel_spmd::SpmdBackend::new + pub static KERNELS: &[Kernel] = &[ + $(Kernel { + id: Op::$op.kernel_id(), + name: Op::$op.name(), + inputs: 1, + elem_bytes: 1, + // A lookup has no lanes: one element a step. + lanes: 1, + run: $run, + }),+ + ]; + }; +} + +kernels! { + Sin => sin, + Cos => cos, + Tan => tan, + Exp => exp, + Log => log, + Sqrt => sqrt, +} + +#[cfg(test)] +mod tests; diff --git a/native/crates/vdev-accel/src/fp8/tables.rs b/native/crates/vdev-accel/src/fp8/tables.rs new file mode 100644 index 0000000..92f6254 --- /dev/null +++ b/native/crates/vdev-accel/src/fp8/tables.rs @@ -0,0 +1,225 @@ +// Generated by tools/fp8_tables.py; do not edit. The generator's +// documentation is the definition of these values, and +// `tools/fp8_tables.py --check ` proves the file current. + +/// `sin` of each E4M3 byte, as an E4M3 byte. +#[rustfmt::skip] +pub static SIN: [u8; 256] = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2e, + 0x2f, 0x31, 0x31, 0x32, 0x33, 0x34, 0x34, 0x35, + 0x35, 0x36, 0x37, 0x38, 0x38, 0x38, 0x38, 0x37, + 0x37, 0x34, 0x32, 0x2c, 0x21, 0x9e, 0xab, 0xb1, + 0xb4, 0xb8, 0xb7, 0xb3, 0xa9, 0x26, 0x33, 0x37, + 0x38, 0x2d, 0xb1, 0xb8, 0xb1, 0x2d, 0x38, 0x32, + 0xa9, 0xb4, 0x37, 0x85, 0xb6, 0x34, 0x29, 0xb8, + 0x31, 0xb8, 0x34, 0x09, 0xb4, 0x38, 0xb0, 0xaa, + 0x37, 0x28, 0xb8, 0x11, 0x38, 0xaa, 0xb6, 0x31, + 0x34, 0xb0, 0x26, 0x19, 0xab, 0x32, 0xb5, 0x37, + 0xb8, 0xb6, 0xae, 0x21, 0x33, 0x37, 0x37, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xae, + 0xaf, 0xb1, 0xb1, 0xb2, 0xb3, 0xb4, 0xb4, 0xb5, + 0xb5, 0xb6, 0xb7, 0xb8, 0xb8, 0xb8, 0xb8, 0xb7, + 0xb7, 0xb4, 0xb2, 0xac, 0xa1, 0x1e, 0x2b, 0x31, + 0x34, 0x38, 0x37, 0x33, 0x29, 0xa6, 0xb3, 0xb7, + 0xb8, 0xad, 0x31, 0x38, 0x31, 0xad, 0xb8, 0xb2, + 0x29, 0x34, 0xb7, 0x05, 0x36, 0xb4, 0xa9, 0x38, + 0xb1, 0x38, 0xb4, 0x89, 0x34, 0xb8, 0x30, 0x2a, + 0xb7, 0xa8, 0x38, 0x91, 0xb8, 0x2a, 0x36, 0xb1, + 0xb4, 0x30, 0xa6, 0x99, 0x2b, 0xb2, 0x35, 0xb7, + 0x38, 0x36, 0x2e, 0xa1, 0xb3, 0xb7, 0xb7, 0x7f, +]; + +/// `cos` of each E4M3 byte, as an E4M3 byte. +#[rustfmt::skip] +pub static COS: [u8; 256] = [ + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x37, 0x37, 0x37, 0x37, 0x37, 0x36, 0x36, + 0x36, 0x36, 0x35, 0x34, 0x34, 0x33, 0x32, 0x31, + 0x31, 0x2e, 0x2a, 0x24, 0x19, 0x96, 0xa3, 0xaa, + 0xad, 0xb2, 0xb5, 0xb7, 0xb8, 0xb8, 0xb7, 0xb5, + 0xb2, 0xa5, 0x29, 0x33, 0x37, 0x38, 0x34, 0x2b, + 0xa1, 0xb7, 0xb5, 0x02, 0x36, 0x37, 0x21, 0xb4, + 0xb7, 0x33, 0x2d, 0xb8, 0x2e, 0x32, 0xb7, 0x22, + 0x35, 0xa0, 0xb3, 0x38, 0xb2, 0xa2, 0x36, 0xb7, + 0x2d, 0xb7, 0x9e, 0x38, 0xa4, 0xb7, 0x2f, 0x35, + 0xb3, 0x36, 0xb8, 0x38, 0xb7, 0x35, 0xb1, 0x2a, + 0x92, 0x30, 0x36, 0x38, 0x34, 0x28, 0xaa, 0x7f, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x37, 0x37, 0x37, 0x37, 0x37, 0x36, 0x36, + 0x36, 0x36, 0x35, 0x34, 0x34, 0x33, 0x32, 0x31, + 0x31, 0x2e, 0x2a, 0x24, 0x19, 0x96, 0xa3, 0xaa, + 0xad, 0xb2, 0xb5, 0xb7, 0xb8, 0xb8, 0xb7, 0xb5, + 0xb2, 0xa5, 0x29, 0x33, 0x37, 0x38, 0x34, 0x2b, + 0xa1, 0xb7, 0xb5, 0x02, 0x36, 0x37, 0x21, 0xb4, + 0xb7, 0x33, 0x2d, 0xb8, 0x2e, 0x32, 0xb7, 0x22, + 0x35, 0xa0, 0xb3, 0x38, 0xb2, 0xa2, 0x36, 0xb7, + 0x2d, 0xb7, 0x9e, 0x38, 0xa4, 0xb7, 0x2f, 0x35, + 0xb3, 0x36, 0xb8, 0x38, 0xb7, 0x35, 0xb1, 0x2a, + 0x92, 0x30, 0x36, 0x38, 0x34, 0x28, 0xaa, 0x7f, +]; + +/// `tan` of each E4M3 byte, as an E4M3 byte. +#[rustfmt::skip] +pub static TAN: [u8; 256] = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2d, 0x2e, 0x2f, 0x30, + 0x31, 0x32, 0x34, 0x35, 0x37, 0x38, 0x3a, 0x3b, + 0x3c, 0x40, 0x44, 0x4a, 0x56, 0xd9, 0xcb, 0xc5, + 0xc1, 0xba, 0xb4, 0xad, 0xa1, 0x1e, 0x2c, 0x33, + 0x39, 0x49, 0xc6, 0xb8, 0xa9, 0x26, 0x36, 0x43, + 0xce, 0xae, 0x32, 0xf6, 0xb2, 0x2f, 0x4e, 0xb6, + 0x2a, 0xb9, 0x41, 0x05, 0xc1, 0x39, 0xa9, 0xcd, + 0x33, 0x50, 0xb9, 0x09, 0x3a, 0xcc, 0xb2, 0x2a, + 0x41, 0xa8, 0x51, 0x11, 0xcb, 0x2b, 0xc0, 0x33, + 0xb8, 0xb1, 0xa6, 0x19, 0x2c, 0x34, 0x3b, 0x44, + 0x5d, 0xbd, 0xaf, 0x21, 0x36, 0x47, 0xc4, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xad, 0xae, 0xaf, 0xb0, + 0xb1, 0xb2, 0xb4, 0xb5, 0xb7, 0xb8, 0xba, 0xbb, + 0xbc, 0xc0, 0xc4, 0xca, 0xd6, 0x59, 0x4b, 0x45, + 0x41, 0x3a, 0x34, 0x2d, 0x21, 0x9e, 0xac, 0xb3, + 0xb9, 0xc9, 0x46, 0x38, 0x29, 0xa6, 0xb6, 0xc3, + 0x4e, 0x2e, 0xb2, 0x76, 0x32, 0xaf, 0xce, 0x36, + 0xaa, 0x39, 0xc1, 0x85, 0x41, 0xb9, 0x29, 0x4d, + 0xb3, 0xd0, 0x39, 0x89, 0xba, 0x4c, 0x32, 0xaa, + 0xc1, 0x28, 0xd1, 0x91, 0x4b, 0xab, 0x40, 0xb3, + 0x38, 0x31, 0x26, 0x99, 0xac, 0xb4, 0xbb, 0xc4, + 0xdd, 0x3d, 0x2f, 0xa1, 0xb6, 0xc7, 0x44, 0x7f, +]; + +/// `exp` of each E4M3 byte, as an E4M3 byte. +#[rustfmt::skip] +pub static EXP: [u8; 256] = [ + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, + 0x39, 0x39, 0x39, 0x3a, 0x3a, 0x3a, 0x3a, 0x3a, + 0x3a, 0x3b, 0x3b, 0x3b, 0x3c, 0x3c, 0x3c, 0x3d, + 0x3d, 0x3e, 0x3f, 0x40, 0x40, 0x41, 0x42, 0x42, + 0x43, 0x44, 0x46, 0x48, 0x49, 0x4a, 0x4c, 0x4d, + 0x4f, 0x51, 0x54, 0x58, 0x5a, 0x5d, 0x60, 0x63, + 0x66, 0x6b, 0x71, 0x77, 0x7d, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7f, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x37, 0x37, 0x37, 0x37, 0x37, 0x37, 0x37, + 0x37, 0x37, 0x37, 0x37, 0x37, 0x36, 0x36, 0x36, + 0x36, 0x36, 0x36, 0x35, 0x35, 0x35, 0x35, 0x35, + 0x34, 0x34, 0x34, 0x33, 0x33, 0x33, 0x32, 0x32, + 0x32, 0x31, 0x31, 0x30, 0x2f, 0x2e, 0x2d, 0x2d, + 0x2c, 0x2a, 0x29, 0x28, 0x26, 0x25, 0x23, 0x22, + 0x21, 0x1d, 0x1b, 0x18, 0x15, 0x12, 0x0f, 0x0c, + 0x09, 0x06, 0x03, 0x02, 0x01, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, +]; + +/// `log` of each E4M3 byte, as an E4M3 byte. +#[rustfmt::skip] +pub static LOG: [u8; 256] = [ + 0xfe, 0xcc, 0xcb, 0xca, 0xca, 0xc9, 0xc9, 0xc9, + 0xc8, 0xc8, 0xc8, 0xc7, 0xc7, 0xc7, 0xc6, 0xc6, + 0xc6, 0xc5, 0xc5, 0xc5, 0xc4, 0xc4, 0xc4, 0xc3, + 0xc3, 0xc3, 0xc2, 0xc2, 0xc1, 0xc1, 0xc1, 0xc1, + 0xc0, 0xc0, 0xbf, 0xbe, 0xbd, 0xbd, 0xbc, 0xbc, + 0xbb, 0xba, 0xb9, 0xb9, 0xb8, 0xb6, 0xb5, 0xb4, + 0xb3, 0xb1, 0xaf, 0xac, 0xa9, 0xa5, 0xa1, 0x98, + 0x00, 0x1f, 0x26, 0x2a, 0x2d, 0x30, 0x31, 0x32, + 0x33, 0x35, 0x37, 0x38, 0x39, 0x39, 0x3a, 0x3b, + 0x3b, 0x3c, 0x3d, 0x3e, 0x3e, 0x3f, 0x40, 0x40, + 0x40, 0x41, 0x41, 0x42, 0x42, 0x42, 0x43, 0x43, + 0x43, 0x44, 0x44, 0x44, 0x45, 0x45, 0x45, 0x46, + 0x46, 0x46, 0x47, 0x47, 0x47, 0x48, 0x48, 0x48, + 0x48, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x4a, + 0x4a, 0x4a, 0x4a, 0x4a, 0x4b, 0x4b, 0x4b, 0x4b, + 0x4b, 0x4b, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x7f, + 0xfe, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, +]; + +/// `sqrt` of each E4M3 byte, as an E4M3 byte. +#[rustfmt::skip] +pub static SQRT: [u8; 256] = [ + 0x00, 0x13, 0x18, 0x1a, 0x1b, 0x1d, 0x1e, 0x1f, + 0x20, 0x20, 0x21, 0x21, 0x22, 0x22, 0x23, 0x23, + 0x23, 0x24, 0x25, 0x25, 0x26, 0x26, 0x27, 0x27, + 0x28, 0x28, 0x29, 0x29, 0x2a, 0x2a, 0x2b, 0x2b, + 0x2b, 0x2c, 0x2d, 0x2d, 0x2e, 0x2e, 0x2f, 0x2f, + 0x30, 0x30, 0x31, 0x31, 0x32, 0x32, 0x33, 0x33, + 0x33, 0x34, 0x35, 0x35, 0x36, 0x36, 0x37, 0x37, + 0x38, 0x38, 0x39, 0x39, 0x3a, 0x3a, 0x3b, 0x3b, + 0x3b, 0x3c, 0x3d, 0x3d, 0x3e, 0x3e, 0x3f, 0x3f, + 0x40, 0x40, 0x41, 0x41, 0x42, 0x42, 0x43, 0x43, + 0x43, 0x44, 0x45, 0x45, 0x46, 0x46, 0x47, 0x47, + 0x48, 0x48, 0x49, 0x49, 0x4a, 0x4a, 0x4b, 0x4b, + 0x4b, 0x4c, 0x4d, 0x4d, 0x4e, 0x4e, 0x4f, 0x4f, + 0x50, 0x50, 0x51, 0x51, 0x52, 0x52, 0x53, 0x53, + 0x53, 0x54, 0x55, 0x55, 0x56, 0x56, 0x57, 0x57, + 0x58, 0x58, 0x59, 0x59, 0x5a, 0x5a, 0x5b, 0x7f, + 0x80, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, +]; diff --git a/native/crates/vdev-accel/src/fp8/tests.rs b/native/crates/vdev-accel/src/fp8/tests.rs new file mode 100644 index 0000000..1bfe6e3 --- /dev/null +++ b/native/crates/vdev-accel/src/fp8/tests.rs @@ -0,0 +1,286 @@ +use super::*; +use kore_svc_accel_spmd::{ + FORMAT_REGISTERED, IDENTITY_MAGIC, REGISTERED_MANIFEST, REGISTERED_RESIDENT_BYTES, SpmdBackend, + SpmdBuffer, SpmdContext, +}; +use virtio_accel_core::{ + Accelerator, AccessMode, ArtifactFormat, ArtifactRef, BackendError, BindingRef, BufferDesc, + BufferRange, BufferUsage, ContextDesc, ContextFlags, EventState, MemoryDomain, QueueDesc, + QueueFlags, TargetIdentity, Timeout, +}; + +const NAN: u8 = 0x7f; +const LARGEST: u8 = 0x7e; + +/// The value of one E4M3 byte; `None` for the NaN. +fn decode(code: u8) -> Option { + let sign = if code & 0x80 != 0 { -1.0 } else { 1.0 }; + let exponent = i32::from((code >> 3) & 0xf); + let mantissa = f64::from(code & 0x7); + if exponent == 0xf && code & 0x7 == 0x7 { + return None; + } + Some(if exponent == 0 { + sign * mantissa * 2f64.powi(-9) + } else { + sign * (8.0 + mantissa) * 2f64.powi(exponent - 10) + }) +} + +/// Round a real to E4M3 by the device's rules: nearest, ties to even, +/// saturating, the sign kept on a zero. +fn encode(value: f64) -> u8 { + if value.is_nan() { + return NAN; + } + let sign = if value.is_sign_negative() { 0x80 } else { 0x00 }; + let magnitude = value.abs(); + if magnitude >= 448.0 { + return sign | LARGEST; + } + let mut best = 0u8; + for code in 1..=LARGEST { + let (candidate, held) = (decode(code).unwrap(), decode(best).unwrap()); + let (near, far) = ((candidate - magnitude).abs(), (held - magnitude).abs()); + if near < far || (near == far && code % 2 == 0) { + best = code; + } + } + sign | best +} + +/// What this machine's own math library says the function is, in binary64. +/// The tables were not made with it; agreeing with it is the check. +fn reference(op: Op, x: f64) -> f64 { + match op { + Op::Sin => x.sin(), + Op::Cos => x.cos(), + Op::Tan => x.tan(), + Op::Exp => x.exp(), + Op::Log => x.ln(), + Op::Sqrt => x.sqrt(), + } +} + +/// The generator's tables against an independent computation of all 1536 +/// entries. Its closest rounding decision is 2.5e-5 of a boundary away +/// (`tools/fp8_tables.py --margin`), eleven orders of magnitude past +/// binary64's error, so agreement here is conclusive and not luck. +#[test] +fn every_entry_is_the_correctly_rounded_result() { + for op in Op::ALL { + for code in 0..=u8::MAX { + let want = match decode(code) { + None => NAN, + // The limit of log at either zero is negative infinity, + // which saturates: `ln(-0.0)` is that in binary64 too. + Some(x) => encode(reference(op, x)), + }; + assert_eq!( + op.apply(code), + want, + "{}({code:#04x} = {:?})", + op.name(), + decode(code) + ); + } + } +} + +/// The rules a driver writer reads, each on a value they can check by hand. +#[test] +fn the_stated_rules_hold_on_values_one_can_check_by_hand() { + let code = |value: f64| encode(value); + let one = code(1.0); + assert_eq!(one, 0x38); + assert_eq!(decode(Op::Sin.apply(one)), Some(0.8125), "sin 1 = 0.8414…"); + assert_eq!(decode(Op::Cos.apply(one)), Some(0.5625), "cos 1 = 0.5403…"); + assert_eq!(decode(Op::Tan.apply(one)), Some(1.5), "tan 1 = 1.5574…"); + assert_eq!(decode(Op::Exp.apply(one)), Some(2.75), "e = 2.7182…"); + assert_eq!(Op::Log.apply(one), 0x00, "log 1 is exactly zero"); + assert_eq!(decode(Op::Sqrt.apply(code(4.0))), Some(2.0)); + assert_eq!(Op::Cos.apply(0x00), one); + assert_eq!(Op::Exp.apply(0x00), one); + + // No infinity: a result past the largest magnitude is that magnitude. + assert_eq!(decode(LARGEST), Some(448.0)); + assert_eq!(Op::Exp.apply(LARGEST), LARGEST); + assert_eq!(Op::Exp.apply(code(8.0)), LARGEST, "e^8 = 2980.9…"); + assert_eq!(Op::Log.apply(0x00), 0x80 | LARGEST, "log 0 is -448"); + assert_eq!(Op::Log.apply(0x80), 0x80 | LARGEST, "and so is log -0"); + assert_eq!(Op::Exp.apply(0x80 | LARGEST), 0x00, "e^-448 rounds to zero"); + + // Outside the reals, and a NaN operand, give the one canonical NaN. + for op in Op::ALL { + assert_eq!(op.apply(0x7f), NAN, "{}", op.name()); + assert_eq!(op.apply(0xff), NAN, "{}", op.name()); + } + assert_eq!(Op::Log.apply(code(-1.0)), NAN); + assert_eq!(Op::Sqrt.apply(code(-1.0)), NAN); + + // A zero keeps its sign where IEEE 754 keeps it. + for op in [Op::Sin, Op::Tan, Op::Sqrt] { + assert_eq!(op.apply(0x00), 0x00, "{}", op.name()); + assert_eq!(op.apply(0x80), 0x80, "{}", op.name()); + } + // sin and tan are odd, cos is even, over every value. + for code in 0..=LARGEST { + assert_eq!(Op::Sin.apply(code | 0x80), Op::Sin.apply(code) ^ 0x80); + assert_eq!(Op::Tan.apply(code | 0x80), Op::Tan.apply(code) ^ 0x80); + assert_eq!(Op::Cos.apply(code | 0x80), Op::Cos.apply(code)); + } +} + +#[test] +fn the_registry_names_each_function_once() { + assert_eq!(KERNELS.len(), Op::ALL.len()); + for (kernel, op) in KERNELS.iter().zip(Op::ALL) { + assert_eq!(kernel.id, op.kernel_id()); + assert_eq!(kernel.name, op.name()); + assert_eq!((kernel.inputs, kernel.elem_bytes), (1, 1)); + assert_eq!(kernel.identity().0[..2], [IDENTITY_MAGIC, op.kernel_id()]); + } + for (index, kernel) in KERNELS.iter().enumerate() { + assert!( + !KERNELS[..index].iter().any(|prior| prior.id == kernel.id), + "duplicate kernel id {:#x}", + kernel.id + ); + } +} + +/// A span is its own share of the grid: two disjoint spans equal one run. +#[test] +fn a_tiled_run_equals_the_whole_run() { + let operands: [u8; 256] = core::array::from_fn(|i| i as u8); + let run = KERNELS[Op::Exp as usize].run; + let launch = kore_driver::accel::LaunchShape::ArtifactDefined; + let mut whole = [0u8; 256]; + run(TileCtx { + tiles: 0..256, + total: 256, + launch, + inputs: &[&operands], + out: &mut whole, + }); + assert_eq!(whole, *Op::Exp.table()); + + let mut split = [0u8; 256]; + let (low, high) = split.split_at_mut(100); + run(TileCtx { + tiles: 0..100, + total: 256, + launch, + inputs: &[&operands], + out: low, + }); + run(TileCtx { + tiles: 100..256, + total: 256, + launch, + inputs: &[&operands], + out: high, + }); + assert_eq!(split, whole); +} + +const BYTES: u64 = 256; + +fn buffer(backend: &SpmdBackend, context: &SpmdContext, usage: BufferUsage) -> SpmdBuffer { + let usage = usage | BufferUsage::TRANSFER_SOURCE | BufferUsage::TRANSFER_DESTINATION; + let desc = BufferDesc::new(BYTES, 1, MemoryDomain::Host, usage).expect("descriptor"); + backend + .allocate_buffer(context, desc) + .expect("allocated") + .into_parts() + .0 +} + +fn artifact(id: u32) -> ArtifactRef<'static> { + let mut words = [0u32; 12]; + words[0] = IDENTITY_MAGIC; + words[1] = id; + ArtifactRef { + format: ArtifactFormat::new(FORMAT_REGISTERED).expect("format"), + target: TargetIdentity(words), + payload: ®ISTERED_MANIFEST, + resident_bytes: REGISTERED_RESIDENT_BYTES, + } +} + +/// The device, as the protocol reaches it: every function over every byte, +/// through the CPU backend's context, buffers, program, queue and event. +#[test] +fn the_backend_runs_each_function_over_every_value() { + let backend = SpmdBackend::new(KERNELS); + let context = backend + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = backend + .create_queue( + &context, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let mut operands = buffer(&backend, &context, BufferUsage::PROGRAM_INPUT); + let results = buffer(&backend, &context, BufferUsage::PROGRAM_OUTPUT); + let every_byte: [u8; 256] = core::array::from_fn(|i| i as u8); + backend + .write_buffer(&mut operands, 0, &every_byte) + .expect("written"); + let whole = BufferRange::new(0, BYTES).expect("range"); + + for op in Op::ALL { + let program = backend + .load_program(&context, artifact(op.kernel_id())) + .expect("loaded"); + let event = backend + .submit( + &queue, + &program, + &[ + BindingRef { + slot: 0, + buffer: &operands, + range: whole, + access: AccessMode::Read, + }, + BindingRef { + slot: 1, + buffer: &results, + range: whole, + access: AccessMode::Write, + }, + ], + Timeout::Infinite, + ) + .map_err(|_| "refused") + .expect("admitted"); + assert_eq!(backend.poll_event(&event), Ok(EventState::Pending)); + assert!(backend.run_pending(), "the admitted launch runs"); + assert_eq!(backend.poll_event(&event), Ok(EventState::Complete)); + let mut read = [0u8; 256]; + backend.read_buffer(&results, 0, &mut read).expect("read"); + assert_eq!(read, *op.table(), "{}", op.name()); + backend.destroy_event(event).map_err(|_| "event").unwrap(); + backend + .unload_program(program) + .map_err(|_| "program") + .unwrap(); + } + + // A kernel of another registry is not one this device has. + assert_eq!( + backend + .load_program( + &context, + artifact(kore_svc_accel_spmd::kernel::KERNEL_ADD_F32) + ) + .err(), + Some(BackendError::Incompatible) + ); +} diff --git a/native/crates/vdev-accel/src/lib.rs b/native/crates/vdev-accel/src/lib.rs new file mode 100644 index 0000000..fe4f5bf --- /dev/null +++ b/native/crates/vdev-accel/src/lib.rs @@ -0,0 +1,25 @@ +//! # kore-vdev-accel: the virtual accelerator +//! +//! A virtio-accel 1.0 device with no hardware behind it. It exists so a +//! driver can be written, run and debugged on a machine that has no +//! accelerator, against a device whose every answer is known in advance. +//! +//! What it computes is deliberately small and exact: six functions of one +//! FP8 (E4M3) value, elementwise over a buffer. See [`fp8`]. An E4M3 value +//! is a byte, so each function is a 256-entry table, and the table *is* the +//! device's definition. A driver is right when it reads back those bytes. +//! +//! The device is an instance of the CPU backend, `kore-svc-accel-spmd`: its +//! programs are that backend's registered kernels, and this crate supplies +//! the registry ([`fp8::KERNELS`]). What a driver exchanges with it is the +//! protocol's own frames, decoded by the protocol's own command engine +//! ([`device::AccelDevice`]). Nothing here knows how a frame reaches the +//! device; that is the transport's business. + +#![no_std] + +pub mod device; +pub mod fp8; + +#[cfg(test)] +extern crate std; diff --git a/native/crates/vdev-accel/tests/session.rs b/native/crates/vdev-accel/tests/session.rs new file mode 100644 index 0000000..ee0d653 --- /dev/null +++ b/native/crates/vdev-accel/tests/session.rs @@ -0,0 +1,438 @@ +//! A driver's whole session with the virtual accelerator, end to end and +//! with nothing of the driver written here: the driver is the protocol's +//! reference guest client (`virtio-accel-guest`), over kore's register +//! block and shared-memory queue (`kore-vdev`'s driver half), against this +//! crate's device behind `kore-vdev`'s transport. +//! +//! What it shows is that the pieces are one device. The reference client +//! knows only the protocol's queue contract; the device model knows only +//! requests and responses; the registers and the ring between them are what +//! a driver in C will map. + +use core::num::{NonZeroU32, NonZeroU64}; +use core::ptr::NonNull; + +use kore_svc_accel_spmd::{ + FORMAT_REGISTERED, IDENTITY_MAGIC, REGISTERED_MANIFEST, REGISTERED_RESIDENT_BYTES, +}; +use kore_vdev::device::VirtualDevice; +use kore_vdev::driver::{self, DriverChain, DriverRing, Layout, Slots}; +use kore_vdev::mem::Window; +use kore_vdev::regs; +use kore_vdev::transport::Transport; +use kore_vdev_accel::device::{AccelDevice, DEVICE_ID}; +use kore_vdev_accel::fp8::Op; +use virtio_accel_guest::{ + AccessMode, Binding, Buffer, BufferDesc, BufferRange, BufferUsage, Completion, Context, + EventState, ExecutionQueue, GuestClient, GuestConfig, MemoryDomain, Operation, ProgramDesc, + RequestPoll, StartResult, SubmissionOutcome, +}; +use virtio_accel_proto::{ + AllocateBufferRequest, CreateContextRequest, CreateQueueRequest, LoadProgramRequest, + ObjectPayload, RequestHeader, ResponseHeader, StatusCode, SubmitRequest, SubmitResponse, + TransferBufferRequest, WireBinding, WireConfig, WireDeviceInfo, WireEventState, read_exact, +}; +use virtio_accel_transport::{DriverChainBuffer, QueueSize}; + +type Ring = DriverRing; +type Chain = DriverChain; +type Client = GuestClient; + +const BUDGET: NonZeroU32 = NonZeroU32::new(8).unwrap(); +const REQUEST: u32 = core::mem::size_of::() as u32; +const RESPONSE: u32 = core::mem::size_of::() as u32; +/// Every E4M3 value, once. +const VALUES: u32 = 256; + +const fn bytes_of() -> u32 { + core::mem::size_of::() as u32 +} + +/// Host memory standing for a lease both sides map. +fn window(bytes: u64) -> Window { + let words: &'static mut [u64] = + Box::leak(vec![0u64; (bytes as usize).div_ceil(8)].into_boxed_slice()); + // SAFETY: a leaked, eight-byte aligned allocation of at least `bytes` + // bytes that nothing else holds a reference into. + unsafe { + Window::new( + NonNull::new(words.as_mut_ptr().cast()).unwrap(), + bytes as usize, + ) + } +} + +/// A driver process and the device's host, in one test. +struct Session { + registers: Window, + memory: Window, + layout: Layout, + transport: Transport, + /// How often the device raised the driver's interrupt. + interrupts: u32, +} + +/// What a bring-up gives the driver. +struct Driver { + client: Client, + slots: Slots, +} + +impl Session { + fn new() -> Self { + // Sixteen descriptors, the device's longest chain, so eight commands + // in flight; a slot holds a request or a response of a page. + let layout = Layout::new(QueueSize::new(16).unwrap(), 4096, 4096).unwrap(); + let registers = window(regs::BYTES); + let memory = window(layout.bytes()); + let device = AccelDevice::new().expect("the device"); + let mut transport = Transport::new(registers, device).expect("a transport"); + // The driver lends its memory before it brings the device up. + transport.lend(memory).expect("a device in its first state"); + Self { + registers, + memory, + layout, + transport, + interrupts: 0, + } + } + + /// The doorbell: the host services the transport. + fn ring(&mut self) { + loop { + let serviced = self.transport.service(BUDGET); + self.interrupts += u32::from(serviced.interrupt); + if !serviced.pending { + return; + } + } + } + + /// What every driver does first: find the device, bring it up and read + /// its configuration. + fn bring_up(&mut self) -> Driver { + let identity = driver::identify(&self.registers).expect("a kore virtual device"); + assert_eq!(identity.device_id, DEVICE_ID); + assert_eq!(identity.vendor_id, regs::VENDOR_KORE); + let mut config = [0u8; core::mem::size_of::()]; + assert_eq!(identity.config_bytes as usize, config.len()); + driver::read_config(&self.registers, &mut config).unwrap(); + let wire: WireConfig = read_exact(&config).unwrap(); + assert_eq!(wire, AccelDevice::CONFIG); + + let (registers, memory, layout) = (self.registers, self.memory, self.layout); + let (ring, slots) = driver::bring_up(®isters, memory, layout, || self.ring()) + .expect("the device comes up"); + // The protocol has no feature of its own: what is left of the offer + // once the transport's bit is taken away must be nothing. + let config = GuestConfig::new(wire, identity.features & !regs::F_VERSION_1, layout.slots()) + .expect("a configuration the reference driver accepts"); + Driver { + client: GuestClient::new(ring, config).expect("the reference driver"), + slots, + } + } + + /// One command: take a slot for a request of `request` payload bytes + /// and a response of `response`, start the operation, ring, and take + /// the answer. + fn call( + &mut self, + driver: &mut Driver, + request: u32, + response: u32, + start: impl FnOnce(&mut Client, Chain) -> StartResult, + ) -> Completion::Error> { + let chain = driver + .slots + .chain(REQUEST + request, RESPONSE + response) + .expect("a free slot"); + let pending = match start(&mut driver.client, chain) { + Ok(pending) => pending, + Err(error) => panic!("the request was not started: {:?}", error.into_parts().2), + }; + let before = self.interrupts; + self.ring(); + assert_eq!(self.interrupts, before + 1, "one command, one interrupt"); + match driver.client.poll(pending) { + RequestPoll::Ready(completion) => completion, + _ => panic!("a serviced command has its answer"), + } + } + + /// A command that must succeed: its output, with the slot returned. + fn ok( + &mut self, + driver: &mut Driver, + request: u32, + response: u32, + start: impl FnOnce(&mut Client, Chain) -> StartResult, + ) -> O::Output { + let (output, chain) = self.ok_with_chain(driver, request, response, start); + driver.slots.recycle(chain); + output + } + + fn ok_with_chain( + &mut self, + driver: &mut Driver, + request: u32, + response: u32, + start: impl FnOnce(&mut Client, Chain) -> StartResult, + ) -> (O::Output, Chain) { + match self.call(driver, request, response, start) { + Completion::Success { output, chain } => (output, chain), + Completion::DeviceError { status, .. } => panic!("the device answered {status:?}"), + Completion::InvalidResponse { error, .. } => panic!("a malformed answer: {error:?}"), + } + } + + /// A command the device must refuse: the status it refused with. + fn refused( + &mut self, + driver: &mut Driver, + request: u32, + response: u32, + start: impl FnOnce(&mut Client, Chain) -> StartResult, + ) -> StatusCode { + match self.call(driver, request, response, start) { + Completion::DeviceError { status, chain, .. } => { + driver.slots.recycle(chain); + status + } + _ => panic!("the device was expected to refuse"), + } + } +} + +/// The objects a session computes with. +struct Objects { + context: Context, + operands: Buffer, + results: Buffer, + queue: ExecutionQueue, +} + +fn objects(session: &mut Session, driver: &mut Driver) -> Objects { + let info = session.ok(driver, 0, bytes_of::(), |client, chain| { + client.get_device_info(chain) + }); + assert_eq!(&info.uuid[..13], b"kore-spmd-cpu"); + + let context = session.ok( + driver, + bytes_of::(), + bytes_of::(), + |client, chain| client.create_context(chain), + ); + let mut buffer = |usage| { + let desc = BufferDesc::new(u64::from(VALUES), 1, MemoryDomain::Host, usage).unwrap(); + session.ok( + driver, + bytes_of::(), + bytes_of::(), + |client, chain| client.allocate_buffer(chain, &context, desc), + ) + }; + let operands = buffer(BufferUsage::PROGRAM_INPUT | BufferUsage::TRANSFER_DESTINATION); + let results = buffer(BufferUsage::PROGRAM_OUTPUT | BufferUsage::TRANSFER_SOURCE); + let queue = session.ok( + driver, + bytes_of::(), + bytes_of::(), + |client, chain| client.create_execution_queue(chain, &context), + ); + Objects { + context, + operands, + results, + queue, + } +} + +/// Run `op` over the operands buffer and answer what the results buffer +/// then holds. +fn compute(session: &mut Session, driver: &mut Driver, objects: &Objects, op: Op) -> Vec { + let mut target = [0u32; 12]; + target[0] = IDENTITY_MAGIC; + target[1] = op.kernel_id(); + let desc = ProgramDesc::new( + NonZeroU32::new(FORMAT_REGISTERED).unwrap(), + target, + NonZeroU64::new(REGISTERED_RESIDENT_BYTES).unwrap(), + ); + let program = session.ok( + driver, + bytes_of::() + REGISTERED_MANIFEST.len() as u32, + bytes_of::(), + |client, chain| { + client.load_program( + chain, + &objects.context, + desc, + REGISTERED_MANIFEST.as_slice(), + ) + }, + ); + + let whole = BufferRange::new(0, u64::from(VALUES)).unwrap(); + let bindings = [ + Binding { + buffer: &objects.operands, + range: whole, + slot: 0, + access: AccessMode::Read, + }, + Binding { + buffer: &objects.results, + range: whole, + slot: 1, + access: AccessMode::Write, + }, + ]; + let outcome = session.ok( + driver, + bytes_of::() + 2 * bytes_of::(), + bytes_of::(), + // Zero: no deadline. + |client, chain| client.submit(chain, &objects.queue, &program, &bindings, 0), + ); + let SubmissionOutcome::Accepted(event) = outcome else { + panic!("the submission was not accepted: {outcome:?}"); + }; + + // The submission was answered and then run in the same service: the + // first poll finds it complete. + let state = session.ok( + driver, + bytes_of::(), + bytes_of::(), + |client, chain| client.poll_event(chain, &event), + ); + assert_eq!(state, EventState::Complete, "{}", op.name()); + + // The results are the event's until the event is destroyed. + let read = |client: &mut Client, chain| client.read_buffer(chain, &objects.results, whole); + assert_eq!( + session.refused(driver, bytes_of::(), VALUES, read), + StatusCode::BUSY + ); + session.ok(driver, bytes_of::(), 0, |client, chain| { + client.destroy_event(chain, event) + }); + let (output, chain) = + session.ok_with_chain(driver, bytes_of::(), VALUES, read); + assert_eq!(output.bytes, u64::from(VALUES)); + let mut results = vec![0u8; VALUES as usize]; + chain + .read_device_writable(u64::from(RESPONSE), &mut results) + .unwrap(); + driver.slots.recycle(chain); + + session.ok(driver, bytes_of::(), 0, |client, chain| { + client.unload_program(chain, program) + }); + results +} + +/// The session a driver is written to hold: bring the device up, make its +/// objects, write every E4M3 value once, and for each of the six functions +/// submit, poll, and read back exactly the function's table. +#[test] +fn the_reference_driver_computes_every_function() { + let mut session = Session::new(); + let mut driver = session.bring_up(); + let objects = objects(&mut session, &mut driver); + + let every_value: [u8; VALUES as usize] = core::array::from_fn(|value| value as u8); + session.ok( + &mut driver, + bytes_of::() + VALUES, + 0, + |client, chain| client.write_buffer(chain, &objects.operands, 0, every_value.as_slice()), + ); + + for op in Op::ALL { + let results = compute(&mut session, &mut driver, &objects, op); + assert_eq!(results.as_slice(), op.table(), "{}", op.name()); + for (value, result) in results.iter().enumerate() { + assert_eq!(*result, op.apply(value as u8)); + } + } + assert_eq!(driver.slots.free(), u32::from(session.layout.slots())); + assert!(!session.transport.device().needs_reset()); + assert!(!session.transport.is_dead()); +} + +/// A second bring-up is a reset. The device forgets every object, an id +/// kept from before names nothing, and the device computes again from its +/// first state. +#[test] +fn a_reset_gives_the_driver_a_device_in_its_first_state() { + let mut session = Session::new(); + let mut driver = session.bring_up(); + let before = objects(&mut session, &mut driver); + + let mut driver = session.bring_up(); + // Discovery again: the client is new, as the device is. + session.ok( + &mut driver, + 0, + bytes_of::(), + |client, chain| client.get_device_info(chain), + ); + let stale = session.refused( + &mut driver, + bytes_of::(), + VALUES, + |client, chain| { + client.read_buffer(chain, &before.results, BufferRange::new(0, 16).unwrap()) + }, + ); + assert_eq!(stale, StatusCode::STALE_OBJECT); + + // The one context the device has is free again, and everything works. + let objects = objects(&mut session, &mut driver); + assert_ne!(objects.context.raw(), before.context.raw()); + let sevens = [0x77u8; VALUES as usize]; + session.ok( + &mut driver, + bytes_of::() + VALUES, + 0, + |client, chain| client.write_buffer(chain, &objects.operands, 0, sevens.as_slice()), + ); + let results = compute(&mut session, &mut driver, &objects, Op::Sqrt); + assert_eq!(results, [Op::Sqrt.apply(0x77); VALUES as usize]); +} + +/// The reference driver with every slot in flight at once: the commands are +/// served in the order they were published, and each answer reaches the +/// request it belongs to. +#[test] +fn every_slot_in_flight_is_answered() { + let mut session = Session::new(); + let mut driver = session.bring_up(); + let slots = session.layout.slots(); + + let pending: Vec<_> = (0..slots) + .map(|_| { + let chain = driver + .slots + .chain(REQUEST, RESPONSE + bytes_of::()) + .unwrap(); + driver.client.get_device_info(chain).ok().expect("started") + }) + .collect(); + assert_eq!(driver.slots.free(), 0); + session.ring(); + for pending in pending { + let RequestPoll::Ready(Completion::Success { output, chain }) = driver.client.poll(pending) + else { + panic!("every command was answered"); + }; + assert_eq!(&output.uuid[..13], b"kore-spmd-cpu"); + driver.slots.recycle(chain); + } + assert_eq!(driver.slots.free(), u32::from(slots)); +} diff --git a/native/crates/vdev/Cargo.toml b/native/crates/vdev/Cargo.toml new file mode 100644 index 0000000..ed6c513 --- /dev/null +++ b/native/crates/vdev/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "kore-vdev" +version = "0.0.1" +edition = "2024" +description = "kore virtual devices: the transport a driver reaches a device with no hardware behind it, as memory it shares with the device's host." +publish = false + +[dependencies] +# Bounded per-chain metadata: a chain's spans and regions are at most the +# queue's size, so they need no heap. +heapless = { version = "0.9.3", default-features = false } +# The queue contracts the ring implements (MicroPerceptron/virtio-accel 1.0): +# `DeviceQueue`, `DeviceChain`, the chain-layout rules and the reset epoch. +# The crate is dependency-free and names no memory model, which is why the +# ring over shared memory lives here and not there. +virtio-accel-transport = { version = "=0.4.1", default-features = false } + +[dev-dependencies] +# The protocol's own in-memory split queue: the oracle the shared-memory +# ring is held to, chain shape by chain shape. +virtio-accel-split-queue = { version = "=0.4.1", default-features = false } diff --git a/native/crates/vdev/src/device.rs b/native/crates/vdev/src/device.rs new file mode 100644 index 0000000..92c49be --- /dev/null +++ b/native/crates/vdev/src/device.rs @@ -0,0 +1,69 @@ +//! **The device model's contract**: what a device with no hardware is, to +//! the transport that carries a driver to it. +//! +//! A model answers commands. A driver sends one command as one chain of +//! descriptors, the bytes it gives the device to read followed by the bytes +//! it gives the device to write, and the model reads a request from the +//! first and writes a response into the second. That is the whole exchange, +//! and it is all a model sees of a driver: it is handed lengths and byte +//! ports, never a device address, a descriptor or a ring. +//! +//! Everything else a driver meets is the transport's +//! ([`Transport`](crate::transport::Transport)): the register block, the +//! status it steps through, the queue, the doorbell and the interrupt. A +//! model is written once and knows none of it. + +pub use virtio_accel_transport::{ChainRegion, ReadableBytes, WritableBytes}; + +/// A reset could not return the model to its first state. The transport +/// stops serving for good, and the host drops the instance. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Unrecoverable; + +/// One device model. +pub trait VirtualDevice { + /// What the driver reads from [`DEVICE_ID`](crate::regs::DEVICE_ID) to + /// tell which model it has been given. + const DEVICE_ID: u32; + + /// The largest queue the model takes: a power of two, and at most + /// [`MAX_QUEUE_SIZE`](crate::ring::MAX_QUEUE_SIZE). + const MAX_QUEUE_SIZE: u16; + + /// The longest chain one command may be, in descriptors: at least two + /// (a request and a response) and at most the queue's size. + const MAX_CHAIN_DESCRIPTORS: u16; + + /// The device configuration a driver reads at + /// [`CONFIG`](crate::regs::CONFIG) before its first command. Fixed for + /// the life of the model. + fn config(&self) -> &[u8]; + + /// One command. `regions` is the chain's layout, `request` its readable + /// bytes and `response` its writable bytes, each as one run however the + /// driver split it. + /// + /// Returns the number of response bytes written, from the start of + /// `response`: what the driver is told the device used. Zero says the + /// chain could not carry a response at all, and nothing was written. + fn command( + &mut self, + regions: &[ChainRegion], + request: &dyn ReadableBytes, + response: &mut dyn WritableBytes, + ) -> u32; + + /// The model's own turn, taken after each command: work a command + /// admitted and answered runs here. Hardware does this by itself; a + /// model does it when the transport says so. Answers whether any ran. + fn drive(&mut self) -> bool; + + /// Whether the model can no longer answer truthfully. The transport + /// then stops taking commands and tells the driver to reset the device. + fn needs_reset(&self) -> bool; + + /// Forget everything a driver made and return to the first state. The + /// transport has stopped taking and completing chains before it calls + /// this, and nothing the driver held from before names anything after. + fn reset(&mut self) -> Result<(), Unrecoverable>; +} diff --git a/native/crates/vdev/src/driver.rs b/native/crates/vdev/src/driver.rs new file mode 100644 index 0000000..d6e3687 --- /dev/null +++ b/native/crates/vdev/src/driver.rs @@ -0,0 +1,624 @@ +//! **The driver's half**: the same registers and the same queue, from the +//! side that owns the memory. +//! +//! A driver written in C does these steps by hand against the layout in +//! [`regs`]. This is those steps in Rust, behind the driver-side queue +//! contract of `virtio-accel-transport` ([`DriverQueue`]), so the protocol's +//! reference guest client runs over a kore virtual device unchanged. +//! +//! # The memory a driver lends +//! +//! One window, laid out by [`Layout`]: the descriptor table, the available +//! ring and the used ring, then a fixed pool of command slots. A slot is two +//! descriptors and the bytes they name, room for one request the device +//! reads and one response it writes. A command is therefore always a chain +//! of two, the shortest the protocol allows, and a driver has as many +//! commands in flight as it has slots. +//! +//! A [`DriverChain`] is one slot, sized for one command. It is taken from +//! the pool ([`Slots::chain`]), filled, published on the ring, taken back +//! used and returned ([`Slots::recycle`]). It is not `Clone`: a slot has +//! one holder at a time, the driver or the device. The pool is apart from +//! the ring because the protocol's client owns the queue it is given and is +//! handed each chain by its caller. +//! +//! # Reset +//! +//! A driver that wants the device back in its first state calls +//! [`bring_up`] again and drops what the last one gave it. The reset is +//! the first step of a bring-up, and the rings are cleared after the device +//! has acknowledged it and before the device is told the queue is ready, so +//! neither side ever sees the other's indices from before. +//! +//! # What the driver trusts +//! +//! The device is the more trusted side, but a used entry is still checked +//! before it is believed: it must name a chain the driver published and has +//! not yet taken back, and a length no longer than the room that chain gave. + +use core::sync::atomic::{Ordering, fence}; + +use virtio_accel_transport::{ + ChainId, DriverChainBuffer, DriverQueue, NotificationHint, NotificationRecheck, PublishError, + PublishErrorKind, PublishedChain, QueueConfigError, QueueEpoch, QueueError, QueuePort, + QueueSize, QueueState, ReclaimedChain, UsedChain, UsedLength, +}; + +use crate::mem::{Memory, MemoryFault}; +use crate::regs::{ + self, F_VERSION_1, STATUS_ACKNOWLEDGE, STATUS_DEVICE_NEEDS_RESET, STATUS_DRIVER, + STATUS_DRIVER_OK, STATUS_FAILED, STATUS_FEATURES_OK, +}; +use crate::ring::{ + AVAIL_ALIGN, AVAIL_ENTRY, DESC_BYTES, MAX_QUEUE_SIZE, RING_FLAGS, RING_HEADER, RING_IDX, + RingAddresses, USED_ALIGN, USED_ENTRY, USED_LEN, VIRTQ_AVAIL_F_NO_INTERRUPT, VIRTQ_DESC_F_NEXT, + VIRTQ_DESC_F_WRITE, VIRTQ_USED_F_NO_NOTIFY, +}; + +/// Descriptors one command slot takes: its request and its response. +const SLOT_DESCRIPTORS: u16 = 2; + +/// The most command slots a queue has: half the largest queue. +pub const MAX_SLOTS: usize = (MAX_QUEUE_SIZE / SLOT_DESCRIPTORS) as usize; + +/// A slot's bytes start on a word, so a frame's fields are aligned for a +/// driver that reads them in place. +const SLOT_ALIGN: u64 = 8; + +/// What the device's half of the register block says it is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Identity { + /// The device model ([`regs::DEVICE_ID`]). + pub device_id: u32, + /// Who made it ([`regs::VENDOR_ID`]). + pub vendor_id: u32, + /// The features it offers. + pub features: u64, + /// The largest queue it takes. + pub queue_num_max: u32, + /// Bytes of device configuration at [`regs::CONFIG`]. + pub config_bytes: u32, +} + +/// Why a device could not be brought up. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BringUpError { + /// The block does not begin with [`regs::MAGIC_VALUE`]. + NotADevice, + /// The block's layout is a version this driver does not know. + Version(u32), + /// The device did not return to status zero when asked to reset. + Reset, + /// The device does not offer [`F_VERSION_1`], or refused it. + Features, + /// The layout does not fit: the queue is larger than the device takes, + /// or the memory is smaller than the layout. + Layout, + /// The device refused the queue. + Queue, + /// A register or the driver's own memory did not answer. + Memory(MemoryFault), +} + +impl From for BringUpError { + fn from(fault: MemoryFault) -> Self { + Self::Memory(fault) + } +} + +/// Read the device's identity, refusing a block that is not a kore virtual +/// device of a layout this driver knows. +pub fn identify(registers: &M) -> Result { + if registers.load_u32(regs::MAGIC)? != regs::MAGIC_VALUE { + return Err(BringUpError::NotADevice); + } + let version = registers.load_u32(regs::VERSION)?; + if version != regs::VERSION_VALUE { + return Err(BringUpError::Version(version)); + } + Ok(Identity { + device_id: registers.load_u32(regs::DEVICE_ID)?, + vendor_id: registers.load_u32(regs::VENDOR_ID)?, + features: registers.load_u64(regs::DEVICE_FEATURES)?, + queue_num_max: registers.load_u32(regs::QUEUE_NUM_MAX)?, + config_bytes: registers.load_u32(regs::CONFIG_BYTES)?, + }) +} + +/// Read the first `target.len()` bytes of the device configuration. +pub fn read_config(registers: &M, target: &mut [u8]) -> Result<(), BringUpError> { + if target.len() as u64 > u64::from(registers.load_u32(regs::CONFIG_BYTES)?) { + return Err(BringUpError::Layout); + } + Ok(registers.read(regs::CONFIG, target)?) +} + +/// How a driver lays out the memory it lends. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Layout { + size: QueueSize, + request_bytes: u32, + response_bytes: u32, +} + +impl Layout { + /// A queue of `size` descriptors, `size / 2` command slots, each with + /// room for a request of `request_bytes` and a response of + /// `response_bytes`. `None` when the queue is larger than a ring holds + /// or a slot has no room. + pub const fn new(size: QueueSize, request_bytes: u32, response_bytes: u32) -> Option { + if size.get() > MAX_QUEUE_SIZE + || size.get() < SLOT_DESCRIPTORS + || request_bytes == 0 + || response_bytes == 0 + { + return None; + } + Some(Self { + size, + request_bytes, + response_bytes, + }) + } + + /// The queue's size. + pub const fn size(&self) -> QueueSize { + self.size + } + + /// Command slots: commands the driver can have in flight. + pub const fn slots(&self) -> u16 { + self.size.get() / SLOT_DESCRIPTORS + } + + /// Where the queue's three parts are. + pub const fn addresses(&self) -> RingAddresses { + let size = self.size.get() as u64; + let available = (size * DESC_BYTES).next_multiple_of(AVAIL_ALIGN); + let used = (available + RING_HEADER + size * AVAIL_ENTRY).next_multiple_of(USED_ALIGN); + RingAddresses { + descriptors: 0, + available, + used, + } + } + + const fn slot_bytes(&self) -> u64 { + (self.request_bytes as u64).next_multiple_of(SLOT_ALIGN) + + (self.response_bytes as u64).next_multiple_of(SLOT_ALIGN) + } + + const fn slots_at(&self) -> u64 { + let size = self.size.get() as u64; + (self.addresses().used + RING_HEADER + size * USED_ENTRY).next_multiple_of(SLOT_ALIGN) + } + + /// Where `slot`'s request and response bytes are. + const fn slot(&self, slot: u16) -> (u64, u64) { + let request = self.slots_at() + slot as u64 * self.slot_bytes(); + ( + request, + request + (self.request_bytes as u64).next_multiple_of(SLOT_ALIGN), + ) + } + + /// Bytes of memory the layout takes. + pub const fn bytes(&self) -> u64 { + self.slots_at() + self.slots() as u64 * self.slot_bytes() + } +} + +/// Bring a device up: reset it, agree features, describe the queue and tell +/// it the driver is ready. `kick` rings the doorbell and waits for the +/// interrupt that answers it. +/// +/// The steps are Virtio 1.3 §3.1.1's. On a refusal the driver marks the +/// device [`STATUS_FAILED`], as that section requires, and says which step +/// was refused. +pub fn bring_up( + registers: &M, + memory: M, + layout: Layout, + mut kick: impl FnMut(), +) -> Result<(DriverRing, Slots), BringUpError> { + let identity = identify(registers)?; + let status = |value| registers.store_u32(regs::DRIVER_STATUS, value); + let answered = || registers.load_u32(regs::DEVICE_STATUS); + let fail = |error| { + // Best effort: the refusal is what the caller needs to hear. + let _ = status(STATUS_FAILED); + Err(error) + }; + + // A device in its reset state, which the driver has announced nothing + // to, has nothing to undo and answers no reset: there is nothing to + // wait for. Any other is reset first, and that is answered. + if registers.load_u32(regs::DRIVER_STATUS)? != 0 || answered()? != 0 { + status(0)?; + kick(); + if answered()? != 0 { + return Err(BringUpError::Reset); + } + } + let mut reached = STATUS_ACKNOWLEDGE | STATUS_DRIVER; + status(reached)?; + + if identity.features & F_VERSION_1 == 0 { + return fail(BringUpError::Features); + } + registers.store_u64(regs::DRIVER_FEATURES, F_VERSION_1)?; + reached |= STATUS_FEATURES_OK; + status(reached)?; + kick(); + if answered()? & STATUS_FEATURES_OK == 0 { + return fail(BringUpError::Features); + } + + if u32::from(layout.size.get()) > identity.queue_num_max || memory.len() < layout.bytes() { + return fail(BringUpError::Layout); + } + let ring = DriverRing::new(memory.clone(), layout)?; + let at = layout.addresses(); + registers.store_u32(regs::QUEUE_NUM, u32::from(layout.size.get()))?; + registers.store_u64(regs::QUEUE_DESC, at.descriptors)?; + registers.store_u64(regs::QUEUE_DRIVER, at.available)?; + registers.store_u64(regs::QUEUE_DEVICE, at.used)?; + registers.store_u32(regs::QUEUE_READY, 1)?; + reached |= STATUS_DRIVER_OK; + status(reached)?; + kick(); + let device = answered()?; + if device & STATUS_DRIVER_OK == 0 || device & STATUS_DEVICE_NEEDS_RESET != 0 { + return fail(BringUpError::Queue); + } + Ok((ring, Slots::new(memory, layout))) +} + +/// Why the driver's ring could not go on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DriverRingError { + /// A used entry names no chain the driver has with the device, or a + /// length longer than the chain's room. + UsedEntry, + /// The driver's own memory did not answer. + Memory(MemoryFault), +} + +impl From for QueueError { + fn from(fault: MemoryFault) -> Self { + Self::Transport(DriverRingError::Memory(fault)) + } +} + +/// One command slot, sized for one command: `readable` bytes the driver +/// writes and the device reads, `writable` bytes the device writes. +#[derive(Debug)] +pub struct DriverChain { + memory: M, + slot: u16, + request: u64, + readable: u32, + response: u64, + writable: u32, +} + +impl DriverChainBuffer for DriverChain { + type Error = MemoryFault; + + fn device_readable_len(&self) -> u64 { + u64::from(self.readable) + } + + fn device_writable_len(&self) -> u64 { + u64::from(self.writable) + } + + fn write_device_readable(&mut self, offset: u64, source: &[u8]) -> Result<(), MemoryFault> { + within(offset, source.len(), self.readable)?; + self.memory.write(self.request + offset, source) + } + + fn read_device_writable(&self, offset: u64, target: &mut [u8]) -> Result<(), MemoryFault> { + within(offset, target.len(), self.writable)?; + self.memory.read(self.response + offset, target) + } +} + +fn within(offset: u64, len: usize, room: u32) -> Result<(), MemoryFault> { + if offset + .checked_add(len as u64) + .is_some_and(|end| end <= u64::from(room)) + { + Ok(()) + } else { + Err(MemoryFault::Range) + } +} + +/// The pool of command slots in the driver's memory. +#[derive(Debug)] +pub struct Slots { + memory: M, + layout: Layout, + /// Slots no chain holds, one bit each. + free: u32, +} + +impl Slots { + fn new(memory: M, layout: Layout) -> Self { + let slots = u32::from(layout.slots()); + Self { + memory, + layout, + free: if slots == u32::BITS { + u32::MAX + } else { + (1 << slots) - 1 + }, + } + } + + /// Slots no chain holds. + pub const fn free(&self) -> u32 { + self.free.count_ones() + } + + /// A free slot sized for a request of `readable` bytes and a response of + /// up to `writable`. `None` when every slot is taken, or the sizes are + /// zero or more than a slot has room for. + pub fn chain(&mut self, readable: u32, writable: u32) -> Option> { + if readable == 0 + || writable == 0 + || readable > self.layout.request_bytes + || writable > self.layout.response_bytes + || self.free == 0 + { + return None; + } + let slot = self.free.trailing_zeros() as u16; + self.free &= !(1 << slot); + let (request, response) = self.layout.slot(slot); + Some(DriverChain { + memory: self.memory.clone(), + slot, + request, + readable, + response, + writable, + }) + } + + /// Return a chain's slot: one the device has used, or one that was + /// never published. + pub fn recycle(&mut self, chain: DriverChain) { + self.free |= 1 << chain.slot; + } +} + +/// The driver's half of one split virtqueue in its own memory. +#[derive(Debug)] +pub struct DriverRing { + memory: M, + layout: Layout, + state: QueueState, + /// Available entries published, and used entries taken back. + published: u16, + taken: u16, + /// The chains the device has, by slot. + held: [Option>; MAX_SLOTS], +} + +impl DriverRing { + /// A ready queue laid out by `layout` in `memory`, with both rings + /// cleared: the memory may have held a queue before a reset. + fn new(memory: M, layout: Layout) -> Result { + let at = layout.addresses(); + for ring in [at.available, at.used] { + memory.store_u16(ring + RING_FLAGS, 0)?; + memory.store_u16(ring + RING_IDX, 0)?; + } + Ok(Self { + memory, + layout, + state: ready(layout.size, QueueEpoch::INITIAL), + published: 0, + taken: 0, + held: [const { None }; MAX_SLOTS], + }) + } + + /// The layout the queue was made with. + pub const fn layout(&self) -> Layout { + self.layout + } + + fn descriptor( + &self, + index: u16, + at: u64, + len: u32, + flags: u16, + next: u16, + ) -> Result<(), MemoryFault> { + let mut bytes = [0u8; DESC_BYTES as usize]; + bytes[..8].copy_from_slice(&at.to_le_bytes()); + bytes[8..12].copy_from_slice(&len.to_le_bytes()); + bytes[12..14].copy_from_slice(&flags.to_le_bytes()); + bytes[14..].copy_from_slice(&next.to_le_bytes()); + self.memory.write( + self.layout.addresses().descriptors + u64::from(index) * DESC_BYTES, + &bytes, + ) + } + + /// Write the chain's two descriptors and publish its head. + fn expose(&mut self, chain: &DriverChain) -> Result { + let at = self.layout.addresses(); + let head = chain.slot * SLOT_DESCRIPTORS; + self.descriptor( + head, + chain.request, + chain.readable, + VIRTQ_DESC_F_NEXT, + head + 1, + )?; + self.descriptor( + head + 1, + chain.response, + chain.writable, + VIRTQ_DESC_F_WRITE, + 0, + )?; + let entry = u64::from(self.published % self.layout.size.get()); + self.memory.write( + at.available + RING_HEADER + entry * AVAIL_ENTRY, + &head.to_le_bytes(), + )?; + self.published = self.published.wrapping_add(1); + // Release: the request bytes, the descriptors and the entry are + // visible to a device that has loaded this index. + self.memory + .store_u16(at.available + RING_IDX, self.published)?; + // The index is published before the device's suppression flag is + // read: a device that clears the flag and rechecks the index cannot + // miss this chain. + fence(Ordering::SeqCst); + Ok( + if self.memory.load_u16(at.used + RING_FLAGS)? & VIRTQ_USED_F_NO_NOTIFY == 0 { + NotificationHint::Notify + } else { + NotificationHint::Suppressed + }, + ) + } +} + +fn ready(size: QueueSize, epoch: QueueEpoch) -> QueueState { + // A configured size no larger than the maximum, which is itself: the + // one shape `QueueState::new` refuses is not this one. + QueueState::new(size, Some(size), true, epoch) + .unwrap_or_else(|_| QueueState::unconfigured(size, epoch)) +} + +impl QueuePort for DriverRing { + fn state(&self) -> QueueState { + self.state + } +} + +impl DriverQueue for DriverRing { + type Chain = DriverChain; + type Reclaimed = heapless::Vec>, MAX_SLOTS>; + type Error = DriverRingError; + + fn publish( + &mut self, + chain: Self::Chain, + ) -> Result> { + if !self.state.ready() { + return Err(PublishError::new(chain, PublishErrorKind::NotReady)); + } + let slot = usize::from(chain.slot); + if self.held[slot].is_some() { + // The slot's descriptors are the device's until it uses them. + return Err(PublishError::new(chain, PublishErrorKind::QueueFull)); + } + match self.expose(&chain) { + Ok(hint) => { + let id = ChainId::new(self.state.epoch(), u64::from(chain.slot * SLOT_DESCRIPTORS)); + self.held[slot] = Some(chain); + Ok(PublishedChain::new(id, hint)) + } + Err(fault) => Err(PublishError::new( + chain, + PublishErrorKind::Transport(DriverRingError::Memory(fault)), + )), + } + } + + fn pop_used(&mut self) -> Result>, QueueError> { + if !self.state.ready() { + return Err(QueueError::NotReady); + } + let at = self.layout.addresses(); + // Acquire: the entry and the response bytes the device wrote before + // it published this index are visible to the reads below. + if self.memory.load_u16(at.used + RING_IDX)? == self.taken { + return Ok(None); + } + let entry = + at.used + RING_HEADER + u64::from(self.taken % self.layout.size.get()) * USED_ENTRY; + let head = self.memory.load_u32(entry)?; + let len = self.memory.load_u32(entry + USED_LEN)?; + self.taken = self.taken.wrapping_add(1); + + let slot = head / u32::from(SLOT_DESCRIPTORS); + let chain = if head % u32::from(SLOT_DESCRIPTORS) == 0 { + self.held + .get_mut(slot as usize) + .and_then(Option::take) + .filter(|chain| len <= chain.writable) + } else { + None + }; + let Some(chain) = chain else { + return Err(QueueError::Transport(DriverRingError::UsedEntry)); + }; + Ok(Some(UsedChain::new( + ChainId::new(self.state.epoch(), u64::from(head)), + UsedLength::new(len), + chain, + ))) + } + + fn disable_used_notifications(&mut self) -> Result<(), QueueError> { + self.memory.store_u16( + self.layout.addresses().available + RING_FLAGS, + VIRTQ_AVAIL_F_NO_INTERRUPT, + )?; + Ok(()) + } + + fn enable_used_notifications( + &mut self, + ) -> Result> { + let at = self.layout.addresses(); + self.memory.store_u16(at.available + RING_FLAGS, 0)?; + // The flag is cleared before the index is read: a device that used + // a chain while interrupts were off is seen here, and one that uses + // a chain after sees the flag clear and interrupts. + fence(Ordering::SeqCst); + Ok(if self.memory.load_u16(at.used + RING_IDX)? == self.taken { + NotificationRecheck::Idle + } else { + NotificationRecheck::WorkPending + }) + } + + fn reset( + &mut self, + next_epoch: QueueEpoch, + ) -> Result> { + if next_epoch <= self.state.epoch() { + return Err(QueueError::InvalidConfiguration( + QueueConfigError::NonIncreasingEpoch, + )); + } + let epoch = self.state.epoch(); + let mut reclaimed = heapless::Vec::new(); + for (slot, held) in self.held.iter_mut().enumerate() { + if let Some(chain) = held.take() { + let head = slot as u64 * u64::from(SLOT_DESCRIPTORS); + // One entry a slot at most, and the vector holds every slot. + let _ = reclaimed.push(ReclaimedChain::new(ChainId::new(epoch, head), chain)); + } + } + // Unconfigured, and it stays so: the queue a driver uses after a + // reset is the one the next `bring_up` makes. + self.state = QueueState::unconfigured(self.layout.size, next_epoch); + Ok(reclaimed) + } +} + +#[cfg(test)] +mod tests; diff --git a/native/crates/vdev/src/driver/tests.rs b/native/crates/vdev/src/driver/tests.rs new file mode 100644 index 0000000..b23cb95 --- /dev/null +++ b/native/crates/vdev/src/driver/tests.rs @@ -0,0 +1,392 @@ +use super::*; +use crate::device::VirtualDevice; +use crate::mem::{Window, tests::window}; +use crate::transport::Transport; +use crate::transport::tests::{ECHO_CONFIG, Echo}; +use core::num::NonZeroU32; +use std::vec; +use std::vec::Vec; + +const BUDGET: NonZeroU32 = NonZeroU32::new(16).unwrap(); + +fn layout() -> Layout { + Layout::new(QueueSize::new(8).unwrap(), 64, 32).unwrap() +} + +/// A device and the two windows a driver is given to it. +struct Bench { + registers: Window, + memory: Window, + transport: Transport, +} + +impl Bench { + fn new() -> Self { + let registers = window(regs::BYTES as usize); + let memory = window(layout().bytes() as usize); + let mut transport = Transport::new(registers, Echo::default()).unwrap(); + transport.lend(memory).unwrap(); + Self { + registers, + memory, + transport, + } + } + + fn bring_up(&mut self) -> Result<(DriverRing, Slots), BringUpError> { + self.bring_up_with(layout()) + } + + fn bring_up_with( + &mut self, + layout: Layout, + ) -> Result<(DriverRing, Slots), BringUpError> { + let transport = &mut self.transport; + bring_up(&self.registers, self.memory, layout, || { + transport.service(BUDGET); + }) + } + + /// Send `request` with `room` for the answer and take the answer back. + fn exchange( + &mut self, + ring: &mut DriverRing, + slots: &mut Slots, + request: &[u8], + room: u32, + ) -> Vec { + let mut chain = slots.chain(request.len() as u32, room).expect("a slot"); + chain.write_device_readable(0, request).unwrap(); + let published = ring.publish(chain).expect("published"); + assert_eq!(published.notification(), NotificationHint::Notify); + assert!(self.transport.service(BUDGET).interrupt); + let used = ring.pop_used().unwrap().expect("used"); + assert_eq!(used.id(), published.id()); + let (_, len, chain) = used.into_parts(); + let mut answer = vec![0u8; len.get() as usize]; + chain.read_device_writable(0, &mut answer).unwrap(); + slots.recycle(chain); + answer + } +} + +#[test] +fn a_layout_keeps_its_parts_apart_and_aligned() { + for size in [2u16, 4, 8, 16, 32, 64] { + for (request, response) in [(1u32, 1u32), (64, 32), (65_536, 65_536), (17, 4093)] { + let layout = Layout::new(QueueSize::new(size).unwrap(), request, response).unwrap(); + let at = layout.addresses(); + let size = u64::from(size); + assert_eq!(at.descriptors % 16, 0); + assert_eq!(at.available % 2, 0); + assert_eq!(at.used % 4, 0); + assert!(at.descriptors + size * 16 <= at.available); + assert!(at.available + 4 + size * 2 <= at.used); + let mut end = at.used + 4 + size * 8; + for slot in 0..layout.slots() { + let (request_at, response_at) = layout.slot(slot); + assert!(end <= request_at, "slot {slot} begins after what precedes"); + assert_eq!(request_at % 8, 0); + assert_eq!(response_at % 8, 0); + assert!(request_at + u64::from(request) <= response_at); + end = response_at + u64::from(response); + } + assert!(end <= layout.bytes()); + assert_eq!(layout.slots(), (size / 2) as u16); + } + } + let size = |size| QueueSize::new(size).unwrap(); + assert!(Layout::new(size(1), 8, 8).is_none(), "no room for a chain"); + assert!( + Layout::new(size(128), 8, 8).is_none(), + "more than a ring holds" + ); + assert!(Layout::new(size(8), 0, 8).is_none()); + assert!(Layout::new(size(8), 8, 0).is_none()); +} + +#[test] +fn a_driver_identifies_brings_up_and_exchanges() { + let mut bench = Bench::new(); + let identity = identify(&bench.registers).unwrap(); + assert_eq!( + identity, + Identity { + device_id: Echo::DEVICE_ID, + vendor_id: regs::VENDOR_KORE, + features: F_VERSION_1, + queue_num_max: 8, + config_bytes: ECHO_CONFIG.len() as u32, + } + ); + let mut config = [0u8; ECHO_CONFIG.len()]; + read_config(&bench.registers, &mut config).unwrap(); + assert_eq!(config, ECHO_CONFIG); + assert_eq!( + read_config(&bench.registers, &mut [0u8; ECHO_CONFIG.len() + 1]), + Err(BringUpError::Layout), + "no more than the device has" + ); + + let (mut ring, mut slots) = bench.bring_up().unwrap(); + assert!(ring.state().ready()); + assert_eq!(ring.layout(), layout()); + assert_eq!(slots.free(), 4); + assert_eq!( + bench.transport.status(), + STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK | STATUS_DRIVER_OK + ); + assert_eq!(bench.transport.features(), F_VERSION_1); + + assert_eq!( + bench.exchange(&mut ring, &mut slots, b"hello", 32), + b"olleh" + ); + assert_eq!(bench.exchange(&mut ring, &mut slots, b"device", 3), b"eci"); + // Enough exchanges to take both rings' indices round the queue. + for round in 0..40u8 { + let request = [round, round.wrapping_mul(7), 3]; + let answer = bench.exchange(&mut ring, &mut slots, &request, 8); + assert_eq!(answer, [3, round.wrapping_mul(7), round]); + } + assert_eq!(slots.free(), 4); + assert_eq!(bench.transport.device().commands, 42); +} + +#[test] +fn a_slot_has_one_holder() { + let mut bench = Bench::new(); + let (mut ring, mut slots) = bench.bring_up().unwrap(); + + assert!(slots.chain(0, 8).is_none()); + assert!(slots.chain(8, 0).is_none()); + assert!(slots.chain(65, 8).is_none(), "more than a slot's request"); + assert!(slots.chain(8, 33).is_none(), "more than a slot's response"); + assert_eq!(slots.free(), 4, "a refusal takes nothing"); + + let mut chains: Vec<_> = (0..4).map(|_| slots.chain(64, 32).unwrap()).collect(); + assert!(slots.chain(1, 1).is_none(), "every slot is taken"); + + // A chain's bytes are its slot's and no other's. + for (index, chain) in chains.iter_mut().enumerate() { + chain.write_device_readable(0, &[index as u8; 64]).unwrap(); + assert_eq!( + chain.write_device_readable(1, &[0; 64]), + Err(MemoryFault::Range) + ); + assert_eq!( + chain.read_device_writable(30, &mut [0; 3]), + Err(MemoryFault::Range) + ); + assert_eq!(chain.device_readable_len(), 64); + assert_eq!(chain.device_writable_len(), 32); + } + // All four in flight at once, answered in order. + let ids: Vec<_> = chains + .drain(..) + .map(|chain| ring.publish(chain).unwrap().id()) + .collect(); + bench.transport.service(BUDGET); + for (index, id) in ids.iter().enumerate() { + let (used, len, chain) = ring.pop_used().unwrap().unwrap().into_parts(); + assert_eq!(used, *id); + assert_eq!(len.get(), 32); + let mut answer = [0u8; 32]; + chain.read_device_writable(0, &mut answer).unwrap(); + assert_eq!(answer, [index as u8; 32]); + slots.recycle(chain); + } + assert!(ring.pop_used().unwrap().is_none()); + assert_eq!(slots.free(), 4); +} + +/// VQ-017 from the driver's side: suppression in either direction loses +/// nothing. +#[test] +fn suppressed_notifications_lose_no_work() { + let mut bench = Bench::new(); + let (mut ring, mut slots) = bench.bring_up().unwrap(); + let one = NonZeroU32::new(1).unwrap(); + + // The driver asks not to be interrupted, and is not. + ring.disable_used_notifications().unwrap(); + let mut chain = slots.chain(2, 8).unwrap(); + chain.write_device_readable(0, b"ab").unwrap(); + ring.publish(chain).unwrap(); + assert!(!bench.transport.service(BUDGET).interrupt); + // Enabling them again finds what was used meanwhile. + assert_eq!( + ring.enable_used_notifications().unwrap(), + NotificationRecheck::WorkPending + ); + let (_, len, chain) = ring.pop_used().unwrap().unwrap().into_parts(); + assert_eq!(len.get(), 2); + slots.recycle(chain); + assert_eq!( + ring.enable_used_notifications().unwrap(), + NotificationRecheck::Idle + ); + + // The device stops listening while its budget is spent: a chain + // published then needs no doorbell, and is served all the same. + for _ in 0..2 { + let mut chain = slots.chain(2, 8).unwrap(); + chain.write_device_readable(0, b"cd").unwrap(); + ring.publish(chain).unwrap(); + } + assert!(bench.transport.service(one).pending); + let mut chain = slots.chain(2, 8).unwrap(); + chain.write_device_readable(0, b"ef").unwrap(); + assert_eq!( + ring.publish(chain).unwrap().notification(), + NotificationHint::Suppressed + ); + while bench.transport.service(one).pending {} + let answers: Vec<_> = core::iter::from_fn(|| ring.pop_used().unwrap()) + .map(|used| { + let (_, _, chain) = used.into_parts(); + let mut answer = [0u8; 2]; + chain.read_device_writable(0, &mut answer).unwrap(); + slots.recycle(chain); + answer + }) + .collect(); + assert_eq!(answers, [*b"dc", *b"dc", *b"fe"]); +} + +#[test] +fn a_used_entry_is_checked_before_it_is_believed() { + // Each forgery: the entry the device's side of the memory is made to + // hold, as `(head, length)`. + for (what, head, len) in [ + ("a head never published", 2u32, 1u32), + ("a slot's second descriptor", 1, 1), + ("no descriptor at all", 4000, 1), + ("more than the room given", 0, 9), + ] { + let mut bench = Bench::new(); + let (mut ring, mut slots) = bench.bring_up().unwrap(); + let mut chain = slots.chain(2, 8).unwrap(); + chain.write_device_readable(0, b"ab").unwrap(); + ring.publish(chain).unwrap(); + + let used = layout().addresses().used; + bench.memory.store_u32(used + 4, head).unwrap(); + bench.memory.store_u32(used + 8, len).unwrap(); + bench.memory.store_u16(used + 2, 1).unwrap(); + assert_eq!( + ring.pop_used().map(|used| used.is_some()), + Err(QueueError::Transport(DriverRingError::UsedEntry)), + "{what}" + ); + } +} + +#[test] +fn a_refused_bring_up_says_which_step() { + // Not a device at all. + let blank = window(regs::BYTES as usize); + let memory = window(layout().bytes() as usize); + let bring = |registers: &Window| bring_up(registers, memory, layout(), || {}).map(|_| ()); + assert_eq!(bring(&blank), Err(BringUpError::NotADevice)); + blank.store_u32(regs::MAGIC, regs::MAGIC_VALUE).unwrap(); + blank.store_u32(regs::VERSION, 2).unwrap(); + assert_eq!(bring(&blank), Err(BringUpError::Version(2))); + + // A device that does not answer a reset. + blank.store_u32(regs::VERSION, regs::VERSION_VALUE).unwrap(); + blank + .store_u32(regs::DEVICE_STATUS, STATUS_DRIVER_OK) + .unwrap(); + assert_eq!(bring(&blank), Err(BringUpError::Reset)); + + // One that offers nothing a driver can accept, and one that never + // answers the features it was sent. The driver gives up on both. + blank.store_u32(regs::DEVICE_STATUS, 0).unwrap(); + assert_eq!(bring(&blank), Err(BringUpError::Features)); + assert_eq!(blank.load_u32(regs::DRIVER_STATUS), Ok(STATUS_FAILED)); + blank.store_u64(regs::DEVICE_FEATURES, F_VERSION_1).unwrap(); + assert_eq!(bring(&blank), Err(BringUpError::Features)); + assert_eq!(blank.load_u32(regs::DRIVER_STATUS), Ok(STATUS_FAILED)); + + // A queue larger than the device takes, and memory smaller than the + // layout. + let mut bench = Bench::new(); + let wide = Layout::new(QueueSize::new(16).unwrap(), 8, 8).unwrap(); + assert_eq!( + bench.bring_up_with(wide).map(|_| ()), + Err(BringUpError::Layout) + ); + let deep = Layout::new(QueueSize::new(8).unwrap(), 4096, 4096).unwrap(); + assert_eq!( + bench.bring_up_with(deep).map(|_| ()), + Err(BringUpError::Layout) + ); + assert_eq!( + bench.registers.load_u32(regs::DRIVER_STATUS), + Ok(STATUS_FAILED) + ); + // The device heard the driver give up, and a new bring-up starts over. + bench.transport.service(BUDGET); + assert_ne!(bench.transport.status() & STATUS_FAILED, 0); + bench.bring_up().expect("after a reset"); + + // A queue the device refuses: its chains are longer than a queue of + // two can hold. + let mut bench = Bench::new(); + let short = Layout::new(QueueSize::new(2).unwrap(), 8, 8).unwrap(); + assert_eq!( + bench.bring_up_with(short).map(|_| ()), + Err(BringUpError::Queue) + ); + assert_ne!(bench.transport.status() & STATUS_DEVICE_NEEDS_RESET, 0); + bench.bring_up().expect("after a reset"); +} + +/// A second bring-up is a reset: the device forgets, both rings start from +/// nothing, and what the first one gave is dead. +#[test] +fn bringing_up_again_resets_the_device() { + let mut bench = Bench::new(); + let (mut ring, mut slots) = bench.bring_up().unwrap(); + for _ in 0..3 { + bench.exchange(&mut ring, &mut slots, b"before", 8); + } + // One left with the device across the reset. + let mut chain = slots.chain(4, 8).unwrap(); + chain.write_device_readable(0, b"lost").unwrap(); + ring.publish(chain).unwrap(); + + let (mut again, mut pool) = bench.bring_up().unwrap(); + assert_eq!(bench.transport.device().resets, 1); + assert_eq!( + bench.transport.device().commands, + 3, + "the lost one never ran" + ); + assert_eq!(pool.free(), 4); + assert!(again.pop_used().unwrap().is_none(), "nothing from before"); + assert_eq!(bench.exchange(&mut again, &mut pool, b"after", 8), b"retfa"); + + // The old ring gives its chain back and serves no more. + let epoch = ring.state().epoch(); + assert_eq!( + ring.reset(epoch).map(|_| ()), + Err(QueueError::InvalidConfiguration( + QueueConfigError::NonIncreasingEpoch + )) + ); + let reclaimed = ring.reset(epoch.checked_next().unwrap()).unwrap(); + assert_eq!(reclaimed.len(), 1); + let (id, chain) = reclaimed.into_iter().next().unwrap().into_parts(); + assert_eq!(id.epoch(), epoch); + assert_eq!(chain.device_readable_len(), 4); + assert!(!ring.state().ready()); + assert_eq!( + ring.pop_used().map(|used| used.is_some()), + Err(QueueError::NotReady) + ); + let (returned, kind) = ring.publish(chain).unwrap_err().into_parts(); + assert_eq!(kind, PublishErrorKind::NotReady); + assert_eq!(returned.device_readable_len(), 4, "handed back unchanged"); +} diff --git a/native/crates/vdev/src/lib.rs b/native/crates/vdev/src/lib.rs new file mode 100644 index 0000000..29924f3 --- /dev/null +++ b/native/crates/vdev/src/lib.rs @@ -0,0 +1,43 @@ +//! # kore-vdev: virtual devices +//! +//! A virtual device is a device with no hardware behind it, hosted by a +//! userland service. A driver reaches it the way it reaches hardware, and +//! each of the three things a driver holds is ordinary shared memory or an +//! ordinary notice: +//! +//! - **registers** are a lease both sides map: the driver reads and writes +//! them as it would a BAR; +//! - **DMA** is a lease the driver owns and lends to the device. A device +//! address is an offset into it, and the device can touch nothing outside +//! it ([`mem`]); +//! - **an interrupt** is a notice the driver parks on. +//! +//! No kernel mechanism is added for any of it, and nothing is emulated: the +//! mappings are real, so the MMU is what a driver's access goes through. +//! +//! What this crate holds is the part every virtual device shares: +//! +//! - [`mem`], the device's view of memory it shares with a driver; +//! - [`regs`], the register block a driver maps and steps through; +//! - [`ring`], the split virtqueue laid out in the driver's memory; +//! - [`transport`], which runs the three for one driver on each doorbell; +//! - [`device`], the contract a device model implements. A model is a +//! separate crate (`kore-vdev-accel` is the first) and sees none of the +//! above: it is handed a request to read and a response to write; +//! - [`driver`], the driver's half of the same registers and queue, for a +//! driver written in Rust. A driver in C writes those steps by hand +//! against the layout in [`regs`]. + +#![no_std] + +extern crate alloc; + +pub mod device; +pub mod driver; +pub mod mem; +pub mod regs; +pub mod ring; +pub mod transport; + +#[cfg(test)] +extern crate std; diff --git a/native/crates/vdev/src/mem.rs b/native/crates/vdev/src/mem.rs new file mode 100644 index 0000000..3312483 --- /dev/null +++ b/native/crates/vdev/src/mem.rs @@ -0,0 +1,294 @@ +//! **The device's view of the driver's memory.** +//! +//! A driver lends a virtual device one window of memory and names places in +//! it by offset: the offset is the *device address*, the only address a +//! driver ever hands a device. The device reaches the window through +//! [`Memory`], which refuses anything outside it, so a driver's mistake, or +//! its malice, stays inside what it lent. +//! +//! The other side of the window is another task that may write any byte at +//! any moment. Nothing here ever forms a Rust reference into it: every +//! access is an atomic load or store, a snapshot the device then validates. +//! A snapshot is not coherent across words; the ring's publication points +//! ([`Memory::load_u16`], [`Memory::store_u16`]) are what order one side's +//! writes before the other's reads. + +use core::ptr::NonNull; +use core::sync::atomic::{AtomicU8, AtomicU16, AtomicU32, AtomicU64, Ordering}; + +/// Why a device address did not name memory the driver lent. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MemoryFault { + /// Some byte of the range is outside the window. + Range, + /// A word access named an address that is not a multiple of its size. + Alignment, +} + +/// A window of driver memory, addressed from zero. +pub trait Memory { + /// Bytes in the window: device addresses are `0..len`. + fn len(&self) -> u64; + + /// Whether the window is empty. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Whether `len` bytes at `at` lie inside the window. + fn contains(&self, at: u64, len: u64) -> bool { + at.checked_add(len).is_some_and(|end| end <= self.len()) + } + + /// Snapshot `target.len()` bytes at `at`. + fn read(&self, at: u64, target: &mut [u8]) -> Result<(), MemoryFault>; + + /// Store `source` at `at`. + fn write(&self, at: u64, source: &[u8]) -> Result<(), MemoryFault>; + + /// Load the little-endian `u16` at `at` with acquire ordering: every + /// write the other side made before it published this word is visible + /// to reads after this one. + fn load_u16(&self, at: u64) -> Result; + + /// Store a little-endian `u16` at `at` with release ordering: every + /// write this side made before it is visible to the other side once it + /// has loaded this word. + fn store_u16(&self, at: u64, value: u16) -> Result<(), MemoryFault>; + + /// Load the little-endian `u32` at `at`, acquire. + fn load_u32(&self, at: u64) -> Result; + + /// Store a little-endian `u32` at `at`, release. + fn store_u32(&self, at: u64, value: u32) -> Result<(), MemoryFault>; + + /// Load the little-endian `u64` at `at`, acquire. + fn load_u64(&self, at: u64) -> Result; + + /// Store a little-endian `u64` at `at`, release. + fn store_u64(&self, at: u64, value: u64) -> Result<(), MemoryFault>; +} + +/// A mapped window: `len` bytes at `base` in this task, shared with a driver. +/// +/// A plain view, copied freely: it owns nothing, and whoever made it keeps +/// the mapping alive ([`Window::new`]). +#[derive(Clone, Copy, Debug)] +pub struct Window { + base: NonNull, + len: usize, +} + +impl Window { + /// A view of `len` bytes at `base`. + /// + /// # Safety + /// `base` is aligned to eight bytes and `len` bytes from it are mapped + /// readable and writable in this task for as long as this view or any + /// copy of it is used. The bytes may be written by another task at any + /// time; nothing else in this task holds a Rust reference into them. + pub const unsafe fn new(base: NonNull, len: usize) -> Self { + Self { base, len } + } + + /// Where the window is mapped in this task. A host uses it to name a + /// word of the window to the kernel, never to form a reference. + pub const fn base(&self) -> NonNull { + self.base + } + + fn range(&self, at: u64, len: usize) -> Result<*mut u8, MemoryFault> { + if !self.contains(at, len as u64) { + return Err(MemoryFault::Range); + } + // SAFETY: `at + len <= self.len`, so the offset stays inside the + // mapping `new` was promised. + Ok(unsafe { self.base.as_ptr().add(at as usize) }) + } + + fn word(&self, at: u64, size: usize) -> Result<*mut u8, MemoryFault> { + if at % size as u64 != 0 { + return Err(MemoryFault::Alignment); + } + // `base` is eight-byte aligned, so an offset aligned to `size` is an + // address aligned to it. + self.range(at, size) + } +} + +/// Bytes moved a word at a time once both sides are word aligned. +const WORD: usize = core::mem::size_of::(); + +impl Memory for Window { + fn len(&self) -> u64 { + self.len as u64 + } + + fn read(&self, at: u64, target: &mut [u8]) -> Result<(), MemoryFault> { + let source = self.range(at, target.len())?; + // Each load is atomic, so a concurrent writer in the driver cannot + // make this a data race; what it can do is change the bytes, which + // is why the caller validates the copy and never the window. + let mut index = 0; + // SAFETY (all three loops): `source.add(index)` is inside the range + // checked above for every `index < target.len()`. + while index < target.len() && (source as usize + index) % WORD != 0 { + target[index] = + unsafe { AtomicU8::from_ptr(source.add(index)) }.load(Ordering::Relaxed); + index += 1; + } + while index + WORD <= target.len() { + let word = + unsafe { AtomicU64::from_ptr(source.add(index).cast()) }.load(Ordering::Relaxed); + target[index..index + WORD].copy_from_slice(&word.to_ne_bytes()); + index += WORD; + } + while index < target.len() { + target[index] = + unsafe { AtomicU8::from_ptr(source.add(index)) }.load(Ordering::Relaxed); + index += 1; + } + Ok(()) + } + + fn write(&self, at: u64, source: &[u8]) -> Result<(), MemoryFault> { + let target = self.range(at, source.len())?; + let mut index = 0; + // SAFETY (all three loops): as in `read`. + while index < source.len() && (target as usize + index) % WORD != 0 { + unsafe { AtomicU8::from_ptr(target.add(index)) } + .store(source[index], Ordering::Relaxed); + index += 1; + } + while index + WORD <= source.len() { + let mut word = [0u8; WORD]; + word.copy_from_slice(&source[index..index + WORD]); + unsafe { AtomicU64::from_ptr(target.add(index).cast()) } + .store(u64::from_ne_bytes(word), Ordering::Relaxed); + index += WORD; + } + while index < source.len() { + unsafe { AtomicU8::from_ptr(target.add(index)) } + .store(source[index], Ordering::Relaxed); + index += 1; + } + Ok(()) + } + + fn load_u16(&self, at: u64) -> Result { + let word = self.word(at, core::mem::size_of::())?; + // SAFETY: in range and aligned for a `u16`. + Ok(u16::from_le( + unsafe { AtomicU16::from_ptr(word.cast()) }.load(Ordering::Acquire), + )) + } + + fn store_u16(&self, at: u64, value: u16) -> Result<(), MemoryFault> { + let word = self.word(at, core::mem::size_of::())?; + // SAFETY: in range and aligned for a `u16`. + unsafe { AtomicU16::from_ptr(word.cast()) }.store(value.to_le(), Ordering::Release); + Ok(()) + } + + fn load_u32(&self, at: u64) -> Result { + let word = self.word(at, core::mem::size_of::())?; + // SAFETY: in range and aligned for a `u32`. + Ok(u32::from_le( + unsafe { AtomicU32::from_ptr(word.cast()) }.load(Ordering::Acquire), + )) + } + + fn store_u32(&self, at: u64, value: u32) -> Result<(), MemoryFault> { + let word = self.word(at, core::mem::size_of::())?; + // SAFETY: in range and aligned for a `u32`. + unsafe { AtomicU32::from_ptr(word.cast()) }.store(value.to_le(), Ordering::Release); + Ok(()) + } + + fn load_u64(&self, at: u64) -> Result { + let word = self.word(at, core::mem::size_of::())?; + // SAFETY: in range and aligned for a `u64`. + Ok(u64::from_le( + unsafe { AtomicU64::from_ptr(word.cast()) }.load(Ordering::Acquire), + )) + } + + fn store_u64(&self, at: u64, value: u64) -> Result<(), MemoryFault> { + let word = self.word(at, core::mem::size_of::())?; + // SAFETY: in range and aligned for a `u64`. + unsafe { AtomicU64::from_ptr(word.cast()) }.store(value.to_le(), Ordering::Release); + Ok(()) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use std::boxed::Box; + use std::vec; + + /// A window over host memory that lives for the test run. + pub(crate) fn window(bytes: usize) -> Window { + let words: &'static mut [u64] = Box::leak(vec![0u64; bytes.div_ceil(8)].into_boxed_slice()); + // SAFETY: a leaked, eight-byte aligned allocation of at least + // `bytes` bytes that nothing else references. + unsafe { Window::new(NonNull::new(words.as_mut_ptr().cast()).unwrap(), bytes) } + } + + #[test] + fn a_window_reaches_exactly_what_was_lent() { + let memory = window(4096); + assert_eq!(memory.len(), 4096); + assert!(memory.contains(4090, 6)); + assert!(!memory.contains(4090, 7)); + assert!(!memory.contains(u64::MAX, 2), "the end does not wrap"); + + let mut out = [0u8; 4]; + assert_eq!(memory.read(4093, &mut out), Err(MemoryFault::Range)); + assert_eq!(memory.write(4096, &[1]), Err(MemoryFault::Range)); + assert_eq!(memory.write(4096, &[]), Ok(()), "nothing, at the end"); + assert_eq!(memory.load_u16(4095), Err(MemoryFault::Alignment)); + assert_eq!(memory.load_u16(4096), Err(MemoryFault::Range)); + assert_eq!(memory.store_u32(2, 0), Err(MemoryFault::Alignment)); + assert_eq!(memory.load_u32(4096), Err(MemoryFault::Range)); + assert_eq!(memory.load_u64(4), Err(MemoryFault::Alignment)); + assert_eq!(memory.store_u64(4096, 0), Err(MemoryFault::Range)); + } + + /// Every alignment of a copy takes the byte head, the word body and the + /// byte tail in some mix, and all of them move the same bytes. + #[test] + fn a_copy_moves_the_same_bytes_at_every_alignment() { + let memory = window(256); + let pattern: std::vec::Vec = (0..64u8).map(|i| i.wrapping_mul(37) ^ 0x5a).collect(); + for at in 0..17u64 { + for len in [0usize, 1, 7, 8, 9, 15, 16, 17, 31, 40] { + memory.write(0, &[0xee; 96]).unwrap(); + memory.write(at, &pattern[..len]).unwrap(); + let mut back = vec![0u8; len]; + memory.read(at, &mut back).unwrap(); + assert_eq!(back, pattern[..len], "at {at}, {len} bytes"); + // The bytes either side are untouched. + let mut around = [0u8; 96]; + memory.read(0, &mut around).unwrap(); + assert!(around[..at as usize].iter().all(|b| *b == 0xee)); + assert!(around[at as usize + len..].iter().all(|b| *b == 0xee)); + } + } + } + + #[test] + fn words_are_little_endian_whatever_the_host_is() { + let memory = window(16); + memory.store_u16(2, 0x1234).unwrap(); + memory.store_u32(4, 0x89ab_cdef).unwrap(); + memory.store_u64(8, 0x0123_4567_89ab_cdef).unwrap(); + let mut bytes = [0u8; 16]; + memory.read(0, &mut bytes).unwrap(); + assert_eq!(bytes[..8], [0, 0, 0x34, 0x12, 0xef, 0xcd, 0xab, 0x89]); + assert_eq!(bytes[8..], [0xef, 0xcd, 0xab, 0x89, 0x67, 0x45, 0x23, 0x01]); + assert_eq!(memory.load_u16(2), Ok(0x1234)); + assert_eq!(memory.load_u32(4), Ok(0x89ab_cdef)); + assert_eq!(memory.load_u64(8), Ok(0x0123_4567_89ab_cdef)); + } +} diff --git a/native/crates/vdev/src/regs.rs b/native/crates/vdev/src/regs.rs new file mode 100644 index 0000000..c4717bc --- /dev/null +++ b/native/crates/vdev/src/regs.rs @@ -0,0 +1,119 @@ +//! **The register block**: what a driver maps where hardware would have a +//! BAR. +//! +//! One page both sides map. Each register belongs to one side: the driver +//! never writes a device register and the device never writes a driver +//! register, so no word has two writers. +//! +//! The layout follows Virtio's own transports in what it carries (identity, +//! feature words, a status the driver steps through, the queue's size and +//! three addresses, a device configuration area) and departs from +//! `virtio-mmio` in one way that is forced: that layout selects what a +//! register shows by a write to another register, which only works when a +//! write traps. A write to shared memory does not, so nothing here is behind +//! a selector, and the status is two words: the one the driver writes and +//! the one the device answers with. +//! +//! # The doorbell +//! +//! The device does not see a write happen. It looks at the driver's +//! registers and at the queue when the driver rings its **doorbell**, the +//! transport's one notification, and it raises the driver's **interrupt** +//! when it has changed [`DEVICE_STATUS`] or used a chain. Virtio leaves +//! notification to each transport (a register on `virtio-mmio`, a hypercall +//! on channel I/O); here it is a notice on a channel, in both directions. +//! +//! The interrupt is a count the driver can sleep on: [`INTERRUPTS`] moves +//! each time the device raises one, so a driver reads it, does its work, +//! and parks until it differs from what it read. An interrupt raised +//! between the read and the park is not lost, because the park compares. +//! +//! A doorbell that announces a write to [`DRIVER_STATUS`] is always answered +//! with an interrupt, whether or not the device took the step: a driver +//! writes its status, rings, waits for the interrupt and reads +//! [`DEVICE_STATUS`]. A step the device refused is one its status does not +//! carry. +//! +//! # Bringing a device up +//! +//! 1. Check [`MAGIC`] and [`VERSION`], read [`DEVICE_ID`]. +//! 2. Write [`STATUS_ACKNOWLEDGE`]` | `[`STATUS_DRIVER`] to +//! [`DRIVER_STATUS`]. +//! 3. Read [`DEVICE_FEATURES`], write the accepted subset to +//! [`DRIVER_FEATURES`], add [`STATUS_FEATURES_OK`], ring the doorbell, +//! and wait for the interrupt. [`DEVICE_STATUS`] carries +//! `FEATURES_OK` only if the device accepted the subset. +//! 4. Lay the queue out in DMA memory; write [`QUEUE_NUM`], the three +//! addresses and [`QUEUE_READY`]. +//! 5. Add [`STATUS_DRIVER_OK`], ring, wait. The device is live when +//! [`DEVICE_STATUS`] carries it. +//! +//! Writing zero to [`DRIVER_STATUS`] and ringing resets the device; the +//! reset is done when [`DEVICE_STATUS`] reads zero. A device that can no +//! longer serve sets [`STATUS_DEVICE_NEEDS_RESET`] there. + +/// Bytes in the register block: one page. +pub const BYTES: u64 = 0x1000; + +/// `u32`, device. [`MAGIC_VALUE`]: this is a kore virtual device. +pub const MAGIC: u64 = 0x00; +/// `u32`, device. [`VERSION_VALUE`]: the layout in this module. +pub const VERSION: u64 = 0x04; +/// `u32`, device. Which device model this is; see the model's own crate. +pub const DEVICE_ID: u64 = 0x08; +/// `u32`, device. Who made it: [`VENDOR_KORE`] for a device kore hosts. +pub const VENDOR_ID: u64 = 0x0c; +/// `u64`, device. The features the device offers. +pub const DEVICE_FEATURES: u64 = 0x10; +/// `u64`, driver. The features the driver accepts, a subset of the offer. +pub const DRIVER_FEATURES: u64 = 0x18; +/// `u32`, driver. The status the driver has reached (`STATUS_*`). +pub const DRIVER_STATUS: u64 = 0x20; +/// `u32`, device. The status the device has accepted, and its own +/// [`STATUS_DEVICE_NEEDS_RESET`]. +pub const DEVICE_STATUS: u64 = 0x24; +/// `u32`, device. The largest queue the device takes. +pub const QUEUE_NUM_MAX: u64 = 0x28; +/// `u32`, driver. The queue's size: a power of two, at most the maximum. +pub const QUEUE_NUM: u64 = 0x2c; +/// `u32`, driver. Nonzero once the queue's size and addresses are written. +pub const QUEUE_READY: u64 = 0x30; +/// `u32`, device. Changes whenever the configuration area does. +pub const CONFIG_GENERATION: u64 = 0x34; +/// `u64`, driver. The descriptor table's device address. +pub const QUEUE_DESC: u64 = 0x38; +/// `u64`, driver. The available ring's device address. +pub const QUEUE_DRIVER: u64 = 0x40; +/// `u64`, driver. The used ring's device address. +pub const QUEUE_DEVICE: u64 = 0x48; +/// `u32`, device. Bytes of device configuration at [`CONFIG`]. +pub const CONFIG_BYTES: u64 = 0x50; +/// `u32`, device. Interrupts the device has raised, wrapping: the word a +/// driver parks on. +pub const INTERRUPTS: u64 = 0x54; +/// The device configuration area, device-written, to the end of the page. +pub const CONFIG: u64 = 0x100; + +/// `"kvdv"`, little-endian: a kore virtual device. +pub const MAGIC_VALUE: u32 = u32::from_le_bytes(*b"kvdv"); +/// The layout's version. +pub const VERSION_VALUE: u32 = 1; +/// `"kore"`, little-endian. +pub const VENDOR_KORE: u32 = u32::from_le_bytes(*b"kore"); + +/// The driver has noticed the device (Virtio 1.3 §2.1). +pub const STATUS_ACKNOWLEDGE: u32 = 1; +/// The driver knows how to drive it. +pub const STATUS_DRIVER: u32 = 2; +/// The driver is set up and the device may serve. +pub const STATUS_DRIVER_OK: u32 = 4; +/// Feature negotiation is complete. +pub const STATUS_FEATURES_OK: u32 = 8; +/// The device cannot go on and must be reset. Device-written. +pub const STATUS_DEVICE_NEEDS_RESET: u32 = 64; +/// The driver has given up on the device. +pub const STATUS_FAILED: u32 = 128; + +/// The device and driver follow Virtio 1.0 or later (feature bit 32). A +/// device offers it and a driver must accept it. +pub const F_VERSION_1: u64 = 1 << 32; diff --git a/native/crates/vdev/src/ring.rs b/native/crates/vdev/src/ring.rs new file mode 100644 index 0000000..2b5da72 --- /dev/null +++ b/native/crates/vdev/src/ring.rs @@ -0,0 +1,630 @@ +//! **The split virtqueue, in the driver's memory**: the device's half. +//! +//! A driver lays a queue out in the memory it lends: a descriptor table, an +//! available ring it writes and a used ring the device writes, in the split +//! layout of Virtio 1.3 §2.7. It tells the device where the three are by +//! device address, publishes a chain of descriptors by adding its head to the +//! available ring, and takes the chain back from the used ring with the +//! number of bytes the device wrote. +//! +//! [`DeviceRing`] is the device's half of that, behind the queue contracts of +//! `virtio-accel-transport` ([`DeviceQueue`], [`DeviceChain`]). The protocol +//! ships a queue for the same contracts, `virtio-accel-split-queue`, but it +//! is a model that owns its rings and every descriptor's bytes on the heap of +//! one task. A driver and a device in two address spaces need the rings in +//! memory both map, which is this. The model is the oracle: the tests here +//! put each chain shape through both and require the same answer. +//! +//! # What the device trusts +//! +//! Nothing the driver wrote. Every index is checked against the queue's size, +//! a chain is walked at most `size` descriptors with a visited set so a loop +//! ends, every span must lie inside the window ([`Memory`]), and a chain is +//! read **once**: the walk snapshots each descriptor, and the ports then use +//! the snapshot, so a driver that rewrites a descriptor after publishing it +//! changes nothing the device does. A chain that is malformed is not an +//! error of the queue: it is handed up like any other, its [`io`] says why, +//! and the device completes it with a used length of zero, as the protocol's +//! command-chain rules require. +//! +//! The profile is the protocol's: direct descriptors only, no indirect +//! tables, no event indices, and the two suppression flags. +//! +//! [`io`]: DeviceChain::io + +use alloc::rc::Rc; +use core::sync::atomic::{AtomicU64, Ordering, fence}; + +use virtio_accel_transport::{ + ByteAccessError, ChainError, ChainId, ChainIo, ChainIoResult, ChainRegion, DeviceChain, + DeviceQueue, MalformedChain, NotificationHint, NotificationRecheck, QueueConfigError, + QueueControl, QueueEpoch, QueueError, QueuePort, QueueSize, QueueState, ReadableBytes, + UsedLength, WritableBytes, validate_chain_layout, +}; + +use crate::mem::{Memory, MemoryFault}; + +/// The largest queue a device offers. +/// +/// A chain is walked with a visited set one machine word wide, so the size +/// is bounded by it; sixty-four descriptors is four of the longest command +/// chains, or thirty-two two-descriptor commands, in flight at once. +pub const MAX_QUEUE_SIZE: u16 = u64::BITS as u16; + +/// One descriptor: `addr: u64, len: u32, flags: u16, next: u16` +/// (Virtio 1.3 §2.7.5). +pub(crate) const DESC_BYTES: u64 = 16; +const DESC_LEN: usize = 8; +const DESC_FLAGS: usize = 12; +const DESC_NEXT: usize = 14; +/// The chain continues at `next`. +pub const VIRTQ_DESC_F_NEXT: u16 = 1 << 0; +/// The device writes this descriptor's bytes; otherwise it reads them. +pub const VIRTQ_DESC_F_WRITE: u16 = 1 << 1; +/// The descriptor names an indirect table: not in this profile. +pub const VIRTQ_DESC_F_INDIRECT: u16 = 1 << 2; + +/// Both rings begin `flags: u16, idx: u16` (§2.7.6, §2.7.8). +pub(crate) const RING_FLAGS: u64 = 0; +pub(crate) const RING_IDX: u64 = 2; +pub(crate) const RING_HEADER: u64 = 4; +/// An available-ring entry is a descriptor index. +pub(crate) const AVAIL_ENTRY: u64 = 2; +/// A used-ring entry is `id: u32, len: u32`. +pub(crate) const USED_ENTRY: u64 = 8; +pub(crate) const USED_LEN: u64 = 4; +/// The driver asks not to be interrupted when the device uses a chain. +pub const VIRTQ_AVAIL_F_NO_INTERRUPT: u16 = 1 << 0; +/// The device asks not to be notified when the driver publishes a chain. +pub const VIRTQ_USED_F_NO_NOTIFY: u16 = 1 << 0; + +/// Alignments the layout requires of each part (§2.7's table). +pub(crate) const DESC_ALIGN: u64 = 16; +pub(crate) const AVAIL_ALIGN: u64 = 2; +pub(crate) const USED_ALIGN: u64 = 4; + +/// Where a driver put a queue's three parts, as device addresses. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RingAddresses { + pub descriptors: u64, + pub available: u64, + pub used: u64, +} + +/// Why the ring could not go on. Each is the driver's doing, and none is one +/// chain's: a device that sees one stops serving the queue and asks for a +/// reset. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RingError { + /// A part of the queue is not where the layout allows: outside the + /// window, or not aligned for its entries. + Layout, + /// The queue was made ready smaller than the longest chain it must hold. + ChainLimitExceedsQueue, + /// The queue cannot be reconfigured while it is ready. + Busy, + /// The available index is further ahead than the queue is long. + AvailableIndex, + /// A head was published that the device already holds. + HeadInFlight, + /// The window stopped answering for a ring the queue validated. + Memory(MemoryFault), +} + +impl From for QueueError { + fn from(fault: MemoryFault) -> Self { + Self::Transport(RingError::Memory(fault)) + } +} + +/// Whether a ring can hold a queue of at most `max_size` whose chains are at +/// most `max_chain_descriptors` long: a chain is two descriptors at least, +/// and no longer than the queue. +pub const fn limits_fit(max_size: QueueSize, max_chain_descriptors: u16) -> bool { + max_size.get() <= MAX_QUEUE_SIZE + && max_chain_descriptors >= 2 + && max_chain_descriptors <= max_size.get() +} + +/// One descriptor's bytes, snapshotted when its chain was taken. +#[derive(Clone, Copy, Debug)] +struct Span { + at: u64, + len: u32, +} + +/// The chain's spans in one direction, as one run of bytes. +#[derive(Clone, Debug)] +struct Port { + memory: M, + spans: heapless::Vec, + len: u64, + epoch: QueueEpoch, + current: Rc, +} + +impl Port { + /// The spans `len` bytes at `offset` fall in, as `(address, from, count)` + /// with `from` the offset into the caller's buffer. + fn each( + &self, + offset: u64, + len: usize, + mut visit: impl FnMut(u64, usize, usize) -> Result<(), MemoryFault>, + ) -> Result<(), ByteAccessError> { + if self.current.load(Ordering::Acquire) != self.epoch.get() { + return Err(ByteAccessError::Reset); + } + if !offset + .checked_add(len as u64) + .is_some_and(|end| end <= self.len) + { + return Err(ByteAccessError::OutOfBounds); + } + let (mut skip, mut done) = (offset, 0usize); + for span in &self.spans { + let span_len = u64::from(span.len); + if skip >= span_len { + skip -= span_len; + continue; + } + if done == len { + break; + } + let count = ((span_len - skip) as usize).min(len - done); + visit(span.at + skip, done, count).map_err(|_| ByteAccessError::Access)?; + done += count; + skip = 0; + } + Ok(()) + } +} + +/// The bytes a chain gives the device to read. +#[derive(Debug)] +pub struct RingSource(Port); + +impl ReadableBytes for RingSource { + fn len(&self) -> u64 { + self.0.len + } + + fn read_at(&self, offset: u64, target: &mut [u8]) -> Result<(), ByteAccessError> { + let memory = &self.0.memory; + let len = target.len(); + self.0.each(offset, len, |at, from, count| { + memory.read(at, &mut target[from..from + count]) + }) + } +} + +/// The bytes a chain gives the device to write. +#[derive(Debug)] +pub struct RingSink(Port); + +impl WritableBytes for RingSink { + fn len(&self) -> u64 { + self.0.len + } + + fn write_at(&mut self, offset: u64, source: &[u8]) -> Result<(), ByteAccessError> { + let memory = &self.0.memory; + self.0.each(offset, source.len(), |at, from, count| { + memory.write(at, &source[from..from + count]) + }) + } +} + +/// One chain the device took from the available ring and has not yet used. +/// +/// Not `Clone`: [`DeviceRing::complete`] consumes it, so a chain is used +/// once. +#[derive(Debug)] +pub struct RingChain { + id: ChainId, + head: u16, + regions: heapless::Vec, + malformed: Option, + request: RingSource, + response: RingSink, +} + +impl DeviceChain for RingChain { + type Request = RingSource; + type Response = RingSink; + type Error = RingError; + + fn id(&self) -> ChainId { + self.id + } + + fn io(&mut self) -> ChainIoResult<'_, Self::Request, Self::Response, Self::Error> { + let current = self.request.0.current.load(Ordering::Acquire); + if current != self.id.epoch().get() { + return Err(ChainError::ResetRace { + chain: self.id.epoch(), + // An epoch is never zero: it starts at one and only grows. + current: QueueEpoch::new(current).unwrap_or(QueueEpoch::INITIAL), + }); + } + if let Some(malformed) = self.malformed { + return Err(ChainError::Malformed(malformed)); + } + Ok(ChainIo::new( + &self.regions, + &self.request, + &mut self.response, + )) + } +} + +/// The device's half of one split virtqueue in a driver's memory. +#[derive(Debug)] +pub struct DeviceRing { + memory: M, + state: QueueState, + max_chain_descriptors: u16, + addresses: RingAddresses, + /// The epoch every chain this queue hands out carries, shared with them + /// so a reset reaches a chain the device still holds. + current: Rc, + /// Available entries taken, and used entries published. + taken: u16, + used: u16, + /// Heads the device holds, one bit each. + in_flight: u64, +} + +impl DeviceRing { + /// A queue in `memory` of at most `max_size` descriptors, whose chains + /// are at most `max_chain_descriptors` long. Unconfigured: the driver + /// sets its size and addresses and makes it ready. + /// + /// `None` when the limits do not fit: a chain is two descriptors at + /// least, and no longer than the queue. + pub fn new(memory: M, max_size: QueueSize, max_chain_descriptors: u16) -> Option { + if !limits_fit(max_size, max_chain_descriptors) { + return None; + } + let epoch = QueueEpoch::INITIAL; + Some(Self { + memory, + state: QueueState::unconfigured(max_size, epoch), + max_chain_descriptors, + addresses: RingAddresses::default(), + current: Rc::new(AtomicU64::new(epoch.get())), + taken: 0, + used: 0, + in_flight: 0, + }) + } + + /// The window the queue is in, given back: the queue is over. + pub fn into_memory(self) -> M { + self.memory + } + + /// Where the driver says the queue's parts are. Taken while the queue is + /// not ready, and checked when it is made so. + pub fn set_addresses(&mut self, addresses: RingAddresses) -> Result<(), QueueError> { + if self.state.ready() { + return Err(QueueError::Transport(RingError::Busy)); + } + self.addresses = addresses; + Ok(()) + } + + /// The addresses the queue was given. + pub const fn addresses(&self) -> RingAddresses { + self.addresses + } + + /// Chains the device holds: taken and not yet used. + pub const fn in_flight(&self) -> u32 { + self.in_flight.count_ones() + } + + fn size(&self) -> Result> { + match (self.state.ready(), self.state.size()) { + (true, Some(size)) => Ok(size.get()), + _ => Err(QueueError::NotReady), + } + } + + /// Whether the three parts lie inside the window where the layout + /// allows them, for a queue of `size`. + fn layout_holds(&self, size: u16) -> bool { + let RingAddresses { + descriptors, + available, + used, + } = self.addresses; + let size = u64::from(size); + descriptors % DESC_ALIGN == 0 + && available % AVAIL_ALIGN == 0 + && used % USED_ALIGN == 0 + && self.memory.contains(descriptors, size * DESC_BYTES) + && self + .memory + .contains(available, RING_HEADER + size * AVAIL_ENTRY) + && self.memory.contains(used, RING_HEADER + size * USED_ENTRY) + } + + /// Walk the chain at `head`, snapshotting each descriptor once. + fn take(&self, head: u16, size: u16) -> Result, QueueError> { + let epoch = self.state.epoch(); + let port = || Port { + memory: self.memory.clone(), + spans: heapless::Vec::new(), + len: 0, + epoch, + current: Rc::clone(&self.current), + }; + let mut chain = RingChain { + id: ChainId::new(epoch, u64::from(head)), + head, + regions: heapless::Vec::new(), + malformed: None, + request: RingSource(port()), + response: RingSink(port()), + }; + chain.malformed = self.walk(&mut chain, head, size)?.err(); + Ok(chain) + } + + /// The walk proper. The outer error is the queue's own memory failing; + /// the inner one is this chain being malformed, which is the chain's + /// business and not the queue's. + fn walk( + &self, + chain: &mut RingChain, + head: u16, + size: u16, + ) -> Result, QueueError> { + let (mut next, mut seen) = (head, 0u64); + // At most `size` descriptors, a hostile loop included: the visited + // set ends it on the first index met twice. + for _ in 0..size { + if next >= size { + return Ok(Err(MalformedChain::DescriptorIndex)); + } + if seen & (1 << next) != 0 { + return Ok(Err(MalformedChain::DescriptorLoop)); + } + seen |= 1 << next; + + let mut descriptor = [0u8; DESC_BYTES as usize]; + self.memory.read( + self.addresses.descriptors + u64::from(next) * DESC_BYTES, + &mut descriptor, + )?; + let word = |at: usize, len: usize| { + let mut bytes = [0u8; 8]; + bytes[..len].copy_from_slice(&descriptor[at..at + len]); + u64::from_le_bytes(bytes) + }; + let (at, len) = (word(0, 8), word(DESC_LEN, 4) as u32); + let flags = word(DESC_FLAGS, 2) as u16; + next = word(DESC_NEXT, 2) as u16; + + if flags & VIRTQ_DESC_F_INDIRECT != 0 { + return Ok(Err(MalformedChain::IndirectUnsupported)); + } + if flags & !(VIRTQ_DESC_F_NEXT | VIRTQ_DESC_F_WRITE) != 0 { + return Ok(Err(MalformedChain::DescriptorFlags)); + } + let writable = flags & VIRTQ_DESC_F_WRITE != 0; + let region = if writable { + ChainRegion::writable(u64::from(len)) + } else { + ChainRegion::readable(u64::from(len)) + }; + // The vectors hold `MAX_QUEUE_SIZE`, which bounds `size`. + let _ = chain.regions.push(region); + let port = if writable { + &mut chain.response.0 + } else { + &mut chain.request.0 + }; + let _ = port.spans.push(Span { at, len }); + port.len += u64::from(len); + + if flags & VIRTQ_DESC_F_NEXT == 0 { + // Count, zero lengths, direction order and totals are the + // protocol's own rule, by its own function. + if let Err(error) = + validate_chain_layout(&chain.regions, self.max_chain_descriptors) + { + return Ok(Err(error.into())); + } + // Every byte the engine may touch is mapped before it runs. + let mapped = |port: &Port| { + port.spans + .iter() + .all(|span| self.memory.contains(span.at, u64::from(span.len))) + }; + if !mapped(&chain.request.0) || !mapped(&chain.response.0) { + return Ok(Err(MalformedChain::Address)); + } + return Ok(Ok(())); + } + } + // `size` descriptors and still a `next`: longer than any chain. + Ok(Err(MalformedChain::DescriptorCount)) + } +} + +impl QueuePort for DeviceRing { + fn state(&self) -> QueueState { + self.state + } +} + +impl QueueControl for DeviceRing { + type Error = RingError; + + fn configure(&mut self, size: QueueSize) -> Result<(), QueueError> { + if self.state.ready() { + return Err(QueueError::Transport(RingError::Busy)); + } + if size > self.state.max_size() { + return Err(QueueError::InvalidConfiguration( + QueueConfigError::SizeExceedsMaximum, + )); + } + if size.get() < self.max_chain_descriptors { + return Err(QueueError::Transport(RingError::ChainLimitExceedsQueue)); + } + self.state = QueueState::new(self.state.max_size(), Some(size), false, self.state.epoch()) + .map_err(QueueError::InvalidConfiguration)?; + Ok(()) + } + + fn set_ready(&mut self, ready: bool) -> Result<(), QueueError> { + if ready { + let Some(size) = self.state.size() else { + return Err(QueueError::InvalidConfiguration( + QueueConfigError::ReadyWithoutSize, + )); + }; + if !self.layout_holds(size.get()) { + return Err(QueueError::Transport(RingError::Layout)); + } + } + self.state = QueueState::new( + self.state.max_size(), + self.state.size(), + ready, + self.state.epoch(), + ) + .map_err(QueueError::InvalidConfiguration)?; + Ok(()) + } +} + +impl DeviceQueue for DeviceRing { + type Chain = RingChain; + type Error = RingError; + + fn pop_available(&mut self) -> Result, QueueError> { + let size = self.size()?; + // Acquire: the entry and the descriptors the driver wrote before it + // published this index are visible to the reads below. + let published = self.memory.load_u16(self.addresses.available + RING_IDX)?; + let pending = published.wrapping_sub(self.taken); + if pending == 0 { + return Ok(None); + } + if pending > size { + return Err(QueueError::Transport(RingError::AvailableIndex)); + } + let slot = u64::from(self.taken % size); + let mut entry = [0u8; AVAIL_ENTRY as usize]; + self.memory.read( + self.addresses.available + RING_HEADER + slot * AVAIL_ENTRY, + &mut entry, + )?; + let head = u16::from_le_bytes(entry); + self.taken = self.taken.wrapping_add(1); + + let chain = self.take(head, size)?; + if head < size { + // The head is a real descriptor: the device holds it until it + // is used, and a second publication of it is the driver reusing + // a descriptor it does not own. + if self.in_flight & (1 << head) != 0 { + return Err(QueueError::Transport(RingError::HeadInFlight)); + } + self.in_flight |= 1 << head; + } + Ok(Some(chain)) + } + + fn complete( + &mut self, + chain: Self::Chain, + used: UsedLength, + ) -> Result> { + let current = self.state.epoch(); + if chain.id.epoch() != current { + return Err(QueueError::ResetRace { + operation: chain.id.epoch(), + current, + }); + } + let size = self.size()?; + let capacity = chain.response.0.len; + if u64::from(used.get()) > capacity { + return Err(QueueError::UsedLengthExceeded { used, capacity }); + } + let slot = u64::from(self.used % size); + let entry = self.addresses.used + RING_HEADER + slot * USED_ENTRY; + self.memory.store_u32(entry, u32::from(chain.head))?; + self.memory.store_u32(entry + USED_LEN, used.get())?; + self.used = self.used.wrapping_add(1); + // Release: the response bytes and the entry above are visible to a + // driver that has loaded this index. + self.memory + .store_u16(self.addresses.used + RING_IDX, self.used)?; + if u32::from(chain.head) < u64::BITS { + self.in_flight &= !(1 << chain.head); + } + // The index is published before the driver's suppression flag is + // read: a driver that clears the flag and then rechecks the index + // cannot miss this entry, and one that set it is not interrupted. + fence(Ordering::SeqCst); + let flags = self + .memory + .load_u16(self.addresses.available + RING_FLAGS)?; + Ok(if flags & VIRTQ_AVAIL_F_NO_INTERRUPT == 0 { + NotificationHint::Notify + } else { + NotificationHint::Suppressed + }) + } + + fn disable_available_notifications(&mut self) -> Result<(), QueueError> { + self.size()?; + self.memory + .store_u16(self.addresses.used + RING_FLAGS, VIRTQ_USED_F_NO_NOTIFY)?; + Ok(()) + } + + fn enable_available_notifications( + &mut self, + ) -> Result> { + self.size()?; + self.memory.store_u16(self.addresses.used + RING_FLAGS, 0)?; + // The flag is cleared before the index is read: a driver that + // published while notifications were off is seen here, and one that + // publishes after sees the flag clear and notifies. + fence(Ordering::SeqCst); + let published = self.memory.load_u16(self.addresses.available + RING_IDX)?; + Ok(if published == self.taken { + NotificationRecheck::Idle + } else { + NotificationRecheck::WorkPending + }) + } + + fn reset(&mut self, next_epoch: QueueEpoch) -> Result<(), QueueError> { + if next_epoch <= self.state.epoch() { + return Err(QueueError::InvalidConfiguration( + QueueConfigError::NonIncreasingEpoch, + )); + } + // Published first: a chain the device still holds refuses its bytes + // and its completion from here on, so nothing of the old epoch is + // written to memory the driver may now reuse. + self.current.store(next_epoch.get(), Ordering::Release); + self.state = QueueState::unconfigured(self.state.max_size(), next_epoch); + self.addresses = RingAddresses::default(); + self.taken = 0; + self.used = 0; + self.in_flight = 0; + Ok(()) + } +} + +#[cfg(test)] +pub(crate) mod tests; diff --git a/native/crates/vdev/src/ring/tests.rs b/native/crates/vdev/src/ring/tests.rs new file mode 100644 index 0000000..c69997c --- /dev/null +++ b/native/crates/vdev/src/ring/tests.rs @@ -0,0 +1,683 @@ +use super::*; +use crate::mem::{Window, tests::window}; +use std::vec; +use std::vec::Vec; +use virtio_accel_split_queue::{Descriptor, DriverChain, SplitQueue}; +use virtio_accel_transport::RegionDirection; + +/// One descriptor a test driver writes: readable bytes, or room to write. +#[derive(Clone, Copy)] +pub(crate) enum Part<'a> { + Read(&'a [u8]), + Write(u32), +} + +/// A driver's half of the queue, written the way a C driver would write it: +/// the three parts laid out in its own memory, and descriptors, available +/// entries and used entries moved by hand. +pub(crate) struct Guest { + pub(crate) memory: Window, + pub(crate) size: u16, + pub(crate) at: RingAddresses, + /// Where the next buffer goes: buffers are never reused in a test. + bump: u64, + published: u16, + seen: u16, +} + +pub(crate) const WINDOW: usize = 64 * 1024; +const BUFFERS: u64 = 0x2000; + +impl Guest { + /// A queue of `size` whose chains are at most `max_chain` long, ready. + fn ready(size: u16, max_chain: u16) -> (Self, DeviceRing) { + let (guest, mut ring) = Self::unready(size, max_chain); + ring.set_ready(true).expect("a layout inside the window"); + (guest, ring) + } + + fn unready(size: u16, max_chain: u16) -> (Self, DeviceRing) { + let guest = Self::over(window(WINDOW), size); + let mut ring = DeviceRing::new( + guest.memory, + QueueSize::new(MAX_QUEUE_SIZE).unwrap(), + max_chain, + ) + .expect("limits fit"); + ring.configure(QueueSize::new(size).unwrap()) + .expect("configured"); + ring.set_addresses(guest.at).expect("addresses"); + (guest, ring) + } + + /// A driver's queue of `size` laid out in `memory`, which no device has + /// been told about yet. + pub(crate) fn over(memory: Window, size: u16) -> Self { + Self { + memory, + size, + at: RingAddresses { + descriptors: 0, + available: 0x800, + used: 0x1000, + }, + bump: BUFFERS, + published: 0, + seen: 0, + } + } + + pub(crate) fn descriptor(&self, index: u16, addr: u64, len: u32, flags: u16, next: u16) { + let mut bytes = [0u8; 16]; + bytes[..8].copy_from_slice(&addr.to_le_bytes()); + bytes[8..12].copy_from_slice(&len.to_le_bytes()); + bytes[12..14].copy_from_slice(&flags.to_le_bytes()); + bytes[14..].copy_from_slice(&next.to_le_bytes()); + self.memory + .write(self.at.descriptors + u64::from(index) * 16, &bytes) + .unwrap(); + } + + fn space(&mut self, len: u32) -> u64 { + let at = self.bump; + self.bump += u64::from(len.max(1)).next_multiple_of(8); + at + } + + /// Write `parts` as a direct chain in descriptors `first..`, and answer + /// where each part's bytes are. + pub(crate) fn chain(&mut self, first: u16, parts: &[Part<'_>]) -> Vec { + let mut places = Vec::new(); + for (offset, part) in parts.iter().enumerate() { + let index = first + offset as u16; + let last = offset + 1 == parts.len(); + let (len, mut flags) = match part { + Part::Read(bytes) => (bytes.len() as u32, 0), + Part::Write(len) => (*len, VIRTQ_DESC_F_WRITE), + }; + let at = self.space(len); + if let Part::Read(bytes) = part { + self.memory.write(at, bytes).unwrap(); + } + if !last { + flags |= VIRTQ_DESC_F_NEXT; + } + self.descriptor(index, at, len, flags, if last { 0 } else { index + 1 }); + places.push(at); + } + places + } + + /// Add `head` to the available ring and publish it. + pub(crate) fn publish(&mut self, head: u16) { + let slot = u64::from(self.published % self.size); + self.memory + .write(self.at.available + 4 + slot * 2, &head.to_le_bytes()) + .unwrap(); + self.published = self.published.wrapping_add(1); + self.memory + .store_u16(self.at.available + 2, self.published) + .unwrap(); + } + + /// The next used entry the driver has not seen: `(head, bytes written)`. + pub(crate) fn used(&mut self) -> Option<(u32, u32)> { + if self.memory.load_u16(self.at.used + 2).unwrap() == self.seen { + return None; + } + let slot = u64::from(self.seen % self.size); + let entry = self.at.used + 4 + slot * 8; + self.seen = self.seen.wrapping_add(1); + Some(( + self.memory.load_u32(entry).unwrap(), + self.memory.load_u32(entry + 4).unwrap(), + )) + } + + pub(crate) fn read(&self, at: u64, len: usize) -> Vec { + let mut bytes = vec![0u8; len]; + self.memory.read(at, &mut bytes).unwrap(); + bytes + } +} + +fn request(chain: &mut RingChain) -> Vec { + let (_, source, _) = chain.io().expect("a valid chain").into_parts(); + let mut bytes = vec![0u8; source.len() as usize]; + source.read_at(0, &mut bytes).unwrap(); + bytes +} + +/// VQ-001, VQ-003, VQ-013: a command crosses in readable descriptors and its +/// response comes back in writable ones, each side one run of bytes whatever +/// the descriptors' split, with the used length exactly what was written. +#[test] +fn a_chain_carries_a_request_out_and_a_response_back() { + let (mut guest, mut ring) = Guest::ready(8, 8); + assert!(ring.pop_available().unwrap().is_none()); + + let places = guest.chain( + 2, + &[ + Part::Read(b"hello, "), + Part::Read(b"device"), + Part::Write(5), + Part::Write(16), + ], + ); + guest.publish(2); + + let mut chain = ring.pop_available().unwrap().expect("one chain"); + assert_eq!(ring.in_flight(), 1); + assert_eq!(request(&mut chain), b"hello, device"); + { + let (regions, source, sink) = chain.io().unwrap().into_parts(); + let shape: Vec<_> = regions.iter().map(|r| (r.direction, r.bytes)).collect(); + assert_eq!( + shape, + [ + (RegionDirection::DeviceReadable, 7), + (RegionDirection::DeviceReadable, 6), + (RegionDirection::DeviceWritable, 5), + (RegionDirection::DeviceWritable, 16), + ] + ); + // A read may start and end anywhere, across the split. + let mut middle = [0u8; 6]; + source.read_at(4, &mut middle).unwrap(); + assert_eq!(&middle, b"o, dev"); + assert_eq!( + source.read_at(8, &mut middle), + Err(ByteAccessError::OutOfBounds) + ); + // The response crosses its split the same way. + assert_eq!(sink.len(), 21); + sink.write_at(0, b"hi, driver").unwrap(); + assert_eq!(sink.write_at(20, b"xy"), Err(ByteAccessError::OutOfBounds)); + } + assert_eq!( + ring.complete(chain, UsedLength::new(10)), + Ok(NotificationHint::Notify) + ); + assert_eq!(ring.in_flight(), 0); + assert_eq!(guest.used(), Some((2, 10))); + assert_eq!(guest.used(), None); + assert_eq!(guest.read(places[2], 5), b"hi, d"); + assert_eq!(guest.read(places[3], 5), b"river"); + // Room past the response is untouched. + assert_eq!(guest.read(places[3] + 5, 11), [0u8; 11]); +} + +/// A chain is read once. A driver that rewrites a descriptor after it +/// published the chain changes nothing the device does with it. +#[test] +fn a_descriptor_rewritten_after_publication_changes_nothing() { + let (mut guest, mut ring) = Guest::ready(4, 4); + let places = guest.chain(0, &[Part::Read(b"the request"), Part::Write(8)]); + let decoy = guest.space(64); + guest.memory.write(decoy, &[0xaa; 64]).unwrap(); + guest.publish(0); + let mut chain = ring.pop_available().unwrap().unwrap(); + + // Point both descriptors at the decoy, longer, and swap their roles. + guest.descriptor(0, decoy, 64, VIRTQ_DESC_F_WRITE | VIRTQ_DESC_F_NEXT, 1); + guest.descriptor(1, decoy, 64, 0, 0); + + assert_eq!(request(&mut chain), b"the request"); + let (_, _, sink) = chain.io().unwrap().into_parts(); + assert_eq!(sink.len(), 8); + sink.write_at(0, b"answered").unwrap(); + ring.complete(chain, UsedLength::new(8)).unwrap(); + assert_eq!(guest.read(places[1], 8), b"answered"); + assert_eq!(guest.read(decoy, 64), [0xaa; 64]); +} + +/// A chain's shape as plain numbers: `(length, flags, next)` per descriptor. +type Shape = &'static [(u32, u16, u16)]; + +const N: u16 = VIRTQ_DESC_F_NEXT; +const W: u16 = VIRTQ_DESC_F_WRITE; + +/// Shapes a driver may publish, good and bad, with what the protocol calls +/// each. The chain limit in these tests is four descriptors. +const SHAPES: &[(&str, Shape, u16, Option)] = &[ + ( + "one readable, one writable", + &[(24, N, 1), (32, W, 0)], + 0, + None, + ), + ( + "two and two", + &[(8, N, 1), (8, N, 2), (8, N | W, 3), (8, W, 0)], + 0, + None, + ), + ( + "a head that is not descriptor zero", + &[(16, W, 0), (16, N, 0)], + 1, + None, + ), + ( + "no writable descriptor", + &[(8, N, 1), (8, 0, 0)], + 0, + Some(MalformedChain::Direction), + ), + ( + "no readable descriptor", + &[(8, N | W, 1), (8, W, 0)], + 0, + Some(MalformedChain::Direction), + ), + ( + "readable after writable", + &[(8, N, 1), (8, N | W, 2), (8, 0, 0)], + 0, + Some(MalformedChain::Direction), + ), + ( + "a single descriptor", + &[(8, W, 0)], + 0, + Some(MalformedChain::DescriptorCount), + ), + ( + "longer than the chain limit", + &[(8, N, 1), (8, N, 2), (8, N, 3), (8, N | W, 4), (8, W, 0)], + 0, + Some(MalformedChain::DescriptorCount), + ), + ( + "a zero-length descriptor", + &[(0, N, 1), (8, W, 0)], + 0, + Some(MalformedChain::ZeroLength), + ), + ( + "a loop", + &[(8, N, 1), (8, N | W, 0)], + 0, + Some(MalformedChain::DescriptorLoop), + ), + ( + "a descriptor that follows itself", + &[(8, N, 0)], + 0, + Some(MalformedChain::DescriptorLoop), + ), + ( + "an indirect descriptor", + &[(16, VIRTQ_DESC_F_INDIRECT, 0)], + 0, + Some(MalformedChain::IndirectUnsupported), + ), + ( + "a flag the profile does not have", + &[(8, N | 0x80, 1), (8, W, 0)], + 0, + Some(MalformedChain::DescriptorFlags), + ), +]; + +/// What a consumed chain says of itself: its regions, or why it is malformed. +type Verdict = Result, MalformedChain>; + +fn verdict(chain: &mut C, name: &str) -> Verdict +where + C::Error: core::fmt::Debug, +{ + match chain.io() { + Ok(io) => Ok(io.into_parts().0.to_vec()), + Err(ChainError::Malformed(malformed)) => Err(malformed), + Err(other) => panic!("{name}: {other:?}"), + } +} + +/// VQ-004, VQ-005, VQ-006, and the oracle: every shape gets the same answer +/// from this ring as from the protocol's own in-memory split queue, and a +/// malformed one is handed up, says why, and is used with nothing written. +#[test] +fn every_chain_shape_is_judged_as_the_protocols_own_queue_judges_it() { + for &(name, shape, head, expected) in SHAPES { + // This ring, in a driver's memory. + let (mut guest, mut ring) = Guest::ready(8, 4); + for (index, &(len, flags, next)) in shape.iter().enumerate() { + let at = guest.space(len); + guest.descriptor(index as u16, at, len, flags, next); + } + guest.publish(head); + let mut chain = ring.pop_available().unwrap().expect(name); + let ours = verdict(&mut chain, name); + assert_eq!(ours.as_ref().err().copied(), expected, "{name}"); + + // The protocol's model, given the same descriptors. + let mut model = SplitQueue::new(QueueSize::new(8).unwrap(), 4).unwrap(); + model.configure(QueueSize::new(8).unwrap()).unwrap(); + QueueControl::set_ready(&mut model, true).unwrap(); + let descriptors = shape + .iter() + .map(|&(len, flags, next)| Descriptor::raw(vec![0; len as usize], flags, next)) + .collect(); + let raw = DriverChain::raw(descriptors, head).expect(name); + model.inject_available(raw).map_err(|_| name).unwrap(); + let mut modelled = DeviceQueue::pop_available(&mut model).unwrap().expect(name); + assert_eq!( + ours, + verdict(&mut modelled, name), + "{name}: this ring against the model" + ); + + // Either way the driver gets its chain back; a malformed one with + // nothing written. + let used = u32::from(ours.is_ok()); + ring.complete(chain, UsedLength::new(used)).expect(name); + assert_eq!(guest.used(), Some((u32::from(head), used)), "{name}"); + } +} + +/// What only a ring in real memory can get wrong: indices and addresses the +/// driver made up. +#[test] +fn indices_and_addresses_outside_what_was_lent_are_malformed_chains() { + let malformed = |build: &dyn Fn(&mut Guest) -> u16| { + let (mut guest, mut ring) = Guest::ready(8, 4); + let head = build(&mut guest); + guest.publish(head); + let mut chain = ring.pop_available().unwrap().unwrap(); + let reason = verdict(&mut chain, "made up").expect_err("malformed"); + // It is still the driver's chain to get back, with nothing written. + ring.complete(chain, UsedLength::new(0)).unwrap(); + assert_eq!(guest.used(), Some((u32::from(head), 0))); + reason + }; + + // A head past the queue. + assert_eq!(malformed(&|_| 8), MalformedChain::DescriptorIndex); + assert_eq!(malformed(&|_| u16::MAX), MalformedChain::DescriptorIndex); + // A `next` past the queue. + assert_eq!( + malformed(&|guest| { + let at = guest.space(8); + guest.descriptor(0, at, 8, N, 9); + 0 + }), + MalformedChain::DescriptorIndex + ); + // Bytes that begin or end outside the window, readable or writable. + for (at, len) in [ + (WINDOW as u64, 8u32), + (WINDOW as u64 - 4, 8), + (u64::MAX - 3, 8), + (0x1_0000_0000, 8), + ] { + assert_eq!( + malformed(&|guest| { + let good = guest.space(8); + guest.descriptor(0, at, len, N, 1); + guest.descriptor(1, good, 8, W, 0); + 0 + }), + MalformedChain::Address, + "readable at {at:#x}" + ); + assert_eq!( + malformed(&|guest| { + let good = guest.space(8); + guest.descriptor(0, good, 8, N, 1); + guest.descriptor(1, at, len, W, 0); + 0 + }), + MalformedChain::Address, + "writable at {at:#x}" + ); + } +} + +/// A used length is the bytes the device wrote, and it cannot claim more +/// than the chain gave it room for. +#[test] +fn a_used_length_past_the_chains_room_is_refused() { + let (mut guest, mut ring) = Guest::ready(4, 4); + guest.chain(0, &[Part::Read(b"q"), Part::Write(4)]); + guest.publish(0); + let chain = ring.pop_available().unwrap().unwrap(); + assert_eq!( + ring.complete(chain, UsedLength::new(5)).err(), + Some(QueueError::UsedLengthExceeded { + used: UsedLength::new(5), + capacity: 4 + }) + ); + assert_eq!(guest.used(), None, "nothing was published"); +} + +/// VQ-014: the device uses chains in whatever order it finishes them. +#[test] +fn chains_are_used_in_the_order_the_device_finishes_them() { + let (mut guest, mut ring) = Guest::ready(8, 4); + guest.chain(0, &[Part::Read(b"first"), Part::Write(4)]); + guest.chain(4, &[Part::Read(b"second"), Part::Write(4)]); + guest.publish(0); + guest.publish(4); + let first = ring.pop_available().unwrap().unwrap(); + let second = ring.pop_available().unwrap().unwrap(); + assert!(ring.pop_available().unwrap().is_none()); + assert_eq!(ring.in_flight(), 2); + ring.complete(second, UsedLength::new(3)).unwrap(); + ring.complete(first, UsedLength::new(1)).unwrap(); + assert_eq!(guest.used(), Some((4, 3))); + assert_eq!(guest.used(), Some((0, 1))); +} + +/// VQ-017: either side may ask not to be woken, and neither loses work for +/// it. +#[test] +fn suppression_is_honoured_and_loses_no_work() { + let (mut guest, mut ring) = Guest::ready(4, 4); + + // The driver asks not to be interrupted. + guest + .memory + .store_u16(guest.at.available, VIRTQ_AVAIL_F_NO_INTERRUPT) + .unwrap(); + guest.chain(0, &[Part::Read(b"q"), Part::Write(1)]); + guest.publish(0); + let chain = ring.pop_available().unwrap().unwrap(); + assert_eq!( + ring.complete(chain, UsedLength::new(1)), + Ok(NotificationHint::Suppressed) + ); + // The entry is there all the same. + assert_eq!(guest.used(), Some((0, 1))); + + // The device asks not to be notified while it drains, and says so where + // the driver reads it. + ring.disable_available_notifications().unwrap(); + assert_eq!( + guest.memory.load_u16(guest.at.used), + Ok(VIRTQ_USED_F_NO_NOTIFY) + ); + assert!(ring.pop_available().unwrap().is_none()); + // A chain published before it listens again is found by the recheck. + guest.chain(2, &[Part::Read(b"r"), Part::Write(1)]); + guest.publish(2); + assert_eq!( + ring.enable_available_notifications(), + Ok(NotificationRecheck::WorkPending) + ); + assert_eq!(guest.memory.load_u16(guest.at.used), Ok(0)); + let chain = ring.pop_available().unwrap().unwrap(); + ring.complete(chain, UsedLength::new(0)).unwrap(); + assert_eq!( + ring.enable_available_notifications(), + Ok(NotificationRecheck::Idle) + ); +} + +/// VQ-018: after a reset nothing of the old epoch is written. A chain the +/// device still holds can neither read, write nor be used. +#[test] +fn a_reset_silences_every_chain_the_device_still_holds() { + let (mut guest, mut ring) = Guest::ready(4, 4); + let places = guest.chain(0, &[Part::Read(b"before"), Part::Write(8)]); + guest.publish(0); + let mut chain = ring.pop_available().unwrap().unwrap(); + let old = chain.id().epoch(); + let next = old.checked_next().unwrap(); + + assert_eq!( + ring.reset(old).err(), + Some(QueueError::InvalidConfiguration( + QueueConfigError::NonIncreasingEpoch + )) + ); + ring.reset(next).unwrap(); + assert!(!ring.state().ready()); + assert_eq!(ring.in_flight(), 0); + + assert!(matches!( + chain.io().map(|_| ()), + Err(ChainError::ResetRace { chain, current }) if chain == old && current == next + )); + assert_eq!( + chain.response.write_at(0, b"too late"), + Err(ByteAccessError::Reset) + ); + let mut bytes = [0u8; 6]; + assert_eq!( + chain.request.read_at(0, &mut bytes), + Err(ByteAccessError::Reset) + ); + assert_eq!( + ring.complete(chain, UsedLength::new(0)).err(), + Some(QueueError::ResetRace { + operation: old, + current: next + }) + ); + assert_eq!(guest.used(), None, "no used entry from the old epoch"); + assert_eq!(guest.read(places[1], 8), [0u8; 8]); + + // The queue serves again once the driver sets it up again. + assert_eq!( + ring.pop_available().map(|chain| chain.is_some()), + Err(QueueError::NotReady) + ); + ring.configure(QueueSize::new(4).unwrap()).unwrap(); + ring.set_addresses(guest.at).unwrap(); + ring.set_ready(true).unwrap(); + // The driver starts its rings over too, as a reset requires of it. + guest.memory.store_u16(guest.at.available + 2, 0).unwrap(); + guest.memory.store_u16(guest.at.used + 2, 0).unwrap(); + (guest.published, guest.seen) = (0, 0); + guest.chain(0, &[Part::Read(b"after"), Part::Write(8)]); + guest.publish(0); + let mut chain = ring.pop_available().unwrap().unwrap(); + assert_eq!(chain.id().epoch(), next); + assert_eq!(request(&mut chain), b"after"); + ring.complete(chain, UsedLength::new(0)).unwrap(); + assert_eq!(guest.used(), Some((0, 0))); +} + +/// The queue's own shape is checked when it is made ready, and a driver that +/// breaks the ring's rules afterwards stops the queue, not the device. +#[test] +fn a_queue_the_driver_set_up_wrongly_is_refused() { + // Limits that do not fit each other. + let memory = window(4096); + let size = |n| QueueSize::new(n).unwrap(); + assert!(DeviceRing::new(memory, size(128), 4).is_none()); + assert!(DeviceRing::new(memory, size(8), 1).is_none()); + assert!(DeviceRing::new(memory, size(8), 16).is_none()); + + let (guest, mut ring) = Guest::unready(8, 4); + // Smaller than the longest chain. + assert_eq!( + ring.configure(size(2)).err(), + Some(QueueError::Transport(RingError::ChainLimitExceedsQueue)) + ); + for wrong in [ + RingAddresses { + descriptors: 8, + ..guest.at + }, + RingAddresses { + available: 0x801, + ..guest.at + }, + RingAddresses { + used: 0x1002, + ..guest.at + }, + RingAddresses { + descriptors: WINDOW as u64 - 16, + ..guest.at + }, + RingAddresses { + used: WINDOW as u64 - 8, + ..guest.at + }, + RingAddresses { + available: u64::MAX - 1, + ..guest.at + }, + ] { + ring.set_addresses(wrong).unwrap(); + assert_eq!( + ring.set_ready(true).err(), + Some(QueueError::Transport(RingError::Layout)), + "{wrong:?}" + ); + assert!(!ring.state().ready()); + } + ring.set_addresses(guest.at).unwrap(); + ring.set_ready(true).unwrap(); + // Ready: its shape is fixed until it is made unready or reset. + assert_eq!( + ring.set_addresses(guest.at).err(), + Some(QueueError::Transport(RingError::Busy)) + ); + assert_eq!( + ring.configure(size(8)).err(), + Some(QueueError::Transport(RingError::Busy)) + ); + + // An available index further ahead than the queue is long. + guest.memory.store_u16(guest.at.available + 2, 9).unwrap(); + assert_eq!( + ring.pop_available().map(|chain| chain.is_some()), + Err(QueueError::Transport(RingError::AvailableIndex)) + ); + + // A head published twice while the device holds it. + let (mut guest, mut ring) = Guest::ready(8, 4); + guest.chain(0, &[Part::Read(b"once"), Part::Write(4)]); + guest.publish(0); + guest.publish(0); + let held = ring.pop_available().unwrap().unwrap(); + assert_eq!( + ring.pop_available().map(|chain| chain.is_some()), + Err(QueueError::Transport(RingError::HeadInFlight)) + ); + drop(held); +} + +/// The ring's indices are sixteen bits and wrap; the queue goes on across +/// the wrap, slot by slot. +#[test] +fn the_indices_wrap_and_the_queue_goes_on() { + let (mut guest, mut ring) = Guest::ready(2, 2); + let places = guest.chain(0, &[Part::Read(b"ping"), Part::Write(4)]); + for round in 0..70_000u32 { + guest.publish(0); + let mut chain = ring.pop_available().unwrap().expect("published"); + let (_, _, sink) = chain.io().unwrap().into_parts(); + sink.write_at(0, &round.to_le_bytes()).unwrap(); + ring.complete(chain, UsedLength::new(4)).unwrap(); + assert_eq!(guest.used(), Some((0, 4)), "round {round}"); + assert_eq!(guest.read(places[1], 4), round.to_le_bytes()); + } + assert!(ring.pop_available().unwrap().is_none()); +} diff --git a/native/crates/vdev/src/transport.rs b/native/crates/vdev/src/transport.rs new file mode 100644 index 0000000..9e0ce10 --- /dev/null +++ b/native/crates/vdev/src/transport.rs @@ -0,0 +1,460 @@ +//! **The transport**: one driver's path to one device model. +//! +//! It owns the three things between them: the register block +//! ([`regs`](crate::regs)), the queue in the driver's memory +//! ([`ring`](crate::ring)) and the model ([`VirtualDevice`]). The host +//! calls [`Transport::service`] when the driver rings its doorbell and +//! raises the driver's interrupt when the answer says to. That is the whole +//! of what a host does, so a host is a loop around two notices. +//! +//! The register block is there from the start, so a driver can find out +//! what the device is before it commits memory to it. The driver's memory +//! arrives when the driver lends it ([`Transport::lend`]), which it must +//! have done by the time it says the queue is ready. +//! +//! # What one service does +//! +//! 1. Reads the status the driver has written. Zero is a reset: the queue +//! takes a new epoch, the model forgets everything, and the status the +//! device publishes returns to zero. +//! 2. Otherwise accepts each step the driver has reached that the device +//! has not, in Virtio's order and only in that order: `ACKNOWLEDGE`, +//! `DRIVER`, `FEATURES_OK` if the features the driver wrote are ones the +//! device offered, `DRIVER_OK` if the queue the driver described is one +//! the device can serve. +//! 3. Once live, takes each chain the driver published, hands it to the +//! model and returns it used. +//! +//! A status write that changes what the driver has announced is always +//! answered with an interrupt, whether or not the device accepted the step: +//! a driver writes its status, rings, waits, and then reads what the device +//! made of it. A refusal is the device's status not carrying the step, +//! never silence. +//! +//! Writing the status the device last saw announces nothing and is not +//! answered, zero included: a device in its reset state that is told to +//! reset has nothing to do. That matters because a look is not always the +//! driver's doing. Whoever hosts the device looks whenever anything wakes +//! it, another driver's doorbell among them, and a device that answered +//! every look in its reset state would interrupt a driver that had asked +//! for nothing. +//! +//! # What the driver cannot do to the device +//! +//! The device keeps its own copy of every decision. It never reads back a +//! register it wrote, it reads the driver's registers once per step and +//! acts on what it read, and a feature set or a queue it accepted stays +//! what it accepted whatever the driver writes afterwards. A driver that +//! describes a queue the device cannot serve, or breaks the queue once it +//! is live, gets [`STATUS_DEVICE_NEEDS_RESET`] and nothing more until it +//! resets; a chain that is merely malformed is one chain's failure, returned +//! used with a length of zero. +//! +//! One driver cannot hold the host either: a service takes at most the +//! budget of chains it is given, a second look at the queue counting as +//! one, and says whether more may be waiting. + +use core::num::NonZeroU32; + +use virtio_accel_transport::{ + ChainError, DeviceChain, DeviceQueue, NotificationHint, NotificationRecheck, QueueControl, + QueuePort, QueueSize, UsedLength, +}; + +use crate::device::VirtualDevice; +use crate::mem::Memory; +use crate::regs::{ + self, F_VERSION_1, STATUS_ACKNOWLEDGE, STATUS_DEVICE_NEEDS_RESET, STATUS_DRIVER, + STATUS_DRIVER_OK, STATUS_FAILED, STATUS_FEATURES_OK, +}; +use crate::ring::{DeviceRing, RingAddresses, limits_fit}; + +/// The features every device here offers. The profile has no optional ring +/// feature (no indirect tables, no event indices, no packed ring), so the +/// offer is the one bit that says the device is not a legacy one. +pub const OFFERED_FEATURES: u64 = F_VERSION_1; + +/// Why a transport could not be made. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AttachError { + /// The register window is not the page the block needs. + Registers, + /// The model's configuration does not fit the block's configuration + /// area. + Config, + /// The model's queue limits are not ones the ring can hold. + Queue, +} + +/// The driver's memory changes only while the device is in its first state: +/// a device that has accepted any step may hold a queue in the memory it +/// has, and is reset before that memory is taken from it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Busy; + +/// What one [`Transport::service`] left for the host to do. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Serviced { + /// Raise the driver's interrupt: the device answered a status write, + /// changed its own status, or used a chain the driver asked to hear + /// about. [`INTERRUPTS`](regs::INTERRUPTS) has already moved; the host + /// wakes whoever is parked on it. + pub interrupt: bool, + /// The budget ran out with chains possibly still waiting. The host + /// calls again without waiting for a doorbell: the driver was told not + /// to ring one. + pub pending: bool, +} + +/// Something under the transport stopped holding: a register that would not +/// load, the queue refusing a step it had validated, or the model using +/// more than it was given. Each ends in [`STATUS_DEVICE_NEEDS_RESET`]. +struct Broken; + +/// One driver's path to one device model. +#[derive(Debug)] +pub struct Transport { + registers: M, + /// The queue, once the driver has lent the memory it is in. + ring: Option>, + max_queue: QueueSize, + device: D, + /// The status the device has accepted and published. + status: u32, + /// The driver's status as last read: a doorbell that finds another + /// value is announcing a status write. + announced: u32, + /// The features accepted at `FEATURES_OK`. + features: u64, + /// A reset failed: the status stays `DEVICE_NEEDS_RESET` for good. + dead: bool, + /// Interrupts raised, as [`regs::INTERRUPTS`] publishes them. + interrupts: u32, +} + +impl Transport { + /// A transport for `device`, with the register block in `registers`. + /// Writes the device's half of the block: the driver may map and read + /// it as soon as this returns. + pub fn new(registers: M, device: D) -> Result { + if registers.len() < regs::BYTES { + return Err(AttachError::Registers); + } + let config = device.config(); + if config.len() as u64 > regs::BYTES - regs::CONFIG { + return Err(AttachError::Config); + } + let max_queue = QueueSize::new(D::MAX_QUEUE_SIZE) + .filter(|size| limits_fit(*size, D::MAX_CHAIN_DESCRIPTORS)) + .ok_or(AttachError::Queue)?; + + let block = |at, value| registers.store_u32(at, value); + // The length was checked above, and every offset is a constant of + // the layout, aligned for its word: a fault here is a window that + // does not keep `Memory`'s contract. + let written = registers + .write(regs::CONFIG, config) + .and_then(|()| block(regs::CONFIG_BYTES, config.len() as u32)) + .and_then(|()| block(regs::CONFIG_GENERATION, 0)) + .and_then(|()| block(regs::QUEUE_NUM_MAX, u32::from(D::MAX_QUEUE_SIZE))) + .and_then(|()| block(regs::DEVICE_STATUS, 0)) + .and_then(|()| block(regs::INTERRUPTS, 0)) + .and_then(|()| registers.store_u64(regs::DEVICE_FEATURES, OFFERED_FEATURES)) + .and_then(|()| block(regs::VENDOR_ID, regs::VENDOR_KORE)) + .and_then(|()| block(regs::DEVICE_ID, D::DEVICE_ID)) + .and_then(|()| block(regs::VERSION, regs::VERSION_VALUE)) + // Last: a driver that sees the magic sees the rest. + .and_then(|()| block(regs::MAGIC, regs::MAGIC_VALUE)); + if written.is_err() { + return Err(AttachError::Registers); + } + Ok(Self { + registers, + ring: None, + max_queue, + device, + status: 0, + announced: 0, + features: 0, + dead: false, + interrupts: 0, + }) + } + + /// The driver lends `memory`: device addresses are offsets into it from + /// here on. Answers the memory it replaces, if the driver had lent some + /// before, which the host may now unmap. + pub fn lend(&mut self, memory: M) -> Result, Busy> { + if self.status != 0 { + return Err(Busy); + } + let before = self.withdraw()?; + // The limits were checked when the transport was made. + self.ring = DeviceRing::new(memory, self.max_queue, D::MAX_CHAIN_DESCRIPTORS); + Ok(before) + } + + /// Take back the memory the driver lent, leaving the device with none. + pub fn withdraw(&mut self) -> Result, Busy> { + if self.status != 0 { + return Err(Busy); + } + Ok(self.ring.take().map(DeviceRing::into_memory)) + } + + /// Whether the driver has lent memory. + pub const fn has_memory(&self) -> bool { + self.ring.is_some() + } + + /// The status the device has accepted (`STATUS_*`). + pub const fn status(&self) -> u32 { + self.status + } + + /// The features the driver and device agreed on; zero before + /// `FEATURES_OK`. + pub const fn features(&self) -> u64 { + self.features + } + + /// A reset could not bring the device back. The host drops this + /// transport; a driver that attaches again gets a new one. + pub const fn is_dead(&self) -> bool { + self.dead + } + + /// The model. + pub const fn device(&self) -> &D { + &self.device + } + + /// The driver rang its doorbell, or the last service left chains + /// pending: look at what the driver wrote and act on it, taking at most + /// `budget` chains. + pub fn service(&mut self, budget: NonZeroU32) -> Serviced { + let before = self.status; + let mut serviced = Serviced::default(); + match self.step(budget) { + Ok(done) => serviced = done, + // Told to the driver below, as the status changing. + Err(Broken) => self.status |= STATUS_DEVICE_NEEDS_RESET, + } + if self.status != before { + // Nothing is left to try if the status itself will not store: + // the driver sees the old one, and the device still refuses to + // serve past what it accepted. + let _ = self.registers.store_u32(regs::DEVICE_STATUS, self.status); + serviced.interrupt = true; + } + if serviced.interrupt { + // After everything the interrupt is about: a driver woken by + // this word finds the status and the used ring already written. + self.interrupts = self.interrupts.wrapping_add(1); + let _ = self.registers.store_u32(regs::INTERRUPTS, self.interrupts); + } + serviced + } + + fn step(&mut self, budget: NonZeroU32) -> Result { + let driver = self.driver_u32(regs::DRIVER_STATUS)?; + // Only a status the device has not seen is an announcement. A look + // that finds the same one, a device in its reset state included, + // answers nothing: the look may be another driver's doing. + let answer = Serviced { + interrupt: driver != self.announced, + pending: false, + }; + self.announced = driver; + if driver == 0 { + self.reset(); + return Ok(answer); + } + self.advance(driver)?; + if !self.live() { + return Ok(answer); + } + let served = self.serve(budget)?; + Ok(Serviced { + interrupt: answer.interrupt || served.interrupt, + pending: served.pending, + }) + } + + fn driver_u32(&self, at: u64) -> Result { + self.registers.load_u32(at).map_err(|_| Broken) + } + + fn driver_u64(&self, at: u64) -> Result { + self.registers.load_u64(at).map_err(|_| Broken) + } + + /// Whether the device takes chains: the driver finished bring-up and + /// neither side has given up. + const fn live(&self) -> bool { + self.status & (STATUS_DRIVER_OK | STATUS_FAILED | STATUS_DEVICE_NEEDS_RESET) + == STATUS_DRIVER_OK + } + + /// The driver wrote zero. Nothing accepted means nothing to undo, which + /// is also what makes a reset idempotent. + fn reset(&mut self) { + if self.status == 0 || self.dead { + return; + } + // The queue first: from the new epoch on nothing of the old one is + // taken, written or completed, so the model is torn down with no + // chain in its hands and the driver may reuse its memory once it + // reads the status as zero. + let queue = self.ring.as_mut().is_none_or(|ring| { + ring.state() + .epoch() + .checked_next() + .is_some_and(|epoch| ring.reset(epoch).is_ok()) + }); + let model = self.device.reset().is_ok(); + self.features = 0; + if queue && model { + self.status = 0; + } else { + self.dead = true; + self.status = STATUS_DEVICE_NEEDS_RESET; + } + } + + /// Accept each step `driver` has reached that the device has not, in + /// order. A step whose predecessor is missing is not accepted, and a + /// step once accepted is not revisited: only a reset takes it back. + fn advance(&mut self, driver: u32) -> Result<(), Broken> { + if self.status & (STATUS_DEVICE_NEEDS_RESET | STATUS_FAILED) != 0 { + return Ok(()); + } + if driver & STATUS_FAILED != 0 { + // The driver gave up: the device stops with it. + self.status |= STATUS_FAILED; + return Ok(()); + } + let reached = |status: u32, step: u32, after: u32| { + driver & step != 0 && status & step == 0 && status & after == after + }; + if reached(self.status, STATUS_ACKNOWLEDGE, 0) { + self.status |= STATUS_ACKNOWLEDGE; + } + if reached(self.status, STATUS_DRIVER, STATUS_ACKNOWLEDGE) { + self.status |= STATUS_DRIVER; + } + let named = STATUS_ACKNOWLEDGE | STATUS_DRIVER; + if reached(self.status, STATUS_FEATURES_OK, named) { + let accepted = self.driver_u64(regs::DRIVER_FEATURES)?; + // A subset of the offer that keeps the one bit a driver must + // accept. Anything else leaves `FEATURES_OK` unset, which is + // how Virtio tells a driver its choice was refused. + if accepted & !OFFERED_FEATURES == 0 && accepted & F_VERSION_1 != 0 { + self.features = accepted; + self.status |= STATUS_FEATURES_OK; + } + } + if reached(self.status, STATUS_DRIVER_OK, named | STATUS_FEATURES_OK) { + if self.bring_up_queue()? { + self.status |= STATUS_DRIVER_OK; + } else { + // The driver said it was done and described a queue that + // cannot be served. There is no later step at which it + // could be put right. + self.status |= STATUS_DEVICE_NEEDS_RESET; + } + } + Ok(()) + } + + /// Take the queue the driver described. `false` when it is not one the + /// device can serve: not ready, a size that is not a power of two the + /// model takes, parts outside the driver's memory, or no memory lent + /// for it to be in. + fn bring_up_queue(&mut self) -> Result { + if self.driver_u32(regs::QUEUE_READY)? == 0 { + return Ok(false); + } + let Some(size) = u16::try_from(self.driver_u32(regs::QUEUE_NUM)?) + .ok() + .and_then(QueueSize::new) + else { + return Ok(false); + }; + let addresses = RingAddresses { + descriptors: self.driver_u64(regs::QUEUE_DESC)?, + available: self.driver_u64(regs::QUEUE_DRIVER)?, + used: self.driver_u64(regs::QUEUE_DEVICE)?, + }; + let Some(ring) = self.ring.as_mut() else { + return Ok(false); + }; + Ok(ring.configure(size).is_ok() + && ring.set_addresses(addresses).is_ok() + && ring.set_ready(true).is_ok()) + } + + /// Take what the driver published, at most `budget` chains. + fn serve(&mut self, budget: NonZeroU32) -> Result { + let mut serviced = Serviced::default(); + let mut left = budget.get(); + // Live means the queue was brought up, which took memory. + let Some(ring) = self.ring.as_mut() else { + return Err(Broken); + }; + // The driver need not ring again while the device is looking. + ring.disable_available_notifications().map_err(|_| Broken)?; + loop { + while left != 0 { + let Some(mut chain) = ring.pop_available().map_err(|_| Broken)? else { + break; + }; + left -= 1; + let used = match chain.io() { + Ok(io) => { + let (regions, request, response) = io.into_parts(); + self.device.command(regions, request, response) + } + // One chain's failure: returned used, nothing written. + Err(ChainError::Malformed(_)) => 0, + // No reset runs inside a service, and the chain's bytes + // were checked when it was taken. + Err(ChainError::ResetRace { .. } | ChainError::Transport(_)) => { + return Err(Broken); + } + }; + let hint = ring + .complete(chain, UsedLength::new(used)) + .map_err(|_| Broken)?; + serviced.interrupt |= hint == NotificationHint::Notify; + // The command is answered before its work runs: a driver + // that submits is told so, and then sees it complete. + self.device.drive(); + if self.device.needs_reset() { + self.status |= STATUS_DEVICE_NEEDS_RESET; + return Ok(serviced); + } + } + if left == 0 { + // Notifications stay off: the host comes back by itself. + serviced.pending = true; + return Ok(serviced); + } + match ring.enable_available_notifications().map_err(|_| Broken)? { + NotificationRecheck::Idle => return Ok(serviced), + // Published while the device was not listening. + NotificationRecheck::WorkPending => { + ring.disable_available_notifications().map_err(|_| Broken)?; + // A look costs what a chain costs. A driver that shows + // an index and takes it back before the device reads + // the entry would otherwise keep the device looking, + // at no cost to the budget, for as long as it liked. + left -= 1; + } + } + } + } +} + +#[cfg(test)] +pub(crate) mod tests; diff --git a/native/crates/vdev/src/transport/tests.rs b/native/crates/vdev/src/transport/tests.rs new file mode 100644 index 0000000..9c6410f --- /dev/null +++ b/native/crates/vdev/src/transport/tests.rs @@ -0,0 +1,1070 @@ +use super::*; +use crate::device::{ChainRegion, ReadableBytes, Unrecoverable, WritableBytes}; +use crate::mem::{Window, tests::window}; +use crate::ring::tests::{Guest, Part, WINDOW}; +use crate::ring::{VIRTQ_AVAIL_F_NO_INTERRUPT, VIRTQ_USED_F_NO_NOTIFY}; +use std::vec; +use std::vec::Vec; + +/// A model that answers a request with its bytes reversed, and misbehaves +/// on request. +#[derive(Debug, Default)] +pub(crate) struct Echo { + pub(crate) commands: u32, + pub(crate) drives: u32, + pub(crate) resets: u32, + /// Set by the request `sick`: the model can no longer answer. + sick: bool, + /// A reset does not bring this model back. + pub(crate) unrecoverable: bool, +} + +pub(crate) const ECHO_CONFIG: &[u8] = b"echo config"; + +impl VirtualDevice for Echo { + const DEVICE_ID: u32 = u32::from_le_bytes(*b"echo"); + const MAX_QUEUE_SIZE: u16 = 8; + const MAX_CHAIN_DESCRIPTORS: u16 = 4; + + fn config(&self) -> &[u8] { + ECHO_CONFIG + } + + fn command( + &mut self, + _regions: &[ChainRegion], + request: &dyn ReadableBytes, + response: &mut dyn WritableBytes, + ) -> u32 { + self.commands += 1; + let mut bytes = vec![0u8; request.len() as usize]; + request.read_at(0, &mut bytes).unwrap(); + match bytes.as_slice() { + b"sick" => self.sick = true, + // Claims more than the chain gave it. + b"over" => return response.len() as u32 + 1, + _ => {} + } + bytes.reverse(); + bytes.truncate(response.len() as usize); + response.write_at(0, &bytes).unwrap(); + bytes.len() as u32 + } + + fn drive(&mut self) -> bool { + self.drives += 1; + false + } + + fn needs_reset(&self) -> bool { + self.sick + } + + fn reset(&mut self) -> Result<(), Unrecoverable> { + self.resets += 1; + if self.unrecoverable { + return Err(Unrecoverable); + } + self.sick = false; + Ok(()) + } +} + +const BUDGET: NonZeroU32 = NonZeroU32::new(16).unwrap(); +const NAMED: u32 = STATUS_ACKNOWLEDGE | STATUS_DRIVER; +const AGREED: u32 = NAMED | STATUS_FEATURES_OK; +const LIVE: u32 = AGREED | STATUS_DRIVER_OK; + +/// A driver written the way a C driver is: the register block and its own +/// memory, with every step done by hand. +struct Bench { + registers: Window, + guest: Guest, + transport: Transport, +} + +impl Bench { + fn new() -> Self { + Self::with(Echo::default()) + } + + fn with(model: Echo) -> Self { + let registers = window(regs::BYTES as usize); + let guest = Guest::over(window(WINDOW), Echo::MAX_QUEUE_SIZE); + let mut transport = Transport::new(registers, model).expect("a transport"); + assert!(transport.lend(guest.memory).unwrap().is_none()); + Self { + registers, + guest, + transport, + } + } + + /// Ring the doorbell. + fn ring(&mut self) -> Serviced { + self.transport.service(BUDGET) + } + + fn write_status(&self, status: u32) { + self.registers + .store_u32(regs::DRIVER_STATUS, status) + .unwrap(); + } + + /// The status the device publishes, which is also what it holds. + fn status(&self) -> u32 { + let published = self.registers.load_u32(regs::DEVICE_STATUS).unwrap(); + assert_eq!(published, self.transport.status()); + published + } + + fn describe_queue(&self, size: u32, at: RingAddresses) { + let registers = &self.registers; + registers.store_u32(regs::QUEUE_NUM, size).unwrap(); + registers + .store_u64(regs::QUEUE_DESC, at.descriptors) + .unwrap(); + registers + .store_u64(regs::QUEUE_DRIVER, at.available) + .unwrap(); + registers.store_u64(regs::QUEUE_DEVICE, at.used).unwrap(); + registers.store_u32(regs::QUEUE_READY, 1).unwrap(); + } + + /// Everything up to `FEATURES_OK`. + fn agree(&mut self) { + self.write_status(NAMED); + self.registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + self.write_status(AGREED); + assert!(self.ring().interrupt); + assert_eq!(self.status(), AGREED); + } + + /// The whole bring-up, with the queue the guest laid out. + fn bring_up(&mut self) { + self.agree(); + self.describe_queue(u32::from(self.guest.size), self.guest.at); + self.write_status(LIVE); + assert!(self.ring().interrupt); + assert_eq!(self.status(), LIVE); + } + + /// Publish `request` with `room` bytes for the answer in descriptors + /// `head` and `head + 1`, and say where the answer goes. + fn send(&mut self, head: u16, request: &[u8], room: u32) -> u64 { + let places = self + .guest + .chain(head, &[Part::Read(request), Part::Write(room)]); + self.guest.publish(head); + places[1] + } + + /// A reset, and a queue with nothing in it for the bring-up after. + fn reset(&mut self) { + self.write_status(0); + assert!( + self.ring().interrupt, + "a reset of a device that is not is answered" + ); + self.guest = Guest::over(self.guest.memory, self.guest.size); + for ring in [self.guest.at.available, self.guest.at.used] { + self.guest.memory.write(ring, &[0; 4]).unwrap(); + } + } +} + +#[test] +fn the_block_says_what_the_device_is() { + let bench = Bench::new(); + let registers = &bench.registers; + assert_eq!(registers.load_u32(regs::MAGIC), Ok(regs::MAGIC_VALUE)); + assert_eq!(registers.load_u32(regs::VERSION), Ok(regs::VERSION_VALUE)); + assert_eq!(registers.load_u32(regs::DEVICE_ID), Ok(Echo::DEVICE_ID)); + assert_eq!(registers.load_u32(regs::VENDOR_ID), Ok(regs::VENDOR_KORE)); + assert_eq!(registers.load_u64(regs::DEVICE_FEATURES), Ok(F_VERSION_1)); + assert_eq!(registers.load_u32(regs::DEVICE_STATUS), Ok(0)); + assert_eq!(registers.load_u32(regs::QUEUE_NUM_MAX), Ok(8)); + assert_eq!(registers.load_u32(regs::CONFIG_GENERATION), Ok(0)); + assert_eq!(registers.load_u32(regs::INTERRUPTS), Ok(0)); + assert_eq!( + registers.load_u32(regs::CONFIG_BYTES), + Ok(ECHO_CONFIG.len() as u32) + ); + let mut config = [0u8; ECHO_CONFIG.len()]; + registers.read(regs::CONFIG, &mut config).unwrap(); + assert_eq!(config, ECHO_CONFIG); + + // The words are where a C header says they are. + let mut head = [0u8; 16]; + registers.read(0, &mut head).unwrap(); + assert_eq!(&head[..4], b"kvdv"); + assert_eq!(head[4..8], 1u32.to_le_bytes()); + assert_eq!(&head[8..12], b"echo"); + assert_eq!(&head[12..], b"kore"); +} + +#[test] +fn a_transport_is_refused_what_it_cannot_carry() { + struct Wide; + impl VirtualDevice for Wide { + const DEVICE_ID: u32 = 0; + const MAX_QUEUE_SIZE: u16 = 8; + const MAX_CHAIN_DESCRIPTORS: u16 = 4; + fn config(&self) -> &[u8] { + &[0; (regs::BYTES - regs::CONFIG) as usize + 1] + } + fn command( + &mut self, + _: &[ChainRegion], + _: &dyn ReadableBytes, + _: &mut dyn WritableBytes, + ) -> u32 { + 0 + } + fn drive(&mut self) -> bool { + false + } + fn needs_reset(&self) -> bool { + false + } + fn reset(&mut self) -> Result<(), Unrecoverable> { + Ok(()) + } + } + struct Odd; + impl VirtualDevice for Odd { + const DEVICE_ID: u32 = 0; + const MAX_QUEUE_SIZE: u16 = SIZE; + const MAX_CHAIN_DESCRIPTORS: u16 = CHAIN; + fn config(&self) -> &[u8] { + &[] + } + fn command( + &mut self, + _: &[ChainRegion], + _: &dyn ReadableBytes, + _: &mut dyn WritableBytes, + ) -> u32 { + 0 + } + fn drive(&mut self) -> bool { + false + } + fn needs_reset(&self) -> bool { + false + } + fn reset(&mut self) -> Result<(), Unrecoverable> { + Ok(()) + } + } + + let page = regs::BYTES as usize; + let attach = |registers: usize| Transport::new(window(registers), Echo::default()).map(|_| ()); + assert_eq!(attach(page - 8), Err(AttachError::Registers)); + assert_eq!(attach(page), Ok(())); + assert_eq!( + Transport::new(window(page), Wide).map(|_| ()), + Err(AttachError::Config) + ); + let odd = |made: Result<(), AttachError>| assert_eq!(made, Err(AttachError::Queue)); + odd(Transport::new(window(page), Odd::<6, 2>).map(|_| ())); + odd(Transport::new(window(page), Odd::<128, 2>).map(|_| ())); + odd(Transport::new(window(page), Odd::<8, 1>).map(|_| ())); + odd(Transport::new(window(page), Odd::<8, 9>).map(|_| ())); +} + +#[test] +fn a_driver_steps_through_bring_up() { + let mut bench = Bench::new(); + // A look at a device in its reset state announces nothing, however + // often it comes: whoever hosts the device looks on every wake, and a + // driver that asked for nothing is not interrupted. Nothing is undone. + for _ in 0..3 { + assert_eq!( + bench.ring(), + Serviced { + interrupt: false, + pending: false + } + ); + } + assert_eq!(bench.registers.load_u32(regs::INTERRUPTS), Ok(0)); + assert_eq!(bench.transport.device().resets, 0); + // Zero written over zero is the same look. + bench.write_status(0); + assert!(!bench.ring().interrupt); + + bench.write_status(STATUS_ACKNOWLEDGE); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), STATUS_ACKNOWLEDGE); + // The same status again announces nothing. + assert!(!bench.ring().interrupt); + + bench.write_status(NAMED); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), NAMED); + assert_eq!(bench.transport.features(), 0); + + bench + .registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + bench.write_status(AGREED); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), AGREED); + assert_eq!(bench.transport.features(), F_VERSION_1); + + bench.describe_queue(8, bench.guest.at); + bench.write_status(LIVE); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), LIVE); + assert_eq!(bench.transport.device().commands, 0); +} + +/// A driver may write every step before it rings once: the device takes +/// them in order all the same. +#[test] +fn steps_written_together_are_taken_in_order() { + let mut bench = Bench::new(); + bench + .registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + bench.describe_queue(8, bench.guest.at); + bench.write_status(LIVE); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), LIVE); +} + +#[test] +fn a_step_whose_predecessor_is_missing_is_not_accepted() { + for skipped in [ + STATUS_DRIVER, + STATUS_FEATURES_OK, + STATUS_DRIVER_OK, + STATUS_ACKNOWLEDGE | STATUS_FEATURES_OK, + STATUS_ACKNOWLEDGE | STATUS_DRIVER_OK, + NAMED | STATUS_DRIVER_OK, + ] { + let mut bench = Bench::new(); + bench + .registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + bench.describe_queue(8, bench.guest.at); + bench.write_status(skipped); + assert!(bench.ring().interrupt, "refused, and told so"); + let accepted = bench.status(); + assert_eq!( + accepted & !skipped, + 0, + "nothing the driver did not write: {skipped:#x}" + ); + assert_eq!( + accepted & (STATUS_DRIVER_OK | STATUS_DEVICE_NEEDS_RESET), + 0, + "not live from {skipped:#x}" + ); + // And nothing is served. + bench.send(0, b"hello", 8); + bench.ring(); + assert!(bench.guest.used().is_none()); + } +} + +#[test] +fn features_the_device_did_not_offer_are_refused() { + for refused in [0, 1, F_VERSION_1 | 1, F_VERSION_1 | 1 << 33, u64::MAX] { + let mut bench = Bench::new(); + bench.write_status(NAMED); + bench + .registers + .store_u64(regs::DRIVER_FEATURES, refused) + .unwrap(); + bench.write_status(AGREED); + assert!(bench.ring().interrupt, "the refusal is an answer"); + assert_eq!(bench.status(), NAMED, "{refused:#x}"); + assert_eq!(bench.transport.features(), 0); + + // The queue cannot go live behind a refusal. + bench.describe_queue(8, bench.guest.at); + bench.write_status(LIVE); + bench.ring(); + assert_eq!(bench.status(), NAMED); + } +} + +#[test] +fn a_queue_the_device_cannot_serve_asks_for_a_reset() { + let good = Guest::over(window(64), 8).at; + let outside = |part: fn(&mut RingAddresses) -> &mut u64| { + let mut at = good; + *part(&mut at) = WINDOW as u64; + at + }; + let cases: [(&str, u32, RingAddresses, bool); 9] = [ + ("not ready", 8, good, false), + ("no size", 0, good, true), + ("not a power of two", 6, good, true), + ("larger than the device takes", 16, good, true), + ("larger than a queue is", 0x1_0008, good, true), + ("shorter than a chain", 2, good, true), + ( + "descriptors outside", + 8, + outside(|at| &mut at.descriptors), + true, + ), + ( + "available outside", + 8, + outside(|at| &mut at.available), + true, + ), + ("used outside", 8, outside(|at| &mut at.used), true), + ]; + for (what, size, at, ready) in cases { + let mut bench = Bench::new(); + bench.agree(); + bench.describe_queue(size, at); + if !ready { + bench.registers.store_u32(regs::QUEUE_READY, 0).unwrap(); + } + bench.write_status(LIVE); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), AGREED | STATUS_DEVICE_NEEDS_RESET, "{what}"); + + // Nothing is served, and putting the queue right is too late. + bench.describe_queue(8, good); + bench.send(0, b"hello", 8); + assert!(!bench.ring().interrupt); + assert!(bench.guest.used().is_none()); + assert_eq!(bench.status(), AGREED | STATUS_DEVICE_NEEDS_RESET); + + // A reset is the way back. + bench.reset(); + assert_eq!(bench.status(), 0); + bench.bring_up(); + } +} + +#[test] +fn commands_are_served_and_answered() { + let mut bench = Bench::new(); + bench.bring_up(); + + let first = bench.send(0, b"hello", 16); + let second = bench.send(2, b"device", 3); + assert_eq!( + bench.ring(), + Serviced { + interrupt: true, + pending: false + } + ); + assert_eq!(bench.guest.used(), Some((0, 5))); + assert_eq!(bench.guest.used(), Some((2, 3))); + assert_eq!(bench.guest.used(), None); + assert_eq!(bench.guest.read(first, 5), b"olleh"); + assert_eq!(bench.guest.read(second, 3), b"eci", "no more than the room"); + assert_eq!(bench.transport.device().commands, 2); + assert_eq!(bench.transport.device().drives, 2, "a turn after each"); + + // A doorbell with nothing new is not answered. + assert_eq!(bench.ring(), Serviced::default()); + + // VQ-017: a driver that asked not to be interrupted is not, and still + // finds its answer. + let flags = bench.guest.at.available; + bench + .guest + .memory + .store_u16(flags, VIRTQ_AVAIL_F_NO_INTERRUPT) + .unwrap(); + let third = bench.send(4, b"quiet", 8); + assert_eq!(bench.ring(), Serviced::default()); + assert_eq!(bench.guest.used(), Some((4, 5))); + assert_eq!(bench.guest.read(third, 5), b"teiuq"); +} + +#[test] +fn a_service_takes_no_more_than_its_budget() { + let mut bench = Bench::new(); + bench.bring_up(); + // Four chains of two descriptors fill the queue of eight. + for head in [0u16, 2, 4, 6] { + bench.send(head, &[b'a' + head as u8; 4], 4); + } + let two = NonZeroU32::new(2).unwrap(); + let notify = bench.guest.at.used; + let suppressed = + |bench: &Bench| bench.guest.memory.load_u16(notify).unwrap() == VIRTQ_USED_F_NO_NOTIFY; + + assert_eq!( + bench.transport.service(two), + Serviced { + interrupt: true, + pending: true + } + ); + assert_eq!(bench.transport.device().commands, 2); + assert!(suppressed(&bench), "the host comes back by itself"); + + // Exactly the budget again: the device cannot yet tell it is done. + assert!(bench.transport.service(two).pending); + assert_eq!(bench.transport.device().commands, 4); + assert!(suppressed(&bench)); + + assert_eq!(bench.transport.service(two), Serviced::default()); + assert!(!suppressed(&bench), "idle: the driver rings again"); + let heads: Vec<_> = core::iter::from_fn(|| bench.guest.used()).collect(); + assert_eq!(heads, [(0, 4), (2, 4), (4, 4), (6, 4)]); +} + +/// VQ-004..VQ-006 through the transport: a chain that cannot carry a +/// command is returned used with nothing written, the model never sees it, +/// and the chains after it are served. +#[test] +fn a_malformed_chain_fails_alone() { + let mut bench = Bench::new(); + bench.bring_up(); + bench + .guest + .chain(0, &[Part::Read(b"no room for an answer")]); + bench.guest.publish(0); + // A head that is no descriptor at all. + bench.guest.publish(200); + let answer = bench.send(2, b"fine", 8); + + assert!(bench.ring().interrupt); + assert_eq!(bench.guest.used(), Some((0, 0))); + assert_eq!(bench.guest.used(), Some((200, 0))); + assert_eq!(bench.guest.used(), Some((2, 4))); + assert_eq!(bench.guest.read(answer, 4), b"enif"); + assert_eq!(bench.transport.device().commands, 1); + assert_eq!( + bench.status(), + LIVE, + "one chain's failure is not the device's" + ); +} + +#[test] +fn a_queue_broken_while_live_asks_for_a_reset() { + let mut bench = Bench::new(); + bench.bring_up(); + bench.send(0, b"hello", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.guest.used(), Some((0, 5))); + + // The driver claims more chains than the queue is long. + let idx = bench.guest.at.available + 2; + bench.guest.memory.store_u16(idx, 1 + 9).unwrap(); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), LIVE | STATUS_DEVICE_NEEDS_RESET); + + // Nothing more is taken, however the driver rings. + bench.guest.memory.store_u16(idx, 1).unwrap(); + bench.send(2, b"again", 8); + assert_eq!(bench.ring(), Serviced::default()); + assert!(bench.guest.used().is_none()); + assert_eq!(bench.transport.device().commands, 1); + + bench.reset(); + assert_eq!(bench.status(), 0); + bench.bring_up(); + let answer = bench.send(0, b"back", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.guest.used(), Some((0, 4))); + assert_eq!(bench.guest.read(answer, 4), b"kcab"); +} + +#[test] +fn a_model_that_cannot_go_on_asks_for_a_reset() { + // It says so itself: the command that made it sick is still answered. + let mut bench = Bench::new(); + bench.bring_up(); + bench.send(0, b"sick", 8); + bench.send(2, b"after", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), LIVE | STATUS_DEVICE_NEEDS_RESET); + assert_eq!(bench.guest.used(), Some((0, 4))); + assert_eq!(bench.guest.used(), None, "nothing after it is taken"); + assert_eq!(bench.transport.device().commands, 1); + bench.reset(); + assert_eq!(bench.status(), 0); + assert_eq!(bench.transport.device().resets, 1); + bench.bring_up(); + bench.send(0, b"well", 8); + bench.ring(); + assert_eq!(bench.guest.used(), Some((0, 4))); + + // Or it claims to have written more than the chain gave it: nothing is + // returned used that would tell the driver to read bytes it never lent. + let mut bench = Bench::new(); + bench.bring_up(); + bench.send(0, b"over", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), LIVE | STATUS_DEVICE_NEEDS_RESET); + assert_eq!(bench.guest.used(), None); +} + +/// VQ-018, VQ-019 at the transport: a reset returns the device to its first +/// state, and nothing of the queue from before is used afterwards. +#[test] +fn a_reset_returns_the_device_to_its_first_state() { + let mut bench = Bench::new(); + bench.bring_up(); + bench.send(0, b"hello", 8); + bench.ring(); + // Published, and never served: the reset comes first. + let unanswered = bench.send(2, b"never", 8); + let used_before = bench + .guest + .memory + .load_u16(bench.guest.at.used + 2) + .unwrap(); + + bench.write_status(0); + assert_eq!( + bench.ring(), + Serviced { + interrupt: true, + pending: false + } + ); + assert_eq!(bench.status(), 0); + assert_eq!(bench.transport.features(), 0); + assert_eq!(bench.transport.device().resets, 1); + assert_eq!(bench.transport.device().commands, 1); + assert_eq!( + bench + .guest + .memory + .load_u16(bench.guest.at.used + 2) + .unwrap(), + used_before, + "no used entry after the reset" + ); + assert_eq!(bench.guest.read(unanswered, 5), [0; 5], "and no bytes"); + + // Zero again announces nothing: nothing was accepted since, so nothing + // is undone, and a reset that asked nothing is not answered. + assert!(!bench.ring().interrupt); + assert_eq!(bench.transport.device().resets, 1); + + // The old queue is not live: ringing without a bring-up serves nothing. + bench.write_status(NAMED); + bench.ring(); + assert_eq!( + bench + .guest + .memory + .load_u16(bench.guest.at.used + 2) + .unwrap(), + used_before + ); + + bench.reset(); + assert_eq!(bench.transport.device().resets, 2); + bench.bring_up(); + let answer = bench.send(0, b"anew", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.guest.used(), Some((0, 4)), "from the first entry"); + assert_eq!(bench.guest.read(answer, 4), b"wena"); +} + +#[test] +fn a_reset_that_fails_is_final() { + let mut bench = Bench::with(Echo { + unrecoverable: true, + ..Echo::default() + }); + bench.bring_up(); + assert!(!bench.transport.is_dead()); + + bench.write_status(0); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), STATUS_DEVICE_NEEDS_RESET); + assert!(bench.transport.is_dead()); + + // No bring-up and no further reset changes it. + bench.write_status(NAMED); + bench.ring(); + assert_eq!(bench.status(), STATUS_DEVICE_NEEDS_RESET); + bench.write_status(0); + assert!(bench.ring().interrupt, "still answered"); + assert_eq!(bench.status(), STATUS_DEVICE_NEEDS_RESET); + assert_eq!(bench.transport.device().resets, 1, "and not tried again"); +} + +#[test] +fn a_driver_that_gives_up_stops_the_device() { + let mut bench = Bench::new(); + bench.bring_up(); + bench.write_status(LIVE | STATUS_FAILED); + bench.send(0, b"hello", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.status(), LIVE | STATUS_FAILED); + assert!(bench.guest.used().is_none()); + + // Taking the bit back does not undo it. + bench.write_status(LIVE); + bench.ring(); + assert_eq!(bench.status(), LIVE | STATUS_FAILED); + assert!(bench.guest.used().is_none()); + + bench.reset(); + assert_eq!(bench.status(), 0); +} + +/// The device acts on what it accepted, not on what the registers say now. +#[test] +fn what_was_accepted_stays_accepted() { + let mut bench = Bench::new(); + bench.bring_up(); + let registers = bench.registers; + + // The driver scribbles over everything it can, the device's own + // registers included, and takes back steps it reached. + registers + .store_u64(regs::DRIVER_FEATURES, u64::MAX) + .unwrap(); + registers.store_u32(regs::QUEUE_NUM, 2).unwrap(); + registers.store_u32(regs::QUEUE_READY, 0).unwrap(); + for part in [regs::QUEUE_DESC, regs::QUEUE_DRIVER, regs::QUEUE_DEVICE] { + registers.store_u64(part, WINDOW as u64).unwrap(); + } + registers.store_u32(regs::DEVICE_STATUS, 0).unwrap(); + registers + .store_u64(regs::DEVICE_FEATURES, u64::MAX) + .unwrap(); + registers.store_u32(regs::QUEUE_NUM_MAX, 0x8000).unwrap(); + bench.write_status(STATUS_ACKNOWLEDGE); + + let answer = bench.send(0, b"still", 8); + assert!(bench.ring().interrupt); + assert_eq!(bench.guest.used(), Some((0, 5))); + assert_eq!(bench.guest.read(answer, 5), b"llits"); + assert_eq!(bench.transport.status(), LIVE); + assert_eq!(bench.transport.features(), F_VERSION_1); +} + +/// Driver memory in which a chain is published at the worst moment: after +/// the device found the queue empty and before it says it is listening +/// again. The driver saw the device not listening and rang no doorbell. +#[derive(Clone, Debug)] +struct Racing { + window: Window, + /// `(the device's listening flag, the available index, its new value)`, + /// until the race has been run. + race: std::rc::Rc>>, +} + +impl Memory for Racing { + fn len(&self) -> u64 { + self.window.len() + } + fn read(&self, at: u64, target: &mut [u8]) -> Result<(), crate::mem::MemoryFault> { + self.window.read(at, target) + } + fn write(&self, at: u64, source: &[u8]) -> Result<(), crate::mem::MemoryFault> { + self.window.write(at, source) + } + fn load_u16(&self, at: u64) -> Result { + self.window.load_u16(at) + } + fn store_u16(&self, at: u64, value: u16) -> Result<(), crate::mem::MemoryFault> { + if let Some((listening, index, published)) = self.race.get() + && at == listening + && value == 0 + { + self.race.set(None); + self.window.store_u16(index, published)?; + } + self.window.store_u16(at, value) + } + fn load_u32(&self, at: u64) -> Result { + self.window.load_u32(at) + } + fn store_u32(&self, at: u64, value: u32) -> Result<(), crate::mem::MemoryFault> { + self.window.store_u32(at, value) + } + fn load_u64(&self, at: u64) -> Result { + self.window.load_u64(at) + } + fn store_u64(&self, at: u64, value: u64) -> Result<(), crate::mem::MemoryFault> { + self.window.store_u64(at, value) + } +} + +/// VQ-017, the half a single thread cannot reach by itself: the device +/// looks once more after it says it is listening, so the chain no doorbell +/// announced is served in the same service. +#[test] +fn a_chain_published_as_the_device_stops_looking_is_served() { + let race = std::rc::Rc::new(core::cell::Cell::new(None)); + let racing = |window| Racing { + window, + race: race.clone(), + }; + let registers = window(regs::BYTES as usize); + let mut guest = Guest::over(window(WINDOW), Echo::MAX_QUEUE_SIZE); + let mut transport = Transport::new(racing(registers), Echo::default()).unwrap(); + transport.lend(racing(guest.memory)).unwrap(); + + registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + registers.store_u32(regs::QUEUE_NUM, 8).unwrap(); + registers + .store_u64(regs::QUEUE_DESC, guest.at.descriptors) + .unwrap(); + registers + .store_u64(regs::QUEUE_DRIVER, guest.at.available) + .unwrap(); + registers + .store_u64(regs::QUEUE_DEVICE, guest.at.used) + .unwrap(); + registers.store_u32(regs::QUEUE_READY, 1).unwrap(); + registers.store_u32(regs::DRIVER_STATUS, LIVE).unwrap(); + transport.service(BUDGET); + assert_eq!(transport.status(), LIVE); + + // One chain announced, and a second written but not yet published. + guest.chain(0, &[Part::Read(b"first"), Part::Write(8)]); + guest.publish(0); + let late = guest.chain(2, &[Part::Read(b"late"), Part::Write(8)])[1]; + guest + .memory + .write(guest.at.available + 4 + 2, &2u16.to_le_bytes()) + .unwrap(); + race.set(Some((guest.at.used, guest.at.available + 2, 2))); + + assert_eq!( + transport.service(BUDGET), + Serviced { + interrupt: true, + pending: false + } + ); + assert!(race.get().is_none(), "the race was run"); + assert_eq!(guest.used(), Some((0, 5))); + assert_eq!(guest.used(), Some((2, 4)), "served without a doorbell"); + assert_eq!(guest.read(late, 4), b"etal"); + assert_eq!( + guest.memory.load_u16(guest.at.used).unwrap(), + 0, + "and the device is listening when it stops" + ); +} + +/// Driver memory whose available index says one chain is published on +/// every second look and none on the looks between: what a driver on +/// another processor can make the device see by writing the index and +/// taking it back. +#[derive(Clone, Debug)] +struct Flapping { + window: Window, + /// The available index, once the queue is live. + index: std::rc::Rc>>, + looks: std::rc::Rc>, +} + +impl Memory for Flapping { + fn len(&self) -> u64 { + self.window.len() + } + fn read(&self, at: u64, target: &mut [u8]) -> Result<(), crate::mem::MemoryFault> { + self.window.read(at, target) + } + fn write(&self, at: u64, source: &[u8]) -> Result<(), crate::mem::MemoryFault> { + self.window.write(at, source) + } + fn load_u16(&self, at: u64) -> Result { + if self.index.get() == Some(at) { + // The device looks to take a chain, then looks again as it + // says it is listening: nothing, then one, and so on. + self.looks.set(self.looks.get() + 1); + return Ok(u16::from(self.looks.get() % 2 == 0)); + } + self.window.load_u16(at) + } + fn store_u16(&self, at: u64, value: u16) -> Result<(), crate::mem::MemoryFault> { + self.window.store_u16(at, value) + } + fn load_u32(&self, at: u64) -> Result { + self.window.load_u32(at) + } + fn store_u32(&self, at: u64, value: u32) -> Result<(), crate::mem::MemoryFault> { + self.window.store_u32(at, value) + } + fn load_u64(&self, at: u64) -> Result { + self.window.load_u64(at) + } + fn store_u64(&self, at: u64, value: u64) -> Result<(), crate::mem::MemoryFault> { + self.window.store_u64(at, value) + } +} + +/// A driver that shows the device a chain and takes it back, over and +/// over, gets the budget's worth of looks from one service and no more. +#[test] +fn a_driver_that_flaps_its_index_cannot_hold_the_device() { + let index = std::rc::Rc::new(core::cell::Cell::new(None)); + let looks = std::rc::Rc::new(core::cell::Cell::new(0)); + let flapping = |window| Flapping { + window, + index: index.clone(), + looks: looks.clone(), + }; + let registers = window(regs::BYTES as usize); + let guest = Guest::over(window(WINDOW), Echo::MAX_QUEUE_SIZE); + let mut transport = Transport::new(flapping(registers), Echo::default()).unwrap(); + transport.lend(flapping(guest.memory)).unwrap(); + + registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + registers.store_u32(regs::QUEUE_NUM, 8).unwrap(); + registers + .store_u64(regs::QUEUE_DESC, guest.at.descriptors) + .unwrap(); + registers + .store_u64(regs::QUEUE_DRIVER, guest.at.available) + .unwrap(); + registers + .store_u64(regs::QUEUE_DEVICE, guest.at.used) + .unwrap(); + registers.store_u32(regs::QUEUE_READY, 1).unwrap(); + registers.store_u32(regs::DRIVER_STATUS, LIVE).unwrap(); + transport.service(BUDGET); + assert_eq!(transport.status(), LIVE); + + index.set(Some(guest.at.available + 2)); + looks.set(0); + let serviced = transport.service(BUDGET); + assert!(serviced.pending, "the host is asked to come back"); + assert_eq!( + looks.get(), + 2 * BUDGET.get(), + "a look to take and a look to listen, the budget's worth" + ); + assert_eq!(transport.device().commands, 0); + assert_eq!(transport.status(), LIVE, "flapping is not a broken queue"); +} + +/// The block is there before any memory is: a driver identifies the device +/// first. A queue made ready with no memory lent is one the device cannot +/// serve. +#[test] +fn a_queue_with_no_memory_lent_asks_for_a_reset() { + let registers = window(regs::BYTES as usize); + let guest = Guest::over(window(WINDOW), Echo::MAX_QUEUE_SIZE); + let mut transport = Transport::new(registers, Echo::default()).unwrap(); + assert!(!transport.has_memory()); + assert_eq!(registers.load_u32(regs::MAGIC), Ok(regs::MAGIC_VALUE)); + assert_eq!(registers.load_u32(regs::DEVICE_ID), Ok(Echo::DEVICE_ID)); + + registers + .store_u64(regs::DRIVER_FEATURES, F_VERSION_1) + .unwrap(); + registers.store_u32(regs::DRIVER_STATUS, AGREED).unwrap(); + assert!(transport.service(BUDGET).interrupt); + assert_eq!(transport.status(), AGREED, "no memory is needed to agree"); + + registers.store_u32(regs::QUEUE_NUM, 8).unwrap(); + registers + .store_u64(regs::QUEUE_DRIVER, guest.at.available) + .unwrap(); + registers + .store_u64(regs::QUEUE_DEVICE, guest.at.used) + .unwrap(); + registers.store_u32(regs::QUEUE_READY, 1).unwrap(); + registers.store_u32(regs::DRIVER_STATUS, LIVE).unwrap(); + assert!(transport.service(BUDGET).interrupt); + assert_eq!(transport.status(), AGREED | STATUS_DEVICE_NEEDS_RESET); + + // Lending now is too late; after the reset it is not. + assert_eq!(transport.lend(guest.memory).map(|_| ()), Err(Busy)); + registers.store_u32(regs::DRIVER_STATUS, 0).unwrap(); + transport.service(BUDGET); + assert_eq!(transport.status(), 0); + assert!(transport.lend(guest.memory).unwrap().is_none()); + registers.store_u32(regs::DRIVER_STATUS, LIVE).unwrap(); + transport.service(BUDGET); + assert_eq!(transport.status(), LIVE); +} + +/// The driver's memory is the device's for as long as the device has +/// accepted anything: it is replaced or taken back only in the first state. +#[test] +fn memory_changes_only_in_the_first_state() { + let mut bench = Bench::new(); + let other = window(WINDOW); + // In the first state it is replaced, and the old is handed back. + assert!(bench.transport.lend(other).unwrap().is_some()); + assert!(bench.transport.lend(bench.guest.memory).unwrap().is_some()); + + bench.write_status(STATUS_ACKNOWLEDGE); + bench.ring(); + assert_eq!(bench.transport.lend(other).map(|_| ()), Err(Busy)); + assert_eq!(bench.transport.withdraw().map(|_| ()), Err(Busy)); + + bench.write_status(0); + bench.ring(); + bench.bring_up(); + let answer = bench.send(0, b"hello", 8); + bench.ring(); + assert_eq!(bench.guest.read(answer, 5), b"olleh"); + assert_eq!(bench.transport.withdraw().map(|_| ()), Err(Busy)); + + // After a reset the device holds no queue, and gives the memory up. + bench.reset(); + assert!(bench.transport.withdraw().unwrap().is_some()); + assert!(!bench.transport.has_memory()); + assert!(bench.transport.withdraw().unwrap().is_none()); +} + +/// The interrupt is a count: it moves exactly when a service says to +/// interrupt, after what the interrupt is about has been written. +#[test] +fn the_interrupt_count_moves_with_each_interrupt() { + let mut bench = Bench::new(); + let count = |bench: &Bench| bench.registers.load_u32(regs::INTERRUPTS).unwrap(); + assert_eq!(count(&bench), 0); + + bench.bring_up(); + let raised = count(&bench); + assert_eq!(raised, 2, "one for the features, one for the queue"); + + // Nothing to answer: no interrupt, and the count stays. + assert!(!bench.ring().interrupt); + assert_eq!(count(&bench), raised); + + // Two commands in one service are one interrupt. + bench.send(0, b"one", 8); + bench.send(2, b"two", 8); + assert!(bench.ring().interrupt); + assert_eq!(count(&bench), raised + 1); + + // A driver that asked not to be interrupted is not counted either. + let flags = bench.guest.at.available; + bench + .guest + .memory + .store_u16(flags, VIRTQ_AVAIL_F_NO_INTERRUPT) + .unwrap(); + bench.send(4, b"quiet", 8); + assert!(!bench.ring().interrupt); + assert_eq!(count(&bench), raised + 1); + assert_eq!(bench.guest.used(), Some((0, 3))); + + // The driver cannot wind the count: the device keeps its own. + bench.registers.store_u32(regs::INTERRUPTS, 0).unwrap(); + bench.write_status(0); + assert!(bench.ring().interrupt); + assert_eq!(count(&bench), raised + 2); +} diff --git a/native/provenance.json b/native/provenance.json index 6832942..77c02e9 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "dfece6447cd331375a28b2bb290691cb8d5ab060", + "revision": "cac17a02528e13f37a363ed4be71aca40f7adc70", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", @@ -17,8 +17,11 @@ "crates/scrub": "kore-scrub", "crates/start": "kore-start", "crates/uart": "kore-uart", + "crates/vdev": "kore-vdev", + "crates/vdev-accel": "kore-vdev-accel", "crates/window": "kore-window", "drivers/nvme": "kore-nvme", + "services/accel-spmd": "kore-svc-accel-spmd", "services/block": "kore-svc-block", "services/dagfs": "kore-svc-dagfs", "services/device": "kore-svc-device", @@ -28,9 +31,10 @@ "services/mm": "kore-svc-mm", "services/output": "kore-svc-output", "services/swap": "kore-svc-swap", + "services/vdev": "kore-svc-vdev", "tools/kore-vol": "kore-vol" }, - "workspace_sha256": "da01593a48d1a10e9995fa7af159b5d8675f3bba89cabb1104ac370934643fde", + "workspace_sha256": "41c4f719ce5aa21bc6919a26cac488e54a270c0454b84257c4d163dd65576222", "files": { "LICENSE": "c9b2b12d3bad9080090798a442b4b2a77c48312ae75089f041281b7c8ad21738", "crates/abi/Cargo.toml": "f9a4811b4b6d06947a47a763354cad3bc739a5ea387194a4dcf22409351f3c7b", @@ -38,10 +42,11 @@ "crates/abi/src/clock.rs": "cce1c2839cba2377d3cc8696d60812f9195f071f422bd97bb4530ea0368972fc", "crates/abi/src/command.rs": "e994b1527c989cc106281aa127df9a54dc03e1c85f3b15c49ba70ad5e5d65721", "crates/abi/src/crash.rs": "f3c9e5a954e06743f6993b0a5eeb6937d2199c50af5fe95b6be446393ccf999b", - "crates/abi/src/device.rs": "06e8891917210870b64fa2ec3d2018067233ad333cf9d1bd13b3ce848fca97ed", + "crates/abi/src/device.rs": "9767bc5990be3e7e13c0542e1f96d738f8f0220d38dbf342742cb4dd3dd1a989", "crates/abi/src/device/census.rs": "6fb54ecaade94b116f41e045b098dc5926e8e1c75eeea33870dd4be01bd1ce23", + "crates/abi/src/device/class.rs": "dacb939e32c175cde259ec2b8ba05132b810583fe6af4615479a0843cc8cc2dc", "crates/abi/src/device/i2c_admission.rs": "1ea2fc04cdb0b15186d2f1d80bde53187860e2197263e225f330965d96e40876", - "crates/abi/src/device/state.rs": "8caf4b751b4cae54f3d78626440bd847a39f1138a4dbc6055d6e8a1a1576ac32", + "crates/abi/src/device/state.rs": "a67c721ce8280c77251eac6d63a3513c8503236e89d2e5ce9d1c9e9245440f9b", "crates/abi/src/execution.rs": "106d679a3f993c3f4e7ad4eb625d40f4399f5b9b50947cb0336ba44a1d6b51af", "crates/abi/src/founding.rs": "801b8e9771763ff9223c3537aabb5653713d1883d1df62ff673f43982aa3e802", "crates/abi/src/guest/aarch64/mod.rs": "1fd24be3fa6cbc344e9873a0e3e75fabd1b919326e477ec4ea4cf148c01b66f8", @@ -50,7 +55,7 @@ "crates/abi/src/guest/wire.rs": "bec45a73cee701b07d40b37f7c296cb5e96f96d2994cd512fd1d9aad7bc7432f", "crates/abi/src/guest/x86_64/mod.rs": "58fe4ec322a03c83e30faf3fef4c7ed6ec06b64d42a33a921e977306148be342", "crates/abi/src/identity.rs": "cf69309ca5e0d53b752f427b8d82eddab874973cfa0a8e120633c326f723f264", - "crates/abi/src/image.rs": "9cc689a9f73015f8e2d3429e57fcc3243dc94a6c929ede623c50515f060065ed", + "crates/abi/src/image.rs": "19b3f6ad647cea4f1bb5a95fc305f93595c059e4dc70a03158d6c9250fc0a3ad", "crates/abi/src/input_status.rs": "5e48ead8883878c52b7ccdb22ff56d7bf04e4d74d7c689573278e7acc5e5b79c", "crates/abi/src/iommu_status.rs": "2a2ef44ec698136ebe213db38b66b08b090b8d2c74a91346a38368ba43c20b02", "crates/abi/src/keyboard.rs": "2a4e763ed9bd76151b1f7450394f3a9a4986a4b4e1a1326124db1e38eb11b8a7", @@ -65,8 +70,8 @@ "crates/abi/src/processor_status.rs": "c963e48f9b07046ab6c82d4cc2735ee2cc6c94561509a4e6e15927c837c0b0c6", "crates/abi/src/session.rs": "ba8e90dc3d1167ac81b52c6b4d6369a7a112b9417342f6697793a7c1673e96b3", "crates/abi/src/stor.rs": "eabad4e5588fd343472660e15cccf32cfcc8d8f4c25eb18dffbd1598b6007db2", - "crates/abi/src/svc.rs": "ea59db02200cdaa8f6b8f242b43ed9f4d5da4e95c0154749812ca4d22c3fbbd0", - "crates/abi/src/task.rs": "a65caa221d4cd7bcf67755fa1bab4391a1cf8cac80985cc750616ed86e50e276", + "crates/abi/src/svc.rs": "1bab27eafe928e27ebf75166f7f9da555addf71a27befdcf7494ec4b2fd81e18", + "crates/abi/src/task.rs": "17a9c177b6b3d2dd569dd1930a8d49e84c1a6d28c8ca4bd9a14163c26740d599", "crates/abi/src/uring.rs": "e98d9db77fbe00d5547b582700a1104d2c5b2546bbe57742dfff8362c32dcc2e", "crates/app/Cargo.toml": "fb793b25eec583da7b5e404b19bd656a6b8bcee2a86a20b0da399be2210a9398", "crates/app/src/lib.rs": "5e7e8d791236f278ee85a02a3c37f41412d879cc41d67f6f6d3d701811973d7d", @@ -190,12 +195,12 @@ "crates/ld/src/store.rs": "ba5a432a2a3e34e64cdd8f02dccc2c69bab5719b6a4223393c72beca0c7a2b18", "crates/ld/src/stub.rs": "b670715026a889f5baaf233a35d3d24c1891223a0e57665782ee326d90e49f5e", "crates/ld/tests/apply_engine.rs": "e01f5f805f34d614717ceb297275d65936c1b48989df90079fcd2f8ec83b6b1c", - "crates/ld/tests/built_images.rs": "400d8f9f4c3a5e62efdbbb790a9eeb3575a6ee3ecfb1aa1ba36c0f8e1fd441fc", + "crates/ld/tests/built_images.rs": "63da044373f87670aa24ec7797cb9cdcf85c4f9762c09bf36ff4a49c2729e223", "crates/ld/tests/check_gate.rs": "0ead133777f18f929d5e9b0353a289a5b4fa9e2af929155f525524096ee5578c", "crates/ld/tests/common/mod.rs": "1e4bdfa37511848d9e9458d9a550a1ff2410154411c6d026e9106035044ea438", "crates/ld/tests/late_layout.rs": "9bfc460d8036d69c1acdc52c430ee475e25062519cdc7dd30f235b882414a339", "crates/ld/tests/link_engine.rs": "a9daca8bbb29b510da1acaccc9bbb24466f9c693bd114b7a9ccb8188f31f2a59", - "crates/ld/tests/shipping_list.rs": "113bf634f5a8f83103e266149c2c8daf834f3c99b86d7e3e0693f520283a1469", + "crates/ld/tests/shipping_list.rs": "443f0560cf481e58f3b72277fb6f1dfec9cf9b58cfb369320d0c2e85e174ddea", "crates/ld/tests/static_boot_images.rs": "25fb47a0a8e5b11dca5b8c31f64b90e8d6a6edc3c4d81b0fb5e1733afea3ebe3", "crates/ring/Cargo.toml": "b0a6e8d0f919494ef9e2f8c229ff64e24709347ca76c3d7f5a6e413387c20bcb", "crates/ring/src/event.rs": "a6db5f78e8472f09da1312456f488cf7e6f9a244b1a2cc63b5ea107eb7cd9020", @@ -217,7 +222,7 @@ "crates/rt/src/cap/verbs.rs": "7d445d43fd71971a313d98b42e39bf4cb21ed528a1f2a15cbcd49ece02ee9007", "crates/rt/src/chan.rs": "fd50f962b23c2bfa15e72a1800790475d5cf6fcc55503ed39269852f025397e5", "crates/rt/src/clock/aarch64.rs": "47828a573f5fc30a988f012a648a8cb55596b45331cdcdfe1b390cd1d8543339", - "crates/rt/src/clock/mod.rs": "d402d793448359338a361563a0fea91ea43d0264a5eb38c75b9abbb87df89af9", + "crates/rt/src/clock/mod.rs": "5eb69e314f8687a265e12e28307b782d2c2cfe9dc10acab4c02e3e0f1cb2f2cc", "crates/rt/src/clock/riscv64.rs": "17d4ecdb2b4f8152c0cf51f71ce28fe331d929a62405810a1b7eb332bb275c3d", "crates/rt/src/clock/x86_64.rs": "ca499d4dc51b7ed0eb196b831295b5fdecb09a1bd2df628a82e749e27011994a", "crates/rt/src/core_id.rs": "cadde39801a1d0ed05d70d0634a9e52d58043878737d790c4130183751b622cf", @@ -261,13 +266,13 @@ "crates/rt/src/star.rs": "91de20ffe5734e7a3c553408b6866087af179b7deddb2d576f05be0ca709ab66", "crates/rt/src/startup.rs": "a005158737d337ea8c270a2860d09762048347e65d2b937734f8b59e3890d324", "crates/rt/src/svc/client.rs": "062b52a0544eb11b9d4f8a938ebe75299bfd6c2ff7f42a1d8d21e8cb487c16d7", - "crates/rt/src/svc/mod.rs": "3c62c81946943268c24aee3a0a8e3778442a5475dc7bf3bdb42780aeaaa958e9", + "crates/rt/src/svc/mod.rs": "208291ede2e702e060f0d735d2ef66034605746c5855c6e5c6d9807de8e15f6a", "crates/rt/src/svc/recyclable.rs": "9e5af8ad316bdd89e001fef7fbb6914b5a936dcdfbab2122740e03c1cf0ee7ec", "crates/rt/src/svc/serve.rs": "01ead15cbff6004fe7d13990c373cb6d50c659526942d1b1cf883f05ec72c8d6", "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", - "crates/rt/src/thread.rs": "06d0eefa92c46fe31392db8b10e891e008e95c3ee4162f9f4034f7dec7d9b35c", - "crates/rt/src/thread_controller.rs": "6ce535ded212d30b4838d15108cf9e9625b4f9e9fd83e0fecde51be833b1326d", + "crates/rt/src/thread.rs": "86278fc96aa71883905fbf1e851ea408c34256501b3712ba9459d98a16bc5131", + "crates/rt/src/thread_controller.rs": "298456742190090b14c718669b437906d1390037204e7d69a10877870029d63b", "crates/rt/src/tls.rs": "85bd95bafb3de73d249d7a2eb8779b5afc929077e27b4baa5f7c7428063063ab", "crates/rt/src/transport/mod.rs": "d5af890c800dd53abdbea8329383068cf43f0bf294ccb6b46dd7d8592ae5f321", "crates/rt/src/upcall.rs": "c0cbd2a59479dfba006850bb1bc5dd4f20f59317d178af776ba678f0e6404f02", @@ -286,6 +291,25 @@ "crates/uart/src/pl011.rs": "920c1ad2b67197ea6508de8355d966384a72b9f2ffb72cdfc1c2cbc6b056b261", "crates/uart/src/tx.rs": "532e0fef75563bf5294211502fc0da6534e51e96b4da0a71896fe629c3e5d7be", "crates/uart/src/uart16550.rs": "689427e689025653727db7bbc30a9eb4d0dbf12cc8202d4ae1aef1978f11555c", + "crates/vdev-accel/Cargo.toml": "88d35110e153faa7dfe68089d901bddc07964c32b4f0767cb23c93497c0f018c", + "crates/vdev-accel/src/device.rs": "0e285a5c781dd515263e862fd7fd7689b991d9215734b8a88d38fb6613168da8", + "crates/vdev-accel/src/device/tests.rs": "43f1871d0965c75f9ff1be052e827c68e43b6c0ae6203a5451b2eedfce770e9e", + "crates/vdev-accel/src/fp8.rs": "89d1a9e034706742ededf4fd7a89a56a3f64e9a9d03e53389914c959b760f088", + "crates/vdev-accel/src/fp8/tables.rs": "d7f6f5d80428935cf5ff8a41c18cd5e20d477cca53a70a4ad711e10777ad4a14", + "crates/vdev-accel/src/fp8/tests.rs": "2b4f879cada7a02ba986f00c54327d9c28a22199887e48e461c1d62d61cd7db4", + "crates/vdev-accel/src/lib.rs": "3ee829e95ad07e54ebad67d6f0681c2406884ad5f946407f8f3ec3e8f3d5ed50", + "crates/vdev-accel/tests/session.rs": "774c6a11da5c446d8685a7db3b3dea2aaf4f1139d7a70d4f90e581e2f6eca559", + "crates/vdev/Cargo.toml": "ba7bf6f814d0b3a8aec36be489c95b33635ce267f7cb91185b1662e90098dbec", + "crates/vdev/src/device.rs": "aee2413e3eb6710041c0c7b72fb943038625547a362dca43fb70c6e24eee1710", + "crates/vdev/src/driver.rs": "b1681703bff927b16e114586babdd40278094eb93c3fdcaebcc55a724c941f6b", + "crates/vdev/src/driver/tests.rs": "f6edc3ef3651b981438cf5dc765c6f5cb78e42d880741ea3ebb8e295fcf413b1", + "crates/vdev/src/lib.rs": "d5e950117b8e6db3b99b303e128fa3b0ccf4211a349af1dc2e5f255a679eac09", + "crates/vdev/src/mem.rs": "1b10cacbc8882c80d2ba7377366002027dbd56253e3944fb7e28d2825dcc116f", + "crates/vdev/src/regs.rs": "6257d31c9469c23b72c2b4ebe32c2558278aaae1d95c118aef051f0e7f7833e4", + "crates/vdev/src/ring.rs": "b027dc22fc2f7aa390722ffcd4bfadc95a3be684b6ed11fb181dbd6753ab90ab", + "crates/vdev/src/ring/tests.rs": "1f8fdb5b0e6741e2db26d1c5b5a06d853ca456bc4e5e2918fd8dae49894a32e4", + "crates/vdev/src/transport.rs": "a791afcb6fa52de8d0e3f3e14837520a644bea1379198ac07bedb4f2d9bfb7b6", + "crates/vdev/src/transport/tests.rs": "db2b0c9e3e92122491ae15b00eb4559370db85040c44a5a09a2efbd2c21e9fea", "crates/window/Cargo.toml": "96c2d4c89e179c3718d4b4e897bb0a44c55ac7ad66341529da4bfd35e203592e", "crates/window/src/appearance.rs": "b22e65286b94215bc4265a09ef280a5118f368ac45120311b870a56f7cc40a81", "crates/window/src/capture.rs": "d793f70fff35dfb9b13547f016b5abf8d851a23e53cfe28c1c24ad9949e6ef44", @@ -319,14 +343,18 @@ "drivers/nvme/src/qpair.rs": "45931cd91894c7e0758947e5dd9180ce12796e9698d3c70067b7af08eb53f2d8", "drivers/nvme/src/regs.rs": "d9410b5c64bb70f99e042f7209119c0f0fdb57f859c6a1c07eba90199f1e98c1", "drivers/nvme/src/swap.rs": "f5e1099b83b2cdeb9d288f4d337f1a234d9dfd16c31e9d9303c10f6f14972141", + "services/accel-spmd/Cargo.toml": "9ddcc01b82f3122a4541f9e36cea8b19e9370d56554a98abe82e39e9598f0a39", + "services/accel-spmd/src/backend.rs": "a332da8b5dd33cddcc261e5b2c024d9212900c8116a877e518016382ed6518fc", + "services/accel-spmd/src/kernel.rs": "d68d6e91faa17bc7c9e922012242bbd79a526afe872ec1655164fb81857a0b1e", + "services/accel-spmd/src/lib.rs": "4e41f01f5a34b9fce68cc9e6a51240ee9066dd495a110d49a999cd452062c56e", "services/block/Cargo.toml": "2729b36369486a6f5c1c29200eaa4603b25c26352f06f52301b70d9d57260e43", - "services/block/boot.toml": "99615c152634c9eb62532eb87c917aa3da99fc26d31a1409c4d269043ceace00", + "services/block/boot.toml": "906dbc8502392782b96449b6dcf604d998a59716a3202c1203a789ec146d0da2", "services/block/src/client.rs": "6aed79a404620f89ffa13078f615e846fdf6f3db5629c5cee5fbccbeaa7636cf", "services/block/src/lib.rs": "cf45960460cf6c9dba72f7a6727b51497275c686166a90be531d675097c275f1", "services/block/src/service.rs": "191e9d887d3f3267d6e2aa96e81996f01321bb2febb85e68581159f6e098b4b5", "services/block/src/wire.rs": "a9d64aef9ec3d58d37ca55b841933fc51b76ab94e18b245278b4519e96c65939", "services/dagfs/Cargo.toml": "d229a126616b1d7638f0f5a6edb4ac37d9d116f32afaa1897ffe484fbb286e5e", - "services/dagfs/boot.toml": "58eda46ef6e6f45bba43df5b72bd7b1c5f45fb039957f9ddedb5d333f10b6b4d", + "services/dagfs/boot.toml": "8af7f22a788922f0fd91e39f701b2a8b36e209727a5feb340e649ef578468b86", "services/dagfs/examples/fsdemo.rs": "1666a14ef62fd4fc46f7f9efa2eaa5021ed79823f273a77f53758307f884e7d0", "services/dagfs/examples/fswire.rs": "994b97d699796b317741e2c87cbe812cc1776ddcd01173323e72a5f74c8ac267", "services/dagfs/examples/model_volume.rs": "0e3623e962da02b2126062a19d198b9e0a8f6e91896fd9a6f7d13f6f30ddc5f1", @@ -369,7 +397,7 @@ "services/dagfs/tests/provider.rs": "ba4070af34ae034c109e391e97bcc4755c6bd70b8e2fb229aaf9d747fef71e2d", "services/dagfs/tests/query_experiment.rs": "6fe147810474b6754ef31ad0fb95493e0b3c0886fbbafd78deb3729d410e8155", "services/device/Cargo.toml": "8f35c9ddd701b58a538f151e95e93ce618f0b346960491adad6d843287165d1c", - "services/device/boot.toml": "4a0c84d22eb06e749e5d0803d51b659e3f8af81c52acb14491f58cec4246e752", + "services/device/boot.toml": "0e0a12ec4f34a65b39ce414661ef2a51c33957a21e5fcef16bbc8d222a7dd48a", "services/device/src/client.rs": "71660deb4a9f51edb7a72fbb3edac94170bf4cccf9ce182cbd478ee3daf406eb", "services/device/src/lib.rs": "215c7c7e81d5d51664ac77b72b2edb0bab739d5a54b30bc9ad08f9dd3a3cfea4", "services/device/src/service/mod.rs": "6038ba0b9375e3c6bbb25c49bf0849a5878a402f4a3a691dc9df1494849d252b", @@ -411,7 +439,7 @@ "services/extract/vendor/jsmn/jsmn.h": "babfcb7413354a43dcd539c600a1e00bd2d1579ad0a6f6c7d8b81dccedb16cd0", "services/extract/vendor/jsmn/kore_jsmn.c": "a666060a7af03c819d1faaccc0d6faa19b76bcfe78e4ef61aeab273ad052c95c", "services/mm/Cargo.toml": "25809b6a03916d96b8de64a5988d005c71c49314fafbe0d920c3ec06f2dc5951", - "services/mm/boot.toml": "8753f22f227455e5fff90674c7295a933b1d035ee6f3c27b7474ca9b99e149c4", + "services/mm/boot.toml": "03aa5d661af5e0e80e730d1b849aabf316a282470db26b8e1faac2bc46e1abd5", "services/mm/src/client.rs": "c6dcc0dc3ed59e0a4cf7532fd306af4c1dc4d4e5ad1edd1d86887def791c5aa6", "services/mm/src/heap.rs": "e55210f3609f2e3e90f30d5497355ddab5801acc60ccbe3f8acbbffff71dbfde", "services/mm/src/lib.rs": "d3820740666ed993e547ee1b3b579231bf439a813d9766860e3b1b57fe815bd6", @@ -423,12 +451,21 @@ "services/output/src/service/mod.rs": "d80c3beb50979b3e1e500abdfa3c0b9d38e354c24e2e34aea443732abbead6f1", "services/output/src/wire.rs": "3752e994eb18714518facae1b78a9ee2b34f99aad8687653f0b89dc8fdeca621", "services/swap/Cargo.toml": "9c0af01403a6ad5a7f4ff69a02b18c8b1b2624a684814d6dc1ac07968c5bbe53", - "services/swap/boot.toml": "bba1d75a7f4418ffa066453fa0b753eebc5ae9d7e69ac5dca2b2a0ed694479eb", + "services/swap/boot.toml": "7bf8b67cf4b04c1fb1ce3e6c6c3bd4e6d0b180b5ab5ebff04d714809020c4389", "services/swap/src/client.rs": "04f735d059b491765d5ba63a4f5f640332735bf336e62dcbae86c17647583e2c", "services/swap/src/extent.rs": "d6705f77951df1c42cda51fc4526ba6262cab6af4a0545405ce06d2d2e1db7b4", "services/swap/src/lib.rs": "883684df54a87ae66aa7895ea19a872cabc8ab7fd4aded4ea65b7e3dd51fe0cf", "services/swap/src/service/mod.rs": "34597aeb2af68085cf129ba6f6641fac52fd9d8fd9eca26dd264c9feadc77dd0", "services/swap/src/wire.rs": "ff1ab3e6b42214d0c009b8cb1bd07ffa5238348b63ca273337b427dd4637ca14", + "services/vdev/Cargo.toml": "e2485be36449450884f91cb8f00741266ca5f2cab3eca9fa21b2bd3bc48bbc05", + "services/vdev/boot.toml": "21765495f72eb2f2950950dc6029028fcd61e061664dea3af503a8ce117c1231", + "services/vdev/src/client.rs": "1bcac2d3c2b8382afdb0d69ea51fbac9198be3c0f7cf71e2da76d331eb5e1002", + "services/vdev/src/face.rs": "e9fc9cc2cf943d13ef9d9e792bee1e872f39dd6c7eb04c00a3cc4cac875fe589", + "services/vdev/src/host.rs": "3c92e97f847393e88b62b2084ca43c8b3122733c5c8761ddd60dee014c087364", + "services/vdev/src/host/tests.rs": "709bb7fd7bb0aca32410abbd776ff8356958f9dcda0438864c7cfb234cee13b0", + "services/vdev/src/lib.rs": "66cb1f43c2e38a87625fa8b8119239b7384b2be86b44aee70d6e69750c6bfbc3", + "services/vdev/src/service.rs": "72772c8f8f94e19afe302ddc8ba977494a22a593ee7c4fcd285391c87ce6ba2c", + "services/vdev/src/wire.rs": "8a9624f64e86fbb0172c6e6b93de9a44eff495575f0a6014e9a2304784ab8478", "tools/kore-vol/Cargo.toml": "d7f3f5c13b1fddd010b5d70d5e16500cec66e8a48221667f5aec84c3ce7af4d7", "tools/kore-vol/src/device.rs": "21011b8ca119bcf987486f0d79bafdd86e7219183577b27b5dd3145f8acb33f2", "tools/kore-vol/src/lib.rs": "41f7fc123e770cc1f34e97de0c2fc6a0ee9211cc29172e23cb42f01e84d2b034", diff --git a/native/services/accel-spmd/Cargo.toml b/native/services/accel-spmd/Cargo.toml new file mode 100644 index 0000000..892974e --- /dev/null +++ b/native/services/accel-spmd/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "kore-svc-accel-spmd" +version = "0.0.1" +edition = "2024" +description = "kore CPU accelerator backend candidate: the CPU-SIMD implementation of the AccelRequest contract, executing registered kernels over kore-rt's SPMD region launch." +publish = false + +[dependencies] +kore-driver = { path = "../../crates/driver", features = ["accel"] } +# The SPMD region launch (`spmd::dispatch_slices`) this row's `submit` is +# built on, and the heap its buffers come from. +kore-rt = { path = "../../crates/rt" } +# The frozen accelerator contracts the backend speaks. Same pin as the +# founded device rows: one protocol and one coherent type family. +virtio-accel-core = { version = "=0.4.1", default-features = false } + +[dev-dependencies] +kore-rt = { path = "../../crates/rt", features = ["std"] } diff --git a/native/services/accel-spmd/src/backend.rs b/native/services/accel-spmd/src/backend.rs new file mode 100644 index 0000000..d3b6e26 --- /dev/null +++ b/native/services/accel-spmd/src/backend.rs @@ -0,0 +1,1563 @@ +//! The **SPMD backend** — `impl Accelerator` over CPU workers. +//! +//! The accel protocol declares one vocabulary for every backend, "NPU / GPU / +//! CPU-SIMD" (`crates/abi/src/svc.rs`). The NPU and GPU have device-side +//! implementations; this crate supplies the CPU backend, but not yet a +//! founded service row. It needs no device, no BAR, no DMA pool and no broker +//! claim — its compute resource is the machine's own cores, reached through +//! [`kore_rt::spmd`]. +//! +//! # Why the compute is not inside `submit` +//! +//! [`Accelerator::submit`]'s contract is explicit: *"synchronous work is +//! limited to validation and admission; the call must not wait for execution +//! to finish."* That rules out running a kernel inline. The R9 resident region +//! now provides the right non-blocking engine shape, but this backend is not +//! yet founded as a service and therefore has no long-lived region owner or +//! event-descriptor storage to bind to it: +//! +//! - [`spmd::dispatch_slices`] is the reusable launch, and it **blocks until +//! done** — it owns a single static anchor and re-stages it only after +//! joining; +//! - [`spmd::found_resident`] can serve a stream of jobs, but its unsafe `env` +//! contract requires each admitted descriptor and everything it borrows to +//! remain owned until the completion generation advances. That owner belongs +//! in the future founded CPU service row, not in a stack-local `submit`. +//! +//! The current unfounded backend therefore keeps the same conformant shape +//! real hardware uses: +//! **admit, then drive, then observe.** `submit` validates and parks the work +//! in an event slot; [`SpmdBackend::run_pending`] performs one admitted +//! launch; `poll_event` reports without blocking. On a device the driving is +//! the device's; here it is the owning service's own turn — which is a +//! scheduling fact about a CPU backend, not a loosening of the contract. +//! +//! Founding the R9 engine changes `run_pending` and the private event owner; +//! no protocol surface moves. +//! +//! # What holds the bound bytes +//! +//! An admitted event records raw spans of its bound buffers rather than +//! borrowing them: the trait requires the caller to keep "every referenced +//! handle alive until the returned event is terminal and destroyed", which is +//! exactly the lifetime a raw span needs and exactly what a Rust borrow cannot +//! express across an admission boundary. The refusals in [`SpmdBackend::submit`] +//! are what make the spans sound — in particular the aliasing check, without +//! which one buffer bound as both input and output would hand the region +//! overlapping `&` and `&mut`. +//! +//! [`Accelerator::submit`]: virtio_accel_core::Accelerator::submit +//! [`spmd::dispatch_slices`]: kore_rt::spmd::dispatch_slices +//! [`spmd::found_resident`]: kore_rt::spmd::found_resident + +extern crate alloc; + +use alloc::vec::Vec; +use core::cell::{Cell, RefCell, UnsafeCell}; +use kore_driver::accel::NativeAccelerator; + +#[allow(deprecated)] +use kore_driver::accel::Dispatch; +use kore_driver::accel::LaunchShape; +use kore_driver::accel::Workgroups; +use virtio_accel_core::{ + AcceleratorClass, AccessMode, AllocatedBuffer, ArtifactRef, BackendError, BindingRef, + BufferDesc, BufferInfo, BufferProperties, Capabilities, ContextDesc, DeviceIdentity, + DeviceInfo, DeviceLimits, EventState, MemoryDomain, QueueDesc, ReleaseFailure, SubmitFailure, + Timeout, +}; + +use crate::kernel::{Kernel, TileCtx, kernel_id, lookup}; + +/// The artifact format this row serves: registered kernels, named by +/// [`TargetIdentity`](virtio_accel_core::TargetIdentity) and carrying a small, +/// fixed manifest. +/// +/// A distinct format rather than a reinterpretation of someone else's, so a +/// client that hands this row an NPU ELF is refused by *format* rather than +/// misparsed. +pub const FORMAT_REGISTERED: u32 = u32::from_le_bytes(*b"KSPM"); + +/// Versioned fixed manifest for a registered-kernel artifact: magic, version, +/// and a reserved-zero word. Keeping it nonzero makes the row admissible +/// through `virtio-accel-device`, whose program accounting correctly refuses +/// zero-byte artifacts. +pub const REGISTERED_MANIFEST: [u8; 8] = [b'K', b'S', b'P', b'M', 1, 0, 0, 0]; + +/// Contexts this backend hosts at once. +const MAX_CONTEXTS: u32 = 1; +/// Buffers per context. +const MAX_BUFFERS: u32 = 256; +/// Resident programs per context — a lookup costs nothing, so this bounds +/// only bookkeeping. +const MAX_PROGRAMS: u32 = 32; +/// Execution queues per context. +const MAX_QUEUES: u32 = 4; +/// Admitted events in flight. +const MAX_EVENTS: usize = 32; +/// Inputs one kernel may bind. Two covers the built-ins with headroom, and a +/// fixed array keeps the admitted record allocation-free. +const MAX_INPUTS: usize = 4; + +/// A host buffer. +/// +/// [`UnsafeCell`] rather than a plain `Vec` because a submission's output is +/// written through a **shared** binding (`BindingRef` carries `&B`), exactly +/// as a device backend writes through a shared DMA mapping. The exclusivity +/// argument is [`SpmdBackend::submit`]'s: one admitted launch at a time, no +/// buffer bound twice in it, and the region's workers touching disjoint +/// element spans of the one output. +pub struct SpmdBuffer { + bytes: UnsafeCell>, + desc: BufferDesc, +} + +impl SpmdBuffer { + fn len(&self) -> usize { + // SAFETY: the length is fixed at allocation and never changes; no + // writer resizes the vector. + unsafe { (*self.bytes.get()).len() } + } + + fn base(&self) -> *mut u8 { + // SAFETY: the allocation outlives every span derived from it — the + // trait pins the handle until its event is terminal and destroyed. + unsafe { (*self.bytes.get()).as_mut_ptr() } + } + + /// Read access for `read_buffer` and for host-side assertions. + fn as_slice(&self) -> &[u8] { + // SAFETY: no launch is in flight when the adapter calls a transfer + // method — `run_pending` is synchronous and the service is + // single-threaded per task. + unsafe { &*self.bytes.get() } + } +} + +/// A context. The CPU has no per-context device state; this exists so the +/// protocol's object graph is honest rather than collapsed. +pub struct SpmdContext(()); + +/// A resident program: one registered kernel. +pub struct SpmdProgram { + kernel: &'static Kernel, +} + +/// Resident bytes retained by a registered program handle. This is the +/// canonical nonzero charge a host should put in `ArtifactRef::resident_bytes`. +pub const REGISTERED_RESIDENT_BYTES: u64 = core::mem::size_of::() as u64; + +impl SpmdProgram { + /// The kernel this program resolved to — its diagnostic name is what a + /// status row prints. + pub const fn name(&self) -> &'static str { + self.kernel.name + } +} + +/// An execution queue. Ordering is per-queue and, with one admitted launch at +/// a time, trivially satisfied. +pub struct SpmdQueue(()); + +/// A submission token — an index into the backend's event table. +pub struct SpmdEvent { + slot: usize, +} + +/// One bound span, recorded at admission. +#[derive(Clone, Copy)] +struct Span { + base: *mut u8, + len: usize, +} + +/// An admitted-but-unrun launch. +struct Admitted { + kernel: &'static Kernel, + inputs: [Span; MAX_INPUTS], + n_inputs: usize, + out: Span, + elems: u32, + launch: LaunchShape, +} + +enum Slot { + Free, + Admitted(Admitted), + Terminal(EventState), +} + +/// The output span, split among the region's workers. +/// +/// [`spmd::dispatch_slices`](kore_rt::spmd::dispatch_slices) hands its body a +/// `Fn(Range) + Sync`, so the body cannot capture a `&mut [u8]`. This +/// type is the disjointness argument made explicit and contained: it carries +/// the whole span once, and [`tile`](Self::tile) hands out exactly one +/// worker's elements. The block split `dispatch_slices` performs is what makes +/// those spans disjoint, so no two workers ever alias — and every kernel above +/// this line stays entirely safe. +struct TileOut { + base: *mut u8, + len: usize, + elem: usize, +} + +// SAFETY: the only access is `tile`, which is called once per worker with that +// worker's own half-open element range. `dispatch_slices` derives those ranges +// from a block split of a single grid, so they are pairwise disjoint; distinct +// disjoint sub-slices of one allocation may be held mutably at once. +unsafe impl Sync for TileOut {} + +impl TileOut { + /// This worker's bytes. + /// + /// # Safety + /// `range` must be one worker's own block-split span for this launch, and + /// no other caller may hold an overlapping span concurrently. + unsafe fn tile(&self, range: core::ops::Range) -> &mut [u8] { + let start = range.start as usize * self.elem; + let end = range.end as usize * self.elem; + debug_assert!(end <= self.len, "a tile span must stay inside the output"); + // SAFETY: the caller's contract gives exclusivity over this span, and + // the bounds were validated at admission (`elems * elem <= out.len`). + unsafe { core::slice::from_raw_parts_mut(self.base.add(start), end - start) } + } +} + +/// The CPU accelerator row. +pub struct SpmdBackend { + registry: &'static [Kernel], + ctx_live: Cell, + buffers: Cell, + programs: Cell, + queues: Cell, + events: RefCell<[Slot; MAX_EVENTS]>, +} + +impl SpmdBackend { + /// A backend serving `registry`. + pub fn new(registry: &'static [Kernel]) -> Self { + Self { + registry, + ctx_live: Cell::new(false), + buffers: Cell::new(0), + programs: Cell::new(0), + queues: Cell::new(0), + events: RefCell::new([const { Slot::Free }; MAX_EVENTS]), + } + } + + /// **Drive one admitted launch**, oldest slot first; `true` when one ran. + /// + /// The execution step `submit` is forbidden to perform (see the module + /// docs). The owning service calls this on its own turn; a test calls it + /// directly. Blocking here is lawful and deliberate — it is the CPU + /// backend's compute, and the trait constrains `submit` and `poll_event`, + /// not the service's own scheduling. + /// + /// Off metal — or when a metal region launch refuses because the shared + /// dispatch anchor is busy or the machine is exhausted — + /// [`spmd::dispatch_slices`](kore_rt::spmd::dispatch_slices) answers + /// `None` and the work runs **serially on this thread**. That is not a + /// degraded mode to be hidden: SPMD is the acceleration here, never the + /// semantics, so the same bytes come out either way and the host suite + /// exercises this backend path. + pub fn run_pending(&self) -> bool { + let admitted = { + let mut slots = self.events.borrow_mut(); + let Some(index) = slots + .iter() + .position(|slot| matches!(slot, Slot::Admitted(_))) + else { + return false; + }; + match core::mem::replace(&mut slots[index], Slot::Free) { + Slot::Admitted(work) => (index, work), + other => { + slots[index] = other; + return false; + } + } + }; + let (index, work) = admitted; + Self::execute(&work); + self.events.borrow_mut()[index] = Slot::Terminal(EventState::Complete); + true + } + + /// Run one admitted launch over the region, or serially where there is + /// none. + fn execute(work: &Admitted) { + let kernel = work.kernel; + let elem = kernel.elem_bytes as usize; + let inputs: [&[u8]; MAX_INPUTS] = core::array::from_fn(|i| { + let span = work.inputs[i]; + if i < work.n_inputs { + // SAFETY: recorded at admission from a live buffer the trait + // pins until this event is terminal; read-only, and the + // aliasing refusal proved it is not the output. + unsafe { core::slice::from_raw_parts(span.base as *const u8, span.len) } + } else { + &[] + } + }); + let inputs = &inputs[..work.n_inputs]; + let out = TileOut { + base: work.out.base, + len: work.out.len, + elem, + }; + let total = work.elems; + let launch = work.launch; + + let body = |range: core::ops::Range| { + // SAFETY: `dispatch_slices` block-splits the grid, so this range + // is this worker's alone and disjoint from every sibling's. + let slice = unsafe { out.tile(range.clone()) }; + (kernel.run)(TileCtx { + tiles: range, + total, + launch, + inputs, + out: slice, + }); + }; + + if kore_rt::spmd::dispatch_slices(total, &body).is_none() { + // The region launch refused: same grid, one CPU worker, this + // thread. The future founded resident row removes this shared- + // anchor contention from steady-state submissions. + body(0..total); + } + } + + fn claim_event(&self) -> Result { + let slots = self.events.borrow(); + if slots.iter().any(|slot| matches!(slot, Slot::Admitted(_))) { + return Err(BackendError::Busy); + } + slots + .iter() + .position(|slot| matches!(slot, Slot::Free)) + .ok_or(BackendError::ResourceLimit) + } + + /// **The admission** both submit paths share. + /// + /// `launch` is R8's exact carried policy. Artifact-defined execution + /// derives its item count from the output; explicit workgroups are checked + /// against it without discarding their six logical axes. + fn admit( + &self, + program: &SpmdProgram, + bindings: &[BindingRef<'_, SpmdBuffer>], + timeout: Timeout, + launch: LaunchShape, + ) -> Result> { + let reject = |e| Err(SubmitFailure::Rejected(e)); + let kernel = program.kernel; + let n_in = kernel.inputs as usize; + + // A deadline would have to be enforced against a launch this backend + // does not yet interrupt, so it is refused rather than accepted and + // ignored — the same posture the reserved wire fields take. + if !matches!(timeout, Timeout::Infinite) { + return reject(BackendError::Unsupported); + } + if n_in > MAX_INPUTS || bindings.len() != n_in + 1 { + return reject(BackendError::InvalidArgument); + } + + let mut inputs = [Span { + base: core::ptr::null_mut(), + len: 0, + }; MAX_INPUTS]; + let mut out: Option = None; + let mut seen = [false; MAX_INPUTS + 1]; + + for binding in bindings { + let slot = binding.slot as usize; + if slot > n_in || seen[slot] { + return reject(BackendError::InvalidArgument); + } + seen[slot] = true; + + let buffer = binding.buffer; + let Ok(offset) = usize::try_from(binding.range.offset) else { + return reject(BackendError::OutOfBounds); + }; + let Ok(len) = usize::try_from(binding.range.bytes()) else { + return reject(BackendError::OutOfBounds); + }; + let end = match offset.checked_add(len) { + Some(end) if end <= buffer.len() => end, + _ => return reject(BackendError::OutOfBounds), + }; + // Defense in depth: the host is required to have checked this, and + // repeating it costs a comparison against a client-shaped mistake + // that would otherwise write through a read-only declaration. + if !buffer.desc.allows_access(binding.access) { + return reject(BackendError::InvalidArgument); + } + let want = if slot == n_in { + AccessMode::Write + } else { + AccessMode::Read + }; + if binding.access != want { + return reject(BackendError::InvalidArgument); + } + let span = Span { + // SAFETY: `offset <= end <= buffer.len()`, so the offset stays + // inside the one allocation this pointer is derived from. + base: unsafe { buffer.base().add(offset) }, + len: end - offset, + }; + if slot == n_in { + out = Some(span); + } else { + inputs[slot] = span; + } + } + + let Some(out) = out else { + return reject(BackendError::InvalidArgument); + }; + + // **The aliasing refusal** — the check the recorded raw spans rest on. + // One buffer bound as both an input and the output would hand the + // region overlapping shared and exclusive views of the same bytes, and + // no amount of care inside the kernel could make that sound. + for input in inputs.iter().take(n_in) { + let lo = input.base as usize; + let hi = lo + input.len; + let olo = out.base as usize; + let ohi = olo + out.len; + if lo < ohi && olo < hi { + return reject(BackendError::InvalidArgument); + } + } + + let elem = kernel.elem_bytes as usize; + if elem == 0 || out.len % elem != 0 { + return reject(BackendError::InvalidArgument); + } + let Ok(derived) = u32::try_from(out.len / elem) else { + return reject(BackendError::OutOfBounds); + }; + let named = match launch { + LaunchShape::ArtifactDefined => None, + LaunchShape::Workgroups(workgroups) => { + let items = workgroups + .group_count() + .into_iter() + .chain(workgroups.local_size()) + .try_fold(1u64, |acc, axis| acc.checked_mul(u64::from(axis))); + let Some(items) = items.and_then(|items| u32::try_from(items).ok()) else { + return reject(BackendError::OutOfBounds); + }; + Some(items) + } + }; + // A carried geometry must describe the launch the bound output can + // actually hold. Silently clamping to the smaller of the two would run + // a different launch than the client asked for and report success. + let elems = match named { + Some(named) if named != derived => return reject(BackendError::InvalidArgument), + Some(named) => named, + None => derived, + }; + // Every input must cover the same grid: a short input would read past + // its own span in the kernel's element indexing. + for input in inputs.iter().take(n_in) { + if input.len < out.len { + return reject(BackendError::OutOfBounds); + } + } + + let slot = self.claim_event().map_err(SubmitFailure::Rejected)?; + self.events.borrow_mut()[slot] = Slot::Admitted(Admitted { + kernel, + inputs, + n_inputs: n_in, + out, + elems, + launch, + }); + Ok(SpmdEvent { slot }) + } +} + +impl virtio_accel_core::Accelerator for SpmdBackend { + type Context = SpmdContext; + type Buffer = SpmdBuffer; + type Program = SpmdProgram; + type Queue = SpmdQueue; + type Event = SpmdEvent; + + fn device_info(&self) -> Result { + Ok(DeviceInfo { + identity: DeviceIdentity { + uuid: *b"kore-spmd-cpu\0\0\0", + // The protocol assigns NPU/GPU/DSP and no CPU value. `OTHER` + // is the honest answer: inventing a discriminant here would + // claim a number the house protocol has not allocated. + class: AcceleratorClass::OTHER, + vendor_id: 0, + device_id: 0, + }, + // Host DRAM is both host-visible and directly bindable; there is + // no device-local domain and no cancellation path. + capabilities: Capabilities::HOST_VISIBLE_MEMORY | Capabilities::SHARED_MEMORY, + limits: DeviceLimits { + max_contexts: MAX_CONTEXTS, + max_buffers_per_context: MAX_BUFFERS, + max_programs_per_context: MAX_PROGRAMS, + max_queues_per_context: MAX_QUEUES, + max_events_per_context: MAX_EVENTS as u32, + max_bindings_per_submission: (MAX_INPUTS + 1) as u32, + max_buffer_bytes: 1 << 31, + max_artifact_bytes: REGISTERED_MANIFEST.len() as u64, + }, + }) + } + + fn create_context(&self, desc: ContextDesc) -> Result { + if !desc.flags.is_empty() { + return Err(BackendError::Unsupported); + } + if self.ctx_live.replace(true) { + return Err(BackendError::ResourceLimit); + } + Ok(SpmdContext(())) + } + + fn destroy_context(&self, context: Self::Context) -> Result<(), ReleaseFailure> { + if self.buffers.get() != 0 || self.programs.get() != 0 || self.queues.get() != 0 { + return Err(ReleaseFailure::Rejected { + error: BackendError::Busy, + resource: context, + }); + } + if self + .events + .borrow() + .iter() + .any(|slot| !matches!(slot, Slot::Free)) + { + return Err(ReleaseFailure::Rejected { + error: BackendError::Busy, + resource: context, + }); + } + self.ctx_live.set(false); + Ok(()) + } + + fn allocate_buffer( + &self, + _context: &Self::Context, + desc: BufferDesc, + ) -> Result, BackendError> { + match desc.domain { + MemoryDomain::Host | MemoryDomain::Shared => {} + MemoryDomain::Device => return Err(BackendError::Unsupported), + } + if self.buffers.get() >= MAX_BUFFERS { + return Err(BackendError::ResourceLimit); + } + let bytes = usize::try_from(desc.bytes()).map_err(|_| BackendError::OutOfBounds)?; + let mut data = Vec::new(); + data.try_reserve_exact(bytes) + .map_err(|_| BackendError::OutOfMemory)?; + data.resize(bytes, 0); + + // The allocator's guarantee, stated rather than assumed: a `Vec` + // is byte-aligned, so a stricter request is refused instead of being + // silently under-served — `BufferInfo::new` would catch it, and + // catching it here names the reason. + let alignment = desc.alignment(); + if alignment > 1 && (data.as_ptr() as u64) % alignment != 0 { + return Err(BackendError::Incompatible); + } + let info = BufferInfo::new( + desc, + desc.bytes(), + alignment, + BufferProperties::HOST_VISIBLE | BufferProperties::DIRECT_BINDING, + )?; + self.buffers.set(self.buffers.get() + 1); + Ok(AllocatedBuffer::new( + SpmdBuffer { + bytes: UnsafeCell::new(data), + desc, + }, + info, + )) + } + + fn write_buffer( + &self, + buffer: &mut Self::Buffer, + offset: u64, + data: &dyn virtio_accel_core::ByteSource, + ) -> Result<(), BackendError> { + if !buffer + .desc + .usage + .contains(virtio_accel_core::BufferUsage::TRANSFER_DESTINATION) + { + return Err(BackendError::PermissionDenied); + } + let at = usize::try_from(offset).map_err(|_| BackendError::OutOfBounds)?; + let len = usize::try_from(data.len()).map_err(|_| BackendError::OutOfBounds)?; + let end = at.checked_add(len).ok_or(BackendError::OutOfBounds)?; + if end > buffer.len() { + return Err(BackendError::OutOfBounds); + } + // SAFETY: `&mut Self::Buffer` is exclusive, and no launch is in flight + // (`run_pending` is synchronous). + let bytes = unsafe { &mut *buffer.bytes.get() }; + data.read_at(0, &mut bytes[at..end]) + } + + fn read_buffer( + &self, + buffer: &Self::Buffer, + offset: u64, + data: &mut dyn virtio_accel_core::ByteSink, + ) -> Result<(), BackendError> { + if !buffer + .desc + .usage + .contains(virtio_accel_core::BufferUsage::TRANSFER_SOURCE) + { + return Err(BackendError::PermissionDenied); + } + let at = usize::try_from(offset).map_err(|_| BackendError::OutOfBounds)?; + let len = usize::try_from(data.len()).map_err(|_| BackendError::OutOfBounds)?; + let end = at.checked_add(len).ok_or(BackendError::OutOfBounds)?; + if end > buffer.len() { + return Err(BackendError::OutOfBounds); + } + data.write_at(0, &buffer.as_slice()[at..end]) + } + + fn free_buffer(&self, buffer: Self::Buffer) -> Result<(), ReleaseFailure> { + drop(buffer); + self.buffers.set(self.buffers.get().saturating_sub(1)); + Ok(()) + } + + fn load_program( + &self, + _context: &Self::Context, + artifact: ArtifactRef<'_>, + ) -> Result { + if artifact.format.get() != FORMAT_REGISTERED { + return Err(BackendError::Unsupported); + } + if artifact.payload.len() != REGISTERED_MANIFEST.len() as u64 { + return Err(BackendError::InvalidArgument); + } + let mut manifest = [0u8; REGISTERED_MANIFEST.len()]; + artifact.payload.read_at(0, &mut manifest)?; + if manifest != REGISTERED_MANIFEST { + return Err(BackendError::InvalidArgument); + } + if artifact.resident_bytes < REGISTERED_RESIDENT_BYTES { + return Err(BackendError::ResourceLimit); + } + let Some(id) = kernel_id(artifact.target) else { + return Err(BackendError::InvalidArgument); + }; + let Some(kernel) = lookup(self.registry, id) else { + return Err(BackendError::Incompatible); + }; + if self.programs.get() >= MAX_PROGRAMS { + return Err(BackendError::ResourceLimit); + } + self.programs.set(self.programs.get() + 1); + Ok(SpmdProgram { kernel }) + } + + fn unload_program(&self, program: Self::Program) -> Result<(), ReleaseFailure> { + drop(program); + self.programs.set(self.programs.get().saturating_sub(1)); + Ok(()) + } + + fn create_queue( + &self, + _context: &Self::Context, + desc: QueueDesc, + ) -> Result { + if !desc.flags.is_empty() { + return Err(BackendError::Unsupported); + } + if self.queues.get() >= MAX_QUEUES { + return Err(BackendError::ResourceLimit); + } + self.queues.set(self.queues.get() + 1); + Ok(SpmdQueue(())) + } + + fn destroy_queue(&self, queue: Self::Queue) -> Result<(), ReleaseFailure> { + drop(queue); + self.queues.set(self.queues.get().saturating_sub(1)); + Ok(()) + } + + fn submit( + &self, + _queue: &Self::Queue, + program: &Self::Program, + bindings: &[BindingRef<'_, Self::Buffer>], + timeout: Timeout, + ) -> Result> { + self.admit(program, bindings, timeout, LaunchShape::ArtifactDefined) + } + + fn poll_event(&self, event: &Self::Event) -> Result { + match &self.events.borrow()[event.slot] { + Slot::Admitted(_) => Ok(EventState::Pending), + Slot::Terminal(state) => Ok(*state), + // A destroyed or never-admitted slot: the adapter is holding an + // event it already retired. + Slot::Free => Err(BackendError::InvalidArgument), + } + } + + fn destroy_event(&self, event: Self::Event) -> Result<(), ReleaseFailure> { + let mut slots = self.events.borrow_mut(); + if matches!(slots[event.slot], Slot::Admitted(_)) { + drop(slots); + return Err(ReleaseFailure::Rejected { + error: BackendError::Busy, + resource: event, + }); + } + slots[event.slot] = Slot::Free; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::kernel::{BUILTIN, IDENTITY_MAGIC, KERNEL_ADD_F32}; + use kore_driver::accel::LaunchShape; + use kore_driver::accel::Workgroups; + use std::vec; + use virtio_accel_core::{ + Accelerator, ArtifactFormat, BufferRange, BufferUsage, ContextDesc, ContextFlags, + QueueFlags, TargetIdentity, + }; + + const ELEMS: usize = 19; // two SIMD steps plus a tail + const WIDE: usize = ELEMS * 4; + const BYTES: u64 = WIDE as u64; + const KERNEL_OBSERVE_LAUNCH: u32 = 0xfeed; + + /// Report the carried launch through the kernel's own output span: + /// element `i` becomes the launch axis at `i % 6`. Each element's value + /// depends only on the launch and its own index, so every tiling of the + /// grid — SPMD workers or the serial fallback — writes the same bytes, + /// and the report lands in the submitting test's buffer rather than + /// process state a concurrently running test could clobber. + fn observe_launch(ctx: TileCtx<'_>) { + let TileCtx { + tiles, launch, out, .. + } = ctx; + let LaunchShape::Workgroups(workgroups) = launch else { + return; + }; + let [gx, gy, gz] = workgroups.group_count(); + let [lx, ly, lz] = workgroups.local_size(); + let axes = [gx, gy, gz, lx, ly, lz]; + for (i, tile) in tiles.enumerate() { + let value = axes[tile as usize % axes.len()] as f32; + out[i * 4..i * 4 + 4].copy_from_slice(&value.to_le_bytes()); + } + } + + static OBSERVING_REGISTRY: &[Kernel] = &[Kernel { + id: KERNEL_OBSERVE_LAUNCH, + name: "observe.launch", + inputs: 0, + elem_bytes: 4, + lanes: 1, + run: observe_launch, + }]; + + /// The six axes [`observe_launch`] reported into `buffer`. + fn observed_launch(b: &SpmdBackend, buffer: &SpmdBuffer) -> [u32; 6] { + let floats = read_floats(b, buffer); + core::array::from_fn(|i| floats[i] as u32) + } + + /// `ByteSource`/`ByteSink` are implemented for sized arrays, so transfers + /// go through one — a `&[u8]` cannot become a `&dyn ByteSource`. + fn packed(values: &[f32]) -> [u8; WIDE] { + let mut out = [0u8; WIDE]; + for (i, v) in values.iter().enumerate() { + out[i * 4..i * 4 + 4].copy_from_slice(&v.to_le_bytes()); + } + out + } + + fn backend() -> SpmdBackend { + SpmdBackend::new(BUILTIN) + } + + fn desc(usage: BufferUsage) -> BufferDesc { + BufferDesc::new(BYTES, 1, MemoryDomain::Host, usage).expect("descriptor") + } + + fn alloc(b: &SpmdBackend, ctx: &SpmdContext, usage: BufferUsage, values: &[f32]) -> SpmdBuffer { + let transfer_usage = + usage | BufferUsage::TRANSFER_SOURCE | BufferUsage::TRANSFER_DESTINATION; + let mut allocated = b + .allocate_buffer(ctx, desc(transfer_usage)) + .expect("allocated"); + b.write_buffer(allocated.buffer_mut(), 0, &packed(values)) + .expect("written"); + allocated.into_parts().0 + } + + fn program(b: &SpmdBackend, ctx: &SpmdContext, id: u32) -> SpmdProgram { + let mut words = [0u32; 12]; + words[0] = IDENTITY_MAGIC; + words[1] = id; + b.load_program( + ctx, + ArtifactRef { + format: ArtifactFormat::new(FORMAT_REGISTERED).expect("format"), + target: TargetIdentity(words), + payload: ®ISTERED_MANIFEST, + resident_bytes: REGISTERED_RESIDENT_BYTES, + }, + ) + .expect("loaded") + } + + fn bind<'a>( + slot: u32, + buffer: &'a SpmdBuffer, + access: AccessMode, + ) -> BindingRef<'a, SpmdBuffer> { + BindingRef { + slot, + buffer, + range: BufferRange::new(0, BYTES).expect("range"), + access, + } + } + + fn read_floats(b: &SpmdBackend, buffer: &SpmdBuffer) -> vec::Vec { + let mut out = [0u8; WIDE]; + b.read_buffer(buffer, 0, &mut out).expect("read"); + out.chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect() + } + + /// The whole path on a machine with no region: context, buffers, program, + /// admission, the drive step, and a terminal poll carrying the right + /// bytes. This is the claim the row exists to make — that the accel + /// protocol admits a non-device tenant — and it runs on any host. + #[test] + fn a_submission_runs_end_to_end_and_produces_the_right_bytes() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + + let lhs: vec::Vec = (0..ELEMS).map(|i| i as f32).collect(); + let rhs: vec::Vec = (0..ELEMS).map(|i| (i * 3) as f32).collect(); + let a = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &lhs); + let c = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &rhs); + let out = alloc(&b, &ctx, BufferUsage::PROGRAM_OUTPUT, &vec![0.0; ELEMS]); + let prog = program(&b, &ctx, KERNEL_ADD_F32); + + let event = b + .submit( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ], + Timeout::Infinite, + ) + .map_err(|_| "submit") + .expect("admitted"); + + assert!( + matches!(b.poll_event(&event), Ok(EventState::Pending)), + "admission must not have executed anything" + ); + assert!(b.run_pending(), "one launch was admitted"); + assert!(!b.run_pending(), "and only one"); + assert!(matches!(b.poll_event(&event), Ok(EventState::Complete))); + + let got = read_floats(&b, &out); + for i in 0..ELEMS { + assert_eq!(got[i], lhs[i] + rhs[i], "element {i}"); + } + b.destroy_event(event) + .map_err(|_| "destroy") + .expect("freed"); + } + + /// `load_program` refuses every shape that is not a registered kernel, + /// and each refusal is distinguishable — a client can tell "wrong format" + /// from "unknown kernel". + #[test] + fn load_program_refuses_everything_that_is_not_a_registered_kernel() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let good = TargetIdentity({ + let mut w = [0u32; 12]; + w[0] = IDENTITY_MAGIC; + w[1] = KERNEL_ADD_F32; + w + }); + let fmt = ArtifactFormat::new(FORMAT_REGISTERED).expect("format"); + + let other_format = ArtifactFormat::new(FORMAT_REGISTERED ^ 1).expect("format"); + assert!(matches!( + b.load_program( + &ctx, + ArtifactRef { + format: other_format, + target: good, + payload: ®ISTERED_MANIFEST, + resident_bytes: REGISTERED_RESIDENT_BYTES + } + ), + Err(BackendError::Unsupported) + )); + + assert!( + matches!( + b.load_program( + &ctx, + ArtifactRef { + format: fmt, + target: good, + payload: b"bytes", + resident_bytes: REGISTERED_RESIDENT_BYTES + } + ), + Err(BackendError::InvalidArgument) + ), + "a malformed manifest must be refused, never silently ignored" + ); + + let mut wrong_version = REGISTERED_MANIFEST; + wrong_version[4] = 2; + assert!(matches!( + b.load_program( + &ctx, + ArtifactRef { + format: fmt, + target: good, + payload: &wrong_version, + resident_bytes: REGISTERED_RESIDENT_BYTES + } + ), + Err(BackendError::InvalidArgument) + )); + + assert!(matches!( + b.load_program( + &ctx, + ArtifactRef { + format: fmt, + target: good, + payload: ®ISTERED_MANIFEST, + resident_bytes: REGISTERED_RESIDENT_BYTES - 1 + } + ), + Err(BackendError::ResourceLimit) + )); + + assert!(matches!( + b.load_program( + &ctx, + ArtifactRef { + format: fmt, + target: TargetIdentity([0; 12]), + payload: ®ISTERED_MANIFEST, + resident_bytes: REGISTERED_RESIDENT_BYTES + } + ), + Err(BackendError::InvalidArgument) + )); + + let unknown = TargetIdentity({ + let mut w = [0u32; 12]; + w[0] = IDENTITY_MAGIC; + w[1] = 0xdead; + w + }); + assert!( + matches!( + b.load_program( + &ctx, + ArtifactRef { + format: fmt, + target: unknown, + payload: ®ISTERED_MANIFEST, + resident_bytes: REGISTERED_RESIDENT_BYTES + } + ), + Err(BackendError::Incompatible) + ), + "a well-formed identity naming no kernel is Incompatible, not invalid" + ); + } + + /// **The refusal the recorded raw spans rest on.** One buffer bound as + /// both an input and the output would give the region overlapping shared + /// and exclusive views of the same bytes. + #[test] + fn submit_refuses_a_buffer_bound_as_both_input_and_output() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let shared = alloc(&b, &ctx, BufferUsage::MUTABLE_STATE, &vec![1.0; ELEMS]); + let other = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![2.0; ELEMS]); + let prog = program(&b, &ctx, KERNEL_ADD_F32); + + let refused = b.submit( + &queue, + &prog, + &[ + bind(0, &shared, AccessMode::Read), + bind(1, &other, AccessMode::Read), + bind(2, &shared, AccessMode::Write), + ], + Timeout::Infinite, + ); + assert!( + matches!( + refused, + Err(SubmitFailure::Rejected(BackendError::InvalidArgument)) + ), + "aliasing the output with an input must refuse" + ); + } + + /// Admission validates shape before it records anything: binding count, + /// access direction, and a deadline this backend cannot enforce. + #[test] + fn submit_refuses_malformed_admissions() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let a = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![1.0; ELEMS]); + let c = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![2.0; ELEMS]); + let out = alloc(&b, &ctx, BufferUsage::PROGRAM_OUTPUT, &vec![0.0; ELEMS]); + let prog = program(&b, &ctx, KERNEL_ADD_F32); + + let short = b.submit( + &queue, + &prog, + &[bind(0, &a, AccessMode::Read), bind(1, &c, AccessMode::Read)], + Timeout::Infinite, + ); + assert!( + matches!( + short, + Err(SubmitFailure::Rejected(BackendError::InvalidArgument)) + ), + "a missing output binding must refuse" + ); + + let wrong_access = b.submit( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Read), + ], + Timeout::Infinite, + ); + assert!( + matches!( + wrong_access, + Err(SubmitFailure::Rejected(BackendError::InvalidArgument)) + ), + "the output slot must be bound writable" + ); + + let deadline = b.submit( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ], + Timeout::AfterNs(core::num::NonZeroU64::new(1_000).expect("nonzero")), + ); + assert!( + matches!( + deadline, + Err(SubmitFailure::Rejected(BackendError::Unsupported)) + ), + "a deadline this backend cannot enforce is refused, not ignored" + ); + } + + /// An admitted event owns its bound spans until it is terminal, so it + /// cannot be retired out from under the launch it authorized. + #[test] + fn an_admitted_event_refuses_destruction_until_it_has_run() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let a = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![1.0; ELEMS]); + let c = alloc(&b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![2.0; ELEMS]); + let out = alloc(&b, &ctx, BufferUsage::PROGRAM_OUTPUT, &vec![0.0; ELEMS]); + let prog = program(&b, &ctx, KERNEL_ADD_F32); + + let event = b + .submit( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ], + Timeout::Infinite, + ) + .map_err(|_| "submit") + .expect("admitted"); + + let event = match b.destroy_event(event) { + Err(ReleaseFailure::Rejected { error, resource }) => { + assert!(matches!(error, BackendError::Busy)); + resource + } + _ => panic!("destroying an unrun event must be rejected, and hand it back"), + }; + assert!(b.run_pending()); + b.destroy_event(event) + .map_err(|_| "destroy") + .expect("terminal events retire"); + } + + /// Raw admitted spans are sound only under the backend's global + /// single-execution rule. A second admission refuses until the first has + /// actually run; a merely terminal event does not consume that execution + /// slot. + #[test] + fn only_one_execution_is_admitted_globally() { + let b = backend(); + let (_ctx, queue, a, c, out, prog) = rig(&b); + let bindings = || { + [ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ] + }; + + let first = b + .submit(&queue, &prog, &bindings(), Timeout::Infinite) + .map_err(|_| "first submit") + .expect("first admission"); + assert!(matches!( + b.submit(&queue, &prog, &bindings(), Timeout::Infinite), + Err(SubmitFailure::Rejected(BackendError::Busy)) + )); + + assert!(b.run_pending()); + let second = b + .submit(&queue, &prog, &bindings(), Timeout::Infinite) + .map_err(|_| "second submit") + .expect("terminal work no longer blocks admission"); + assert!(b.run_pending()); + b.destroy_event(first) + .map_err(|_| "destroy first") + .expect("first terminal event retires"); + b.destroy_event(second) + .map_err(|_| "destroy second") + .expect("second terminal event retires"); + } + + /// Fixture: a context, a queue, two inputs, an output, and a program. + fn rig( + b: &SpmdBackend, + ) -> ( + SpmdContext, + SpmdQueue, + SpmdBuffer, + SpmdBuffer, + SpmdBuffer, + SpmdProgram, + ) { + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let a = alloc(b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![1.0; ELEMS]); + let c = alloc(b, &ctx, BufferUsage::PROGRAM_INPUT, &vec![2.0; ELEMS]); + let out = alloc(b, &ctx, BufferUsage::PROGRAM_OUTPUT, &vec![0.0; ELEMS]); + let prog = program(b, &ctx, KERNEL_ADD_F32); + (ctx, queue, a, c, out, prog) + } + + /// **R8's path**: a geometry whose work-item count matches the bound + /// output admits, and the launch computes the same result the + /// geometry-free path does. + #[test] + fn a_carried_geometry_matching_the_output_admits_and_runs() { + let b = backend(); + let (_ctx, queue, a, c, out, prog) = rig(&b); + // 19 items as a 19×1×1 grid of 1×1×1 groups. + let workgroups = Workgroups::new([ELEMS as u32, 1, 1], [1, 1, 1]).expect("geometry"); + + let event = b + .submit_with_launch( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ], + LaunchShape::Workgroups(workgroups), + Timeout::Infinite, + ) + .map_err(|_| "submit_with_launch") + .expect("admitted"); + assert!(b.run_pending()); + assert!(matches!(b.poll_event(&event), Ok(EventState::Complete))); + assert!( + read_floats(&b, &out).iter().all(|&v| v == 3.0), + "1.0 + 2.0 across the whole carried grid" + ); + } + + /// Equal products are not equal launch shapes. Preserve all six logical + /// axes through admission and the CPU worker scheduler into the registered + /// kernel's `TileCtx`. + #[test] + fn equal_product_workgroup_shapes_reach_the_kernel_distinctly() { + let b = SpmdBackend::new(OBSERVING_REGISTRY); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let out = alloc(&b, &ctx, BufferUsage::PROGRAM_OUTPUT, &vec![0.0; ELEMS]); + let prog = program(&b, &ctx, KERNEL_OBSERVE_LAUNCH); + + let shapes = [ + Workgroups::new([ELEMS as u32, 1, 1], [1, 1, 1]).unwrap(), + Workgroups::new([1, 1, 1], [ELEMS as u32, 1, 1]).unwrap(), + ]; + let expected = [[ELEMS as u32, 1, 1, 1, 1, 1], [1, 1, 1, ELEMS as u32, 1, 1]]; + + for (shape, expected) in shapes.into_iter().zip(expected) { + let event = b + .submit_with_launch( + &queue, + &prog, + &[bind(0, &out, AccessMode::Write)], + LaunchShape::Workgroups(shape), + Timeout::Infinite, + ) + .map_err(|_| "submit_with_launch") + .expect("same-sized explicit shape admits"); + assert!(b.run_pending()); + assert_eq!(observed_launch(&b, &out), expected); + b.destroy_event(event) + .map_err(|_| "destroy") + .expect("terminal event retires"); + } + } + + /// Deployed OP13 geometry stays exact to the CPU backend: equal products + /// retain their distinct axes, while a client SIMD width that differs from + /// the compiled registered kernel is refused before admission. + #[test] + #[allow(deprecated)] + fn legacy_dispatch_requires_compiled_simd_and_preserves_axes() { + let b = SpmdBackend::new(OBSERVING_REGISTRY); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let queue = b + .create_queue( + &ctx, + QueueDesc { + flags: QueueFlags::empty(), + }, + ) + .expect("queue"); + let out = alloc(&b, &ctx, BufferUsage::PROGRAM_OUTPUT, &vec![0.0; ELEMS]); + let prog = program(&b, &ctx, KERNEL_OBSERVE_LAUNCH); + + let exact = Dispatch::new([1, 1, 1], [ELEMS as u32, 1, 1], 1).unwrap(); + let event = b + .submit_dispatch( + &queue, + &prog, + &[bind(0, &out, AccessMode::Write)], + exact, + Timeout::Infinite, + ) + .map_err(|_| "legacy submit") + .expect("compiled width admits"); + assert!(b.run_pending()); + assert_eq!(observed_launch(&b, &out), [1, 1, 1, ELEMS as u32, 1, 1]); + b.destroy_event(event) + .map_err(|_| "destroy") + .expect("terminal event retires"); + + let wrong_simd = Dispatch::new([1, 1, 1], [ELEMS as u32, 1, 1], 2).unwrap(); + assert!(matches!( + b.submit_dispatch( + &queue, + &prog, + &[bind(0, &out, AccessMode::Write)], + wrong_simd, + Timeout::Infinite, + ), + Err(SubmitFailure::Rejected(BackendError::Incompatible)) + )); + } + + /// A grid the bound output cannot hold is refused rather than clamped — + /// clamping would run a different launch than the client asked for and + /// report success. + #[test] + fn a_geometry_the_output_cannot_hold_is_refused() { + let b = backend(); + let (_ctx, queue, a, c, out, prog) = rig(&b); + let too_big = Workgroups::new([ELEMS as u32 + 1, 1, 1], [1, 1, 1]).expect("geometry"); + assert!(matches!( + b.submit_with_launch( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ], + LaunchShape::Workgroups(too_big), + Timeout::Infinite, + ), + Err(SubmitFailure::Rejected(BackendError::InvalidArgument)) + )); + } + + /// Explicit one-item geometry is literal and cannot alias the separate + /// artifact-defined mode. + #[test] + fn explicit_one_item_launch_does_not_mean_artifact_defined() { + let b = backend(); + let (_ctx, queue, a, c, out, prog) = rig(&b); + assert!(matches!( + b.submit_with_launch( + &queue, + &prog, + &[ + bind(0, &a, AccessMode::Read), + bind(1, &c, AccessMode::Read), + bind(2, &out, AccessMode::Write), + ], + LaunchShape::Workgroups(Workgroups::new([1; 3], [1; 3]).unwrap()), + Timeout::Infinite, + ), + Err(SubmitFailure::Rejected(BackendError::InvalidArgument)) + )); + } + + /// A device-local request has no honest answer on a CPU row. + #[test] + fn device_local_memory_is_refused_rather_than_faked() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let device = BufferDesc::new(BYTES, 1, MemoryDomain::Device, BufferUsage::PROGRAM_INPUT) + .expect("descriptor"); + assert!(matches!( + b.allocate_buffer(&ctx, device), + Err(BackendError::Unsupported) + )); + } + + /// Transfer permissions are a provider contract too, not merely a check + /// supplied by the generic command service. Keeping the check here makes + /// direct providers and future non-virtio callers preserve the descriptor's + /// authority boundary. + #[test] + fn transfers_require_declared_usage() { + let b = backend(); + let ctx = b + .create_context(ContextDesc { + flags: ContextFlags::empty(), + }) + .expect("context"); + let mut source_only = b + .allocate_buffer(&ctx, desc(BufferUsage::TRANSFER_SOURCE)) + .expect("source buffer"); + let mut destination_only = b + .allocate_buffer(&ctx, desc(BufferUsage::TRANSFER_DESTINATION)) + .expect("destination buffer"); + let bytes = [0x5au8; WIDE]; + let mut sink = [0u8; WIDE]; + + assert_eq!( + b.write_buffer(source_only.buffer_mut(), 0, &bytes), + Err(BackendError::PermissionDenied) + ); + assert_eq!( + b.read_buffer(destination_only.buffer(), 0, &mut sink), + Err(BackendError::PermissionDenied) + ); + b.write_buffer(destination_only.buffer_mut(), 0, &bytes) + .expect("write-enabled descriptor"); + b.read_buffer(source_only.buffer(), 0, &mut sink) + .expect("read-enabled descriptor"); + } + + /// The row advertises its bounded nonzero manifest and a class the + /// protocol has actually allocated. + #[test] + fn device_info_advertises_the_registered_manifest_bound() { + let info = backend().device_info().expect("info"); + assert_eq!( + info.validate(), + Ok(()), + "the generic host can admit this row" + ); + assert_eq!( + info.limits.max_artifact_bytes, + REGISTERED_MANIFEST.len() as u64 + ); + assert!(REGISTERED_RESIDENT_BYTES > 0); + assert!(::EXPLICIT_WORKGROUPS); + assert_eq!(info.identity.class, AcceleratorClass::OTHER); + } +} + +impl NativeAccelerator for SpmdBackend { + const EXPLICIT_WORKGROUPS: bool = true; + /// Honor the deployed OP13 dialect without letting its client SIMD become + /// the preferred portable contract. The width is accepted only when it is + /// exactly the compiled registered kernel's lane width; only after that + /// proof are the unchanged grid/group axes lowered to logical workgroups. + #[allow(deprecated)] + fn submit_dispatch( + &self, + _queue: &Self::Queue, + program: &Self::Program, + bindings: &[BindingRef<'_, Self::Buffer>], + dispatch: Dispatch, + timeout: Timeout, + ) -> Result> { + if dispatch.is_scalar() { + return self.admit(program, bindings, timeout, LaunchShape::ArtifactDefined); + } + if dispatch.simd_width() != program.kernel.lanes { + return Err(SubmitFailure::Rejected(BackendError::Incompatible)); + } + let workgroups = + Workgroups::new(dispatch.grid(), dispatch.group()).map_err(SubmitFailure::Rejected)?; + self.admit( + program, + bindings, + timeout, + LaunchShape::Workgroups(workgroups), + ) + } + fn submit_with_launch( + &self, + _queue: &Self::Queue, + program: &Self::Program, + bindings: &[BindingRef<'_, Self::Buffer>], + launch: LaunchShape, + timeout: Timeout, + ) -> Result> { + self.admit(program, bindings, timeout, launch) + } +} diff --git a/native/services/accel-spmd/src/kernel.rs b/native/services/accel-spmd/src/kernel.rs new file mode 100644 index 0000000..b6dce53 --- /dev/null +++ b/native/services/accel-spmd/src/kernel.rs @@ -0,0 +1,362 @@ +//! **Registered kernels** — what a "program" is on the CPU row. +//! +//! The NPU row loads a compiled artifact: an ELF the NPU compiler emitted, +//! placed in device memory and relocated. There is no CPU compiler in this +//! tree, so this row takes the other option the protocol already leaves open. +//! [`ArtifactRef::target`] is twelve **opaque** words whose "schema belongs to +//! the artifact format, not this crate" — so a format whose schema says *the +//! words name a kernel this build already carries* is conformant, not a +//! workaround. `load_program` validates a fixed versioned manifest, then does +//! a lookup rather than loading executable bytes; any other payload is refused +//! rather than ignored. +//! +//! That choice is a v1 posture, not a ceiling: an artifact-loading CPU row can +//! land beside this one under a second [`ArtifactFormat`] without moving a +//! byte of protocol, and the W^X executable-lease path (#660) is what it would +//! stand on. +//! +//! # The tiling contract +//! +//! A kernel declares the size of one **output element**, and that is the unit +//! the region's workers are split over (`res/SPMD.md` R1 — the launch extent +//! is the *work* grid, not a thread count). The backend does the split; a +//! kernel body receives its own element span and a `&mut [u8]` that is exactly +//! those elements' bytes. Two workers therefore cannot alias, and a kernel +//! never reasons about the region at all. +//! +//! Inputs are shared for the whole grid: an elementwise body indexes them by +//! the same element numbers it was given. +//! +//! # What this shape deliberately does not cover +//! +//! Reductions. A sum has one output element, so the tiling above degenerates +//! to a single worker and the parallelism is lost. That is not a gap to paper +//! over here — `spmd::reduce` is a *different launch shape* (map to a partial, +//! then combine), and it wants its own kernel class rather than a reduction +//! squeezed through an elementwise signature. The registry is the place that +//! class will be added. +//! +//! [`ArtifactRef::target`]: virtio_accel_core::ArtifactRef +//! [`ArtifactFormat`]: virtio_accel_core::ArtifactFormat + +use core::ops::Range; +use core::simd::{Simd, num::SimdFloat}; + +use kore_driver::accel::LaunchShape; +use virtio_accel_core::TargetIdentity; + +/// Lanes per SIMD step. Eight `f32` is 256 bits — the v3 user profile's +/// natural width, and the widest the tree's baseline profile can assume. +pub(crate) const LANES: usize = 8; + +/// The identity word every kernel this row serves begins with — `"KSPM"`. +/// +/// A client naming any other first word is naming an artifact for some other +/// format, and gets [`Unsupported`](virtio_accel_core::BackendError::Unsupported) +/// rather than a lookup miss: the two are different failures and a client can +/// act on the difference. +pub const IDENTITY_MAGIC: u32 = u32::from_le_bytes(*b"KSPM"); + +/// One tile span's view of a kernel launch. +/// +/// `out` is **exactly** this span's bytes — the backend has already done the +/// split, so a body writes its whole slice and cannot reach a sibling's. +pub struct TileCtx<'a> { + /// This worker's element span within the grid. + pub tiles: Range, + /// The whole grid's element count. + pub total: u32, + /// The exact launch policy admitted for this execution. + /// + /// Explicit workgroup axes remain intact here even though the CPU worker + /// tier schedules a contiguous linear span. A registered kernel can derive + /// logical group/local ids from the original shape; equal-product shapes + /// are never collapsed into the same kernel invocation. No barrier model + /// is implied by this field (`res/SPMD.md` R8). + pub launch: LaunchShape, + /// The bound inputs, whole (not split) — indexed by grid element. + pub inputs: &'a [&'a [u8]], + /// This span's output bytes, and only this span's. + pub out: &'a mut [u8], +} + +/// A kernel this build carries. +pub struct Kernel { + /// The registry key: `[IDENTITY_MAGIC, id, 0, 0, …]`. + pub id: u32, + /// Diagnostic name — what a status row prints, never a lookup key. + pub name: &'static str, + /// Input bindings this kernel requires, exactly. + pub inputs: u32, + /// Bytes in one output element — the tiling unit. + pub elem_bytes: u32, + /// Lanes this kernel's body runs in lockstep. + /// + /// A property of the compiled body, not a client choice or wire field. + /// Providers may use it for diagnostics and scheduling decisions. + pub lanes: u32, + /// Run one span. Called once per worker, concurrently. + pub run: fn(TileCtx<'_>), +} + +impl Kernel { + /// The twelve-word identity a client names this kernel by. + pub const fn identity(&self) -> TargetIdentity { + let mut words = [0u32; 12]; + words[0] = IDENTITY_MAGIC; + words[1] = self.id; + TargetIdentity(words) + } +} + +/// Split a target identity into its kernel id, or `None` when the words are +/// not this format's. +/// +/// Fail-closed on the reserved tail: words 2..12 must be zero. A client that +/// set one is asking for something this build does not implement, and +/// accepting the request while ignoring the words would be the dishonest +/// reading — the same posture every sibling wire form in the tree takes. +pub fn kernel_id(target: TargetIdentity) -> Option { + let TargetIdentity(words) = target; + if words[0] != IDENTITY_MAGIC { + return None; + } + if words[2..].iter().any(|&w| w != 0) { + return None; + } + Some(words[1]) +} + +/// Find a kernel by id. +pub fn lookup(registry: &'static [Kernel], id: u32) -> Option<&'static Kernel> { + registry.iter().find(|k| k.id == id) +} + +// ── the built-in registry ─────────────────────────────────────────────── + +/// `add.f32` — elementwise `out[i] = in0[i] + in1[i]`. +pub const KERNEL_ADD_F32: u32 = 1; +/// `mul.f32` — elementwise `out[i] = in0[i] * in1[i]`. +pub const KERNEL_MUL_F32: u32 = 2; + +/// The kernels this build carries. +/// +/// Two, deliberately: enough to prove the seam end to end (binding validation, +/// tiling, the region launch, the serial fallback) without this row pretending +/// to be a compute library. A kernel set is a separate question from a backend, +/// and answering it here would make the backend hard to review. +pub static BUILTIN: &[Kernel] = &[ + Kernel { + id: KERNEL_ADD_F32, + name: "add.f32", + inputs: 2, + elem_bytes: 4, + lanes: LANES as u32, + run: add_f32, + }, + Kernel { + id: KERNEL_MUL_F32, + name: "mul.f32", + inputs: 2, + elem_bytes: 4, + lanes: LANES as u32, + run: mul_f32, + }, +]; + +/// Read `count` `f32`s starting at element `first` from a bound input. +/// +/// Bindings arrive as bytes: the wire carries offsets into shared windows and +/// says nothing about element types, so a kernel decodes its own operands. +/// Unaligned by construction — a client's offset need not be 4-byte aligned — +/// which is why this reads through `from_le_bytes` rather than casting. +#[inline] +fn load_f32(src: &[u8], index: usize) -> f32 { + let at = index * 4; + let mut word = [0u8; 4]; + word.copy_from_slice(&src[at..at + 4]); + f32::from_le_bytes(word) +} + +#[inline] +fn store_f32(dst: &mut [u8], index: usize, value: f32) { + let at = index * 4; + dst[at..at + 4].copy_from_slice(&value.to_le_bytes()); +} + +/// The shared body of both elementwise kernels — `op` applied lane-wise. +/// +/// Vectorised through [`core::simd`] rather than a scalar loop +/// (`kore-design-doctrine` §8: byte-at-a-time is a red flag, and the portable +/// API composes with the trait-seam discipline instead of fighting it). The +/// tail below `LANES` runs scalar, which is the ordinary shape and not a +/// fallback. +#[inline] +fn elementwise(ctx: TileCtx<'_>, op: fn(Simd, Simd) -> Simd) { + let a = ctx.inputs[0]; + let b = ctx.inputs[1]; + let span = (ctx.tiles.end - ctx.tiles.start) as usize; + let base = ctx.tiles.start as usize; + + let mut i = 0; + while i + LANES <= span { + let mut lhs = [0f32; LANES]; + let mut rhs = [0f32; LANES]; + for lane in 0..LANES { + lhs[lane] = load_f32(a, base + i + lane); + rhs[lane] = load_f32(b, base + i + lane); + } + let out = op(Simd::from_array(lhs), Simd::from_array(rhs)).to_array(); + for lane in 0..LANES { + store_f32(ctx.out, i + lane, out[lane]); + } + i += LANES; + } + while i < span { + let v = op( + Simd::splat(load_f32(a, base + i)), + Simd::splat(load_f32(b, base + i)), + )[0]; + store_f32(ctx.out, i, v); + i += 1; + } +} + +fn add_f32(ctx: TileCtx<'_>) { + elementwise(ctx, |a, b| a + b); +} + +fn mul_f32(ctx: TileCtx<'_>) { + elementwise(ctx, |a, b| a * b); +} + +/// Keep the `SimdFloat` import load-bearing rather than speculative: the trait +/// is what makes the lane-wise arithmetic above resolve to the portable ops. +const _: fn() = || { + let _ = Simd::::splat(0.0).reduce_sum(); +}; + +#[cfg(test)] +mod tests { + use super::*; + use std::vec; + + fn bytes_of(values: &[f32]) -> vec::Vec { + values.iter().flat_map(|v| v.to_le_bytes()).collect() + } + + fn floats_of(bytes: &[u8]) -> vec::Vec { + bytes + .chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect() + } + + /// The identity round-trips, and every non-conforming shape refuses. + #[test] + fn identities_round_trip_and_fail_closed() { + let kernel = &BUILTIN[0]; + assert_eq!(kernel_id(kernel.identity()), Some(kernel.id)); + + assert_eq!( + kernel_id(TargetIdentity([0; 12])), + None, + "a zero identity is not this format's" + ); + + let mut wrong_magic = kernel.identity(); + wrong_magic.0[0] ^= 1; + assert_eq!(kernel_id(wrong_magic), None, "another format's artifact"); + + let mut reserved_set = kernel.identity(); + reserved_set.0[7] = 1; + assert_eq!( + kernel_id(reserved_set), + None, + "a set reserved word must refuse, not be ignored" + ); + } + + /// A kernel run over one whole span computes the elementwise result — + /// including the tail below the SIMD width. + #[test] + fn add_computes_over_a_full_span_including_the_tail() { + // 19 elements: two full 8-lane steps plus a 3-element tail. + let a: vec::Vec = (0..19).map(|i| i as f32).collect(); + let b: vec::Vec = (0..19).map(|i| (i * 2) as f32).collect(); + let src_a = bytes_of(&a); + let src_b = bytes_of(&b); + let mut out = vec![0u8; 19 * 4]; + + add_f32(TileCtx { + tiles: 0..19, + total: 19, + launch: LaunchShape::ArtifactDefined, + inputs: &[&src_a, &src_b], + out: &mut out, + }); + + let got = floats_of(&out); + for i in 0..19 { + assert_eq!(got[i], a[i] + b[i], "element {i}"); + } + } + + /// The split is the whole safety argument, so it has to compose: running + /// two disjoint spans into their own slices must equal one whole run. + #[test] + fn disjoint_spans_compose_into_the_whole_result() { + let a: vec::Vec = (0..20).map(|i| i as f32).collect(); + let b: vec::Vec = (0..20).map(|i| (i + 7) as f32).collect(); + let src_a = bytes_of(&a); + let src_b = bytes_of(&b); + + let mut whole = vec![0u8; 20 * 4]; + mul_f32(TileCtx { + tiles: 0..20, + total: 20, + launch: LaunchShape::ArtifactDefined, + inputs: &[&src_a, &src_b], + out: &mut whole, + }); + + let mut split = vec![0u8; 20 * 4]; + { + let (lo, hi) = split.split_at_mut(9 * 4); + mul_f32(TileCtx { + tiles: 0..9, + total: 20, + launch: LaunchShape::ArtifactDefined, + inputs: &[&src_a, &src_b], + out: lo, + }); + mul_f32(TileCtx { + tiles: 9..20, + total: 20, + launch: LaunchShape::ArtifactDefined, + inputs: &[&src_a, &src_b], + out: hi, + }); + } + assert_eq!(whole, split, "a tiled run equals the whole run"); + } + + /// Every registered kernel is reachable by its own identity, and ids are + /// unique — a duplicate would make `lookup` silently prefer one. + #[test] + fn the_registry_is_a_function() { + for kernel in BUILTIN { + let found = lookup(BUILTIN, kernel.id).expect("registered"); + assert_eq!(found.name, kernel.name); + } + for (i, k) in BUILTIN.iter().enumerate() { + assert!( + !BUILTIN[..i].iter().any(|prev| prev.id == k.id), + "duplicate kernel id {}", + k.id + ); + assert!(k.elem_bytes > 0, "{} needs a tiling unit", k.name); + } + assert!(lookup(BUILTIN, 0xdead_beef).is_none(), "unknown id misses"); + } +} diff --git a/native/services/accel-spmd/src/lib.rs b/native/services/accel-spmd/src/lib.rs new file mode 100644 index 0000000..494344d --- /dev/null +++ b/native/services/accel-spmd/src/lib.rs @@ -0,0 +1,86 @@ +//! # kore-svc-accel-spmd — the CPU backend for the accel protocol +//! +//! `crates/abi/src/svc.rs` describes the accel request wire form as "one +//! vocabulary for every backend (**NPU / GPU / CPU-SIMD**)". The NPU and GPU +//! have device-side implementations in tree; this crate supplies the CPU +//! backend candidate. It is not yet a founded service row. Its compute +//! resource is the machine's own cores, reached through [`kore_rt::spmd`]'s +//! region launch. +//! +//! ## Why the CPU backend is worth its own tenancy +//! +//! Not because a CPU computes faster than an NPU. The accel protocol targets +//! one client vocabulary across providers, and that target needs a CPU +//! implementation. The host tests here drive this backend through context → +//! buffer → program → submit → poll, while `kore-svc-accel` separately gates +//! the generic service and wire. That is a backend-contract witness, not yet +//! proof of a founded CPU service, common executable artifacts, scheduler +//! placement, or production portability across CPU, GPU, and NPU. +//! `res/AXIOM_INTEGRATION.md` defines the ownership boundary: `.ax` lowering +//! and CPU partition compilation belong to `ax`/`ax-exec`; this crate is the +//! provider tenancy and queue adapter. Its `KSPM` built-ins are bootstrap +//! witnesses, not a parallel graph or kernel catalog. +//! +//! ## R8: this backend is the `CPU-SIMD` half of one geometry +//! +//! `res/SPMD.md` R8 settles where the matrix backend lives: the SPMD launch +//! geometry is *both* a direct CPU API and a device-portable one. Legacy scalar +//! `OP_SUBMIT` is artifact-defined; a non-scalar compatibility record is +//! accepted only when its SIMD width equals the registered kernel's compiled +//! lanes. The additive explicit-launch operation carries only workgroup count +//! and local size, where SIMD remains purely a provider compilation/scheduling +//! choice. +//! +//! [`SpmdBackend`] implements both the deprecated exact compatibility seam and +//! `submit_with_launch`: artifact-defined execution derives its extent from the +//! output, while explicit workgroups must describe exactly the output the +//! registered kernel can hold. +//! +//! ## Two structural findings this row surfaced +//! +//! **A program need not be a compiled artifact.** [`ArtifactRef::target`] is +//! twelve *opaque* words whose "schema belongs to the artifact format, not +//! this crate" — so a format whose schema says the words name a kernel this +//! build already carries is conformant, and needs no CPU compiler to exist. +//! See [`kernel`]. +//! +//! **`submit` and SPMD's one-shot launch did not compose — R9 fixes the +//! mechanism.** The `Accelerator` trait forbids `submit` from waiting for +//! execution, and SPMD's only reusable launch (`dispatch_slices`) blocked +//! on its join. `res/SPMD.md` R9 lands the resident region — the founded- +//! once engine with a non-blocking `submit`, a completion word (`poll`), +//! and a `retire` — which IS the shape this row wants (and the device rows +//! already have). So the compute still lives in +//! [`SpmdBackend::run_pending`] — admit, drive, observe — but that one +//! function becomes a resident client the day this crate is founded as a +//! service: `found_resident` once, `submit` per launch, `poll` for +//! completion, no gang re-founded per submit. It is deliberately NOT +//! rewired yet: with no serve loop driving it, a resident path would be +//! metal code no gate exercises (the metal-real rule), so the rewire lands +//! with the service that drives it. Until then `run_pending` keeps the +//! `dispatch_slices` path, which produces identical bytes. See [`backend`]. +//! +//! ## What is not here +//! +//! A `SystemService` row. This crate is the backend and its kernels; founding +//! it as a boot service (manifest row, endowment, founding order, archive +//! measurement) is its own slice — and it is the slice that turns +//! [`SpmdBackend::run_pending`] into an R9 resident client (above), because +//! only then is that path metal-driven and provable. +//! +//! [`ArtifactRef::target`]: virtio_accel_core::ArtifactRef + +#![no_std] +#![feature(portable_simd)] + +pub mod backend; +pub mod kernel; + +pub use backend::{ + FORMAT_REGISTERED, REGISTERED_MANIFEST, REGISTERED_RESIDENT_BYTES, SpmdBackend, SpmdBuffer, + SpmdContext, SpmdEvent, SpmdProgram, SpmdQueue, +}; +pub use kernel::{BUILTIN, IDENTITY_MAGIC, Kernel, TileCtx}; + +#[cfg(test)] +extern crate std; diff --git a/native/services/block/boot.toml b/native/services/block/boot.toml index 65fdc1b..be7e276 100644 --- a/native/services/block/boot.toml +++ b/native/services/block/boot.toml @@ -12,5 +12,5 @@ chain = true endowment = "kore_svc_block::BlockEndowment" protocol = "kore_svc_block::BlockService" endows = ["DMA_POOL@1"] -digest_x86_64 = "blake3:d9f8829568372dbcdccc3028bd4b302850c814eb4d44233290c5bdb165c67cef" -digest_aarch64 = "blake3:d004acf08fb53d2d42656f0df24bf54318e3eda87ec79a6778c2e186b61ade79" +digest_x86_64 = "blake3:e48b79857534c5630964141cadce6985ccfdc9864a819276917efc8ed415ec50" +digest_aarch64 = "blake3:f33c632b85bf9adc9094e709710606d2a56c10449e76d22919697cdba6f08775" diff --git a/native/services/dagfs/boot.toml b/native/services/dagfs/boot.toml index 37866c1..15b963a 100644 --- a/native/services/dagfs/boot.toml +++ b/native/services/dagfs/boot.toml @@ -47,5 +47,5 @@ exec_client = true # `dagfs_heap_supply_holds_every_session` holds this row to it; the rest is # room for the proofs open handles carry, allocated per open at their depth. heap_supply_bytes = 0x00D0_0000 -digest_x86_64 = "blake3:dc2fe387db38e271eac4f5d93ac4b5d815060a82cee8cdc81c16bd363d6e9eed" -digest_aarch64 = "blake3:d27bf4cc470afee6a1b5c7c80127dc7e0f8366569b56b74b09c75719e45a3e23" +digest_x86_64 = "blake3:10d0cc3c987cd5303a7bdafe82961f3ce30ff9c067a21d96894a7c68fba5c291" +digest_aarch64 = "blake3:981d8318b9a9b415dcbc0e331aa984750134d35828ea499e0a4b53557aa4c548" diff --git a/native/services/device/boot.toml b/native/services/device/boot.toml index 96c7f02..7d444ce 100644 --- a/native/services/device/boot.toml +++ b/native/services/device/boot.toml @@ -47,5 +47,5 @@ protocol = "kore_svc_device::DeviceBroker" # row's own claimant band. instances = 8 endows = ["ROOT_COMPLEX@0", "PLATFORM_BUS@0", "FACE_SUPPLY@0"] -digest_x86_64 = "blake3:5a3ff2252bef814e79727f873c89de62565ab491f8065599ada4db348c829223" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:f00fd8b3aad6308541c51f785e3651dc793b60652a821ffee9aa7d79b49eb615" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:f4ef9270f4c48990e242ce05f4d7c7b5c62cb2da959e148290ed073df9075379" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:a4d0b40e6d7ef5d6ce5d270ba91f1c09aee246d5b16e99f864001b92dbe570f4" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/mm/boot.toml b/native/services/mm/boot.toml index b6412e0..7771a32 100644 --- a/native/services/mm/boot.toml +++ b/native/services/mm/boot.toml @@ -54,5 +54,5 @@ supply_quota_bytes = 27917287424 # 26 GiB endows = ["MEM_SUPPLY@0", "MEM_REGION@0", "SYSTEM_OBSERVE@0"] # review mirror; asserted === PROFILE.roles shell_client = true exec_client = true -digest_x86_64 = "blake3:041a33515a21fff195c4cab320e9c8e64cbf775f04f01b7dbe40fa90be5a31a7" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:e17d59e57476b1db9554bc889640ffd3f02b6e2c98d509d93dea1c7efadb2a2a" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:2ebee97309e197da6bcf90b34d7fbdd42eafa679b6afd770e3599500eefed040" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:693bd018cbfffac602028bcbbac3dcbc734b6e3347106f1529a9934b32c92d0b" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/swap/boot.toml b/native/services/swap/boot.toml index 0a1028a..314360e 100644 --- a/native/services/swap/boot.toml +++ b/native/services/swap/boot.toml @@ -9,5 +9,5 @@ image = "kore-svc-swap" endowment = "kore_svc_swap::ZramEndowment" protocol = "kore_svc_swap::ZramSwap" endows = ["MEM_SUPPLY@0"] -digest_x86_64 = "blake3:bc3ff6fbb123b3b44645e959346e320519b9ab5f43b5c999d0685ee9c7fed141" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:6495b930db0d4e4ca6ce41b0d19580ea1d816748358cc9c8bceef150ec0ccc30" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:c6363f610b2e977482dca6d876e201b689667ba62f45052a6d2fc49ce425b754" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:7c9b860f6572ab954cebd6579ef63f38a9256e93206127f9cd0c1bfc93a72d7d" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/vdev/Cargo.toml b/native/services/vdev/Cargo.toml new file mode 100644 index 0000000..7d66c36 --- /dev/null +++ b/native/services/vdev/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "kore-svc-vdev" +version = "0.0.0" +edition = "2024" +description = "kore virtual-device service: hosts device models with no hardware behind them, one per driver, over memory the driver shares." +publish = false + +[dependencies] +kore-abi = { path = "../../crates/abi" } +kore-rt = { path = "../../crates/rt" } +# The transport a driver reaches a device model through. +kore-vdev = { path = "../../crates/vdev" } +# The first model. Only the service hosts it. +kore-vdev-accel = { path = "../../crates/vdev-accel", optional = true } + +[features] +default = ["host"] +# The service itself: the device table and the models it hosts. A crate that +# only speaks to the service (a driver's library, which carries no device +# model) builds without it and keeps the wire, the face and the client. +host = ["dep:kore-vdev-accel"] + +[dev-dependencies] +kore-rt = { path = "../../crates/rt", features = ["std"] } +# The driver the host tests drive a hosted device with: the protocol's +# reference guest, over kore-vdev's driver half. +virtio-accel-guest = { version = "=0.4.1", default-features = false } +virtio-accel-proto = { version = "=0.4.1", default-features = false } +virtio-accel-transport = { version = "=0.4.1", default-features = false } diff --git a/native/services/vdev/boot.toml b/native/services/vdev/boot.toml new file mode 100644 index 0000000..9ff6c63 --- /dev/null +++ b/native/services/vdev/boot.toml @@ -0,0 +1,30 @@ +# services/vdev/boot.toml — this service's boot-manifest row (the design +# lives in res/boot.toml's header). The row rides beside the crate it +# endows; whether and WHEN it founds is the root manifest's `services` list. + +# The virtual-device service (res/VIRTUAL_DEVICE.md): hosts a device model +# with no hardware behind it for each driver that opens one. +# +# Unchained, and before the device..exec retained-founder chain for the +# reason power is: exec owns pair one's client face and its retained return +# leg, so the row must exist before exec's founder can publish that face. +# +# Pair zero is the founder's control face. Pair one belongs to exec, which +# uses it only to admit and retire one recyclable face per program launched +# with the virtual-device profile. The shell receives no face. +# +# The row is endowed with nothing: no device, no authority over the machine. +# Its heap is where every open device lives. Four devices may be open at +# once, each holding at most 2 MiB of buffers for its driver +# (`kore_svc_vdev::host::DEVICE_HEAP_BYTES`, 8 MiB); the lease below covers +# that with 4 MiB over for the engines' own tables, the seat table and the +# runtime. The service's tests hold the two numbers together. +[service] +image = "kore-svc-vdev" +endowment = "kore_svc_vdev::VdevEndowment" +protocol = "kore_svc_vdev::VdevService" +endows = [] +exec_client = true # pair 1 administers per-launch driver faces +heap_supply_bytes = 0x00C0_0000 +digest_x86_64 = "blake3:33b4bd37c5e76b836ff3bb9f041b37184483f33ba7c37c4b04b09e11cc4b1b00" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:344e1f6da34c1ef515b0c0b9988b5e3b9e673cf08df6f6065ecd5bc661fe42de" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/vdev/src/client.rs b/native/services/vdev/src/client.rs new file mode 100644 index 0000000..e21b772 --- /dev/null +++ b/native/services/vdev/src/client.rs @@ -0,0 +1,439 @@ +//! The driver's side of the service: the face a program is launched with, +//! and exec's administration of such faces. +//! +//! A driver does four things with its face, and only the first two are +//! requests: +//! +//! 1. [`open`](ProcessVdev::open) a device model. The register block, which +//! the driver has had mapped since it started, is written before the +//! reply. +//! 2. [`lend`](ProcessVdev::lend) the memory its queue and buffers are in. +//! The driver keeps the lease and its own mapping; the service receives +//! a view of exactly the bytes named. +//! 3. Ring the [`doorbell`](ProcessVdev::doorbell): a notice, with no reply. +//! 4. [`wait`](ProcessVdev::wait) for the interrupt: a park on the register +//! block's interrupt count, which any thread of the driver may do. +//! +//! Everything else a driver does with its device it does in the memory the +//! two share, through `kore_vdev::driver` or by hand. +//! +//! # A driver with more than one thread +//! +//! Requests belong to the thread the program was launched on: the face's +//! rings have one producer, and the replies wake that thread. The doorbell +//! and the interrupt are any thread's, but a capability is a thread's own, so +//! a thread that rings or parks must have been founded holding its share of +//! the face ([`THREAD_SHARES`], [`ThreadVdev`]). The register block itself is +//! mapped once, by the launched thread, and every thread of the program sees +//! that mapping. + +use kore_abi::cap::{ResourceKind, Rights}; +use kore_abi::session::SlotRef; +use kore_abi::task::role; +use kore_rt::svc::{ + CallError, PeerBundlePart, PeerId, ReturnDeriveError, ReturnDeriver, StagedReturn, + StagedReturnError, SvcClient, +}; +use kore_rt::thread_controller::Handed; +use kore_rt::transport::Transport; + +use crate::VdevService; +use crate::wire::{VdevRefusal, VdevRequest, VdevResponse}; + +/// Where a program launched with the virtual-device profile finds its face +/// in its own boot block. +/// +/// The layout exec stamps and the program binds, in one place. Indices +/// 0 to 5 of these role namespaces belong to the filesystem, window, +/// memory, accelerator and power faces a launch may also carry; exec pins +/// the disjointness where it sees every face. +pub mod process_face { + /// The doorbell, and the request kick (`role::SVC_SEND`). + pub const REQUEST_KICK: u16 = 8; + /// The face itself: request rings, then the register block + /// (`role::SVC_RING`). + pub const CONTROL_RING: u16 = 8; + /// The return leg a driver lends memory on (`role::SVC_RETURN`). + pub const RETURN_SEND: u16 = 8; + /// Correlation datum for the service's landing on that leg + /// (`role::SVC_RETURN`). + pub const RETURN_LAND: u16 = 9; +} + +/// The cookie of a doorbell notice: `"vdevbell"`. The service looks at its +/// devices on any wake, so the cookie only says what the notice was for. +pub const COOKIE_DOORBELL: u64 = u64::from_le_bytes(*b"vdevbell"); + +/// Rights of the view of lent memory the service receives: it reads and +/// writes, and can pass it nowhere. +pub const LENT_RIGHTS: Rights = Rights::ACCESS.union(Rights::MODIFY); +/// The driver's staged view carries `DERIVE` as well, so the rendezvous can +/// attenuate it to [`LENT_RIGHTS`]. +#[cfg(target_os = "none")] +const LENT_STAGED_RIGHTS: Rights = LENT_RIGHTS.union(Rights::DERIVE); + +/// What a thread of a driver holds of the face, beyond the thread the +/// program was launched on: the face's memory, to park on the interrupt +/// count in it, and the kick, to ring the doorbell. It reads and notifies; +/// it maps nothing and writes nothing through either. +/// +/// A program hands these on to the threads it founds +/// (`kore_rt::thread::share_endowment` for one it founds itself, +/// `Controller::hand_on` for the threads a controller founds). +pub const THREAD_SHARES: [Handed; 2] = [ + Handed { + role: role::SVC_RING, + index: process_face::CONTROL_RING, + kind: ResourceKind::MemLease, + rights: Rights::ACCESS, + }, + Handed { + role: role::SVC_SEND, + index: process_face::REQUEST_KICK, + kind: ResourceKind::ChanSend, + rights: Rights::ACCESS, + }, +]; + +/// How a wait for the interrupt ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Waited { + /// The interrupt count moved, to this. + Interrupt(u32), + /// The deadline passed with the count unmoved. + Deadline, + /// The caller's own condition ended the wait. + Stopped, + /// This thread cannot wait on the device: its share of the face is + /// gone, or the kernel will not park it. Waiting again would not help. + Gone, +} + +/// The longest one park of an open-ended wait: the kernel's word wait is +/// never unbounded, so a wait with no deadline is a loop of bounded parks +/// and each re-reads the count. A quarter of a second, kore-rt's figure for +/// the same loop (`futex`, `event`): long enough that a waiter is not paced +/// by it, short enough that the caller's condition, which no wake +/// announces, is seen soon after it changes. +#[cfg(target_os = "none")] +const PARK_SLICE_NS: u64 = 250_000_000; + +/// The ring client over the generic [`SvcClient`]. +pub struct RingVdev<'w, 't, T: Transport>(pub SvcClient<'w, 't, T, VdevService>); + +impl RingVdev<'_, '_, T> { + /// One request. A call that never reached the service answers + /// [`VdevRefusal::Unavailable`]: to a driver the two are the same, a + /// device it cannot have now. + pub async fn request(&mut self, request: VdevRequest) -> VdevResponse { + match self.0.call(&request).await { + Ok(reply) => reply.response, + Err(_) => VdevResponse::Refused(VdevRefusal::Unavailable), + } + } +} + +/// Exec's fixed administration face, with its retained return leg. +pub struct RingVdevAdmin<'w, 't, T: Transport> { + pub client: RingVdev<'w, 't, T>, + return_tx: SlotRef, + return_land: SlotRef, +} + +impl<'w, 't, T: Transport> RingVdevAdmin<'w, 't, T> { + pub const fn new( + client: RingVdev<'w, 't, T>, + return_tx: SlotRef, + return_land: SlotRef, + ) -> Self { + Self { + client, + return_tx, + return_land, + } + } + + /// Convey one component of a driver's face. + pub async fn admit_component_staged( + &mut self, + part: PeerBundlePart, + rights: Rights, + stage: F, + ) -> Result + where + F: FnOnce(ReturnDeriver<'_>) -> Result, + { + let request = VdevRequest::AdmitPeer { + part, + announced: self.return_land, + }; + self.client + .0 + .call_with_staged_return(&request, self.return_tx, rights, stage) + .await + .map(|reply| reply.response) + } + + pub async fn take_admission(&mut self) -> Result { + self.call(VdevRequest::TakeAdmission).await + } + + pub async fn retire_peer(&mut self, peer: PeerId) -> Result { + self.call(VdevRequest::RetirePeer { peer }).await + } + + pub async fn confirm_retired(&mut self, peer: PeerId) -> Result { + self.call(VdevRequest::ConfirmRetired { peer }).await + } + + async fn call(&mut self, request: VdevRequest) -> Result { + self.client + .0 + .call(&request) + .await + .map(|reply| reply.response) + } +} + +#[cfg(target_os = "none")] +pub use metal::{ProcessVdev, ThreadVdev, attach_process_face, attach_thread_face}; + +#[cfg(target_os = "none")] +mod metal { + use super::*; + use kore_rt::cap::verbs::{WordWait, WordWaitError}; + use kore_rt::cap::{LeaseToken, verbs}; + use kore_rt::service::MAX_GRANTS; + use kore_rt::svc::{ClientEndpoints, REP_CAP, REQ_CAP, ReturnLeaseSource, client_rings}; + use kore_rt::transport::TrapTransport; + use kore_vdev::mem::{Memory, Window}; + use kore_vdev::regs; + + use crate::face::{PEER_BYTES, REGISTERS_AT}; + + // A notice needs exactly what a thread's share of the kick carries. + const _: () = assert!(THREAD_SHARES[1].rights.bits() == verbs::NOTICE_RIGHTS.bits()); + + /// One thread's part of the face: the doorbell and the interrupt. + /// + /// Both name capabilities in the calling thread's own arena, so a value + /// of this is its thread's and is attached on the thread that uses it. + #[derive(Clone, Copy, Debug)] + pub struct ThreadVdev { + kick: SlotRef, + /// The interrupt count, named to the kernel: this thread's holding + /// of the face's memory and the count's offset in it. + interrupts: (SlotRef, u32), + } + + /// Attach the calling thread's part of the face from its own boot + /// block. `None` on a thread founded without it. + pub fn attach_thread_face(rt: &kore_rt::entry::Rt) -> Option { + let boot = rt.bootstrap(); + Some(ThreadVdev { + kick: boot.handle(role::SVC_SEND, process_face::REQUEST_KICK)?, + interrupts: ( + boot.handle(role::SVC_RING, process_face::CONTROL_RING)?, + u32::try_from(REGISTERS_AT + regs::INTERRUPTS).ok()?, + ), + }) + } + + impl ThreadVdev { + /// Ring the doorbell. `false` when the notice was not taken: the + /// service's doorbell is full, which means it has a wake standing + /// and will look anyway. + pub fn doorbell(&self) -> bool { + verbs::post_notice(self.kick, COOKIE_DOORBELL, 0) + } + + /// Park until the interrupt count is no longer `seen`, or until + /// `deadline_ns` (absolute monotonic nanoseconds): one park, and + /// what ended it. The caller reads the count from the register + /// block afterwards, whatever this answers. + /// + /// The park compares in the kernel, so an interrupt raised between + /// the caller's read of `seen` and this call is not lost. + pub fn park(&self, seen: u32, deadline_ns: u64) -> Result { + let (subject, offset) = self.interrupts; + verbs::wait_word(subject, offset, seen, deadline_ns) + } + + /// Wait until the interrupt count in `registers` is no longer + /// `seen`: until `deadline_ns` (absolute monotonic nanoseconds) + /// when one is given, and for as long as `go_on` answers `true`, + /// which is asked before every park. + /// + /// Every wait is a park in the kernel; nothing here spins. A park + /// the kernel has no room for now is a short cede and another look. + pub fn wait( + &self, + registers: Window, + seen: u32, + deadline_ns: Option, + mut go_on: impl FnMut() -> bool, + ) -> Waited { + loop { + match registers.load_u32(regs::INTERRUPTS) { + Ok(count) if count != seen => return Waited::Interrupt(count), + Ok(_) => {} + Err(_) => return Waited::Gone, + } + if !go_on() { + return Waited::Stopped; + } + let Some(now) = kore_rt::clock::deadline_base_nanos() else { + // No way to name an instant, and the kernel takes no + // unbounded park. + return Waited::Gone; + }; + let slice = now.saturating_add(PARK_SLICE_NS); + let bound = match deadline_ns { + Some(at) if now >= at => return Waited::Deadline, + Some(at) => at.min(slice), + None => slice, + }; + match self.park(seen, bound) { + Ok(WordWait::Changed | WordWait::Deadline) => {} + Ok(WordWait::WouldBlock) => { + if !kore_rt::pace::cede() { + return Waited::Gone; + } + } + Ok(WordWait::Revoked) | Err(_) => return Waited::Gone, + } + } + } + } + + /// A program's face with the service: its request rings, the register + /// block of its device, the leg it lends memory on, and the launched + /// thread's own doorbell and interrupt. + pub struct ProcessVdev<'t> { + client: RingVdev<'static, 't, TrapTransport>, + registers: Window, + thread: ThreadVdev, + return_tx: SlotRef, + return_land: SlotRef, + /// Keeps the face mapped for the life of the program. + _ring: LeaseToken, + } + + /// Attach the face from a profiled program's boot block. `None` when the + /// program was launched without one: there is no device to open. + pub fn attach_process_face<'t>( + rt: &kore_rt::entry::Rt, + transport: &'t TrapTransport, + ) -> Option> { + let boot = rt.bootstrap(); + let thread = attach_thread_face(rt)?; + let ring = verbs::take_lease(boot.handle(role::SVC_RING, process_face::CONTROL_RING)?)?; + let window = verbs::map_held_rw(ring.slot())?; + if (window.len() as u64) < PEER_BYTES { + return None; + } + let base = window.cast::(); + // SAFETY: the first page is the face's rings, and this is the + // program's sole client attachment over the task-lifetime lease exec + // delivered. The slice stops short of the register block, so no + // reference covers what the device writes. + let rings: &'static mut [u8] = + unsafe { core::slice::from_raw_parts_mut(base.as_ptr(), REGISTERS_AT as usize) }; + let rings = client_rings::(rings, REQ_CAP, REP_CAP).ok()?; + // SAFETY: the second page of the same mapping, page-aligned, mapped + // read-write for as long as `ring` is held, which is as long as + // this value lives. The service writes it concurrently, which is + // what a `Window` is for. + let registers = + unsafe { Window::new(base.add(REGISTERS_AT as usize), regs::BYTES as usize) }; + Some(ProcessVdev { + client: RingVdev(SvcClient::new( + transport, + rings, + ClientEndpoints { + kick: thread.kick, + grants: [const { None }; MAX_GRANTS], + }, + )), + registers, + thread, + return_tx: boot.handle(role::SVC_RETURN, process_face::RETURN_SEND)?, + return_land: SlotRef::from_wire( + boot.datum(role::SVC_RETURN, process_face::RETURN_LAND)? as u32, + ), + _ring: ring, + }) + } + + impl ProcessVdev<'_> { + /// The register block of this program's device. It says a device is + /// there once [`open`](Self::open) has been accepted. The mapping is + /// the process's: every thread of the program may use it for as long + /// as this value lives. + pub const fn registers(&self) -> Window { + self.registers + } + + /// The launched thread's own doorbell and interrupt. + pub const fn thread(&self) -> ThreadVdev { + self.thread + } + + /// Open the device model with this register-block id. + pub async fn open(&mut self, device: u32) -> VdevResponse { + self.client.request(VdevRequest::Open { device }).await + } + + /// Lend the first `bytes` of `source`: device addresses are offsets + /// into them from here on. The caller keeps the lease and its own + /// mapping of it. + pub async fn lend(&mut self, source: &S, bytes: u64) -> VdevResponse { + let request = VdevRequest::Lend { + announced: self.return_land, + }; + let reply = self + .client + .0 + .call_with_staged_return(&request, self.return_tx, LENT_RIGHTS, |derive| { + derive.lease_window(source, LENT_STAGED_RIGHTS, 0, bytes) + }) + .await; + match reply { + Ok(reply) => reply.response, + Err(_) => VdevResponse::Refused(VdevRefusal::Unavailable), + } + } + + /// Give the device up, and take back the memory lent to it. + pub async fn close(&mut self) -> VdevResponse { + self.client.request(VdevRequest::Close).await + } + + /// Ring the doorbell ([`ThreadVdev::doorbell`]). + pub fn doorbell(&self) -> bool { + self.thread.doorbell() + } + + /// The interrupt count now. + pub fn interrupts(&self) -> u32 { + // The block is a page of a mapping this value keeps: the load + // cannot fault. + self.registers.load_u32(regs::INTERRUPTS).unwrap_or(0) + } + + /// Wait until the interrupt count is no longer `seen`, or until + /// `deadline_ns` (absolute monotonic nanoseconds), and answer the + /// count. `seen` again means no interrupt came: the deadline + /// passed, or this thread cannot wait on the device + /// ([`ThreadVdev::wait`]). + pub fn wait(&self, seen: u32, deadline_ns: u64) -> u32 { + match self + .thread + .wait(self.registers, seen, Some(deadline_ns), || true) + { + Waited::Interrupt(count) => count, + Waited::Deadline | Waited::Stopped | Waited::Gone => seen, + } + } + } +} diff --git a/native/services/vdev/src/face.rs b/native/services/vdev/src/face.rs new file mode 100644 index 0000000..d8e84c4 --- /dev/null +++ b/native/services/vdev/src/face.rs @@ -0,0 +1,29 @@ +//! A driver's face with the service: what both sides agree is where. +//! +//! A face is two pages exec supplies when it launches a program with the +//! virtual-device profile: the request rings, then the register block. The +//! driver has both mapped from its first instruction; the service is handed +//! the second when the face is admitted. + +use kore_rt::svc::{REP_CAP, REQ_CAP, window_bytes}; +use kore_vdev::regs; + +#[cfg(feature = "host")] +use crate::service::VdevService; + +/// Where the register block is in a driver's face: the page after the +/// request rings. +pub const REGISTERS_AT: u64 = 0x1000; + +/// A driver's face: the rings' page and the register block's. +pub const PEER_BYTES: u64 = REGISTERS_AT + regs::BYTES; + +// The rings fit the first page, so the block starts on the second. +const _: () = assert!(window_bytes::(REQ_CAP, REP_CAP) as u64 <= REGISTERS_AT); + +/// The service, to a crate that speaks to it and does not host it: the name +/// its protocol hangs on, which nothing can make a value of. The service's +/// own build has the real one (`service::VdevService`). +#[cfg(not(feature = "host"))] +#[derive(Debug)] +pub enum VdevService {} diff --git a/native/services/vdev/src/host.rs b/native/services/vdev/src/host.rs new file mode 100644 index 0000000..d03e0ea --- /dev/null +++ b/native/services/vdev/src/host.rs @@ -0,0 +1,259 @@ +//! **The devices the service hosts.** +//! +//! Every driver face has a *seat*: the register page of that face, and the +//! device the driver opened over it, if it has opened one. A seat costs a +//! few words, so there is one for every program that holds a face. A device +//! is a model with its own state on the service's heap, so only a few are +//! open at once ([`MAX_DEVICES`]). +//! +//! The table is plain data. What the runtime supplies comes in as +//! arguments: the register window when a face is admitted, the mapped +//! window and its owner when a driver lends memory, and a closure that +//! wakes a driver when its device interrupts. That is what lets the whole +//! of it run off metal. + +use alloc::vec::Vec; +use core::num::NonZeroU32; + +use kore_vdev::mem::{Memory, Window}; +use kore_vdev::regs; +use kore_vdev::transport::Transport; +use kore_vdev_accel::device::{AccelDevice, DEVICE_ID, MAX_BUFFER_BACKING_BYTES}; + +/// Devices open at once. +/// +/// A device may hold [`MAX_BUFFER_BACKING_BYTES`] of buffers for its driver +/// on the service's heap, and the heap is one founded lease +/// (`heap_supply_bytes` in the service's boot manifest row). Four devices +/// are 8 MiB of buffers at most, which that lease covers with room for the +/// engines' own tables; a fifth driver is told the service is full and may +/// ask again when another closes. +pub const MAX_DEVICES: usize = 4; + +/// The heap the open devices can ask for between them. The boot manifest +/// row's `heap_supply_bytes` must cover it; the service's tests hold the +/// two together. +pub const DEVICE_HEAP_BYTES: u64 = MAX_DEVICES as u64 * MAX_BUFFER_BACKING_BYTES; + +/// Seats there can ever be: the runtime names a peer's slot in eight bits. +pub const MAX_SEATS: usize = 1 << u8::BITS; + +/// Chains one device is served per look. A driver has at most thirty-two +/// commands in flight (half the largest queue), so sixteen is two looks for +/// the fullest queue, and no driver keeps the service from the next for +/// longer than sixteen commands take. +pub const SERVICE_BUDGET: NonZeroU32 = NonZeroU32::new(16).unwrap(); + +/// The page size a lent window is counted in on the wire. +pub const PAGE_BYTES: u64 = 0x1000; + +/// Why a driver's request was not done. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refusal { + /// No device model has that id. + NoDevice, + /// The device is not in a state for this: already open, not open, or + /// holding a queue in the memory it has. + Busy, + /// Every device the service hosts at once is taken. + Limit, + /// The model would not start. + Unavailable, + /// What was lent is not memory a device can address. + Memory, +} + +/// One open device. The transport is declared first so that it is dropped +/// first: every view into the lent memory goes before the owner that keeps +/// that memory mapped. +#[derive(Debug)] +struct Hosted { + transport: Transport, + /// What keeps the lent memory mapped, for as long as the device has it. + lent: Option, +} + +/// One driver face. +#[derive(Debug)] +struct Seat { + /// The runtime's name for this occupant of the slot: a later occupant + /// has another, so a request from one is never taken for the other's. + arrival: u16, + registers: Window, + device: Option>, +} + +/// Every seat, by the runtime's slot for the face. `O` is whatever keeps a +/// lent window mapped; dropping it gives the window back. +#[derive(Debug)] +pub struct Devices { + seats: Vec>>, + open: usize, +} + +impl Default for Devices { + fn default() -> Self { + Self::new() + } +} + +impl Devices { + pub const fn new() -> Self { + Self { + seats: Vec::new(), + open: 0, + } + } + + /// Devices open now. + pub const fn open(&self) -> usize { + self.open + } + + /// Whether `slot` is seated. + pub fn seated(&self, slot: usize) -> bool { + self.seats.get(slot).is_some_and(Option::is_some) + } + + /// The arrival seated in `slot`. + pub fn arrival(&self, slot: usize) -> Option { + self.seats.get(slot)?.as_ref().map(|seat| seat.arrival) + } + + /// Seat the face that arrived as `arrival` in `slot`, over its register + /// page. `false` when the slot is taken, is not one the runtime can + /// name, or the table could not grow to hold it. + /// + /// # Safety of the window + /// `registers` stays mapped until [`unseat`](Self::unseat) returns for + /// this slot: the caller owns the mapping and asks before it unmaps. + pub fn seat(&mut self, slot: usize, arrival: u16, registers: Window) -> bool { + if slot >= MAX_SEATS || registers.len() < regs::BYTES || self.seated(slot) { + return false; + } + if slot >= self.seats.len() { + let more = slot + 1 - self.seats.len(); + if self.seats.try_reserve(more).is_err() { + return false; + } + self.seats.resize_with(slot + 1, || None); + } + // A page that held another face's device says nothing of it. + let _ = registers.store_u32(regs::MAGIC, 0); + self.seats[slot] = Some(Seat { + arrival, + registers, + device: None, + }); + true + } + + /// The face in `slot` is going: close its device and forget its + /// register page. After this nothing here touches either window. + pub fn unseat(&mut self, slot: usize) { + if let Some(seat) = self.seats.get_mut(slot).and_then(Option::take) + && seat.device.is_some() + { + self.open -= 1; + } + } + + fn seat_mut(&mut self, slot: usize) -> Option<&mut Seat> { + self.seats.get_mut(slot).and_then(Option::as_mut) + } + + /// Open the device model `device` for the face in `slot`: its register + /// block is written before this returns. + pub fn open_device(&mut self, slot: usize, device: u32) -> Result<(), Refusal> { + let full = self.open >= MAX_DEVICES; + let seat = self.seat_mut(slot).ok_or(Refusal::Unavailable)?; + if device != DEVICE_ID { + return Err(Refusal::NoDevice); + } + if seat.device.is_some() { + return Err(Refusal::Busy); + } + if full { + return Err(Refusal::Limit); + } + let model = AccelDevice::new().map_err(|_| Refusal::Unavailable)?; + let transport = Transport::new(seat.registers, model).map_err(|_| Refusal::Unavailable)?; + seat.device = Some(Hosted { + transport, + lent: None, + }); + self.open += 1; + Ok(()) + } + + /// The driver in `slot` lends `memory`, which `owner` keeps mapped. + /// Answers the pages lent. On a refusal `owner` is dropped, which gives + /// the memory back; so is the owner of any memory this replaces. + pub fn lend(&mut self, slot: usize, memory: Window, owner: O) -> Result { + let hosted = self + .seat_mut(slot) + .and_then(|seat| seat.device.as_mut()) + .ok_or(Refusal::Busy)?; + let pages = memory.len() / PAGE_BYTES; + if pages == 0 || memory.len() % PAGE_BYTES != 0 { + return Err(Refusal::Memory); + } + let pages = u32::try_from(pages).map_err(|_| Refusal::Memory)?; + // The view the device held goes inside `lend`; its owner after. + hosted.transport.lend(memory).map_err(|_| Refusal::Busy)?; + hosted.lent = Some(owner); + Ok(pages) + } + + /// The driver in `slot` gives its device up. The memory it lent goes + /// back, and the register block stops saying a device is there. + pub fn close_device(&mut self, slot: usize) -> Result<(), Refusal> { + let seat = self.seat_mut(slot).ok_or(Refusal::Busy)?; + if seat.device.take().is_none() { + return Err(Refusal::Busy); + } + let _ = seat.registers.store_u32(regs::MAGIC, 0); + self.open -= 1; + Ok(()) + } + + /// A doorbell: look at every open device. `interrupt` is called with the + /// slot and register window of each device that raised one, after its + /// interrupt count moved. Answers whether any device has chains waiting + /// that its budget did not reach, in which case the caller comes back + /// without waiting for another doorbell. + /// + /// A doorbell names no device, so each open one is looked at: at most + /// [`MAX_DEVICES`], and a look at an idle device is three loads. + pub fn service(&mut self, mut interrupt: impl FnMut(usize, &Window)) -> bool { + let mut pending = false; + let mut left = self.open; + for (slot, seat) in self.seats.iter_mut().enumerate() { + if left == 0 { + break; + } + let Some(seat) = seat else { continue }; + let Some(hosted) = seat.device.as_mut() else { + continue; + }; + left -= 1; + let serviced = hosted.transport.service(SERVICE_BUDGET); + let dead = hosted.transport.is_dead(); + if serviced.interrupt { + interrupt(slot, &seat.registers); + } + pending |= serviced.pending; + if dead { + // A model that could not be reset is not brought back. The + // block keeps the status that says so; the driver opens + // again for a new device. + seat.device = None; + self.open -= 1; + } + } + pending + } +} + +#[cfg(test)] +mod tests; diff --git a/native/services/vdev/src/host/tests.rs b/native/services/vdev/src/host/tests.rs new file mode 100644 index 0000000..36c43a4 --- /dev/null +++ b/native/services/vdev/src/host/tests.rs @@ -0,0 +1,397 @@ +use super::*; +use core::cell::Cell; +use core::ptr::NonNull; +use kore_vdev::driver::{self, DriverChain, DriverRing, Layout, Slots}; +use kore_vdev::regs::{ + STATUS_ACKNOWLEDGE, STATUS_DEVICE_NEEDS_RESET, STATUS_DRIVER, STATUS_DRIVER_OK, + STATUS_FEATURES_OK, +}; +use std::boxed::Box; +use std::rc::Rc; +use std::vec; +use std::vec::Vec; +use virtio_accel_guest::{Completion, GuestClient, GuestConfig, RequestPoll}; +use virtio_accel_proto::{RequestHeader, ResponseHeader, WireDeviceInfo}; +use virtio_accel_transport::QueueSize; + +/// Host memory standing for a lease both sides map. +fn window(bytes: u64) -> Window { + let words: &'static mut [u64] = + Box::leak(vec![0u64; (bytes as usize).div_ceil(8)].into_boxed_slice()); + // SAFETY: a leaked, eight-byte aligned allocation of at least `bytes` + // bytes that nothing else holds a reference into. + unsafe { + Window::new( + NonNull::new(words.as_mut_ptr().cast()).unwrap(), + bytes as usize, + ) + } +} + +fn page() -> Window { + window(regs::BYTES) +} + +/// What keeps lent memory mapped, in a test: it counts its own end. +#[derive(Debug)] +struct Owner(Rc>); + +impl Drop for Owner { + fn drop(&mut self) { + self.0.set(self.0.get() + 1); + } +} + +/// A count of owners dropped, and a way to make one. +fn owners() -> (Rc>, impl Fn() -> Owner) { + let dropped = Rc::new(Cell::new(0)); + let make = { + let dropped = dropped.clone(); + move || Owner(dropped.clone()) + }; + (dropped, make) +} + +fn magic(registers: &Window) -> u32 { + registers.load_u32(regs::MAGIC).unwrap() +} + +const LIVE: u32 = STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK | STATUS_DRIVER_OK; + +#[test] +fn a_seat_is_one_faces_and_names_its_occupant() { + let mut devices = Devices::::new(); + let registers = page(); + registers.store_u32(regs::MAGIC, 0xdead_beef).unwrap(); + + assert!(!devices.seat(MAX_SEATS, 7, registers), "no such slot"); + assert!(!devices.seat(2, 7, window(regs::BYTES - 8)), "not a page"); + assert!(!devices.seated(2)); + assert_eq!(devices.arrival(2), None); + + assert!(devices.seat(2, 7, registers)); + assert_eq!( + magic(®isters), + 0, + "an earlier face's device is not shown" + ); + assert!(devices.seated(2)); + assert_eq!(devices.arrival(2), Some(7)); + assert!(!devices.seat(2, 8, page()), "the slot has its occupant"); + assert_eq!(devices.arrival(2), Some(7)); + assert!(!devices.seated(1) && !devices.seated(3)); + + // The table grows to whatever slot the runtime names. + assert!(devices.seat(MAX_SEATS - 1, 9, page())); + assert_eq!(devices.arrival(MAX_SEATS - 1), Some(9)); + + devices.unseat(2); + assert!(!devices.seated(2)); + devices.unseat(2); + devices.unseat(MAX_SEATS + 5); + assert!(devices.seat(2, 8, page()), "and the slot is free again"); + assert_eq!(devices.arrival(2), Some(8)); +} + +#[test] +fn a_driver_opens_the_device_it_names_once() { + let mut devices = Devices::::new(); + let registers = page(); + assert_eq!(devices.open_device(0, DEVICE_ID), Err(Refusal::Unavailable)); + assert!(devices.seat(0, 1, registers)); + + assert_eq!(devices.open_device(0, 0), Err(Refusal::NoDevice)); + assert_eq!( + devices.open_device(0, u32::from_le_bytes(*b"echo")), + Err(Refusal::NoDevice) + ); + assert_eq!(magic(®isters), 0); + assert_eq!(devices.open(), 0); + + assert_eq!(devices.open_device(0, DEVICE_ID), Ok(())); + assert_eq!(devices.open(), 1); + let identity = driver::identify(®isters).expect("the block is written"); + assert_eq!(identity.device_id, DEVICE_ID); + assert_eq!(identity.vendor_id, regs::VENDOR_KORE); + assert_eq!(devices.open_device(0, DEVICE_ID), Err(Refusal::Busy)); + assert_eq!(devices.open(), 1); + + assert_eq!(devices.close_device(0), Ok(())); + assert_eq!(magic(®isters), 0, "no device is shown once it is closed"); + assert_eq!(devices.open(), 0); + assert_eq!(devices.close_device(0), Err(Refusal::Busy)); + assert_eq!(devices.close_device(9), Err(Refusal::Busy)); + assert_eq!(devices.open_device(0, DEVICE_ID), Ok(()), "and opens again"); +} + +#[test] +fn only_so_many_devices_are_open_at_once() { + let mut devices = Devices::::new(); + for slot in 0..=MAX_DEVICES { + assert!(devices.seat(slot, slot as u16, page())); + } + for slot in 0..MAX_DEVICES { + assert_eq!(devices.open_device(slot, DEVICE_ID), Ok(())); + } + assert_eq!(devices.open(), MAX_DEVICES); + assert_eq!( + devices.open_device(MAX_DEVICES, DEVICE_ID), + Err(Refusal::Limit) + ); + // The seat is still the driver's: it asks again when another closes, + // or when another's face is retired with its device open. + assert_eq!(devices.close_device(1), Ok(())); + assert_eq!(devices.open_device(MAX_DEVICES, DEVICE_ID), Ok(())); + assert_eq!(devices.open_device(1, DEVICE_ID), Err(Refusal::Limit)); + devices.unseat(0); + assert_eq!(devices.open(), MAX_DEVICES - 1); + assert_eq!(devices.open_device(1, DEVICE_ID), Ok(())); + // The heap the row is founded with is sized by this. + assert_eq!(DEVICE_HEAP_BYTES, 8 << 20); +} + +#[test] +fn lent_memory_is_kept_exactly_as_long_as_the_device_has_it() { + let (dropped, owner) = owners(); + let mut devices = Devices::::new(); + let registers = page(); + assert!(devices.seat(0, 1, registers)); + let memory = window(4 * PAGE_BYTES); + + // Nothing to lend to, and the memory goes straight back. + assert_eq!(devices.lend(0, memory, owner()), Err(Refusal::Busy)); + assert_eq!(devices.lend(3, memory, owner()), Err(Refusal::Busy)); + assert_eq!(dropped.get(), 2); + + devices.open_device(0, DEVICE_ID).unwrap(); + for bad in [0, 8, PAGE_BYTES - 8, PAGE_BYTES + 8] { + assert_eq!( + devices.lend(0, window(bad), owner()), + Err(Refusal::Memory), + "{bad} bytes" + ); + } + assert_eq!(dropped.get(), 6); + + assert_eq!(devices.lend(0, memory, owner()), Ok(4)); + assert_eq!(dropped.get(), 6, "kept while the device has it"); + // Lent again in the first state: the old memory goes back. + assert_eq!(devices.lend(0, window(PAGE_BYTES), owner()), Ok(1)); + assert_eq!(dropped.get(), 7); + + // Once the driver has taken a step, the device may be using it. + registers + .store_u32(regs::DRIVER_STATUS, STATUS_ACKNOWLEDGE) + .unwrap(); + devices.service(|_, _| {}); + assert_eq!(devices.lend(0, memory, owner()), Err(Refusal::Busy)); + assert_eq!(dropped.get(), 8, "the refused memory, not the lent"); + + // Closing the device gives it back; so does retiring the face. + devices.close_device(0).unwrap(); + assert_eq!(dropped.get(), 9); + devices.open_device(0, DEVICE_ID).unwrap(); + assert_eq!(devices.lend(0, memory, owner()), Ok(4)); + devices.unseat(0); + assert_eq!(dropped.get(), 10); +} + +type Client = GuestClient>; + +/// One driver program: its two windows, and what a bring-up gave it. +struct Program { + slot: usize, + registers: Window, + client: Client, + slots: Slots, +} + +const REQUEST: u32 = core::mem::size_of::() as u32; +const INFO: u32 = + (core::mem::size_of::() + core::mem::size_of::()) as u32; + +/// Seat a driver in `slot`, open the accelerator, lend it memory and bring +/// it up, with the reference guest client over the queue. +fn program(devices: &mut Devices, slot: usize, owner: Owner, size: u16) -> Program { + let registers = page(); + assert!(devices.seat(slot, slot as u16 + 100, registers)); + devices.open_device(slot, DEVICE_ID).unwrap(); + let layout = Layout::new(QueueSize::new(size).unwrap(), 256, 256).unwrap(); + let memory = window(layout.bytes().next_multiple_of(PAGE_BYTES)); + devices.lend(slot, memory, owner).unwrap(); + + let identity = driver::identify(®isters).unwrap(); + let mut config = [0u8; 16]; + driver::read_config(®isters, &mut config).unwrap(); + let wire = virtio_accel_proto::read_exact(&config).unwrap(); + let (ring, slots) = driver::bring_up(®isters, memory, layout, || { + devices.service(|_, _| {}); + }) + .expect("the device comes up"); + let config = GuestConfig::new(wire, identity.features & !regs::F_VERSION_1, layout.slots()) + .expect("a configuration the reference driver accepts"); + Program { + slot, + registers, + client: GuestClient::new(ring, config).unwrap(), + slots, + } +} + +impl Program { + /// Publish one discovery command. + fn ask(&mut self) -> virtio_accel_guest::Pending { + let chain: DriverChain = self.slots.chain(REQUEST, INFO).expect("a free slot"); + self.client.get_device_info(chain).ok().expect("started") + } + + /// Take one answer, which must be there. + fn answer(&mut self, pending: virtio_accel_guest::Pending) { + let RequestPoll::Ready(Completion::Success { output, chain }) = self.client.poll(pending) + else { + panic!("the command was answered"); + }; + assert_eq!(&output.uuid[..13], b"kore-spmd-cpu"); + self.slots.recycle(chain); + } + + fn interrupts(&self) -> u32 { + self.registers.load_u32(regs::INTERRUPTS).unwrap() + } +} + +/// One doorbell serves every open device, and each driver is interrupted +/// for its own work and no other's. +#[test] +fn a_doorbell_reaches_every_open_device() { + let (_, owner) = owners(); + let mut devices = Devices::::new(); + let mut first = program(&mut devices, 1, owner(), 16); + let mut second = program(&mut devices, 4, owner(), 16); + // A face with no device open is not looked at. + assert!(devices.seat(2, 50, page())); + let (before_first, before_second) = (first.interrupts(), second.interrupts()); + + let asked_first = first.ask(); + let asked_second = [second.ask(), second.ask()]; + let mut raised = Vec::new(); + assert!(!devices.service(|slot, registers| { + raised.push((slot, registers.load_u32(regs::INTERRUPTS).unwrap())); + })); + assert_eq!( + raised, + [(1, before_first + 1), (4, before_second + 1)], + "each told once, its count already moved" + ); + first.answer(asked_first); + for asked in asked_second { + second.answer(asked); + } + + // Only the driver with work is interrupted. + let asked = first.ask(); + let mut raised = Vec::new(); + devices.service(|slot, _| raised.push(slot)); + assert_eq!(raised, [first.slot]); + first.answer(asked); + assert_eq!(second.interrupts(), before_second + 1); + + // And an idle doorbell interrupts nobody. + assert!(!devices.service(|slot, _| panic!("seat {slot} was interrupted"))); +} + +/// A queue fuller than the budget is served over as many looks as it +/// takes, and the table says to come back until it is done. +#[test] +fn a_full_queue_is_served_over_several_looks() { + let (_, owner) = owners(); + let mut devices = Devices::::new(); + let mut driver = program(&mut devices, 0, owner(), 64); + let commands = SERVICE_BUDGET.get() + 4; + let asked: Vec<_> = (0..commands).map(|_| driver.ask()).collect(); + + assert!(devices.service(|_, _| {}), "the budget ran out"); + let mut looks = 1; + while devices.service(|_, _| {}) { + looks += 1; + assert!(looks < 8, "the queue drains"); + } + for asked in asked { + driver.answer(asked); + } +} + +/// A device reset past its last namespace cannot be brought back: the +/// table drops it, the block says it needs a reset, and the driver opens a +/// new one. +#[test] +fn a_device_that_cannot_be_reset_is_dropped() { + let (dropped, owner) = owners(); + let mut devices = Devices::::new(); + let registers = page(); + assert!(devices.seat(0, 1, registers)); + devices.open_device(0, DEVICE_ID).unwrap(); + devices.lend(0, window(PAGE_BYTES), owner()).unwrap(); + + let status = |value| registers.store_u32(regs::DRIVER_STATUS, value).unwrap(); + for _ in 1..u32::from(u16::MAX) { + status(STATUS_ACKNOWLEDGE); + devices.service(|_, _| {}); + status(0); + devices.service(|_, _| {}); + } + assert_eq!(devices.open(), 1); + assert_eq!(registers.load_u32(regs::DEVICE_STATUS), Ok(0)); + + status(STATUS_ACKNOWLEDGE); + devices.service(|_, _| {}); + status(0); + let mut told = 0; + devices.service(|_, _| told += 1); + assert_eq!(told, 1, "the driver hears of it"); + assert_eq!( + registers.load_u32(regs::DEVICE_STATUS), + Ok(STATUS_DEVICE_NEEDS_RESET) + ); + assert_eq!(devices.open(), 0, "the dead device is gone"); + assert_eq!(dropped.get(), 1, "and its memory went back"); + + assert_eq!(devices.close_device(0), Err(Refusal::Busy)); + devices.open_device(0, DEVICE_ID).unwrap(); + assert_eq!(registers.load_u32(regs::DEVICE_STATUS), Ok(0)); + registers + .store_u64(regs::DRIVER_FEATURES, regs::F_VERSION_1) + .unwrap(); + registers.store_u32(regs::QUEUE_NUM, 16).unwrap(); + registers.store_u32(regs::QUEUE_READY, 1).unwrap(); + status(LIVE); + devices.service(|_, _| {}); + assert_ne!( + registers.load_u32(regs::DEVICE_STATUS).unwrap() & STATUS_DEVICE_NEEDS_RESET, + 0, + "a new device, which was lent nothing" + ); +} + +/// The row's founded heap covers every device the table will open, with +/// room over for what is not buffers. The two numbers live in two files; +/// this is where they are held together. +#[test] +fn the_founded_heap_covers_every_open_device() { + let row = include_str!("../../boot.toml"); + let value = row + .lines() + .find_map(|line| line.strip_prefix("heap_supply_bytes")) + .and_then(|rest| rest.split('=').nth(1)) + .map(|value| value.split('#').next().unwrap().trim().replace('_', "")) + .expect("the row sizes its heap"); + let heap = u64::from_str_radix(value.trim_start_matches("0x"), 16).expect("a hex size"); + // The engines' tables, the seat table and the runtime's own boxes. + const OVERHEAD: u64 = 2 << 20; + assert!( + heap >= DEVICE_HEAP_BYTES + OVERHEAD, + "{heap:#x} does not cover {MAX_DEVICES} devices of buffers and the overhead" + ); + // One lease, mapped at one stride of the founder's heap band. + assert!(heap <= 16 << 20); +} diff --git a/native/services/vdev/src/lib.rs b/native/services/vdev/src/lib.rs new file mode 100644 index 0000000..d57f05a --- /dev/null +++ b/native/services/vdev/src/lib.rs @@ -0,0 +1,53 @@ +//! # kore-svc-vdev: the virtual-device service +//! +//! A virtual device is a device with no hardware behind it (`kore-vdev`). +//! This service is where one lives: it hosts a device model for each driver +//! that asks for one, and what passes between a driver and its device is +//! memory the two share and two wakes. +//! +//! - **Registers** are a page of the driver's own face with the service, so +//! the driver has them mapped from its first instruction and the device +//! writes them when the driver opens it ([`VdevRequest::Open`]). +//! - **DMA** is memory the driver owns and lends ([`VdevRequest::Lend`]). +//! A device address is an offset into it; the service maps exactly what +//! was lent and can reach nothing else of the driver's. +//! - **The doorbell** is a notice on the face's kick. It carries nothing: +//! the service looks at every device it hosts when one arrives. +//! - **The interrupt** is the register block's interrupt count +//! (`kore_vdev::regs::INTERRUPTS`), which the service moves and then +//! wakes by the memory. A driver parks on the word. +//! +//! No kernel mechanism is specific to any of it. The face is one exec +//! mints for a program at launch and retires when the program ends, as it +//! does the power face ([`service`] documents the administration half). +//! +//! The device table ([`host`]) is plain data over `kore-vdev`'s transport +//! and is tested off metal; [`service`] is the shell that feeds it from the +//! runtime. + +#![no_std] +// The peer table grows with the programs that hold a face, from the +// service's own heap through the `Allocator` seam. +#![cfg_attr(feature = "host", feature(allocator_api))] + +extern crate alloc; + +#[cfg(test)] +extern crate std; + +pub mod client; +pub mod face; +#[cfg(feature = "host")] +pub mod host; +#[cfg(feature = "host")] +pub mod service; +pub mod wire; + +#[cfg(target_os = "none")] +pub use client::{ProcessVdev, ThreadVdev, attach_process_face, attach_thread_face}; +pub use client::{RingVdev, RingVdevAdmin, THREAD_SHARES, Waited, process_face}; +#[cfg(not(feature = "host"))] +pub use face::VdevService; +#[cfg(feature = "host")] +pub use service::{VdevEndowment, VdevService}; +pub use wire::{VdevRefusal, VdevRequest, VdevResponse}; diff --git a/native/services/vdev/src/service.rs b/native/services/vdev/src/service.rs new file mode 100644 index 0000000..06b9162 --- /dev/null +++ b/native/services/vdev/src/service.rs @@ -0,0 +1,389 @@ +//! The service: the device table, fed from the runtime. +//! +//! # Whose face is whose +//! +//! Pair zero is the founder's. Pair one is exec's, the *administration* +//! pair: exec builds a face for each program it launches with the +//! virtual-device profile, conveys its components here one request at a +//! time ([`VdevRequest::AdmitPeer`]), and retires it when the program ends +//! ([`VdevRequest::RetirePeer`]). Every other pair is such a face, a +//! driver's. +//! +//! A driver's face is two pages exec supplies: the request rings, then the +//! register block ([`REGISTERS_AT`]). The driver has both mapped before its +//! first instruction; the service is handed the second when the face is +//! admitted, and seats the driver over it. +//! +//! # What wakes the service +//! +//! A request on any pair, or a doorbell: a bare notice on a driver's kick. +//! Either way the service then looks at every open device +//! ([`Devices::service`]), so a doorbell needs no request and no reply. A +//! device whose budget ran out with chains still waiting is looked at again +//! at once, without waiting for another doorbell. + +use kore_abi::task::geom; +use kore_rt::service::{PeerClose, ReturnMode, Serve, StartFailure, SystemService}; +use kore_rt::svc::{ + PeerAdmissionRefusal, PeerBundleAssembler, PeerBundlePart, PeerId, PeerMapPlan, REP_CAP, + REQ_CAP, window_bytes, +}; +use kore_vdev::mem::Window; +use kore_vdev::regs; + +pub use crate::face::{PEER_BYTES, REGISTERS_AT}; +use crate::host::{Devices, MAX_SEATS}; +pub use crate::wire::{VdevRefusal, VdevRequest, VdevResponse}; + +/// The service is endowed with nothing: it holds no device and no +/// authority over the machine, only its own heap and the faces exec +/// conveys to it. +#[derive(Debug, kore_rt::Bundle)] +#[bundle(name = "vdev")] +pub struct VdevEndowment; + +/// The pair exec administers driver faces on. +const ADMIN_VIA: u16 = 1; + +/// What keeps a lent window mapped in this task: the capability the driver +/// delivered. Dropping it revokes the view, and the mapping with it. +#[cfg(target_os = "none")] +type Lent = kore_rt::cap::ReturnedCapability; +/// Off metal nothing is ever lent: there is no arena to map from. +#[cfg(not(target_os = "none"))] +type Lent = core::convert::Infallible; + +/// The virtual-device service. +#[derive(Debug)] +pub struct VdevService { + devices: Devices, + /// A device ran out of budget with chains waiting: look again at once. + pending: bool, + peer_assembler: PeerBundleAssembler, + peer_map: Option, + last_admission: Option>, +} + +impl VdevService { + /// The seat of the driver a request arrived from, if it is a driver's + /// face and still the occupant of its slot. + fn seat_of(&self, via: u16) -> Option { + let slot = PeerId::from_arrival(via)?.slot(); + (self.devices.arrival(slot) == Some(via)).then_some(slot) + } + + /// The register block inside a face's tail: `aux` is the face after its + /// rings. + fn registers_in(aux: &'static mut [u8]) -> Option { + let rings = window_bytes::(REQ_CAP, REP_CAP) as u64; + let skip = usize::try_from(REGISTERS_AT - rings).ok()?; + let block = aux.get_mut(skip..)?; + if (block.len() as u64) < regs::BYTES { + return None; + } + let base = core::ptr::NonNull::new(block.as_mut_ptr())?; + if base.as_ptr() as usize % REGISTERS_AT as usize != 0 { + return None; + } + // SAFETY: the host mapped the face read-write at a page boundary and + // keeps it mapped until it has called `peer_detaching`, where the + // seat and with it every copy of this view is dropped. The slice + // this was made from is not used again, so nothing in this task + // holds a reference into the page while the driver writes it. + Some(unsafe { Window::new(base, regs::BYTES as usize) }) + } + + fn lend(&mut self, seat: usize, lent: Option) -> VdevResponse { + let Some(owner) = lent else { + // Nothing arrived on the return leg to lend. + return VdevResponse::Refused(VdevRefusal::Memory); + }; + match map_lent(owner) { + Some((memory, owner)) => match self.devices.lend(seat, memory, owner) { + Ok(pages) => VdevResponse::Lent { pages }, + Err(refusal) => VdevResponse::Refused(refusal.into()), + }, + None => VdevResponse::Refused(VdevRefusal::Memory), + } + } +} + +/// Map what a driver lent. The owner comes back with the window so the two +/// are kept, and dropped, together. `None` drops the owner, which gives the +/// memory back. +#[cfg(target_os = "none")] +fn map_lent(owner: Lent) -> Option<(Window, Lent)> { + use kore_rt::cap::verbs; + let view = verbs::take_view(owner.slot()?)?; + // A sink mapping: should the driver die with its memory lent, the + // device's next look reads a sink page, which the transport takes for + // what it is (a status of zero is a reset, an index of zero is an empty + // queue), instead of faulting on a hole before the face is retired. + let mapped = verbs::map_view_sink(view)?; + let base = mapped.cast::(); + if base.as_ptr() as usize % core::mem::align_of::() != 0 { + return None; + } + // SAFETY: the kernel installed `[base, +len)` read-write in this task + // over the lent view and keeps it there while `owner` lives. The window + // is handed to the device table with `owner`, which drops every view + // before it drops the owner. The driver writes the same bytes at will, + // which is what `Window` is for: no reference is ever formed into them. + Some((unsafe { Window::new(base, mapped.len()) }, owner)) +} + +#[cfg(not(target_os = "none"))] +fn map_lent(owner: Lent) -> Option<(Window, Lent)> { + match owner {} +} + +/// Wake the driver parked on its device's interrupt count. +#[cfg(target_os = "none")] +fn interrupt(registers: &Window) { + // The word's address, to name it to the kernel: never dereferenced. + let word = registers + .base() + .as_ptr() + .wrapping_add(regs::INTERRUPTS as usize); + // The count moved before this. A driver not yet parked reads it again + // before it parks; one that is parked is woken here. A wake the kernel + // refuses is a lost kick, which the driver's deadline covers. + if let Some((subject, offset)) = kore_rt::cap::mapped::locate(word) { + let _ = kore_rt::cap::verbs::wake_word(subject, offset, u32::MAX); + } +} + +#[cfg(not(target_os = "none"))] +fn interrupt(_registers: &Window) {} + +impl SystemService for VdevService { + const NAME: &'static str = "vdev"; + type Endowment = VdevEndowment; + + /// Both return legs deliver here: the administration pair's carries a + /// face's components, a driver's carries the memory it lends. + const RETURN_MODE: ReturnMode = ReturnMode::RetainedAny; + + fn bind(_endowment: VdevEndowment) -> Self { + Self { + devices: Devices::new(), + pending: false, + peer_assembler: PeerBundleAssembler::new(0, true), + peer_map: None, + last_admission: None, + } + } + + /// One face for each program that drives a device, and many programs + /// may run at once: the peer table grows from the service's heap. + fn peer_arena(&self) -> Option<&'static (dyn core::alloc::Allocator + Sync)> { + static GLOBAL: alloc::alloc::Global = alloc::alloc::Global; + Some(&GLOBAL) + } + + fn start_on_metal(&mut self, rt: &kore_rt::entry::Rt) -> Result<(), StartFailure> { + // Where driver faces are mapped: one stride a slot, in the + // transient window, for every slot the runtime can name. + let window = rt + .geom() + .and_then(|geometry| geometry.window_of(geom::TRANSIENT)) + .filter(|window| { + window.base() != 0 + && window.base() & 0xfff == 0 + && window.len() >= PEER_BYTES * MAX_SEATS as u64 + }) + .ok_or(StartFailure::new(1))?; + self.peer_map = Some(PeerMapPlan { + base: window.base(), + stride: PEER_BYTES, + }); + Ok(()) + } + + /// Runs on every wake, a doorbell included: look at every open device. + fn poll_background(&mut self) { + self.pending = self.devices.service(|_, registers| interrupt(registers)); + } + + /// A device with chains its budget did not reach is due now. + fn next_deadline(&self) -> Option { + self.pending.then_some(0) + } + + fn drive_deadlines(&mut self, _now_ns: u64) -> u16 { + // The look itself is `poll_background`'s, on the same pass. + 0 + } + + fn drive_host( + host: kore_rt::svc::ServeHost<'_, '_, T, Self>, + ) -> impl core::future::Future { + async move { + host.run_deadline_driven().await; + } + } + + fn serve(&mut self, req: VdevRequest, sv: &mut Serve<'_, VdevResponse>) { + let resp = match req { + VdevRequest::AdmitPeer { part, announced } => { + // A `Ring` begins a face. One arriving while the assembler + // still owns components means the last face was abandoned + // by an asker that gave up: retire what it left, and refuse + // the new one while any of those owners is still closing. + // Administration only, and before anything else touches the + // assembler. + let abandoned = sv.via() == ADMIN_VIA + && matches!(part, PeerBundlePart::Ring) + && !self.peer_assembler.is_empty() + && !self.peer_assembler.retire_incomplete(); + if sv.via() != ADMIN_VIA || self.last_admission.is_some() || abandoned { + VdevResponse::Refused(VdevRefusal::Conflict) + } else if let (Some(map), Some(mut input)) = + (self.peer_map, sv.returned_input(announced)) + { + let accepted = input + .capability() + .and_then(|capability| self.peer_assembler.accepts(part, capability).ok()) + .is_some(); + if !accepted { + VdevResponse::Refused(VdevRefusal::Denied) + } else { + let capability = input.take().expect("validated peer component remains"); + let bundle = self + .peer_assembler + .push(part, capability) + .expect("synchronous peer validation remains stable"); + if let Some(bundle) = bundle { + sv.admit_peer(bundle, map); + } + VdevResponse::Accepted + } + } else { + VdevResponse::Refused(VdevRefusal::Unavailable) + } + } + VdevRequest::TakeAdmission => { + if sv.via() != ADMIN_VIA { + VdevResponse::Refused(VdevRefusal::Denied) + } else { + match self.last_admission.take() { + Some(Ok(peer)) => VdevResponse::PeerAdmitted { peer }, + Some(Err(_)) => VdevResponse::Refused(VdevRefusal::Unavailable), + // No verdict and components still held: the face + // never completed, so no verdict will ever come. + // `Pending` would be a wait with no end. + None if !self.peer_assembler.is_empty() => { + let _ = self.peer_assembler.retire_incomplete(); + VdevResponse::Refused(VdevRefusal::Conflict) + } + // Complete, and the host has not admitted it yet. + None => VdevResponse::Refused(VdevRefusal::Pending), + } + } + } + VdevRequest::RetirePeer { peer } => { + if sv.via() == ADMIN_VIA + && self.devices.arrival(peer.slot()) == Some(peer.arrival()) + { + sv.retire_peer(peer, PeerClose::Orderly); + VdevResponse::Accepted + } else { + VdevResponse::Refused(VdevRefusal::Denied) + } + } + VdevRequest::ConfirmRetired { peer } => { + if sv.via() == ADMIN_VIA + && self.devices.arrival(peer.slot()) != Some(peer.arrival()) + { + VdevResponse::Accepted + } else { + VdevResponse::Refused(VdevRefusal::Conflict) + } + } + VdevRequest::ClosePeer => { + if self.seat_of(sv.via()).is_some() { + sv.close_peer(PeerClose::Orderly); + VdevResponse::Accepted + } else { + VdevResponse::Refused(VdevRefusal::Denied) + } + } + VdevRequest::Open { device } => match self.seat_of(sv.via()) { + Some(seat) => match self.devices.open_device(seat, device) { + Ok(()) => VdevResponse::Accepted, + Err(refusal) => VdevResponse::Refused(refusal.into()), + }, + None => VdevResponse::Refused(VdevRefusal::Denied), + }, + VdevRequest::Lend { announced } => { + // Taken whatever the answer: a capability lent to a request + // that is refused is dropped, which gives it back. + let lent = sv + .returned_input(announced) + .and_then(|mut input| input.take()); + match self.seat_of(sv.via()) { + Some(seat) => self.lend(seat, lent.and_then(lent_owner)), + None => VdevResponse::Refused(VdevRefusal::Denied), + } + } + VdevRequest::Close => match self.seat_of(sv.via()) { + Some(seat) => match self.devices.close_device(seat) { + Ok(()) => VdevResponse::Accepted, + Err(refusal) => VdevResponse::Refused(refusal.into()), + }, + None => VdevResponse::Refused(VdevRefusal::Denied), + }, + }; + sv.reply(resp); + } + + /// The face's tail is its register block: seat the driver over it. + fn attach_peer_aux(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + match Self::registers_in(aux) { + Some(registers) => self.devices.seat(peer.slot(), peer.arrival(), registers), + None => false, + } + } + + /// The host is about to unmap the face: nothing here may touch its + /// register block, or the memory its driver lent, after this. + fn peer_detaching(&mut self, peer: PeerId) { + if self.devices.arrival(peer.slot()) == Some(peer.arrival()) { + self.devices.unseat(peer.slot()); + } + } + + fn peer_admitted(&mut self, result: Result) -> bool { + if let Ok(peer) = result + && self.devices.arrival(peer.slot()) != Some(peer.arrival()) + { + // Admitted with no seat: the face's tail was never attached. + self.last_admission = Some(Err(PeerAdmissionRefusal::PolicyCollision)); + return false; + } + self.last_admission = Some(result); + true + } + + fn peer_closed(&mut self, via: u16, _reason: PeerClose) -> bool { + // `peer_detaching` has run for a face that was seated; this covers + // one that closed before it was. + if let Some(seat) = self.seat_of(via) { + self.devices.unseat(seat); + } + true + } +} + +/// The capability a driver delivered, as what keeps its window mapped. +#[cfg(target_os = "none")] +fn lent_owner(capability: kore_rt::cap::ReturnedCapability) -> Option { + Some(capability) +} + +/// Off metal a delivered capability maps nothing: it is dropped, and the +/// request is answered as one that lent no memory. +#[cfg(not(target_os = "none"))] +fn lent_owner(_capability: kore_rt::cap::ReturnedCapability) -> Option { + None +} diff --git a/native/services/vdev/src/wire.rs b/native/services/vdev/src/wire.rs new file mode 100644 index 0000000..af51e2a --- /dev/null +++ b/native/services/vdev/src/wire.rs @@ -0,0 +1,465 @@ +//! The request vocabulary and its crossing shape. +//! +//! Two kinds of asker share one ring layout. The administration pair +//! (exec's) conveys the components of a driver's face and retires it when +//! the program ends; it never asks for a device. A driver's own face asks +//! for a device and lends it memory; it cannot administer. Which is which +//! is the pair a request arrived on, never a byte of the request. + +use kore_abi::session::SlotRef; +use kore_abi::svc::{VdevRepWire, VdevReqWire}; +use kore_rt::svc::{PeerBundlePart, PeerId, SvcProtocol}; + +use crate::VdevService; + +/// What a face may ask. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VdevRequest { + /// Administration: one component of a driver's face, delivered on the + /// administration pair's return leg. + AdmitPeer { + part: PeerBundlePart, + announced: SlotRef, + }, + /// Administration: the verdict on the face whose last component was + /// conveyed. + TakeAdmission, + /// Administration: the program holding this face has ended. + RetirePeer { peer: PeerId }, + /// Administration: has that face gone? + ConfirmRetired { peer: PeerId }, + /// A driver ends its own face. + ClosePeer, + /// A driver asks for the device model with this register-block id. + Open { device: u32 }, + /// A driver lends the memory its device addresses, delivered on the + /// face's return leg. + Lend { announced: SlotRef }, + /// A driver gives its device up and takes its memory back. + Close, +} + +/// What a face is told. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VdevResponse { + Accepted, + Refused(VdevRefusal), + PeerAdmitted { + peer: PeerId, + }, + /// The device addresses this many 4 KiB pages from here on. + Lent { + pages: u32, + }, +} + +/// Why a request was not done. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VdevRefusal { + /// The request is not this face's to make. + Denied, + /// The admission result is not published yet. + Pending, + /// An earlier admission must be settled first. + Conflict, + /// The service cannot do this now. + Unavailable, + /// No device model has that id. + NoDevice, + /// The device is not in a state for this. + Busy, + /// Every device the service hosts at once is taken. + Limit, + /// What was lent is not memory a device can address. + Memory, +} + +#[cfg(feature = "host")] +impl From for VdevRefusal { + fn from(refusal: crate::host::Refusal) -> Self { + use crate::host::Refusal; + match refusal { + Refusal::NoDevice => Self::NoDevice, + Refusal::Busy => Self::Busy, + Refusal::Limit => Self::Limit, + Refusal::Unavailable => Self::Unavailable, + Refusal::Memory => Self::Memory, + } + } +} + +const PART_RING: u8 = 0; +const PART_KICK: u8 = 1; +const PART_RETURN: u8 = 2; +const PART_GRANT: u8 = 0x80; + +/// Lower a request (total). +pub fn encode_req(req: &VdevRequest) -> VdevReqWire { + let mut wire = VdevReqWire { + op: 0, + reserved: [0; 7], + }; + wire.op = match *req { + VdevRequest::AdmitPeer { part, announced } => { + wire.reserved[0] = match part { + PeerBundlePart::Ring => PART_RING, + PeerBundlePart::Kick => PART_KICK, + PeerBundlePart::ReturnRecv => PART_RETURN, + PeerBundlePart::Grant(index) => PART_GRANT | index, + }; + wire.reserved[1..5].copy_from_slice(&announced.index().to_le_bytes()); + VdevReqWire::OP_ADMIT_PEER + } + VdevRequest::TakeAdmission => VdevReqWire::OP_TAKE_ADMISSION, + VdevRequest::RetirePeer { peer } => { + wire.reserved[..2].copy_from_slice(&peer.arrival().to_le_bytes()); + VdevReqWire::OP_RETIRE_PEER + } + VdevRequest::ConfirmRetired { peer } => { + wire.reserved[..2].copy_from_slice(&peer.arrival().to_le_bytes()); + VdevReqWire::OP_CONFIRM_RETIRED + } + VdevRequest::ClosePeer => VdevReqWire::OP_CLOSE_PEER, + VdevRequest::Open { device } => { + wire.reserved[..4].copy_from_slice(&device.to_le_bytes()); + VdevReqWire::OP_OPEN + } + VdevRequest::Lend { announced } => { + wire.reserved[..4].copy_from_slice(&announced.index().to_le_bytes()); + VdevReqWire::OP_LEND + } + VdevRequest::Close => VdevReqWire::OP_CLOSE, + }; + wire +} + +fn word(bytes: &[u8; 7], at: usize) -> u32 { + u32::from_le_bytes([bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]]) +} + +fn decode_peer(bytes: &[u8; 7]) -> Option { + if bytes[2..].iter().any(|byte| *byte != 0) { + return None; + } + PeerId::from_arrival(u16::from_le_bytes([bytes[0], bytes[1]])) +} + +/// Lift a wire request, fail-closed: only each opcode's declared payload is +/// accepted, and every byte beyond it must be zero. +pub fn decode_req(w: &VdevReqWire) -> Option { + let empty = w.reserved == [0; 7]; + Some(match w.op { + VdevReqWire::OP_ADMIT_PEER if w.reserved[5..] == [0; 2] => VdevRequest::AdmitPeer { + part: match w.reserved[0] { + PART_RING => PeerBundlePart::Ring, + PART_KICK => PeerBundlePart::Kick, + PART_RETURN => PeerBundlePart::ReturnRecv, + value if value & PART_GRANT != 0 => PeerBundlePart::Grant(value & !PART_GRANT), + _ => return None, + }, + announced: SlotRef::from_wire(word(&w.reserved, 1)), + }, + VdevReqWire::OP_TAKE_ADMISSION if empty => VdevRequest::TakeAdmission, + VdevReqWire::OP_RETIRE_PEER => VdevRequest::RetirePeer { + peer: decode_peer(&w.reserved)?, + }, + VdevReqWire::OP_CONFIRM_RETIRED => VdevRequest::ConfirmRetired { + peer: decode_peer(&w.reserved)?, + }, + VdevReqWire::OP_CLOSE_PEER if empty => VdevRequest::ClosePeer, + VdevReqWire::OP_OPEN if w.reserved[4..] == [0; 3] => VdevRequest::Open { + device: word(&w.reserved, 0), + }, + VdevReqWire::OP_LEND if w.reserved[4..] == [0; 3] => VdevRequest::Lend { + announced: SlotRef::from_wire(word(&w.reserved, 0)), + }, + VdevReqWire::OP_CLOSE if empty => VdevRequest::Close, + _ => return None, + }) +} + +/// Lower a response (total). The service grants nothing, so `grants` is +/// written zero whatever was asked. +pub fn encode_rep(rep: &VdevResponse, _grants: u8) -> VdevRepWire { + let (tag, refusal, reserved) = match *rep { + VdevResponse::Accepted => (VdevRepWire::TAG_ACCEPTED, 0, [0; 5]), + VdevResponse::PeerAdmitted { peer } => { + let arrival = peer.arrival().to_le_bytes(); + ( + VdevRepWire::TAG_PEER_ADMITTED, + 0, + [arrival[0], arrival[1], 0, 0, 0], + ) + } + VdevResponse::Lent { pages } => { + let pages = pages.to_le_bytes(); + ( + VdevRepWire::TAG_LENT, + 0, + [pages[0], pages[1], pages[2], pages[3], 0], + ) + } + VdevResponse::Refused(refusal) => ( + VdevRepWire::TAG_REFUSED, + match refusal { + VdevRefusal::Denied => VdevRepWire::REFUSE_DENIED, + VdevRefusal::Pending => VdevRepWire::REFUSE_PENDING, + VdevRefusal::Conflict => VdevRepWire::REFUSE_CONFLICT, + VdevRefusal::Unavailable => VdevRepWire::REFUSE_UNAVAILABLE, + VdevRefusal::NoDevice => VdevRepWire::REFUSE_NO_DEVICE, + VdevRefusal::Busy => VdevRepWire::REFUSE_BUSY, + VdevRefusal::Limit => VdevRepWire::REFUSE_LIMIT, + VdevRefusal::Memory => VdevRepWire::REFUSE_MEMORY, + }, + [0; 5], + ), + }; + VdevRepWire { + tag, + refusal, + grants: 0, + reserved, + } +} + +/// Lift a wire reply, fail-closed: a non-zero `grants`, a refusal byte on +/// a success tag and a set reserved byte each refuse. +pub fn decode_rep(w: &VdevRepWire) -> Option<(VdevResponse, u8)> { + if w.grants != 0 { + return None; + } + let response = match (w.tag, w.refusal) { + (VdevRepWire::TAG_ACCEPTED, 0) if w.reserved == [0; 5] => VdevResponse::Accepted, + (VdevRepWire::TAG_PEER_ADMITTED, 0) if w.reserved[2..] == [0; 3] => { + VdevResponse::PeerAdmitted { + peer: PeerId::from_arrival(u16::from_le_bytes([w.reserved[0], w.reserved[1]]))?, + } + } + (VdevRepWire::TAG_LENT, 0) if w.reserved[4] == 0 => VdevResponse::Lent { + pages: u32::from_le_bytes([w.reserved[0], w.reserved[1], w.reserved[2], w.reserved[3]]), + }, + (VdevRepWire::TAG_REFUSED, refusal) if w.reserved == [0; 5] => { + VdevResponse::Refused(match refusal { + VdevRepWire::REFUSE_DENIED => VdevRefusal::Denied, + VdevRepWire::REFUSE_PENDING => VdevRefusal::Pending, + VdevRepWire::REFUSE_CONFLICT => VdevRefusal::Conflict, + VdevRepWire::REFUSE_UNAVAILABLE => VdevRefusal::Unavailable, + VdevRepWire::REFUSE_NO_DEVICE => VdevRefusal::NoDevice, + VdevRepWire::REFUSE_BUSY => VdevRefusal::Busy, + VdevRepWire::REFUSE_LIMIT => VdevRefusal::Limit, + VdevRepWire::REFUSE_MEMORY => VdevRefusal::Memory, + _ => return None, + }) + } + _ => return None, + }; + Some((response, 0)) +} + +impl SvcProtocol for VdevService { + type Request = VdevRequest; + type Response = VdevResponse; + type ReqWire = VdevReqWire; + type RepWire = VdevRepWire; + + /// The service mints no authority: structural, not a convention. + const REPLY_GRANTS: usize = 0; + /// Pair zero is the founder's control face; pair one is exec's, which + /// administers the driver faces. + const SERVE_PAIRS: usize = 2; + + fn reply_matches(request: &VdevReqWire, response: &VdevResponse, grants: u8) -> bool { + grants == 0 + && match (decode_req(request), response) { + (Some(VdevRequest::TakeAdmission), VdevResponse::PeerAdmitted { .. }) => true, + (Some(VdevRequest::Lend { .. }), VdevResponse::Lent { .. }) => true, + (Some(VdevRequest::Lend { .. }), VdevResponse::Accepted) => false, + (Some(_), VdevResponse::Accepted | VdevResponse::Refused(_)) => true, + _ => false, + } + } + + fn encode_req(req: &VdevRequest) -> VdevReqWire { + encode_req(req) + } + fn decode_req(wire: &VdevReqWire) -> Option { + decode_req(wire) + } + fn encode_rep(rep: &VdevResponse, grants: u8) -> VdevRepWire { + encode_rep(rep, grants) + } + fn decode_rep(wire: &VdevRepWire) -> Option<(VdevResponse, u8)> { + decode_rep(wire) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn peer() -> PeerId { + PeerId::from_arrival(PeerId::SLOT_CAPACITY as u16).expect("the first recyclable pair") + } + + const REQUESTS: [fn() -> VdevRequest; 11] = [ + || VdevRequest::AdmitPeer { + part: PeerBundlePart::Ring, + announced: SlotRef::from_wire(27), + }, + || VdevRequest::AdmitPeer { + part: PeerBundlePart::Kick, + announced: SlotRef::from_wire(28), + }, + || VdevRequest::AdmitPeer { + part: PeerBundlePart::ReturnRecv, + announced: SlotRef::from_wire(u32::MAX), + }, + || VdevRequest::AdmitPeer { + part: PeerBundlePart::Grant(3), + announced: SlotRef::from_wire(0), + }, + || VdevRequest::TakeAdmission, + || VdevRequest::RetirePeer { peer: peer() }, + || VdevRequest::ConfirmRetired { peer: peer() }, + || VdevRequest::ClosePeer, + || VdevRequest::Open { + device: 0x6c63_6361, + }, + || VdevRequest::Lend { + announced: SlotRef::from_wire(0x0102_0304), + }, + || VdevRequest::Close, + ]; + + const RESPONSES: [fn() -> VdevResponse; 12] = [ + || VdevResponse::Accepted, + || VdevResponse::PeerAdmitted { peer: peer() }, + || VdevResponse::Lent { pages: 1 }, + || VdevResponse::Lent { pages: u32::MAX }, + || VdevResponse::Refused(VdevRefusal::Denied), + || VdevResponse::Refused(VdevRefusal::Pending), + || VdevResponse::Refused(VdevRefusal::Conflict), + || VdevResponse::Refused(VdevRefusal::Unavailable), + || VdevResponse::Refused(VdevRefusal::NoDevice), + || VdevResponse::Refused(VdevRefusal::Busy), + || VdevResponse::Refused(VdevRefusal::Limit), + || VdevResponse::Refused(VdevRefusal::Memory), + ]; + + #[test] + fn every_request_and_response_round_trips() { + for request in REQUESTS { + let request = request(); + assert_eq!(decode_req(&encode_req(&request)), Some(request)); + } + for response in RESPONSES { + let response = response(); + assert_eq!(decode_rep(&encode_rep(&response, 0)), Some((response, 0))); + // Whatever was asked, nothing is granted. + assert_eq!(encode_rep(&response, 3).grants, 0); + } + } + + /// A zeroed record, an unknown opcode and a byte set beyond an opcode's + /// payload each refuse: nothing is guessed. + #[test] + fn a_request_decodes_only_from_exactly_its_own_bytes() { + let zero = VdevReqWire { + op: 0, + reserved: [0; 7], + }; + assert_eq!(decode_req(&zero), None, "a zeroed record asks nothing"); + assert_eq!(decode_req(&VdevReqWire { op: 9, ..zero }), None); + assert_eq!(decode_req(&VdevReqWire { op: 255, ..zero }), None); + + for request in REQUESTS { + let request = request(); + let wire = encode_req(&request); + // How many leading bytes of the payload this opcode owns. + let owned = match request { + VdevRequest::AdmitPeer { .. } => 5, + VdevRequest::RetirePeer { .. } | VdevRequest::ConfirmRetired { .. } => 2, + VdevRequest::Open { .. } | VdevRequest::Lend { .. } => 4, + VdevRequest::TakeAdmission | VdevRequest::ClosePeer | VdevRequest::Close => 0, + }; + for at in owned..7 { + let mut stray = wire; + stray.reserved[at] = 1; + assert_eq!(decode_req(&stray), None, "{request:?}, byte {at}"); + } + } + // An unknown part of a face is not a part. + let mut part = encode_req(&REQUESTS[0]()); + part.reserved[0] = 3; + assert_eq!(decode_req(&part), None); + // A fixed pair is not a face exec can retire. + let mut fixed = encode_req(&VdevRequest::RetirePeer { peer: peer() }); + fixed.reserved[..2].copy_from_slice(&1u16.to_le_bytes()); + assert_eq!(decode_req(&fixed), None); + } + + #[test] + fn a_reply_decodes_only_from_exactly_its_own_bytes() { + let zero = VdevRepWire { + tag: 0, + refusal: 0, + grants: 0, + reserved: [0; 5], + }; + assert_eq!(decode_rep(&zero), None); + for response in RESPONSES { + let response = response(); + let wire = encode_rep(&response, 0); + assert_eq!(decode_rep(&VdevRepWire { grants: 1, ..wire }), None); + if !matches!(response, VdevResponse::Refused(_)) { + let refused = VdevRepWire { refusal: 1, ..wire }; + assert_eq!(decode_rep(&refused), None, "{response:?}"); + } + let owned = match response { + VdevResponse::PeerAdmitted { .. } => 2, + VdevResponse::Lent { .. } => 4, + VdevResponse::Accepted | VdevResponse::Refused(_) => 0, + }; + for at in owned..5 { + let mut stray = wire; + stray.reserved[at] = 1; + assert_eq!(decode_rep(&stray), None, "{response:?}, byte {at}"); + } + } + let unknown = VdevRepWire { + tag: VdevRepWire::TAG_REFUSED, + refusal: 9, + ..zero + }; + assert_eq!(decode_rep(&unknown), None); + } + + /// A reply is believed only if it answers what was asked. + #[test] + fn a_reply_must_answer_its_request() { + let matches = |request: VdevRequest, response: VdevResponse| { + ::reply_matches(&encode_req(&request), &response, 0) + }; + let lend = VdevRequest::Lend { + announced: SlotRef::from_wire(1), + }; + assert!(matches(lend, VdevResponse::Lent { pages: 4 })); + assert!(!matches(lend, VdevResponse::Accepted)); + assert!(matches(lend, VdevResponse::Refused(VdevRefusal::Busy))); + assert!(!matches(lend, VdevResponse::PeerAdmitted { peer: peer() })); + assert!(matches( + VdevRequest::TakeAdmission, + VdevResponse::PeerAdmitted { peer: peer() } + )); + assert!(!matches( + VdevRequest::Close, + VdevResponse::Lent { pages: 4 } + )); + assert!(matches(VdevRequest::Close, VdevResponse::Accepted)); + assert!(!::reply_matches( + &encode_req(&VdevRequest::Close), + &VdevResponse::Accepted, + 1 + )); + } +} diff --git a/src/lib.rs b/src/lib.rs index 3396e29..17a396e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,6 +29,17 @@ pub use kore_rt::{thread, thread_controller}; #[cfg(feature = "driver")] pub use kore_driver as driver; +/// A driver's side of a virtual device, a device with no hardware behind it +/// that Kore's virtual-device service hosts for a driver to be written +/// against: the register block and the views of shared memory +/// (`kore-vdev`), and the service's client (`kore-svc-vdev`, built without +/// the service or any device model). +#[cfg(feature = "vdev")] +pub mod vdev { + pub use kore_svc_vdev::{client, face, wire}; + pub use kore_vdev::{driver, mem, regs}; +} + /// Native clocks and deadlines; wall time never coerces into monotonic time. pub mod time { pub use core::time::Duration; From 6e08ec1f014335e7aab3dc24fd7be101f3ddb288 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 10:08:42 +0000 Subject: [PATCH 18/18] c: a virtual device behind a descriptor A C program launched with Kore's virtual-device profile can hold its device, in the shape Linux's UIO gives a userspace driver: - `open("/kore/device/virtual/accel")` has the service start the model and write its register block. One device is open at a time, and `close` gives it up. - `mmap` at offset 0 answers the register block; at offset one page it answers fresh, zeroed memory the driver lends the device, which cannot be unmapped while the device is open. - `read` of one 32-bit word waits for an interrupt and answers the interrupt count, each interrupt once. `write` of one word rings the doorbell. `poll` waits for the interrupt with a timeout. - `write` on descriptors 1 and 2 writes to `stdout` and `stderr`, in order with what writes. The interrupt and the doorbell are capabilities, and capabilities are per thread, so the host founds the main thread holding its own share of the face and has the thread controller hand the share on to every pthread: `read` parks the calling thread in the kernel on the interrupt count and `write` posts its notice directly. `open`, the loan and `close` are requests to the service, which thread 0's face carries; they are posted to it and waited for as a mapping of the memory supply is, raced against the program's end so a service that does not answer cannot hold an exit. `KoreVdevRegisters` and the `KORE_VDEV_*` constants in are the register block's layout, held to kore-vdev's own constants by compile-time assertions, and a test holds every constant the headers carry to a shift C reads as Rust wrote it. `kore_fd_describe` answers `KORE_FD_DEVICE`. libkore.so exports `read`, `write` and `poll` (133 symbols). Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01ACHC7gwT8vVUKqPGAjXfMM --- crates/c/Cargo.toml | 2 +- crates/c/cbindgen.toml | 9 +- crates/c/src/api.rs | 193 ++++++- crates/c/src/device.rs | 883 ++++++++++++++++++++++++++++++++ crates/c/src/fd.rs | 72 ++- crates/c/src/lib.rs | 6 +- crates/c/src/mmap.rs | 20 +- crates/c/src/posix.rs | 308 ++++++++++- crates/c/src/program.rs | 32 +- crates/c/src/stdio.rs | 18 +- crates/ffi/src/thread.rs | 9 + docs/languages.md | 51 +- extern/c/include/kore/program.h | 205 +++++++- tools/check-program.py | 10 +- 14 files changed, 1763 insertions(+), 55 deletions(-) create mode 100644 crates/c/src/device.rs diff --git a/crates/c/Cargo.toml b/crates/c/Cargo.toml index 2188bbc..ca3fa09 100644 --- a/crates/c/Cargo.toml +++ b/crates/c/Cargo.toml @@ -19,7 +19,7 @@ name = "kore_c" path = "src/lib.rs" [dependencies] -kore-std = { package = "kore-std", path = "../..", features = ["heap", "fs"] } +kore-std = { package = "kore-std", path = "../..", features = ["heap", "fs", "vdev"] } # The C ABI itself, with the standard C allocation and thread names a C # program calls (`malloc`, `pthread_create`, ...) exported as real symbols. kore-ffi = { path = "../ffi", features = ["c-stdlib"] } diff --git a/crates/c/cbindgen.toml b/crates/c/cbindgen.toml index 0c770e4..c5f2d2d 100644 --- a/crates/c/cbindgen.toml +++ b/crates/c/cbindgen.toml @@ -15,9 +15,12 @@ parse_deps = false # these), not an application API: the C library declares them itself, as # its declares `abort`. `KoreFsStep` is kore.h's typedef, which # this header includes. -# `open`, `close`, `mmap` and `munmap` are POSIX's, declared by , -# and . -exclude = ["__kore_program_start", "__kore_program_exit", "__kore_program_arguments", "abort", "open", "close", "mmap", "munmap", "KoreFsStep"] +# `open`, `close`, `read`, `write`, `poll`, `mmap` and `munmap` are POSIX's, +# declared by , , and ; so is +# `struct pollfd`. +exclude = ["__kore_program_start", "__kore_program_exit", "__kore_program_arguments", "abort", "open", "close", "read", "write", "poll", "PollFd", "mmap", "munmap", "KoreFsStep"] +# No function takes the register block: a program maps it. +include = ["KoreVdevRegisters"] [enum] prefix_with_name = true rename_variants = "ScreamingSnakeCase" diff --git a/crates/c/src/api.rs b/crates/c/src/api.rs index aa16bb3..a4d97b8 100644 --- a/crates/c/src/api.rs +++ b/crates/c/src/api.rs @@ -71,24 +71,183 @@ pub unsafe extern "C" fn kore_fs_run( /// [`KoreFdInfo::kind`]: the memory supply the program's allocator grows /// on. `mmap` answers a fresh, zeroed region of it. pub const KORE_FD_MEMORY: u32 = 1; +/// [`KoreFdInfo::kind`]: a virtual device (``). `mmap` answers +/// its register block and memory to lend it, `read` its interrupt, and +/// `write` rings its doorbell. +pub const KORE_FD_DEVICE: u32 = 2; /// [`KoreFdInfo::access`]: the capability may be read. pub const KORE_FD_READ: u32 = 1 << 0; /// [`KoreFdInfo::access`]: the capability may be written. pub const KORE_FD_WRITE: u32 = 1 << 1; +/// Bytes in a virtual device's register block: one page. +pub const KORE_VDEV_REGISTERS_BYTES: usize = 4096; +/// `mmap` offset on a virtual device's descriptor: the register block +/// ([`KoreVdevRegisters`]). +pub const KORE_VDEV_MAP_REGISTERS: i64 = 0; +/// `mmap` offset on a virtual device's descriptor: fresh, zeroed memory the +/// driver owns and lends the device. A device address is an offset into it. +/// The device takes a loan only in its reset state, so map it before the +/// first status write. +pub const KORE_VDEV_MAP_MEMORY: i64 = 4096; +/// [`KoreVdevRegisters::magic`]: `"kvdv"`, a Kore virtual device. +pub const KORE_VDEV_MAGIC: u32 = 0x7664_766b; +/// [`KoreVdevRegisters::version`]: this layout. +pub const KORE_VDEV_VERSION: u32 = 1; +/// [`KoreVdevRegisters::vendor_id`]: `"kore"`, a device Kore hosts. +pub const KORE_VDEV_VENDOR_KORE: u32 = 0x6572_6f6b; +/// [`KoreVdevRegisters::device_id`]: `"accl"`, the virtual accelerator, a +/// virtio-accel 1.0 device. Its name is `/kore/device/virtual/accel`. +pub const KORE_VDEV_ID_ACCEL: u32 = 0x6c63_6361; +/// Status: the driver has noticed the device (Virtio 1.3 section 2.1). +pub const KORE_VDEV_STATUS_ACKNOWLEDGE: u32 = 1; +/// Status: the driver knows how to drive it. +pub const KORE_VDEV_STATUS_DRIVER: u32 = 2; +/// Status: the driver is set up and the device may serve. +pub const KORE_VDEV_STATUS_DRIVER_OK: u32 = 4; +/// Status: feature negotiation is complete. +pub const KORE_VDEV_STATUS_FEATURES_OK: u32 = 8; +/// Status, written by the device: it cannot go on and must be reset. +pub const KORE_VDEV_STATUS_DEVICE_NEEDS_RESET: u32 = 64; +/// Status: the driver has given up on the device. +pub const KORE_VDEV_STATUS_FAILED: u32 = 128; +/// Feature bit 32, `VIRTIO_F_VERSION_1`: every device offers it and a +/// driver must accept it. Written out, because the header carries this +/// expression as it stands and `1 << 32` is not a 64-bit value in C. +pub const KORE_VDEV_F_VERSION_1: u64 = 0x1_0000_0000; +/// Bytes between the last register and the device configuration. +pub const KORE_VDEV_RESERVED_BYTES: usize = 168; +/// Bytes of the device configuration area, to the end of the block. +pub const KORE_VDEV_CONFIG_AREA_BYTES: usize = 3840; + +/// The register block of a Kore virtual device: what `mmap` at +/// `KORE_VDEV_MAP_REGISTERS` answers, declared `volatile` by a driver. +/// +/// Every word is little-endian and has one writer. A driver never writes a +/// device register, and what it writes to its own the device reads once, at +/// the step that uses it, when the driver rings the doorbell (a `write` of +/// one 32-bit word to the descriptor). The device answers by raising an +/// interrupt: it increments `interrupts`, and a `read` of one 32-bit word +/// from the descriptor returns once the count has moved. +/// +/// Bringing the device up is Virtio 1.3 section 3.1.1 over these registers, +/// with the memory lent first (`KORE_VDEV_MAP_MEMORY`). A device just opened +/// is in its reset state, both status words zero; one that is not is reset +/// first, by writing 0 to `driver_status` and ringing. Then write +/// `ACKNOWLEDGE | DRIVER` and ring; write the accepted features, add +/// `FEATURES_OK`, ring, and check `device_status` carries it; lay the queue +/// out in the lent memory, write `queue_num`, the three addresses and +/// `queue_ready`; add `DRIVER_OK` and ring. A new value of `driver_status` +/// is always answered with an interrupt, and a step the device refused is +/// one its `device_status` does not carry. The value the device last saw +/// announces nothing and is not answered, so a driver waits only for what +/// it asked. +#[repr(C)] +pub struct KoreVdevRegisters { + /// Device: `KORE_VDEV_MAGIC`. + pub magic: u32, + /// Device: `KORE_VDEV_VERSION`. + pub version: u32, + /// Device: which model this is (`KORE_VDEV_ID_ACCEL`). + pub device_id: u32, + /// Device: `KORE_VDEV_VENDOR_KORE`. + pub vendor_id: u32, + /// Device: the features it offers. + pub device_features: u64, + /// Driver: the features it accepts, a subset of the offer. + pub driver_features: u64, + /// Driver: the status it has reached (`KORE_VDEV_STATUS_*`). + pub driver_status: u32, + /// Device: the status it has accepted, and its own + /// `KORE_VDEV_STATUS_DEVICE_NEEDS_RESET`. + pub device_status: u32, + /// Device: the largest queue it takes. + pub queue_num_max: u32, + /// Driver: the queue's size, a power of two between the longest chain + /// the device takes (its configuration says how long) and + /// `queue_num_max`. + pub queue_num: u32, + /// Driver: nonzero once the queue's size and addresses are written. + pub queue_ready: u32, + /// Device: changes whenever the configuration area does. + pub config_generation: u32, + /// Driver: the descriptor table's device address. + pub queue_desc: u64, + /// Driver: the available ring's device address. + pub queue_driver: u64, + /// Driver: the used ring's device address. + pub queue_device: u64, + /// Device: bytes of device configuration in `config`. + pub config_bytes: u32, + /// Device: interrupts raised, wrapping. + pub interrupts: u32, + /// Unused. + pub reserved: [u8; KORE_VDEV_RESERVED_BYTES], + /// Device: the device configuration. + pub config: [u8; KORE_VDEV_CONFIG_AREA_BYTES], +} + +// The declaration is `kore-vdev`'s register block, offset for offset, and +// the constants are its constants: a drift fails this build. +const _: () = { + use core::mem::{offset_of, size_of}; + use kore_std::vdev::regs; + macro_rules! at { + ($($field:ident = $offset:expr),* $(,)?) => { + $(assert!(offset_of!(KoreVdevRegisters, $field) as u64 == $offset);)* + }; + } + at! { + magic = regs::MAGIC, + version = regs::VERSION, + device_id = regs::DEVICE_ID, + vendor_id = regs::VENDOR_ID, + device_features = regs::DEVICE_FEATURES, + driver_features = regs::DRIVER_FEATURES, + driver_status = regs::DRIVER_STATUS, + device_status = regs::DEVICE_STATUS, + queue_num_max = regs::QUEUE_NUM_MAX, + queue_num = regs::QUEUE_NUM, + queue_ready = regs::QUEUE_READY, + config_generation = regs::CONFIG_GENERATION, + queue_desc = regs::QUEUE_DESC, + queue_driver = regs::QUEUE_DRIVER, + queue_device = regs::QUEUE_DEVICE, + config_bytes = regs::CONFIG_BYTES, + interrupts = regs::INTERRUPTS, + config = regs::CONFIG, + } + assert!(size_of::() as u64 == regs::BYTES); + assert!(KORE_VDEV_REGISTERS_BYTES as u64 == regs::BYTES); + assert!(KORE_VDEV_MAP_REGISTERS == crate::device::MAP_REGISTERS); + assert!(KORE_VDEV_MAP_MEMORY == crate::device::MAP_MEMORY); + assert!(KORE_VDEV_MAGIC == regs::MAGIC_VALUE); + assert!(KORE_VDEV_VERSION == regs::VERSION_VALUE); + assert!(KORE_VDEV_VENDOR_KORE == regs::VENDOR_KORE); + assert!(KORE_VDEV_ID_ACCEL == crate::device::MODEL_ACCEL); + assert!(KORE_VDEV_STATUS_ACKNOWLEDGE == regs::STATUS_ACKNOWLEDGE); + assert!(KORE_VDEV_STATUS_DRIVER == regs::STATUS_DRIVER); + assert!(KORE_VDEV_STATUS_DRIVER_OK == regs::STATUS_DRIVER_OK); + assert!(KORE_VDEV_STATUS_FEATURES_OK == regs::STATUS_FEATURES_OK); + assert!(KORE_VDEV_STATUS_DEVICE_NEEDS_RESET == regs::STATUS_DEVICE_NEEDS_RESET); + assert!(KORE_VDEV_STATUS_FAILED == regs::STATUS_FAILED); + assert!(KORE_VDEV_F_VERSION_1 == regs::F_VERSION_1); +}; + /// What a descriptor names, as the calling thread holds it /// ([`kore_fd_describe`]). #[repr(C)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct KoreFdInfo { - /// What kind of capability it is: `KORE_FD_MEMORY`. + /// What kind of capability it is: `KORE_FD_MEMORY` or `KORE_FD_DEVICE`. pub kind: u32, /// What the capability's own rights permit this thread, as /// `KORE_FD_READ | KORE_FD_WRITE` bits: the rights it was founded with, /// which the access `open` asked for can only narrow. pub access: u32, /// For `KORE_FD_MEMORY`, the supply's whole length in bytes: what the - /// allocator and every mapping share, not what is still free. + /// allocator and every mapping share, not what is still free. For + /// `KORE_FD_DEVICE`, the register block's length. pub length: u64, } @@ -137,6 +296,7 @@ fn describe(name: &crate::fd::Name) -> Option { KORE_FD_MEMORY, kore_std::runtime::cap::verbs::lease_geometry(held.slot)?.1, ), + Kind::Device { .. } => (KORE_FD_DEVICE, kore_std::vdev::regs::BYTES), }; Some(KoreFdInfo { kind, @@ -164,6 +324,35 @@ mod tests { panic!("a session ran with no program to host"); } + /// The headers carry a constant's expression as Rust wrote it, and C + /// reads `1 << n` as an `int`: a shift that does not fit one is not the + /// value Rust meant. `KORE_VDEV_F_VERSION_1` was `(1 << 32)` once, which + /// a driver compiled as zero. + #[test] + fn no_header_constant_shifts_past_an_int() { + for (name, header) in [ + ( + "kore/program.h", + include_str!("../../../extern/c/include/kore/program.h"), + ), + ("kore.h", include_str!("../../../extern/c/include/kore.h")), + ] { + for line in header.lines().filter(|line| line.starts_with("#define ")) { + let mut rest = line; + while let Some(at) = rest.find("<<") { + rest = &rest[at + 2..]; + let digits: std::string::String = rest + .trim_start() + .chars() + .take_while(char::is_ascii_digit) + .collect(); + let shift: u32 = digits.parse().unwrap_or(u32::MAX); + assert!(shift < 31, "{name}: `{line}` shifts an int by {digits}"); + } + } + } + } + #[test] fn a_null_step_is_an_invalid_argument_and_the_error_is_written() { let mut error = KoreFsError::Busy; diff --git a/crates/c/src/device.rs b/crates/c/src/device.rs new file mode 100644 index 0000000..4de9b3d --- /dev/null +++ b/crates/c/src/device.rs @@ -0,0 +1,883 @@ +//! A virtual device behind a descriptor. +//! +//! A program launched with Kore's virtual-device profile has a face with the +//! virtual-device service: a device with no hardware behind it, for a driver +//! to be written against (Kore's `res/VIRTUAL_DEVICE.md`). This module is +//! C's spelling for holding one, in the shape Linux's UIO gives a userspace +//! driver: +//! +//! - **`open`** a name of the device model. The service starts the model and +//! writes its register block. One descriptor at a time: the face is one +//! device's. +//! - **`mmap` at offset 0** answers the register block, one page the program +//! and the device both write. +//! - **`mmap` at offset one page** answers fresh, zeroed memory the driver +//! owns and lends the device: a device address is an offset into it. One +//! such region at a time, and it cannot be unmapped while the device is +//! open, because the device may be using it. +//! - **`read`** of one 32-bit word waits for an interrupt and answers the +//! interrupt count. Each interrupt is read once. +//! - **`write`** of one 32-bit word rings the doorbell. The word is not read. +//! - **`poll`** says whether an interrupt is waiting to be read, with a +//! timeout. +//! - **`close`** gives the device up. The service drops the model and its +//! view of the lent memory, which stays the program's until it is unmapped. +//! +//! # Whose capability does what +//! +//! Capabilities are per thread. Thread 0 was launched holding the face, so +//! everything that is a request to the service (open, lend, close) is posted +//! to it and waited for, as a mapping of the memory supply is. The interrupt +//! and the doorbell are the hot path and never cross a thread: this library +//! founds the program's threads holding their own share of the face +//! ([`shares`]), so `read` parks the calling thread in the kernel on the +//! interrupt count and `write` posts its notice directly. A thread founded +//! without a share is refused with `ENXIO`, whoever opened the descriptor. +//! +//! The request rules, the refusals and the state a descriptor's words keep +//! are plain data, so the host suite exercises the code the target runs; the +//! face is the metal half. + +use core::ffi::c_int; +use core::sync::atomic::{AtomicU32, Ordering}; + +use kore_std::vdev::client::process_face; +use kore_std::vdev::wire::VdevRefusal; + +use crate::fd::errno; +use crate::mmap::{MAP_FIXED, MAP_SHARED, PAGE, PROT_READ, PROT_WRITE, pages}; + +/// The face's instance in `role::SVC_RING`: the coordinate a device name +/// stands for. +pub(crate) const FACE: u16 = process_face::CONTROL_RING; +/// The face's kick in `role::SVC_SEND`: what a thread rings the doorbell on. +pub(crate) const KICK: u16 = process_face::REQUEST_KICK; + +/// The virtual accelerator's register-block id, `"accl"`. +pub(crate) const MODEL_ACCEL: u32 = u32::from_le_bytes(*b"accl"); + +/// `mmap` offsets on a device descriptor, UIO's convention: map `N` is at +/// offset `N` pages. +pub(crate) const MAP_REGISTERS: i64 = 0; +pub(crate) const MAP_MEMORY: i64 = PAGE as i64; + +/// What `read` answers and `write` takes: one 32-bit word. +pub(crate) const WORD: usize = core::mem::size_of::(); + +/// What an `mmap` on a device descriptor asks for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Map { + /// The register block. + Registers, + /// Memory to lend the device: `asked` whole pages' bytes. + Memory { asked: usize }, +} + +impl Map { + /// What `mmap`'s arguments ask of a device, or the `errno` that refuses + /// them. + /// + /// `EINVAL`: no length, an offset that is neither map, a register + /// mapping longer than the block, or anything but `MAP_SHARED` (the + /// device sees every store, so there is no private copy to make). + /// `ENOTSUP`: `MAP_FIXED`, and any protection but read or read-write. + /// `ENOMEM`: a length that does not round to pages. + pub(crate) fn of(len: usize, prot: c_int, flags: c_int, offset: i64) -> Result { + if len == 0 { + return Err(errno::EINVAL); + } + if flags & MAP_FIXED != 0 { + return Err(errno::ENOTSUP); + } + if flags != MAP_SHARED { + return Err(errno::EINVAL); + } + if prot != PROT_READ && prot != PROT_WRITE && prot != PROT_READ | PROT_WRITE { + return Err(errno::ENOTSUP); + } + match offset { + MAP_REGISTERS if len <= PAGE => Ok(Self::Registers), + MAP_MEMORY => pages(len) + .map(|asked| Self::Memory { asked }) + .ok_or(errno::ENOMEM), + _ => Err(errno::EINVAL), + } + } + + /// Whether the mapping needs the descriptor opened for writing: any + /// writable one, and lent memory always, since lending is an act and the + /// driver's queue lives there. + pub(crate) fn writes(self, prot: c_int) -> bool { + matches!(self, Self::Memory { .. }) || prot & PROT_WRITE != 0 + } +} + +/// The `errno` for a request the service refused. +pub(crate) fn refused(refusal: VdevRefusal) -> c_int { + match refusal { + // No model has that id. + VdevRefusal::NoDevice => errno::ENXIO, + // The device is not in a state for this: already open, or holding a + // queue in the memory it has. + VdevRefusal::Busy => errno::EBUSY, + // Every device the service hosts at once is taken: ask again when + // another driver closes. + VdevRefusal::Limit => errno::EAGAIN, + // What was lent is not memory a device can address. + VdevRefusal::Memory => errno::EINVAL, + // This face may not ask that. + VdevRefusal::Denied => errno::EACCES, + // The service did not answer, or answered out of turn. + VdevRefusal::Unavailable | VdevRefusal::Pending | VdevRefusal::Conflict => errno::EIO, + } +} + +/// The open device's words: which opening this is, and the interrupt count +/// last read. +/// +/// The generation is odd while a device is open. A blocked `read` belongs to +/// the opening it began under, and ends with `EBADF` when that opening does. +pub(crate) struct Opening { + generation: AtomicU32, + seen: AtomicU32, +} + +impl Opening { + pub(crate) const fn new() -> Self { + Self { + generation: AtomicU32::new(0), + seen: AtomicU32::new(0), + } + } + + /// The opening now, if a device is open. + pub(crate) fn current(&self) -> Option { + let generation = self.generation.load(Ordering::Acquire); + (generation & 1 != 0).then_some(generation) + } + + /// Whether `generation` is still the opening. + pub(crate) fn is(&self, generation: u32) -> bool { + self.generation.load(Ordering::Acquire) == generation + } + + /// A device opened with its interrupt count at `count`: interrupts + /// before now are not this opening's to read. `false` when one is open. + pub(crate) fn begin(&self, count: u32) -> bool { + let generation = self.generation.load(Ordering::Acquire); + if generation & 1 != 0 { + return false; + } + self.seen.store(count, Ordering::Release); + self.generation + .store(generation.wrapping_add(1), Ordering::Release); + true + } + + /// The device closed. `false` when none was open. + pub(crate) fn end(&self) -> bool { + let generation = self.generation.load(Ordering::Acquire); + if generation & 1 == 0 { + return false; + } + self.generation + .store(generation.wrapping_add(1), Ordering::Release); + true + } + + /// The interrupt count last read. + pub(crate) fn seen(&self) -> u32 { + self.seen.load(Ordering::Acquire) + } + + /// Take the interrupt that moved the count from `seen` to `count`. + /// `false` when another reader took it first. + pub(crate) fn take(&self, seen: u32, count: u32) -> bool { + self.seen + .compare_exchange(seen, count, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } +} + +#[cfg(target_os = "none")] +pub(crate) use metal::{ + Host, close, holds_kick, map, open, posted, read, readable, ring, shares, unmap, wait_readable, +}; + +/// Off Kore no thread holds a face. +#[cfg(not(target_os = "none"))] +pub(crate) fn holds_kick() -> bool { + false +} + +#[cfg(target_os = "none")] +mod metal { + extern crate alloc; + + use core::cell::UnsafeCell; + use core::ffi::c_int; + use core::ptr::NonNull; + use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; + use core::time::Duration; + + use kore_std::abi::cap::{ResourceKind, Rights}; + use kore_std::abi::task::role; + use kore_std::runtime::cap::{MappedLeaseView, RetireOutcome, StashedLease, verbs}; + use kore_std::runtime::entry::{self, Rt}; + use kore_std::runtime::futex::{self, WaitOutcome}; + use kore_std::runtime::pace; + use kore_std::runtime::thread::{self, WorkerCap}; + use kore_std::runtime::transport::TrapTransport; + use kore_std::vdev::client::{ + ProcessVdev, THREAD_SHARES, ThreadVdev, Waited, attach_process_face, attach_thread_face, + }; + use kore_std::vdev::mem::{Memory as _, Window}; + use kore_std::vdev::regs; + use kore_std::vdev::wire::VdevResponse; + + use super::{FACE, KICK, Map, Opening, PAGE, refused}; + use crate::fd::errno; + use crate::lock::Lock; + + /// The device this program has open, if it has one. + static OPENING: Opening = Opening::new(); + + /// The register block's address in this process, once thread 0 has + /// attached the face; zero before. The mapping is thread 0's and lasts + /// the program. + static REGISTERS: AtomicUsize = AtomicUsize::new(0); + + /// The lent memory's address while the program holds a mapping of it; + /// zero otherwise. + static MEMORY: AtomicUsize = AtomicUsize::new(0); + + /// The register block, to read the interrupt count through. + fn registers() -> Option { + let base = NonNull::new(REGISTERS.load(Ordering::Acquire) as *mut u8)?; + // SAFETY: thread 0 published the address of the face's second page, + // which it keeps mapped read-write for the life of the process. The + // device writes it concurrently, which is what a `Window` is for. + Some(unsafe { Window::new(base, regs::BYTES as usize) }) + } + + /// The calling thread's own part of the face. + fn face() -> Option { + attach_thread_face(&entry::thread_context()?) + } + + /// Whether the calling thread holds the face's kick with the right a + /// notice needs. + pub(crate) fn holds_kick() -> bool { + entry::thread_context() + .and_then(|rt| rt.bootstrap().handle(role::SVC_SEND, KICK)) + .and_then(|kick| verbs::capability_rights(kick, ResourceKind::ChanSend)) + .is_some_and(|rights| rights.contains(verbs::NOTICE_RIGHTS)) + } + + /// The endowment that founds a thread holding its share of the face: + /// what thread 0 gives the main thread, with `DERIVE` so the main thread + /// can found the thread controller holding it too. Empty for a launch + /// without a face. + pub(crate) fn shares() -> impl Iterator { + THREAD_SHARES.iter().filter_map(|share| { + thread::share_endowment( + share.role, + share.index, + share.kind, + share.rights | Rights::DERIVE, + ) + }) + } + + // The table's row names the same coordinate the shares do. + const _: () = assert!(THREAD_SHARES[0].index == FACE && THREAD_SHARES[1].index == KICK); + + // ── The threads' half ─────────────────────────────────────────────── + + /// `read`: wait for an interrupt of this opening and take it. + /// + /// `EBADF`: no device is open, or it closed while this thread waited. + /// `ENXIO`: the calling thread holds no share of the face. `EIO`: the + /// thread cannot wait on the device (its share is gone, or the kernel + /// will not park it). + pub(crate) fn read() -> Result { + let generation = OPENING.current().ok_or(errno::EBADF)?; + let registers = registers().ok_or(errno::EBADF)?; + let face = face().ok_or(errno::ENXIO)?; + loop { + let seen = OPENING.seen(); + match face.wait(registers, seen, None, || OPENING.is(generation)) { + // Another reader may have taken this interrupt: then wait + // for the next. + Waited::Interrupt(count) => { + if OPENING.take(seen, count) { + return Ok(count); + } + } + Waited::Stopped => return Err(errno::EBADF), + Waited::Deadline | Waited::Gone => return Err(errno::EIO), + } + } + } + + /// `write`: ring the doorbell. A notice the service's doorbell had no + /// room for is not an error: it has a wake standing and looks anyway. + /// + /// `EBADF`: no device is open. `ENXIO`: the calling thread holds no + /// share of the face. + pub(crate) fn ring() -> Result<(), c_int> { + OPENING.current().ok_or(errno::EBADF)?; + let _ = face().ok_or(errno::ENXIO)?.doorbell(); + Ok(()) + } + + /// Whether an interrupt is waiting to be read. `None` when no device is + /// open. + pub(crate) fn readable() -> Option { + OPENING.current()?; + let count = registers()?.load_u32(regs::INTERRUPTS).ok()?; + Some(count != OPENING.seen()) + } + + /// `poll`'s wait: until an interrupt is waiting to be read or + /// `deadline_ns` passes. `Ok(true)`: one is waiting. `Ok(false)`: the + /// deadline passed. `Err`: as [`read`], and the caller reports it on the + /// descriptor. + pub(crate) fn wait_readable(deadline_ns: Option) -> Result { + let generation = OPENING.current().ok_or(errno::EBADF)?; + let registers = registers().ok_or(errno::EBADF)?; + let face = face().ok_or(errno::ENXIO)?; + match face.wait(registers, OPENING.seen(), deadline_ns, || { + OPENING.is(generation) + }) { + Waited::Interrupt(_) => Ok(true), + Waited::Deadline => Ok(false), + Waited::Stopped => Err(errno::EBADF), + Waited::Gone => Err(errno::EIO), + } + } + + /// `open`: have the service start the model with this id. + /// + /// `EBUSY`: this program has a device open. `ENXIO`: the launch carried + /// no face, the program has no host to ask, or no model has that id. + /// `ENOTSUP`: asked from thread 0 itself, which cannot wait for its own + /// answer. Otherwise as the service refuses ([`refused`]). + pub(crate) fn open(model: u32) -> Result<(), c_int> { + ask(Op::Open { model }).map(|_| ()) + } + + /// `close`: give the device up. Nothing is reported: a device that could + /// not be closed ends with the program. + pub(crate) fn close() { + let _ = ask(Op::Close); + } + + /// `mmap` on the open device. + /// + /// `EBADF`: no device is open. `EBUSY`: the program already holds a + /// mapping of device memory. `ENOMEM`: the supply or the link window has + /// no room for the region, or the kernel refused the map. Otherwise as + /// the service refuses the loan ([`refused`]). + pub(crate) fn map(map: Map) -> Result<*mut u8, c_int> { + OPENING.current().ok_or(errno::EBADF)?; + match map { + Map::Registers => NonNull::new(REGISTERS.load(Ordering::Acquire) as *mut u8) + .map(NonNull::as_ptr) + .ok_or(errno::EBADF), + Map::Memory { asked } => ask(Op::Lend { asked }).map(|base| base as *mut u8), + } + } + + /// `munmap` of a device mapping. `None` when `base` is not one, and the + /// memory supply's rules apply. + /// + /// The register block is the process's for its life: unmapping it is + /// accepted and changes nothing. `EBUSY`: the lent memory, while the + /// device is open. `EINVAL`: `len` is not the mapping's length. + /// `EAGAIN`: the kernel did not confirm the unmap. + pub(crate) fn unmap(base: usize, len: usize) -> Option> { + if base == 0 { + return None; + } + if base == REGISTERS.load(Ordering::Acquire) { + return Some(if len == 0 || len > PAGE { + Err(errno::EINVAL) + } else { + Ok(()) + }); + } + if base != MEMORY.load(Ordering::Acquire) { + return None; + } + Some(ask(Op::Release { base, len }).map(|_| ())) + } + + // ── Reaching thread 0 ─────────────────────────────────────────────── + + /// One request of thread 0. + #[derive(Clone, Copy)] + enum Op { + Open { model: u32 }, + Close, + Lend { asked: usize }, + Release { base: usize, len: usize }, + } + + // The cell's states. A caller owns it at IDLE (under [`CALLERS`]) and at + // SERVED, thread 0 at POSTED. + const IDLE: u32 = 0; + const POSTED: u32 = 1; + const SERVED: u32 = 2; + + /// One request at a time: the caller holding [`CALLERS`] writes `op` and + /// posts, thread 0 writes `answer` and serves. + struct Call { + state: AtomicU32, + op: UnsafeCell, + answer: UnsafeCell>, + } + // SAFETY: `op` and `answer` are touched only by the side `state` hands + // the cell to. + unsafe impl Sync for Call {} + static CALL: Call = Call { + state: AtomicU32::new(IDLE), + op: UnsafeCell::new(Op::Close), + answer: UnsafeCell::new(Ok(0)), + }; + + /// Callers take turns at the cell. + static CALLERS: Lock = Lock::new(); + + /// How long a caller waits before it re-reads the cell: thread 0 kicks + /// the main thread when it has answered, and holds no send into a + /// pthread's doorbell, which rides this slice (the mapping cell's rule). + const SLICE: Duration = Duration::from_nanos(pace::CEDE_NS); + + /// Whether a request waits for thread 0. + pub(crate) fn posted() -> bool { + CALL.state.load(Ordering::Acquire) == POSTED + } + + /// Have thread 0 carry `op` out, and wait for its answer. + fn ask(op: Op) -> Result { + if crate::program::on_host() { + // Thread 0 answers these between its own turns: it cannot wait + // here for an answer only it can give. + return Err(errno::ENOTSUP); + } + if !crate::program::hosted() { + return Err(errno::ENXIO); + } + CALLERS.acquire(); + // SAFETY: IDLE under the lock hands the cell to this caller. + unsafe { *CALL.op.get() = op }; + CALL.state.store(POSTED, Ordering::SeqCst); + // Refused only when no post stood, and then thread 0 re-reads the + // cell before it parks. + let _ = thread::kick_parent(); + loop { + match CALL.state.load(Ordering::Acquire) { + SERVED => break, + state => { + if futex::wait(&CALL.state, state, Some(SLICE)) == WaitOutcome::Unsupported { + // No doorbell to park on: the deadline park any + // thread can. + let _ = pace::cede(); + } + } + } + } + // SAFETY: SERVED hands the cell back to this caller. + let answer = unsafe { *CALL.answer.get() }; + CALL.state.store(IDLE, Ordering::Release); + CALLERS.release(); + answer + } + + // ── Thread 0's half ───────────────────────────────────────────────── + + /// The memory lent to the device, or kept from the last loan. + enum Lent { + /// Nothing carved. + Nothing, + /// Unmapped. Its lease and address serve the next loan of its + /// length: the supply's cursors and the window's only advance. + Spare { + lease: StashedLease, + base: usize, + len: usize, + }, + /// Mapped in this process. `lent` while the device has it. + Mapped { + lease: StashedLease, + view: MappedLeaseView, + base: usize, + len: usize, + lent: bool, + }, + } + + /// Thread 0's side of the program's device: the face, attached on first + /// use, and the memory lent through it. + pub(crate) struct Host<'r, 't> { + rt: &'r Rt, + transport: &'t TrapTransport, + face: Option>, + /// The launch carried no face: every later request is refused the + /// same way without asking again. + absent: bool, + memory: Lent, + } + + impl<'r, 't> Host<'r, 't> { + pub(crate) fn new(rt: &'r Rt, transport: &'t TrapTransport) -> Self { + Self { + rt, + transport, + face: None, + absent: false, + memory: Lent::Nothing, + } + } + + /// Answer the posted request, if there is one. `true` when one was + /// answered. Dropped mid-request when the program ends first; the + /// caller it leaves waiting is stopped with the rest of the program. + pub(crate) async fn serve(&mut self) -> bool { + if !posted() { + return false; + } + // SAFETY: POSTED hands the cell to thread 0 until it stores + // SERVED. + let op = unsafe { *CALL.op.get() }; + let answer = self.perform(op).await; + // SAFETY: as above. + unsafe { *CALL.answer.get() = answer }; + // SeqCst, as the futex contract asks of the store before a wake. + CALL.state.store(SERVED, Ordering::SeqCst); + futex::wake(&CALL.state); + true + } + + fn face(&mut self) -> Result<&mut ProcessVdev<'t>, c_int> { + if self.absent { + return Err(errno::ENXIO); + } + if self.face.is_none() { + match attach_process_face(self.rt, self.transport) { + Some(face) => { + REGISTERS + .store(face.registers().base().as_ptr() as usize, Ordering::Release); + self.face = Some(face); + } + None => { + self.absent = true; + return Err(errno::ENXIO); + } + } + } + self.face.as_mut().ok_or(errno::ENXIO) + } + + async fn perform(&mut self, op: Op) -> Result { + match op { + Op::Open { model } => self.open(model).await, + Op::Close => self.close().await, + Op::Lend { asked } => self.lend(asked).await, + Op::Release { base, len } => self.release(base, len), + } + } + + async fn open(&mut self, model: u32) -> Result { + if OPENING.current().is_some() { + return Err(errno::EBUSY); + } + let face = self.face()?; + match face.open(model).await { + VdevResponse::Accepted => { + // Interrupts from before this opening are not its to + // read. + OPENING.begin(face.interrupts()); + Ok(0) + } + VdevResponse::Refused(refusal) => Err(refused(refusal)), + _ => Err(errno::EIO), + } + } + + async fn close(&mut self) -> Result { + if !OPENING.end() { + return Ok(0); + } + let face = self.face()?; + // The service drops the model and its view of what was lent. + // Only its word for that frees the memory to be unmapped: a + // close it did not answer leaves the loan standing, and the + // face ends with the program. + let closed = matches!(face.close().await, VdevResponse::Accepted); + if closed && let Lent::Mapped { lent, .. } = &mut self.memory { + *lent = false; + } + Ok(0) + } + + async fn lend(&mut self, asked: usize) -> Result { + if OPENING.current().is_none() { + return Err(errno::EBADF); + } + // A mapping the program was never given (a loan the service + // refused, whose unmap the kernel did not confirm) is retired + // before anything else is tried. + if MEMORY.load(Ordering::Acquire) == 0 { + self.retire_unpublished(); + } + let (lease, base, len) = match core::mem::replace(&mut self.memory, Lent::Nothing) { + Lent::Spare { lease, base, len } if len == asked => (lease, base, len), + mapped @ Lent::Mapped { .. } => { + self.memory = mapped; + return Err(errno::EBUSY); + } + // A spare of another length is given up for good, and + // nothing carved is nothing to reuse. + Lent::Spare { .. } | Lent::Nothing => { + let lease = kore_std::runtime::heap::reserve_from_top(asked as u64) + .ok_or(errno::ENOMEM)?; + let len = lease + .slot() + .and_then(verbs::lease_geometry) + .and_then(|(_, len)| usize::try_from(len).ok()) + .ok_or(errno::ENOMEM)?; + let base = crate::mmap::place(len).ok_or(errno::ENOMEM)?; + (lease, base, len) + } + }; + let Some(view) = crate::mmap::map_zeroed(&lease, base, len, true) else { + self.memory = Lent::Spare { lease, base, len }; + return Err(errno::ENOMEM); + }; + let face = match self.face() { + Ok(face) => face, + Err(code) => { + self.memory = Lent::Mapped { + lease, + view, + base, + len, + lent: false, + }; + self.retire_unpublished(); + return Err(code); + } + }; + let answer = face.lend(&lease, len as u64).await; + let lent = matches!( + answer, + VdevResponse::Lent { pages } if pages as usize == len / PAGE + ); + self.memory = Lent::Mapped { + lease, + view, + base, + len, + lent, + }; + if lent { + MEMORY.store(base, Ordering::Release); + return Ok(base); + } + self.retire_unpublished(); + Err(match answer { + VdevResponse::Refused(refusal) => refused(refusal), + _ => errno::EIO, + }) + } + + /// Unmap a mapping the program does not hold and the device does not + /// have. One the kernel does not confirm stays as it is, to be tried + /// again. + fn retire_unpublished(&mut self) { + self.memory = match core::mem::replace(&mut self.memory, Lent::Nothing) { + Lent::Mapped { + lease, + mut view, + base, + len, + lent: false, + } => match view.try_retire_outcome() { + RetireOutcome::Retired | RetireOutcome::AlreadyDead => { + Lent::Spare { lease, base, len } + } + RetireOutcome::Retryable => Lent::Mapped { + lease, + view, + base, + len, + lent: false, + }, + }, + other => other, + }; + } + + fn release(&mut self, at: usize, asked: usize) -> Result { + match &self.memory { + Lent::Mapped { base, .. } if *base != at => return Err(errno::EINVAL), + Lent::Mapped { lent: true, .. } => return Err(errno::EBUSY), + Lent::Mapped { len, .. } => { + if crate::mmap::pages(asked) != Some(*len) { + return Err(errno::EINVAL); + } + } + Lent::Spare { .. } | Lent::Nothing => return Err(errno::EINVAL), + } + // The program's mapping ends here whether or not the kernel + // confirms it at once: the address is no longer its to use. + MEMORY.store(0, Ordering::Release); + self.retire_unpublished(); + match self.memory { + Lent::Mapped { base, .. } => { + // Still mapped: the program may ask again. + MEMORY.store(base, Ordering::Release); + Err(errno::EAGAIN) + } + _ => Ok(0), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const RW: c_int = PROT_READ | PROT_WRITE; + const MAP_PRIVATE: c_int = crate::mmap::MAP_PRIVATE; + + #[test] + fn offset_zero_is_the_register_block_and_one_page_is_memory() { + assert_eq!(Map::of(PAGE, RW, MAP_SHARED, 0), Ok(Map::Registers)); + // A shorter length is still the one page. + assert_eq!(Map::of(0x58, PROT_READ, MAP_SHARED, 0), Ok(Map::Registers)); + assert_eq!( + Map::of(1 << 20, RW, MAP_SHARED, MAP_MEMORY), + Ok(Map::Memory { asked: 1 << 20 }) + ); + // Lent memory rounds up to whole pages, as any mapping does. + assert_eq!( + Map::of(PAGE + 1, RW, MAP_SHARED, MAP_MEMORY), + Ok(Map::Memory { asked: 2 * PAGE }) + ); + } + + #[test] + fn a_device_mapping_is_refused_by_cause() { + assert_eq!(Map::of(0, RW, MAP_SHARED, 0), Err(errno::EINVAL)); + // The block is one page. + assert_eq!(Map::of(PAGE + 1, RW, MAP_SHARED, 0), Err(errno::EINVAL)); + // No third map, and no offset inside one. + for offset in [-1, 1, 2 * PAGE as i64, MAP_MEMORY + 1] { + assert_eq!( + Map::of(PAGE, RW, MAP_SHARED, offset), + Err(errno::EINVAL), + "{offset}" + ); + } + // The device sees every store: there is no private copy. + assert_eq!(Map::of(PAGE, RW, MAP_PRIVATE, 0), Err(errno::EINVAL)); + assert_eq!( + Map::of(PAGE, RW, MAP_SHARED | MAP_PRIVATE, 0), + Err(errno::EINVAL) + ); + assert_eq!( + Map::of(PAGE, RW, MAP_SHARED | MAP_FIXED, 0), + Err(errno::ENOTSUP) + ); + // PROT_NONE and PROT_EXEC. + for prot in [0, 4, RW | 4] { + assert_eq!( + Map::of(PAGE, prot, MAP_SHARED, 0), + Err(errno::ENOTSUP), + "{prot}" + ); + } + assert_eq!( + Map::of(usize::MAX, RW, MAP_SHARED, MAP_MEMORY), + Err(errno::ENOMEM) + ); + } + + #[test] + fn lent_memory_and_a_writable_block_need_a_writable_descriptor() { + assert!(!Map::Registers.writes(PROT_READ)); + assert!(Map::Registers.writes(RW)); + assert!(Map::Registers.writes(PROT_WRITE)); + // Lending is an act, whatever protection the mapping asks. + assert!(Map::Memory { asked: PAGE }.writes(PROT_READ)); + assert!(Map::Memory { asked: PAGE }.writes(RW)); + } + + #[test] + fn every_refusal_has_an_errno() { + assert_eq!(refused(VdevRefusal::NoDevice), errno::ENXIO); + assert_eq!(refused(VdevRefusal::Busy), errno::EBUSY); + assert_eq!(refused(VdevRefusal::Limit), errno::EAGAIN); + assert_eq!(refused(VdevRefusal::Memory), errno::EINVAL); + assert_eq!(refused(VdevRefusal::Denied), errno::EACCES); + for refusal in [ + VdevRefusal::Unavailable, + VdevRefusal::Pending, + VdevRefusal::Conflict, + ] { + assert_eq!(refused(refusal), errno::EIO, "{refusal:?}"); + } + } + + #[test] + fn one_device_is_open_at_a_time_and_each_opening_is_its_own() { + let opening = Opening::new(); + assert_eq!(opening.current(), None); + assert!(!opening.end(), "nothing was open"); + assert!(opening.begin(7)); + let first = opening.current().expect("open"); + assert!(opening.is(first)); + assert_eq!( + opening.seen(), + 7, + "interrupts before the opening are not its" + ); + assert!(!opening.begin(9), "a second device"); + assert_eq!(opening.seen(), 7, "a refused opening changes nothing"); + assert!(opening.end()); + assert_eq!(opening.current(), None); + assert!(!opening.is(first), "a reader of the first opening ends"); + assert!(opening.begin(0)); + let second = opening.current().expect("open again"); + assert_ne!(first, second); + assert!(!opening.is(first)); + } + + #[test] + fn an_interrupt_is_read_once() { + let opening = Opening::new(); + assert!(opening.begin(0)); + // Two readers saw the count at zero and woke on the same interrupt. + assert!(opening.take(0, 1)); + assert!(!opening.take(0, 1), "the second waits for the next"); + assert_eq!(opening.seen(), 1); + // The count wraps. + assert!(opening.take(1, u32::MAX)); + assert!(opening.take(u32::MAX, 0)); + assert_eq!(opening.seen(), 0); + } + + #[test] + fn the_names_coordinates_are_the_faces() { + assert_eq!(MODEL_ACCEL, 0x6c63_6361); + assert_eq!(MAP_MEMORY, 0x1000); + assert_eq!(WORD, 4); + let shares = kore_std::vdev::client::THREAD_SHARES; + assert_eq!(shares[0].index, FACE); + assert_eq!(shares[1].index, KICK); + } +} diff --git a/crates/c/src/fd.rs b/crates/c/src/fd.rs index 91d271f..94cca39 100644 --- a/crates/c/src/fd.rs +++ b/crates/c/src/fd.rs @@ -35,12 +35,14 @@ use kore_std::abi::task::role; pub(crate) mod errno { use core::ffi::c_int; pub(crate) const ENOENT: c_int = 2; + pub(crate) const EIO: c_int = 5; pub(crate) const ENXIO: c_int = 6; pub(crate) const EBADF: c_int = 9; pub(crate) const EAGAIN: c_int = 11; pub(crate) const ENOMEM: c_int = 12; pub(crate) const EACCES: c_int = 13; pub(crate) const EFAULT: c_int = 14; + pub(crate) const EBUSY: c_int = 16; pub(crate) const EINVAL: c_int = 22; pub(crate) const EMFILE: c_int = 24; pub(crate) const ENOTSUP: c_int = 95; @@ -54,6 +56,14 @@ pub(crate) enum Kind { /// region off it, through the runtime's own reservation so the two never /// meet ([`crate::mmap`]). Memory, + /// A virtual device of the launch's virtual-device face, by its + /// register-block id: registers and lent memory through `mmap`, the + /// interrupt through `read`, the doorbell through `write` + /// ([`crate::device`]). + Device { + /// The device model's id, the register block's `DEVICE_ID`. + model: u32, + }, } /// One row: a name and the bootstrap coordinate it stands for. @@ -75,12 +85,29 @@ pub(crate) struct Name { /// `/kore/memory` is `role::HEAP@0`: the launch's memory supply, which /// kore-rt endows every thread that may allocate with a share of /// (`WorkerCap::heap`). -pub(crate) static NAMES: [Name; 1] = [Name { - path: b"/kore/memory", - role: role::HEAP, - index: 0, - kind: Kind::Memory, -}]; +/// +/// `/kore/device/virtual/accel` is the virtual accelerator on the launch's +/// virtual-device face. Its row's coordinate is the face's memory, whose +/// second page is the register block: the share of it this library founds +/// the program's threads holding is what a thread parks on the interrupt +/// through. The face's kick, beside it, is what a thread rings the doorbell +/// through ([`crate::device::KICK`]). +pub(crate) static NAMES: [Name; 2] = [ + Name { + path: b"/kore/memory", + role: role::HEAP, + index: 0, + kind: Kind::Memory, + }, + Name { + path: b"/kore/device/virtual/accel", + role: role::SVC_RING, + index: crate::device::FACE, + kind: Kind::Device { + model: crate::device::MODEL_ACCEL, + }, + }, +]; /// The row `path` names. pub(crate) fn resolve(path: &[u8]) -> Option { @@ -241,14 +268,17 @@ pub(crate) fn holding(name: &Name) -> Option { let slot = kore_std::runtime::entry::thread_context()? .bootstrap() .handle(name.role, name.index)?; - let kind = match name.kind { - Kind::Memory => ResourceKind::MemLease, + let rights = verbs::capability_rights(slot, ResourceKind::MemLease)?; + let writable = match name.kind { + Kind::Memory => rights.contains(Rights::MODIFY), + // What a thread writes of a device is its doorbell: a notice on the + // face's kick, which is a capability beside the face's memory. + Kind::Device { .. } => crate::device::holds_kick(), }; - let rights = verbs::capability_rights(slot, kind)?; Some(Holding { slot, readable: rights.contains(Rights::ACCESS), - writable: rights.contains(Rights::MODIFY), + writable, }) } @@ -266,7 +296,17 @@ pub(crate) fn open(path: &[u8], flags: c_int) -> Result { if !access.within(held.readable, held.writable) { return Err(errno::EACCES); } - TABLE.claim(row, access) + match NAMES[row].kind { + Kind::Memory => TABLE.claim(row, access), + // A device is opened as well as named: the service starts the model + // and writes its register block. + Kind::Device { model } => { + crate::device::open(model)?; + TABLE + .claim(row, access) + .inspect_err(|_| crate::device::close()) + } + } } #[cfg(test)] @@ -289,6 +329,16 @@ mod tests { assert_eq!(resolve(path), None, "{:?}", core::str::from_utf8(path)); } assert_eq!((NAMES[0].role, NAMES[0].index), (role::HEAP, 0)); + assert_eq!(resolve(b"/kore/device/virtual/accel"), Some(1)); + assert_eq!(resolve(b"/kore/device/virtual"), None); + assert_eq!(resolve(b"/kore/device/virtual/accl"), None); + assert_eq!(NAMES[1].role, role::SVC_RING); + assert_eq!( + NAMES[1].kind, + Kind::Device { + model: u32::from_le_bytes(*b"accl") + } + ); } #[test] diff --git a/crates/c/src/lib.rs b/crates/c/src/lib.rs index 11eced7..aa84b4d 100644 --- a/crates/c/src/lib.rs +++ b/crates/c/src/lib.rs @@ -48,6 +48,10 @@ pub mod api; #[cfg(any(test, target_os = "none"))] #[cfg_attr(not(target_os = "none"), allow(dead_code))] mod args; +// Off Kore only the descriptor table's rows and the host suite read this: +// the face is the Kore program's. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +mod device; #[cfg(any(test, target_os = "none"))] mod exit; // Off Kore only `kore_fd_describe` and the host suite read these two: the @@ -56,7 +60,7 @@ mod exit; mod fd; #[cfg(target_os = "none")] mod lock; -#[cfg(any(test, target_os = "none"))] +// Off Kore only the host suite and the device's mapping rules read this. #[cfg_attr(not(target_os = "none"), allow(dead_code))] mod mmap; #[cfg(target_os = "none")] diff --git a/crates/c/src/mmap.rs b/crates/c/src/mmap.rs index cebe56a..9f40a21 100644 --- a/crates/c/src/mmap.rs +++ b/crates/c/src/mmap.rs @@ -187,7 +187,7 @@ impl Regions { } #[cfg(target_os = "none")] -pub(crate) use metal::{arm, map, posted, serve, unmap}; +pub(crate) use metal::{arm, map, map_zeroed, place, posted, serve, unmap}; #[cfg(target_os = "none")] mod metal { @@ -228,13 +228,23 @@ mod metal { } } + /// An address for `len` bytes in the window regions land in, which is + /// spent for the life of the program. `None` when the window is unarmed + /// or has no room. + pub(crate) fn place(len: usize) -> Option { + WINDOW + .choose(len as u64, PAGE as u64) + .ok() + .and_then(|base| usize::try_from(base).ok()) + } + /// Map `lease` whole at `base`, zeroed, with the request's access. /// /// The lease is mapped writable to be zeroed whatever was asked: a parked /// region holds what its last user left, and a fresh one is zeroed too /// rather than trusted to be. A read-only request is then mapped again /// without write. - fn map_zeroed( + pub(crate) fn map_zeroed( lease: &StashedLease, base: usize, len: usize, @@ -307,11 +317,7 @@ mod metal { .and_then(verbs::lease_geometry) .and_then(|(_, len)| usize::try_from(len).ok()) .ok_or(errno::ENOMEM)?; - let base = WINDOW - .choose(len as u64, PAGE as u64) - .ok() - .and_then(|base| usize::try_from(base).ok()) - .ok_or(errno::ENOMEM)?; + let base = place(len).ok_or(errno::ENOMEM)?; (row, lease, base, len) } }; diff --git a/crates/c/src/posix.rs b/crates/c/src/posix.rs index 49da7bc..113018d 100644 --- a/crates/c/src/posix.rs +++ b/crates/c/src/posix.rs @@ -1,15 +1,16 @@ -//! The POSIX names over [`crate::fd`] and [`crate::mmap`]: `open`, `close`, -//! `mmap` and `munmap`, as ``, `` and `` -//! declare them. Each sets `errno` and answers `-1` (`MAP_FAILED` for `mmap`) -//! when it refuses. +//! The POSIX names over [`crate::fd`], [`crate::mmap`] and [`crate::device`]: +//! `open`, `close`, `read`, `write`, `poll`, `mmap` and `munmap`, as +//! ``, ``, `` and `` declare them. Each +//! sets `errno` and answers `-1` (`MAP_FAILED` for `mmap`) when it refuses. //! //! Exported on Kore alone, like the rest of the C library: on a host these //! names are the host libc's. -use core::ffi::{CStr, c_char, c_int, c_void}; +use core::ffi::{CStr, c_char, c_int, c_short, c_ulong, c_void}; use kore_c_shim::errno::set as set_errno; +use crate::device; use crate::fd::{self, Kind, NAMES, TABLE, errno}; use crate::mmap::{self, Request}; @@ -38,17 +39,278 @@ pub unsafe extern "C" fn open(path: *const c_char, flags: c_int, _mode: ...) -> } /// `close(fd)`: forget the descriptor. The capability it named stays the -/// program's, and mappings made through it stand. +/// program's, and mappings made through it stand. A device's descriptor +/// gives the device up as well ([`device`]). #[unsafe(no_mangle)] pub extern "C" fn close(fd: c_int) -> c_int { + let device = matches!( + TABLE.get(fd), + Ok((row, _)) if matches!(NAMES[row].kind, Kind::Device { .. }) + ); match TABLE.release(fd) { - Ok(()) => 0, + Ok(()) => { + if device { + device::close(); + } + 0 + } Err(code) => refused(code), } } -/// `mmap(addr, len, prot, flags, fd, offset)` ([`mmap`]'s module docs). -/// `addr` is a hint POSIX lets an implementation ignore, and it is. +/// The numbers of `stdout` and `stderr`, which no `open` hands out. +const STDOUT: c_int = 1; +const STDERR: c_int = 2; + +/// `read(fd, buf, count)`: on a device's descriptor, wait for an interrupt +/// and answer the device's interrupt count, one 32-bit word, as a UIO device +/// does ([`device::read`]). +/// +/// `EBADF`: `fd` is not open for reading. `EINVAL`: `count` is not four, or +/// the descriptor names something that is not read (the memory supply). +/// `EFAULT`: `buf` is null. Otherwise as [`device::read`] refuses. +/// +/// # Safety +/// `buf` is null or valid for `count` bytes of writes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn read(fd: c_int, buf: *mut c_void, count: usize) -> isize { + let answer = TABLE.get(fd).and_then(|(row, access)| { + if !access.read { + return Err(errno::EBADF); + } + match NAMES[row].kind { + Kind::Memory => Err(errno::EINVAL), + Kind::Device { .. } => { + if count != device::WORD { + return Err(errno::EINVAL); + } + if buf.is_null() { + return Err(errno::EFAULT); + } + device::read() + } + } + }); + match answer { + Ok(interrupts) => { + // SAFETY: the caller's contract: four writable bytes, of any + // alignment. + unsafe { buf.cast::().write_unaligned(interrupts) }; + device::WORD as isize + } + Err(code) => refused(code) as isize, + } +} + +/// `write(fd, buf, count)`: on descriptor 1 or 2, `count` bytes to `stdout` +/// or `stderr`, in order with what `` writes; on a device's +/// descriptor, one 32-bit word rings the doorbell ([`device::ring`]). The +/// word is not read. +/// +/// `EBADF`: `fd` is not open for writing, or the launch carried no stream. +/// `EINVAL`: a device was written anything but four bytes, or the descriptor +/// names something that is not written (the memory supply). `EFAULT`: `buf` +/// is null and `count` is not zero. Otherwise as [`device::ring`] refuses. +/// +/// # Safety +/// `buf` is null or valid for `count` bytes of reads. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn write(fd: c_int, buf: *const c_void, count: usize) -> isize { + if fd == STDOUT || fd == STDERR { + if count == 0 { + return 0; + } + if buf.is_null() { + return refused(errno::EFAULT) as isize; + } + // POSIX leaves a count past `SSIZE_MAX` to the implementation: this + // one writes what the return value can report. + let count = count.min(isize::MAX as usize); + // SAFETY: the caller's contract: `count` readable bytes. + let bytes = unsafe { core::slice::from_raw_parts(buf.cast::(), count) }; + return if crate::stdio::write_standard(fd == STDERR, bytes) { + count as isize + } else { + refused(errno::EBADF) as isize + }; + } + let answer = TABLE.get(fd).and_then(|(row, access)| { + if !access.write { + return Err(errno::EBADF); + } + match NAMES[row].kind { + Kind::Memory => Err(errno::EINVAL), + Kind::Device { .. } => { + if count != device::WORD { + return Err(errno::EINVAL); + } + if buf.is_null() { + return Err(errno::EFAULT); + } + device::ring() + } + } + }); + match answer { + Ok(()) => device::WORD as isize, + Err(code) => refused(code) as isize, + } +} + +/// ``'s `struct pollfd`, musl's layout. +#[repr(C)] +pub struct PollFd { + fd: c_int, + events: c_short, + revents: c_short, +} + +/// ``'s events, musl's values. +const POLLIN: c_short = 0x001; +const POLLOUT: c_short = 0x004; +const POLLERR: c_short = 0x008; +const POLLNVAL: c_short = 0x020; + +/// What each entry can report now, and whether one of them asks for a +/// device's interrupt and does not have it: the one thing a `poll` here can +/// wait for. +/// +/// # Safety +/// `fds` is valid for `nfds` entries. +unsafe fn poll_pass(fds: *mut PollFd, nfds: usize) -> (c_int, bool) { + let mut ready = 0; + let mut waits = false; + for at in 0..nfds { + // SAFETY: the caller's contract; one entry is borrowed at a time. + let entry = unsafe { &mut *fds.add(at) }; + entry.revents = 0; + if entry.fd < 0 { + // POSIX: a negative descriptor is skipped. + continue; + } + entry.revents = match TABLE.get(entry.fd) { + // A standard stream takes a write whenever it is asked. + Err(_) if entry.fd == STDOUT || entry.fd == STDERR => entry.events & POLLOUT, + Err(_) => POLLNVAL, + Ok((row, access)) => match NAMES[row].kind { + // Mapping a supply never waits. + Kind::Memory => entry.events & (POLLIN | POLLOUT), + Kind::Device { .. } => { + // The doorbell never waits. + let mut events = if access.write { + entry.events & POLLOUT + } else { + 0 + }; + if access.read && entry.events & POLLIN != 0 { + match device::readable() { + Some(true) => events |= POLLIN, + Some(false) => waits = true, + None => events |= POLLNVAL, + } + } + events + } + }, + }; + if entry.revents != 0 { + ready += 1; + } + } + (ready, waits) +} + +/// `poll(fds, nfds, timeout)`: which descriptors are ready, waiting up to +/// `timeout` milliseconds (forever when negative) for a device's interrupt. +/// +/// `POLLIN` on a device's descriptor is an interrupt waiting to be `read`; +/// `POLLOUT` is always ready, there and on the standard output streams; a +/// memory supply's descriptor is always ready; a number that is not open +/// reports `POLLNVAL`. A device that closes during the wait reports +/// `POLLNVAL`, and one this thread cannot wait on `POLLERR`. +/// +/// `EFAULT`: `fds` is null and `nfds` is not zero. `EINVAL`: more entries +/// than there are descriptors, or a wait with no timeout that nothing in the +/// set can end. A wait with a timeout and nothing to wait for sleeps it out +/// and answers zero. +/// +/// # Safety +/// `fds` is null or valid for `nfds` entries of reads and writes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn poll(fds: *mut PollFd, nfds: c_ulong, timeout: c_int) -> c_int { + let Some(nfds) = usize::try_from(nfds) + .ok() + .filter(|&nfds| nfds <= fd::CAPACITY + fd::FIRST as usize) + else { + return refused(errno::EINVAL); + }; + if nfds != 0 && fds.is_null() { + return refused(errno::EFAULT); + } + // SAFETY: the caller's contract. + let (ready, waits) = unsafe { poll_pass(fds, nfds) }; + if ready != 0 || timeout == 0 { + return ready; + } + let deadline = match u64::try_from(timeout) { + Ok(milliseconds) => { + // The calibrated counter, or the kernel's own stamp on a target + // without one: a timeout is honoured wherever a thread can park. + let Some(now) = kore_std::runtime::clock::deadline_base_nanos() else { + return refused(errno::EINVAL); + }; + Some(now.saturating_add(milliseconds.saturating_mul(1_000_000))) + } + Err(_) => None, + }; + if !waits { + return match deadline { + // Nothing here can become ready: the timeout is the whole wait. + Some(at) => { + // A deadline park of this thread; a thread that cannot park + // answers at once, which the caller's own loop absorbs. + let _ = kore_std::runtime::cap::verbs::sleep_until(at); + 0 + } + None => refused(errno::EINVAL), + }; + } + match device::wait_readable(deadline) { + // An interrupt, or the timeout: either way the entries say which. + // SAFETY: the caller's contract. + Ok(_) => unsafe { poll_pass(fds, nfds) }.0, + Err(code) => { + // The device closed, or cannot be waited on: say so on its + // entries, which is what ended the wait. + let event = if code == errno::EBADF { + POLLNVAL + } else { + POLLERR + }; + let mut ready = 0; + for at in 0..nfds { + // SAFETY: the caller's contract. + let entry = unsafe { &mut *fds.add(at) }; + let device = matches!( + TABLE.get(entry.fd), + Ok((row, _)) if matches!(NAMES[row].kind, Kind::Device { .. }) + ); + if device && entry.events & POLLIN != 0 { + entry.revents |= event; + } + if entry.revents != 0 { + ready += 1; + } + } + ready + } + } +} + +/// `mmap(addr, len, prot, flags, fd, offset)`: of the memory supply +/// ([`mmap`]'s module docs), or of a device, whose offset says which of its +/// maps is meant ([`device::Map`]). `addr` is a hint POSIX lets an +/// implementation ignore, and it is. /// /// `EBADF`: `fd` is not open. `EACCES`: it was not opened for reading, or a /// writable mapping was asked of one not opened for writing. `ENXIO`: the @@ -64,14 +326,25 @@ pub extern "C" fn mmap( offset: i64, ) -> *mut c_void { let mapped = TABLE.get(fd).and_then(|(row, access)| { - let request = Request::of(len, prot, flags, offset)?; - if !access.read || (request.write && !access.write) { - return Err(errno::EACCES); - } let name = &NAMES[row]; - let held = fd::holding(name).ok_or(errno::ENXIO)?; match name.kind { - Kind::Memory => mmap::map(held, request), + Kind::Memory => { + let request = Request::of(len, prot, flags, offset)?; + if !access.read || (request.write && !access.write) { + return Err(errno::EACCES); + } + let held = fd::holding(name).ok_or(errno::ENXIO)?; + mmap::map(held, request) + } + Kind::Device { .. } => { + let map = device::Map::of(len, prot, flags, offset)?; + if !access.read || (map.writes(prot) && !access.write) { + return Err(errno::EACCES); + } + // The calling thread's own share, as for any descriptor. + fd::holding(name).ok_or(errno::ENXIO)?; + device::map(map) + } } }); match mapped { @@ -84,10 +357,11 @@ pub extern "C" fn mmap( } /// `munmap(addr, len)`: unmap the program's mapping at `addr`, whole -/// ([`mmap::unmap`]). +/// ([`mmap::unmap`], or [`device::unmap`] for a device's). #[unsafe(no_mangle)] pub extern "C" fn munmap(addr: *mut c_void, len: usize) -> c_int { - match mmap::unmap(addr.addr(), len) { + let unmapped = device::unmap(addr.addr(), len).unwrap_or_else(|| mmap::unmap(addr.addr(), len)); + match unmapped { Ok(()) => 0, Err(code) => refused(code), } diff --git a/crates/c/src/program.rs b/crates/c/src/program.rs index 657040d..96a8055 100644 --- a/crates/c/src/program.rs +++ b/crates/c/src/program.rs @@ -167,11 +167,13 @@ impl Request { } } -/// Anything the host must answer: an end, a session, or a mapping. +/// Anything the host must answer: an end, a session, a mapping, or a +/// request of the program's device. fn pending() -> bool { EXIT.posted().is_some() || SESSION.state.load(Ordering::Acquire) == POSTED || crate::mmap::posted() + || crate::device::posted() } // ── Entry points the C library calls ──────────────────────────────────── @@ -392,10 +394,22 @@ impl Founded { // thread starts kore-rt's thread controller there, which mints the // doorbells its threads park on (`thread_controller`). Without one the // controller still runs, taking the deadline park. - let grant = thread::channel_space_endowment(); + // + // And its share of the launch's virtual-device face, when it carried + // one: the interrupt and the doorbell are a thread's own + // capabilities, so each thread of the program is founded holding + // them. This thread gives the main thread its share here; the + // controller the main thread starts hands it on to every thread it + // founds. + let mut endowment: alloc::vec::Vec = + thread::channel_space_endowment().into_iter().collect(); + endowment.extend(crate::device::shares()); + for share in kore_std::vdev::client::THREAD_SHARES { + let _ = kore_ffi::hand_on_to_threads(share); + } match thread::spawn_with_doorbell_on( main_thread, - grant.as_slice(), + &endowment, WorkerStack::Bytes(crate::api::KORE_MAIN_STACK_BYTES), ) { Some(worker) => Some(Self { @@ -480,6 +494,7 @@ async fn host( ) -> Result { let mut wake = NoticeRx::new(transport, worker.wake_recv()); let mut filesystem = Filesystem::new(rt, transport); + let mut device = crate::device::Host::new(rt, transport); // The one writer the host can kick, woken when its room is made. let wake_main = || { let _ = worker.kick(); @@ -501,6 +516,17 @@ async fn host( wake_main(); continue; } + // A thread's request of the program's device: this thread holds + // the face. The service's answer is waited for like a session's, so + // an end requested meanwhile still ends the program. + if crate::device::posted() { + let request = pin!(device.serve()); + let ended = pin!(until_exit(&mut wake, output, &wake_main)); + if race(request, ended).await.is_some() { + wake_main(); + } + continue; + } if SESSION.state.load(Ordering::Acquire) == POSTED { // SAFETY: POSTED hands the cell to the host until it stores SERVED. let (step, user) = { diff --git a/crates/c/src/stdio.rs b/crates/c/src/stdio.rs index 7e6f712..44964ab 100644 --- a/crates/c/src/stdio.rs +++ b/crates/c/src/stdio.rs @@ -276,7 +276,9 @@ pub(crate) fn held_is_next(held_after: u32, head: u32) -> bool { extern crate alloc; #[cfg(target_os = "none")] -pub(crate) use metal::{Endpoint, abandon, arm, drain, finish, flush_now, install, report}; +pub(crate) use metal::{ + Endpoint, abandon, arm, drain, finish, flush_now, install, report, write_standard, +}; #[cfg(target_os = "none")] mod metal { @@ -340,6 +342,20 @@ mod metal { let _ = sink(Standard::Error, line); } + /// `write` on a standard stream's descriptor: `bytes` to `stdout`, or to + /// `stderr` when `error`, in order with what `` writes. `false` + /// when the launch carried no stream or it has closed. + pub(crate) fn write_standard(error: bool, bytes: &[u8]) -> bool { + sink( + if error { + Standard::Error + } else { + Standard::Output + }, + bytes, + ) + } + pub(crate) fn host_pending() -> bool { host().is_some_and(|endpoint| !endpoint.held.borrow().is_empty()) } diff --git a/crates/ffi/src/thread.rs b/crates/ffi/src/thread.rs index 9e05906..14e7d1e 100644 --- a/crates/ffi/src/thread.rs +++ b/crates/ffi/src/thread.rs @@ -31,6 +31,15 @@ pub const KORE_THREAD_ECANCELED: i32 = 125; pub const KORE_THREAD_ENOTSUP: i32 = 95; static THREADS: Controller = Controller::new(); + +/// Have every thread this process starts through `pthread_create` hold +/// `share`: a capability its founder holds at a boot-block coordinate, which +/// each founding re-derives (`Controller::hand_on`). For a program's host to +/// call before the first thread is created; `false` once the controller has +/// started, or when it can hand on no more. +pub fn hand_on_to_threads(share: kore::runtime::thread_controller::Handed) -> bool { + THREADS.hand_on(share) +} struct Ticket { id: Id, control: Arc, diff --git a/docs/languages.md b/docs/languages.md index 0086c3f..59d6ded 100644 --- a/docs/languages.md +++ b/docs/languages.md @@ -180,8 +180,8 @@ close(fd); are the process's, as C expects, from 3 (0 to 2 are the streams'), and no authority crosses a thread through one. - **`kore_fd_describe(fd, &info)`** reads what the calling thread holds - there: its kind (`KORE_FD_MEMORY`), what its rights permit, and for a - memory supply its whole length. + there: its kind (`KORE_FD_MEMORY` or `KORE_FD_DEVICE`), what its rights + permit, and for a memory supply its whole length. `mmap` on the memory supply answers fresh, zeroed memory, as mapping `/dev/zero` does elsewhere. It is three of kore-rt's verbs, the sequence a @@ -213,6 +213,53 @@ mapped there. The rules follow from that: end with `ENOMEM`. A program holds sixteen regions, mapped and parked together. +### A virtual device + +A program launched with Kore's virtual-device profile has a face with the +virtual-device service: a device with no hardware behind it, for a driver to +be written against (Kore's `res/VIRTUAL_DEVICE.md`). Its name is a second row +of the table, and its descriptor has the shape Linux's UIO gives a userspace +driver (`crates/c/src/device.rs`): + +```c +int fd = open("/kore/device/virtual/accel", O_RDWR); +volatile KoreVdevRegisters *regs = + mmap(NULL, KORE_VDEV_REGISTERS_BYTES, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, KORE_VDEV_MAP_REGISTERS); +uint8_t *lent = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, KORE_VDEV_MAP_MEMORY); +... +uint32_t word = 1; +write(fd, &word, sizeof word); /* ring the doorbell */ +read(fd, &word, sizeof word); /* wait for the interrupt */ +``` + +- **`open` starts the device.** The service starts the model the name + stands for and writes its register block. A program has one device open at + a time (`EBUSY`), and `close` gives it up. +- **Offset 0 is the register block**, one page the program and the device + both write. `KoreVdevRegisters` (``) is its layout, held to + `kore-vdev`'s register constants by this crate's build. +- **Offset one page is memory to lend.** It is fresh and zeroed, the + program's own, and a device address is an offset into it. The device takes + a loan only in its reset state, so a driver maps it before its first status + write. One region at a time, and it cannot be unmapped while the device is + open (`EBUSY`), since the device may be using it. +- **`read` waits for an interrupt** and answers the device's interrupt count, + one 32-bit word. Each interrupt is read once. **`write`** of one word rings + the doorbell. **`poll`** waits for the interrupt with a timeout. +- **The interrupt and the doorbell never cross a thread.** They are + capabilities, and capabilities are per thread, so the host founds the main + thread holding its own share of the face and the thread controller hands + the share on to every `pthread`. `read` parks the calling thread in the + kernel on the interrupt count, and `write` posts its notice directly. + `open`, the loan and `close` are requests to the service, which thread 0's + face carries: they are posted to it and waited for, as a mapping of the + memory supply is. + +`write` on descriptors 1 and 2 writes to `stdout` and `stderr`, in order +with what `` writes. + **`main` gets its arguments.** The launcher's page carries the name it resolved the program by and the rest of the command line. The host reads it once, on thread 0 before the main thread exists, and splits the rest as every diff --git a/extern/c/include/kore/program.h b/extern/c/include/kore/program.h index b8e3f05..9eda7e8 100644 --- a/extern/c/include/kore/program.h +++ b/extern/c/include/kore/program.h @@ -28,6 +28,13 @@ */ #define KORE_FD_MEMORY 1 +/** + * [`KoreFdInfo::kind`]: a virtual device (``). `mmap` answers + * its register block and memory to lend it, `read` its interrupt, and + * `write` rings its doorbell. + */ +#define KORE_FD_DEVICE 2 + /** * [`KoreFdInfo::access`]: the capability may be read. */ @@ -38,13 +45,100 @@ */ #define KORE_FD_WRITE (1 << 1) +/** + * Bytes in a virtual device's register block: one page. + */ +#define KORE_VDEV_REGISTERS_BYTES 4096 + +/** + * `mmap` offset on a virtual device's descriptor: the register block + * ([`KoreVdevRegisters`]). + */ +#define KORE_VDEV_MAP_REGISTERS 0 + +/** + * `mmap` offset on a virtual device's descriptor: fresh, zeroed memory the + * driver owns and lends the device. A device address is an offset into it. + * The device takes a loan only in its reset state, so map it before the + * first status write. + */ +#define KORE_VDEV_MAP_MEMORY 4096 + +/** + * [`KoreVdevRegisters::magic`]: `"kvdv"`, a Kore virtual device. + */ +#define KORE_VDEV_MAGIC 1986295403 + +/** + * [`KoreVdevRegisters::version`]: this layout. + */ +#define KORE_VDEV_VERSION 1 + +/** + * [`KoreVdevRegisters::vendor_id`]: `"kore"`, a device Kore hosts. + */ +#define KORE_VDEV_VENDOR_KORE 1701998443 + +/** + * [`KoreVdevRegisters::device_id`]: `"accl"`, the virtual accelerator, a + * virtio-accel 1.0 device. Its name is `/kore/device/virtual/accel`. + */ +#define KORE_VDEV_ID_ACCEL 1818452833 + +/** + * Status: the driver has noticed the device (Virtio 1.3 section 2.1). + */ +#define KORE_VDEV_STATUS_ACKNOWLEDGE 1 + +/** + * Status: the driver knows how to drive it. + */ +#define KORE_VDEV_STATUS_DRIVER 2 + +/** + * Status: the driver is set up and the device may serve. + */ +#define KORE_VDEV_STATUS_DRIVER_OK 4 + +/** + * Status: feature negotiation is complete. + */ +#define KORE_VDEV_STATUS_FEATURES_OK 8 + +/** + * Status, written by the device: it cannot go on and must be reset. + */ +#define KORE_VDEV_STATUS_DEVICE_NEEDS_RESET 64 + +/** + * Status: the driver has given up on the device. + */ +#define KORE_VDEV_STATUS_FAILED 128 + +/** + * Feature bit 32, `VIRTIO_F_VERSION_1`: every device offers it and a + * driver must accept it. Written out, because the header carries this + * expression as it stands and `1 << 32` is not a 64-bit value in C. + */ +#define KORE_VDEV_F_VERSION_1 4294967296 + +/** + * Bytes between the last register and the device configuration. + */ +#define KORE_VDEV_RESERVED_BYTES 168 + +/** + * Bytes of the device configuration area, to the end of the block. + */ +#define KORE_VDEV_CONFIG_AREA_BYTES 3840 + /** * What a descriptor names, as the calling thread holds it * ([`kore_fd_describe`]). */ typedef struct KoreFdInfo { /** - * What kind of capability it is: `KORE_FD_MEMORY`. + * What kind of capability it is: `KORE_FD_MEMORY` or `KORE_FD_DEVICE`. */ uint32_t kind; /** @@ -55,11 +149,118 @@ typedef struct KoreFdInfo { uint32_t access; /** * For `KORE_FD_MEMORY`, the supply's whole length in bytes: what the - * allocator and every mapping share, not what is still free. + * allocator and every mapping share, not what is still free. For + * `KORE_FD_DEVICE`, the register block's length. */ uint64_t length; } KoreFdInfo; +/** + * The register block of a Kore virtual device: what `mmap` at + * `KORE_VDEV_MAP_REGISTERS` answers, declared `volatile` by a driver. + * + * Every word is little-endian and has one writer. A driver never writes a + * device register, and what it writes to its own the device reads once, at + * the step that uses it, when the driver rings the doorbell (a `write` of + * one 32-bit word to the descriptor). The device answers by raising an + * interrupt: it increments `interrupts`, and a `read` of one 32-bit word + * from the descriptor returns once the count has moved. + * + * Bringing the device up is Virtio 1.3 section 3.1.1 over these registers, + * with the memory lent first (`KORE_VDEV_MAP_MEMORY`). A device just opened + * is in its reset state, both status words zero; one that is not is reset + * first, by writing 0 to `driver_status` and ringing. Then write + * `ACKNOWLEDGE | DRIVER` and ring; write the accepted features, add + * `FEATURES_OK`, ring, and check `device_status` carries it; lay the queue + * out in the lent memory, write `queue_num`, the three addresses and + * `queue_ready`; add `DRIVER_OK` and ring. A new value of `driver_status` + * is always answered with an interrupt, and a step the device refused is + * one its `device_status` does not carry. The value the device last saw + * announces nothing and is not answered, so a driver waits only for what + * it asked. + */ +typedef struct KoreVdevRegisters { + /** + * Device: `KORE_VDEV_MAGIC`. + */ + uint32_t magic; + /** + * Device: `KORE_VDEV_VERSION`. + */ + uint32_t version; + /** + * Device: which model this is (`KORE_VDEV_ID_ACCEL`). + */ + uint32_t device_id; + /** + * Device: `KORE_VDEV_VENDOR_KORE`. + */ + uint32_t vendor_id; + /** + * Device: the features it offers. + */ + uint64_t device_features; + /** + * Driver: the features it accepts, a subset of the offer. + */ + uint64_t driver_features; + /** + * Driver: the status it has reached (`KORE_VDEV_STATUS_*`). + */ + uint32_t driver_status; + /** + * Device: the status it has accepted, and its own + * `KORE_VDEV_STATUS_DEVICE_NEEDS_RESET`. + */ + uint32_t device_status; + /** + * Device: the largest queue it takes. + */ + uint32_t queue_num_max; + /** + * Driver: the queue's size, a power of two between the longest chain + * the device takes (its configuration says how long) and + * `queue_num_max`. + */ + uint32_t queue_num; + /** + * Driver: nonzero once the queue's size and addresses are written. + */ + uint32_t queue_ready; + /** + * Device: changes whenever the configuration area does. + */ + uint32_t config_generation; + /** + * Driver: the descriptor table's device address. + */ + uint64_t queue_desc; + /** + * Driver: the available ring's device address. + */ + uint64_t queue_driver; + /** + * Driver: the used ring's device address. + */ + uint64_t queue_device; + /** + * Device: bytes of device configuration in `config`. + */ + uint32_t config_bytes; + /** + * Device: interrupts raised, wrapping. + */ + uint32_t interrupts; + /** + * Unused. + */ + uint8_t reserved[KORE_VDEV_RESERVED_BYTES]; + /** + * Device: the device configuration. + */ + uint8_t config[KORE_VDEV_CONFIG_AREA_BYTES]; +} KoreVdevRegisters; + #ifdef __cplusplus extern "C" { #endif // __cplusplus diff --git a/tools/check-program.py b/tools/check-program.py index c6b0a1b..0e12146 100755 --- a/tools/check-program.py +++ b/tools/check-program.py @@ -40,10 +40,10 @@ # The rest of the C library a program's headers declare (Kore's TCC # `include/`: , , , ), which libkore.so # implements: kore-c-shim's printf family, streams, parsers, string routines -# and per-thread errno, and kore-c's own abort; and , and -# 's open, close, mmap and munmap over the capabilities the -# program holds. The program's libc.a holds the rest: exit, atexit and the -# routines a compiler assumes. +# and per-thread errno, and kore-c's own abort; and , , +# and 's open, close, read, write, poll, mmap and munmap +# over the capabilities the program holds. The program's libc.a holds the +# rest: exit, atexit and the routines a compiler assumes. C_LIBRARY = {'printf', 'fprintf', 'sprintf', 'snprintf', 'vprintf', 'vfprintf', 'vsprintf', 'vsnprintf', 'fputc', 'putc', 'putchar', 'fputs', 'puts', 'fwrite', 'fflush', 'fclose', 'stdout', 'stderr', @@ -52,7 +52,7 @@ 'memchr', 'strchr', 'strrchr', 'strcmp', 'strncmp', 'strcpy', 'strncpy', 'strncat', 'strstr', '__errno_location', - 'open', 'close', 'mmap', 'munmap'} + 'open', 'close', 'read', 'write', 'poll', 'mmap', 'munmap'} DYNAMIC_TLS = re.compile(r'TLSDESC|DTPMOD|DTPOFF|TLSGD|TLSLD|TLS_DTP')