From aa411c56131f6bb187fb22bdb587a9739a1ab9c5 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 19:04:55 +0200 Subject: [PATCH 1/8] toyos-transport: the one transport core; blockring's rings are rebuilt on it Stage T1 of the unified transport. A new pure crate, `toyos-transport` (no_std, no alloc, forbid(unsafe_code), rustc-dep-of-std, the domain lint line, one dependency: toyos-untrusted), holds what every client/server session will run over: - `Word`, the port: load, store, and a SeqCst fence in the word's own memory model, so loom sees the fence. - `Producer`/`Consumer`: SPSC queues of E-word entries. Entries come out as `[Untrusted; E]`; a peer's cursor is bounded against what this end released or published (`Violation::TailPastDepth`/`HeadPastTail`), and one moved back within bounds costs only its owner. Each end re-reads the peer's cursor only when what it saw is spent. - `StreamTx`/`StreamRx`: byte rings that answer `Span`s and never touch a byte, with a producer-written `end` word stored after the tail and loaded before it. - `Wake`/`before_sleep`: the sleeper stores `sleep`, fences, looks again; the publisher stores its tail, fences, loads `sleep`, and answers `Wake::Peer` only then. - `Geometry::decode`, `Run::decode`, and `Own`/`Lent`/`Held`, none Clone. - `Inflight`: a tag is a slot index under the slot's own sequence, so an answered, ended or replayed tag is `Violation::Tag`; `end` answers every tag in flight, once, eagerly. - `Schema`: LAYOUT, decode_request, decode_reply. blockring's `ring.rs` is deleted. Its places, `client`/`server` and the ring types live in `layout.rs` over the core; a consumer's `sleep` word takes the free word after its head (1 and 33), the page otherwise as it was. `Request::decode`/`Completion::decode` decode `Untrusted` words, and `entry::Block` is the protocol as a schema. `ServerSession::take_entry` takes the popped words; the model's `take` wraps its raw words as a peer's, so `src/model.rs` is unchanged byte for byte and its end-state counts match main's for every bound. blockd compiles against the rebuilt rings with its doorbells as they were: it rings on any publish, whatever the wake says. One difference: a completion that finds no room, or a client head past what was posted, now ends that session instead of panicking blockd, whose old `push` asserted. The loom publication check moves into the core (`tests/loom.rs`) beside a lost-wake model (futex as park/unpark) and two hostile-peer models; an exhaustive session model (crash, reconnect, replayed tags) holds Inflight and the rings together. Four mutation features, each a CONTROLS row that reds: publish-relaxed, no-sleep-fence, no-clamp, end-keeps-inflight; blockring's mutate-ring-publish-relaxed goes with the file it reverted. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 14 +- Cargo.toml | 1 + src/build.rs | 6 +- src/ci.rs | 12 +- tests/toyos-rust-tests/Cargo.lock | 12 ++ toyos-blockring/Cargo.toml | 23 +- toyos-blockring/src/entry.rs | 122 ++++++----- toyos-blockring/src/layout.rs | 40 +++- toyos-blockring/src/lib.rs | 11 +- toyos-blockring/src/ring.rs | 241 --------------------- toyos-blockring/src/server.rs | 11 +- toyos-blockring/tests/loom_ring.rs | 69 ------ toyos-transport/Cargo.toml | 46 ++++ toyos-transport/src/arena.rs | 180 ++++++++++++++++ toyos-transport/src/inflight.rs | 115 ++++++++++ toyos-transport/src/lib.rs | 195 +++++++++++++++++ toyos-transport/src/model.rs | 328 +++++++++++++++++++++++++++++ toyos-transport/src/queue.rs | 271 ++++++++++++++++++++++++ toyos-transport/src/stream.rs | 283 +++++++++++++++++++++++++ toyos-transport/tests/loom.rs | 175 +++++++++++++++ toyos-untrusted/Cargo.toml | 7 + userland/Cargo.lock | 12 ++ userland/blockd/src/main.rs | 32 ++- userland/blockd/src/session.rs | 14 +- 24 files changed, 1803 insertions(+), 417 deletions(-) delete mode 100644 toyos-blockring/src/ring.rs delete mode 100644 toyos-blockring/tests/loom_ring.rs create mode 100644 toyos-transport/Cargo.toml create mode 100644 toyos-transport/src/arena.rs create mode 100644 toyos-transport/src/inflight.rs create mode 100644 toyos-transport/src/lib.rs create mode 100644 toyos-transport/src/model.rs create mode 100644 toyos-transport/src/queue.rs create mode 100644 toyos-transport/src/stream.rs create mode 100644 toyos-transport/tests/loom.rs diff --git a/Cargo.lock b/Cargo.lock index 802affd915b..1846b3b1776 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1188,8 +1188,8 @@ version = "0.1.0" name = "toyos-blockring" version = "0.1.0" dependencies = [ - "loom", "toyos-blockhold", + "toyos-transport", ] [[package]] @@ -1424,9 +1424,21 @@ version = "0.1.0" name = "toyos-tmpdir" version = "0.1.0" +[[package]] +name = "toyos-transport" +version = "0.1.0" +dependencies = [ + "loom", + "rustc-std-workspace-core", + "toyos-untrusted", +] + [[package]] name = "toyos-untrusted" version = "0.1.0" +dependencies = [ + "rustc-std-workspace-core", +] [[package]] name = "toyos-update" diff --git a/Cargo.toml b/Cargo.toml index 0ec14bd71dc..6a7c3a02ffb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ members = [ "toyos-symbols", "toyos-tco", "toyos-tmpdir", + "toyos-transport", "toyos-untrusted", "toyos-update", "toyos-userbound", diff --git a/src/build.rs b/src/build.rs index a65b75e2709..2cc8c728f1f 100644 --- a/src/build.rs +++ b/src/build.rs @@ -2739,13 +2739,14 @@ mod tests { /// /// `loom` selects loom's instrumented atomics; `check`, `protocol-port`, /// `tripwire` and `std` mirror `toyos-sched`'s own features so the shared - /// sources compile identically and name nothing a model turns on. Everything + /// sources compile identically and name nothing a model turns on; + /// `rustc-dep-of-std` builds a crate under std. Everything /// else declared in any of these files is, by construction, a /// `--features ` command that must red a named model — each file's own /// comment beside the name carries the argument for why. fn declared_model_controls(root: &Path) -> Vec<(&'static str, String)> { const NOT_A_CONTROL: &[&str] = - &["loom", "check", "protocol-port", "tripwire", "std", "default"]; + &["loom", "check", "protocol-port", "tripwire", "std", "default", "rustc-dep-of-std"]; let mut out = Vec::new(); for (crate_name, manifest) in [ ("kernel-loom", "kernel-loom/Cargo.toml"), @@ -2753,6 +2754,7 @@ mod tests { ("toyos-sched-sim", "toyos-sched/sim/Cargo.toml"), ("toyos-proclife", "toyos-proclife/Cargo.toml"), ("toyos-blockring", "toyos-blockring/Cargo.toml"), + ("toyos-transport", "toyos-transport/Cargo.toml"), ] { let path = root.join(manifest); let text = fs::read_to_string(&path) diff --git a/src/ci.rs b/src/ci.rs index 69f6b80181b..b52aaeac7a9 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -230,6 +230,7 @@ const SCHED_LOOM: &[&str] = &["-p", "toyos-sched-loom"]; const SCHED_SIM: &[&str] = &["-p", "toyos-sched-sim"]; const PROCLIFE: &[&str] = &["-p", "toyos-proclife"]; const BLOCKRING: &[&str] = &["-p", "toyos-blockring"]; +const TRANSPORT: &[&str] = &["-p", "toyos-transport"]; const fn red( krate: &'static [&'static str], @@ -378,9 +379,16 @@ pub(crate) const CONTROLS: &[Control] = &[ red(BLOCKRING, "mutate-no-reissue-after-loss", None, &[ "what_a_flush_calls_durable_is_on_the_medium ... FAILED", ]), - red(BLOCKRING, "mutate-ring-publish-relaxed", Some("loom_ring"), &[ - "a_published_request_is_read_whole ... FAILED", + // The transport's four: a tail published before its entry, a wake both + // sides miss, a peer's cursor believed, and a session's tags outliving it. + red(TRANSPORT, "publish-relaxed", Some("loom"), &["a_published_entry_is_read_whole ... FAILED"]), + // A lost wake is a consumer parked for good: loom's deadlock, whose unwind + // panics again before the harness prints a `FAILED` line. + red(TRANSPORT, "no-sleep-fence", Some("loom"), &[ + "deadlock; threads = [(Id(0), Blocked(Location(None))), (Id(1), Blocked(Location(None)))]", ]), + red(TRANSPORT, "no-clamp", Some("loom"), &["a_hostile_producer_yields_entries_or_a_violation ... FAILED"]), + red(TRANSPORT, "end-keeps-inflight", None, &["every_tag_is_answered_exactly_once ... FAILED"]), ]; /// Whether a control's run showed its teeth. diff --git a/tests/toyos-rust-tests/Cargo.lock b/tests/toyos-rust-tests/Cargo.lock index bc2b2c2b2b1..5879429cd2b 100644 --- a/tests/toyos-rust-tests/Cargo.lock +++ b/tests/toyos-rust-tests/Cargo.lock @@ -2102,6 +2102,7 @@ name = "toyos-blockring" version = "0.1.0" dependencies = [ "toyos-blockhold", + "toyos-transport", ] [[package]] @@ -2174,6 +2175,17 @@ dependencies = [ name = "toyos-tco" version = "0.1.0" +[[package]] +name = "toyos-transport" +version = "0.1.0" +dependencies = [ + "toyos-untrusted", +] + +[[package]] +name = "toyos-untrusted" +version = "0.1.0" + [[package]] name = "toyos-wallclock" version = "0.1.0" diff --git a/toyos-blockring/Cargo.toml b/toyos-blockring/Cargo.toml index ae73456fa9f..7f23ba250e7 100644 --- a/toyos-blockring/Cargo.toml +++ b/toyos-blockring/Cargo.toml @@ -1,15 +1,15 @@ # A member of the host workspace (root `Cargo.toml`). What lives here is the # block protocol between a block service and its client — the shared session -# page's layout, the two rings on it, the words a request and a completion are, -# the control frames a session is opened with, and every decision -# either end makes about what a completion means — with nothing that touches a -# device, a handle or a mapping. `userland/blockd` serves it and its client -# glue speaks it; both map the page and hand this crate the words. +# page's layout, where its two `toyos-transport` rings are, the words a request +# and a completion are, the control frames a session is opened with, and every +# decision either end makes about what a completion means — with nothing that +# touches a device, a handle or a mapping. `userland/blockd` serves it and its +# client glue speaks it; both map the page and hand this crate the words. # # Its defects are orders — a completion racing a crash, a flush racing a reset, # a reconnect racing a reissue — so `src/model.rs` enumerates every ordering of -# a scripted client against a server, a device and a crash, and -# `tests/loom_ring.rs` checks the rings' publication under loom. +# a scripted client against a server, a device and a crash; the rings' +# publication is `toyos-transport`'s, and checked under loom there. [package] name = "toyos-blockring" @@ -32,19 +32,14 @@ mutate-abort-keeps-inflight = [] # re-issues nothing: writes it had acknowledged are gone and a later flush # still says durable. mutate-no-reissue-after-loss = [] -# The rings publish a tail with `Relaxed`, so the consumer can see the index -# before the entry's words. -mutate-ring-publish-relaxed = [] [dependencies] # Whose flush answers for which writes the disk lost, and who holds which span: # the server half's bookkeeping, decided where the kernel's block layer decides # it today. toyos-blockhold = { path = "../toyos-blockhold" } - -[dev-dependencies] -# Already resolved in this workspace for `kernel-loom` and `toyos-sched/loom`. -loom = "0.7" +# The rings on the session page, and the schema the protocol is to them. +toyos-transport = { path = "../toyos-transport" } [lints.rust] warnings = "deny" diff --git a/toyos-blockring/src/entry.rs b/toyos-blockring/src/entry.rs index 1e8b8b8e177..8084b97d8d7 100644 --- a/toyos-blockring/src/entry.rs +++ b/toyos-blockring/src/entry.rs @@ -4,7 +4,9 @@ //! request is bounded here against the arena and the partition, and a word //! this protocol does not define is a refusal, never a default. -use crate::layout::{ARENA_BLOCKS, CQE_WORDS, MAX_REQUEST_BLOCKS, SQE_WORDS}; +use toyos_transport::{Layout, Schema, Untrusted, Violation}; + +use crate::layout::{ARENA_BLOCKS, BLOCK_BYTES, CQE_WORDS, MAX_REQUEST_BLOCKS, SQE_WORDS}; /// What a request asks of the partition. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -25,15 +27,6 @@ impl Op { Self::Flush => 3, } } - - const fn from_word(word: u32) -> Option { - match word { - 1 => Some(Self::Read), - 2 => Some(Self::Write), - 3 => Some(Self::Flush), - _ => None, - } - } } /// One request. `lba` is the partition's own block number, from 0: nothing in @@ -78,37 +71,36 @@ impl Request { /// `partition_blocks`: every block it names is inside both the arena and /// the partition, a transfer moves at least one block and at most /// [`MAX_REQUEST_BLOCKS`], and a flush names nothing. - pub fn decode(words: [u32; SQE_WORDS], partition_blocks: u64) -> Result { - let tag = words[1]; + pub fn decode(words: [Untrusted; SQE_WORDS], partition_blocks: u64) -> Result { + let [op, tag, lba_low, lba_high, blocks, arena, reserved @ ..] = words; + let tag = opaque(tag); let refused = Refused::Malformed { tag }; - let op = Op::from_word(words[0]).ok_or(refused)?; - let lba = u64::from(words[2]) | (u64::from(words[3]) << 32); - let (blocks, arena) = (words[4], words[5]); - if words[6] != 0 || words[7] != 0 { + let op = [Op::Read, Op::Write, Op::Flush].into_iter().find(|o| op.is(o.word())).ok_or(refused)?; + if !reserved.iter().all(|word| word.is(0)) { return Err(refused); } - match op { - Op::Flush => { - if lba != 0 || blocks != 0 || arena != 0 { - return Err(refused); - } - } - Op::Read | Op::Write => { - if blocks == 0 || blocks > MAX_REQUEST_BLOCKS { - return Err(refused); - } - if arena.checked_add(blocks).is_none_or(|end| end > ARENA_BLOCKS) { - return Err(refused); - } - if lba.checked_add(u64::from(blocks)).is_none_or(|end| end > partition_blocks) { - return Err(refused); - } + let lba = u64::from(opaque(lba_low)) | (u64::from(opaque(lba_high)) << 32); + if op == Op::Flush { + if lba != 0 || !blocks.is(0) || !arena.is(0) { + return Err(refused); } + return Ok(Self { op, tag, lba, blocks: 0, arena: 0 }); + } + let blocks = blocks.at_most(MAX_REQUEST_BLOCKS.into()).ok().filter(|&b| b > 0).ok_or(refused)? as u32; + let arena = arena.at_most((ARENA_BLOCKS - blocks).into()).map_err(|_| refused)? as u32; + if lba.checked_add(u64::from(blocks)).is_none_or(|end| end > partition_blocks) { + return Err(refused); } Ok(Self { op, tag, lba, blocks, arena }) } } +/// A word every value of which means something: a tag its answer echoes, or +/// half of an `lba` the partition bounds whole. +fn opaque(word: Untrusted) -> u32 { + word.at_most(u32::MAX.into()).map_or(0, |word| word as u32) +} + /// What a completion says of its request. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Status { @@ -135,16 +127,6 @@ impl Status { Self::Lost => 3, } } - - const fn from_word(word: u32) -> Option { - match word { - 0 => Some(Self::Ok), - 1 => Some(Self::Invalid), - 2 => Some(Self::Device), - 3 => Some(Self::Lost), - _ => None, - } - } } /// One completion. @@ -161,20 +143,58 @@ impl Completion { /// `None` for words no server of this protocol writes: the client treats a /// server that wrote them as one that broke the session. - pub fn decode(words: [u32; CQE_WORDS]) -> Option { - if words[2] != 0 || words[3] != 0 { + pub fn decode(words: [Untrusted; CQE_WORDS]) -> Option { + let [tag, status, reserved @ ..] = words; + if !reserved.iter().all(|word| word.is(0)) { return None; } - Some(Self { tag: words[0], status: Status::from_word(words[1])? }) + let status = + [Status::Ok, Status::Invalid, Status::Device, Status::Lost].into_iter().find(|s| status.is(s.word()))?; + Some(Self { tag: opaque(tag), status }) + } +} + +/// The block protocol as a transport schema, over a partition `blocks` long. +pub struct Block { + pub blocks: u64, +} + +impl Schema for Block { + const LAYOUT: Layout = Layout::Region { slot_bytes: BLOCK_BYTES as u32 }; + type Request = Request; + type Reply = Completion; + type Refusal = Refused; + + fn decode_request(&self, words: [Untrusted; SQE_WORDS]) -> Result { + Request::decode(words, self.blocks) + } + + fn decode_reply(&self, words: [Untrusted; CQE_WORDS]) -> Result { + Completion::decode(words).ok_or(Violation::Entry) } } #[cfg(test)] mod tests { use super::*; + use crate::layout::{arena_byte, SESSION_BYTES}; const PARTITION: u64 = 1000; + fn peer(words: [u32; N]) -> [Untrusted; N] { + words.map(Untrusted::new) + } + + /// The transport's geometry of the schema's layout is this crate's arena, + /// block for block. + #[test] + fn the_schemas_arena_is_the_sessions() { + let geometry = toyos_transport::Geometry::decode(Untrusted::new(SESSION_BYTES as u64), Block::LAYOUT).unwrap(); + assert_eq!(geometry.slots(), ARENA_BLOCKS); + let last = geometry.own(ARENA_BLOCKS - 1, 1).unwrap(); + assert_eq!(last.run().span().offset, arena_byte(ARENA_BLOCKS - 1)); + } + #[test] fn a_request_survives_its_words() { for request in [ @@ -182,10 +202,10 @@ mod tests { Request { op: Op::Write, tag: u32::MAX, lba: 0, blocks: MAX_REQUEST_BLOCKS, arena: ARENA_BLOCKS - MAX_REQUEST_BLOCKS }, Request { op: Op::Flush, tag: 0, lba: 0, blocks: 0, arena: 0 }, ] { - assert_eq!(Request::decode(request.encode(), PARTITION), Ok(request)); + assert_eq!(Request::decode(peer(request.encode()), PARTITION), Ok(request)); } let wide = Request { op: Op::Read, tag: 1, lba: 1 << 40, blocks: 2, arena: 3 }; - assert_eq!(Request::decode(wide.encode(), u64::MAX), Ok(wide)); + assert_eq!(Request::decode(peer(wide.encode()), u64::MAX), Ok(wide)); } /// Every field a hostile client can set out of range is refused, and the @@ -207,7 +227,7 @@ mod tests { ]; for (what, words) in cases { assert_eq!( - Request::decode(words, PARTITION), + Request::decode(peer(words), PARTITION), Err(Refused::Malformed { tag: 42 }), "{what} was not refused" ); @@ -218,9 +238,9 @@ mod tests { fn a_completion_survives_its_words_and_refuses_what_it_does_not_define() { for status in [Status::Ok, Status::Invalid, Status::Device, Status::Lost] { let c = Completion { tag: 9, status }; - assert_eq!(Completion::decode(c.encode()), Some(c)); + assert_eq!(Completion::decode(peer(c.encode())), Some(c)); } - assert_eq!(Completion::decode([1, 4, 0, 0]), None); - assert_eq!(Completion::decode([1, 0, 1, 0]), None); + assert_eq!(Completion::decode(peer([1, 4, 0, 0])), None); + assert_eq!(Completion::decode(peer([1, 0, 1, 0])), None); } } diff --git a/toyos-blockring/src/layout.rs b/toyos-blockring/src/layout.rs index 1d36547d028..7df5fe666c3 100644 --- a/toyos-blockring/src/layout.rs +++ b/toyos-blockring/src/layout.rs @@ -1,7 +1,10 @@ //! Where everything is on a session's region, in 32-bit words from its start. //! -//! The four ring indices sit on cache lines of their own, so the client's -//! stores to its two and the server's to its two never share a line. +//! The four ring indices sit on cache lines of their own, each consumer's +//! `sleep` word on its head's, so the client's stores and the server's never +//! share a line. + +use toyos_transport::{Consumer, Cursors, Place, Producer, Violation, Word}; /// A session's whole region: the one size shared memory comes in. pub const SESSION_BYTES: usize = 2 * 1024 * 1024; @@ -10,18 +13,19 @@ pub const SESSION_BYTES: usize = 2 * 1024 * 1024; pub const BLOCK_BYTES: usize = 4096; /// How many requests, and so how many completions, one session has in flight. -/// A power of two, so an index is its ring position masked. pub const DEPTH: u32 = 64; /// The most blocks one request moves. A driver whose device takes less in one /// command splits it; one that takes more is still asked for no more than this. pub const MAX_REQUEST_BLOCKS: u32 = 32; -/// Index words. The server writes [`SQ_HEAD`] and [`CQ_TAIL`], the client the -/// other two. +/// Index words. The server writes [`SQ_HEAD`], [`SQ_SLEEP`] and [`CQ_TAIL`], +/// the client the other three. pub const SQ_HEAD: usize = 0; +pub const SQ_SLEEP: usize = 1; pub const SQ_TAIL: usize = 16; pub const CQ_HEAD: usize = 32; +pub const CQ_SLEEP: usize = 33; pub const CQ_TAIL: usize = 48; /// Words per request entry, and where the request ring starts. @@ -35,13 +39,37 @@ pub const CQ_BASE: usize = SQ_BASE + DEPTH as usize * SQE_WORDS; /// Every word the rings use; the page they are on is the first block. pub const RING_WORDS: usize = CQ_BASE + DEPTH as usize * CQE_WORDS; +/// The request ring and the completion ring. +pub const REQUESTS: Place = + Place { cursors: Cursors { head: SQ_HEAD, tail: SQ_TAIL, sleep: SQ_SLEEP }, entries: SQ_BASE }; +pub const COMPLETIONS: Place = + Place { cursors: Cursors { head: CQ_HEAD, tail: CQ_TAIL, sleep: CQ_SLEEP }, entries: CQ_BASE }; + +/// A client's two ends: requests out, completions in. +pub type ClientRings = (Producer, Consumer); + +/// A server's two ends: requests in, completions out. +pub type ServerRings = (Consumer, Producer); + +/// The client's ends of a session page, every word it owns set to 0. Done +/// before the page is sent to a server, and again before it is sent to the +/// next one. +pub fn client(page: &[W]) -> Result { + Ok((Producer::new(page, REQUESTS)?, Consumer::new(page, COMPLETIONS)?)) +} + +/// The server's ends of a session page it was sent, every word it owns set to +/// 0. Whatever the client left in its own is bounded when first looked at. +pub fn server(page: &[W]) -> Result { + Ok((Consumer::new(page, REQUESTS)?, Producer::new(page, COMPLETIONS)?)) +} + /// Where the arena starts, in bytes: the block after the rings' page. pub const ARENA_OFFSET: usize = BLOCK_BYTES; /// The arena's blocks; a request's `arena` is an index below this. pub const ARENA_BLOCKS: u32 = ((SESSION_BYTES - ARENA_OFFSET) / BLOCK_BYTES) as u32; -const _: () = assert!(DEPTH.is_power_of_two()); const _: () = assert!(RING_WORDS * 4 <= ARENA_OFFSET); const _: () = assert!(MAX_REQUEST_BLOCKS <= ARENA_BLOCKS); diff --git a/toyos-blockring/src/lib.rs b/toyos-blockring/src/lib.rs index 57ee1dbd950..3988cdaba32 100644 --- a/toyos-blockring/src/lib.rs +++ b/toyos-blockring/src/lib.rs @@ -7,8 +7,8 @@ //! [`BLOCK_BYTES`] blocks a request names by index and the device moves data //! into and out of directly. Nothing on the page is a pointer and nothing on it //! is trusted by the end that did not write it: a consumer bounds every index -//! and every field before it acts ([`entry::Request::decode`], -//! [`ring::Consumer`]). +//! and every field before it acts ([`entry::Request::decode`], the +//! transport's cursor bounds); [`entry::Block`] is the protocol as its schema. //! //! **A doorbell is a byte on the session's connection**, written after the //! entries it announces are published. The connection is also what tells each @@ -28,9 +28,9 @@ //! **Requests in flight at once are unordered**, as on the device: a client //! that needs one to follow another waits for the first's answer. //! -//! Pure: `alloc` and `toyos-blockhold`, no `unsafe`. The ends that map the -//! page — `userland/blockd` and its client — hand this crate the page as -//! words ([`ring::Word`]) and act on what it answers. +//! Pure: `alloc`, `toyos-blockhold` and `toyos-transport`, no `unsafe`. The +//! ends that map the page — `userland/blockd` and its client — hand it the +//! page as words ([`toyos_transport::Word`]) and act on what it answers. #![cfg_attr(not(test), no_std)] #![forbid(unsafe_code)] @@ -40,7 +40,6 @@ extern crate alloc; pub mod client; pub mod entry; pub mod layout; -pub mod ring; pub mod server; pub mod wire; diff --git a/toyos-blockring/src/ring.rs b/toyos-blockring/src/ring.rs deleted file mode 100644 index 9b148d44529..00000000000 --- a/toyos-blockring/src/ring.rs +++ /dev/null @@ -1,241 +0,0 @@ -//! The two single-producer rings on a session's page. -//! -//! **A producer writes an entry's words, then publishes the tail with -//! `Release`; a consumer loads the tail with `Acquire`, then reads the words.** -//! The head goes back the other way: a consumer publishes it with `Release` -//! only after the entries below it are read, and a producer loads it with -//! `Acquire` before it writes over them. `tests/loom_ring.rs` holds the -//! first edge; the second is the same pair turned round. -//! -//! Each end keeps its own index in a local and only ever *stores* the shared -//! one, so nothing the peer writes into its own index can move ours; what the -//! peer's index claims is bounded against ours before it is believed -//! ([`Violation`]). - -use core::sync::atomic::{AtomicU32, Ordering}; - -use crate::layout::{CQE_WORDS, CQ_BASE, CQ_HEAD, CQ_TAIL, DEPTH, RING_WORDS, SQE_WORDS, SQ_BASE, SQ_HEAD, SQ_TAIL}; - -/// One shared 32-bit word: an atomic over the mapped page, or a model's. -pub trait Word { - fn load(&self, order: Ordering) -> u32; - fn store(&self, value: u32, order: Ordering); -} - -impl Word for AtomicU32 { - fn load(&self, order: Ordering) -> u32 { - AtomicU32::load(self, order) - } - fn store(&self, value: u32, order: Ordering) { - AtomicU32::store(self, value, order) - } -} - -/// The peer's index says something no peer of this protocol can: more entries -/// published than the ring holds, or more consumed than were produced. The -/// session is over. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Violation; - -/// The publishing order a tail is stored with. -#[cfg(not(feature = "mutate-ring-publish-relaxed"))] -const PUBLISH: Ordering = Ordering::Release; -#[cfg(feature = "mutate-ring-publish-relaxed")] -const PUBLISH: Ordering = Ordering::Relaxed; - -/// Where one ring is on the page, in words. -#[derive(Clone, Copy, Debug)] -struct Place { - head: usize, - tail: usize, - entries: usize, -} - -const REQUESTS: Place = Place { head: SQ_HEAD, tail: SQ_TAIL, entries: SQ_BASE }; -const COMPLETIONS: Place = Place { head: CQ_HEAD, tail: CQ_TAIL, entries: CQ_BASE }; - -fn checked(page: &[W]) -> &[W] { - assert!(page.len() >= RING_WORDS, "a session page holds every ring word"); - page -} - -/// The end of a ring that writes entries. It holds its indices and not the -/// page, so its owner keeps the mapping beside it; every call is given the -/// page. -#[derive(Debug)] -pub struct Producer { - place: Place, - local: u32, - published: u32, -} - -impl Producer { - fn new(page: &[W], place: Place) -> Self { - checked(page)[place.tail].store(0, Ordering::Release); - Self { place, local: 0, published: 0 } - } - - /// How many entries may be pushed before the consumer frees more. - pub fn space(&self, page: &[W]) -> Result { - let head = checked(page)[self.place.head].load(Ordering::Acquire); - let used = self.local.wrapping_sub(head); - if used > DEPTH { - return Err(Violation); - } - Ok(DEPTH - used) - } - - /// Write one entry. It is the consumer's only once [`Self::publish`] runs. - /// - /// # Panics - /// When the ring has no space: the caller asks [`Self::space`] first. - pub fn push(&mut self, page: &[W], words: [u32; N]) { - assert!(self.space(page).is_ok_and(|space| space > 0), "a push into a full ring"); - let at = self.place.entries + (self.local % DEPTH) as usize * N; - for (i, word) in words.into_iter().enumerate() { - page[at + i].store(word, Ordering::Relaxed); - } - self.local = self.local.wrapping_add(1); - } - - /// Publish every entry pushed so far; answers whether there was any. - pub fn publish(&mut self, page: &[W]) -> bool { - if self.published == self.local { - return false; - } - checked(page)[self.place.tail].store(self.local, PUBLISH); - self.published = self.local; - true - } -} - -/// The end of a ring that reads entries; like [`Producer`], it holds indices -/// and is given the page. -#[derive(Debug)] -pub struct Consumer { - place: Place, - local: u32, - released: u32, -} - -impl Consumer { - fn new(page: &[W], place: Place) -> Self { - checked(page)[place.head].store(0, Ordering::Release); - Self { place, local: 0, released: 0 } - } - - /// The next published entry, `None` for none, or the producer's tail - /// claiming more than the ring holds. - pub fn pop(&mut self, page: &[W]) -> Result, Violation> { - let tail = checked(page)[self.place.tail].load(Ordering::Acquire); - let ready = tail.wrapping_sub(self.local); - if ready > DEPTH { - return Err(Violation); - } - if ready == 0 { - return Ok(None); - } - let at = self.place.entries + (self.local % DEPTH) as usize * N; - let words = core::array::from_fn(|i| page[at + i].load(Ordering::Relaxed)); - self.local = self.local.wrapping_add(1); - Ok(Some(words)) - } - - /// Give every entry popped so far back to the producer. - pub fn release(&mut self, page: &[W]) { - if self.released != self.local { - checked(page)[self.place.head].store(self.local, Ordering::Release); - self.released = self.local; - } - } -} - -/// A client's two ends: requests out, completions in. -pub type ClientRings = (Producer, Consumer); - -/// A server's two ends: requests in, completions out. -pub type ServerRings = (Consumer, Producer); - -/// The client's ends of a session page, both indices it owns set to 0. Done -/// before the page is sent to a server, and again before it is sent to the -/// next one. -pub fn client(page: &[W]) -> ClientRings { - (Producer::new(page, REQUESTS), Consumer::new(page, COMPLETIONS)) -} - -/// The server's ends of a session page it was sent, both indices it owns set -/// to 0. Whatever the client left in its own two is bounded by the first -/// [`Consumer::pop`] and [`Producer::space`]. -pub fn server(page: &[W]) -> ServerRings { - (Consumer::new(page, REQUESTS), Producer::new(page, COMPLETIONS)) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::entry::{Completion, Op, Request, Status}; - use alloc::vec::Vec; - - fn page() -> Vec { - (0..RING_WORDS).map(|_| AtomicU32::new(0)).collect() - } - - #[test] - fn a_request_crosses_and_its_completion_comes_back() { - let page = page(); - let (mut sq, mut cq) = client(&page); - let (mut rq, mut cp) = server(&page); - let request = Request { op: Op::Write, tag: 3, lba: 8, blocks: 2, arena: 1 }; - sq.push(&page, request.encode()); - assert_eq!(rq.pop(&page), Ok(None), "an entry is nobody's before it is published"); - assert!(sq.publish(&page)); - assert_eq!(rq.pop(&page).map(|w| w.map(|w| Request::decode(w, 100))), Ok(Some(Ok(request)))); - rq.release(&page); - cp.push(&page, Completion { tag: 3, status: Status::Ok }.encode()); - cp.publish(&page); - assert_eq!(cq.pop(&page).map(|w| w.and_then(Completion::decode)), Ok(Some(Completion { tag: 3, status: Status::Ok }))); - } - - /// The ring wraps many times over, and space is exactly what the consumer - /// has given back. - #[test] - fn the_ring_wraps_and_counts_its_space() { - let page = page(); - let (mut sq, _) = client(&page); - let (mut rq, _) = server(&page); - let (mut pushed, mut popped) = (0u32, 0u32); - for round in 0..5 * DEPTH { - // Uneven batches both ways, so the two indices cross every slot - // at every distance. - for _ in 0..1 + round % 9 { - assert_eq!(sq.space(&page), Ok(DEPTH - (pushed - popped))); - if pushed - popped == DEPTH { - break; - } - sq.push(&page, Request { op: Op::Read, tag: pushed, lba: 0, blocks: 1, arena: 0 }.encode()); - pushed += 1; - } - sq.publish(&page); - for _ in 0..1 + round % 7 { - let Ok(Some(words)) = rq.pop(&page) else { break }; - assert_eq!(words[1], popped, "entries come out in the order they went in"); - popped += 1; - } - rq.release(&page); - } - assert!(pushed > 2 * DEPTH, "the ring wrapped"); - } - - /// A hostile producer's tail far ahead of the consumer, and a hostile - /// consumer's head ahead of what was produced, both end the session. - #[test] - fn a_peer_index_out_of_reach_is_a_violation() { - let page = page(); - let (sq, _) = client(&page); - let (mut rq, _) = server(&page); - page[SQ_TAIL].store(DEPTH + 1, Ordering::Release); - assert_eq!(rq.pop(&page), Err(Violation)); - page[SQ_HEAD].store(5, Ordering::Release); - assert_eq!(sq.space(&page), Err(Violation)); - } -} diff --git a/toyos-blockring/src/server.rs b/toyos-blockring/src/server.rs index 6157ffd6ad2..6c927cd2b06 100644 --- a/toyos-blockring/src/server.rs +++ b/toyos-blockring/src/server.rs @@ -19,6 +19,8 @@ use alloc::collections::BTreeMap; use toyos_blockhold::{Holds, Writer}; +use toyos_transport::Untrusted; + use crate::entry::{Completion, Op, Refused, Request, Status}; use crate::layout::SQE_WORDS; @@ -75,7 +77,7 @@ impl ServerSession { /// A malformed entry, and a tag already in flight, are answered at once /// and never reach the device: the second would make one tag two /// requests, and the client could not tell which answer was whose. - pub fn take(&mut self, words: [u32; SQE_WORDS]) -> Taken { + pub fn take_entry(&mut self, words: [Untrusted; SQE_WORDS]) -> Taken { let request = match Request::decode(words, self.blocks) { Ok(request) => request, Err(Refused::Malformed { tag }) => { @@ -89,6 +91,13 @@ impl ServerSession { Taken::Issue(request) } + /// [`Self::take_entry`] of words a test wrote as the client, which arrive + /// as a peer's do. + #[cfg(test)] + pub fn take(&mut self, words: [u32; SQE_WORDS]) -> Taken { + self.take_entry(words.map(Untrusted::new)) + } + /// The device answered the request `tag` with `done` (whether it did it), /// while its loss count was `losses`. `None` for a tag no longer in /// flight: a reset has already answered it. diff --git a/toyos-blockring/tests/loom_ring.rs b/toyos-blockring/tests/loom_ring.rs deleted file mode 100644 index 26bcef0b3e0..00000000000 --- a/toyos-blockring/tests/loom_ring.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! The rings' publication edge under loom: a consumer that sees a tail sees -//! every word of the entries below it. -//! -//! The client and the server are two processes on two CPUs over one shared -//! page, so this is the one property of the rings no host test that runs both -//! ends on one thread can reach. `mutate-ring-publish-relaxed` takes the edge -//! away and this must red: -//! -//! cargo test -p toyos-blockring --features mutate-ring-publish-relaxed --test loom_ring - -use core::sync::atomic::Ordering; - -use loom::sync::atomic::AtomicU32; -use loom::sync::Arc; -use toyos_blockring::entry::{Op, Request}; -use toyos_blockring::layout::RING_WORDS; -use toyos_blockring::ring::{self, Word}; - -/// A loom atomic as a page word: the trait is this crate's and the type is -/// loom's, so the two meet through a wrapper. -struct Shared(AtomicU32); - -impl Word for Shared { - fn load(&self, order: Ordering) -> u32 { - self.0.load(order) - } - fn store(&self, value: u32, order: Ordering) { - self.0.store(value, order) - } -} - -fn page() -> Arc> { - Arc::new((0..RING_WORDS).map(|_| Shared(AtomicU32::new(0))).collect()) -} - -const PARTITION: u64 = 1 << 20; - -fn request(n: u32) -> Request { - Request { op: Op::Write, tag: 100 + n, lba: 7 + u64::from(n), blocks: 1 + n, arena: 3 * n } -} - -/// Two requests published one at a time, read by the other end as they -/// arrive: each is whole, in order, and exactly what was written. -#[test] -fn a_published_request_is_read_whole() { - loom::model(|| { - let page = page(); - let server_page = Arc::clone(&page); - let server = loom::thread::spawn(move || { - let (mut requests, _) = ring::server(&server_page); - let mut read = Vec::new(); - while read.len() < 2 { - match requests.pop(&server_page).expect("the client keeps the protocol") { - Some(words) => read.push(Request::decode(words, PARTITION)), - None => loom::thread::yield_now(), - } - } - requests.release(&server_page); - read - }); - let (mut requests, _) = ring::client(&page); - for n in 0..2 { - requests.push(&page, request(n).encode()); - requests.publish(&page); - } - let read = server.join().expect("the server thread"); - assert_eq!(read, [Ok(request(0)), Ok(request(1))], "a request was read before its words"); - }); -} diff --git a/toyos-transport/Cargo.toml b/toyos-transport/Cargo.toml new file mode 100644 index 00000000000..53a4c41dc36 --- /dev/null +++ b/toyos-transport/Cargo.toml @@ -0,0 +1,46 @@ +# A member of the host workspace (root `Cargo.toml`). The one transport every +# client/server session runs over: the rings, the byte streams, the arena's +# ownership tokens, the tag table and the wake, as decisions over words an +# adapter hands in. Nothing here maps, copies or blocks. +# +# Its defects are orders and hostile peers, so `tests/loom.rs` holds the +# publication edge, the wake and a peer that scribbles every word it can reach, +# and `src/model.rs` enumerates a session through crash, restart and a late +# completion. + +[package] +name = "toyos-transport" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false + +[features] +rustc-dep-of-std = ["core", "toyos-untrusted/rustc-dep-of-std"] +# Declared, never enabled by any build that ships: each reverts one decision so +# the model that refuses it is shown able to red at all. `src/ci.rs`'s +# `CONTROLS` runs each and demands the named test's own FAILED line. +# +# A producer publishes its tail `Relaxed`, so a consumer can see the index +# before the entry's words. +publish-relaxed = [] +# Neither side of the wake fences between its store and its load, so a +# producer can miss a consumer's `sleep` while the consumer misses its tail. +no-sleep-fence = [] +# A peer's cursor is believed however far it claims to be, so a hostile +# producer hands the consumer more entries than the ring holds. +no-clamp = [] +# `Inflight::end` answers nothing and keeps every tag in flight, so a session's +# requests outlive it unanswered and a late completion answers one. +end-keeps-inflight = [] + +[dependencies] +core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } +toyos-untrusted = { path = "../toyos-untrusted" } + +[dev-dependencies] +# Already resolved in this workspace for `kernel-loom` and `toyos-sched/loom`. +loom = "0.7" + +[lints.rust] +warnings = "deny" diff --git a/toyos-transport/src/arena.rs b/toyos-transport/src/arena.rs new file mode 100644 index 00000000000..01883f7cc1c --- /dev/null +++ b/toyos-transport/src/arena.rs @@ -0,0 +1,180 @@ +//! A region's arena, and who may touch which run of it. +//! +//! **A run exists only once it is bounded by the [`Geometry`]**, and it +//! travels as one of three tokens, none of them `Clone`: [`Own`] on the side +//! that allocated it, which lending consumes into [`Lent`] until a completion +//! naming it hands it back; and [`Held`] on the side that decoded it from an +//! entry ([`Run::decode`]). The adapter reaches a run's bytes only through a +//! token's [`Run::span`]. + +use crate::{Span, Untrusted, Violation}; + +/// What a schema's region is: none, or a header page and an arena of slots of +/// the schema's size. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Layout { + /// Entries travel as frames on the connection; a region sent is refused. + Inline, + Region { slot_bytes: u32 }, +} + +/// A region's arena, decoded once. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Geometry { + slot_bytes: u32, + slots: u32, +} + +impl Geometry { + /// Every region is this long: the granule shared memory comes in. + pub const BYTES: u64 = 0x20_0000; + /// The header page the queues and cursors are on; the arena follows it. + pub const HEADER_BYTES: u32 = 0x1000; + const ARENA_BYTES: u32 = 0x20_0000 - Self::HEADER_BYTES; + + /// The arena of a region `bytes` long, cut as `layout` says. + pub fn decode(bytes: Untrusted, layout: Layout) -> Result { + let Layout::Region { slot_bytes } = layout else { return Err(Violation::Region) }; + bytes.exactly(Self::BYTES).map_err(|_| Violation::Region)?; + let slots = Self::ARENA_BYTES.checked_div(slot_bytes).filter(|&n| n > 0).ok_or(Violation::Region)?; + Ok(Self { slot_bytes, slots }) + } + + pub fn slots(&self) -> u32 { + self.slots + } + + /// Slots `first..first + count`, for the allocator that hands them out + /// once each to own; `None` for none or past the arena. + pub fn own(&self, first: u32, count: u32) -> Option { + self.run(first, count).map(Own) + } + + fn run(&self, first: u32, count: u32) -> Option { + if count == 0 || first.checked_add(count)? > self.slots { + return None; + } + let offset = first.checked_mul(self.slot_bytes)?.checked_add(Self::HEADER_BYTES)?; + let len = count.checked_mul(self.slot_bytes)?; + Some(Run { first, count, span: Span { offset: offset.try_into().ok()?, len: len.try_into().ok()? } }) + } +} + +/// Slots of the arena, bounded by its geometry. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Run { + first: u32, + count: u32, + span: Span, +} + +impl Run { + /// The run an entry's two words name, held until its answer. + pub fn decode(first: Untrusted, count: Untrusted, geometry: &Geometry) -> Result { + let bounded = |word: Untrusted| word.at_most(u64::from(geometry.slots)).ok()?.try_into().ok(); + bounded(first) + .zip(bounded(count)) + .and_then(|(first, count)| geometry.run(first, count)) + .map(Held) + .ok_or(Violation::Run) + } + + pub fn first(&self) -> u32 { + self.first + } + + pub fn count(&self) -> u32 { + self.count + } + + /// Its bytes in the region. + pub fn span(&self) -> Span { + self.span + } +} + +/// A run this side allocated and has not lent. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct Own(Run); + +impl Own { + pub fn run(&self) -> &Run { + &self.0 + } + + /// Put it in an entry: the run is the peer's until its answer. + pub fn lend(self) -> (Lent, Run) { + (Lent(self.0), self.0) + } +} + +/// A run in an entry the peer has not answered. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct Lent(Run); + +impl Lent { + /// The completion that names it came back: it is this side's again. + pub fn back(self) -> Own { + Own(self.0) + } +} + +/// A run the peer lent, bounded, until this side answers the entry that named +/// it. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct Held(Run); + +impl Held { + pub fn run(&self) -> &Run { + &self.0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const PAGES: Layout = Layout::Region { slot_bytes: 4096 }; + + fn geometry() -> Geometry { + Geometry::decode(Untrusted::new(Geometry::BYTES), PAGES).unwrap() + } + + #[test] + fn a_region_is_the_granule_and_a_schema_with_one() { + assert_eq!(geometry().slots(), 511); + for bytes in [0, Geometry::BYTES - 1, Geometry::BYTES + 1, 2 * Geometry::BYTES] { + assert_eq!(Geometry::decode(Untrusted::new(bytes), PAGES), Err(Violation::Region)); + } + assert_eq!(Geometry::decode(Untrusted::new(Geometry::BYTES), Layout::Inline), Err(Violation::Region)); + assert_eq!( + Geometry::decode(Untrusted::new(Geometry::BYTES), Layout::Region { slot_bytes: 0 }), + Err(Violation::Region) + ); + } + + /// Every run a peer can name is inside the arena, whole slots of it; the + /// last slot is a run and one past it is not, however the sum wraps. + #[test] + fn a_run_a_peer_names_is_inside_the_arena() { + let g = geometry(); + let held = Run::decode(Untrusted::new(510), Untrusted::new(1), &g).unwrap(); + assert_eq!(held.run().span(), Span { offset: 0x1000 + 510 * 4096, len: 4096 }); + for (first, count) in [(510, 2), (0, 0), (511, 1), (1, u32::MAX), (u32::MAX, 1), (0, 512)] { + assert_eq!( + Run::decode(Untrusted::new(first), Untrusted::new(count), &g), + Err(Violation::Run), + "{first}+{count}" + ); + } + } + + #[test] + fn lending_consumes_and_the_answer_gives_back() { + let own = geometry().own(3, 2).unwrap(); + let (lent, run) = own.lend(); + assert_eq!((run.first(), run.count()), (3, 2)); + assert_eq!(lent.back().run(), &run); + assert_eq!(geometry().own(510, 2), None); + } +} diff --git a/toyos-transport/src/inflight.rs b/toyos-transport/src/inflight.rs new file mode 100644 index 00000000000..3545f5dde4b --- /dev/null +++ b/toyos-transport/src/inflight.rs @@ -0,0 +1,115 @@ +//! The requests a client has on the wire, by tag. +//! +//! **Each tag is answered exactly once**: by the completion that names it +//! ([`Inflight::answer`]) or, when the session ends, by [`Inflight::end`] — +//! never by both, and never by a completion naming a tag from before its +//! slot was last filled. A tag is the slot's index under the slot's own +//! sequence, so a tag answered, ended or replayed names nothing. + +use crate::{Untrusted, Violation}; + +const INDEX_BITS: u32 = 16; +const INDEX_MASK: u32 = (1 << INDEX_BITS) - 1; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +struct Slot { + seq: u16, + value: Option, +} + +/// At most `D` requests in flight, each carrying what its answer needs. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Inflight { + slots: [Slot; D], +} + +fn tag(seq: u16, index: u32) -> u32 { + u32::from(seq).wrapping_shl(INDEX_BITS) | index +} + +impl Inflight { + pub fn new() -> Self { + const { assert!(D > 0 && D <= 1 << INDEX_BITS, "a tag's index is 16 bits") }; + Self { slots: core::array::from_fn(|_| Slot { seq: 0, value: None }) } + } + + /// Keep `value` in flight under a tag no earlier request had in this slot; + /// `Err(value)` with `D` in flight. + pub fn insert(&mut self, value: T) -> Result { + let Some((index, slot)) = (0u32..).zip(self.slots.iter_mut()).find(|(_, slot)| slot.value.is_none()) else { + return Err(value); + }; + slot.seq = slot.seq.wrapping_add(1); + slot.value = Some(value); + Ok(tag(slot.seq, index)) + } + + /// What the peer's `tag` answers; a tag nothing is in flight under is a + /// violation. + pub fn answer(&mut self, tag: Untrusted) -> Result { + let index = tag.map(|t| t & INDEX_MASK).index(D).map_err(|_| Violation::Tag)?; + let slot = self.slots.get_mut(index).ok_or(Violation::Tag)?; + if !tag.map(|t| t.wrapping_shr(INDEX_BITS)).is(u32::from(slot.seq)) { + return Err(Violation::Tag); + } + slot.value.take().ok_or(Violation::Tag) + } + + /// The session ended: `each` is given every tag in flight, once, with what + /// it carried, and none of them is answered again. + pub fn end(&mut self, mut each: impl FnMut(u32, T)) { + for (index, slot) in (0u32..).zip(self.slots.iter_mut()) { + #[cfg(not(feature = "end-keeps-inflight"))] + let value = slot.value.take(); + #[cfg(feature = "end-keeps-inflight")] + let value = None; + if let Some(value) = value { + each(tag(slot.seq, index), value); + } + } + } +} + +impl Default for Inflight { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_tag_is_answered_once_and_its_slot_takes_a_new_one() { + let mut inflight = Inflight::<&str, 2>::new(); + let a = inflight.insert("a").unwrap(); + let b = inflight.insert("b").unwrap(); + assert_eq!(inflight.insert("c"), Err("c")); + assert_eq!(inflight.answer(Untrusted::new(a)), Ok("a")); + assert_eq!(inflight.answer(Untrusted::new(a)), Err(Violation::Tag), "answered twice"); + let c = inflight.insert("c").unwrap(); + assert_ne!(c, a, "the slot's next tag is not its last"); + assert_eq!(inflight.answer(Untrusted::new(a)), Err(Violation::Tag), "a replay answers nothing"); + assert_eq!(inflight.answer(Untrusted::new(2)), Err(Violation::Tag), "an index past the table"); + assert_eq!(inflight.answer(Untrusted::new(b)), Ok("b")); + assert_eq!(inflight.answer(Untrusted::new(c)), Ok("c")); + assert!(inflight.insert("d").is_ok() && inflight.insert("e").is_ok(), "every slot is free again"); + } + + #[test] + fn an_end_answers_every_tag_once_and_a_late_completion_nothing() { + let mut inflight = Inflight::::new(); + let tags: Vec = (0..3).map(|n| inflight.insert(n).unwrap()).collect(); + assert_eq!(inflight.answer(Untrusted::new(tags[1])), Ok(1)); + let mut ended = Vec::new(); + inflight.end(|tag, value| ended.push((tag, value))); + assert_eq!(ended, [(tags[0], 0), (tags[2], 2)]); + for tag in tags { + assert_eq!(inflight.answer(Untrusted::new(tag)), Err(Violation::Tag)); + } + let mut again = 0; + inflight.end(|_, _| again += 1); + assert_eq!(again, 0, "an end answers nothing a first one answered"); + } +} diff --git a/toyos-transport/src/lib.rs b/toyos-transport/src/lib.rs new file mode 100644 index 00000000000..41f7f3e314e --- /dev/null +++ b/toyos-transport/src/lib.rs @@ -0,0 +1,195 @@ +//! The transport every client/server session runs over. +//! +//! **A session is one connection and, at most, one region.** The connection +//! carries the hello, handles, doorbells and the hang-up; the region carries +//! [`Producer`]/[`Consumer`] queues of fixed-size entries, [`StreamTx`]/ +//! [`StreamRx`] byte rings, and an arena whose runs pass between the ends only +//! as [`Own`] → [`Lent`] on one side and [`Held`] on the other. A schema +//! ([`Schema`]) says what the entries mean; this crate says only what is safe. +//! +//! **Nothing here holds the region.** An adapter hands every call the region's +//! words as a slice of [`Word`]s and copies bytes itself, through the [`Span`]s +//! answered here, once: no reference to the peer's bytes is formed. +//! +//! **What the peer writes is untrusted until decoded.** An entry comes out as +//! [`Untrusted`] words; a peer's cursor is bounded against the ring before it +//! is believed, and the variant of [`Violation`] names what it claimed; a run +//! is bounded against the [`Geometry`], and a tag against the [`Inflight`] +//! table. An end keeps its own cursor locally and only stores the shared one, +//! so nothing the peer writes moves it, and a cursor the peer moves backwards +//! within bounds costs only the peer. +//! +//! **Publication is `Release`/`Acquire`; the wake is a pair of `SeqCst` +//! fences.** A consumer that finds nothing stores its `sleep` word, fences and +//! looks again ([`Consumer::before_sleep`]); a producer stores its tail, +//! fences and loads `sleep`, and asks for a wake ([`Wake::Peer`]) only if it is +//! set. A busy consumer costs a producer no syscall, and a hostile `sleep` only +//! costs a wake. +//! +//! **A restart is seen only as a hang-up**, and [`Inflight::end`] answers every +//! tag that was in flight, once; a tag from before it answers nothing. + +#![cfg_attr(not(test), no_std)] +#![forbid(unsafe_code)] +#![cfg_attr(not(test), forbid(clippy::arithmetic_side_effects, clippy::indexing_slicing, clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::as_conversions))] + +mod arena; +mod inflight; +#[cfg(test)] +mod model; +mod queue; +mod stream; + +use core::sync::atomic::{AtomicU32, Ordering}; + +pub use arena::{Geometry, Held, Layout, Lent, Own, Run}; +pub use inflight::Inflight; +pub use queue::{Consumer, Place, Producer}; +pub use stream::{End, StreamPlace, StreamRx, StreamTx}; +pub use toyos_untrusted::Untrusted; + +/// One shared 32-bit word of a region: an atomic over the mapping, or a +/// model's. +pub trait Word { + fn load(&self, order: Ordering) -> u32; + fn store(&self, value: u32, order: Ordering); + /// A `SeqCst` fence, in the memory model this word lives in. + fn fence(); +} + +impl Word for AtomicU32 { + fn load(&self, order: Ordering) -> u32 { + AtomicU32::load(self, order) + } + fn store(&self, value: u32, order: Ordering) { + AtomicU32::store(self, value, order) + } + fn fence() { + core::sync::atomic::fence(Ordering::SeqCst) + } +} + +/// The peer wrote what no peer of the protocol writes, or the region is not +/// one this session can use. The session is over; the variant is its name. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Violation { + /// A producer's tail more than the ring holds past what was released. + TailPastDepth, + /// A consumer's head past what was published, or more than the ring holds + /// behind it. + HeadPastTail, + /// A stream's end word that is no [`End`]. + End, + /// A reply no server of the schema writes. + Entry, + /// A run outside the arena. + Run, + /// A tag nothing is in flight under. + Tag, + /// A region no [`Geometry`] describes, or a place outside the words given. + Region, +} + +/// What a publish asks of the adapter. +#[must_use] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Wake { + /// The consumer said it sleeps: wake it. + Peer, + /// The consumer is awake and will find what was published. + Busy, +} + +/// A consumer that found nothing and said so: it waits while word `word` +/// holds `value`, as a futex does. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Asleep { + pub word: usize, + pub value: u32, +} + +/// Bytes `offset..offset + len` of the region. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Span { + pub offset: usize, + pub len: usize, +} + +/// Where a ring's two cursors and its consumer's `sleep` word are, in words. +/// The producer stores `tail`, the consumer `head` and `sleep`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Cursors { + pub head: usize, + pub tail: usize, + pub sleep: usize, +} + +/// A protocol over this transport: its region's layout, and the one place its +/// entries' words become its own types. A request that does not decode is +/// answered by tag with the schema's refusal; a reply that does not is the +/// server ending the session. +pub trait Schema { + const LAYOUT: Layout; + type Request; + type Reply; + type Refusal; + fn decode_request(&self, words: [Untrusted; SQE]) -> Result; + fn decode_reply(&self, words: [Untrusted; CQE]) -> Result; +} + +#[cfg(not(feature = "publish-relaxed"))] +const PUBLISH: Ordering = Ordering::Release; +#[cfg(feature = "publish-relaxed")] +const PUBLISH: Ordering = Ordering::Relaxed; + +fn word(page: &[W], at: usize) -> Result<&W, Violation> { + page.get(at).ok_or(Violation::Region) +} + +/// A distance the peer's cursor claims, believed only up to `cap`. +fn clamp(claimed: Untrusted, cap: u32, broken: Violation) -> Result { + let cap = if cfg!(feature = "no-clamp") { u32::MAX } else { cap }; + claimed.at_most(u64::from(cap)).ok().and_then(|n| u32::try_from(n).ok()).ok_or(broken) +} + +impl Cursors { + /// How far past `released` the producer's tail is: at most `cap`. + fn published(&self, page: &[W], released: u32, cap: u32) -> Result { + let tail = word(page, self.tail)?.load(Ordering::Acquire); + clamp(Untrusted::new(tail).map(|t| t.wrapping_sub(released)), cap, Violation::TailPastDepth) + } + + /// How far behind `published` the consumer's head is: at most `cap`. + fn unreleased(&self, page: &[W], published: u32, cap: u32) -> Result { + let head = word(page, self.head)?.load(Ordering::Acquire); + clamp(Untrusted::new(head).map(|h| published.wrapping_sub(h)), cap, Violation::HeadPastTail) + } + + fn publish(&self, page: &[W], tail: u32) -> Result { + word(page, self.tail)?.store(tail, PUBLISH); + self.wake(page) + } + + /// The producer's half of the wake, after what it published is stored. + fn wake(&self, page: &[W]) -> Result { + #[cfg(not(feature = "no-sleep-fence"))] + W::fence(); + Ok(match word(page, self.sleep)?.load(Ordering::Relaxed) { + 0 => Wake::Busy, + _ => Wake::Peer, + }) + } + + /// The consumer's half: say it sleeps, before it looks again. + fn sleep(&self, page: &[W]) -> Result<(), Violation> { + word(page, self.sleep)?.store(1, Ordering::Relaxed); + #[cfg(not(feature = "no-sleep-fence"))] + W::fence(); + Ok(()) + } + + fn awake(&self, page: &[W]) -> Result<(), Violation> { + word(page, self.sleep)?.store(0, Ordering::Relaxed); + Ok(()) + } +} diff --git a/toyos-transport/src/model.rs b/toyos-transport/src/model.rs new file mode 100644 index 00000000000..56c245b00d4 --- /dev/null +++ b/toyos-transport/src/model.rs @@ -0,0 +1,328 @@ +//! Every ordering of one client, a server, its death, the client's reconnect, +//! and a server that answers a tag it has answered before. +//! +//! The client puts requests on a real [`Producer`] and reads answers off a +//! real [`Consumer`], over words of its own, keeping them in a real +//! [`Inflight`]. [`explore`] runs, depth first and exhaustively, every +//! interleaving of: the client sending its next request, the server taking +//! one, answering any one it holds, **answering again any tag it has ever +//! taken**, **dying**, the client reading an answer, noticing the hang-up, +//! and reconnecting over the same words to a fresh server. A client that reads +//! a violation ends the session as it would a hang-up. +//! +//! The law: every request is answered exactly once — never twice, and by the +//! end of every run — and an answer read off the ring answers only a request +//! of the session it was sent in. + +use std::cell::Cell; +use std::collections::HashSet; +use std::format; +use std::string::String; +use std::vec::Vec; + +use crate::{Consumer, Cursors, Inflight, Place, Producer, Untrusted, Word}; + +const D: u32 = 2; +const REQUESTS: usize = 3; +const SQ: Place = Place { cursors: Cursors { head: 0, tail: 1, sleep: 2 }, entries: 3 }; +const CQ: Place = Place { cursors: Cursors { head: 5, tail: 6, sleep: 7 }, entries: 8 }; +const WORDS: usize = 10; + +/// A word of a model that runs on one thread: every order is the program's. +#[derive(Clone, Debug)] +struct Shared(Cell); + +impl Word for Shared { + fn load(&self, _: core::sync::atomic::Ordering) -> u32 { + self.0.get() + } + fn store(&self, value: u32, _: core::sync::atomic::Ordering) { + self.0.set(value) + } + fn fence() {} +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Answer { + /// Read off the ring in session `session`. + Read { session: u32 }, + /// Given by the session's end. + Ended, +} + +#[derive(Clone, Debug)] +struct Server { + requests: Consumer<1, D>, + replies: Producer<1, D>, + held: Vec, +} + +#[derive(Clone, Debug)] +struct World { + page: Vec, + requests: Producer<1, D>, + replies: Consumer<1, D>, + inflight: Inflight, + next: usize, + session: u32, + /// The session each request was sent in. + sent: [Option; REQUESTS], + answers: [Vec; REQUESTS], + server: Option, + /// The server died and the client has not noticed. + hung_up: bool, + /// Every tag any server took, which a replay names. + taken: Vec, + crashes: u8, + replays: u8, +} + +/// What a run found. +pub struct Explored { + pub broken: Option, + pub states: usize, + pub ends: usize, + /// Answers the client refused as naming nothing in flight. + pub refused: usize, +} + +struct Run { + seen: HashSet, + broken: Option, + ends: usize, + refused: usize, + path: Vec, +} + +fn plain(word: Untrusted) -> u32 { + word.at_most(u32::MAX.into()).unwrap() as u32 +} + +impl World { + fn new(crashes: u8, replays: u8) -> Self { + let page: Vec = (0..WORDS).map(|_| Shared(Cell::new(0))).collect(); + let mut world = Self { + requests: Producer::new(&page, SQ).unwrap(), + replies: Consumer::new(&page, CQ).unwrap(), + page, + inflight: Inflight::new(), + next: 0, + session: 0, + sent: [None; REQUESTS], + answers: Default::default(), + server: None, + hung_up: false, + taken: Vec::new(), + crashes, + replays, + }; + world.connect(); + world + } + + /// A fresh server's ends, and the client's again, over the same words. + fn connect(&mut self) { + self.requests = Producer::new(&self.page, SQ).unwrap(); + self.replies = Consumer::new(&self.page, CQ).unwrap(); + self.server = Some(Server { + requests: Consumer::new(&self.page, SQ).unwrap(), + replies: Producer::new(&self.page, CQ).unwrap(), + held: Vec::new(), + }); + } + + /// Whether the client thinks it has a session. + fn up(&self) -> bool { + self.server.is_some() || self.hung_up + } + + /// The session is over: every request in flight is answered by its end. + fn end(&mut self) { + self.server = None; + self.hung_up = false; + let answers = &mut self.answers; + self.inflight.end(|_, request| answers[request].push(Answer::Ended)); + } + + /// The server posts an answer naming `tag`; `false` if the ring is full. + fn post(&mut self, tag: u32) -> bool { + let server = self.server.as_mut().unwrap(); + if !server.replies.push(&self.page, [tag]).unwrap() { + return false; + } + let _ = server.replies.publish(&self.page).unwrap(); + true + } +} + +fn go(run: &mut Run, step: String, world: World) { + run.path.push(step); + dfs(run, world); + run.path.pop(); +} + +fn fail(run: &mut Run, why: String) { + if run.broken.is_none() { + run.broken = Some(format!("{why}, after {}", run.path.join(" > "))); + } +} + +fn dfs(run: &mut Run, world: World) { + if run.broken.is_some() || !run.seen.insert(format!("{world:?}")) { + return; + } + let mut moved = false; + + // The client sends its next request, into a ring nobody may be reading. + if world.up() && world.next < REQUESTS { + let mut w = world.clone(); + if let Ok(tag) = w.inflight.insert(w.next) { + assert!(w.requests.push(&w.page, [tag]).unwrap(), "more requests on the ring than in flight"); + let _ = w.requests.publish(&w.page).unwrap(); + w.sent[w.next] = Some(w.session); + w.next += 1; + moved = true; + go(run, format!("send {}#{tag:x}", world.next), w); + } + } + + if let Some(server) = &world.server { + // The server takes the oldest request. + let mut w = world.clone(); + let s = w.server.as_mut().unwrap(); + if let Some([tag]) = s.requests.pop(&w.page).unwrap() { + let tag = plain(tag); + s.requests.release(&w.page).unwrap(); + s.held.push(tag); + w.taken.push(tag); + moved = true; + go(run, format!("take #{tag:x}"), w); + } + + // It answers any one it holds. + for i in 0..server.held.len() { + let mut w = world.clone(); + let tag = w.server.as_mut().unwrap().held.remove(i); + if w.post(tag) { + moved = true; + go(run, format!("answer #{tag:x}"), w); + } + } + + // It answers again a tag it took, in this session or an earlier one. + if world.replays > 0 { + for &tag in &world.taken { + let mut w = world.clone(); + w.replays -= 1; + if w.post(tag) { + moved = true; + go(run, format!("replay #{tag:x}"), w); + } + } + } + + // It dies; the client has not noticed. + if world.crashes > 0 { + let mut w = world.clone(); + w.crashes -= 1; + w.server = None; + w.hung_up = true; + moved = true; + go(run, "crash".into(), w); + } + } + + // The client reads the oldest answer, the server alive or not. + if world.up() { + let mut w = world.clone(); + if let Some([tag]) = w.replies.pop(&w.page).unwrap() { + w.replies.release(&w.page).unwrap(); + let step = format!("read #{:x}", plain(tag)); + match w.inflight.answer(tag) { + Ok(request) => { + if w.sent[request] != Some(w.session) { + let sent = w.sent[request]; + return fail(run, format!("request {request} of session {sent:?} answered in {}", w.session)); + } + w.answers[request].push(Answer::Read { session: w.session }); + if w.answers[request].len() > 1 { + return fail(run, format!("request {request} answered {:?}", w.answers[request])); + } + } + Err(_) => { + run.refused += 1; + w.end(); + } + } + moved = true; + go(run, step, w); + } + } + + // It notices the hang-up. + if world.hung_up { + let mut w = world.clone(); + w.end(); + moved = true; + go(run, "notice".into(), w); + } + + // It reconnects over the same words, as a new session. + if !world.up() { + let mut w = world.clone(); + w.session += 1; + w.connect(); + moved = true; + go(run, "reconnect".into(), w); + } + + if !moved { + run.ends += 1; + for (request, answers) in world.answers.iter().enumerate() { + if answers.len() != 1 { + return fail(run, format!("request {request} ended answered {answers:?}")); + } + } + } +} + +/// Explore every run with at most `crashes` deaths and `replays` replays. +pub fn explore(crashes: u8, replays: u8) -> Explored { + let mut run = Run { seen: HashSet::new(), broken: None, ends: 0, refused: 0, path: Vec::new() }; + dfs(&mut run, World::new(crashes, replays)); + Explored { broken: run.broken, states: run.seen.len(), ends: run.ends, refused: run.refused } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// What the session is held under: no failure, a death, a replay, and + /// both twice. + const BOUNDS: [(u8, u8); 4] = [(0, 0), (1, 0), (0, 1), (2, 2)]; + + #[test] + fn every_tag_is_answered_exactly_once() { + for (crashes, replays) in BOUNDS { + let explored = explore(crashes, replays); + std::println!( + "{crashes} crashes, {replays} replays: {} states, {} end states, {} answers refused", + explored.states, + explored.ends, + explored.refused + ); + assert!(explored.ends > 0, "the model reached no end"); + if let Some(why) = explored.broken { + panic!("{crashes} crashes, {replays} replays: {why}"); + } + } + } + + /// The model is not vacuous: a replay reaches the client and is refused. + #[test] + fn the_model_reaches_a_replay_refused() { + let explored = explore(0, 1); + assert_eq!(explored.broken, None); + assert!(explored.refused > 0, "no replay reached the client"); + } +} diff --git a/toyos-transport/src/queue.rs b/toyos-transport/src/queue.rs new file mode 100644 index 00000000000..7d0a458ebbb --- /dev/null +++ b/toyos-transport/src/queue.rs @@ -0,0 +1,271 @@ +//! A single-producer queue of `E`-word entries, `D` deep. +//! +//! **A producer writes an entry's words, then publishes the tail with +//! `Release`; a consumer loads the tail with `Acquire`, then reads the words.** +//! The head goes back the other way. Each end looks at the peer's cursor when +//! what it last saw is spent, not once per entry, and stores its own once per +//! batch. + +use core::sync::atomic::Ordering; + +use crate::{word, Asleep, Cursors, Untrusted, Violation, Wake, Word}; + +/// Where one queue is in a region, in words: its cursors, and its first entry. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Place { + pub cursors: Cursors, + pub entries: usize, +} + +impl Place { + /// The words of the entry at ring position `at`. + fn entry<'a, W, const E: usize, const D: u32>(&self, page: &'a [W], at: u32) -> Result<&'a [W], Violation> { + let first = usize::try_from(at & D.wrapping_sub(1)) + .ok() + .and_then(|slot| slot.checked_mul(E)) + .and_then(|offset| offset.checked_add(self.entries)); + first.and_then(|first| page.get(first..first.checked_add(E)?)).ok_or(Violation::Region) + } + + /// Every word the queue uses is in `page`. + fn check(&self, page: &[W]) -> Result<(), Violation> { + const { assert!(D.is_power_of_two() && E > 0, "a queue is a power of two deep, of entries of a word or more") }; + word(page, self.cursors.head)?; + word(page, self.cursors.tail)?; + word(page, self.cursors.sleep)?; + self.entry::(page, D.wrapping_sub(1)).map(|_| ()) + } +} + +/// The end of a queue that writes entries. It holds its cursors and not the +/// region; every call is given the region's words. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Producer { + place: Place, + local: u32, + published: u32, + /// Entries that may be pushed before the head is looked at again. + room: u32, +} + +impl Producer { + /// This end of the queue at `place`, its tail stored 0. + pub fn new(page: &[W], place: Place) -> Result { + place.check::(page)?; + word(page, place.cursors.tail)?.store(0, Ordering::Release); + Ok(Self { place, local: 0, published: 0, room: D }) + } + + /// How many entries may be pushed before the consumer frees more. + pub fn space(&mut self, page: &[W]) -> Result { + let unreleased = self.place.cursors.unreleased(page, self.published, D)?; + let pending = self.local.wrapping_sub(self.published); + self.room = D.saturating_sub(pending.saturating_add(unreleased)); + Ok(self.room) + } + + /// Write one entry, or answer `false` for a full queue and write nothing. + /// It is the consumer's once [`Self::publish`] runs. + pub fn push(&mut self, page: &[W], words: [u32; E]) -> Result { + if self.room == 0 && self.space(page)? == 0 { + return Ok(false); + } + for (shared, value) in self.place.entry::(page, self.local)?.iter().zip(words) { + shared.store(value, Ordering::Relaxed); + } + self.local = self.local.wrapping_add(1); + self.room = self.room.wrapping_sub(1); + Ok(true) + } + + /// Publish every entry pushed so far; `None` if there was none. + pub fn publish(&mut self, page: &[W]) -> Result, Violation> { + if self.published == self.local { + return Ok(None); + } + let wake = self.place.cursors.publish(page, self.local)?; + self.published = self.local; + Ok(Some(wake)) + } +} + +/// The end of a queue that reads entries; like [`Producer`], it holds cursors +/// and is given the region's words. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Consumer { + place: Place, + local: u32, + released: u32, + /// Entries published and not yet popped, as last seen. + ready: u32, + /// This end stored its `sleep` word and has not cleared it. + asleep: bool, +} + +impl Consumer { + /// This end of the queue at `place`, its head and `sleep` stored 0. + pub fn new(page: &[W], place: Place) -> Result { + place.check::(page)?; + word(page, place.cursors.head)?.store(0, Ordering::Release); + place.cursors.awake(page)?; + Ok(Self { place, local: 0, released: 0, ready: 0, asleep: false }) + } + + /// Look at the tail again, and answer it; what of it is not popped is + /// `ready`. + fn observe(&mut self, page: &[W]) -> Result { + let published = self.place.cursors.published(page, self.released, D)?; + self.ready = published.saturating_sub(self.local.wrapping_sub(self.released)); + Ok(self.released.wrapping_add(published)) + } + + /// The next published entry, or `None` for none. + pub fn pop(&mut self, page: &[W]) -> Result; E]>, Violation> { + if self.ready == 0 { + if self.asleep { + self.place.cursors.awake(page)?; + self.asleep = false; + } + self.observe(page)?; + if self.ready == 0 { + return Ok(None); + } + } + let mut words = [Untrusted::new(0); E]; + for (out, shared) in words.iter_mut().zip(self.place.entry::(page, self.local)?) { + *out = Untrusted::new(shared.load(Ordering::Relaxed)); + } + self.local = self.local.wrapping_add(1); + self.ready = self.ready.wrapping_sub(1); + Ok(Some(words)) + } + + /// Give every entry popped so far back to the producer. + pub fn release(&mut self, page: &[W]) -> Result<(), Violation> { + if self.released != self.local { + word(page, self.place.cursors.head)?.store(self.local, Ordering::Release); + self.released = self.local; + } + Ok(()) + } + + /// Say this end sleeps, and look once more: `None` if an entry is there + /// after all, or where to wait. A producer that publishes after this is + /// answered [`Wake::Peer`]; the next [`Self::pop`] says this end is awake. + pub fn before_sleep(&mut self, page: &[W]) -> Result, Violation> { + if self.ready > 0 { + return Ok(None); + } + self.place.cursors.sleep(page)?; + self.asleep = true; + let tail = self.observe(page)?; + if self.ready > 0 { + self.place.cursors.awake(page)?; + self.asleep = false; + return Ok(None); + } + Ok(Some(Asleep { word: self.place.cursors.tail, value: tail })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use core::sync::atomic::AtomicU32; + + const D: u32 = 8; + const PLACE: Place = Place { cursors: Cursors { head: 0, tail: 16, sleep: 1 }, entries: 32 }; + + fn page() -> Vec { + (0..32 + 2 * D as usize).map(|_| AtomicU32::new(0)).collect() + } + + fn ends(page: &[AtomicU32]) -> (Producer<2, D>, Consumer<2, D>) { + (Producer::new(page, PLACE).unwrap(), Consumer::new(page, PLACE).unwrap()) + } + + fn plain(words: Option<[Untrusted; 2]>) -> Option<[u32; 2]> { + words.map(|w| w.map(|w| w.at_most(u32::MAX.into()).unwrap() as u32)) + } + + #[test] + fn an_entry_is_nobodys_before_it_is_published() { + let page = page(); + let (mut tx, mut rx) = ends(&page); + assert_eq!(tx.push(&page, [3, 4]), Ok(true)); + assert_eq!(rx.pop(&page).map(plain), Ok(None)); + assert_eq!(tx.publish(&page), Ok(Some(Wake::Busy))); + assert_eq!(tx.publish(&page), Ok(None), "nothing new, nothing published"); + assert_eq!(rx.pop(&page).map(plain), Ok(Some([3, 4]))); + assert_eq!(rx.pop(&page).map(plain), Ok(None)); + } + + /// The ring wraps many times over, and space is exactly what the consumer + /// has given back. + #[test] + fn the_ring_wraps_and_counts_its_space() { + let page = page(); + let (mut tx, mut rx) = ends(&page); + let (mut pushed, mut popped) = (0u32, 0u32); + for round in 0..5 * D { + for _ in 0..1 + round % 9 { + assert_eq!(tx.space(&page), Ok(D - (pushed - popped))); + if !tx.push(&page, [pushed, !pushed]).unwrap() { + assert_eq!(pushed - popped, D); + break; + } + pushed += 1; + } + let _ = tx.publish(&page).unwrap(); + for _ in 0..1 + round % 7 { + let Some(words) = plain(rx.pop(&page).unwrap()) else { break }; + assert_eq!(words, [popped, !popped], "entries come out whole, in the order they went in"); + popped += 1; + } + rx.release(&page).unwrap(); + } + assert!(pushed > 2 * D, "the ring wrapped"); + } + + /// A tail more than the ring past what was released, and a head past what + /// was published, each end the session by name; a cursor moved backwards + /// within bounds costs only its owner. + #[test] + fn a_peer_cursor_out_of_reach_is_a_violation() { + let page = page(); + let (mut tx, mut rx) = ends(&page); + page[16].store(D + 1, Ordering::Release); + assert_eq!(rx.pop(&page), Err(Violation::TailPastDepth)); + page[16].store(u32::MAX, Ordering::Release); + assert_eq!(rx.pop(&page), Err(Violation::TailPastDepth), "a tail behind what was released"); + page[0].store(1, Ordering::Release); + assert_eq!(tx.space(&page), Err(Violation::HeadPastTail)); + page[0].store(u32::MAX, Ordering::Release); + assert_eq!(tx.space(&page), Ok(D - 1), "a head moved back costs its consumer the room"); + } + + #[test] + fn a_place_outside_the_words_is_refused() { + let page = page(); + let short = &page[..32 + 2 * D as usize - 1]; + assert_eq!(Producer::<2, D>::new(short, PLACE).err(), Some(Violation::Region)); + assert_eq!(Consumer::<2, D>::new(short, PLACE).err(), Some(Violation::Region)); + } + + /// The wake's two halves on one thread: a consumer that said it sleeps is + /// woken by the next publish and by no later one once it has popped. + #[test] + fn a_sleeper_is_woken_once_and_a_busy_one_never() { + let page = page(); + let (mut tx, mut rx) = ends(&page); + assert_eq!(rx.before_sleep(&page), Ok(Some(Asleep { word: 16, value: 0 }))); + tx.push(&page, [1, 1]).unwrap(); + assert_eq!(tx.publish(&page), Ok(Some(Wake::Peer))); + assert!(rx.pop(&page).unwrap().is_some()); + assert_eq!(rx.pop(&page), Ok(None), "the pop that found nothing said this end is awake"); + tx.push(&page, [2, 2]).unwrap(); + assert_eq!(tx.publish(&page), Ok(Some(Wake::Busy))); + assert_eq!(rx.before_sleep(&page), Ok(None), "an entry was there after all"); + assert_eq!(tx.publish(&page), Ok(None)); + } +} diff --git a/toyos-transport/src/stream.rs b/toyos-transport/src/stream.rs new file mode 100644 index 00000000000..a122aa20e14 --- /dev/null +++ b/toyos-transport/src/stream.rs @@ -0,0 +1,283 @@ +//! A single-producer byte ring with free-running cursors, and the producer's +//! `end` word. +//! +//! **This crate never touches a byte.** Each end answers the [`Span`]s of the +//! region the adapter copies into or out of, once, and the cursors publish +//! them as a queue's do. The ring's capacity is a power of two, so a cursor +//! wrapping at 2³² lands on the same byte. +//! +//! **The end is stored after the tail and loaded before it**, so a reader that +//! sees [`End::Fin`] and no bytes has seen the stream's last byte. + +use core::sync::atomic::Ordering; + +use crate::{word, Asleep, Cursors, Span, Untrusted, Violation, Wake, Word}; + +/// Where one stream is: its cursors, its `end` word, and its bytes — a fixed +/// span of the region or an arena run's ([`crate::Run::span`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct StreamPlace { + pub cursors: Cursors, + pub end: usize, + pub data: Span, +} + +impl StreamPlace { + /// The capacity, once every word is in `page` and the bytes are a power of + /// two. + fn check(&self, page: &[W]) -> Result { + for at in [self.cursors.head, self.cursors.tail, self.cursors.sleep, self.end] { + word(page, at)?; + } + u32::try_from(self.data.len).ok().filter(|cap| cap.is_power_of_two()).ok_or(Violation::Region) + } + + /// The bytes from cursor `at`, `len` long, where they are: past the ring's + /// end they go on from its start. + fn spans(&self, cap: u32, at: u32, len: u32) -> Result<[Span; 2], Violation> { + let from = at & cap.wrapping_sub(1); + let first = len.min(cap.wrapping_sub(from)); + let bytes = |n: u32| usize::try_from(n).map_err(|_| Violation::Region); + let offset = self.data.offset.checked_add(bytes(from)?).ok_or(Violation::Region)?; + Ok([ + Span { offset, len: bytes(first)? }, + Span { offset: self.data.offset, len: bytes(len.wrapping_sub(first))? }, + ]) + } +} + +/// What the producer has said of the bytes after the last it published. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum End { + /// More may follow. + Open, + /// None follow: the stream ended whole. + Fin, + /// None follow, and what was not read is abandoned. + Reset, +} + +impl End { + const fn word(self) -> u32 { + match self { + Self::Open => 0, + Self::Fin => 1, + Self::Reset => 2, + } + } + + fn decode(word: Untrusted) -> Result { + [Self::Open, Self::Fin, Self::Reset].into_iter().find(|end| word.is(end.word())).ok_or(Violation::End) + } +} + +/// The end of a stream that writes bytes. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct StreamTx { + place: StreamPlace, + cap: u32, + local: u32, + published: u32, +} + +impl StreamTx { + /// This end of the stream at `place`, its tail stored 0 and its end open. + pub fn new(page: &[W], place: StreamPlace) -> Result { + let cap = place.check(page)?; + word(page, place.cursors.tail)?.store(0, Ordering::Release); + word(page, place.end)?.store(End::Open.word(), Ordering::Release); + Ok(Self { place, cap, local: 0, published: 0 }) + } + + /// Room for at most `want` bytes, taken now: the caller fills the spans, + /// then publishes. + pub fn write(&mut self, page: &[W], want: u32) -> Result<[Span; 2], Violation> { + let unreleased = self.place.cursors.unreleased(page, self.published, self.cap)?; + let pending = self.local.wrapping_sub(self.published); + let len = want.min(self.cap.saturating_sub(pending.saturating_add(unreleased))); + let spans = self.place.spans(self.cap, self.local, len)?; + self.local = self.local.wrapping_add(len); + Ok(spans) + } + + /// Publish every byte written so far; `None` if there was none. + pub fn publish(&mut self, page: &[W]) -> Result, Violation> { + if self.published == self.local { + return Ok(None); + } + let wake = self.place.cursors.publish(page, self.local)?; + self.published = self.local; + Ok(Some(wake)) + } + + /// Publish what was written and say what follows it. + pub fn close(&mut self, page: &[W], end: End) -> Result { + word(page, self.place.cursors.tail)?.store(self.local, Ordering::Release); + self.published = self.local; + word(page, self.place.end)?.store(end.word(), Ordering::Release); + self.place.cursors.wake(page) + } +} + +/// The end of a stream that reads bytes. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct StreamRx { + place: StreamPlace, + cap: u32, + local: u32, + released: u32, + asleep: bool, +} + +impl StreamRx { + /// This end of the stream at `place`, its head and `sleep` stored 0. + pub fn new(page: &[W], place: StreamPlace) -> Result { + let cap = place.check(page)?; + word(page, place.cursors.head)?.store(0, Ordering::Release); + place.cursors.awake(page)?; + Ok(Self { place, cap, local: 0, released: 0, asleep: false }) + } + + /// What was said of the end, then how many bytes are ready past what was + /// read, and the tail they end at. + fn observe(&self, page: &[W]) -> Result<(End, u32, u32), Violation> { + let end = End::decode(Untrusted::new(word(page, self.place.end)?.load(Ordering::Acquire)))?; + let published = self.place.cursors.published(page, self.released, self.cap)?; + let ready = published.saturating_sub(self.local.wrapping_sub(self.released)); + Ok((end, ready, self.released.wrapping_add(published))) + } + + /// At most `want` bytes, taken now, and what follows them: the caller + /// copies the spans out, then releases. + pub fn read(&mut self, page: &[W], want: u32) -> Result<([Span; 2], End), Violation> { + if self.asleep { + self.place.cursors.awake(page)?; + self.asleep = false; + } + let (end, ready, _) = self.observe(page)?; + let len = want.min(ready); + let spans = self.place.spans(self.cap, self.local, len)?; + self.local = self.local.wrapping_add(len); + Ok((spans, end)) + } + + /// Give every byte read so far back to the producer. + pub fn release(&mut self, page: &[W]) -> Result<(), Violation> { + if self.released != self.local { + word(page, self.place.cursors.head)?.store(self.local, Ordering::Release); + self.released = self.local; + } + Ok(()) + } + + /// As [`crate::Consumer::before_sleep`]; an end said is something to read. + pub fn before_sleep(&mut self, page: &[W]) -> Result, Violation> { + self.place.cursors.sleep(page)?; + self.asleep = true; + let (end, ready, tail) = self.observe(page)?; + if ready > 0 || end != End::Open { + self.place.cursors.awake(page)?; + self.asleep = false; + return Ok(None); + } + Ok(Some(Asleep { word: self.place.cursors.tail, value: tail })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use core::sync::atomic::AtomicU32; + + const PLACE: StreamPlace = StreamPlace { + cursors: Cursors { head: 0, tail: 16, sleep: 1 }, + end: 17, + data: Span { offset: 4096, len: 16 }, + }; + + fn page() -> Vec { + (0..32).map(|_| AtomicU32::new(0)).collect() + } + + /// The bytes behind the spans, as the adapter holds them. + fn copy(ring: &mut [u8; 16], spans: [Span; 2], bytes: &[u8], into: bool) -> Vec { + let mut out = Vec::new(); + let mut at = 0; + for span in spans { + let ring = &mut ring[span.offset - 4096..span.offset - 4096 + span.len]; + if into { + ring.copy_from_slice(&bytes[at..at + span.len]); + } + out.extend_from_slice(ring); + at += span.len; + } + out + } + + /// Bytes cross whole and in order as the cursors wrap the ring, a write is + /// never given more room than was released, and `Fin` arrives after the + /// last byte. + #[test] + fn bytes_cross_whole_around_the_ring_and_end_after_the_last() { + let page = page(); + let mut ring = [0u8; 16]; + let mut tx = StreamTx::new(&page, PLACE).unwrap(); + let mut rx = StreamRx::new(&page, PLACE).unwrap(); + let (mut sent, mut got) = (Vec::new(), Vec::new()); + for n in 0..40u8 { + let chunk: Vec = (0..n % 11).map(|i| n.wrapping_mul(31).wrapping_add(i)).collect(); + let spans = tx.write(&page, chunk.len() as u32).unwrap(); + let len: usize = spans.iter().map(|s| s.len).sum(); + assert!(len <= 16 - (sent.len() - got.len()), "room past what was released"); + sent.extend_from_slice(©(&mut ring, spans, &chunk[..len], true)); + let _ = tx.publish(&page).unwrap(); + let (spans, end) = rx.read(&page, u32::from(n % 5)).unwrap(); + assert_eq!(end, End::Open); + got.extend(copy(&mut ring, spans, &[], false)); + rx.release(&page).unwrap(); + } + assert_eq!(tx.close(&page, End::Fin), Ok(Wake::Busy)); + loop { + let (spans, end) = rx.read(&page, 16).unwrap(); + let chunk = copy(&mut ring, spans, &[], false); + got.extend_from_slice(&chunk); + rx.release(&page).unwrap(); + if chunk.is_empty() { + assert_eq!(end, End::Fin); + break; + } + } + assert_eq!(got, sent); + assert!(sent.len() > 64, "the ring wrapped"); + } + + #[test] + fn an_end_word_no_producer_writes_is_a_violation() { + let page = page(); + let mut rx = StreamRx::new(&page, PLACE).unwrap(); + page[17].store(3, Ordering::Release); + assert_eq!(rx.read(&page, 1), Err(Violation::End)); + page[17].store(0, Ordering::Release); + page[16].store(17, Ordering::Release); + assert_eq!(rx.read(&page, 1), Err(Violation::TailPastDepth)); + } + + #[test] + fn a_ring_that_is_not_a_power_of_two_is_refused() { + let page = page(); + let place = StreamPlace { data: Span { offset: 4096, len: 24 }, ..PLACE }; + assert_eq!(StreamTx::new(&page, place).err(), Some(Violation::Region)); + } + + /// A reader that said it sleeps is woken by the close, and one that looks + /// after a close does not sleep. + #[test] + fn a_close_wakes_a_sleeper_and_keeps_the_next_awake() { + let page = page(); + let mut tx = StreamTx::new(&page, PLACE).unwrap(); + let mut rx = StreamRx::new(&page, PLACE).unwrap(); + assert_eq!(rx.before_sleep(&page), Ok(Some(Asleep { word: 16, value: 0 }))); + assert_eq!(tx.close(&page, End::Reset), Ok(Wake::Peer)); + assert_eq!(rx.before_sleep(&page), Ok(None)); + } +} diff --git a/toyos-transport/tests/loom.rs b/toyos-transport/tests/loom.rs new file mode 100644 index 00000000000..00fca81d788 --- /dev/null +++ b/toyos-transport/tests/loom.rs @@ -0,0 +1,175 @@ +//! The transport's two ends on two CPUs, under loom: what no host test that +//! runs both ends on one thread can reach. +//! +//! - **Publication**: a consumer that sees a tail sees every word of the +//! entries below it. `publish-relaxed` takes the edge away. +//! - **No lost wake**: a consumer that says it sleeps and a producer that +//! publishes cannot both miss the other; the futex is park and unpark, and +//! a lost wake is a consumer parked for good. `no-sleep-fence` takes the +//! fences away. +//! - **A hostile peer** that stores to every word it can reach, at every step, +//! leaves the honest end with entries, nothing, or a named violation, and +//! never more entries than the ring holds. `no-clamp` believes the peer. +//! +//! cargo test -p toyos-transport --features --test loom + +use core::sync::atomic::Ordering; + +use loom::sync::atomic::{fence, AtomicU32}; +use loom::sync::Arc; +use toyos_transport::{Consumer, Cursors, Place, Producer, Untrusted, Violation, Wake, Word}; + +/// A loom atomic as a region word: the trait is this crate's and the type is +/// loom's, so the two meet through a wrapper. +struct Shared(AtomicU32); + +impl Word for Shared { + fn load(&self, order: Ordering) -> u32 { + self.0.load(order) + } + fn store(&self, value: u32, order: Ordering) { + self.0.store(value, order) + } + fn fence() { + fence(Ordering::SeqCst) + } +} + +const E: usize = 2; +const D: u32 = 2; +const PLACE: Place = Place { cursors: Cursors { head: 0, tail: 1, sleep: 2 }, entries: 3 }; +const WORDS: usize = 3 + E * D as usize; + +fn page() -> Arc> { + Arc::new((0..WORDS).map(|_| Shared(AtomicU32::new(0))).collect()) +} + +fn ends(page: &[Shared]) -> (Producer, Consumer) { + (Producer::new(page, PLACE).unwrap(), Consumer::new(page, PLACE).unwrap()) +} + +fn entry(n: u32) -> [u32; E] { + [100 + n, 7 * n + 3] +} + +fn plain(words: [Untrusted; E]) -> [u32; E] { + words.map(|w| w.at_most(u32::MAX.into()).unwrap() as u32) +} + +/// Two entries published one at a time, read by the other end as they +/// arrive: each is whole, in order, and exactly what was written. +#[test] +fn a_published_entry_is_read_whole() { + loom::model(|| { + let page = page(); + let (mut tx, mut rx) = ends(&page); + let consumer_page = Arc::clone(&page); + let consumer = loom::thread::spawn(move || { + let mut read = Vec::new(); + while read.len() < 2 { + match rx.pop(&consumer_page).expect("the producer keeps the protocol") { + Some(words) => read.push(plain(words)), + None => loom::thread::yield_now(), + } + } + rx.release(&consumer_page).unwrap(); + read + }); + for n in 0..2 { + assert!(tx.push(&page, entry(n)).unwrap()); + let _ = tx.publish(&page).unwrap(); + } + let read = consumer.join().expect("the consumer thread"); + assert_eq!(read, [entry(0), entry(1)], "an entry was read before its words"); + }); +} + +/// A consumer that finds nothing sleeps as a futex does — parked while the +/// tail still holds what it saw — and is woken only when a publish answers +/// [`Wake::Peer`]. Whatever the schedule, it gets the entry. +#[test] +fn a_publish_and_a_sleep_cannot_both_miss() { + loom::model(|| { + let page = page(); + let (mut tx, mut rx) = ends(&page); + let consumer_page = Arc::clone(&page); + let consumer = loom::thread::spawn(move || loop { + if let Some(words) = rx.pop(&consumer_page).unwrap() { + return plain(words); + } + if let Some(asleep) = rx.before_sleep(&consumer_page).unwrap() { + if consumer_page[asleep.word].load(Ordering::Relaxed) == asleep.value { + loom::thread::park(); + } + } + }); + assert!(tx.push(&page, entry(1)).unwrap()); + if tx.publish(&page).unwrap() == Some(Wake::Peer) { + consumer.thread().unpark(); + } + assert_eq!(consumer.join().expect("the consumer parked over a published entry"), entry(1)); + }); +} + +/// A producer that stores a tail past the ring, one behind what was released, +/// and garbage into the entries and into the consumer's own head and `sleep`: +/// every pop is an entry, nothing, or [`Violation::TailPastDepth`], and no +/// more than the ring's depth of entries is taken without a release. +#[test] +fn a_hostile_producer_yields_entries_or_a_violation() { + loom::model(|| { + let page = page(); + let (_, mut rx) = ends(&page); + let hostile_page = Arc::clone(&page); + let hostile = loom::thread::spawn(move || { + for (at, value) in [(1, D + 1), (3, 9), (0, 5), (1, u32::MAX), (2, 7)] { + hostile_page[at].store(value, Ordering::Release); + } + }); + let mut taken = 0; + for _ in 0..D + 2 { + match rx.pop(&page) { + Ok(Some(_)) => taken += 1, + Ok(None) => {} + Err(violation) => { + assert_eq!(violation, Violation::TailPastDepth); + break; + } + } + } + hostile.join().unwrap(); + assert!(taken <= D, "took {taken} entries from a ring of {D} without releasing one"); + }); +} + +/// A consumer that stores a head past what was published, one wrapped far +/// behind, and garbage into the producer's own tail and the entries: every +/// push is room, a full ring, or [`Violation::HeadPastTail`], and no more +/// than the ring's depth is ever pushed ahead of a head that was never moved. +#[test] +fn a_hostile_consumer_yields_room_or_a_violation() { + loom::model(|| { + let page = page(); + let (mut tx, _) = ends(&page); + let hostile_page = Arc::clone(&page); + let hostile = loom::thread::spawn(move || { + for (at, value) in [(0, 5), (1, 9), (0, u32::MAX), (2, 1), (4, 6)] { + hostile_page[at].store(value, Ordering::Release); + } + }); + let mut pushed = 0; + for n in 0..D + 2 { + match tx.push(&page, entry(n)) { + Ok(true) => pushed += 1, + Ok(false) => {} + Err(violation) => { + assert_eq!(violation, Violation::HeadPastTail); + break; + } + } + let _ = tx.publish(&page).unwrap(); + } + hostile.join().unwrap(); + assert!(pushed <= D, "pushed {pushed} into a ring of {D} nobody released"); + }); +} diff --git a/toyos-untrusted/Cargo.toml b/toyos-untrusted/Cargo.toml index 83e3dc98028..3ded1fbbf7f 100644 --- a/toyos-untrusted/Cargo.toml +++ b/toyos-untrusted/Cargo.toml @@ -15,3 +15,10 @@ version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" publish = false + +[features] +# `toyos-transport` sits under std, and so does what it bounds with. +rustc-dep-of-std = ["core"] + +[dependencies] +core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } diff --git a/userland/Cargo.lock b/userland/Cargo.lock index a1147e1c2b2..30245ea0bae 100644 --- a/userland/Cargo.lock +++ b/userland/Cargo.lock @@ -4058,6 +4058,7 @@ name = "toyos-blockring" version = "0.1.0" dependencies = [ "toyos-blockhold", + "toyos-transport", ] [[package]] @@ -4150,6 +4151,17 @@ version = "0.1.0" name = "toyos-tmpdir" version = "0.1.0" +[[package]] +name = "toyos-transport" +version = "0.1.0" +dependencies = [ + "toyos-untrusted", +] + +[[package]] +name = "toyos-untrusted" +version = "0.1.0" + [[package]] name = "toyos-update" version = "0.1.0" diff --git a/userland/blockd/src/main.rs b/userland/blockd/src/main.rs index c6a18a93192..5a671cc1671 100644 --- a/userland/blockd/src/main.rs +++ b/userland/blockd/src/main.rs @@ -47,8 +47,7 @@ use toyos_abi::part::{PartGuid, GUID_TEXT_LEN}; use toyos_abi::syscall::{DEV_PREFIX, SyscallError}; use toyos_blockhold::Holds; use toyos_blockring::entry::{Completion, Op}; -use toyos_blockring::layout::{arena_byte, DEPTH}; -use toyos_blockring::ring::{self, ServerRings}; +use toyos_blockring::layout::{self, arena_byte, ServerRings, DEPTH}; use toyos_blockring::server::{ServerSession, Taken}; use toyos_blockring::wire::{self, Opened, Refusal}; use toyos_blockring::{BLOCK_BYTES, PORT, SESSION_BYTES}; @@ -249,7 +248,7 @@ impl Service { fn admit(&mut self, opening: Opening, conn: Connection) -> u64 { let id = self.next_id; self.next_id += 1; - let rings = ring::server(opening.region.words()); + let rings = layout::server(opening.region.words()).expect("blockd: the region holds every ring word"); self.sessions.insert( id, Served { @@ -276,11 +275,14 @@ impl Service { self.holds.release(opening.first); } - /// Put `c` on its session's completion ring. + /// Put `c` on its session's completion ring. [`Self::pull`] leaves room for + /// every answer, so a ring without it is a client whose head went back or + /// past what was posted: the session ends. fn post(session: &mut Served, c: Completion) { - let page = session.region.words(); - session.rings.1.push(page, c.encode()); - session.posted = true; + match session.rings.1.push(session.region.words(), c.encode()) { + Ok(true) => session.posted = true, + Ok(false) | Err(_) => session.closing = true, + } } /// Hand one device answer to the session it is for. @@ -307,11 +309,11 @@ impl Service { /// session's completion ring have room for it. fn pull(&mut self, id: u64) { let s = self.sessions.get_mut(&id).expect("a live session"); - let page = s.region.words(); loop { if s.closing { break; } + let page = s.region.words(); // Room for every answer: what is on the device, and what is posted // and not yet read, never exceeds the completion ring. let Ok(space) = s.rings.1.space(page) else { @@ -331,11 +333,8 @@ impl Service { } }; s.requests += 1; - match s.state.take(words) { - Taken::Answer(c) => { - s.rings.1.push(page, c.encode()); - s.posted = true; - } + match s.state.take_entry(words) { + Taken::Answer(c) => Self::post(s, c), Taken::Issue(req) => { let owner = Owner::Session { session: id, tag: req.tag, write: req.op == Op::Write }; match req.op { @@ -349,14 +348,13 @@ impl Service { // medium already. Op::Flush => { let c = s.state.complete(req.tag, true, &mut self.holds, self.losses); - s.rings.1.push(page, c.expect("just taken").encode()); - s.posted = true; + Self::post(s, c.expect("just taken")); } } } } } - s.rings.0.release(page); + s.rings.0.release(s.region.words()).expect("blockd: the region holds every ring word"); } /// Publish what each session was answered, and ring its doorbell. @@ -366,7 +364,7 @@ impl Service { continue; } s.posted = false; - if s.rings.1.publish(s.region.words()) { + if s.rings.1.publish(s.region.words()).expect("blockd: the region holds every ring word").is_some() { match s.conn.write_nonblock(&[1]) { Ok(_) | Err(SyscallError::WouldBlock) => {} Err(_) => s.closing = true, diff --git a/userland/blockd/src/session.rs b/userland/blockd/src/session.rs index 2a9aea1d4f3..62362ab3340 100644 --- a/userland/blockd/src/session.rs +++ b/userland/blockd/src/session.rs @@ -23,8 +23,7 @@ use toyos::poller::{Poller, READABLE}; use toyos_abi::syscall::SyscallError; use toyos_blockring::client::{Client, Outcome, Ticket}; use toyos_blockring::entry::{Completion, Op}; -use toyos_blockring::layout::{ARENA_BLOCKS, MAX_REQUEST_BLOCKS}; -use toyos_blockring::ring::{self, ClientRings}; +use toyos_blockring::layout::{self, ClientRings, ARENA_BLOCKS, MAX_REQUEST_BLOCKS}; use toyos_blockring::wire::{self, Opened, Refusal}; use toyos_blockring::BLOCK_BYTES; @@ -135,7 +134,7 @@ impl Session { /// `names` calls `service`. pub fn open(names: Namespace, service: &str, guid: [u8; wire::GUID_BYTES]) -> Result { let region = Region::create().map_err(Error::Kernel)?; - let rings = ring::client(region.words()); + let rings = layout::client(region.words()).expect("blockd: the region holds every ring word"); let (conn, opened) = handshake(&names, service, guid, ®ion)?; let mut client = Client::new(); client.session_started(); @@ -231,10 +230,11 @@ impl Session { Err(_) => break, } let Some(request) = self.client.next_request() else { break }; - self.rings.0.push(page, request.encode()); + let pushed = self.rings.0.push(page, request.encode()); + assert_eq!(pushed, Ok(true), "blockd: a request past the room just counted"); } self.peak = self.peak.max(self.client.on_the_wire()); - if self.rings.0.publish(page) { + if self.rings.0.publish(page).expect("blockd: the region holds every ring word").is_some() { // A full pipe is a doorbell already rung; a gone one is a server // that has ended, which the wait finds. let _ = conn.write_nonblock(&[1]); @@ -301,7 +301,7 @@ impl Session { } } } - self.rings.1.release(page); + self.rings.1.release(page).expect("blockd: the region holds every ring word"); violated } @@ -355,7 +355,7 @@ impl Session { assert!(self.conn.is_none(), "blockd: reconnect while a session is open"); // Before the region goes to the new server: it must find this end's // two indices at zero, as it will set its own. - self.rings = ring::client(self.region.words()); + self.rings = layout::client(self.region.words()).expect("blockd: the region holds every ring word"); let (conn, opened) = handshake(&self.names, &self.service, self.guid, &self.region)?; if opened != self.opened { return Err(Error::Protocol); From dc71e92c4f325b7c7e423685a32a0c60517d76ef Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 20:02:47 +0200 Subject: [PATCH 2/8] review #552: blockring decodes through the core, and the core holds only what it has a user for Blockers of review r1: - The stream is deleted (stream.rs, End, StreamPlace, Violation::End): nothing used it before T6/T9, and its close left the futex word unchanged, a lost wake. That defect is recorded for T6 in the transport design, not the tree. - The clamps have tests that can fail: a producer that steps its tail one past each pop with nothing released, and a consumer that steps its head onto each unpublished push. Each goes red under the reviewer's mutation (tail bounded against `local`, head bounded against `local` without `pending`). - blockd_io gains `hostile-head`: with a write on the device, the client moves CQ_HEAD to CQ_TAIL - DEPTH; blockd must end that session and serve the next. blockd_survives_its_death runs it first. - The conversion is finished here rather than in T2: - The arena is declared once, as `Geometry`/`Run`. `layout::ARENA` is the block geometry; SESSION_BYTES derives from `Geometry::BYTES`; ARENA_BLOCKS, ARENA_OFFSET and arena_byte are gone. A request carries the `Run` its words decode to, so blockd's device address and the client's arena window come from its span, and region.rs's own bound check goes. - The tag table is declared once: the client's wire is an `Inflight`, which gains `values()` for the two places the client looks across what is in flight. next_tag, the BTreeMap and the client's own Violation go; a tag table that is full holds the next request back. - Schema, entry::Block, Layout and Geometry::decode had no caller and go. - Own/Lent/Held protected nothing (anyone could mint an Own, and Lent::back took no completion) and go; `Run::decode` answers a bounded `Run`. - The test-only `take` shim goes and `take_entry` is `take` again. - blockring's model keeps its VecDeque queues as a reference and drives the session page's own rings beside them, holding every entry either end takes equal to the queue's. A pop that reads the wrong slot reds all four model tests. Notes: - The no-sleep-fence control is split: no-wake-fence takes the producer's fence away and no-sleep-fence the consumer's, each red on its own. The wake model no longer parks; the consumer reports whether it slept, so the verdict is the test's FAILED line and not loom's deadlock. - `opaque` fails fast instead of defaulting to 0 on a branch that cannot run. - The 16-bit tag sequence's wrap is stated at the site as the invariant it is: a replay a wrap later answers what a server could answer by name anyway. - rustc-dep-of-std leaves the transport and toyos-untrusted: nothing builds either under std before T2, which adds it with its user. - Two issues filed: a block publish pays a fence and a load for a wake nobody asks for, and the head's Release/Acquire has no oracle. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 4 - ...publish-pays-for-a-wake-nobody-asks-for.md | 19 ++ ...ransport-heads-orderings-have-no-oracle.md | 18 ++ src/build.rs | 5 +- src/ci.rs | 9 +- tests/common/blockd.rs | 10 +- tests/toyos-rust-tests/src/bin/blockd_io.rs | 86 +++++- toyos-blockring/src/client.rs | 127 ++++---- toyos-blockring/src/entry.rs | 104 +++---- toyos-blockring/src/layout.rs | 26 +- toyos-blockring/src/lib.rs | 11 +- toyos-blockring/src/model.rs | 162 ++++++++-- toyos-blockring/src/server.rs | 19 +- toyos-transport/Cargo.toml | 20 +- toyos-transport/src/arena.rs | 132 ++------ toyos-transport/src/inflight.rs | 13 +- toyos-transport/src/lib.rs | 43 +-- toyos-transport/src/queue.rs | 39 +++ toyos-transport/src/stream.rs | 283 ------------------ toyos-transport/tests/loom.rs | 43 +-- toyos-untrusted/Cargo.toml | 7 - userland/blockd/src/main.rs | 9 +- userland/blockd/src/region.rs | 14 +- userland/blockd/src/session.rs | 43 +-- 24 files changed, 536 insertions(+), 710 deletions(-) create mode 100644 issues/design-debt/a-block-publish-pays-for-a-wake-nobody-asks-for.md create mode 100644 issues/design-debt/the-transport-heads-orderings-have-no-oracle.md delete mode 100644 toyos-transport/src/stream.rs diff --git a/Cargo.lock b/Cargo.lock index cc17ada12f9..fd2b4377bc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1432,16 +1432,12 @@ name = "toyos-transport" version = "0.1.0" dependencies = [ "loom", - "rustc-std-workspace-core", "toyos-untrusted", ] [[package]] name = "toyos-untrusted" version = "0.1.0" -dependencies = [ - "rustc-std-workspace-core", -] [[package]] name = "toyos-update" diff --git a/issues/design-debt/a-block-publish-pays-for-a-wake-nobody-asks-for.md b/issues/design-debt/a-block-publish-pays-for-a-wake-nobody-asks-for.md new file mode 100644 index 00000000000..77b87e78a50 --- /dev/null +++ b/issues/design-debt/a-block-publish-pays-for-a-wake-nobody-asks-for.md @@ -0,0 +1,19 @@ +--- +status: open +kind: defect +opened: 2026-09-27 +--- + +# A block publish pays for a wake nobody asks for + +Every `Producer::publish` in `toyos-transport` stores its tail, runs a `SeqCst` +fence and loads the consumer's `sleep` word, so that it can answer +`Wake::Peer` or `Wake::Busy`. blockd (`userland/blockd/src/main.rs`, +`publish`) and its client (`userland/blockd/src/session.rs`, `pump`) ring the +connection on every publish whatever the answer, and neither end calls +`Consumer::before_sleep`, so the answer is always `Busy`: the fence and the +load are paid once per batch and buy nothing. Their cost has not been +measured. + +**Exit condition.** blockd and its client sleep through `before_sleep` and +ring their peer only on `Wake::Peer`. diff --git a/issues/design-debt/the-transport-heads-orderings-have-no-oracle.md b/issues/design-debt/the-transport-heads-orderings-have-no-oracle.md new file mode 100644 index 00000000000..29f06c94e32 --- /dev/null +++ b/issues/design-debt/the-transport-heads-orderings-have-no-oracle.md @@ -0,0 +1,18 @@ +--- +status: open +kind: tooling +opened: 2026-09-27 +--- + +# The transport head's orderings have no oracle + +A consumer stores its head `Release` after it has loaded the entries below it, +and a producer loads the head `Acquire` before it writes over them +(`toyos-transport/src/queue.rs`, `Consumer::release` and `Producer::space`). +No test reds if either is `Relaxed`: what the pair forbids is load buffering — +a consumer's load of an entry reading the producer's later overwrite — which +loom does not model, and every other test runs both ends on one thread. The +tail's edge has its control (`publish-relaxed`); the head's has none. + +**Exit condition.** A control that relaxes the head's two orderings, and a +model or a run on a weakly ordered CPU that goes red under it. diff --git a/src/build.rs b/src/build.rs index 2cc8c728f1f..f536a1e8336 100644 --- a/src/build.rs +++ b/src/build.rs @@ -2739,14 +2739,13 @@ mod tests { /// /// `loom` selects loom's instrumented atomics; `check`, `protocol-port`, /// `tripwire` and `std` mirror `toyos-sched`'s own features so the shared - /// sources compile identically and name nothing a model turns on; - /// `rustc-dep-of-std` builds a crate under std. Everything + /// sources compile identically and name nothing a model turns on. Everything /// else declared in any of these files is, by construction, a /// `--features ` command that must red a named model — each file's own /// comment beside the name carries the argument for why. fn declared_model_controls(root: &Path) -> Vec<(&'static str, String)> { const NOT_A_CONTROL: &[&str] = - &["loom", "check", "protocol-port", "tripwire", "std", "default", "rustc-dep-of-std"]; + &["loom", "check", "protocol-port", "tripwire", "std", "default"]; let mut out = Vec::new(); for (crate_name, manifest) in [ ("kernel-loom", "kernel-loom/Cargo.toml"), diff --git a/src/ci.rs b/src/ci.rs index 48375a597d8..c4ddf71b98a 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -368,14 +368,9 @@ pub(crate) const CONTROLS: &[Control] = &[ red(BLOCKRING, "mutate-no-reissue-after-loss", None, &[ "what_a_flush_calls_durable_is_on_the_medium ... FAILED", ]), - // The transport's four: a tail published before its entry, a wake both - // sides miss, a peer's cursor believed, and a session's tags outliving it. red(TRANSPORT, "publish-relaxed", Some("loom"), &["a_published_entry_is_read_whole ... FAILED"]), - // A lost wake is a consumer parked for good: loom's deadlock, whose unwind - // panics again before the harness prints a `FAILED` line. - red(TRANSPORT, "no-sleep-fence", Some("loom"), &[ - "deadlock; threads = [(Id(0), Blocked(Location(None))), (Id(1), Blocked(Location(None)))]", - ]), + red(TRANSPORT, "no-wake-fence", Some("loom"), &["a_publish_and_a_sleep_cannot_both_miss ... FAILED"]), + red(TRANSPORT, "no-sleep-fence", Some("loom"), &["a_publish_and_a_sleep_cannot_both_miss ... FAILED"]), red(TRANSPORT, "no-clamp", Some("loom"), &["a_hostile_producer_yields_entries_or_a_violation ... FAILED"]), red(TRANSPORT, "end-keeps-inflight", None, &["every_tag_is_answered_exactly_once ... FAILED"]), ]; diff --git a/tests/common/blockd.rs b/tests/common/blockd.rs index 10390e75b8f..40520cc34e3 100644 --- a/tests/common/blockd.rs +++ b/tests/common/blockd.rs @@ -432,8 +432,12 @@ pub fn blockd_serves_partitions( Ok(()) } -/// blockd's two failures, each survived by its client. +/// blockd's two failures, each survived by its client, and a client that +/// breaks the protocol, survived by blockd. /// +/// - `hostile-head`: with a write on the device, a client moves its completion +/// ring's head a ring behind blockd's tail; the answer finds no room, blockd +/// ends that session, and serves the next. /// - `reset`: blockd withholds its second write's answer; the silence ends in /// a controller reset; the withheld write is answered not done; and the /// write acknowledged before it, which the reset may have lost, is on the @@ -454,6 +458,7 @@ pub fn blockd_survives_its_death( rust_bins: &[(String, Vec)], ) -> Result<(), String> { let (mut qemu, layout, disk, trace, before) = boot(c_bins, rust_bins, "blockd-death", &[])?; + let hostile = role(&mut qemu, "hostile-head", Duration::from_secs(120))?; let reset = role(&mut qemu, "reset", Duration::from_secs(240))?; for want in [ "blockd: WITHHELD the device's answer to a write", @@ -477,7 +482,8 @@ pub fn blockd_survives_its_death( } let tail = partclaim::shut_down(qemu); partclaim::no_panic("on the way down", &tail)?; - let mut log = Serial::named("blockd_survives_its_death", format!("{}{}", reset.serial, crash.serial)); + let mut log = + Serial::named("blockd_survives_its_death", format!("{}{}{}", hostile.serial, reset.serial, crash.serial)); log.push(&tail); log.must_be_clean()?; diff --git a/tests/toyos-rust-tests/src/bin/blockd_io.rs b/tests/toyos-rust-tests/src/bin/blockd_io.rs index c80803d460c..b746acb6e64 100644 --- a/tests/toyos-rust-tests/src/bin/blockd_io.rs +++ b/tests/toyos-rust-tests/src/bin/blockd_io.rs @@ -15,6 +15,9 @@ //! was answered; //! - `bench` — the same bytes through the kernel's driver (a partition claim on //! the first controller) and through blockd, timed; +//! - `hostile-head` — a client that, with a write on the device, moves its +//! completion ring's head a ring behind blockd's tail: the session is +//! ended, and blockd serves the next one; //! - `reset` — blockd started withholding its second answer: the silence ends //! in a controller reset, the withheld write is answered not done, and the //! write acknowledged before it is on the medium after the next flush; @@ -32,9 +35,12 @@ use std::io::{BufRead, BufReader, Write}; use std::os::toyos::process::{ChildExt, CommandExt}; use std::process::{Child, Command, Stdio}; +use std::sync::atomic::Ordering; +use std::sync::mpsc::{self, Receiver}; use std::time::{Duration, Instant}; use blockd::nvme::{Controller, Owner}; +use blockd::region::Region; use blockd::{Error, Outcome, Session, Unsent}; use toyos::endow::Endowments; use toyos::poller::{Poller, READABLE}; @@ -45,6 +51,7 @@ use toyos::syscap::SysCap; use toyos::AsHandle; use toyos_abi::part::PartGuid; use toyos_abi::syscall::{DeviceType, PciId, SyscallError, DEV_PREFIX, SERVE_PREFIX, SYSCAP_LABEL}; +use toyos_blockring::layout::{CQ_HEAD, CQ_TAIL, DEPTH, SQ_BASE, SQ_TAIL}; use toyos_blockring::wire::{self, Refusal}; use toyos_blockring::{BLOCK_BYTES, MAX_REQUEST_BLOCKS, PORT}; @@ -91,6 +98,11 @@ const NARROW: u64 = 128 * 1024 * 1024; /// a liveness bound, far past what one block takes. const AIMED: Duration = Duration::from_secs(10); +/// How long `hostile-head` waits for blockd to withhold its write's answer, and +/// then for the reset that ends its session: a liveness bound past blockd's +/// ten seconds of silence. +const SILENCE_ENDS: Duration = Duration::from_secs(30); + fn guid(text: &str) -> [u8; 16] { PartGuid::parse(text).unwrap_or_else(|| panic!("{text} is no GUID")).0 } @@ -121,6 +133,8 @@ struct Blockd { acceptor: Acceptor, connector: Connector, child: Option, + /// Says once the running blockd has withheld a write's answer. + withheld: Option>, } impl Blockd { @@ -130,7 +144,7 @@ impl Blockd { fn with(syscap: SysCap, args: &[&str]) -> Self { let (acceptor, connector) = port::create().unwrap_or_else(|e| fail(format!("no port: {e:?}"))); - let mut blockd = Self { syscap, acceptor, connector, child: None }; + let mut blockd = Self { syscap, acceptor, connector, child: None, withheld: None }; blockd.spawn(args, false); blockd } @@ -164,6 +178,7 @@ impl Blockd { command.endow(&format!("{SERVE_PREFIX}{PORT}"), acceptor.0); let mut child = command.spawn().unwrap_or_else(|e| fail(format!("blockd did not start: {e}"))); let out = child.stdout.take().expect("piped"); + let (said, heard) = mpsc::channel(); let mut kill = kill_on_withheld.then(|| { toyos_abi::syscall::dup(toyos_abi::RawHandle(child.as_raw_handle())) .unwrap_or_else(|e| fail(format!("blockd's handle would not duplicate: {e:?}"))) @@ -172,6 +187,7 @@ impl Blockd { for line in BufReader::new(out).lines().map_while(Result::ok) { println!("{line}"); if line.contains("WITHHELD") { + let _ = said.send(()); if let Some(handle) = kill.take() { let _ = toyos_abi::syscall::process_kill(handle); println!("blockd_io: blockd killed with the withheld write done on the device"); @@ -180,6 +196,7 @@ impl Blockd { } }); self.child = Some(child); + self.withheld = Some(heard); } /// End the running blockd, if one is, and wait for it to be gone. @@ -497,6 +514,72 @@ fn reset() { println!("blockd_io: PASS reset"); } +/// A client whose write is on the device moves its completion ring's head a +/// ring's depth behind the tail blockd published, so the answer finds no room: +/// blockd ends that session, and serves the next. +fn hostile_head() { + let mut blockd = Blockd::start(&["--silence-write", "1"]); + let region = Region::create().unwrap_or_else(|e| fail(format!("a region: {e:?}"))); + let conn = blockd.names().open(PORT).unwrap_or_else(|e| fail(format!("the port: {e:?}"))); + let shared = region.share().unwrap_or_else(|e| fail(format!("a second handle: {e:?}"))); + conn.send_bytes_with_handles(&[shared], wire::MSG_OPEN, &guid(TARGET)) + .unwrap_or_else(|e| fail(format!("the open: {e:?}"))); + let header = conn.recv_header().unwrap_or_else(|e| fail(format!("the answer: {e:?}"))); + let mut payload = [0u8; 64]; + conn.recv_bytes(&header, &mut payload).unwrap_or_else(|e| fail(format!("the answer: {e:?}"))); + if header.msg_type != wire::MSG_OPENED { + fail(format!("the slot's open was answered {}", header.msg_type)); + } + // A write of the slot's block 0 from arena block 0 under tag 1, as the + // words a client puts on the request ring, published and rung. + let words = region.words(); + for (at, word) in [2, 1, 0, 0, 1, 0, 0, 0].into_iter().enumerate() { + words[SQ_BASE + at].store(word, Ordering::Relaxed); + } + words[SQ_TAIL].store(1, Ordering::Release); + conn.write_nonblock(&[1]).unwrap_or_else(|e| fail(format!("the doorbell: {e:?}"))); + let withheld = blockd.withheld.as_ref().expect("spawned"); + if withheld.recv_timeout(SILENCE_ENDS).is_err() { + fail(format!("blockd withheld no write's answer in {SILENCE_ENDS:?}")); + } + let tail = words[CQ_TAIL].load(Ordering::Acquire); + words[CQ_HEAD].store(tail.wrapping_sub(DEPTH), Ordering::Release); + conn.write_nonblock(&[1]).unwrap_or_else(|e| fail(format!("the doorbell: {e:?}"))); + println!("blockd_io: with a write on the device, the client moved its completion head {DEPTH} behind the tail"); + let poller = Poller::new(1); + let asked = Instant::now(); + loop { + match conn.read_nonblock(&mut [0u8; 8]) { + Ok(0) => break, + Err(SyscallError::WouldBlock) => {} + Ok(_) => fail("blockd rang a session whose answer had no room".into()), + Err(_) => break, + } + let Some(left) = SILENCE_ENDS.checked_sub(asked.elapsed()) else { + fail(format!("blockd did not end the session in {SILENCE_ENDS:?}")); + }; + poller.watch(&conn, READABLE, 0); + poller.wait(1, left.as_nanos() as u64, |_| {}); + } + match blockd.child.as_mut().expect("spawned").try_wait() { + Ok(None) => println!("blockd_io: blockd ended the session and runs on"), + other => fail(format!("blockd ended with the session: {other:?}")), + } + let mut next = open(blockd.names(), TARGET); + let block = pattern(0x6B, 0); + match next.write(0, &block) { + Ok(Outcome::Done) => {} + other => fail(format!("the next session's write was answered {other:?}")), + } + flushed(&mut next); + match next.read(0, 1) { + Ok((Outcome::Done, Some(data))) if data == block => {} + other => fail(format!("the next session read back {:?}", other.map(|(o, _)| o))), + } + println!("blockd_io: the next session wrote, flushed and read back the slot's block 0"); + println!("blockd_io: PASS hostile-head"); +} + /// The FAT32 volume's device: a session, with blockd's supervisor beside it. struct Volume { blockd: Blockd, @@ -964,6 +1047,7 @@ fn main() { Some("claims") => claims(), Some("holder") => holder_role(args.get(2).map_or("", String::as_str)), Some("bench") => bench(), + Some("hostile-head") => hostile_head(), Some("reset") => reset(), Some("crash") => crash(), Some(role @ ("dma-inside" | "dma-outside" | "dma-revoked" | "dma-after")) => dma(role), diff --git a/toyos-blockring/src/client.rs b/toyos-blockring/src/client.rs index 933665c7850..3ecce711e14 100644 --- a/toyos-blockring/src/client.rs +++ b/toyos-blockring/src/client.rs @@ -5,7 +5,7 @@ //! **Every request asked for is answered exactly once**, by [`Client::complete`] //! or by [`Client::session_ended`], and never twice: a completion for a tag //! that is not on the wire is the server breaking the session -//! ([`Violation`]), not a second answer. +//! ([`Violation::Tag`]), not a second answer. //! //! **An acknowledged write is kept until a flush covers it.** Its arena blocks //! stay pinned ([`Client::take_released`] is when they come back) because a @@ -36,7 +36,10 @@ use alloc::collections::{BTreeMap, VecDeque}; use alloc::vec::Vec; +use toyos_transport::{Inflight, Run, Untrusted, Violation}; + use crate::entry::{Completion, Op, Request, Status}; +use crate::layout::DEPTH; /// The caller's name for one thing it asked for. pub type Ticket = u64; @@ -65,17 +68,11 @@ pub enum Outcome { Refused, } -/// The server said something no server of this protocol says. The session is -/// over, and [`Client::session_ended`] is what the caller does next. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Violation; - /// A write acknowledged and not yet covered by a flush. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] struct Acked { lba: u64, - blocks: u32, - arena: u32, + run: Run, /// When it was first acknowledged, which is the order it is issued again /// in. first: u64, @@ -86,7 +83,7 @@ struct Acked { impl Acked { fn overlaps_range(&self, lba: u64, blocks: u32) -> bool { - self.lba < lba + u64::from(blocks) && lba < self.lba + u64::from(self.blocks) + self.lba < lba + u64::from(blocks) && lba < self.lba + u64::from(self.run.count()) } } @@ -106,8 +103,7 @@ enum Kind { struct Queued { op: Op, lba: u64, - blocks: u32, - arena: u32, + run: Option, kind: Kind, } @@ -126,16 +122,15 @@ pub struct Client { /// Not yet on the wire, in order. Writes being issued again are always at /// the front, then any flush attempt waiting on them. outbox: VecDeque, - wire: BTreeMap, + wire: Inflight, acked: VecDeque, - next_tag: u32, next_seq: u64, /// Losses taken so far: a flush sent before one says nothing about what it /// lost. losses: u64, attempts: BTreeMap, answers: VecDeque<(Ticket, Outcome)>, - released: VecDeque<(u32, u32)>, + released: VecDeque, /// Writes put on the wire again after a loss, over the client's life. reissued: u64, /// An acknowledged write was given up: no flush is durable again. @@ -149,18 +144,15 @@ impl Client { Self::default() } - /// Ask for a read or write of `blocks` at `lba`, through arena blocks - /// from `arena`, or for a flush (whose range is ignored). - pub fn submit(&mut self, ticket: Ticket, op: Op, lba: u64, blocks: u32, arena: u32) { - let (lba, blocks, arena) = match op { - Op::Flush => (0, 0, 0), - Op::Read | Op::Write => (lba, blocks, arena), - }; + /// Ask for a read or write at `lba` through the arena blocks `run`, or for + /// a flush, which names neither. + pub fn submit(&mut self, ticket: Ticket, op: Op, lba: u64, run: Option) { + assert_eq!(run.is_some(), op != Op::Flush, "a read or a write names a run, and a flush none"); let kind = match op { Op::Flush => Kind::Flush(ticket), Op::Read | Op::Write => Kind::User(ticket), }; - self.outbox.push_back(Queued { op, lba, blocks, arena, kind }); + self.outbox.push_back(Queued { op, lba, run, kind }); } fn reissuing(&self) -> bool { @@ -169,8 +161,8 @@ impl Client { } /// The next request to put on the wire, tagged; `None` while there is no - /// session, nothing to send, or what is next must wait for writes being - /// issued again. + /// session, nothing to send, [`DEPTH`] on the wire, or what is next must + /// wait for writes being issued again. pub fn next_request(&mut self) -> Option { if !self.up { return None; @@ -184,7 +176,7 @@ impl Client { Kind::Reissue(acked) => { let blocked = self.wire.values().any(|sent| { let q = sent.queued; - q.op == Op::Write && acked.overlaps_range(q.lba, q.blocks) + q.op == Op::Write && q.run.is_some_and(|run| acked.overlaps_range(q.lba, run.count())) }); if blocked { return None; @@ -196,28 +188,25 @@ impl Client { } } } + let tag = self.wire.insert(Sent { queued: front, covers: self.next_seq, losses: self.losses }).ok()?; self.outbox.pop_front(); if matches!(front.kind, Kind::Reissue(_)) { self.reissued += 1; } - let tag = self.next_tag; - self.next_tag = self.next_tag.wrapping_add(1); - let (covers, losses) = (self.next_seq, self.losses); - self.wire.insert(tag, Sent { queued: front, covers, losses }); - Some(Request { op: front.op, tag, lba: front.lba, blocks: front.blocks, arena: front.arena }) + Some(Request { op: front.op, tag, lba: front.lba, run: front.run }) } /// What the server answered. pub fn complete(&mut self, completion: Completion) -> Result<(), Violation> { - let sent = self.wire.remove(&completion.tag).ok_or(Violation)?; + let sent = self.wire.answer(Untrusted::new(completion.tag))?; let q = sent.queued; match (q.kind, completion.status) { (Kind::User(ticket), Status::Ok) => { match q.op { Op::Write => { let seq = self.bump(); - let acked = - Acked { lba: q.lba, blocks: q.blocks, arena: q.arena, first: seq, seq, attempts: 0 }; + let run = q.run.expect("a write names its run"); + let acked = Acked { lba: q.lba, run, first: seq, seq, attempts: 0 }; // Sent before a loss it did not see: the device may // have taken it and lost it, and an earlier write it // overlaps is being issued again — so it is issued @@ -228,19 +217,19 @@ impl Client { self.acked.push_back(acked); } } - Op::Read => self.released.push_back((q.arena, q.blocks)), - Op::Flush => return Err(Violation), + Op::Read => self.released.extend(q.run), + Op::Flush => return Err(Violation::Entry), } self.answers.push_back((ticket, Outcome::Done)); } (Kind::User(ticket), status) => { - self.released.push_back((q.arena, q.blocks)); + self.released.extend(q.run); let outcome = match status { Status::Invalid => Outcome::Invalid, Status::Device => Outcome::Device, // A read or a write answered `Lost` is a server that does // not know which op it was. - Status::Lost | Status::Ok => return Err(Violation), + Status::Lost | Status::Ok => return Err(Violation::Entry), }; self.answers.push_back((ticket, outcome)); } @@ -259,7 +248,7 @@ impl Client { if acked.attempts > MAX_ATTEMPTS { // The device will not take the only copy there is: every // flush waiting is told so, and the write is gone. - self.released.push_back((acked.arena, acked.blocks)); + self.released.push_back(acked.run); self.gave_up = true; self.fail_waiting_flushes(); } else { @@ -274,7 +263,7 @@ impl Client { (Kind::Flush(ticket), Status::Ok) => { while self.acked.front().is_some_and(|a| a.seq < sent.covers) { let a = self.acked.pop_front().expect("just seen"); - self.released.push_back((a.arena, a.blocks)); + self.released.push_back(a.run); } self.attempts.remove(&ticket); let outcome = if self.gave_up { Outcome::Device } else { Outcome::Durable }; @@ -298,15 +287,16 @@ impl Client { /// The session is over: the server hung up, broke the protocol, or died. pub fn session_ended(&mut self) { self.up = false; - let wire = core::mem::take(&mut self.wire); + let mut wire = Vec::new(); + self.wire.end(|_, sent| wire.push(sent)); let mut flushes = Vec::new(); - for sent in wire.into_values() { + for sent in wire { let q = sent.queued; match q.kind { Kind::User(ticket) => { #[cfg(not(feature = "mutate-session-end-forgets"))] { - self.released.push_back((q.arena, q.blocks)); + self.released.extend(q.run); self.answers.push_back((ticket, Outcome::Refused)); } #[cfg(feature = "mutate-session-end-forgets")] @@ -338,19 +328,19 @@ impl Client { self.answers.drain(..) } - /// Arena runs `(first, blocks)` nothing will read or write again. - pub fn take_released(&mut self) -> impl Iterator + '_ { + /// Arena runs nothing will read or write again. + pub fn take_released(&mut self) -> impl Iterator + '_ { self.released.drain(..) } /// Nothing is waiting, on the wire, or held for a flush. pub fn quiet(&self) -> bool { - self.outbox.is_empty() && self.wire.is_empty() && self.acked.is_empty() + self.outbox.is_empty() && self.wire.values().next().is_none() && self.acked.is_empty() } /// How many requests are on the wire. pub fn on_the_wire(&self) -> usize { - self.wire.len() + self.wire.values().count() } /// How many writes have gone on the wire again after a loss. @@ -380,13 +370,7 @@ impl Client { Kind::User(_) | Kind::Flush(_) => true, }) .unwrap_or(self.outbox.len()); - self.outbox.insert(at, Queued { - op: Op::Write, - lba: acked.lba, - blocks: acked.blocks, - arena: acked.arena, - kind: Kind::Reissue(acked), - }); + self.outbox.insert(at, Queued { op: Op::Write, lba: acked.lba, run: Some(acked.run), kind: Kind::Reissue(acked) }); } /// Ask the flush `ticket` again, once every write being issued again is @@ -397,7 +381,7 @@ impl Client { .iter() .position(|q| !matches!(q.kind, Kind::Reissue(_) | Kind::Flush(_))) .unwrap_or(self.outbox.len()); - self.outbox.insert(at, Queued { op: Op::Flush, lba: 0, blocks: 0, arena: 0, kind: Kind::Flush(ticket) }); + self.outbox.insert(at, Queued { op: Op::Flush, lba: 0, run: None, kind: Kind::Flush(ticket) }); } /// The device may no longer hold any write acknowledged and not covered: @@ -433,6 +417,7 @@ impl Client { #[cfg(test)] mod tests { use super::*; + use crate::layout::ARENA; fn answer(client: &mut Client, request: Request, status: Status) { client.complete(Completion { tag: request.tag, status }).unwrap(); @@ -441,7 +426,7 @@ mod tests { #[test] fn nothing_goes_out_before_a_session() { let mut client = Client::new(); - client.submit(1, Op::Read, 0, 1, 0); + client.submit(1, Op::Read, 0, ARENA.run(0, 1)); assert_eq!(client.next_request(), None); client.session_started(); assert!(client.next_request().is_some()); @@ -451,10 +436,10 @@ mod tests { fn a_second_answer_for_one_tag_is_a_violation() { let mut client = Client::new(); client.session_started(); - client.submit(1, Op::Read, 0, 1, 0); + client.submit(1, Op::Read, 0, ARENA.run(0, 1)); let r = client.next_request().unwrap(); answer(&mut client, r, Status::Ok); - assert_eq!(client.complete(Completion { tag: r.tag, status: Status::Ok }), Err(Violation)); + assert_eq!(client.complete(Completion { tag: r.tag, status: Status::Ok }), Err(Violation::Tag)); } /// A write acknowledged, then a flush answered `Lost`: the write goes out @@ -464,14 +449,14 @@ mod tests { fn a_lost_flush_reissues_then_asks_again() { let mut client = Client::new(); client.session_started(); - client.submit(1, Op::Write, 5, 2, 3); + client.submit(1, Op::Write, 5, ARENA.run(3, 2)); let w = client.next_request().unwrap(); answer(&mut client, w, Status::Ok); - client.submit(2, Op::Flush, 0, 0, 0); + client.submit(2, Op::Flush, 0, None); let f = client.next_request().unwrap(); answer(&mut client, f, Status::Lost); let again = client.next_request().unwrap(); - assert_eq!((again.op, again.lba, again.blocks, again.arena), (Op::Write, 5, 2, 3)); + assert_eq!((again.op, again.lba, again.run), (Op::Write, 5, ARENA.run(3, 2))); assert_eq!(client.next_request(), None, "the flush waits for the write"); answer(&mut client, again, Status::Ok); let f2 = client.next_request().unwrap(); @@ -479,7 +464,7 @@ mod tests { answer(&mut client, f2, Status::Ok); let answers: Vec<_> = client.take_answers().collect(); assert_eq!(answers, [(1, Outcome::Done), (2, Outcome::Durable)]); - assert_eq!(client.take_released().collect::>(), [(3, 2)]); + assert_eq!(client.take_released().collect::>(), ARENA.run(3, 2).into_iter().collect::>()); assert!(client.quiet()); } @@ -489,20 +474,20 @@ mod tests { fn overlapping_writes_go_out_again_in_order() { let mut client = Client::new(); client.session_started(); - client.submit(1, Op::Write, 0, 2, 0); + client.submit(1, Op::Write, 0, ARENA.run(0, 2)); let a = client.next_request().unwrap(); answer(&mut client, a, Status::Ok); - client.submit(2, Op::Write, 1, 1, 2); + client.submit(2, Op::Write, 1, ARENA.run(2, 1)); let b = client.next_request().unwrap(); answer(&mut client, b, Status::Ok); client.session_ended(); client.session_started(); let first = client.next_request().unwrap(); - assert_eq!((first.lba, first.arena), (0, 0)); + assert_eq!((first.lba, first.run), (0, ARENA.run(0, 2))); assert_eq!(client.next_request(), None, "the overlapping one waits"); answer(&mut client, first, Status::Ok); let second = client.next_request().unwrap(); - assert_eq!((second.lba, second.arena), (1, 2)); + assert_eq!((second.lba, second.run), (1, ARENA.run(2, 1))); } /// A write the device refused on every reissue is gone, and its caller was @@ -512,7 +497,7 @@ mod tests { fn a_write_given_up_poisons_every_later_flush() { let mut client = Client::new(); client.session_started(); - client.submit(1, Op::Write, 0, 1, 0); + client.submit(1, Op::Write, 0, ARENA.run(0, 1)); let w = client.next_request().unwrap(); answer(&mut client, w, Status::Ok); client.session_ended(); @@ -522,12 +507,12 @@ mod tests { assert_eq!((again.op, again.lba), (Op::Write, 0)); answer(&mut client, again, Status::Device); } - client.submit(2, Op::Flush, 0, 0, 0); + client.submit(2, Op::Flush, 0, None); let f = client.next_request().unwrap(); assert_eq!(f.op, Op::Flush); answer(&mut client, f, Status::Ok); assert_eq!(client.take_answers().collect::>(), [(1, Outcome::Done), (2, Outcome::Device)]); - client.submit(3, Op::Flush, 0, 0, 0); + client.submit(3, Op::Flush, 0, None); let f = client.next_request().unwrap(); answer(&mut client, f, Status::Ok); assert_eq!(client.take_answers().collect::>(), [(3, Outcome::Device)]); @@ -537,12 +522,12 @@ mod tests { fn what_was_on_the_wire_at_the_end_is_refused_and_what_was_not_waits() { let mut client = Client::new(); client.session_started(); - client.submit(1, Op::Write, 0, 1, 0); - client.submit(2, Op::Read, 4, 1, 1); + client.submit(1, Op::Write, 0, ARENA.run(0, 1)); + client.submit(2, Op::Read, 4, ARENA.run(1, 1)); let _ = client.next_request().unwrap(); client.session_ended(); assert_eq!(client.take_answers().collect::>(), [(1, Outcome::Refused)]); - assert_eq!(client.take_released().collect::>(), [(0, 1)]); + assert_eq!(client.take_released().collect::>(), ARENA.run(0, 1).into_iter().collect::>()); client.session_started(); let r = client.next_request().unwrap(); assert_eq!((r.op, r.lba), (Op::Read, 4)); diff --git a/toyos-blockring/src/entry.rs b/toyos-blockring/src/entry.rs index 8084b97d8d7..5fdd0948848 100644 --- a/toyos-blockring/src/entry.rs +++ b/toyos-blockring/src/entry.rs @@ -4,16 +4,16 @@ //! request is bounded here against the arena and the partition, and a word //! this protocol does not define is a refusal, never a default. -use toyos_transport::{Layout, Schema, Untrusted, Violation}; +use toyos_transport::{Run, Untrusted}; -use crate::layout::{ARENA_BLOCKS, BLOCK_BYTES, CQE_WORDS, MAX_REQUEST_BLOCKS, SQE_WORDS}; +use crate::layout::{ARENA, CQE_WORDS, MAX_REQUEST_BLOCKS, SQE_WORDS}; /// What a request asks of the partition. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Op { - /// `blocks` from the partition's block `lba` into the arena. + /// The run's blocks from the partition's block `lba` into the arena. Read, - /// `blocks` from the arena to the partition's block `lba`. + /// The run's blocks from the arena to the partition's block `lba`. Write, /// Every write acknowledged before this was submitted, onto the medium. Flush, @@ -31,17 +31,15 @@ impl Op { /// One request. `lba` is the partition's own block number, from 0: nothing in /// this protocol names a device block, so a neighbour's blocks have no -/// spelling. A flush carries no range, and its `lba`, `blocks` and `arena` are -/// zero. +/// spelling. A flush carries no range: its `lba` is zero and it names no run. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Request { pub op: Op, /// The client's name for it, echoed by its completion. pub tag: u32, pub lba: u64, - pub blocks: u32, - /// The first arena block the data is in or goes to. - pub arena: u32, + /// The arena blocks the data is in or goes to. + pub run: Option, } /// Why a request was answered without being done. @@ -55,16 +53,8 @@ pub enum Refused { impl Request { pub fn encode(&self) -> [u32; SQE_WORDS] { - [ - self.op.word(), - self.tag, - self.lba as u32, - (self.lba >> 32) as u32, - self.blocks, - self.arena, - 0, - 0, - ] + let (first, count) = self.run.map_or((0, 0), |run| (run.first(), run.count())); + [self.op.word(), self.tag, self.lba as u32, (self.lba >> 32) as u32, count, first, 0, 0] } /// The request these words are, bounded against a partition of @@ -84,21 +74,24 @@ impl Request { if lba != 0 || !blocks.is(0) || !arena.is(0) { return Err(refused); } - return Ok(Self { op, tag, lba, blocks: 0, arena: 0 }); + return Ok(Self { op, tag, lba, run: None }); } - let blocks = blocks.at_most(MAX_REQUEST_BLOCKS.into()).ok().filter(|&b| b > 0).ok_or(refused)? as u32; - let arena = arena.at_most((ARENA_BLOCKS - blocks).into()).map_err(|_| refused)? as u32; - if lba.checked_add(u64::from(blocks)).is_none_or(|end| end > partition_blocks) { + let run = Run::decode(arena, blocks, &ARENA).map_err(|_| refused)?; + let blocks = run.count(); + if blocks > MAX_REQUEST_BLOCKS || lba.checked_add(u64::from(blocks)).is_none_or(|end| end > partition_blocks) { return Err(refused); } - Ok(Self { op, tag, lba, blocks, arena }) + Ok(Self { op, tag, lba, run: Some(run) }) } } /// A word every value of which means something: a tag its answer echoes, or /// half of an `lba` the partition bounds whole. fn opaque(word: Untrusted) -> u32 { - word.at_most(u32::MAX.into()).map_or(0, |word| word as u32) + word.at_most(u32::MAX.into()) + .ok() + .and_then(|word| u32::try_from(word).ok()) + .expect("every u32 is at most u32::MAX") } /// What a completion says of its request. @@ -154,30 +147,9 @@ impl Completion { } } -/// The block protocol as a transport schema, over a partition `blocks` long. -pub struct Block { - pub blocks: u64, -} - -impl Schema for Block { - const LAYOUT: Layout = Layout::Region { slot_bytes: BLOCK_BYTES as u32 }; - type Request = Request; - type Reply = Completion; - type Refusal = Refused; - - fn decode_request(&self, words: [Untrusted; SQE_WORDS]) -> Result { - Request::decode(words, self.blocks) - } - - fn decode_reply(&self, words: [Untrusted; CQE_WORDS]) -> Result { - Completion::decode(words).ok_or(Violation::Entry) - } -} - #[cfg(test)] mod tests { use super::*; - use crate::layout::{arena_byte, SESSION_BYTES}; const PARTITION: u64 = 1000; @@ -185,26 +157,17 @@ mod tests { words.map(Untrusted::new) } - /// The transport's geometry of the schema's layout is this crate's arena, - /// block for block. - #[test] - fn the_schemas_arena_is_the_sessions() { - let geometry = toyos_transport::Geometry::decode(Untrusted::new(SESSION_BYTES as u64), Block::LAYOUT).unwrap(); - assert_eq!(geometry.slots(), ARENA_BLOCKS); - let last = geometry.own(ARENA_BLOCKS - 1, 1).unwrap(); - assert_eq!(last.run().span().offset, arena_byte(ARENA_BLOCKS - 1)); - } - #[test] fn a_request_survives_its_words() { + let last = ARENA.slots() - MAX_REQUEST_BLOCKS; for request in [ - Request { op: Op::Read, tag: 7, lba: 999, blocks: 1, arena: 0 }, - Request { op: Op::Write, tag: u32::MAX, lba: 0, blocks: MAX_REQUEST_BLOCKS, arena: ARENA_BLOCKS - MAX_REQUEST_BLOCKS }, - Request { op: Op::Flush, tag: 0, lba: 0, blocks: 0, arena: 0 }, + Request { op: Op::Read, tag: 7, lba: 999, run: ARENA.run(0, 1) }, + Request { op: Op::Write, tag: u32::MAX, lba: 0, run: ARENA.run(last, MAX_REQUEST_BLOCKS) }, + Request { op: Op::Flush, tag: 0, lba: 0, run: None }, ] { assert_eq!(Request::decode(peer(request.encode()), PARTITION), Ok(request)); } - let wide = Request { op: Op::Read, tag: 1, lba: 1 << 40, blocks: 2, arena: 3 }; + let wide = Request { op: Op::Read, tag: 1, lba: 1 << 40, run: ARENA.run(3, 2) }; assert_eq!(Request::decode(peer(wide.encode()), u64::MAX), Ok(wide)); } @@ -212,17 +175,22 @@ mod tests { /// refusal still carries its tag. #[test] fn a_request_outside_its_bounds_is_refused_by_tag() { - let good = Request { op: Op::Write, tag: 42, lba: 10, blocks: 2, arena: 5 }; + let good = Request { op: Op::Write, tag: 42, lba: 10, run: ARENA.run(5, 2) }; + let with = |at: usize, word: u32| { + let mut words = good.encode(); + words[at] = word; + words + }; let cases: [(&str, [u32; SQE_WORDS]); 10] = [ - ("op 0", { let mut w = good.encode(); w[0] = 0; w }), - ("op 4", { let mut w = good.encode(); w[0] = 4; w }), - ("no blocks", Request { blocks: 0, ..good }.encode()), - ("too many blocks", Request { blocks: MAX_REQUEST_BLOCKS + 1, ..good }.encode()), - ("past the arena", Request { arena: ARENA_BLOCKS - 1, ..good }.encode()), - ("arena wraps", Request { arena: u32::MAX, ..good }.encode()), + ("op 0", with(0, 0)), + ("op 4", with(0, 4)), + ("no blocks", with(4, 0)), + ("too many blocks", with(4, MAX_REQUEST_BLOCKS + 1)), + ("past the arena", with(5, ARENA.slots() - 1)), + ("arena wraps", with(5, u32::MAX)), ("past the partition", Request { lba: PARTITION - 1, ..good }.encode()), ("lba wraps", Request { lba: u64::MAX, ..good }.encode()), - ("reserved word", { let mut w = good.encode(); w[7] = 1; w }), + ("reserved word", with(7, 1)), ("a flush with a range", Request { op: Op::Flush, ..good }.encode()), ]; for (what, words) in cases { diff --git a/toyos-blockring/src/layout.rs b/toyos-blockring/src/layout.rs index 7df5fe666c3..457be042b46 100644 --- a/toyos-blockring/src/layout.rs +++ b/toyos-blockring/src/layout.rs @@ -4,14 +4,21 @@ //! `sleep` word on its head's, so the client's stores and the server's never //! share a line. -use toyos_transport::{Consumer, Cursors, Place, Producer, Violation, Word}; +use toyos_transport::{Consumer, Cursors, Geometry, Place, Producer, Violation, Word}; /// A session's whole region: the one size shared memory comes in. -pub const SESSION_BYTES: usize = 2 * 1024 * 1024; +pub const SESSION_BYTES: usize = Geometry::BYTES as usize; /// The unit every request is in, and the unit the arena is cut into. pub const BLOCK_BYTES: usize = 4096; +/// The arena: whole blocks after the rings' page, which a request names by +/// run. +pub const ARENA: Geometry = match Geometry::new(BLOCK_BYTES as u32) { + Some(arena) => arena, + None => panic!("a block is no longer than the arena"), +}; + /// How many requests, and so how many completions, one session has in flight. pub const DEPTH: u32 = 64; @@ -64,16 +71,5 @@ pub fn server(page: &[W]) -> Result { Ok((Consumer::new(page, REQUESTS)?, Producer::new(page, COMPLETIONS)?)) } -/// Where the arena starts, in bytes: the block after the rings' page. -pub const ARENA_OFFSET: usize = BLOCK_BYTES; - -/// The arena's blocks; a request's `arena` is an index below this. -pub const ARENA_BLOCKS: u32 = ((SESSION_BYTES - ARENA_OFFSET) / BLOCK_BYTES) as u32; - -const _: () = assert!(RING_WORDS * 4 <= ARENA_OFFSET); -const _: () = assert!(MAX_REQUEST_BLOCKS <= ARENA_BLOCKS); - -/// The byte offset of arena block `block` in the region. -pub const fn arena_byte(block: u32) -> usize { - ARENA_OFFSET + block as usize * BLOCK_BYTES -} +const _: () = assert!(RING_WORDS * 4 <= Geometry::HEADER_BYTES as usize); +const _: () = assert!(MAX_REQUEST_BLOCKS <= ARENA.slots()); diff --git a/toyos-blockring/src/lib.rs b/toyos-blockring/src/lib.rs index 3988cdaba32..dca1b010a16 100644 --- a/toyos-blockring/src/lib.rs +++ b/toyos-blockring/src/lib.rs @@ -3,12 +3,12 @@ //! **A session is one shared region**, [`SESSION_BYTES`] long, that the client //! makes and sends. Its first page holds two single-producer rings — requests //! from the client ([`layout::SQ_BASE`]), completions from the server -//! ([`layout::CQ_BASE`]) — and the rest is the *arena*: whole -//! [`BLOCK_BYTES`] blocks a request names by index and the device moves data -//! into and out of directly. Nothing on the page is a pointer and nothing on it +//! ([`layout::CQ_BASE`]) — and the rest is the *arena* ([`layout::ARENA`]): +//! whole [`BLOCK_BYTES`] blocks a request names by run and the device moves +//! data into and out of directly. Nothing on the page is a pointer and nothing on it //! is trusted by the end that did not write it: a consumer bounds every index //! and every field before it acts ([`entry::Request::decode`], the -//! transport's cursor bounds); [`entry::Block`] is the protocol as its schema. +//! transport's cursor bounds). //! //! **A doorbell is a byte on the session's connection**, written after the //! entries it announces are published. The connection is also what tells each @@ -47,7 +47,8 @@ pub mod wire; mod model; pub use entry::{Completion, Op, Request, Status}; -pub use layout::{ARENA_BLOCKS, BLOCK_BYTES, DEPTH, MAX_REQUEST_BLOCKS, SESSION_BYTES}; +pub use toyos_transport::Run; +pub use layout::{BLOCK_BYTES, DEPTH, MAX_REQUEST_BLOCKS, SESSION_BYTES}; /// The name a block service is served under. A holder of its connector may /// open any partition the service has. diff --git a/toyos-blockring/src/model.rs b/toyos-blockring/src/model.rs index 72786ae985a..662d6b593b6 100644 --- a/toyos-blockring/src/model.rs +++ b/toyos-blockring/src/model.rs @@ -1,12 +1,13 @@ //! Every ordering of a client, a server, a device and their failures. //! //! A scripted caller asks for writes and flushes; the rings between the client -//! and the server are queues; the server is [`ServerSession`] over a device of -//! two blocks with a volatile cache. [`explore`] runs, depth first and -//! exhaustively, every interleaving of: the caller asking for its next step, -//! the server taking a request, the device completing any one it holds, **the -//! device failing any one it holds** (not done, answered so), the client -//! reading a completion, **the device being reset** under whatever is in +//! and the server are queues, and the session page's own rings are driven +//! beside them and held to them ([`Queues`]); the server is [`ServerSession`] +//! over a device of two blocks with a volatile cache. [`explore`] runs, depth +//! first and exhaustively, every interleaving of: the caller asking for its +//! next step, the server taking a request, the device completing any one it +//! holds, **the device failing any one it holds** (not done, answered so), the +//! client reading a completion, **the device being reset** under whatever is in //! flight (each command dropped, or run before the stop with its completion //! read or not; the cache kept or dropped), **the server dying** (each command //! dropped or applied; the cache kept or dropped; the rings left as they @@ -29,12 +30,17 @@ use alloc::collections::{BTreeMap, VecDeque}; use alloc::format; use alloc::string::String; use alloc::vec::Vec; +use core::cell::Cell; +use core::fmt; +use core::sync::atomic::Ordering; use std::collections::HashSet; use toyos_blockhold::Holds; +use toyos_transport::{Untrusted, Word}; use crate::client::{Client, Outcome, Ticket, MAX_ATTEMPTS}; use crate::entry::{Completion, Op, Request}; +use crate::layout::{self, ClientRings, ServerRings, ARENA, CQE_WORDS, RING_WORDS, SQE_WORDS}; use crate::server::{ServerSession, Taken}; const BLOCKS: usize = 2; @@ -79,6 +85,92 @@ pub enum Law { Durable, } +/// One word of the session page. The model runs on one thread, so every order +/// is the program's. +#[derive(Clone)] +struct Shared(Cell); + +impl Word for Shared { + fn load(&self, _: Ordering) -> u32 { + self.0.get() + } + fn store(&self, value: u32, _: Ordering) { + self.0.set(value) + } + fn fence() {} +} + +const RING: &str = "the page holds every ring word"; + +/// The rings between the client and the server, twice: as queues, the +/// reference, and as the session page's own rings, whose every entry either +/// end takes is held equal to the queue's. A state is rendered by its queues +/// alone, so the search tells apart what the protocol can, not where on the +/// page it is. +#[derive(Clone)] +struct Queues { + sq: VecDeque, + cq: VecDeque, + page: Vec, + client: ClientRings, + server: ServerRings, +} + +impl fmt::Debug for Queues { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Queues").field("sq", &self.sq).field("cq", &self.cq).finish() + } +} + +impl Queues { + fn new() -> Self { + let page: Vec = (0..RING_WORDS).map(|_| Shared(Cell::new(0))).collect(); + let (client, server) = (layout::client(&page).expect(RING), layout::server(&page).expect(RING)); + Self { sq: VecDeque::new(), cq: VecDeque::new(), page, client, server } + } + + /// The session is over: what either queue held is gone, and the next + /// session's two ends start over the same page. + fn reset(&mut self) { + self.sq.clear(); + self.cq.clear(); + self.client = layout::client(&self.page).expect(RING); + self.server = layout::server(&self.page).expect(RING); + } + + fn send(&mut self, request: Request) { + assert_eq!(self.client.0.push(&self.page, request.encode()), Ok(true), "a request past the ring's depth"); + let _ = self.client.0.publish(&self.page).expect(RING); + self.sq.push_back(request); + } + + /// The oldest request, as the server's ring gives it. + fn take(&mut self) -> Option<[Untrusted; SQE_WORDS]> { + let want = self.sq.pop_front()?; + let words = self.server.0.pop(&self.page).expect("an honest client's tail"); + let words = words.expect("the ring holds what the queue does"); + self.server.0.release(&self.page).expect(RING); + assert_eq!(Request::decode(words, u64::MAX), Ok(want), "the ring gave the server what the queue holds"); + Some(words) + } + + fn post(&mut self, c: Completion) { + assert_eq!(self.server.1.push(&self.page, c.encode()), Ok(true), "a completion past the ring's depth"); + let _ = self.server.1.publish(&self.page).expect(RING); + self.cq.push_back(c); + } + + /// The oldest completion, as the client's ring gives it. + fn read(&mut self) -> Option { + let want = self.cq.pop_front()?; + let words: [Untrusted; CQE_WORDS] = + self.client.1.pop(&self.page).expect("an honest server's tail").expect("the ring holds what the queue does"); + self.client.1.release(&self.page).expect(RING); + assert_eq!(Completion::decode(words), Some(want), "the ring gave the client what the queue holds"); + Some(want) + } +} + #[derive(Clone, Debug)] struct Server { session: ServerSession, @@ -87,15 +179,16 @@ struct Server { #[derive(Clone, Debug)] struct World { - client: Client, + /// Boxed: its tag table has a slot for every entry of the ring, and a + /// frame of the search holds several worlds. + client: Box, next: usize, /// Every answer each ticket has had. answers: BTreeMap>, /// For each flush ticket, the write tickets answered `Done` before it was /// asked for. acked_before: BTreeMap>, - sq: VecDeque, - cq: VecDeque, + queues: Queues, server: Option, /// The connection: false once the server has died, until a reconnect. alive: bool, @@ -127,12 +220,11 @@ struct Run<'a> { /// Explore `script` against at most `failures`. pub fn explore(script: &[Step], failures: Failures) -> Explored { let mut world = World { - client: Client::new(), + client: Box::new(Client::new()), next: 0, answers: BTreeMap::new(), acked_before: BTreeMap::new(), - sq: VecDeque::new(), - cq: VecDeque::new(), + queues: Queues::new(), server: None, alive: false, losses: 0, @@ -164,7 +256,7 @@ fn connect(world: &mut World) { /// goes onto the ring. fn pump(world: &mut World) { while let Some(request) = world.client.next_request() { - world.sq.push_back(request); + world.queues.send(request); } } @@ -181,7 +273,8 @@ fn write_of(script: &[Step], ticket: Ticket) -> Option<(u64, u8)> { fn apply(script: &[Step], cache: &mut [Option; BLOCKS], media: &mut [u8; BLOCKS], request: Request) { match request.op { Op::Write => { - let (_, value) = write_of(script, u64::from(request.arena)).expect("a write's arena is its ticket"); + let (_, value) = write_of(script, u64::from(request.run.expect("a write names its run").first())) + .expect("a write's arena block is its ticket"); cache[request.lba as usize] = Some(value); } Op::Flush => { @@ -330,7 +423,7 @@ fn dfs(run: &mut Run, world: World) { if !busy { let mut w = world.clone(); match script[world.next] { - Step::Write { block, .. } => w.client.submit(ticket, Op::Write, block, 1, ticket as u32), + Step::Write { block, .. } => w.client.submit(ticket, Op::Write, block, ARENA.run(ticket as u32, 1)), Step::Flush => { let acked = w .answers @@ -339,7 +432,7 @@ fn dfs(run: &mut Run, world: World) { .map(|(t, _)| *t) .collect(); w.acked_before.insert(ticket, acked); - w.client.submit(ticket, Op::Flush, 0, 0, 0); + w.client.submit(ticket, Op::Flush, 0, None); } } w.next += 1; @@ -350,16 +443,22 @@ fn dfs(run: &mut Run, world: World) { } // The server takes the oldest request. - if world.alive && !world.sq.is_empty() { + if world.alive && !world.queues.sq.is_empty() { let mut w = world.clone(); - let request = w.sq.pop_front().expect("just seen"); + let words = w.queues.take().expect("just seen"); let server = w.server.as_mut().expect("alive"); - match server.session.take(request.encode()) { - Taken::Issue(request) => w.device.push((request.tag, request)), - Taken::Answer(c) => w.cq.push_back(c), - } + let step = match server.session.take(words) { + Taken::Issue(request) => { + w.device.push((request.tag, request)); + format!("take {:?}#{}", request.op, request.tag) + } + Taken::Answer(c) => { + w.queues.post(c); + format!("refuse #{}", c.tag) + } + }; moved = true; - go(run, format!("take {:?}#{}", request.op, request.tag), w); + go(run, step, w); } // The device completes any one command it holds. @@ -370,7 +469,7 @@ fn dfs(run: &mut Run, world: World) { let losses = w.losses; if let Some(server) = w.server.as_mut() { if let Some(c) = server.session.complete(tag, true, &mut server.holds, losses) { - w.cq.push_back(c); + w.queues.post(c); } } moved = true; @@ -386,7 +485,7 @@ fn dfs(run: &mut Run, world: World) { let losses = w.losses; if let Some(server) = w.server.as_mut() { if let Some(c) = server.session.complete(tag, false, &mut server.holds, losses) { - w.cq.push_back(c); + w.queues.post(c); } } moved = true; @@ -395,9 +494,9 @@ fn dfs(run: &mut Run, world: World) { } // The client reads the oldest completion. - if world.client.up() && !world.cq.is_empty() { + if world.client.up() && !world.queues.cq.is_empty() { let mut w = world.clone(); - let c = w.cq.pop_front().expect("just seen"); + let c = w.queues.read().expect("just seen"); if w.client.complete(c).is_err() { fail(run, Law::Answers, format!("the client met a second completion for tag {}", c.tag)); return; @@ -415,7 +514,7 @@ fn dfs(run: &mut Run, world: World) { let losses = w.losses; let server = w.server.as_mut().expect("alive"); if let Some(c) = server.session.complete(tag, true, &mut server.holds, losses) { - w.cq.push_back(c); + w.queues.post(c); } moved = true; go(run, format!("posted #{tag}"), w); @@ -432,7 +531,9 @@ fn dfs(run: &mut Run, world: World) { w.media = media; w.losses += 1; let server = w.server.as_mut().expect("alive"); - w.cq.extend(server.session.abort_all()); + for c in server.session.abort_all() { + w.queues.post(c); + } moved = true; go(run, format!("reset({posted:?} {cache:?} {media:?})"), w); } @@ -458,8 +559,7 @@ fn dfs(run: &mut Run, world: World) { if !world.alive && world.client.up() { let mut w = world.clone(); w.client.session_ended(); - w.sq.clear(); - w.cq.clear(); + w.queues.reset(); collect(run, &mut w); moved = true; go(run, "notice".into(), w); diff --git a/toyos-blockring/src/server.rs b/toyos-blockring/src/server.rs index 6c927cd2b06..bd4561d78ef 100644 --- a/toyos-blockring/src/server.rs +++ b/toyos-blockring/src/server.rs @@ -18,7 +18,6 @@ use alloc::collections::BTreeMap; use toyos_blockhold::{Holds, Writer}; - use toyos_transport::Untrusted; use crate::entry::{Completion, Op, Refused, Request, Status}; @@ -77,7 +76,7 @@ impl ServerSession { /// A malformed entry, and a tag already in flight, are answered at once /// and never reach the device: the second would make one tag two /// requests, and the client could not tell which answer was whose. - pub fn take_entry(&mut self, words: [Untrusted; SQE_WORDS]) -> Taken { + pub fn take(&mut self, words: [Untrusted; SQE_WORDS]) -> Taken { let request = match Request::decode(words, self.blocks) { Ok(request) => request, Err(Refused::Malformed { tag }) => { @@ -91,13 +90,6 @@ impl ServerSession { Taken::Issue(request) } - /// [`Self::take_entry`] of words a test wrote as the client, which arrive - /// as a peer's do. - #[cfg(test)] - pub fn take(&mut self, words: [u32; SQE_WORDS]) -> Taken { - self.take_entry(words.map(Untrusted::new)) - } - /// The device answered the request `tag` with `done` (whether it did it), /// while its loss count was `losses`. `None` for a tag no longer in /// flight: a reset has already answered it. @@ -142,12 +134,13 @@ impl ServerSession { #[cfg(test)] mod tests { use super::*; + use crate::layout::ARENA; - fn write(tag: u32) -> [u32; SQE_WORDS] { - Request { op: Op::Write, tag, lba: 0, blocks: 1, arena: 0 }.encode() + fn write(tag: u32) -> [Untrusted; SQE_WORDS] { + Request { op: Op::Write, tag, lba: 0, run: ARENA.run(0, 1) }.encode().map(Untrusted::new) } - fn flush(tag: u32) -> [u32; SQE_WORDS] { - Request { op: Op::Flush, tag, lba: 0, blocks: 0, arena: 0 }.encode() + fn flush(tag: u32) -> [Untrusted; SQE_WORDS] { + Request { op: Op::Flush, tag, lba: 0, run: None }.encode().map(Untrusted::new) } #[test] diff --git a/toyos-transport/Cargo.toml b/toyos-transport/Cargo.toml index 53a4c41dc36..483227af1e3 100644 --- a/toyos-transport/Cargo.toml +++ b/toyos-transport/Cargo.toml @@ -1,12 +1,11 @@ # A member of the host workspace (root `Cargo.toml`). The one transport every -# client/server session runs over: the rings, the byte streams, the arena's -# ownership tokens, the tag table and the wake, as decisions over words an -# adapter hands in. Nothing here maps, copies or blocks. +# client/server session runs over: the rings, the arena's runs, the tag table +# and the wake, as decisions over words an adapter hands in. Nothing here maps, +# copies or blocks. # # Its defects are orders and hostile peers, so `tests/loom.rs` holds the -# publication edge, the wake and a peer that scribbles every word it can reach, -# and `src/model.rs` enumerates a session through crash, restart and a late -# completion. +# publication edge, the wake and a hostile peer, and `src/model.rs` enumerates +# a session through crash, restart and a late completion. [package] name = "toyos-transport" @@ -16,7 +15,6 @@ license = "MIT OR Apache-2.0" publish = false [features] -rustc-dep-of-std = ["core", "toyos-untrusted/rustc-dep-of-std"] # Declared, never enabled by any build that ships: each reverts one decision so # the model that refuses it is shown able to red at all. `src/ci.rs`'s # `CONTROLS` runs each and demands the named test's own FAILED line. @@ -24,8 +22,11 @@ rustc-dep-of-std = ["core", "toyos-untrusted/rustc-dep-of-std"] # A producer publishes its tail `Relaxed`, so a consumer can see the index # before the entry's words. publish-relaxed = [] -# Neither side of the wake fences between its store and its load, so a -# producer can miss a consumer's `sleep` while the consumer misses its tail. +# The producer does not fence between its tail and its load of `sleep`, so it +# can miss a consumer's `sleep` while the consumer misses its tail. +no-wake-fence = [] +# The consumer does not fence between its `sleep` and its load of the tail: +# the same miss, from the other side. no-sleep-fence = [] # A peer's cursor is believed however far it claims to be, so a hostile # producer hands the consumer more entries than the ring holds. @@ -35,7 +36,6 @@ no-clamp = [] end-keeps-inflight = [] [dependencies] -core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } toyos-untrusted = { path = "../toyos-untrusted" } [dev-dependencies] diff --git a/toyos-transport/src/arena.rs b/toyos-transport/src/arena.rs index 01883f7cc1c..6a4a363f821 100644 --- a/toyos-transport/src/arena.rs +++ b/toyos-transport/src/arena.rs @@ -1,24 +1,19 @@ -//! A region's arena, and who may touch which run of it. +//! A region's arena, cut into slots, and the runs of it an entry names. //! -//! **A run exists only once it is bounded by the [`Geometry`]**, and it -//! travels as one of three tokens, none of them `Clone`: [`Own`] on the side -//! that allocated it, which lending consumes into [`Lent`] until a completion -//! naming it hands it back; and [`Held`] on the side that decoded it from an -//! entry ([`Run::decode`]). The adapter reaches a run's bytes only through a -//! token's [`Run::span`]. +//! **A run exists only once it is bounded by the [`Geometry`]**: a peer's is +//! decoded ([`Run::decode`]) and this side's is cut ([`Geometry::run`]). The +//! adapter reaches a run's bytes only through its [`Span`]. -use crate::{Span, Untrusted, Violation}; +use crate::{Untrusted, Violation}; -/// What a schema's region is: none, or a header page and an arena of slots of -/// the schema's size. +/// Bytes `offset..offset + len` of the region. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum Layout { - /// Entries travel as frames on the connection; a region sent is refused. - Inline, - Region { slot_bytes: u32 }, +pub struct Span { + pub offset: usize, + pub len: usize, } -/// A region's arena, decoded once. +/// A region's arena: whole slots after its header page. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Geometry { slot_bytes: u32, @@ -27,30 +22,25 @@ pub struct Geometry { impl Geometry { /// Every region is this long: the granule shared memory comes in. - pub const BYTES: u64 = 0x20_0000; + pub const BYTES: u32 = 0x20_0000; /// The header page the queues and cursors are on; the arena follows it. pub const HEADER_BYTES: u32 = 0x1000; - const ARENA_BYTES: u32 = 0x20_0000 - Self::HEADER_BYTES; - - /// The arena of a region `bytes` long, cut as `layout` says. - pub fn decode(bytes: Untrusted, layout: Layout) -> Result { - let Layout::Region { slot_bytes } = layout else { return Err(Violation::Region) }; - bytes.exactly(Self::BYTES).map_err(|_| Violation::Region)?; - let slots = Self::ARENA_BYTES.checked_div(slot_bytes).filter(|&n| n > 0).ok_or(Violation::Region)?; - Ok(Self { slot_bytes, slots }) - } + const ARENA_BYTES: u32 = Self::BYTES - Self::HEADER_BYTES; - pub fn slots(&self) -> u32 { - self.slots + /// The arena cut into slots of `slot_bytes`; `None` if not one fits. + pub const fn new(slot_bytes: u32) -> Option { + match Self::ARENA_BYTES.checked_div(slot_bytes) { + Some(slots) if slots > 0 => Some(Self { slot_bytes, slots }), + _ => None, + } } - /// Slots `first..first + count`, for the allocator that hands them out - /// once each to own; `None` for none or past the arena. - pub fn own(&self, first: u32, count: u32) -> Option { - self.run(first, count).map(Own) + pub const fn slots(&self) -> u32 { + self.slots } - fn run(&self, first: u32, count: u32) -> Option { + /// Slots `first..first + count`; `None` for none, or past the arena. + pub fn run(&self, first: u32, count: u32) -> Option { if count == 0 || first.checked_add(count)? > self.slots { return None; } @@ -69,13 +59,12 @@ pub struct Run { } impl Run { - /// The run an entry's two words name, held until its answer. - pub fn decode(first: Untrusted, count: Untrusted, geometry: &Geometry) -> Result { + /// The run a peer's two words name. + pub fn decode(first: Untrusted, count: Untrusted, geometry: &Geometry) -> Result { let bounded = |word: Untrusted| word.at_most(u64::from(geometry.slots)).ok()?.try_into().ok(); bounded(first) .zip(bounded(count)) .and_then(|(first, count)| geometry.run(first, count)) - .map(Held) .ok_or(Violation::Run) } @@ -93,88 +82,31 @@ impl Run { } } -/// A run this side allocated and has not lent. -#[derive(Debug, PartialEq, Eq, Hash)] -pub struct Own(Run); - -impl Own { - pub fn run(&self) -> &Run { - &self.0 - } - - /// Put it in an entry: the run is the peer's until its answer. - pub fn lend(self) -> (Lent, Run) { - (Lent(self.0), self.0) - } -} - -/// A run in an entry the peer has not answered. -#[derive(Debug, PartialEq, Eq, Hash)] -pub struct Lent(Run); - -impl Lent { - /// The completion that names it came back: it is this side's again. - pub fn back(self) -> Own { - Own(self.0) - } -} - -/// A run the peer lent, bounded, until this side answers the entry that named -/// it. -#[derive(Debug, PartialEq, Eq, Hash)] -pub struct Held(Run); - -impl Held { - pub fn run(&self) -> &Run { - &self.0 - } -} - #[cfg(test)] mod tests { use super::*; - const PAGES: Layout = Layout::Region { slot_bytes: 4096 }; - - fn geometry() -> Geometry { - Geometry::decode(Untrusted::new(Geometry::BYTES), PAGES).unwrap() - } + const PAGES: Geometry = Geometry::new(4096).unwrap(); #[test] - fn a_region_is_the_granule_and_a_schema_with_one() { - assert_eq!(geometry().slots(), 511); - for bytes in [0, Geometry::BYTES - 1, Geometry::BYTES + 1, 2 * Geometry::BYTES] { - assert_eq!(Geometry::decode(Untrusted::new(bytes), PAGES), Err(Violation::Region)); - } - assert_eq!(Geometry::decode(Untrusted::new(Geometry::BYTES), Layout::Inline), Err(Violation::Region)); - assert_eq!( - Geometry::decode(Untrusted::new(Geometry::BYTES), Layout::Region { slot_bytes: 0 }), - Err(Violation::Region) - ); + fn an_arena_is_whole_slots_after_the_header() { + assert_eq!(PAGES.slots(), 511); + assert_eq!(Geometry::new(0), None); + assert_eq!(Geometry::new(Geometry::BYTES), None, "a slot longer than the arena"); } /// Every run a peer can name is inside the arena, whole slots of it; the /// last slot is a run and one past it is not, however the sum wraps. #[test] fn a_run_a_peer_names_is_inside_the_arena() { - let g = geometry(); - let held = Run::decode(Untrusted::new(510), Untrusted::new(1), &g).unwrap(); - assert_eq!(held.run().span(), Span { offset: 0x1000 + 510 * 4096, len: 4096 }); + let run = Run::decode(Untrusted::new(510), Untrusted::new(1), &PAGES).unwrap(); + assert_eq!(run.span(), Span { offset: 0x1000 + 510 * 4096, len: 4096 }); for (first, count) in [(510, 2), (0, 0), (511, 1), (1, u32::MAX), (u32::MAX, 1), (0, 512)] { assert_eq!( - Run::decode(Untrusted::new(first), Untrusted::new(count), &g), + Run::decode(Untrusted::new(first), Untrusted::new(count), &PAGES), Err(Violation::Run), "{first}+{count}" ); } } - - #[test] - fn lending_consumes_and_the_answer_gives_back() { - let own = geometry().own(3, 2).unwrap(); - let (lent, run) = own.lend(); - assert_eq!((run.first(), run.count()), (3, 2)); - assert_eq!(lent.back().run(), &run); - assert_eq!(geometry().own(510, 2), None); - } } diff --git a/toyos-transport/src/inflight.rs b/toyos-transport/src/inflight.rs index 3545f5dde4b..70a0208fa88 100644 --- a/toyos-transport/src/inflight.rs +++ b/toyos-transport/src/inflight.rs @@ -4,7 +4,10 @@ //! ([`Inflight::answer`]) or, when the session ends, by [`Inflight::end`] — //! never by both, and never by a completion naming a tag from before its //! slot was last filled. A tag is the slot's index under the slot's own -//! sequence, so a tag answered, ended or replayed names nothing. +//! sequence, so a tag answered, ended or replayed names nothing. The sequence +//! wraps: a tag replayed a wrap later answers the request then in its slot, +//! which a server could answer by naming that request's own tag, so it grants +//! the peer nothing. use crate::{Untrusted, Violation}; @@ -55,6 +58,11 @@ impl Inflight { slot.value.take().ok_or(Violation::Tag) } + /// What is in flight, in no order. + pub fn values(&self) -> impl Iterator { + self.slots.iter().filter_map(|slot| slot.value.as_ref()) + } + /// The session ended: `each` is given every tag in flight, once, with what /// it carried, and none of them is answered again. pub fn end(&mut self, mut each: impl FnMut(u32, T)) { @@ -90,6 +98,9 @@ mod tests { assert_eq!(inflight.answer(Untrusted::new(a)), Err(Violation::Tag), "answered twice"); let c = inflight.insert("c").unwrap(); assert_ne!(c, a, "the slot's next tag is not its last"); + let mut held: Vec<&str> = inflight.values().copied().collect(); + held.sort_unstable(); + assert_eq!(held, ["b", "c"], "what is in flight, and only that"); assert_eq!(inflight.answer(Untrusted::new(a)), Err(Violation::Tag), "a replay answers nothing"); assert_eq!(inflight.answer(Untrusted::new(2)), Err(Violation::Tag), "an index past the table"); assert_eq!(inflight.answer(Untrusted::new(b)), Ok("b")); diff --git a/toyos-transport/src/lib.rs b/toyos-transport/src/lib.rs index 41f7f3e314e..ce43f6a0736 100644 --- a/toyos-transport/src/lib.rs +++ b/toyos-transport/src/lib.rs @@ -2,14 +2,13 @@ //! //! **A session is one connection and, at most, one region.** The connection //! carries the hello, handles, doorbells and the hang-up; the region carries -//! [`Producer`]/[`Consumer`] queues of fixed-size entries, [`StreamTx`]/ -//! [`StreamRx`] byte rings, and an arena whose runs pass between the ends only -//! as [`Own`] → [`Lent`] on one side and [`Held`] on the other. A schema -//! ([`Schema`]) says what the entries mean; this crate says only what is safe. +//! [`Producer`]/[`Consumer`] queues of fixed-size entries and an arena of +//! [`Run`]s. A protocol says what the entries mean; this crate says only what +//! is safe. //! //! **Nothing here holds the region.** An adapter hands every call the region's -//! words as a slice of [`Word`]s and copies bytes itself, through the [`Span`]s -//! answered here, once: no reference to the peer's bytes is formed. +//! words as a slice of [`Word`]s and copies bytes itself, through a run's +//! [`Span`], once: no reference to the peer's bytes is formed. //! //! **What the peer writes is untrusted until decoded.** An entry comes out as //! [`Untrusted`] words; a peer's cursor is bounded against the ring before it @@ -38,14 +37,12 @@ mod inflight; #[cfg(test)] mod model; mod queue; -mod stream; use core::sync::atomic::{AtomicU32, Ordering}; -pub use arena::{Geometry, Held, Layout, Lent, Own, Run}; +pub use arena::{Geometry, Run, Span}; pub use inflight::Inflight; pub use queue::{Consumer, Place, Producer}; -pub use stream::{End, StreamPlace, StreamRx, StreamTx}; pub use toyos_untrusted::Untrusted; /// One shared 32-bit word of a region: an atomic over the mapping, or a @@ -78,15 +75,13 @@ pub enum Violation { /// A consumer's head past what was published, or more than the ring holds /// behind it. HeadPastTail, - /// A stream's end word that is no [`End`]. - End, - /// A reply no server of the schema writes. + /// A reply no server of the protocol writes. Entry, /// A run outside the arena. Run, /// A tag nothing is in flight under. Tag, - /// A region no [`Geometry`] describes, or a place outside the words given. + /// A place outside the words given. Region, } @@ -108,13 +103,6 @@ pub struct Asleep { pub value: u32, } -/// Bytes `offset..offset + len` of the region. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct Span { - pub offset: usize, - pub len: usize, -} - /// Where a ring's two cursors and its consumer's `sleep` word are, in words. /// The producer stores `tail`, the consumer `head` and `sleep`. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -124,19 +112,6 @@ pub struct Cursors { pub sleep: usize, } -/// A protocol over this transport: its region's layout, and the one place its -/// entries' words become its own types. A request that does not decode is -/// answered by tag with the schema's refusal; a reply that does not is the -/// server ending the session. -pub trait Schema { - const LAYOUT: Layout; - type Request; - type Reply; - type Refusal; - fn decode_request(&self, words: [Untrusted; SQE]) -> Result; - fn decode_reply(&self, words: [Untrusted; CQE]) -> Result; -} - #[cfg(not(feature = "publish-relaxed"))] const PUBLISH: Ordering = Ordering::Release; #[cfg(feature = "publish-relaxed")] @@ -172,7 +147,7 @@ impl Cursors { /// The producer's half of the wake, after what it published is stored. fn wake(&self, page: &[W]) -> Result { - #[cfg(not(feature = "no-sleep-fence"))] + #[cfg(not(feature = "no-wake-fence"))] W::fence(); Ok(match word(page, self.sleep)?.load(Ordering::Relaxed) { 0 => Wake::Busy, diff --git a/toyos-transport/src/queue.rs b/toyos-transport/src/queue.rs index 7d0a458ebbb..d6a6d1acc8b 100644 --- a/toyos-transport/src/queue.rs +++ b/toyos-transport/src/queue.rs @@ -244,6 +244,45 @@ mod tests { assert_eq!(tx.space(&page), Ok(D - 1), "a head moved back costs its consumer the room"); } + /// A producer that steps its tail one past each entry popped, none of them + /// released: the ring's depth is taken, and the next tail is past it. + #[test] + fn a_tail_stepped_past_each_pop_is_refused_at_the_depth() { + let page = page(); + let (_, mut rx) = ends(&page); + let mut taken = 0; + let refused = loop { + page[16].store(taken + 1, Ordering::Release); + match rx.pop(&page) { + Ok(Some(_)) => taken += 1, + Ok(None) => panic!("a published entry was not taken"), + Err(violation) => break violation, + } + assert!(taken <= D, "took {taken} entries from a ring of {D} without releasing one"); + }; + assert_eq!((taken, refused), (D, Violation::TailPastDepth)); + } + + /// A consumer that steps its head onto each entry pushed, none of them + /// published: the ring's depth is pushed, and the next head is past what + /// was published. + #[test] + fn a_head_stepped_past_each_push_is_refused_at_the_depth() { + let page = page(); + let (mut tx, _) = ends(&page); + let mut pushed = 0; + let refused = loop { + match tx.push(&page, [pushed, pushed]) { + Ok(true) => pushed += 1, + Ok(false) => panic!("a ring with nothing published was full"), + Err(violation) => break violation, + } + page[0].store(pushed, Ordering::Release); + assert!(pushed <= D, "pushed {pushed} into a ring of {D} with none published"); + }; + assert_eq!((pushed, refused), (D, Violation::HeadPastTail)); + } + #[test] fn a_place_outside_the_words_is_refused() { let page = page(); diff --git a/toyos-transport/src/stream.rs b/toyos-transport/src/stream.rs deleted file mode 100644 index a122aa20e14..00000000000 --- a/toyos-transport/src/stream.rs +++ /dev/null @@ -1,283 +0,0 @@ -//! A single-producer byte ring with free-running cursors, and the producer's -//! `end` word. -//! -//! **This crate never touches a byte.** Each end answers the [`Span`]s of the -//! region the adapter copies into or out of, once, and the cursors publish -//! them as a queue's do. The ring's capacity is a power of two, so a cursor -//! wrapping at 2³² lands on the same byte. -//! -//! **The end is stored after the tail and loaded before it**, so a reader that -//! sees [`End::Fin`] and no bytes has seen the stream's last byte. - -use core::sync::atomic::Ordering; - -use crate::{word, Asleep, Cursors, Span, Untrusted, Violation, Wake, Word}; - -/// Where one stream is: its cursors, its `end` word, and its bytes — a fixed -/// span of the region or an arena run's ([`crate::Run::span`]). -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct StreamPlace { - pub cursors: Cursors, - pub end: usize, - pub data: Span, -} - -impl StreamPlace { - /// The capacity, once every word is in `page` and the bytes are a power of - /// two. - fn check(&self, page: &[W]) -> Result { - for at in [self.cursors.head, self.cursors.tail, self.cursors.sleep, self.end] { - word(page, at)?; - } - u32::try_from(self.data.len).ok().filter(|cap| cap.is_power_of_two()).ok_or(Violation::Region) - } - - /// The bytes from cursor `at`, `len` long, where they are: past the ring's - /// end they go on from its start. - fn spans(&self, cap: u32, at: u32, len: u32) -> Result<[Span; 2], Violation> { - let from = at & cap.wrapping_sub(1); - let first = len.min(cap.wrapping_sub(from)); - let bytes = |n: u32| usize::try_from(n).map_err(|_| Violation::Region); - let offset = self.data.offset.checked_add(bytes(from)?).ok_or(Violation::Region)?; - Ok([ - Span { offset, len: bytes(first)? }, - Span { offset: self.data.offset, len: bytes(len.wrapping_sub(first))? }, - ]) - } -} - -/// What the producer has said of the bytes after the last it published. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum End { - /// More may follow. - Open, - /// None follow: the stream ended whole. - Fin, - /// None follow, and what was not read is abandoned. - Reset, -} - -impl End { - const fn word(self) -> u32 { - match self { - Self::Open => 0, - Self::Fin => 1, - Self::Reset => 2, - } - } - - fn decode(word: Untrusted) -> Result { - [Self::Open, Self::Fin, Self::Reset].into_iter().find(|end| word.is(end.word())).ok_or(Violation::End) - } -} - -/// The end of a stream that writes bytes. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct StreamTx { - place: StreamPlace, - cap: u32, - local: u32, - published: u32, -} - -impl StreamTx { - /// This end of the stream at `place`, its tail stored 0 and its end open. - pub fn new(page: &[W], place: StreamPlace) -> Result { - let cap = place.check(page)?; - word(page, place.cursors.tail)?.store(0, Ordering::Release); - word(page, place.end)?.store(End::Open.word(), Ordering::Release); - Ok(Self { place, cap, local: 0, published: 0 }) - } - - /// Room for at most `want` bytes, taken now: the caller fills the spans, - /// then publishes. - pub fn write(&mut self, page: &[W], want: u32) -> Result<[Span; 2], Violation> { - let unreleased = self.place.cursors.unreleased(page, self.published, self.cap)?; - let pending = self.local.wrapping_sub(self.published); - let len = want.min(self.cap.saturating_sub(pending.saturating_add(unreleased))); - let spans = self.place.spans(self.cap, self.local, len)?; - self.local = self.local.wrapping_add(len); - Ok(spans) - } - - /// Publish every byte written so far; `None` if there was none. - pub fn publish(&mut self, page: &[W]) -> Result, Violation> { - if self.published == self.local { - return Ok(None); - } - let wake = self.place.cursors.publish(page, self.local)?; - self.published = self.local; - Ok(Some(wake)) - } - - /// Publish what was written and say what follows it. - pub fn close(&mut self, page: &[W], end: End) -> Result { - word(page, self.place.cursors.tail)?.store(self.local, Ordering::Release); - self.published = self.local; - word(page, self.place.end)?.store(end.word(), Ordering::Release); - self.place.cursors.wake(page) - } -} - -/// The end of a stream that reads bytes. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct StreamRx { - place: StreamPlace, - cap: u32, - local: u32, - released: u32, - asleep: bool, -} - -impl StreamRx { - /// This end of the stream at `place`, its head and `sleep` stored 0. - pub fn new(page: &[W], place: StreamPlace) -> Result { - let cap = place.check(page)?; - word(page, place.cursors.head)?.store(0, Ordering::Release); - place.cursors.awake(page)?; - Ok(Self { place, cap, local: 0, released: 0, asleep: false }) - } - - /// What was said of the end, then how many bytes are ready past what was - /// read, and the tail they end at. - fn observe(&self, page: &[W]) -> Result<(End, u32, u32), Violation> { - let end = End::decode(Untrusted::new(word(page, self.place.end)?.load(Ordering::Acquire)))?; - let published = self.place.cursors.published(page, self.released, self.cap)?; - let ready = published.saturating_sub(self.local.wrapping_sub(self.released)); - Ok((end, ready, self.released.wrapping_add(published))) - } - - /// At most `want` bytes, taken now, and what follows them: the caller - /// copies the spans out, then releases. - pub fn read(&mut self, page: &[W], want: u32) -> Result<([Span; 2], End), Violation> { - if self.asleep { - self.place.cursors.awake(page)?; - self.asleep = false; - } - let (end, ready, _) = self.observe(page)?; - let len = want.min(ready); - let spans = self.place.spans(self.cap, self.local, len)?; - self.local = self.local.wrapping_add(len); - Ok((spans, end)) - } - - /// Give every byte read so far back to the producer. - pub fn release(&mut self, page: &[W]) -> Result<(), Violation> { - if self.released != self.local { - word(page, self.place.cursors.head)?.store(self.local, Ordering::Release); - self.released = self.local; - } - Ok(()) - } - - /// As [`crate::Consumer::before_sleep`]; an end said is something to read. - pub fn before_sleep(&mut self, page: &[W]) -> Result, Violation> { - self.place.cursors.sleep(page)?; - self.asleep = true; - let (end, ready, tail) = self.observe(page)?; - if ready > 0 || end != End::Open { - self.place.cursors.awake(page)?; - self.asleep = false; - return Ok(None); - } - Ok(Some(Asleep { word: self.place.cursors.tail, value: tail })) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use core::sync::atomic::AtomicU32; - - const PLACE: StreamPlace = StreamPlace { - cursors: Cursors { head: 0, tail: 16, sleep: 1 }, - end: 17, - data: Span { offset: 4096, len: 16 }, - }; - - fn page() -> Vec { - (0..32).map(|_| AtomicU32::new(0)).collect() - } - - /// The bytes behind the spans, as the adapter holds them. - fn copy(ring: &mut [u8; 16], spans: [Span; 2], bytes: &[u8], into: bool) -> Vec { - let mut out = Vec::new(); - let mut at = 0; - for span in spans { - let ring = &mut ring[span.offset - 4096..span.offset - 4096 + span.len]; - if into { - ring.copy_from_slice(&bytes[at..at + span.len]); - } - out.extend_from_slice(ring); - at += span.len; - } - out - } - - /// Bytes cross whole and in order as the cursors wrap the ring, a write is - /// never given more room than was released, and `Fin` arrives after the - /// last byte. - #[test] - fn bytes_cross_whole_around_the_ring_and_end_after_the_last() { - let page = page(); - let mut ring = [0u8; 16]; - let mut tx = StreamTx::new(&page, PLACE).unwrap(); - let mut rx = StreamRx::new(&page, PLACE).unwrap(); - let (mut sent, mut got) = (Vec::new(), Vec::new()); - for n in 0..40u8 { - let chunk: Vec = (0..n % 11).map(|i| n.wrapping_mul(31).wrapping_add(i)).collect(); - let spans = tx.write(&page, chunk.len() as u32).unwrap(); - let len: usize = spans.iter().map(|s| s.len).sum(); - assert!(len <= 16 - (sent.len() - got.len()), "room past what was released"); - sent.extend_from_slice(©(&mut ring, spans, &chunk[..len], true)); - let _ = tx.publish(&page).unwrap(); - let (spans, end) = rx.read(&page, u32::from(n % 5)).unwrap(); - assert_eq!(end, End::Open); - got.extend(copy(&mut ring, spans, &[], false)); - rx.release(&page).unwrap(); - } - assert_eq!(tx.close(&page, End::Fin), Ok(Wake::Busy)); - loop { - let (spans, end) = rx.read(&page, 16).unwrap(); - let chunk = copy(&mut ring, spans, &[], false); - got.extend_from_slice(&chunk); - rx.release(&page).unwrap(); - if chunk.is_empty() { - assert_eq!(end, End::Fin); - break; - } - } - assert_eq!(got, sent); - assert!(sent.len() > 64, "the ring wrapped"); - } - - #[test] - fn an_end_word_no_producer_writes_is_a_violation() { - let page = page(); - let mut rx = StreamRx::new(&page, PLACE).unwrap(); - page[17].store(3, Ordering::Release); - assert_eq!(rx.read(&page, 1), Err(Violation::End)); - page[17].store(0, Ordering::Release); - page[16].store(17, Ordering::Release); - assert_eq!(rx.read(&page, 1), Err(Violation::TailPastDepth)); - } - - #[test] - fn a_ring_that_is_not_a_power_of_two_is_refused() { - let page = page(); - let place = StreamPlace { data: Span { offset: 4096, len: 24 }, ..PLACE }; - assert_eq!(StreamTx::new(&page, place).err(), Some(Violation::Region)); - } - - /// A reader that said it sleeps is woken by the close, and one that looks - /// after a close does not sleep. - #[test] - fn a_close_wakes_a_sleeper_and_keeps_the_next_awake() { - let page = page(); - let mut tx = StreamTx::new(&page, PLACE).unwrap(); - let mut rx = StreamRx::new(&page, PLACE).unwrap(); - assert_eq!(rx.before_sleep(&page), Ok(Some(Asleep { word: 16, value: 0 }))); - assert_eq!(tx.close(&page, End::Reset), Ok(Wake::Peer)); - assert_eq!(rx.before_sleep(&page), Ok(None)); - } -} diff --git a/toyos-transport/tests/loom.rs b/toyos-transport/tests/loom.rs index 00fca81d788..241a024256b 100644 --- a/toyos-transport/tests/loom.rs +++ b/toyos-transport/tests/loom.rs @@ -4,12 +4,13 @@ //! - **Publication**: a consumer that sees a tail sees every word of the //! entries below it. `publish-relaxed` takes the edge away. //! - **No lost wake**: a consumer that says it sleeps and a producer that -//! publishes cannot both miss the other; the futex is park and unpark, and -//! a lost wake is a consumer parked for good. `no-sleep-fence` takes the -//! fences away. -//! - **A hostile peer** that stores to every word it can reach, at every step, -//! leaves the honest end with entries, nothing, or a named violation, and -//! never more entries than the ring holds. `no-clamp` believes the peer. +//! publishes cannot both miss the other; the futex is a load of the word it +//! waits on, and a lost wake is a consumer that slept over a publish that +//! answered [`Wake::Busy`]. `no-wake-fence` takes the producer's fence away +//! and `no-sleep-fence` the consumer's. +//! - **A hostile peer** leaves the honest end with entries, nothing, or a +//! named violation, and never more entries than the ring holds. `no-clamp` +//! believes the peer. //! //! cargo test -p toyos-transport --features --test loom @@ -84,30 +85,30 @@ fn a_published_entry_is_read_whole() { }); } -/// A consumer that finds nothing sleeps as a futex does — parked while the -/// tail still holds what it saw — and is woken only when a publish answers -/// [`Wake::Peer`]. Whatever the schedule, it gets the entry. +/// A consumer that finds nothing sleeps as a futex does — only while the tail +/// still holds what it saw — and is woken only when a publish answers +/// [`Wake::Peer`]. Whatever the schedule, a consumer that slept is woken. #[test] fn a_publish_and_a_sleep_cannot_both_miss() { loom::model(|| { let page = page(); let (mut tx, mut rx) = ends(&page); let consumer_page = Arc::clone(&page); - let consumer = loom::thread::spawn(move || loop { - if let Some(words) = rx.pop(&consumer_page).unwrap() { - return plain(words); - } - if let Some(asleep) = rx.before_sleep(&consumer_page).unwrap() { - if consumer_page[asleep.word].load(Ordering::Relaxed) == asleep.value { - loom::thread::park(); - } + let consumer = loom::thread::spawn(move || { + if rx.pop(&consumer_page).unwrap().is_some() { + return false; } + rx.before_sleep(&consumer_page) + .unwrap() + .is_some_and(|asleep| consumer_page[asleep.word].load(Ordering::Relaxed) == asleep.value) }); assert!(tx.push(&page, entry(1)).unwrap()); - if tx.publish(&page).unwrap() == Some(Wake::Peer) { - consumer.thread().unpark(); - } - assert_eq!(consumer.join().expect("the consumer parked over a published entry"), entry(1)); + let wake = tx.publish(&page).unwrap(); + let slept = consumer.join().expect("the consumer thread"); + assert!( + !slept || wake == Some(Wake::Peer), + "the consumer slept over a published entry and the publish answered {wake:?}" + ); }); } diff --git a/toyos-untrusted/Cargo.toml b/toyos-untrusted/Cargo.toml index 3ded1fbbf7f..83e3dc98028 100644 --- a/toyos-untrusted/Cargo.toml +++ b/toyos-untrusted/Cargo.toml @@ -15,10 +15,3 @@ version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" publish = false - -[features] -# `toyos-transport` sits under std, and so does what it bounds with. -rustc-dep-of-std = ["core"] - -[dependencies] -core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } diff --git a/userland/blockd/src/main.rs b/userland/blockd/src/main.rs index d49578c565e..dcb6f984b05 100644 --- a/userland/blockd/src/main.rs +++ b/userland/blockd/src/main.rs @@ -47,7 +47,7 @@ use toyos_abi::part::{PartGuid, GUID_TEXT_LEN}; use toyos_abi::syscall::{DEV_PREFIX, SyscallError}; use toyos_blockhold::Holds; use toyos_blockring::entry::{Completion, Op}; -use toyos_blockring::layout::{self, arena_byte, ServerRings, DEPTH}; +use toyos_blockring::layout::{self, ServerRings, DEPTH}; use toyos_blockring::server::{ServerSession, Taken}; use toyos_blockring::wire::{self, Opened, Refusal}; use toyos_blockring::{BLOCK_BYTES, PORT, SESSION_BYTES}; @@ -330,15 +330,16 @@ impl Service { } }; s.requests += 1; - match s.state.take_entry(words) { + match s.state.take(words) { Taken::Answer(c) => Self::post(s, c), Taken::Issue(req) => { let owner = Owner::Session { session: id, tag: req.tag, write: req.op == Op::Write }; match req.op { Op::Read | Op::Write => { - let at = s.device_addr + arena_byte(req.arena) as u64; + let run = req.run.expect("blockd: a transfer names its run"); + let at = s.device_addr + run.span().offset as u64; let block = s.state.first() + req.lba; - self.ctrl.submit_io(req.op == Op::Write, block, req.blocks, at, owner); + self.ctrl.submit_io(req.op == Op::Write, block, run.count(), at, owner); } Op::Flush if self.ctrl.vwc => self.ctrl.submit_flush(owner), // No volatile cache: every write answered is on the diff --git a/userland/blockd/src/region.rs b/userland/blockd/src/region.rs index c4fba33dd08..f6d112c29b1 100644 --- a/userland/blockd/src/region.rs +++ b/userland/blockd/src/region.rs @@ -9,8 +9,8 @@ use core::sync::atomic::AtomicU32; use toyos::shm::SharedMemory; use toyos_abi::syscall::SyscallError; -use toyos_blockring::layout::{arena_byte, ARENA_BLOCKS, RING_WORDS, SESSION_BYTES}; -use toyos_blockring::BLOCK_BYTES; +use toyos_blockring::layout::{RING_WORDS, SESSION_BYTES}; +use toyos_blockring::Run; use toyos::volatile::Window; @@ -42,16 +42,12 @@ impl Region { unsafe { core::slice::from_raw_parts(base as *const AtomicU32, RING_WORDS) } } - /// Arena blocks `first..first + blocks`. - pub fn arena(&self, first: u32, blocks: u32) -> Window { - assert!( - first.checked_add(blocks).is_some_and(|end| end <= ARENA_BLOCKS), - "blockd: arena blocks {first}+{blocks} past the arena" - ); + /// The arena blocks of `run`. + pub fn arena(&self, run: &Run) -> Window { // SAFETY: `SESSION_BYTES` mapped for as long as `self.memory`, and // every window over it is used while the region is held. let whole = unsafe { Window::new(self.memory.as_ptr(), SESSION_BYTES) }; - whole.sub(arena_byte(first), blocks as usize * BLOCK_BYTES) + whole.sub(run.span().offset, run.span().len) } /// A second handle to the region, for a send. diff --git a/userland/blockd/src/session.rs b/userland/blockd/src/session.rs index 62362ab3340..9b514a6b788 100644 --- a/userland/blockd/src/session.rs +++ b/userland/blockd/src/session.rs @@ -23,9 +23,9 @@ use toyos::poller::{Poller, READABLE}; use toyos_abi::syscall::SyscallError; use toyos_blockring::client::{Client, Outcome, Ticket}; use toyos_blockring::entry::{Completion, Op}; -use toyos_blockring::layout::{self, ClientRings, ARENA_BLOCKS, MAX_REQUEST_BLOCKS}; +use toyos_blockring::layout::{self, ClientRings, ARENA, MAX_REQUEST_BLOCKS}; use toyos_blockring::wire::{self, Opened, Refusal}; -use toyos_blockring::BLOCK_BYTES; +use toyos_blockring::{Run, BLOCK_BYTES}; use crate::region::Region; @@ -78,10 +78,10 @@ struct Arena { impl Arena { fn new() -> Self { - Self { free: vec![true; ARENA_BLOCKS as usize] } + Self { free: vec![true; ARENA.slots() as usize] } } - fn alloc(&mut self, blocks: u32) -> Option { + fn alloc(&mut self, blocks: u32) -> Option { let n = blocks as usize; let mut run = 0; for i in 0..self.free.len() { @@ -89,22 +89,23 @@ impl Arena { if run == n { let first = i + 1 - n; self.free[first..=i].iter_mut().for_each(|b| *b = false); - return Some(first as u32); + return ARENA.run(first as u32, blocks); } } None } - fn release(&mut self, first: u32, blocks: u32) { - for b in &mut self.free[first as usize..(first + blocks) as usize] { - assert!(!*b, "blockd: arena block {first}+{blocks} released twice"); + fn release(&mut self, run: Run) { + let first = run.first() as usize; + for b in &mut self.free[first..first + run.count() as usize] { + assert!(!*b, "blockd: arena run {run:?} released twice"); *b = true; } } } enum Pending { - Read { arena: u32, blocks: u32 }, + Read { run: Run }, Write, Flush, } @@ -183,10 +184,10 @@ impl Session { if self.conn.is_none() { return Err(Unsent::Ended); } - let arena = self.arena.alloc(blocks).ok_or(Unsent::ArenaFull)?; - self.region.arena(arena, blocks).copy_in(0, data); + let run = self.arena.alloc(blocks).ok_or(Unsent::ArenaFull)?; + self.region.arena(&run).copy_in(0, data); let ticket = self.ticket(); - self.client.submit(ticket, Op::Write, lba, blocks, arena); + self.client.submit(ticket, Op::Write, lba, Some(run)); self.pending.insert(ticket, Pending::Write); Ok(ticket) } @@ -197,10 +198,10 @@ impl Session { if self.conn.is_none() { return Err(Unsent::Ended); } - let arena = self.arena.alloc(blocks).ok_or(Unsent::ArenaFull)?; + let run = self.arena.alloc(blocks).ok_or(Unsent::ArenaFull)?; let ticket = self.ticket(); - self.client.submit(ticket, Op::Read, lba, blocks, arena); - self.pending.insert(ticket, Pending::Read { arena, blocks }); + self.client.submit(ticket, Op::Read, lba, Some(run)); + self.pending.insert(ticket, Pending::Read { run }); Ok(ticket) } @@ -210,7 +211,7 @@ impl Session { return Err(Unsent::Ended); } let ticket = self.ticket(); - self.client.submit(ticket, Op::Flush, 0, 0, 0); + self.client.submit(ticket, Op::Flush, 0, None); self.pending.insert(ticket, Pending::Flush); Ok(ticket) } @@ -324,9 +325,9 @@ impl Session { for (ticket, outcome) in decided { let pending = self.pending.remove(&ticket).expect("blockd: an answer for no ticket"); let data = match (pending, outcome) { - (Pending::Read { arena, blocks }, Outcome::Done) => { - let mut data = vec![0u8; blocks as usize * BLOCK_BYTES]; - self.region.arena(arena, blocks).copy_out(0, &mut data); + (Pending::Read { run }, Outcome::Done) => { + let mut data = vec![0u8; run.span().len]; + self.region.arena(&run).copy_out(0, &mut data); Some(data) } _ => None, @@ -334,8 +335,8 @@ impl Session { answers.push(Answer { ticket, outcome, data }); } let released: Vec<_> = self.client.take_released().collect(); - for (first, blocks) in released { - self.arena.release(first, blocks); + for run in released { + self.arena.release(run); } } From b69deab11e67be23efaab46d7564e771831971d0 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 20:20:43 +0200 Subject: [PATCH 3/8] blockd_io hostile-head: wait on blockd's own words, not on the hang-up The panic arm showed the race: the client saw its connection close while `try_wait` still found blockd running, so the role went on to open a session nobody would answer and ended by the host's timeout instead of by name. The supervisor now hands every line blockd says to the role, which waits for `WITHHELD` and then for the session's `closed after`; a blockd that ends first disconnects the channel and the role fails saying so. Co-Authored-By: Claude Opus 5.5 --- tests/toyos-rust-tests/src/bin/blockd_io.rs | 58 ++++++++++----------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/tests/toyos-rust-tests/src/bin/blockd_io.rs b/tests/toyos-rust-tests/src/bin/blockd_io.rs index b746acb6e64..5682f93bd80 100644 --- a/tests/toyos-rust-tests/src/bin/blockd_io.rs +++ b/tests/toyos-rust-tests/src/bin/blockd_io.rs @@ -36,7 +36,7 @@ use std::io::{BufRead, BufReader, Write}; use std::os::toyos::process::{ChildExt, CommandExt}; use std::process::{Child, Command, Stdio}; use std::sync::atomic::Ordering; -use std::sync::mpsc::{self, Receiver}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; use std::time::{Duration, Instant}; use blockd::nvme::{Controller, Owner}; @@ -133,8 +133,8 @@ struct Blockd { acceptor: Acceptor, connector: Connector, child: Option, - /// Says once the running blockd has withheld a write's answer. - withheld: Option>, + /// Every line the running blockd says. + said: Option>, } impl Blockd { @@ -144,7 +144,7 @@ impl Blockd { fn with(syscap: SysCap, args: &[&str]) -> Self { let (acceptor, connector) = port::create().unwrap_or_else(|e| fail(format!("no port: {e:?}"))); - let mut blockd = Self { syscap, acceptor, connector, child: None, withheld: None }; + let mut blockd = Self { syscap, acceptor, connector, child: None, said: None }; blockd.spawn(args, false); blockd } @@ -178,7 +178,7 @@ impl Blockd { command.endow(&format!("{SERVE_PREFIX}{PORT}"), acceptor.0); let mut child = command.spawn().unwrap_or_else(|e| fail(format!("blockd did not start: {e}"))); let out = child.stdout.take().expect("piped"); - let (said, heard) = mpsc::channel(); + let (says, said) = mpsc::channel(); let mut kill = kill_on_withheld.then(|| { toyos_abi::syscall::dup(toyos_abi::RawHandle(child.as_raw_handle())) .unwrap_or_else(|e| fail(format!("blockd's handle would not duplicate: {e:?}"))) @@ -187,16 +187,32 @@ impl Blockd { for line in BufReader::new(out).lines().map_while(Result::ok) { println!("{line}"); if line.contains("WITHHELD") { - let _ = said.send(()); if let Some(handle) = kill.take() { let _ = toyos_abi::syscall::process_kill(handle); println!("blockd_io: blockd killed with the withheld write done on the device"); } } + let _ = says.send(line); } }); self.child = Some(child); - self.withheld = Some(heard); + self.said = Some(said); + } + + /// Wait, at most `bound`, for the running blockd to say a line holding + /// `needle`. + fn says(&self, needle: &str, bound: Duration) { + let said = self.said.as_ref().expect("spawned"); + let asked = Instant::now(); + loop { + let left = bound.saturating_sub(asked.elapsed()); + match said.recv_timeout(left) { + Ok(line) if line.contains(needle) => return, + Ok(_) => {} + Err(RecvTimeoutError::Timeout) => fail(format!("blockd did not say {needle:?} in {bound:?}")), + Err(RecvTimeoutError::Disconnected) => fail(format!("blockd ended before it said {needle:?}")), + } + } } /// End the running blockd, if one is, and wait for it to be gone. @@ -518,7 +534,7 @@ fn reset() { /// ring's depth behind the tail blockd published, so the answer finds no room: /// blockd ends that session, and serves the next. fn hostile_head() { - let mut blockd = Blockd::start(&["--silence-write", "1"]); + let blockd = Blockd::start(&["--silence-write", "1"]); let region = Region::create().unwrap_or_else(|e| fail(format!("a region: {e:?}"))); let conn = blockd.names().open(PORT).unwrap_or_else(|e| fail(format!("the port: {e:?}"))); let shared = region.share().unwrap_or_else(|e| fail(format!("a second handle: {e:?}"))); @@ -538,33 +554,13 @@ fn hostile_head() { } words[SQ_TAIL].store(1, Ordering::Release); conn.write_nonblock(&[1]).unwrap_or_else(|e| fail(format!("the doorbell: {e:?}"))); - let withheld = blockd.withheld.as_ref().expect("spawned"); - if withheld.recv_timeout(SILENCE_ENDS).is_err() { - fail(format!("blockd withheld no write's answer in {SILENCE_ENDS:?}")); - } + blockd.says("WITHHELD", SILENCE_ENDS); let tail = words[CQ_TAIL].load(Ordering::Acquire); words[CQ_HEAD].store(tail.wrapping_sub(DEPTH), Ordering::Release); conn.write_nonblock(&[1]).unwrap_or_else(|e| fail(format!("the doorbell: {e:?}"))); println!("blockd_io: with a write on the device, the client moved its completion head {DEPTH} behind the tail"); - let poller = Poller::new(1); - let asked = Instant::now(); - loop { - match conn.read_nonblock(&mut [0u8; 8]) { - Ok(0) => break, - Err(SyscallError::WouldBlock) => {} - Ok(_) => fail("blockd rang a session whose answer had no room".into()), - Err(_) => break, - } - let Some(left) = SILENCE_ENDS.checked_sub(asked.elapsed()) else { - fail(format!("blockd did not end the session in {SILENCE_ENDS:?}")); - }; - poller.watch(&conn, READABLE, 0); - poller.wait(1, left.as_nanos() as u64, |_| {}); - } - match blockd.child.as_mut().expect("spawned").try_wait() { - Ok(None) => println!("blockd_io: blockd ended the session and runs on"), - other => fail(format!("blockd ended with the session: {other:?}")), - } + blockd.says("closed after", SILENCE_ENDS); + println!("blockd_io: blockd ended the session and runs on"); let mut next = open(blockd.names(), TARGET); let block = pattern(0x6B, 0); match next.write(0, &block) { From 167b07c267e8140a3cdacb2a4336d877590b6a9c Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 21:21:12 +0200 Subject: [PATCH 4/8] review #552 r2: delete the wake and the second model; the block model back to main's speed, held to its queues both ways The wake had no caller: nobody called `before_sleep`, and both publish sites threw the `Wake` away. `Wake`, `Asleep`, the `sleep` words and their fences, `Word::fence`, the two fence controls, their loom test and issues/design-debt/a-block-publish-pays-for-a-wake-nobody-asks-for.md are deleted; the wake lands in T2 with its first caller. toyos-transport/src/model.rs was a second session model beside blockring's, which runs the same `Inflight`, `Producer` and `Consumer` through crash and reconnect. It is deleted, and `end-keeps-inflight` names `inflight::tests::an_end_answers_every_tag_once_and_a_late_completion_nothing`. Ring ends are made over `&[W; N]`: `new` holds the place to `N`, so `publish` and `release` cannot fail, and blockd's four `expect`s on them go. The one index into the words is `word`, under the invariant `new` checked. A request's run is its op's: `Op::Read(Run) | Op::Write(Run) | Op::Flush`. `run: Option` and the `expect`s that guarded it go. A tag's index takes the bits the table needs (6 for 64 slots) and the sequence the rest (26), so one slot's sequence no longer wraps every 2^16 fills. The block model: its state key names tags by first appearance and renders the client's table by its filled slots; its rings are four deep, so each fills and wraps within the bounds; the client's table is as deep as the rings; and a ring whose queue is empty must pop nothing. Deleting `Consumer::pop`'s `ready` decrement now reds all four model tests. The bounds are main's. blockd_io's hostile request is `Request::encode`d. Filed issues/filesystem/a-read-or-write-answered-lost-is-never-answered.md, pre-existing on main. Co-Authored-By: Claude Opus 5.5 --- ...publish-pays-for-a-wake-nobody-asks-for.md | 19 - ...r-write-answered-lost-is-never-answered.md | 24 ++ src/ci.rs | 6 +- tests/toyos-rust-tests/src/bin/blockd_io.rs | 11 +- toyos-blockring/Cargo.toml | 2 +- toyos-blockring/src/client.rs | 99 +++--- toyos-blockring/src/entry.rs | 65 ++-- toyos-blockring/src/layout.rs | 23 +- toyos-blockring/src/model.rs | 172 ++++++--- toyos-blockring/src/server.rs | 16 +- toyos-transport/Cargo.toml | 15 +- toyos-transport/src/inflight.rs | 72 ++-- toyos-transport/src/lib.rs | 88 +---- toyos-transport/src/model.rs | 328 ------------------ toyos-transport/src/queue.rs | 165 ++++----- toyos-transport/tests/loom.rs | 61 +--- userland/blockd/src/main.rs | 16 +- userland/blockd/src/region.rs | 4 +- userland/blockd/src/session.rs | 12 +- 19 files changed, 428 insertions(+), 770 deletions(-) delete mode 100644 issues/design-debt/a-block-publish-pays-for-a-wake-nobody-asks-for.md create mode 100644 issues/filesystem/a-read-or-write-answered-lost-is-never-answered.md delete mode 100644 toyos-transport/src/model.rs diff --git a/issues/design-debt/a-block-publish-pays-for-a-wake-nobody-asks-for.md b/issues/design-debt/a-block-publish-pays-for-a-wake-nobody-asks-for.md deleted file mode 100644 index 77b87e78a50..00000000000 --- a/issues/design-debt/a-block-publish-pays-for-a-wake-nobody-asks-for.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-27 ---- - -# A block publish pays for a wake nobody asks for - -Every `Producer::publish` in `toyos-transport` stores its tail, runs a `SeqCst` -fence and loads the consumer's `sleep` word, so that it can answer -`Wake::Peer` or `Wake::Busy`. blockd (`userland/blockd/src/main.rs`, -`publish`) and its client (`userland/blockd/src/session.rs`, `pump`) ring the -connection on every publish whatever the answer, and neither end calls -`Consumer::before_sleep`, so the answer is always `Busy`: the fence and the -load are paid once per batch and buy nothing. Their cost has not been -measured. - -**Exit condition.** blockd and its client sleep through `before_sleep` and -ring their peer only on `Wake::Peer`. diff --git a/issues/filesystem/a-read-or-write-answered-lost-is-never-answered.md b/issues/filesystem/a-read-or-write-answered-lost-is-never-answered.md new file mode 100644 index 00000000000..15b6be578c3 --- /dev/null +++ b/issues/filesystem/a-read-or-write-answered-lost-is-never-answered.md @@ -0,0 +1,24 @@ +--- +status: assigned +kind: defect +opened: 2026-09-27 +--- + +# A read or write answered Lost is never answered + +`Client::complete` (`toyos-blockring/src/client.rs`) takes a completion's tag +off the wire before it looks at the status. A user read or write answered +`Status::Lost` is then refused as `Violation::Entry`, and the session ends; but +the tag is no longer on the wire, so `Client::session_ended` does not answer it +`Refused`, and nothing ever answers its ticket. blockd's client +(`userland/blockd/src/session.rs`) keeps that ticket in `pending` for good, +across every reconnect. That breaks the module's own "every request asked for +is answered exactly once". Only a server that breaks the protocol writes that +completion; blockd does not. + +Held by the orchestrator. + +**Exit condition.** A completion the client refuses leaves its tag answered by +the session's end — refused before the tag is taken off the wire, or answered +where it is refused — and a model or unit test in which a server answers a +write `Lost` goes red without the fix and green with it. diff --git a/src/ci.rs b/src/ci.rs index c4ddf71b98a..4210b98a9a9 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -369,10 +369,10 @@ pub(crate) const CONTROLS: &[Control] = &[ "what_a_flush_calls_durable_is_on_the_medium ... FAILED", ]), red(TRANSPORT, "publish-relaxed", Some("loom"), &["a_published_entry_is_read_whole ... FAILED"]), - red(TRANSPORT, "no-wake-fence", Some("loom"), &["a_publish_and_a_sleep_cannot_both_miss ... FAILED"]), - red(TRANSPORT, "no-sleep-fence", Some("loom"), &["a_publish_and_a_sleep_cannot_both_miss ... FAILED"]), red(TRANSPORT, "no-clamp", Some("loom"), &["a_hostile_producer_yields_entries_or_a_violation ... FAILED"]), - red(TRANSPORT, "end-keeps-inflight", None, &["every_tag_is_answered_exactly_once ... FAILED"]), + red(TRANSPORT, "end-keeps-inflight", None, &[ + "an_end_answers_every_tag_once_and_a_late_completion_nothing ... FAILED", + ]), ]; /// Whether a control's run showed its teeth. diff --git a/tests/toyos-rust-tests/src/bin/blockd_io.rs b/tests/toyos-rust-tests/src/bin/blockd_io.rs index 5682f93bd80..8892dac216a 100644 --- a/tests/toyos-rust-tests/src/bin/blockd_io.rs +++ b/tests/toyos-rust-tests/src/bin/blockd_io.rs @@ -51,9 +51,9 @@ use toyos::syscap::SysCap; use toyos::AsHandle; use toyos_abi::part::PartGuid; use toyos_abi::syscall::{DeviceType, PciId, SyscallError, DEV_PREFIX, SERVE_PREFIX, SYSCAP_LABEL}; -use toyos_blockring::layout::{CQ_HEAD, CQ_TAIL, DEPTH, SQ_BASE, SQ_TAIL}; +use toyos_blockring::layout::{ARENA, CQ_HEAD, CQ_TAIL, DEPTH, SQ_BASE, SQ_TAIL}; use toyos_blockring::wire::{self, Refusal}; -use toyos_blockring::{BLOCK_BYTES, MAX_REQUEST_BLOCKS, PORT}; +use toyos_blockring::{Op, Request, BLOCK_BYTES, MAX_REQUEST_BLOCKS, PORT}; const SELF: &str = "/system/bin/test_rs_blockd_io"; @@ -99,8 +99,7 @@ const NARROW: u64 = 128 * 1024 * 1024; const AIMED: Duration = Duration::from_secs(10); /// How long `hostile-head` waits for blockd to withhold its write's answer, and -/// then for the reset that ends its session: a liveness bound past blockd's -/// ten seconds of silence. +/// then for the reset that ends its session: a liveness bound. const SILENCE_ENDS: Duration = Duration::from_secs(30); fn guid(text: &str) -> [u8; 16] { @@ -549,7 +548,9 @@ fn hostile_head() { // A write of the slot's block 0 from arena block 0 under tag 1, as the // words a client puts on the request ring, published and rung. let words = region.words(); - for (at, word) in [2, 1, 0, 0, 1, 0, 0, 0].into_iter().enumerate() { + let run = ARENA.run(0, 1).unwrap_or_else(|| fail("arena block 0 is no run".into())); + let write = Request { op: Op::Write(run), tag: 1, lba: 0 }; + for (at, word) in write.encode().into_iter().enumerate() { words[SQ_BASE + at].store(word, Ordering::Relaxed); } words[SQ_TAIL].store(1, Ordering::Release); diff --git a/toyos-blockring/Cargo.toml b/toyos-blockring/Cargo.toml index 7f23ba250e7..f9cbb2b18ca 100644 --- a/toyos-blockring/Cargo.toml +++ b/toyos-blockring/Cargo.toml @@ -38,7 +38,7 @@ mutate-no-reissue-after-loss = [] # the server half's bookkeeping, decided where the kernel's block layer decides # it today. toyos-blockhold = { path = "../toyos-blockhold" } -# The rings on the session page, and the schema the protocol is to them. +# The rings on the session page. toyos-transport = { path = "../toyos-transport" } [lints.rust] diff --git a/toyos-blockring/src/client.rs b/toyos-blockring/src/client.rs index 3ecce711e14..9bae51b169f 100644 --- a/toyos-blockring/src/client.rs +++ b/toyos-blockring/src/client.rs @@ -103,7 +103,6 @@ enum Kind { struct Queued { op: Op, lba: u64, - run: Option, kind: Kind, } @@ -116,13 +115,14 @@ struct Sent { losses: u64, } +/// `D` is the most requests it has on the wire at once: a session's [`DEPTH`]. #[derive(Clone, Debug, PartialEq, Eq, Hash, Default)] -pub struct Client { +pub struct Client { up: bool, /// Not yet on the wire, in order. Writes being issued again are always at /// the front, then any flush attempt waiting on them. outbox: VecDeque, - wire: Inflight, + wire: Inflight, acked: VecDeque, next_seq: u64, /// Losses taken so far: a flush sent before one says nothing about what it @@ -137,22 +137,20 @@ pub struct Client { gave_up: bool, } -impl Client { +impl Client { /// A client with no session yet: requests wait until /// [`Self::session_started`]. pub fn new() -> Self { Self::default() } - /// Ask for a read or write at `lba` through the arena blocks `run`, or for - /// a flush, which names neither. - pub fn submit(&mut self, ticket: Ticket, op: Op, lba: u64, run: Option) { - assert_eq!(run.is_some(), op != Op::Flush, "a read or a write names a run, and a flush none"); + /// Ask for a read or write at `lba`, or for a flush. + pub fn submit(&mut self, ticket: Ticket, op: Op, lba: u64) { let kind = match op { Op::Flush => Kind::Flush(ticket), - Op::Read | Op::Write => Kind::User(ticket), + Op::Read(_) | Op::Write(_) => Kind::User(ticket), }; - self.outbox.push_back(Queued { op, lba, run, kind }); + self.outbox.push_back(Queued { op, lba, kind }); } fn reissuing(&self) -> bool { @@ -161,7 +159,7 @@ impl Client { } /// The next request to put on the wire, tagged; `None` while there is no - /// session, nothing to send, [`DEPTH`] on the wire, or what is next must + /// session, nothing to send, `D` on the wire, or what is next must /// wait for writes being issued again. pub fn next_request(&mut self) -> Option { if !self.up { @@ -176,7 +174,7 @@ impl Client { Kind::Reissue(acked) => { let blocked = self.wire.values().any(|sent| { let q = sent.queued; - q.op == Op::Write && q.run.is_some_and(|run| acked.overlaps_range(q.lba, run.count())) + matches!(q.op, Op::Write(run) if acked.overlaps_range(q.lba, run.count())) }); if blocked { return None; @@ -193,7 +191,7 @@ impl Client { if matches!(front.kind, Kind::Reissue(_)) { self.reissued += 1; } - Some(Request { op: front.op, tag, lba: front.lba, run: front.run }) + Some(Request { op: front.op, tag, lba: front.lba }) } /// What the server answered. @@ -203,9 +201,8 @@ impl Client { match (q.kind, completion.status) { (Kind::User(ticket), Status::Ok) => { match q.op { - Op::Write => { + Op::Write(run) => { let seq = self.bump(); - let run = q.run.expect("a write names its run"); let acked = Acked { lba: q.lba, run, first: seq, seq, attempts: 0 }; // Sent before a loss it did not see: the device may // have taken it and lost it, and an earlier write it @@ -217,13 +214,15 @@ impl Client { self.acked.push_back(acked); } } - Op::Read => self.released.extend(q.run), + Op::Read(run) => self.released.push_back(run), Op::Flush => return Err(Violation::Entry), } self.answers.push_back((ticket, Outcome::Done)); } (Kind::User(ticket), status) => { - self.released.extend(q.run); + if let Op::Read(run) | Op::Write(run) = q.op { + self.released.push_back(run); + } let outcome = match status { Status::Invalid => Outcome::Invalid, Status::Device => Outcome::Device, @@ -296,7 +295,9 @@ impl Client { Kind::User(ticket) => { #[cfg(not(feature = "mutate-session-end-forgets"))] { - self.released.extend(q.run); + if let Op::Read(run) | Op::Write(run) = q.op { + self.released.push_back(run); + } self.answers.push_back((ticket, Outcome::Refused)); } #[cfg(feature = "mutate-session-end-forgets")] @@ -338,9 +339,9 @@ impl Client { self.outbox.is_empty() && self.wire.values().next().is_none() && self.acked.is_empty() } - /// How many requests are on the wire. - pub fn on_the_wire(&self) -> usize { - self.wire.values().count() + /// The tags on the wire. + pub fn on_the_wire(&self) -> impl Iterator + '_ { + self.wire.tags() } /// How many writes have gone on the wire again after a loss. @@ -370,7 +371,7 @@ impl Client { Kind::User(_) | Kind::Flush(_) => true, }) .unwrap_or(self.outbox.len()); - self.outbox.insert(at, Queued { op: Op::Write, lba: acked.lba, run: Some(acked.run), kind: Kind::Reissue(acked) }); + self.outbox.insert(at, Queued { op: Op::Write(acked.run), lba: acked.lba, kind: Kind::Reissue(acked) }); } /// Ask the flush `ticket` again, once every write being issued again is @@ -381,7 +382,7 @@ impl Client { .iter() .position(|q| !matches!(q.kind, Kind::Reissue(_) | Kind::Flush(_))) .unwrap_or(self.outbox.len()); - self.outbox.insert(at, Queued { op: Op::Flush, lba: 0, run: None, kind: Kind::Flush(ticket) }); + self.outbox.insert(at, Queued { op: Op::Flush, lba: 0, kind: Kind::Flush(ticket) }); } /// The device may no longer hold any write acknowledged and not covered: @@ -419,14 +420,18 @@ mod tests { use super::*; use crate::layout::ARENA; + fn run(first: u32, count: u32) -> Run { + ARENA.run(first, count).unwrap() + } + fn answer(client: &mut Client, request: Request, status: Status) { client.complete(Completion { tag: request.tag, status }).unwrap(); } #[test] fn nothing_goes_out_before_a_session() { - let mut client = Client::new(); - client.submit(1, Op::Read, 0, ARENA.run(0, 1)); + let mut client: Client = Client::new(); + client.submit(1, Op::Read(run(0, 1)), 0); assert_eq!(client.next_request(), None); client.session_started(); assert!(client.next_request().is_some()); @@ -434,9 +439,9 @@ mod tests { #[test] fn a_second_answer_for_one_tag_is_a_violation() { - let mut client = Client::new(); + let mut client: Client = Client::new(); client.session_started(); - client.submit(1, Op::Read, 0, ARENA.run(0, 1)); + client.submit(1, Op::Read(run(0, 1)), 0); let r = client.next_request().unwrap(); answer(&mut client, r, Status::Ok); assert_eq!(client.complete(Completion { tag: r.tag, status: Status::Ok }), Err(Violation::Tag)); @@ -447,16 +452,16 @@ mod tests { /// `Durable` only from the second. #[test] fn a_lost_flush_reissues_then_asks_again() { - let mut client = Client::new(); + let mut client: Client = Client::new(); client.session_started(); - client.submit(1, Op::Write, 5, ARENA.run(3, 2)); + client.submit(1, Op::Write(run(3, 2)), 5); let w = client.next_request().unwrap(); answer(&mut client, w, Status::Ok); - client.submit(2, Op::Flush, 0, None); + client.submit(2, Op::Flush, 0); let f = client.next_request().unwrap(); answer(&mut client, f, Status::Lost); let again = client.next_request().unwrap(); - assert_eq!((again.op, again.lba, again.run), (Op::Write, 5, ARENA.run(3, 2))); + assert_eq!((again.op, again.lba), (Op::Write(run(3, 2)), 5)); assert_eq!(client.next_request(), None, "the flush waits for the write"); answer(&mut client, again, Status::Ok); let f2 = client.next_request().unwrap(); @@ -464,7 +469,7 @@ mod tests { answer(&mut client, f2, Status::Ok); let answers: Vec<_> = client.take_answers().collect(); assert_eq!(answers, [(1, Outcome::Done), (2, Outcome::Durable)]); - assert_eq!(client.take_released().collect::>(), ARENA.run(3, 2).into_iter().collect::>()); + assert_eq!(client.take_released().collect::>(), [run(3, 2)]); assert!(client.quiet()); } @@ -472,22 +477,22 @@ mod tests { /// other, in the order they were first acknowledged. #[test] fn overlapping_writes_go_out_again_in_order() { - let mut client = Client::new(); + let mut client: Client = Client::new(); client.session_started(); - client.submit(1, Op::Write, 0, ARENA.run(0, 2)); + client.submit(1, Op::Write(run(0, 2)), 0); let a = client.next_request().unwrap(); answer(&mut client, a, Status::Ok); - client.submit(2, Op::Write, 1, ARENA.run(2, 1)); + client.submit(2, Op::Write(run(2, 1)), 1); let b = client.next_request().unwrap(); answer(&mut client, b, Status::Ok); client.session_ended(); client.session_started(); let first = client.next_request().unwrap(); - assert_eq!((first.lba, first.run), (0, ARENA.run(0, 2))); + assert_eq!((first.op, first.lba), (Op::Write(run(0, 2)), 0)); assert_eq!(client.next_request(), None, "the overlapping one waits"); answer(&mut client, first, Status::Ok); let second = client.next_request().unwrap(); - assert_eq!((second.lba, second.run), (1, ARENA.run(2, 1))); + assert_eq!((second.op, second.lba), (Op::Write(run(2, 1)), 1)); } /// A write the device refused on every reissue is gone, and its caller was @@ -495,24 +500,24 @@ mod tests { /// never durable. #[test] fn a_write_given_up_poisons_every_later_flush() { - let mut client = Client::new(); + let mut client: Client = Client::new(); client.session_started(); - client.submit(1, Op::Write, 0, ARENA.run(0, 1)); + client.submit(1, Op::Write(run(0, 1)), 0); let w = client.next_request().unwrap(); answer(&mut client, w, Status::Ok); client.session_ended(); for _ in 0..=MAX_ATTEMPTS { client.session_started(); let again = client.next_request().unwrap(); - assert_eq!((again.op, again.lba), (Op::Write, 0)); + assert_eq!((again.op, again.lba), (Op::Write(run(0, 1)), 0)); answer(&mut client, again, Status::Device); } - client.submit(2, Op::Flush, 0, None); + client.submit(2, Op::Flush, 0); let f = client.next_request().unwrap(); assert_eq!(f.op, Op::Flush); answer(&mut client, f, Status::Ok); assert_eq!(client.take_answers().collect::>(), [(1, Outcome::Done), (2, Outcome::Device)]); - client.submit(3, Op::Flush, 0, None); + client.submit(3, Op::Flush, 0); let f = client.next_request().unwrap(); answer(&mut client, f, Status::Ok); assert_eq!(client.take_answers().collect::>(), [(3, Outcome::Device)]); @@ -520,16 +525,16 @@ mod tests { #[test] fn what_was_on_the_wire_at_the_end_is_refused_and_what_was_not_waits() { - let mut client = Client::new(); + let mut client: Client = Client::new(); client.session_started(); - client.submit(1, Op::Write, 0, ARENA.run(0, 1)); - client.submit(2, Op::Read, 4, ARENA.run(1, 1)); + client.submit(1, Op::Write(run(0, 1)), 0); + client.submit(2, Op::Read(run(1, 1)), 4); let _ = client.next_request().unwrap(); client.session_ended(); assert_eq!(client.take_answers().collect::>(), [(1, Outcome::Refused)]); - assert_eq!(client.take_released().collect::>(), ARENA.run(0, 1).into_iter().collect::>()); + assert_eq!(client.take_released().collect::>(), [run(0, 1)]); client.session_started(); let r = client.next_request().unwrap(); - assert_eq!((r.op, r.lba), (Op::Read, 4)); + assert_eq!((r.op, r.lba), (Op::Read(run(1, 1)), 4)); } } diff --git a/toyos-blockring/src/entry.rs b/toyos-blockring/src/entry.rs index 5fdd0948848..7dc9948a303 100644 --- a/toyos-blockring/src/entry.rs +++ b/toyos-blockring/src/entry.rs @@ -8,38 +8,31 @@ use toyos_transport::{Run, Untrusted}; use crate::layout::{ARENA, CQE_WORDS, MAX_REQUEST_BLOCKS, SQE_WORDS}; -/// What a request asks of the partition. +/// What a request asks of the partition, and the arena blocks the data is in +/// or goes to. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Op { /// The run's blocks from the partition's block `lba` into the arena. - Read, + Read(Run), /// The run's blocks from the arena to the partition's block `lba`. - Write, + Write(Run), /// Every write acknowledged before this was submitted, onto the medium. Flush, } -impl Op { - const fn word(self) -> u32 { - match self { - Self::Read => 1, - Self::Write => 2, - Self::Flush => 3, - } - } -} +const READ: u32 = 1; +const WRITE: u32 = 2; +const FLUSH: u32 = 3; /// One request. `lba` is the partition's own block number, from 0: nothing in /// this protocol names a device block, so a neighbour's blocks have no -/// spelling. A flush carries no range: its `lba` is zero and it names no run. +/// spelling. A flush carries no range: its `lba` is zero. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Request { pub op: Op, /// The client's name for it, echoed by its completion. pub tag: u32, pub lba: u64, - /// The arena blocks the data is in or goes to. - pub run: Option, } /// Why a request was answered without being done. @@ -53,8 +46,12 @@ pub enum Refused { impl Request { pub fn encode(&self) -> [u32; SQE_WORDS] { - let (first, count) = self.run.map_or((0, 0), |run| (run.first(), run.count())); - [self.op.word(), self.tag, self.lba as u32, (self.lba >> 32) as u32, count, first, 0, 0] + let (op, first, count) = match self.op { + Op::Read(run) => (READ, run.first(), run.count()), + Op::Write(run) => (WRITE, run.first(), run.count()), + Op::Flush => (FLUSH, 0, 0), + }; + [op, self.tag, self.lba as u32, (self.lba >> 32) as u32, count, first, 0, 0] } /// The request these words are, bounded against a partition of @@ -65,23 +62,25 @@ impl Request { let [op, tag, lba_low, lba_high, blocks, arena, reserved @ ..] = words; let tag = opaque(tag); let refused = Refused::Malformed { tag }; - let op = [Op::Read, Op::Write, Op::Flush].into_iter().find(|o| op.is(o.word())).ok_or(refused)?; if !reserved.iter().all(|word| word.is(0)) { return Err(refused); } let lba = u64::from(opaque(lba_low)) | (u64::from(opaque(lba_high)) << 32); - if op == Op::Flush { - if lba != 0 || !blocks.is(0) || !arena.is(0) { - return Err(refused); - } - return Ok(Self { op, tag, lba, run: None }); - } + let transfer: fn(Run) -> Op = if op.is(READ) { + Op::Read + } else if op.is(WRITE) { + Op::Write + } else if op.is(FLUSH) && lba == 0 && blocks.is(0) && arena.is(0) { + return Ok(Self { op: Op::Flush, tag, lba }); + } else { + return Err(refused); + }; let run = Run::decode(arena, blocks, &ARENA).map_err(|_| refused)?; let blocks = run.count(); if blocks > MAX_REQUEST_BLOCKS || lba.checked_add(u64::from(blocks)).is_none_or(|end| end > partition_blocks) { return Err(refused); } - Ok(Self { op, tag, lba, run: Some(run) }) + Ok(Self { op: transfer(run), tag, lba }) } } @@ -153,6 +152,10 @@ mod tests { const PARTITION: u64 = 1000; + fn run(first: u32, count: u32) -> Run { + ARENA.run(first, count).unwrap() + } + fn peer(words: [u32; N]) -> [Untrusted; N] { words.map(Untrusted::new) } @@ -161,13 +164,13 @@ mod tests { fn a_request_survives_its_words() { let last = ARENA.slots() - MAX_REQUEST_BLOCKS; for request in [ - Request { op: Op::Read, tag: 7, lba: 999, run: ARENA.run(0, 1) }, - Request { op: Op::Write, tag: u32::MAX, lba: 0, run: ARENA.run(last, MAX_REQUEST_BLOCKS) }, - Request { op: Op::Flush, tag: 0, lba: 0, run: None }, + Request { op: Op::Read(run(0, 1)), tag: 7, lba: 999 }, + Request { op: Op::Write(run(last, MAX_REQUEST_BLOCKS)), tag: u32::MAX, lba: 0 }, + Request { op: Op::Flush, tag: 0, lba: 0 }, ] { assert_eq!(Request::decode(peer(request.encode()), PARTITION), Ok(request)); } - let wide = Request { op: Op::Read, tag: 1, lba: 1 << 40, run: ARENA.run(3, 2) }; + let wide = Request { op: Op::Read(run(3, 2)), tag: 1, lba: 1 << 40 }; assert_eq!(Request::decode(peer(wide.encode()), u64::MAX), Ok(wide)); } @@ -175,7 +178,7 @@ mod tests { /// refusal still carries its tag. #[test] fn a_request_outside_its_bounds_is_refused_by_tag() { - let good = Request { op: Op::Write, tag: 42, lba: 10, run: ARENA.run(5, 2) }; + let good = Request { op: Op::Write(run(5, 2)), tag: 42, lba: 10 }; let with = |at: usize, word: u32| { let mut words = good.encode(); words[at] = word; @@ -191,7 +194,7 @@ mod tests { ("past the partition", Request { lba: PARTITION - 1, ..good }.encode()), ("lba wraps", Request { lba: u64::MAX, ..good }.encode()), ("reserved word", with(7, 1)), - ("a flush with a range", Request { op: Op::Flush, ..good }.encode()), + ("a flush with a range", with(0, 3)), ]; for (what, words) in cases { assert_eq!( diff --git a/toyos-blockring/src/layout.rs b/toyos-blockring/src/layout.rs index 457be042b46..f0d108a5d71 100644 --- a/toyos-blockring/src/layout.rs +++ b/toyos-blockring/src/layout.rs @@ -1,8 +1,7 @@ //! Where everything is on a session's region, in 32-bit words from its start. //! -//! The four ring indices sit on cache lines of their own, each consumer's -//! `sleep` word on its head's, so the client's stores and the server's never -//! share a line. +//! The four ring indices sit on cache lines of their own, so the client's +//! stores to its two and the server's to its two never share a line. use toyos_transport::{Consumer, Cursors, Geometry, Place, Producer, Violation, Word}; @@ -26,13 +25,11 @@ pub const DEPTH: u32 = 64; /// command splits it; one that takes more is still asked for no more than this. pub const MAX_REQUEST_BLOCKS: u32 = 32; -/// Index words. The server writes [`SQ_HEAD`], [`SQ_SLEEP`] and [`CQ_TAIL`], -/// the client the other three. +/// Index words. The server writes [`SQ_HEAD`] and [`CQ_TAIL`], the client the +/// other two. pub const SQ_HEAD: usize = 0; -pub const SQ_SLEEP: usize = 1; pub const SQ_TAIL: usize = 16; pub const CQ_HEAD: usize = 32; -pub const CQ_SLEEP: usize = 33; pub const CQ_TAIL: usize = 48; /// Words per request entry, and where the request ring starts. @@ -48,26 +45,26 @@ pub const RING_WORDS: usize = CQ_BASE + DEPTH as usize * CQE_WORDS; /// The request ring and the completion ring. pub const REQUESTS: Place = - Place { cursors: Cursors { head: SQ_HEAD, tail: SQ_TAIL, sleep: SQ_SLEEP }, entries: SQ_BASE }; + Place { cursors: Cursors { head: SQ_HEAD, tail: SQ_TAIL }, entries: SQ_BASE }; pub const COMPLETIONS: Place = - Place { cursors: Cursors { head: CQ_HEAD, tail: CQ_TAIL, sleep: CQ_SLEEP }, entries: CQ_BASE }; + Place { cursors: Cursors { head: CQ_HEAD, tail: CQ_TAIL }, entries: CQ_BASE }; /// A client's two ends: requests out, completions in. -pub type ClientRings = (Producer, Consumer); +pub type ClientRings = (Producer, Consumer); /// A server's two ends: requests in, completions out. -pub type ServerRings = (Consumer, Producer); +pub type ServerRings = (Consumer, Producer); /// The client's ends of a session page, every word it owns set to 0. Done /// before the page is sent to a server, and again before it is sent to the /// next one. -pub fn client(page: &[W]) -> Result { +pub fn client(page: &[W; RING_WORDS]) -> Result { Ok((Producer::new(page, REQUESTS)?, Consumer::new(page, COMPLETIONS)?)) } /// The server's ends of a session page it was sent, every word it owns set to /// 0. Whatever the client left in its own is bounded when first looked at. -pub fn server(page: &[W]) -> Result { +pub fn server(page: &[W; RING_WORDS]) -> Result { Ok((Consumer::new(page, REQUESTS)?, Producer::new(page, COMPLETIONS)?)) } diff --git a/toyos-blockring/src/model.rs b/toyos-blockring/src/model.rs index 662d6b593b6..5217ef58daf 100644 --- a/toyos-blockring/src/model.rs +++ b/toyos-blockring/src/model.rs @@ -1,8 +1,8 @@ //! Every ordering of a client, a server, a device and their failures. //! //! A scripted caller asks for writes and flushes; the rings between the client -//! and the server are queues, and the session page's own rings are driven -//! beside them and held to them ([`Queues`]); the server is [`ServerSession`] +//! and the server are queues, and rings of the transport's own are driven +//! beside them and held to them both ways ([`Queues`]); the server is [`ServerSession`] //! over a device of two blocks with a volatile cache. [`explore`] runs, depth //! first and exhaustively, every interleaving of: the caller asking for its //! next step, the server taking a request, the device completing any one it @@ -31,16 +31,15 @@ use alloc::format; use alloc::string::String; use alloc::vec::Vec; use core::cell::Cell; -use core::fmt; use core::sync::atomic::Ordering; use std::collections::HashSet; use toyos_blockhold::Holds; -use toyos_transport::{Untrusted, Word}; +use toyos_transport::{Consumer, Cursors, Place, Producer, Untrusted, Word}; use crate::client::{Client, Outcome, Ticket, MAX_ATTEMPTS}; use crate::entry::{Completion, Op, Request}; -use crate::layout::{self, ClientRings, ServerRings, ARENA, CQE_WORDS, RING_WORDS, SQE_WORDS}; +use crate::layout::{ARENA, CQE_WORDS, SQE_WORDS}; use crate::server::{ServerSession, Taken}; const BLOCKS: usize = 2; @@ -69,6 +68,8 @@ pub struct Explored { /// End states where a flush was answered the device's refusal: the client /// gave a write or a flush up. pub given_up: usize, + /// Whether the request ring, and the completion ring, held [`DEPTH`]. + pub filled: [bool; 2], } /// One thing the scripted caller asks for. @@ -97,50 +98,57 @@ impl Word for Shared { fn store(&self, value: u32, _: Ordering) { self.0.set(value) } - fn fence() {} } +/// How deep the model's rings are: shallow enough that a script fills and +/// wraps each. +const DEPTH: u32 = 4; +const SQ: Place = Place { cursors: Cursors { head: 0, tail: 1 }, entries: 4 }; +const CQ: Place = Place { cursors: Cursors { head: 2, tail: 3 }, entries: SQ.entries + DEPTH as usize * SQE_WORDS }; +const WORDS: usize = CQ.entries + DEPTH as usize * CQE_WORDS; const RING: &str = "the page holds every ring word"; +type ClientEnds = (Producer, Consumer); +type ServerEnds = (Consumer, Producer); + /// The rings between the client and the server, twice: as queues, the -/// reference, and as the session page's own rings, whose every entry either -/// end takes is held equal to the queue's. A state is rendered by its queues -/// alone, so the search tells apart what the protocol can, not where on the -/// page it is. +/// reference, and as the transport's rings, which give either end exactly what +/// the queue does — every entry it holds, and nothing when it holds none. A +/// state is keyed by its queues alone ([`key`]), so the search tells apart +/// what the protocol can, not where on the page it is. #[derive(Clone)] struct Queues { sq: VecDeque, cq: VecDeque, - page: Vec, - client: ClientRings, - server: ServerRings, -} - -impl fmt::Debug for Queues { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Queues").field("sq", &self.sq).field("cq", &self.cq).finish() - } + page: Box<[Shared; WORDS]>, + client: ClientEnds, + server: ServerEnds, } impl Queues { fn new() -> Self { - let page: Vec = (0..RING_WORDS).map(|_| Shared(Cell::new(0))).collect(); - let (client, server) = (layout::client(&page).expect(RING), layout::server(&page).expect(RING)); + let page = Box::new(core::array::from_fn(|_| Shared(Cell::new(0)))); + let (client, server) = Self::ends(&page); Self { sq: VecDeque::new(), cq: VecDeque::new(), page, client, server } } + /// Both ends over `page`, every cursor stored 0. + fn ends(page: &[Shared; WORDS]) -> (ClientEnds, ServerEnds) { + let client = (Producer::new(page, SQ).expect(RING), Consumer::new(page, CQ).expect(RING)); + (client, (Consumer::new(page, SQ).expect(RING), Producer::new(page, CQ).expect(RING))) + } + /// The session is over: what either queue held is gone, and the next /// session's two ends start over the same page. fn reset(&mut self) { self.sq.clear(); self.cq.clear(); - self.client = layout::client(&self.page).expect(RING); - self.server = layout::server(&self.page).expect(RING); + (self.client, self.server) = Self::ends(&self.page); } fn send(&mut self, request: Request) { assert_eq!(self.client.0.push(&self.page, request.encode()), Ok(true), "a request past the ring's depth"); - let _ = self.client.0.publish(&self.page).expect(RING); + self.client.0.publish(&self.page); self.sq.push_back(request); } @@ -149,14 +157,14 @@ impl Queues { let want = self.sq.pop_front()?; let words = self.server.0.pop(&self.page).expect("an honest client's tail"); let words = words.expect("the ring holds what the queue does"); - self.server.0.release(&self.page).expect(RING); + self.server.0.release(&self.page); assert_eq!(Request::decode(words, u64::MAX), Ok(want), "the ring gave the server what the queue holds"); Some(words) } fn post(&mut self, c: Completion) { assert_eq!(self.server.1.push(&self.page, c.encode()), Ok(true), "a completion past the ring's depth"); - let _ = self.server.1.publish(&self.page).expect(RING); + self.server.1.publish(&self.page); self.cq.push_back(c); } @@ -165,23 +173,31 @@ impl Queues { let want = self.cq.pop_front()?; let words: [Untrusted; CQE_WORDS] = self.client.1.pop(&self.page).expect("an honest server's tail").expect("the ring holds what the queue does"); - self.client.1.release(&self.page).expect(RING); + self.client.1.release(&self.page); assert_eq!(Completion::decode(words), Some(want), "the ring gave the client what the queue holds"); Some(want) } + + /// A ring whose queue is empty gives nothing. + fn hold_empty(&mut self) { + if self.sq.is_empty() { + assert_eq!(self.server.0.pop(&self.page), Ok(None), "the request ring gave what the queue does not hold"); + } + if self.cq.is_empty() { + assert_eq!(self.client.1.pop(&self.page), Ok(None), "the completion ring gave what the queue does not hold"); + } + } } -#[derive(Clone, Debug)] +#[derive(Clone)] struct Server { session: ServerSession, holds: Holds, } -#[derive(Clone, Debug)] +#[derive(Clone)] struct World { - /// Boxed: its tag table has a slot for every entry of the ring, and a - /// frame of the search holds several worlds. - client: Box, + client: Client<{ DEPTH as usize }>, next: usize, /// Every answer each ticket has had. answers: BTreeMap>, @@ -205,12 +221,12 @@ struct World { struct Run<'a> { script: &'a [Step], - /// Visited states, keyed by their whole rendering: `Holds` carries no - /// `Hash`, and a rendering is exact. + /// Visited states, by [`key`]. seen: HashSet, broken: Option<(Law, String)>, ends: usize, given_up: usize, + filled: [bool; 2], /// Whether a flush may end answered `Device`. may_give_up: bool, /// The steps from the start to here, for a failure to name. @@ -220,7 +236,7 @@ struct Run<'a> { /// Explore `script` against at most `failures`. pub fn explore(script: &[Step], failures: Failures) -> Explored { let mut world = World { - client: Box::new(Client::new()), + client: Client::new(), next: 0, answers: BTreeMap::new(), acked_before: BTreeMap::new(), @@ -236,10 +252,18 @@ pub fn explore(script: &[Step], failures: Failures) -> Explored { }; connect(&mut world); let may_give_up = failures.total() > MAX_ATTEMPTS; - let mut run = - Run { script, seen: HashSet::new(), broken: None, ends: 0, given_up: 0, may_give_up, path: Vec::new() }; + let mut run = Run { + script, + seen: HashSet::new(), + broken: None, + ends: 0, + given_up: 0, + filled: [false; 2], + may_give_up, + path: Vec::new(), + }; dfs(&mut run, world); - Explored { broken: run.broken, ends: run.ends, given_up: run.given_up } + Explored { broken: run.broken, ends: run.ends, given_up: run.given_up, filled: run.filled } } fn connect(world: &mut World) { @@ -272,9 +296,8 @@ fn write_of(script: &[Step], ticket: Ticket) -> Option<(u64, u8)> { /// the cache onto the medium. fn apply(script: &[Step], cache: &mut [Option; BLOCKS], media: &mut [u8; BLOCKS], request: Request) { match request.op { - Op::Write => { - let (_, value) = write_of(script, u64::from(request.run.expect("a write names its run").first())) - .expect("a write's arena block is its ticket"); + Op::Write(run) => { + let (_, value) = write_of(script, u64::from(run.first())).expect("a write's arena block is its ticket"); cache[request.lba as usize] = Some(value); } Op::Flush => { @@ -284,7 +307,7 @@ fn apply(script: &[Step], cache: &mut [Option; BLOCKS], media: &mut [u8; BLO } } } - Op::Read => {} + Op::Read(_) => {} } } @@ -404,10 +427,58 @@ fn end(run: &mut Run, world: &World) { } } -fn dfs(run: &mut Run, world: World) { - if run.broken.is_some() || !run.seen.insert(format!("{world:?}")) { +/// Names a state's tags by first appearance. +#[derive(Default)] +struct Names(Vec); + +impl Names { + fn of(&mut self, tag: u32) -> u32 { + let at = self.0.iter().position(|&t| t == tag).unwrap_or_else(|| { + self.0.push(tag); + self.0.len() - 1 + }); + at as u32 + } +} + +/// A state's name in the search: everything it holds, with every tag renamed +/// by first appearance, the client's first in slot order, and the client's +/// table rendered by its filled slots. A fresh tag equals none present, so +/// states named alike differ only in their tags' numbers, which the protocol +/// compares for equality and orders only in a reset's answers. +fn key(world: &World) -> String { + let mut names = Names::default(); + let wire: Vec = world.client.on_the_wire().map(|t| names.of(t)).collect(); + let mut request = |r: &Request| Request { tag: names.of(r.tag), ..*r }; + let sq: Vec = world.queues.sq.iter().map(&mut request).collect(); + let device: Vec = world.device.iter().map(|(_, r)| request(r)).collect(); + let cq: Vec = world.queues.cq.iter().map(|c| Completion { tag: names.of(c.tag), ..*c }).collect(); + let server = world.server.as_ref().map(|s| { + let inflight: Vec<(u32, Op)> = s.session.inflight().map(|(t, op)| (names.of(t), op)).collect(); + format!("{inflight:?} {:?}", s.holds) + }); + let posted: Vec = world.posted.iter().map(|&t| names.of(t)).collect(); + format!( + "{:?} {wire:?} {sq:?} {device:?} {cq:?} {server:?} {posted:?} {} {:?} {:?} {} {} {:?} {:?} {:?}", + world.client, + world.next, + world.answers, + world.acked_before, + world.alive, + world.losses, + world.cache, + world.media, + world.left + ) +} + +fn dfs(run: &mut Run, mut world: World) { + world.queues.hold_empty(); + if run.broken.is_some() || !run.seen.insert(key(&world)) { return; } + run.filled[0] |= world.queues.sq.len() == DEPTH as usize; + run.filled[1] |= world.queues.cq.len() == DEPTH as usize; let mut moved = false; let script = run.script; @@ -423,7 +494,10 @@ fn dfs(run: &mut Run, world: World) { if !busy { let mut w = world.clone(); match script[world.next] { - Step::Write { block, .. } => w.client.submit(ticket, Op::Write, block, ARENA.run(ticket as u32, 1)), + Step::Write { block, .. } => { + let run = ARENA.run(ticket as u32, 1).expect("a script's ticket is an arena block"); + w.client.submit(ticket, Op::Write(run), block); + } Step::Flush => { let acked = w .answers @@ -432,7 +506,7 @@ fn dfs(run: &mut Run, world: World) { .map(|(t, _)| *t) .collect(); w.acked_before.insert(ticket, acked); - w.client.submit(ticket, Op::Flush, 0, None); + w.client.submit(ticket, Op::Flush, 0); } } w.next += 1; @@ -667,12 +741,16 @@ mod tests { } /// The model is not vacuous: with no failure at all, a run ends with both - /// flushes durable and the medium holding the script's last values. + /// flushes durable and the medium holding the script's last values; each + /// ring holds its depth, and wraps, because one session carries more than + /// that. #[test] fn the_model_reaches_the_end_it_should() { let explored = verdict(&SCRIPT, at_most(0, 0, 0)); assert_eq!(explored.broken, None); assert!(explored.ends >= 1); + assert_eq!(explored.filled, [true, true], "a ring never held its depth"); + assert!(SCRIPT.len() > DEPTH as usize, "a ring never wraps"); } /// Nor is the give-up path out of its reach: past [`MAX_ATTEMPTS`] failures diff --git a/toyos-blockring/src/server.rs b/toyos-blockring/src/server.rs index bd4561d78ef..ac41a160fd9 100644 --- a/toyos-blockring/src/server.rs +++ b/toyos-blockring/src/server.rs @@ -66,9 +66,9 @@ impl ServerSession { self.blocks } - /// Requests taken and not yet answered. - pub fn inflight(&self) -> usize { - self.inflight.len() + /// Requests taken and not yet answered, by tag. + pub fn inflight(&self) -> impl ExactSizeIterator + '_ { + self.inflight.iter().map(|(&tag, &op)| (tag, op)) } /// Decide what one entry the client published is. @@ -103,8 +103,8 @@ impl ServerSession { let op = self.inflight.remove(&tag)?; let status = match (op, done) { (_, false) => Status::Device, - (Op::Read, true) => Status::Ok, - (Op::Write, true) => { + (Op::Read(_), true) => Status::Ok, + (Op::Write(_), true) => { holds.wrote(self.writer(), losses); Status::Ok } @@ -137,10 +137,10 @@ mod tests { use crate::layout::ARENA; fn write(tag: u32) -> [Untrusted; SQE_WORDS] { - Request { op: Op::Write, tag, lba: 0, run: ARENA.run(0, 1) }.encode().map(Untrusted::new) + Request { op: Op::Write(ARENA.run(0, 1).unwrap()), tag, lba: 0 }.encode().map(Untrusted::new) } fn flush(tag: u32) -> [Untrusted; SQE_WORDS] { - Request { op: Op::Flush, tag, lba: 0, run: None }.encode().map(Untrusted::new) + Request { op: Op::Flush, tag, lba: 0 }.encode().map(Untrusted::new) } #[test] @@ -158,7 +158,7 @@ mod tests { let mut session = ServerSession::new(0, 10); assert!(matches!(session.take(write(4)), Taken::Issue(_))); assert_eq!(session.take(write(4)), Taken::Answer(Completion { tag: 4, status: Status::Invalid })); - assert_eq!(session.inflight(), 1); + assert_eq!(session.inflight().len(), 1); } /// A write acknowledged before a reset bumped the loss count is a loss the diff --git a/toyos-transport/Cargo.toml b/toyos-transport/Cargo.toml index 483227af1e3..56018c4b77c 100644 --- a/toyos-transport/Cargo.toml +++ b/toyos-transport/Cargo.toml @@ -1,11 +1,10 @@ # A member of the host workspace (root `Cargo.toml`). The one transport every -# client/server session runs over: the rings, the arena's runs, the tag table -# and the wake, as decisions over words an adapter hands in. Nothing here maps, -# copies or blocks. +# client/server session runs over: the rings, the arena's runs and the tag +# table, as decisions over words an adapter hands in. Nothing here maps, copies +# or blocks. # # Its defects are orders and hostile peers, so `tests/loom.rs` holds the -# publication edge, the wake and a hostile peer, and `src/model.rs` enumerates -# a session through crash, restart and a late completion. +# publication edge and a hostile peer. [package] name = "toyos-transport" @@ -22,12 +21,6 @@ publish = false # A producer publishes its tail `Relaxed`, so a consumer can see the index # before the entry's words. publish-relaxed = [] -# The producer does not fence between its tail and its load of `sleep`, so it -# can miss a consumer's `sleep` while the consumer misses its tail. -no-wake-fence = [] -# The consumer does not fence between its `sleep` and its load of the tail: -# the same miss, from the other side. -no-sleep-fence = [] # A peer's cursor is believed however far it claims to be, so a hostile # producer hands the consumer more entries than the ring holds. no-clamp = [] diff --git a/toyos-transport/src/inflight.rs b/toyos-transport/src/inflight.rs index 70a0208fa88..9a45176bdb9 100644 --- a/toyos-transport/src/inflight.rs +++ b/toyos-transport/src/inflight.rs @@ -4,55 +4,59 @@ //! ([`Inflight::answer`]) or, when the session ends, by [`Inflight::end`] — //! never by both, and never by a completion naming a tag from before its //! slot was last filled. A tag is the slot's index under the slot's own -//! sequence, so a tag answered, ended or replayed names nothing. The sequence -//! wraps: a tag replayed a wrap later answers the request then in its slot, -//! which a server could answer by naming that request's own tag, so it grants -//! the peer nothing. +//! sequence, so a tag answered, ended or replayed names nothing. The index +//! takes the bits the table needs and the sequence the rest, and wraps: a tag +//! replayed a wrap later answers the request then in its slot, which a server +//! could answer by naming that request's own tag, so it grants the peer +//! nothing. -use crate::{Untrusted, Violation}; +use core::fmt; -const INDEX_BITS: u32 = 16; -const INDEX_MASK: u32 = (1 << INDEX_BITS) - 1; +use crate::{Untrusted, Violation}; -#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, PartialEq, Eq, Hash)] struct Slot { - seq: u16, + seq: u32, value: Option, } /// At most `D` requests in flight, each carrying what its answer needs. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, PartialEq, Eq, Hash)] pub struct Inflight { slots: [Slot; D], } -fn tag(seq: u16, index: u32) -> u32 { - u32::from(seq).wrapping_shl(INDEX_BITS) | index -} - impl Inflight { + const INDEX_BITS: u32 = usize::BITS.wrapping_sub(D.saturating_sub(1).leading_zeros()); + const INDEX_MASK: u32 = 1u32.wrapping_shl(Self::INDEX_BITS).wrapping_sub(1); + const SEQ_MASK: u32 = u32::MAX.wrapping_shr(Self::INDEX_BITS); + pub fn new() -> Self { - const { assert!(D > 0 && D <= 1 << INDEX_BITS, "a tag's index is 16 bits") }; + const { assert!(D > 0 && Self::INDEX_BITS <= 16, "a tag keeps at least 16 bits of sequence") }; Self { slots: core::array::from_fn(|_| Slot { seq: 0, value: None }) } } + fn tag(seq: u32, index: u32) -> u32 { + seq.wrapping_shl(Self::INDEX_BITS) | index + } + /// Keep `value` in flight under a tag no earlier request had in this slot; /// `Err(value)` with `D` in flight. pub fn insert(&mut self, value: T) -> Result { let Some((index, slot)) = (0u32..).zip(self.slots.iter_mut()).find(|(_, slot)| slot.value.is_none()) else { return Err(value); }; - slot.seq = slot.seq.wrapping_add(1); + slot.seq = slot.seq.wrapping_add(1) & Self::SEQ_MASK; slot.value = Some(value); - Ok(tag(slot.seq, index)) + Ok(Self::tag(slot.seq, index)) } /// What the peer's `tag` answers; a tag nothing is in flight under is a /// violation. pub fn answer(&mut self, tag: Untrusted) -> Result { - let index = tag.map(|t| t & INDEX_MASK).index(D).map_err(|_| Violation::Tag)?; + let index = tag.map(|t| t & Self::INDEX_MASK).index(D).map_err(|_| Violation::Tag)?; let slot = self.slots.get_mut(index).ok_or(Violation::Tag)?; - if !tag.map(|t| t.wrapping_shr(INDEX_BITS)).is(u32::from(slot.seq)) { + if !tag.map(|t| t.wrapping_shr(Self::INDEX_BITS)).is(slot.seq) { return Err(Violation::Tag); } slot.value.take().ok_or(Violation::Tag) @@ -63,6 +67,11 @@ impl Inflight { self.slots.iter().filter_map(|slot| slot.value.as_ref()) } + /// Every tag in flight, by its slot. + pub fn tags(&self) -> impl Iterator + '_ { + (0u32..).zip(&self.slots).filter(|(_, slot)| slot.value.is_some()).map(|(index, slot)| Self::tag(slot.seq, index)) + } + /// The session ended: `each` is given every tag in flight, once, with what /// it carried, and none of them is answered again. pub fn end(&mut self, mut each: impl FnMut(u32, T)) { @@ -72,7 +81,7 @@ impl Inflight { #[cfg(feature = "end-keeps-inflight")] let value = None; if let Some(value) = value { - each(tag(slot.seq, index), value); + each(Self::tag(slot.seq, index), value); } } } @@ -84,6 +93,13 @@ impl Default for Inflight { } } +/// Only what is in flight, by slot. +impl fmt::Debug for Inflight { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_map().entries((0u32..).zip(&self.slots).filter_map(|(i, slot)| Some((i, slot.value.as_ref()?)))).finish() + } +} + #[cfg(test)] mod tests { use super::*; @@ -101,6 +117,7 @@ mod tests { let mut held: Vec<&str> = inflight.values().copied().collect(); held.sort_unstable(); assert_eq!(held, ["b", "c"], "what is in flight, and only that"); + assert_eq!(inflight.tags().collect::>(), [c, b], "the tags in flight, by slot"); assert_eq!(inflight.answer(Untrusted::new(a)), Err(Violation::Tag), "a replay answers nothing"); assert_eq!(inflight.answer(Untrusted::new(2)), Err(Violation::Tag), "an index past the table"); assert_eq!(inflight.answer(Untrusted::new(b)), Ok("b")); @@ -108,6 +125,21 @@ mod tests { assert!(inflight.insert("d").is_ok() && inflight.insert("e").is_ok(), "every slot is free again"); } + /// A slot filled 2^16 times over still refuses its first tag: the index + /// takes six bits of a 64-slot table's tag, not sixteen. + #[test] + fn a_slot_refilled_past_sixteen_bits_refuses_its_first_tag() { + let mut inflight = Inflight::<(), 64>::new(); + let first = inflight.insert(()).unwrap(); + inflight.answer(Untrusted::new(first)).unwrap(); + for _ in 0..u16::MAX { + let tag = inflight.insert(()).unwrap(); + inflight.answer(Untrusted::new(tag)).unwrap(); + } + inflight.insert(()).unwrap(); + assert_eq!(inflight.answer(Untrusted::new(first)), Err(Violation::Tag)); + } + #[test] fn an_end_answers_every_tag_once_and_a_late_completion_nothing() { let mut inflight = Inflight::::new(); diff --git a/toyos-transport/src/lib.rs b/toyos-transport/src/lib.rs index ce43f6a0736..c81d65ebdbd 100644 --- a/toyos-transport/src/lib.rs +++ b/toyos-transport/src/lib.rs @@ -6,8 +6,8 @@ //! [`Run`]s. A protocol says what the entries mean; this crate says only what //! is safe. //! -//! **Nothing here holds the region.** An adapter hands every call the region's -//! words as a slice of [`Word`]s and copies bytes itself, through a run's +//! **Nothing here holds the region.** An adapter hands every call the `N` +//! [`Word`]s an end was made over, and copies bytes itself, through a run's //! [`Span`], once: no reference to the peer's bytes is formed. //! //! **What the peer writes is untrusted until decoded.** An entry comes out as @@ -18,24 +18,16 @@ //! so nothing the peer writes moves it, and a cursor the peer moves backwards //! within bounds costs only the peer. //! -//! **Publication is `Release`/`Acquire`; the wake is a pair of `SeqCst` -//! fences.** A consumer that finds nothing stores its `sleep` word, fences and -//! looks again ([`Consumer::before_sleep`]); a producer stores its tail, -//! fences and loads `sleep`, and asks for a wake ([`Wake::Peer`]) only if it is -//! set. A busy consumer costs a producer no syscall, and a hostile `sleep` only -//! costs a wake. -//! //! **A restart is seen only as a hang-up**, and [`Inflight::end`] answers every //! tag that was in flight, once; a tag from before it answers nothing. #![cfg_attr(not(test), no_std)] #![forbid(unsafe_code)] -#![cfg_attr(not(test), forbid(clippy::arithmetic_side_effects, clippy::indexing_slicing, clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::as_conversions))] +#![cfg_attr(not(test), forbid(clippy::arithmetic_side_effects, clippy::unwrap_used, clippy::expect_used, clippy::panic))] +#![cfg_attr(not(test), deny(clippy::indexing_slicing, clippy::as_conversions))] mod arena; mod inflight; -#[cfg(test)] -mod model; mod queue; use core::sync::atomic::{AtomicU32, Ordering}; @@ -50,8 +42,6 @@ pub use toyos_untrusted::Untrusted; pub trait Word { fn load(&self, order: Ordering) -> u32; fn store(&self, value: u32, order: Ordering); - /// A `SeqCst` fence, in the memory model this word lives in. - fn fence(); } impl Word for AtomicU32 { @@ -61,9 +51,6 @@ impl Word for AtomicU32 { fn store(&self, value: u32, order: Ordering) { AtomicU32::store(self, value, order) } - fn fence() { - core::sync::atomic::fence(Ordering::SeqCst) - } } /// The peer wrote what no peer of the protocol writes, or the region is not @@ -85,31 +72,12 @@ pub enum Violation { Region, } -/// What a publish asks of the adapter. -#[must_use] -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum Wake { - /// The consumer said it sleeps: wake it. - Peer, - /// The consumer is awake and will find what was published. - Busy, -} - -/// A consumer that found nothing and said so: it waits while word `word` -/// holds `value`, as a futex does. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct Asleep { - pub word: usize, - pub value: u32, -} - -/// Where a ring's two cursors and its consumer's `sleep` word are, in words. -/// The producer stores `tail`, the consumer `head` and `sleep`. +/// Where a ring's two cursors are, in words. The producer stores `tail`, the +/// consumer `head`. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Cursors { pub head: usize, pub tail: usize, - pub sleep: usize, } #[cfg(not(feature = "publish-relaxed"))] @@ -117,8 +85,12 @@ const PUBLISH: Ordering = Ordering::Release; #[cfg(feature = "publish-relaxed")] const PUBLISH: Ordering = Ordering::Relaxed; -fn word(page: &[W], at: usize) -> Result<&W, Violation> { - page.get(at).ok_or(Violation::Region) +/// Word `at` of an end's words, and the one index into them: every `at` an end +/// forms is below `N`, because its place was held to `N` when the end was +/// made and a ring position is masked below the depth. +#[allow(clippy::indexing_slicing)] +fn word(page: &[W; N], at: usize) -> &W { + &page[at] } /// A distance the peer's cursor claims, believed only up to `cap`. @@ -129,42 +101,14 @@ fn clamp(claimed: Untrusted, cap: u32, broken: Violation) -> Result(&self, page: &[W], released: u32, cap: u32) -> Result { - let tail = word(page, self.tail)?.load(Ordering::Acquire); + fn published(&self, page: &[W; N], released: u32, cap: u32) -> Result { + let tail = word(page, self.tail).load(Ordering::Acquire); clamp(Untrusted::new(tail).map(|t| t.wrapping_sub(released)), cap, Violation::TailPastDepth) } /// How far behind `published` the consumer's head is: at most `cap`. - fn unreleased(&self, page: &[W], published: u32, cap: u32) -> Result { - let head = word(page, self.head)?.load(Ordering::Acquire); + fn unreleased(&self, page: &[W; N], published: u32, cap: u32) -> Result { + let head = word(page, self.head).load(Ordering::Acquire); clamp(Untrusted::new(head).map(|h| published.wrapping_sub(h)), cap, Violation::HeadPastTail) } - - fn publish(&self, page: &[W], tail: u32) -> Result { - word(page, self.tail)?.store(tail, PUBLISH); - self.wake(page) - } - - /// The producer's half of the wake, after what it published is stored. - fn wake(&self, page: &[W]) -> Result { - #[cfg(not(feature = "no-wake-fence"))] - W::fence(); - Ok(match word(page, self.sleep)?.load(Ordering::Relaxed) { - 0 => Wake::Busy, - _ => Wake::Peer, - }) - } - - /// The consumer's half: say it sleeps, before it looks again. - fn sleep(&self, page: &[W]) -> Result<(), Violation> { - word(page, self.sleep)?.store(1, Ordering::Relaxed); - #[cfg(not(feature = "no-sleep-fence"))] - W::fence(); - Ok(()) - } - - fn awake(&self, page: &[W]) -> Result<(), Violation> { - word(page, self.sleep)?.store(0, Ordering::Relaxed); - Ok(()) - } } diff --git a/toyos-transport/src/model.rs b/toyos-transport/src/model.rs deleted file mode 100644 index 56c245b00d4..00000000000 --- a/toyos-transport/src/model.rs +++ /dev/null @@ -1,328 +0,0 @@ -//! Every ordering of one client, a server, its death, the client's reconnect, -//! and a server that answers a tag it has answered before. -//! -//! The client puts requests on a real [`Producer`] and reads answers off a -//! real [`Consumer`], over words of its own, keeping them in a real -//! [`Inflight`]. [`explore`] runs, depth first and exhaustively, every -//! interleaving of: the client sending its next request, the server taking -//! one, answering any one it holds, **answering again any tag it has ever -//! taken**, **dying**, the client reading an answer, noticing the hang-up, -//! and reconnecting over the same words to a fresh server. A client that reads -//! a violation ends the session as it would a hang-up. -//! -//! The law: every request is answered exactly once — never twice, and by the -//! end of every run — and an answer read off the ring answers only a request -//! of the session it was sent in. - -use std::cell::Cell; -use std::collections::HashSet; -use std::format; -use std::string::String; -use std::vec::Vec; - -use crate::{Consumer, Cursors, Inflight, Place, Producer, Untrusted, Word}; - -const D: u32 = 2; -const REQUESTS: usize = 3; -const SQ: Place = Place { cursors: Cursors { head: 0, tail: 1, sleep: 2 }, entries: 3 }; -const CQ: Place = Place { cursors: Cursors { head: 5, tail: 6, sleep: 7 }, entries: 8 }; -const WORDS: usize = 10; - -/// A word of a model that runs on one thread: every order is the program's. -#[derive(Clone, Debug)] -struct Shared(Cell); - -impl Word for Shared { - fn load(&self, _: core::sync::atomic::Ordering) -> u32 { - self.0.get() - } - fn store(&self, value: u32, _: core::sync::atomic::Ordering) { - self.0.set(value) - } - fn fence() {} -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Answer { - /// Read off the ring in session `session`. - Read { session: u32 }, - /// Given by the session's end. - Ended, -} - -#[derive(Clone, Debug)] -struct Server { - requests: Consumer<1, D>, - replies: Producer<1, D>, - held: Vec, -} - -#[derive(Clone, Debug)] -struct World { - page: Vec, - requests: Producer<1, D>, - replies: Consumer<1, D>, - inflight: Inflight, - next: usize, - session: u32, - /// The session each request was sent in. - sent: [Option; REQUESTS], - answers: [Vec; REQUESTS], - server: Option, - /// The server died and the client has not noticed. - hung_up: bool, - /// Every tag any server took, which a replay names. - taken: Vec, - crashes: u8, - replays: u8, -} - -/// What a run found. -pub struct Explored { - pub broken: Option, - pub states: usize, - pub ends: usize, - /// Answers the client refused as naming nothing in flight. - pub refused: usize, -} - -struct Run { - seen: HashSet, - broken: Option, - ends: usize, - refused: usize, - path: Vec, -} - -fn plain(word: Untrusted) -> u32 { - word.at_most(u32::MAX.into()).unwrap() as u32 -} - -impl World { - fn new(crashes: u8, replays: u8) -> Self { - let page: Vec = (0..WORDS).map(|_| Shared(Cell::new(0))).collect(); - let mut world = Self { - requests: Producer::new(&page, SQ).unwrap(), - replies: Consumer::new(&page, CQ).unwrap(), - page, - inflight: Inflight::new(), - next: 0, - session: 0, - sent: [None; REQUESTS], - answers: Default::default(), - server: None, - hung_up: false, - taken: Vec::new(), - crashes, - replays, - }; - world.connect(); - world - } - - /// A fresh server's ends, and the client's again, over the same words. - fn connect(&mut self) { - self.requests = Producer::new(&self.page, SQ).unwrap(); - self.replies = Consumer::new(&self.page, CQ).unwrap(); - self.server = Some(Server { - requests: Consumer::new(&self.page, SQ).unwrap(), - replies: Producer::new(&self.page, CQ).unwrap(), - held: Vec::new(), - }); - } - - /// Whether the client thinks it has a session. - fn up(&self) -> bool { - self.server.is_some() || self.hung_up - } - - /// The session is over: every request in flight is answered by its end. - fn end(&mut self) { - self.server = None; - self.hung_up = false; - let answers = &mut self.answers; - self.inflight.end(|_, request| answers[request].push(Answer::Ended)); - } - - /// The server posts an answer naming `tag`; `false` if the ring is full. - fn post(&mut self, tag: u32) -> bool { - let server = self.server.as_mut().unwrap(); - if !server.replies.push(&self.page, [tag]).unwrap() { - return false; - } - let _ = server.replies.publish(&self.page).unwrap(); - true - } -} - -fn go(run: &mut Run, step: String, world: World) { - run.path.push(step); - dfs(run, world); - run.path.pop(); -} - -fn fail(run: &mut Run, why: String) { - if run.broken.is_none() { - run.broken = Some(format!("{why}, after {}", run.path.join(" > "))); - } -} - -fn dfs(run: &mut Run, world: World) { - if run.broken.is_some() || !run.seen.insert(format!("{world:?}")) { - return; - } - let mut moved = false; - - // The client sends its next request, into a ring nobody may be reading. - if world.up() && world.next < REQUESTS { - let mut w = world.clone(); - if let Ok(tag) = w.inflight.insert(w.next) { - assert!(w.requests.push(&w.page, [tag]).unwrap(), "more requests on the ring than in flight"); - let _ = w.requests.publish(&w.page).unwrap(); - w.sent[w.next] = Some(w.session); - w.next += 1; - moved = true; - go(run, format!("send {}#{tag:x}", world.next), w); - } - } - - if let Some(server) = &world.server { - // The server takes the oldest request. - let mut w = world.clone(); - let s = w.server.as_mut().unwrap(); - if let Some([tag]) = s.requests.pop(&w.page).unwrap() { - let tag = plain(tag); - s.requests.release(&w.page).unwrap(); - s.held.push(tag); - w.taken.push(tag); - moved = true; - go(run, format!("take #{tag:x}"), w); - } - - // It answers any one it holds. - for i in 0..server.held.len() { - let mut w = world.clone(); - let tag = w.server.as_mut().unwrap().held.remove(i); - if w.post(tag) { - moved = true; - go(run, format!("answer #{tag:x}"), w); - } - } - - // It answers again a tag it took, in this session or an earlier one. - if world.replays > 0 { - for &tag in &world.taken { - let mut w = world.clone(); - w.replays -= 1; - if w.post(tag) { - moved = true; - go(run, format!("replay #{tag:x}"), w); - } - } - } - - // It dies; the client has not noticed. - if world.crashes > 0 { - let mut w = world.clone(); - w.crashes -= 1; - w.server = None; - w.hung_up = true; - moved = true; - go(run, "crash".into(), w); - } - } - - // The client reads the oldest answer, the server alive or not. - if world.up() { - let mut w = world.clone(); - if let Some([tag]) = w.replies.pop(&w.page).unwrap() { - w.replies.release(&w.page).unwrap(); - let step = format!("read #{:x}", plain(tag)); - match w.inflight.answer(tag) { - Ok(request) => { - if w.sent[request] != Some(w.session) { - let sent = w.sent[request]; - return fail(run, format!("request {request} of session {sent:?} answered in {}", w.session)); - } - w.answers[request].push(Answer::Read { session: w.session }); - if w.answers[request].len() > 1 { - return fail(run, format!("request {request} answered {:?}", w.answers[request])); - } - } - Err(_) => { - run.refused += 1; - w.end(); - } - } - moved = true; - go(run, step, w); - } - } - - // It notices the hang-up. - if world.hung_up { - let mut w = world.clone(); - w.end(); - moved = true; - go(run, "notice".into(), w); - } - - // It reconnects over the same words, as a new session. - if !world.up() { - let mut w = world.clone(); - w.session += 1; - w.connect(); - moved = true; - go(run, "reconnect".into(), w); - } - - if !moved { - run.ends += 1; - for (request, answers) in world.answers.iter().enumerate() { - if answers.len() != 1 { - return fail(run, format!("request {request} ended answered {answers:?}")); - } - } - } -} - -/// Explore every run with at most `crashes` deaths and `replays` replays. -pub fn explore(crashes: u8, replays: u8) -> Explored { - let mut run = Run { seen: HashSet::new(), broken: None, ends: 0, refused: 0, path: Vec::new() }; - dfs(&mut run, World::new(crashes, replays)); - Explored { broken: run.broken, states: run.seen.len(), ends: run.ends, refused: run.refused } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// What the session is held under: no failure, a death, a replay, and - /// both twice. - const BOUNDS: [(u8, u8); 4] = [(0, 0), (1, 0), (0, 1), (2, 2)]; - - #[test] - fn every_tag_is_answered_exactly_once() { - for (crashes, replays) in BOUNDS { - let explored = explore(crashes, replays); - std::println!( - "{crashes} crashes, {replays} replays: {} states, {} end states, {} answers refused", - explored.states, - explored.ends, - explored.refused - ); - assert!(explored.ends > 0, "the model reached no end"); - if let Some(why) = explored.broken { - panic!("{crashes} crashes, {replays} replays: {why}"); - } - } - } - - /// The model is not vacuous: a replay reaches the client and is refused. - #[test] - fn the_model_reaches_a_replay_refused() { - let explored = explore(0, 1); - assert_eq!(explored.broken, None); - assert!(explored.refused > 0, "no replay reached the client"); - } -} diff --git a/toyos-transport/src/queue.rs b/toyos-transport/src/queue.rs index d6a6d1acc8b..e67bb0dcc1a 100644 --- a/toyos-transport/src/queue.rs +++ b/toyos-transport/src/queue.rs @@ -1,4 +1,4 @@ -//! A single-producer queue of `E`-word entries, `D` deep. +//! A single-producer queue of `E`-word entries, `D` deep, among `N` words. //! //! **A producer writes an entry's words, then publishes the tail with //! `Release`; a consumer loads the tail with `Acquire`, then reads the words.** @@ -8,7 +8,7 @@ use core::sync::atomic::Ordering; -use crate::{word, Asleep, Cursors, Untrusted, Violation, Wake, Word}; +use crate::{word, Cursors, Untrusted, Violation, Word, PUBLISH}; /// Where one queue is in a region, in words: its cursors, and its first entry. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -19,28 +19,35 @@ pub struct Place { impl Place { /// The words of the entry at ring position `at`. - fn entry<'a, W, const E: usize, const D: u32>(&self, page: &'a [W], at: u32) -> Result<&'a [W], Violation> { - let first = usize::try_from(at & D.wrapping_sub(1)) - .ok() - .and_then(|slot| slot.checked_mul(E)) - .and_then(|offset| offset.checked_add(self.entries)); - first.and_then(|first| page.get(first..first.checked_add(E)?)).ok_or(Violation::Region) + fn entry<'a, W, const E: usize, const D: u32, const N: usize>( + &self, + page: &'a [W; N], + at: u32, + ) -> impl Iterator { + // `usize` holds every `u32`, so the slot is exact. + #[allow(clippy::as_conversions)] + let slot = (at & D.wrapping_sub(1)) as usize; + let first = self.entries.wrapping_add(slot.wrapping_mul(E)); + (0..E).map(move |k| word(page, first.wrapping_add(k))) } - /// Every word the queue uses is in `page`. - fn check(&self, page: &[W]) -> Result<(), Violation> { + /// Every word the queue uses is below `N`. + fn check(&self) -> Result<(), Violation> { const { assert!(D.is_power_of_two() && E > 0, "a queue is a power of two deep, of entries of a word or more") }; - word(page, self.cursors.head)?; - word(page, self.cursors.tail)?; - word(page, self.cursors.sleep)?; - self.entry::(page, D.wrapping_sub(1)).map(|_| ()) + const { assert!(usize::BITS >= u32::BITS, "a ring position is a u32") }; + let entries_end = + usize::try_from(D).ok().and_then(|d| d.checked_mul(E)).and_then(|words| words.checked_add(self.entries)); + match entries_end { + Some(end) if end <= N && self.cursors.head < N && self.cursors.tail < N => Ok(()), + _ => Err(Violation::Region), + } } } /// The end of a queue that writes entries. It holds its cursors and not the /// region; every call is given the region's words. #[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct Producer { +pub struct Producer { place: Place, local: u32, published: u32, @@ -48,16 +55,16 @@ pub struct Producer { room: u32, } -impl Producer { +impl Producer { /// This end of the queue at `place`, its tail stored 0. - pub fn new(page: &[W], place: Place) -> Result { - place.check::(page)?; - word(page, place.cursors.tail)?.store(0, Ordering::Release); + pub fn new(page: &[W; N], place: Place) -> Result { + place.check::()?; + word(page, place.cursors.tail).store(0, Ordering::Release); Ok(Self { place, local: 0, published: 0, room: D }) } /// How many entries may be pushed before the consumer frees more. - pub fn space(&mut self, page: &[W]) -> Result { + pub fn space(&mut self, page: &[W; N]) -> Result { let unreleased = self.place.cursors.unreleased(page, self.published, D)?; let pending = self.local.wrapping_sub(self.published); self.room = D.saturating_sub(pending.saturating_add(unreleased)); @@ -66,11 +73,11 @@ impl Producer { /// Write one entry, or answer `false` for a full queue and write nothing. /// It is the consumer's once [`Self::publish`] runs. - pub fn push(&mut self, page: &[W], words: [u32; E]) -> Result { + pub fn push(&mut self, page: &[W; N], words: [u32; E]) -> Result { if self.room == 0 && self.space(page)? == 0 { return Ok(false); } - for (shared, value) in self.place.entry::(page, self.local)?.iter().zip(words) { + for (shared, value) in self.place.entry::(page, self.local).zip(words) { shared.store(value, Ordering::Relaxed); } self.local = self.local.wrapping_add(1); @@ -78,61 +85,47 @@ impl Producer { Ok(true) } - /// Publish every entry pushed so far; `None` if there was none. - pub fn publish(&mut self, page: &[W]) -> Result, Violation> { + /// Publish every entry pushed so far; `false` if there was none. + pub fn publish(&mut self, page: &[W; N]) -> bool { if self.published == self.local { - return Ok(None); + return false; } - let wake = self.place.cursors.publish(page, self.local)?; + word(page, self.place.cursors.tail).store(self.local, PUBLISH); self.published = self.local; - Ok(Some(wake)) + true } } /// The end of a queue that reads entries; like [`Producer`], it holds cursors /// and is given the region's words. #[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct Consumer { +pub struct Consumer { place: Place, local: u32, released: u32, /// Entries published and not yet popped, as last seen. ready: u32, - /// This end stored its `sleep` word and has not cleared it. - asleep: bool, } -impl Consumer { - /// This end of the queue at `place`, its head and `sleep` stored 0. - pub fn new(page: &[W], place: Place) -> Result { - place.check::(page)?; - word(page, place.cursors.head)?.store(0, Ordering::Release); - place.cursors.awake(page)?; - Ok(Self { place, local: 0, released: 0, ready: 0, asleep: false }) - } - - /// Look at the tail again, and answer it; what of it is not popped is - /// `ready`. - fn observe(&mut self, page: &[W]) -> Result { - let published = self.place.cursors.published(page, self.released, D)?; - self.ready = published.saturating_sub(self.local.wrapping_sub(self.released)); - Ok(self.released.wrapping_add(published)) +impl Consumer { + /// This end of the queue at `place`, its head stored 0. + pub fn new(page: &[W; N], place: Place) -> Result { + place.check::()?; + word(page, place.cursors.head).store(0, Ordering::Release); + Ok(Self { place, local: 0, released: 0, ready: 0 }) } /// The next published entry, or `None` for none. - pub fn pop(&mut self, page: &[W]) -> Result; E]>, Violation> { + pub fn pop(&mut self, page: &[W; N]) -> Result; E]>, Violation> { if self.ready == 0 { - if self.asleep { - self.place.cursors.awake(page)?; - self.asleep = false; - } - self.observe(page)?; + let published = self.place.cursors.published(page, self.released, D)?; + self.ready = published.saturating_sub(self.local.wrapping_sub(self.released)); if self.ready == 0 { return Ok(None); } } let mut words = [Untrusted::new(0); E]; - for (out, shared) in words.iter_mut().zip(self.place.entry::(page, self.local)?) { + for (out, shared) in words.iter_mut().zip(self.place.entry::(page, self.local)) { *out = Untrusted::new(shared.load(Ordering::Relaxed)); } self.local = self.local.wrapping_add(1); @@ -141,30 +134,11 @@ impl Consumer { } /// Give every entry popped so far back to the producer. - pub fn release(&mut self, page: &[W]) -> Result<(), Violation> { + pub fn release(&mut self, page: &[W; N]) { if self.released != self.local { - word(page, self.place.cursors.head)?.store(self.local, Ordering::Release); + word(page, self.place.cursors.head).store(self.local, Ordering::Release); self.released = self.local; } - Ok(()) - } - - /// Say this end sleeps, and look once more: `None` if an entry is there - /// after all, or where to wait. A producer that publishes after this is - /// answered [`Wake::Peer`]; the next [`Self::pop`] says this end is awake. - pub fn before_sleep(&mut self, page: &[W]) -> Result, Violation> { - if self.ready > 0 { - return Ok(None); - } - self.place.cursors.sleep(page)?; - self.asleep = true; - let tail = self.observe(page)?; - if self.ready > 0 { - self.place.cursors.awake(page)?; - self.asleep = false; - return Ok(None); - } - Ok(Some(Asleep { word: self.place.cursors.tail, value: tail })) } } @@ -174,13 +148,14 @@ mod tests { use core::sync::atomic::AtomicU32; const D: u32 = 8; - const PLACE: Place = Place { cursors: Cursors { head: 0, tail: 16, sleep: 1 }, entries: 32 }; + const PLACE: Place = Place { cursors: Cursors { head: 0, tail: 16 }, entries: 32 }; + const WORDS: usize = 32 + 2 * D as usize; - fn page() -> Vec { - (0..32 + 2 * D as usize).map(|_| AtomicU32::new(0)).collect() + fn page() -> [AtomicU32; WORDS] { + core::array::from_fn(|_| AtomicU32::new(0)) } - fn ends(page: &[AtomicU32]) -> (Producer<2, D>, Consumer<2, D>) { + fn ends(page: &[AtomicU32; WORDS]) -> (Producer<2, D, WORDS>, Consumer<2, D, WORDS>) { (Producer::new(page, PLACE).unwrap(), Consumer::new(page, PLACE).unwrap()) } @@ -194,8 +169,8 @@ mod tests { let (mut tx, mut rx) = ends(&page); assert_eq!(tx.push(&page, [3, 4]), Ok(true)); assert_eq!(rx.pop(&page).map(plain), Ok(None)); - assert_eq!(tx.publish(&page), Ok(Some(Wake::Busy))); - assert_eq!(tx.publish(&page), Ok(None), "nothing new, nothing published"); + assert!(tx.publish(&page)); + assert!(!tx.publish(&page), "nothing new, nothing published"); assert_eq!(rx.pop(&page).map(plain), Ok(Some([3, 4]))); assert_eq!(rx.pop(&page).map(plain), Ok(None)); } @@ -216,13 +191,13 @@ mod tests { } pushed += 1; } - let _ = tx.publish(&page).unwrap(); + tx.publish(&page); for _ in 0..1 + round % 7 { let Some(words) = plain(rx.pop(&page).unwrap()) else { break }; assert_eq!(words, [popped, !popped], "entries come out whole, in the order they went in"); popped += 1; } - rx.release(&page).unwrap(); + rx.release(&page); } assert!(pushed > 2 * D, "the ring wrapped"); } @@ -286,25 +261,13 @@ mod tests { #[test] fn a_place_outside_the_words_is_refused() { let page = page(); - let short = &page[..32 + 2 * D as usize - 1]; - assert_eq!(Producer::<2, D>::new(short, PLACE).err(), Some(Violation::Region)); - assert_eq!(Consumer::<2, D>::new(short, PLACE).err(), Some(Violation::Region)); - } - - /// The wake's two halves on one thread: a consumer that said it sleeps is - /// woken by the next publish and by no later one once it has popped. - #[test] - fn a_sleeper_is_woken_once_and_a_busy_one_never() { - let page = page(); - let (mut tx, mut rx) = ends(&page); - assert_eq!(rx.before_sleep(&page), Ok(Some(Asleep { word: 16, value: 0 }))); - tx.push(&page, [1, 1]).unwrap(); - assert_eq!(tx.publish(&page), Ok(Some(Wake::Peer))); - assert!(rx.pop(&page).unwrap().is_some()); - assert_eq!(rx.pop(&page), Ok(None), "the pop that found nothing said this end is awake"); - tx.push(&page, [2, 2]).unwrap(); - assert_eq!(tx.publish(&page), Ok(Some(Wake::Busy))); - assert_eq!(rx.before_sleep(&page), Ok(None), "an entry was there after all"); - assert_eq!(tx.publish(&page), Ok(None)); + for place in [ + Place { entries: 33, ..PLACE }, + Place { cursors: Cursors { head: WORDS, tail: 16 }, ..PLACE }, + Place { cursors: Cursors { head: 0, tail: WORDS }, ..PLACE }, + ] { + assert_eq!(Producer::<2, D, WORDS>::new(&page, place).err(), Some(Violation::Region)); + assert_eq!(Consumer::<2, D, WORDS>::new(&page, place).err(), Some(Violation::Region)); + } } } diff --git a/toyos-transport/tests/loom.rs b/toyos-transport/tests/loom.rs index 241a024256b..a529456436e 100644 --- a/toyos-transport/tests/loom.rs +++ b/toyos-transport/tests/loom.rs @@ -3,11 +3,6 @@ //! //! - **Publication**: a consumer that sees a tail sees every word of the //! entries below it. `publish-relaxed` takes the edge away. -//! - **No lost wake**: a consumer that says it sleeps and a producer that -//! publishes cannot both miss the other; the futex is a load of the word it -//! waits on, and a lost wake is a consumer that slept over a publish that -//! answered [`Wake::Busy`]. `no-wake-fence` takes the producer's fence away -//! and `no-sleep-fence` the consumer's. //! - **A hostile peer** leaves the honest end with entries, nothing, or a //! named violation, and never more entries than the ring holds. `no-clamp` //! believes the peer. @@ -16,9 +11,9 @@ use core::sync::atomic::Ordering; -use loom::sync::atomic::{fence, AtomicU32}; +use loom::sync::atomic::AtomicU32; use loom::sync::Arc; -use toyos_transport::{Consumer, Cursors, Place, Producer, Untrusted, Violation, Wake, Word}; +use toyos_transport::{Consumer, Cursors, Place, Producer, Untrusted, Violation, Word}; /// A loom atomic as a region word: the trait is this crate's and the type is /// loom's, so the two meet through a wrapper. @@ -31,21 +26,18 @@ impl Word for Shared { fn store(&self, value: u32, order: Ordering) { self.0.store(value, order) } - fn fence() { - fence(Ordering::SeqCst) - } } const E: usize = 2; const D: u32 = 2; -const PLACE: Place = Place { cursors: Cursors { head: 0, tail: 1, sleep: 2 }, entries: 3 }; -const WORDS: usize = 3 + E * D as usize; +const PLACE: Place = Place { cursors: Cursors { head: 0, tail: 1 }, entries: 2 }; +const WORDS: usize = 2 + E * D as usize; -fn page() -> Arc> { - Arc::new((0..WORDS).map(|_| Shared(AtomicU32::new(0))).collect()) +fn page() -> Arc<[Shared; WORDS]> { + Arc::new(core::array::from_fn(|_| Shared(AtomicU32::new(0)))) } -fn ends(page: &[Shared]) -> (Producer, Consumer) { +fn ends(page: &[Shared; WORDS]) -> (Producer, Consumer) { (Producer::new(page, PLACE).unwrap(), Consumer::new(page, PLACE).unwrap()) } @@ -73,49 +65,22 @@ fn a_published_entry_is_read_whole() { None => loom::thread::yield_now(), } } - rx.release(&consumer_page).unwrap(); + rx.release(&consumer_page); read }); for n in 0..2 { assert!(tx.push(&page, entry(n)).unwrap()); - let _ = tx.publish(&page).unwrap(); + tx.publish(&page); } let read = consumer.join().expect("the consumer thread"); assert_eq!(read, [entry(0), entry(1)], "an entry was read before its words"); }); } -/// A consumer that finds nothing sleeps as a futex does — only while the tail -/// still holds what it saw — and is woken only when a publish answers -/// [`Wake::Peer`]. Whatever the schedule, a consumer that slept is woken. -#[test] -fn a_publish_and_a_sleep_cannot_both_miss() { - loom::model(|| { - let page = page(); - let (mut tx, mut rx) = ends(&page); - let consumer_page = Arc::clone(&page); - let consumer = loom::thread::spawn(move || { - if rx.pop(&consumer_page).unwrap().is_some() { - return false; - } - rx.before_sleep(&consumer_page) - .unwrap() - .is_some_and(|asleep| consumer_page[asleep.word].load(Ordering::Relaxed) == asleep.value) - }); - assert!(tx.push(&page, entry(1)).unwrap()); - let wake = tx.publish(&page).unwrap(); - let slept = consumer.join().expect("the consumer thread"); - assert!( - !slept || wake == Some(Wake::Peer), - "the consumer slept over a published entry and the publish answered {wake:?}" - ); - }); -} - /// A producer that stores a tail past the ring, one behind what was released, -/// and garbage into the entries and into the consumer's own head and `sleep`: -/// every pop is an entry, nothing, or [`Violation::TailPastDepth`], and no -/// more than the ring's depth of entries is taken without a release. +/// and garbage into the entries and into the consumer's own head: every pop is +/// an entry, nothing, or [`Violation::TailPastDepth`], and no more than the +/// ring's depth of entries is taken without a release. #[test] fn a_hostile_producer_yields_entries_or_a_violation() { loom::model(|| { @@ -168,7 +133,7 @@ fn a_hostile_consumer_yields_room_or_a_violation() { break; } } - let _ = tx.publish(&page).unwrap(); + tx.publish(&page); } hostile.join().unwrap(); assert!(pushed <= D, "pushed {pushed} into a ring of {D} nobody released"); diff --git a/userland/blockd/src/main.rs b/userland/blockd/src/main.rs index dcb6f984b05..8bfaea41f6b 100644 --- a/userland/blockd/src/main.rs +++ b/userland/blockd/src/main.rs @@ -318,7 +318,7 @@ impl Service { break; }; let unread = (DEPTH - space) as usize; - if s.state.inflight() + unread >= DEPTH as usize || !self.ctrl.has_room() { + if s.state.inflight().len() + unread >= DEPTH as usize || !self.ctrl.has_room() { break; } let words = match s.rings.0.pop(page) { @@ -333,13 +333,13 @@ impl Service { match s.state.take(words) { Taken::Answer(c) => Self::post(s, c), Taken::Issue(req) => { - let owner = Owner::Session { session: id, tag: req.tag, write: req.op == Op::Write }; + let write = matches!(req.op, Op::Write(_)); + let owner = Owner::Session { session: id, tag: req.tag, write }; match req.op { - Op::Read | Op::Write => { - let run = req.run.expect("blockd: a transfer names its run"); + Op::Read(run) | Op::Write(run) => { let at = s.device_addr + run.span().offset as u64; let block = s.state.first() + req.lba; - self.ctrl.submit_io(req.op == Op::Write, block, run.count(), at, owner); + self.ctrl.submit_io(write, block, run.count(), at, owner); } Op::Flush if self.ctrl.vwc => self.ctrl.submit_flush(owner), // No volatile cache: every write answered is on the @@ -352,7 +352,7 @@ impl Service { } } } - s.rings.0.release(s.region.words()).expect("blockd: the region holds every ring word"); + s.rings.0.release(s.region.words()); } /// Publish what each session was answered, and ring its doorbell. @@ -362,7 +362,7 @@ impl Service { continue; } s.posted = false; - if s.rings.1.publish(s.region.words()).expect("blockd: the region holds every ring word").is_some() { + if s.rings.1.publish(s.region.words()) { match s.conn.write_nonblock(&[1]) { Ok(_) | Err(SyscallError::WouldBlock) => {} Err(_) => s.closing = true, @@ -377,7 +377,7 @@ impl Service { let done: Vec = self .sessions .iter() - .filter(|(_, s)| s.closing && s.state.inflight() == 0) + .filter(|(_, s)| s.closing && s.state.inflight().len() == 0) .map(|(id, _)| *id) .collect(); for id in done { diff --git a/userland/blockd/src/region.rs b/userland/blockd/src/region.rs index f6d112c29b1..d268aec9bc9 100644 --- a/userland/blockd/src/region.rs +++ b/userland/blockd/src/region.rs @@ -31,7 +31,7 @@ impl Region { } /// The ring words, as the rings take them. - pub fn words(&self) -> &[AtomicU32] { + pub fn words(&self) -> &[AtomicU32; RING_WORDS] { let base = self.memory.as_ptr(); assert!(base as usize % align_of::() == 0); // SAFETY: the mapping is `SESSION_BYTES` long and lives as long as @@ -39,7 +39,7 @@ impl Region { // `RING_WORDS` words of it are inside it (`layout`'s own assertion); // the base is 2 MiB aligned; and an atomic is the one type that may // alias memory another process writes. - unsafe { core::slice::from_raw_parts(base as *const AtomicU32, RING_WORDS) } + unsafe { &*(base as *const [AtomicU32; RING_WORDS]) } } /// The arena blocks of `run`. diff --git a/userland/blockd/src/session.rs b/userland/blockd/src/session.rs index 9b514a6b788..16c9fae0e76 100644 --- a/userland/blockd/src/session.rs +++ b/userland/blockd/src/session.rs @@ -187,7 +187,7 @@ impl Session { let run = self.arena.alloc(blocks).ok_or(Unsent::ArenaFull)?; self.region.arena(&run).copy_in(0, data); let ticket = self.ticket(); - self.client.submit(ticket, Op::Write, lba, Some(run)); + self.client.submit(ticket, Op::Write(run), lba); self.pending.insert(ticket, Pending::Write); Ok(ticket) } @@ -200,7 +200,7 @@ impl Session { } let run = self.arena.alloc(blocks).ok_or(Unsent::ArenaFull)?; let ticket = self.ticket(); - self.client.submit(ticket, Op::Read, lba, Some(run)); + self.client.submit(ticket, Op::Read(run), lba); self.pending.insert(ticket, Pending::Read { run }); Ok(ticket) } @@ -211,7 +211,7 @@ impl Session { return Err(Unsent::Ended); } let ticket = self.ticket(); - self.client.submit(ticket, Op::Flush, 0, None); + self.client.submit(ticket, Op::Flush, 0); self.pending.insert(ticket, Pending::Flush); Ok(ticket) } @@ -234,8 +234,8 @@ impl Session { let pushed = self.rings.0.push(page, request.encode()); assert_eq!(pushed, Ok(true), "blockd: a request past the room just counted"); } - self.peak = self.peak.max(self.client.on_the_wire()); - if self.rings.0.publish(page).expect("blockd: the region holds every ring word").is_some() { + self.peak = self.peak.max(self.client.on_the_wire().count()); + if self.rings.0.publish(page) { // A full pipe is a doorbell already rung; a gone one is a server // that has ended, which the wait finds. let _ = conn.write_nonblock(&[1]); @@ -302,7 +302,7 @@ impl Session { } } } - self.rings.1.release(page).expect("blockd: the region holds every ring word"); + self.rings.1.release(page); violated } From 64d7149f7a6fc83e99a2e8e9b7f254f050e07414 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 21:31:43 +0200 Subject: [PATCH 5/8] toyos-transport: a run renders by its slots; the block model keeps its page inline A run's span follows from its first slot and count under the one geometry, so `Run`'s Debug prints only those. The block model keys every state by a rendering full of runs, and the shorter one takes its two law tests from about 6.2 s to about 5.5 s of user time on this host. Co-Authored-By: Claude Opus 5.5 --- toyos-blockring/src/model.rs | 4 ++-- toyos-transport/src/arena.rs | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/toyos-blockring/src/model.rs b/toyos-blockring/src/model.rs index 5217ef58daf..b5c468777a7 100644 --- a/toyos-blockring/src/model.rs +++ b/toyos-blockring/src/model.rs @@ -120,14 +120,14 @@ type ServerEnds = (Consumer, Producer, cq: VecDeque, - page: Box<[Shared; WORDS]>, + page: [Shared; WORDS], client: ClientEnds, server: ServerEnds, } impl Queues { fn new() -> Self { - let page = Box::new(core::array::from_fn(|_| Shared(Cell::new(0)))); + let page = core::array::from_fn(|_| Shared(Cell::new(0))); let (client, server) = Self::ends(&page); Self { sq: VecDeque::new(), cq: VecDeque::new(), page, client, server } } diff --git a/toyos-transport/src/arena.rs b/toyos-transport/src/arena.rs index 6a4a363f821..d85fbe9fe9c 100644 --- a/toyos-transport/src/arena.rs +++ b/toyos-transport/src/arena.rs @@ -4,6 +4,8 @@ //! decoded ([`Run::decode`]) and this side's is cut ([`Geometry::run`]). The //! adapter reaches a run's bytes only through its [`Span`]. +use core::fmt; + use crate::{Untrusted, Violation}; /// Bytes `offset..offset + len` of the region. @@ -51,13 +53,20 @@ impl Geometry { } /// Slots of the arena, bounded by its geometry. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct Run { first: u32, count: u32, span: Span, } +/// Its slots alone: the span follows from them. +impl fmt::Debug for Run { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Run").field("first", &self.first).field("count", &self.count).finish() + } +} + impl Run { /// The run a peer's two words name. pub fn decode(first: Untrusted, count: Untrusted, geometry: &Geometry) -> Result { From 0a117e1cd267ecc78b2aa648b0a78182109e2547 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 22:15:20 +0200 Subject: [PATCH 6/8] blockring answers a reset in take order; ring places checked at compile time; lba in the op The model's key renamed tags by first appearance while `ServerSession` kept its requests in a `BTreeMap` and `abort_all` answered in tag order. Two states that differ only in free slots' sequences then got one name and answered a later reset in different orders, and the search pruned the second of them. `ServerSession` now keeps its requests in take order and answers a reset in that order. A tag is now compared only for equality, so renaming by first appearance is exact. - `server::tests::a_reset_answers_in_the_order_taken`: takes 9, then 4, and demands [9, 4]. - `model::tests::states_named_alike_act_alike`: stages the two states, shows the key names them alike, then takes W3 and resets both and demands they are still named alike. To walk named paths the search's events are now one function, `next`, that `dfs` and the test both call. The law checks after an event return a `Result` instead of writing into the run. - Under `ServerSession` restored to round 3's `BTreeMap`, both tests red and the search reproduces that version's end-state counts exactly. `Place` is now made by `Place::new::()`, whose inline const refuses a word at or past `N` (E0080). `new` of each end is infallible: `Violation::Region`, the `Result`s of the ends' `new` and of `layout::client`/`server`, blockd's three `expect`s and the model's `RING` expects are gone. `Cursors` folds into `Place`; `word`, `clamp` and `PUBLISH` move to `queue.rs`, their one user. `Op::Read { run, lba } | Op::Write { run, lba } | Op::Flush`: a flush can no longer carry an `lba`, and `Request` and `Client::submit` lose their separate one. Co-Authored-By: Claude Opus 5.5 --- tests/toyos-rust-tests/src/bin/blockd_io.rs | 2 +- toyos-blockring/src/client.rs | 60 +++--- toyos-blockring/src/entry.rs | 48 ++--- toyos-blockring/src/layout.rs | 16 +- toyos-blockring/src/model.rs | 209 +++++++++++--------- toyos-blockring/src/server.rs | 52 +++-- toyos-transport/Cargo.toml | 1 - toyos-transport/src/lib.rs | 47 +---- toyos-transport/src/queue.rs | 140 ++++++++----- toyos-transport/tests/loom.rs | 6 +- userland/blockd/src/main.rs | 8 +- userland/blockd/src/session.rs | 10 +- 12 files changed, 310 insertions(+), 289 deletions(-) diff --git a/tests/toyos-rust-tests/src/bin/blockd_io.rs b/tests/toyos-rust-tests/src/bin/blockd_io.rs index 8892dac216a..0fea755af1b 100644 --- a/tests/toyos-rust-tests/src/bin/blockd_io.rs +++ b/tests/toyos-rust-tests/src/bin/blockd_io.rs @@ -549,7 +549,7 @@ fn hostile_head() { // words a client puts on the request ring, published and rung. let words = region.words(); let run = ARENA.run(0, 1).unwrap_or_else(|| fail("arena block 0 is no run".into())); - let write = Request { op: Op::Write(run), tag: 1, lba: 0 }; + let write = Request { op: Op::Write { run, lba: 0 }, tag: 1 }; for (at, word) in write.encode().into_iter().enumerate() { words[SQ_BASE + at].store(word, Ordering::Relaxed); } diff --git a/toyos-blockring/src/client.rs b/toyos-blockring/src/client.rs index 9bae51b169f..191cf9c3004 100644 --- a/toyos-blockring/src/client.rs +++ b/toyos-blockring/src/client.rs @@ -102,7 +102,6 @@ enum Kind { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] struct Queued { op: Op, - lba: u64, kind: Kind, } @@ -144,13 +143,13 @@ impl Client { Self::default() } - /// Ask for a read or write at `lba`, or for a flush. - pub fn submit(&mut self, ticket: Ticket, op: Op, lba: u64) { + /// Ask for a read, a write or a flush. + pub fn submit(&mut self, ticket: Ticket, op: Op) { let kind = match op { Op::Flush => Kind::Flush(ticket), - Op::Read(_) | Op::Write(_) => Kind::User(ticket), + Op::Read { .. } | Op::Write { .. } => Kind::User(ticket), }; - self.outbox.push_back(Queued { op, lba, kind }); + self.outbox.push_back(Queued { op, kind }); } fn reissuing(&self) -> bool { @@ -173,8 +172,7 @@ impl Client { // later one of the caller's. Kind::Reissue(acked) => { let blocked = self.wire.values().any(|sent| { - let q = sent.queued; - matches!(q.op, Op::Write(run) if acked.overlaps_range(q.lba, run.count())) + matches!(sent.queued.op, Op::Write { run, lba } if acked.overlaps_range(lba, run.count())) }); if blocked { return None; @@ -191,7 +189,7 @@ impl Client { if matches!(front.kind, Kind::Reissue(_)) { self.reissued += 1; } - Some(Request { op: front.op, tag, lba: front.lba }) + Some(Request { op: front.op, tag }) } /// What the server answered. @@ -201,9 +199,9 @@ impl Client { match (q.kind, completion.status) { (Kind::User(ticket), Status::Ok) => { match q.op { - Op::Write(run) => { + Op::Write { run, lba } => { let seq = self.bump(); - let acked = Acked { lba: q.lba, run, first: seq, seq, attempts: 0 }; + let acked = Acked { lba, run, first: seq, seq, attempts: 0 }; // Sent before a loss it did not see: the device may // have taken it and lost it, and an earlier write it // overlaps is being issued again — so it is issued @@ -214,13 +212,13 @@ impl Client { self.acked.push_back(acked); } } - Op::Read(run) => self.released.push_back(run), + Op::Read { run, .. } => self.released.push_back(run), Op::Flush => return Err(Violation::Entry), } self.answers.push_back((ticket, Outcome::Done)); } (Kind::User(ticket), status) => { - if let Op::Read(run) | Op::Write(run) = q.op { + if let Op::Read { run, .. } | Op::Write { run, .. } = q.op { self.released.push_back(run); } let outcome = match status { @@ -295,7 +293,7 @@ impl Client { Kind::User(ticket) => { #[cfg(not(feature = "mutate-session-end-forgets"))] { - if let Op::Read(run) | Op::Write(run) = q.op { + if let Op::Read { run, .. } | Op::Write { run, .. } = q.op { self.released.push_back(run); } self.answers.push_back((ticket, Outcome::Refused)); @@ -371,7 +369,7 @@ impl Client { Kind::User(_) | Kind::Flush(_) => true, }) .unwrap_or(self.outbox.len()); - self.outbox.insert(at, Queued { op: Op::Write(acked.run), lba: acked.lba, kind: Kind::Reissue(acked) }); + self.outbox.insert(at, Queued { op: Op::Write { run: acked.run, lba: acked.lba }, kind: Kind::Reissue(acked) }); } /// Ask the flush `ticket` again, once every write being issued again is @@ -382,7 +380,7 @@ impl Client { .iter() .position(|q| !matches!(q.kind, Kind::Reissue(_) | Kind::Flush(_))) .unwrap_or(self.outbox.len()); - self.outbox.insert(at, Queued { op: Op::Flush, lba: 0, kind: Kind::Flush(ticket) }); + self.outbox.insert(at, Queued { op: Op::Flush, kind: Kind::Flush(ticket) }); } /// The device may no longer hold any write acknowledged and not covered: @@ -431,7 +429,7 @@ mod tests { #[test] fn nothing_goes_out_before_a_session() { let mut client: Client = Client::new(); - client.submit(1, Op::Read(run(0, 1)), 0); + client.submit(1, Op::Read { run: run(0, 1), lba: 0 }); assert_eq!(client.next_request(), None); client.session_started(); assert!(client.next_request().is_some()); @@ -441,7 +439,7 @@ mod tests { fn a_second_answer_for_one_tag_is_a_violation() { let mut client: Client = Client::new(); client.session_started(); - client.submit(1, Op::Read(run(0, 1)), 0); + client.submit(1, Op::Read { run: run(0, 1), lba: 0 }); let r = client.next_request().unwrap(); answer(&mut client, r, Status::Ok); assert_eq!(client.complete(Completion { tag: r.tag, status: Status::Ok }), Err(Violation::Tag)); @@ -454,14 +452,14 @@ mod tests { fn a_lost_flush_reissues_then_asks_again() { let mut client: Client = Client::new(); client.session_started(); - client.submit(1, Op::Write(run(3, 2)), 5); + client.submit(1, Op::Write { run: run(3, 2), lba: 5 }); let w = client.next_request().unwrap(); answer(&mut client, w, Status::Ok); - client.submit(2, Op::Flush, 0); + client.submit(2, Op::Flush); let f = client.next_request().unwrap(); answer(&mut client, f, Status::Lost); let again = client.next_request().unwrap(); - assert_eq!((again.op, again.lba), (Op::Write(run(3, 2)), 5)); + assert_eq!(again.op, Op::Write { run: run(3, 2), lba: 5 }); assert_eq!(client.next_request(), None, "the flush waits for the write"); answer(&mut client, again, Status::Ok); let f2 = client.next_request().unwrap(); @@ -479,20 +477,20 @@ mod tests { fn overlapping_writes_go_out_again_in_order() { let mut client: Client = Client::new(); client.session_started(); - client.submit(1, Op::Write(run(0, 2)), 0); + client.submit(1, Op::Write { run: run(0, 2), lba: 0 }); let a = client.next_request().unwrap(); answer(&mut client, a, Status::Ok); - client.submit(2, Op::Write(run(2, 1)), 1); + client.submit(2, Op::Write { run: run(2, 1), lba: 1 }); let b = client.next_request().unwrap(); answer(&mut client, b, Status::Ok); client.session_ended(); client.session_started(); let first = client.next_request().unwrap(); - assert_eq!((first.op, first.lba), (Op::Write(run(0, 2)), 0)); + assert_eq!(first.op, Op::Write { run: run(0, 2), lba: 0 }); assert_eq!(client.next_request(), None, "the overlapping one waits"); answer(&mut client, first, Status::Ok); let second = client.next_request().unwrap(); - assert_eq!((second.op, second.lba), (Op::Write(run(2, 1)), 1)); + assert_eq!(second.op, Op::Write { run: run(2, 1), lba: 1 }); } /// A write the device refused on every reissue is gone, and its caller was @@ -502,22 +500,22 @@ mod tests { fn a_write_given_up_poisons_every_later_flush() { let mut client: Client = Client::new(); client.session_started(); - client.submit(1, Op::Write(run(0, 1)), 0); + client.submit(1, Op::Write { run: run(0, 1), lba: 0 }); let w = client.next_request().unwrap(); answer(&mut client, w, Status::Ok); client.session_ended(); for _ in 0..=MAX_ATTEMPTS { client.session_started(); let again = client.next_request().unwrap(); - assert_eq!((again.op, again.lba), (Op::Write(run(0, 1)), 0)); + assert_eq!(again.op, Op::Write { run: run(0, 1), lba: 0 }); answer(&mut client, again, Status::Device); } - client.submit(2, Op::Flush, 0); + client.submit(2, Op::Flush); let f = client.next_request().unwrap(); assert_eq!(f.op, Op::Flush); answer(&mut client, f, Status::Ok); assert_eq!(client.take_answers().collect::>(), [(1, Outcome::Done), (2, Outcome::Device)]); - client.submit(3, Op::Flush, 0); + client.submit(3, Op::Flush); let f = client.next_request().unwrap(); answer(&mut client, f, Status::Ok); assert_eq!(client.take_answers().collect::>(), [(3, Outcome::Device)]); @@ -527,14 +525,14 @@ mod tests { fn what_was_on_the_wire_at_the_end_is_refused_and_what_was_not_waits() { let mut client: Client = Client::new(); client.session_started(); - client.submit(1, Op::Write(run(0, 1)), 0); - client.submit(2, Op::Read(run(1, 1)), 4); + client.submit(1, Op::Write { run: run(0, 1), lba: 0 }); + client.submit(2, Op::Read { run: run(1, 1), lba: 4 }); let _ = client.next_request().unwrap(); client.session_ended(); assert_eq!(client.take_answers().collect::>(), [(1, Outcome::Refused)]); assert_eq!(client.take_released().collect::>(), [run(0, 1)]); client.session_started(); let r = client.next_request().unwrap(); - assert_eq!((r.op, r.lba), (Op::Read(run(1, 1)), 4)); + assert_eq!(r.op, Op::Read { run: run(1, 1), lba: 4 }); } } diff --git a/toyos-blockring/src/entry.rs b/toyos-blockring/src/entry.rs index 7dc9948a303..f39788b4c00 100644 --- a/toyos-blockring/src/entry.rs +++ b/toyos-blockring/src/entry.rs @@ -9,13 +9,15 @@ use toyos_transport::{Run, Untrusted}; use crate::layout::{ARENA, CQE_WORDS, MAX_REQUEST_BLOCKS, SQE_WORDS}; /// What a request asks of the partition, and the arena blocks the data is in -/// or goes to. +/// or goes to. `lba` is the partition's own block number, from 0: nothing in +/// this protocol names a device block, so a neighbour's blocks have no +/// spelling. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Op { /// The run's blocks from the partition's block `lba` into the arena. - Read(Run), + Read { run: Run, lba: u64 }, /// The run's blocks from the arena to the partition's block `lba`. - Write(Run), + Write { run: Run, lba: u64 }, /// Every write acknowledged before this was submitted, onto the medium. Flush, } @@ -24,15 +26,12 @@ const READ: u32 = 1; const WRITE: u32 = 2; const FLUSH: u32 = 3; -/// One request. `lba` is the partition's own block number, from 0: nothing in -/// this protocol names a device block, so a neighbour's blocks have no -/// spelling. A flush carries no range: its `lba` is zero. +/// One request. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Request { pub op: Op, /// The client's name for it, echoed by its completion. pub tag: u32, - pub lba: u64, } /// Why a request was answered without being done. @@ -46,12 +45,12 @@ pub enum Refused { impl Request { pub fn encode(&self) -> [u32; SQE_WORDS] { - let (op, first, count) = match self.op { - Op::Read(run) => (READ, run.first(), run.count()), - Op::Write(run) => (WRITE, run.first(), run.count()), - Op::Flush => (FLUSH, 0, 0), + let (op, lba, first, count) = match self.op { + Op::Read { run, lba } => (READ, lba, run.first(), run.count()), + Op::Write { run, lba } => (WRITE, lba, run.first(), run.count()), + Op::Flush => (FLUSH, 0, 0, 0), }; - [op, self.tag, self.lba as u32, (self.lba >> 32) as u32, count, first, 0, 0] + [op, self.tag, lba as u32, (lba >> 32) as u32, count, first, 0, 0] } /// The request these words are, bounded against a partition of @@ -66,12 +65,12 @@ impl Request { return Err(refused); } let lba = u64::from(opaque(lba_low)) | (u64::from(opaque(lba_high)) << 32); - let transfer: fn(Run) -> Op = if op.is(READ) { - Op::Read + let transfer: fn(Run, u64) -> Op = if op.is(READ) { + |run, lba| Op::Read { run, lba } } else if op.is(WRITE) { - Op::Write + |run, lba| Op::Write { run, lba } } else if op.is(FLUSH) && lba == 0 && blocks.is(0) && arena.is(0) { - return Ok(Self { op: Op::Flush, tag, lba }); + return Ok(Self { op: Op::Flush, tag }); } else { return Err(refused); }; @@ -80,7 +79,7 @@ impl Request { if blocks > MAX_REQUEST_BLOCKS || lba.checked_add(u64::from(blocks)).is_none_or(|end| end > partition_blocks) { return Err(refused); } - Ok(Self { op: transfer(run), tag, lba }) + Ok(Self { op: transfer(run, lba), tag }) } } @@ -164,13 +163,13 @@ mod tests { fn a_request_survives_its_words() { let last = ARENA.slots() - MAX_REQUEST_BLOCKS; for request in [ - Request { op: Op::Read(run(0, 1)), tag: 7, lba: 999 }, - Request { op: Op::Write(run(last, MAX_REQUEST_BLOCKS)), tag: u32::MAX, lba: 0 }, - Request { op: Op::Flush, tag: 0, lba: 0 }, + Request { op: Op::Read { run: run(0, 1), lba: 999 }, tag: 7 }, + Request { op: Op::Write { run: run(last, MAX_REQUEST_BLOCKS), lba: 0 }, tag: u32::MAX }, + Request { op: Op::Flush, tag: 0 }, ] { assert_eq!(Request::decode(peer(request.encode()), PARTITION), Ok(request)); } - let wide = Request { op: Op::Read(run(3, 2)), tag: 1, lba: 1 << 40 }; + let wide = Request { op: Op::Read { run: run(3, 2), lba: 1 << 40 }, tag: 1 }; assert_eq!(Request::decode(peer(wide.encode()), u64::MAX), Ok(wide)); } @@ -178,7 +177,8 @@ mod tests { /// refusal still carries its tag. #[test] fn a_request_outside_its_bounds_is_refused_by_tag() { - let good = Request { op: Op::Write(run(5, 2)), tag: 42, lba: 10 }; + let write = |lba| Request { op: Op::Write { run: run(5, 2), lba }, tag: 42 }; + let good = write(10); let with = |at: usize, word: u32| { let mut words = good.encode(); words[at] = word; @@ -191,8 +191,8 @@ mod tests { ("too many blocks", with(4, MAX_REQUEST_BLOCKS + 1)), ("past the arena", with(5, ARENA.slots() - 1)), ("arena wraps", with(5, u32::MAX)), - ("past the partition", Request { lba: PARTITION - 1, ..good }.encode()), - ("lba wraps", Request { lba: u64::MAX, ..good }.encode()), + ("past the partition", write(PARTITION - 1).encode()), + ("lba wraps", write(u64::MAX).encode()), ("reserved word", with(7, 1)), ("a flush with a range", with(0, 3)), ]; diff --git a/toyos-blockring/src/layout.rs b/toyos-blockring/src/layout.rs index f0d108a5d71..05f60c90d85 100644 --- a/toyos-blockring/src/layout.rs +++ b/toyos-blockring/src/layout.rs @@ -3,7 +3,7 @@ //! The four ring indices sit on cache lines of their own, so the client's //! stores to its two and the server's to its two never share a line. -use toyos_transport::{Consumer, Cursors, Geometry, Place, Producer, Violation, Word}; +use toyos_transport::{Consumer, Geometry, Place, Producer, Word}; /// A session's whole region: the one size shared memory comes in. pub const SESSION_BYTES: usize = Geometry::BYTES as usize; @@ -44,10 +44,8 @@ pub const CQ_BASE: usize = SQ_BASE + DEPTH as usize * SQE_WORDS; pub const RING_WORDS: usize = CQ_BASE + DEPTH as usize * CQE_WORDS; /// The request ring and the completion ring. -pub const REQUESTS: Place = - Place { cursors: Cursors { head: SQ_HEAD, tail: SQ_TAIL }, entries: SQ_BASE }; -pub const COMPLETIONS: Place = - Place { cursors: Cursors { head: CQ_HEAD, tail: CQ_TAIL }, entries: CQ_BASE }; +pub const REQUESTS: Place = Place::new::(); +pub const COMPLETIONS: Place = Place::new::(); /// A client's two ends: requests out, completions in. pub type ClientRings = (Producer, Consumer); @@ -58,14 +56,14 @@ pub type ServerRings = (Consumer, Producer(page: &[W; RING_WORDS]) -> Result { - Ok((Producer::new(page, REQUESTS)?, Consumer::new(page, COMPLETIONS)?)) +pub fn client(page: &[W; RING_WORDS]) -> ClientRings { + (Producer::new(page, REQUESTS), Consumer::new(page, COMPLETIONS)) } /// The server's ends of a session page it was sent, every word it owns set to /// 0. Whatever the client left in its own is bounded when first looked at. -pub fn server(page: &[W; RING_WORDS]) -> Result { - Ok((Consumer::new(page, REQUESTS)?, Producer::new(page, COMPLETIONS)?)) +pub fn server(page: &[W; RING_WORDS]) -> ServerRings { + (Consumer::new(page, REQUESTS), Producer::new(page, COMPLETIONS)) } const _: () = assert!(RING_WORDS * 4 <= Geometry::HEADER_BYTES as usize); diff --git a/toyos-blockring/src/model.rs b/toyos-blockring/src/model.rs index b5c468777a7..0edc7115fba 100644 --- a/toyos-blockring/src/model.rs +++ b/toyos-blockring/src/model.rs @@ -35,7 +35,7 @@ use core::sync::atomic::Ordering; use std::collections::HashSet; use toyos_blockhold::Holds; -use toyos_transport::{Consumer, Cursors, Place, Producer, Untrusted, Word}; +use toyos_transport::{Consumer, Place, Producer, Untrusted, Word}; use crate::client::{Client, Outcome, Ticket, MAX_ATTEMPTS}; use crate::entry::{Completion, Op, Request}; @@ -103,10 +103,11 @@ impl Word for Shared { /// How deep the model's rings are: shallow enough that a script fills and /// wraps each. const DEPTH: u32 = 4; -const SQ: Place = Place { cursors: Cursors { head: 0, tail: 1 }, entries: 4 }; -const CQ: Place = Place { cursors: Cursors { head: 2, tail: 3 }, entries: SQ.entries + DEPTH as usize * SQE_WORDS }; -const WORDS: usize = CQ.entries + DEPTH as usize * CQE_WORDS; -const RING: &str = "the page holds every ring word"; +const SQ_BASE: usize = 4; +const CQ_BASE: usize = SQ_BASE + DEPTH as usize * SQE_WORDS; +const WORDS: usize = CQ_BASE + DEPTH as usize * CQE_WORDS; +const SQ: Place = Place::new::<0, 1, SQ_BASE>(); +const CQ: Place = Place::new::<2, 3, CQ_BASE>(); type ClientEnds = (Producer, Consumer); type ServerEnds = (Consumer, Producer); @@ -134,8 +135,7 @@ impl Queues { /// Both ends over `page`, every cursor stored 0. fn ends(page: &[Shared; WORDS]) -> (ClientEnds, ServerEnds) { - let client = (Producer::new(page, SQ).expect(RING), Consumer::new(page, CQ).expect(RING)); - (client, (Consumer::new(page, SQ).expect(RING), Producer::new(page, CQ).expect(RING))) + ((Producer::new(page, SQ), Consumer::new(page, CQ)), (Consumer::new(page, SQ), Producer::new(page, CQ))) } /// The session is over: what either queue held is gone, and the next @@ -235,6 +235,22 @@ struct Run<'a> { /// Explore `script` against at most `failures`. pub fn explore(script: &[Step], failures: Failures) -> Explored { + let mut run = Run { + script, + seen: HashSet::new(), + broken: None, + ends: 0, + given_up: 0, + filled: [false; 2], + may_give_up: failures.total() > MAX_ATTEMPTS, + path: Vec::new(), + }; + dfs(&mut run, start(failures)); + Explored { broken: run.broken, ends: run.ends, given_up: run.given_up, filled: run.filled } +} + +/// The client connected to a fresh server, with nothing asked yet. +fn start(failures: Failures) -> World { let mut world = World { client: Client::new(), next: 0, @@ -251,19 +267,7 @@ pub fn explore(script: &[Step], failures: Failures) -> Explored { left: failures, }; connect(&mut world); - let may_give_up = failures.total() > MAX_ATTEMPTS; - let mut run = Run { - script, - seen: HashSet::new(), - broken: None, - ends: 0, - given_up: 0, - filled: [false; 2], - may_give_up, - path: Vec::new(), - }; - dfs(&mut run, world); - Explored { broken: run.broken, ends: run.ends, given_up: run.given_up, filled: run.filled } + world } fn connect(world: &mut World) { @@ -276,8 +280,7 @@ fn connect(world: &mut World) { pump(world); } -/// What the glue does after every event: every request the client will send -/// goes onto the ring. +/// Every request the client will send goes onto the ring. fn pump(world: &mut World) { while let Some(request) = world.client.next_request() { world.queues.send(request); @@ -296,9 +299,9 @@ fn write_of(script: &[Step], ticket: Ticket) -> Option<(u64, u8)> { /// the cache onto the medium. fn apply(script: &[Step], cache: &mut [Option; BLOCKS], media: &mut [u8; BLOCKS], request: Request) { match request.op { - Op::Write(run) => { + Op::Write { run, lba } => { let (_, value) = write_of(script, u64::from(run.first())).expect("a write's arena block is its ticket"); - cache[request.lba as usize] = Some(value); + cache[lba as usize] = Some(value); } Op::Flush => { for (b, slot) in cache.iter_mut().enumerate() { @@ -307,7 +310,7 @@ fn apply(script: &[Step], cache: &mut [Option; BLOCKS], media: &mut [u8; BLO } } } - Op::Read(_) => {} + Op::Read { .. } => {} } } @@ -351,56 +354,44 @@ fn fail(run: &mut Run, law: Law, why: String) { } } -fn go(run: &mut Run, step: String, world: World) { - run.path.push(step); - dfs(run, world); - run.path.pop(); -} - -/// Take the client's answers and hold each against the laws. -fn collect(run: &mut Run, world: &mut World) { +/// What the glue does after every event: take the client's answers, hold each +/// against the laws, and pump. +fn settle(script: &[Step], world: &mut World) -> Result<(), (Law, String)> { let answers: Vec<_> = world.client.take_answers().collect(); let _ = world.client.take_released().count(); for (ticket, outcome) in answers { let had = world.answers.entry(ticket).or_default(); had.push(outcome); if had.len() > 1 { - let had = had.clone(); - fail(run, Law::Answers, format!("ticket {ticket} answered twice: {had:?}")); - return; + return Err((Law::Answers, format!("ticket {ticket} answered twice: {had:?}"))); } if outcome == Outcome::Durable { - durable(run, world, ticket); + durable(script, world, ticket).map_err(|why| (Law::Durable, why))?; } } pump(world); + Ok(()) } /// What the flush `ticket`, just answered durable, promised is on the medium. -fn durable(run: &mut Run, world: &World, flush: Ticket) { +fn durable(script: &[Step], world: &World, flush: Ticket) -> Result<(), String> { let before = &world.acked_before[&flush]; for block in 0..BLOCKS as u64 { - let on_block = |t: &Ticket| write_of(run.script, *t).is_some_and(|(b, _)| b == block); + let on_block = |t: &Ticket| write_of(script, *t).is_some_and(|(b, _)| b == block); let Some(last) = before.iter().copied().filter(on_block).max() else { continue }; - let (_, want) = write_of(run.script, last).expect("a write"); + let (_, want) = write_of(script, last).expect("a write"); let allowed: Vec = core::iter::once(want) - .chain((last + 1..run.script.len() as u64).filter(on_block).filter_map(|t| { - write_of(run.script, t).map(|(_, v)| v) - })) + .chain((last + 1..script.len() as u64).filter(on_block).filter_map(|t| write_of(script, t).map(|(_, v)| v))) .collect(); let on = world.media[block as usize]; if !allowed.contains(&on) { - fail( - run, - Law::Durable, - format!( - "flush {flush} was answered durable with block {block} holding {on}, not the \ - {want} acknowledged before it (or a later one of {allowed:?})" - ), - ); - return; + return Err(format!( + "flush {flush} was answered durable with block {block} holding {on}, not the \ + {want} acknowledged before it (or a later one of {allowed:?})" + )); } } + Ok(()) } /// Nothing more can happen: every ticket was answered, and every flush said @@ -443,9 +434,7 @@ impl Names { /// A state's name in the search: everything it holds, with every tag renamed /// by first appearance, the client's first in slot order, and the client's -/// table rendered by its filled slots. A fresh tag equals none present, so -/// states named alike differ only in their tags' numbers, which the protocol -/// compares for equality and orders only in a reset's answers. +/// table rendered by its filled slots. fn key(world: &World) -> String { let mut names = Names::default(); let wire: Vec = world.client.on_the_wire().map(|t| names.of(t)).collect(); @@ -479,8 +468,30 @@ fn dfs(run: &mut Run, mut world: World) { } run.filled[0] |= world.queues.sq.len() == DEPTH as usize; run.filled[1] |= world.queues.cq.len() == DEPTH as usize; - let mut moved = false; - let script = run.script; + let next = next(run.script, &world); + if next.is_empty() { + run.ends += 1; + end(run, &world); + } + for (step, after) in next { + run.path.push(step); + match after { + Ok(world) => dfs(run, world), + Err((law, why)) => fail(run, law, why), + } + run.path.pop(); + } +} + +/// The world after an event, or the law it broke. +type After = Result; + +/// Every event that can happen next, named, and what it leads to. +fn next(script: &[Step], world: &World) -> Vec<(String, After)> { + let mut next = Vec::new(); + let mut after = |step: String, after: After| { + next.push((step, after.and_then(|mut w| settle(script, &mut w).map(|()| w)))); + }; // The caller asks for its next step, never a write to a block with a write // still unanswered: two writes in flight to one block are unordered. @@ -496,7 +507,7 @@ fn dfs(run: &mut Run, mut world: World) { match script[world.next] { Step::Write { block, .. } => { let run = ARENA.run(ticket as u32, 1).expect("a script's ticket is an arena block"); - w.client.submit(ticket, Op::Write(run), block); + w.client.submit(ticket, Op::Write { run, lba: block }); } Step::Flush => { let acked = w @@ -506,13 +517,11 @@ fn dfs(run: &mut Run, mut world: World) { .map(|(t, _)| *t) .collect(); w.acked_before.insert(ticket, acked); - w.client.submit(ticket, Op::Flush, 0); + w.client.submit(ticket, Op::Flush); } } w.next += 1; - pump(&mut w); - moved = true; - go(run, format!("ask {}", world.next), w); + after(format!("ask {}", world.next), Ok(w)); } } @@ -531,8 +540,7 @@ fn dfs(run: &mut Run, mut world: World) { format!("refuse #{}", c.tag) } }; - moved = true; - go(run, step, w); + after(step, Ok(w)); } // The device completes any one command it holds. @@ -546,8 +554,7 @@ fn dfs(run: &mut Run, mut world: World) { w.queues.post(c); } } - moved = true; - go(run, format!("done {:?}#{tag}", request.op), w); + after(format!("done {:?}#{tag}", request.op), Ok(w)); } // The device fails any one command it holds: not done, and answered so. @@ -562,8 +569,7 @@ fn dfs(run: &mut Run, mut world: World) { w.queues.post(c); } } - moved = true; - go(run, format!("fail {:?}#{tag}", request.op), w); + after(format!("fail {:?}#{tag}", request.op), Ok(w)); } } @@ -571,13 +577,10 @@ fn dfs(run: &mut Run, mut world: World) { if world.client.up() && !world.queues.cq.is_empty() { let mut w = world.clone(); let c = w.queues.read().expect("just seen"); - if w.client.complete(c).is_err() { - fail(run, Law::Answers, format!("the client met a second completion for tag {}", c.tag)); - return; - } - collect(run, &mut w); - moved = true; - go(run, format!("read #{} {:?}", c.tag, c.status), w); + let read = w.client.complete(c).map(|()| w).map_err(|_| { + (Law::Answers, format!("the client met a second completion for tag {}", c.tag)) + }); + after(format!("read #{} {:?}", c.tag, c.status), read); } // The server reads a completion the device posted before it was reset: @@ -590,13 +593,12 @@ fn dfs(run: &mut Run, mut world: World) { if let Some(c) = server.session.complete(tag, true, &mut server.holds, losses) { w.queues.post(c); } - moved = true; - go(run, format!("posted #{tag}"), w); + after(format!("posted #{tag}"), Ok(w)); } // The device is reset under everything in flight. if world.left.resets > 0 && world.alive { - for (posted, cache, media) in fates(script, &world, true) { + for (posted, cache, media) in fates(script, world, true) { let mut w = world.clone(); w.left.resets -= 1; w.device.clear(); @@ -608,14 +610,13 @@ fn dfs(run: &mut Run, mut world: World) { for c in server.session.abort_all() { w.queues.post(c); } - moved = true; - go(run, format!("reset({posted:?} {cache:?} {media:?})"), w); + after(format!("reset({posted:?} {cache:?} {media:?})"), Ok(w)); } } // The server dies. if world.left.crashes > 0 && world.alive { - for (_, cache, media) in fates(script, &world, false) { + for (_, cache, media) in fates(script, world, false) { let mut w = world.clone(); w.left.crashes -= 1; w.device.clear(); @@ -624,8 +625,7 @@ fn dfs(run: &mut Run, mut world: World) { w.media = media; w.server = None; w.alive = false; - moved = true; - go(run, format!("crash({cache:?} {media:?})"), w); + after(format!("crash({cache:?} {media:?})"), Ok(w)); } } @@ -634,23 +634,17 @@ fn dfs(run: &mut Run, mut world: World) { let mut w = world.clone(); w.client.session_ended(); w.queues.reset(); - collect(run, &mut w); - moved = true; - go(run, "notice".into(), w); + after("notice".into(), Ok(w)); } // It reconnects to the server started in its place. if !world.alive && !world.client.up() { let mut w = world.clone(); connect(&mut w); - moved = true; - go(run, "reconnect".into(), w); + after("reconnect".into(), Ok(w)); } - if !moved { - run.ends += 1; - end(run, &world); - } + next } #[cfg(test)] @@ -664,6 +658,7 @@ mod tests { Step::Write { block: 0, value: 3 }, Step::Flush, ]; + const _: () = assert!(SCRIPT.len() > DEPTH as usize, "a ring never wraps"); const fn at_most(resets: u8, crashes: u8, errors: u8) -> Failures { Failures { resets, crashes, errors } @@ -750,7 +745,6 @@ mod tests { assert_eq!(explored.broken, None); assert!(explored.ends >= 1); assert_eq!(explored.filled, [true, true], "a ring never held its depth"); - assert!(SCRIPT.len() > DEPTH as usize, "a ring never wraps"); } /// Nor is the give-up path out of its reach: past [`MAX_ATTEMPTS`] failures @@ -762,4 +756,33 @@ mod tests { assert_eq!(explored.broken, None); assert!(explored.given_up > 0, "no run gave a write up"); } + + /// The world after each event in turn, each the first whose name starts with + /// the word given. + fn walk(mut world: World, events: &[&str]) -> World { + for event in events { + let (_, after) = next(&SCRIPT, &world) + .into_iter() + .find(|(step, _)| step.starts_with(event)) + .unwrap_or_else(|| panic!("no {event} after {}", key(&world))); + world = after.expect("no law is broken on the way"); + } + world + } + + /// Two states a key that orders tags by value merges, though a reset parts + /// them: F2 on the device and slot 1 free, reached with W1 asked after W0 + /// was answered, and with the two in flight together. Named alike, they act + /// alike: with W3 taken and the device reset, they are still named alike. + #[test] + fn states_named_alike_act_alike() { + let fresh = start(at_most(1, 0, 0)); + let answered_first = + walk(fresh.clone(), &["ask", "take", "done", "read", "ask", "take", "done", "read", "ask", "take"]); + let together = walk(fresh, &["ask", "ask", "take", "done", "read", "take", "done", "read", "ask", "take"]); + assert_ne!(answered_first.device, together.device, "the flush on the device has one tag in both"); + assert_eq!(key(&answered_first), key(&together), "the search tells the two apart"); + let [answered_first, together] = [answered_first, together].map(|w| walk(w, &["ask", "take", "reset"])); + assert_eq!(key(&answered_first), key(&together), "a reset answered the two in different orders"); + } } diff --git a/toyos-blockring/src/server.rs b/toyos-blockring/src/server.rs index ac41a160fd9..24ca1703bf2 100644 --- a/toyos-blockring/src/server.rs +++ b/toyos-blockring/src/server.rs @@ -15,7 +15,7 @@ //! own since it last heard. The loss count is the device's, bumped by whoever //! resets it. -use alloc::collections::BTreeMap; +use alloc::vec::Vec; use toyos_blockhold::{Holds, Writer}; use toyos_transport::Untrusted; @@ -31,8 +31,9 @@ pub struct ServerSession { blocks: u64, /// The span of the device this session holds, which is its writer. first: u64, - /// By tag: the op each request in flight asked for. - inflight: BTreeMap, + /// Each request in flight, in the order it was taken: a tag is only ever + /// compared for equality. + inflight: Vec<(u32, Op)>, } /// A request the device is to be asked for. @@ -48,7 +49,7 @@ impl ServerSession { /// A session over the partition at device block `first`, `blocks` long, /// which `holds` already holds for it. pub fn new(first: u64, blocks: u64) -> Self { - Self { blocks, first, inflight: BTreeMap::new() } + Self { blocks, first, inflight: Vec::new() } } /// The writer this session's writes and flushes are accounted to. @@ -66,9 +67,9 @@ impl ServerSession { self.blocks } - /// Requests taken and not yet answered, by tag. + /// Requests taken and not yet answered, in the order they were taken. pub fn inflight(&self) -> impl ExactSizeIterator + '_ { - self.inflight.iter().map(|(&tag, &op)| (tag, op)) + self.inflight.iter().copied() } /// Decide what one entry the client published is. @@ -83,10 +84,10 @@ impl ServerSession { return Taken::Answer(Completion { tag, status: Status::Invalid }) } }; - if self.inflight.contains_key(&request.tag) { + if self.inflight.iter().any(|&(tag, _)| tag == request.tag) { return Taken::Answer(Completion { tag: request.tag, status: Status::Invalid }); } - self.inflight.insert(request.tag, request.op); + self.inflight.push((request.tag, request.op)); Taken::Issue(request) } @@ -100,11 +101,12 @@ impl ServerSession { holds: &mut Holds, losses: u64, ) -> Option { - let op = self.inflight.remove(&tag)?; + let at = self.inflight.iter().position(|&(t, _)| t == tag)?; + let (_, op) = self.inflight.remove(at); let status = match (op, done) { (_, false) => Status::Device, - (Op::Read(_), true) => Status::Ok, - (Op::Write(_), true) => { + (Op::Read { .. }, true) => Status::Ok, + (Op::Write { .. }, true) => { holds.wrote(self.writer(), losses); Status::Ok } @@ -117,14 +119,10 @@ impl ServerSession { } /// The device was reset under every request in flight: each is answered - /// [`Status::Device`], and a late answer from the device for any of them - /// finds nothing to complete. - pub fn abort_all(&mut self) -> alloc::vec::Vec { - let answered = self - .inflight - .keys() - .map(|&tag| Completion { tag, status: Status::Device }) - .collect(); + /// [`Status::Device`], in the order it was taken, and a late answer from + /// the device for any of them finds nothing to complete. + pub fn abort_all(&mut self) -> Vec { + let answered = self.inflight.iter().map(|&(tag, _)| Completion { tag, status: Status::Device }).collect(); #[cfg(not(feature = "mutate-abort-keeps-inflight"))] self.inflight.clear(); answered @@ -137,10 +135,10 @@ mod tests { use crate::layout::ARENA; fn write(tag: u32) -> [Untrusted; SQE_WORDS] { - Request { op: Op::Write(ARENA.run(0, 1).unwrap()), tag, lba: 0 }.encode().map(Untrusted::new) + Request { op: Op::Write { run: ARENA.run(0, 1).unwrap(), lba: 0 }, tag }.encode().map(Untrusted::new) } fn flush(tag: u32) -> [Untrusted; SQE_WORDS] { - Request { op: Op::Flush, tag, lba: 0 }.encode().map(Untrusted::new) + Request { op: Op::Flush, tag }.encode().map(Untrusted::new) } #[test] @@ -153,6 +151,18 @@ mod tests { assert_eq!(session.complete(1, true, &mut holds, 1), None); } + /// A reset answers in the order the requests were taken, whatever their + /// tags' values: nothing orders a tag but its arrival. + #[test] + fn a_reset_answers_in_the_order_taken() { + let mut session = ServerSession::new(0, 10); + for tag in [9, 4] { + assert!(matches!(session.take(write(tag)), Taken::Issue(_))); + } + let answered: Vec = session.abort_all().iter().map(|c| c.tag).collect(); + assert_eq!(answered, [9, 4]); + } + #[test] fn a_tag_in_flight_twice_is_refused_unissued() { let mut session = ServerSession::new(0, 10); diff --git a/toyos-transport/Cargo.toml b/toyos-transport/Cargo.toml index 56018c4b77c..cd1fbf10a09 100644 --- a/toyos-transport/Cargo.toml +++ b/toyos-transport/Cargo.toml @@ -32,7 +32,6 @@ end-keeps-inflight = [] toyos-untrusted = { path = "../toyos-untrusted" } [dev-dependencies] -# Already resolved in this workspace for `kernel-loom` and `toyos-sched/loom`. loom = "0.7" [lints.rust] diff --git a/toyos-transport/src/lib.rs b/toyos-transport/src/lib.rs index c81d65ebdbd..1fbfcbfecb1 100644 --- a/toyos-transport/src/lib.rs +++ b/toyos-transport/src/lib.rs @@ -53,8 +53,8 @@ impl Word for AtomicU32 { } } -/// The peer wrote what no peer of the protocol writes, or the region is not -/// one this session can use. The session is over; the variant is its name. +/// The peer wrote what no peer of the protocol writes. The session is over; +/// the variant is its name. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Violation { /// A producer's tail more than the ring holds past what was released. @@ -68,47 +68,4 @@ pub enum Violation { Run, /// A tag nothing is in flight under. Tag, - /// A place outside the words given. - Region, -} - -/// Where a ring's two cursors are, in words. The producer stores `tail`, the -/// consumer `head`. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct Cursors { - pub head: usize, - pub tail: usize, -} - -#[cfg(not(feature = "publish-relaxed"))] -const PUBLISH: Ordering = Ordering::Release; -#[cfg(feature = "publish-relaxed")] -const PUBLISH: Ordering = Ordering::Relaxed; - -/// Word `at` of an end's words, and the one index into them: every `at` an end -/// forms is below `N`, because its place was held to `N` when the end was -/// made and a ring position is masked below the depth. -#[allow(clippy::indexing_slicing)] -fn word(page: &[W; N], at: usize) -> &W { - &page[at] -} - -/// A distance the peer's cursor claims, believed only up to `cap`. -fn clamp(claimed: Untrusted, cap: u32, broken: Violation) -> Result { - let cap = if cfg!(feature = "no-clamp") { u32::MAX } else { cap }; - claimed.at_most(u64::from(cap)).ok().and_then(|n| u32::try_from(n).ok()).ok_or(broken) -} - -impl Cursors { - /// How far past `released` the producer's tail is: at most `cap`. - fn published(&self, page: &[W; N], released: u32, cap: u32) -> Result { - let tail = word(page, self.tail).load(Ordering::Acquire); - clamp(Untrusted::new(tail).map(|t| t.wrapping_sub(released)), cap, Violation::TailPastDepth) - } - - /// How far behind `published` the consumer's head is: at most `cap`. - fn unreleased(&self, page: &[W; N], published: u32, cap: u32) -> Result { - let head = word(page, self.head).load(Ordering::Acquire); - clamp(Untrusted::new(head).map(|h| published.wrapping_sub(h)), cap, Violation::HeadPastTail) - } } diff --git a/toyos-transport/src/queue.rs b/toyos-transport/src/queue.rs index e67bb0dcc1a..681b101b243 100644 --- a/toyos-transport/src/queue.rs +++ b/toyos-transport/src/queue.rs @@ -8,22 +8,73 @@ use core::sync::atomic::Ordering; -use crate::{word, Cursors, Untrusted, Violation, Word, PUBLISH}; +use crate::{Untrusted, Violation, Word}; -/// Where one queue is in a region, in words: its cursors, and its first entry. +#[cfg(not(feature = "publish-relaxed"))] +const PUBLISH: Ordering = Ordering::Release; +#[cfg(feature = "publish-relaxed")] +const PUBLISH: Ordering = Ordering::Relaxed; + +/// Word `at` of an end's words, and the one index into them: every `at` an end +/// forms is below `N`, because its [`Place`] is and a ring position is masked +/// below the depth. +#[allow(clippy::indexing_slicing)] +fn word(page: &[W; N], at: usize) -> &W { + &page[at] +} + +/// A distance the peer's cursor claims, believed only up to `cap`. +fn clamp(claimed: Untrusted, cap: u32, broken: Violation) -> Result { + let cap = if cfg!(feature = "no-clamp") { u32::MAX } else { cap }; + claimed.at_most(u64::from(cap)).ok().and_then(|n| u32::try_from(n).ok()).ok_or(broken) +} + +/// Where a queue of `E`-word entries, `D` deep, is among `N` words: the word +/// its consumer stores its head in, the word its producer stores its tail in, +/// and its first entry. +/// +/// ``` +/// use toyos_transport::Place; +/// const QUEUE: Place<2, 4, 10> = Place::new::<0, 1, 2>(); +/// ``` +/// +/// A place with a word at `N` or past it does not compile: +/// +/// ```compile_fail,E0080 +/// use toyos_transport::Place; +/// const QUEUE: Place<2, 4, 10> = Place::new::<0, 1, 3>(); +/// ``` +/// +/// ```compile_fail,E0080 +/// use toyos_transport::Place; +/// const QUEUE: Place<2, 4, 10> = Place::new::<10, 1, 2>(); +/// ``` +/// +/// ```compile_fail,E0080 +/// use toyos_transport::Place; +/// const QUEUE: Place<2, 4, 10> = Place::new::<0, 10, 2>(); +/// ``` #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct Place { - pub cursors: Cursors, - pub entries: usize, +pub struct Place { + head: usize, + tail: usize, + entries: usize, } -impl Place { +impl Place { + pub const fn new() -> Self { + const { + assert!(D.is_power_of_two() && E > 0, "a queue is a power of two deep, of entries of a word or more"); + assert!(usize::BITS >= u32::BITS, "a ring position is a u32"); + #[allow(clippy::as_conversions)] + let entries_end = ENTRIES + D as usize * E; + assert!(HEAD < N && TAIL < N && entries_end <= N, "a place names a word past its words"); + } + Self { head: HEAD, tail: TAIL, entries: ENTRIES } + } + /// The words of the entry at ring position `at`. - fn entry<'a, W, const E: usize, const D: u32, const N: usize>( - &self, - page: &'a [W; N], - at: u32, - ) -> impl Iterator { + fn entry<'a, W>(&self, page: &'a [W; N], at: u32) -> impl Iterator { // `usize` holds every `u32`, so the slot is exact. #[allow(clippy::as_conversions)] let slot = (at & D.wrapping_sub(1)) as usize; @@ -31,16 +82,16 @@ impl Place { (0..E).map(move |k| word(page, first.wrapping_add(k))) } - /// Every word the queue uses is below `N`. - fn check(&self) -> Result<(), Violation> { - const { assert!(D.is_power_of_two() && E > 0, "a queue is a power of two deep, of entries of a word or more") }; - const { assert!(usize::BITS >= u32::BITS, "a ring position is a u32") }; - let entries_end = - usize::try_from(D).ok().and_then(|d| d.checked_mul(E)).and_then(|words| words.checked_add(self.entries)); - match entries_end { - Some(end) if end <= N && self.cursors.head < N && self.cursors.tail < N => Ok(()), - _ => Err(Violation::Region), - } + /// How far past `released` the producer's tail is: at most `D`. + fn published(&self, page: &[W; N], released: u32) -> Result { + let tail = word(page, self.tail).load(Ordering::Acquire); + clamp(Untrusted::new(tail).map(|t| t.wrapping_sub(released)), D, Violation::TailPastDepth) + } + + /// How far behind `published` the consumer's head is: at most `D`. + fn unreleased(&self, page: &[W; N], published: u32) -> Result { + let head = word(page, self.head).load(Ordering::Acquire); + clamp(Untrusted::new(head).map(|h| published.wrapping_sub(h)), D, Violation::HeadPastTail) } } @@ -48,7 +99,7 @@ impl Place { /// region; every call is given the region's words. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct Producer { - place: Place, + place: Place, local: u32, published: u32, /// Entries that may be pushed before the head is looked at again. @@ -57,15 +108,14 @@ pub struct Producer { impl Producer { /// This end of the queue at `place`, its tail stored 0. - pub fn new(page: &[W; N], place: Place) -> Result { - place.check::()?; - word(page, place.cursors.tail).store(0, Ordering::Release); - Ok(Self { place, local: 0, published: 0, room: D }) + pub fn new(page: &[W; N], place: Place) -> Self { + word(page, place.tail).store(0, Ordering::Release); + Self { place, local: 0, published: 0, room: D } } /// How many entries may be pushed before the consumer frees more. pub fn space(&mut self, page: &[W; N]) -> Result { - let unreleased = self.place.cursors.unreleased(page, self.published, D)?; + let unreleased = self.place.unreleased(page, self.published)?; let pending = self.local.wrapping_sub(self.published); self.room = D.saturating_sub(pending.saturating_add(unreleased)); Ok(self.room) @@ -77,7 +127,7 @@ impl Producer { if self.room == 0 && self.space(page)? == 0 { return Ok(false); } - for (shared, value) in self.place.entry::(page, self.local).zip(words) { + for (shared, value) in self.place.entry(page, self.local).zip(words) { shared.store(value, Ordering::Relaxed); } self.local = self.local.wrapping_add(1); @@ -90,7 +140,7 @@ impl Producer { if self.published == self.local { return false; } - word(page, self.place.cursors.tail).store(self.local, PUBLISH); + word(page, self.place.tail).store(self.local, PUBLISH); self.published = self.local; true } @@ -100,7 +150,7 @@ impl Producer { /// and is given the region's words. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct Consumer { - place: Place, + place: Place, local: u32, released: u32, /// Entries published and not yet popped, as last seen. @@ -109,23 +159,22 @@ pub struct Consumer { impl Consumer { /// This end of the queue at `place`, its head stored 0. - pub fn new(page: &[W; N], place: Place) -> Result { - place.check::()?; - word(page, place.cursors.head).store(0, Ordering::Release); - Ok(Self { place, local: 0, released: 0, ready: 0 }) + pub fn new(page: &[W; N], place: Place) -> Self { + word(page, place.head).store(0, Ordering::Release); + Self { place, local: 0, released: 0, ready: 0 } } /// The next published entry, or `None` for none. pub fn pop(&mut self, page: &[W; N]) -> Result; E]>, Violation> { if self.ready == 0 { - let published = self.place.cursors.published(page, self.released, D)?; + let published = self.place.published(page, self.released)?; self.ready = published.saturating_sub(self.local.wrapping_sub(self.released)); if self.ready == 0 { return Ok(None); } } let mut words = [Untrusted::new(0); E]; - for (out, shared) in words.iter_mut().zip(self.place.entry::(page, self.local)) { + for (out, shared) in words.iter_mut().zip(self.place.entry(page, self.local)) { *out = Untrusted::new(shared.load(Ordering::Relaxed)); } self.local = self.local.wrapping_add(1); @@ -136,7 +185,7 @@ impl Consumer { /// Give every entry popped so far back to the producer. pub fn release(&mut self, page: &[W; N]) { if self.released != self.local { - word(page, self.place.cursors.head).store(self.local, Ordering::Release); + word(page, self.place.head).store(self.local, Ordering::Release); self.released = self.local; } } @@ -148,15 +197,15 @@ mod tests { use core::sync::atomic::AtomicU32; const D: u32 = 8; - const PLACE: Place = Place { cursors: Cursors { head: 0, tail: 16 }, entries: 32 }; const WORDS: usize = 32 + 2 * D as usize; + const PLACE: Place<2, D, WORDS> = Place::new::<0, 16, 32>(); fn page() -> [AtomicU32; WORDS] { core::array::from_fn(|_| AtomicU32::new(0)) } fn ends(page: &[AtomicU32; WORDS]) -> (Producer<2, D, WORDS>, Consumer<2, D, WORDS>) { - (Producer::new(page, PLACE).unwrap(), Consumer::new(page, PLACE).unwrap()) + (Producer::new(page, PLACE), Consumer::new(page, PLACE)) } fn plain(words: Option<[Untrusted; 2]>) -> Option<[u32; 2]> { @@ -257,17 +306,4 @@ mod tests { }; assert_eq!((pushed, refused), (D, Violation::HeadPastTail)); } - - #[test] - fn a_place_outside_the_words_is_refused() { - let page = page(); - for place in [ - Place { entries: 33, ..PLACE }, - Place { cursors: Cursors { head: WORDS, tail: 16 }, ..PLACE }, - Place { cursors: Cursors { head: 0, tail: WORDS }, ..PLACE }, - ] { - assert_eq!(Producer::<2, D, WORDS>::new(&page, place).err(), Some(Violation::Region)); - assert_eq!(Consumer::<2, D, WORDS>::new(&page, place).err(), Some(Violation::Region)); - } - } } diff --git a/toyos-transport/tests/loom.rs b/toyos-transport/tests/loom.rs index a529456436e..8051dc92a57 100644 --- a/toyos-transport/tests/loom.rs +++ b/toyos-transport/tests/loom.rs @@ -13,7 +13,7 @@ use core::sync::atomic::Ordering; use loom::sync::atomic::AtomicU32; use loom::sync::Arc; -use toyos_transport::{Consumer, Cursors, Place, Producer, Untrusted, Violation, Word}; +use toyos_transport::{Consumer, Place, Producer, Untrusted, Violation, Word}; /// A loom atomic as a region word: the trait is this crate's and the type is /// loom's, so the two meet through a wrapper. @@ -30,15 +30,15 @@ impl Word for Shared { const E: usize = 2; const D: u32 = 2; -const PLACE: Place = Place { cursors: Cursors { head: 0, tail: 1 }, entries: 2 }; const WORDS: usize = 2 + E * D as usize; +const PLACE: Place = Place::new::<0, 1, 2>(); fn page() -> Arc<[Shared; WORDS]> { Arc::new(core::array::from_fn(|_| Shared(AtomicU32::new(0)))) } fn ends(page: &[Shared; WORDS]) -> (Producer, Consumer) { - (Producer::new(page, PLACE).unwrap(), Consumer::new(page, PLACE).unwrap()) + (Producer::new(page, PLACE), Consumer::new(page, PLACE)) } fn entry(n: u32) -> [u32; E] { diff --git a/userland/blockd/src/main.rs b/userland/blockd/src/main.rs index 8bfaea41f6b..ad65b88890a 100644 --- a/userland/blockd/src/main.rs +++ b/userland/blockd/src/main.rs @@ -245,7 +245,7 @@ impl Service { fn admit(&mut self, opening: Opening, conn: Connection) -> u64 { let id = self.next_id; self.next_id += 1; - let rings = layout::server(opening.region.words()).expect("blockd: the region holds every ring word"); + let rings = layout::server(opening.region.words()); self.sessions.insert( id, Served { @@ -333,12 +333,12 @@ impl Service { match s.state.take(words) { Taken::Answer(c) => Self::post(s, c), Taken::Issue(req) => { - let write = matches!(req.op, Op::Write(_)); + let write = matches!(req.op, Op::Write { .. }); let owner = Owner::Session { session: id, tag: req.tag, write }; match req.op { - Op::Read(run) | Op::Write(run) => { + Op::Read { run, lba } | Op::Write { run, lba } => { let at = s.device_addr + run.span().offset as u64; - let block = s.state.first() + req.lba; + let block = s.state.first() + lba; self.ctrl.submit_io(write, block, run.count(), at, owner); } Op::Flush if self.ctrl.vwc => self.ctrl.submit_flush(owner), diff --git a/userland/blockd/src/session.rs b/userland/blockd/src/session.rs index 16c9fae0e76..a5b83eed6a5 100644 --- a/userland/blockd/src/session.rs +++ b/userland/blockd/src/session.rs @@ -135,7 +135,7 @@ impl Session { /// `names` calls `service`. pub fn open(names: Namespace, service: &str, guid: [u8; wire::GUID_BYTES]) -> Result { let region = Region::create().map_err(Error::Kernel)?; - let rings = layout::client(region.words()).expect("blockd: the region holds every ring word"); + let rings = layout::client(region.words()); let (conn, opened) = handshake(&names, service, guid, ®ion)?; let mut client = Client::new(); client.session_started(); @@ -187,7 +187,7 @@ impl Session { let run = self.arena.alloc(blocks).ok_or(Unsent::ArenaFull)?; self.region.arena(&run).copy_in(0, data); let ticket = self.ticket(); - self.client.submit(ticket, Op::Write(run), lba); + self.client.submit(ticket, Op::Write { run, lba }); self.pending.insert(ticket, Pending::Write); Ok(ticket) } @@ -200,7 +200,7 @@ impl Session { } let run = self.arena.alloc(blocks).ok_or(Unsent::ArenaFull)?; let ticket = self.ticket(); - self.client.submit(ticket, Op::Read(run), lba); + self.client.submit(ticket, Op::Read { run, lba }); self.pending.insert(ticket, Pending::Read { run }); Ok(ticket) } @@ -211,7 +211,7 @@ impl Session { return Err(Unsent::Ended); } let ticket = self.ticket(); - self.client.submit(ticket, Op::Flush, 0); + self.client.submit(ticket, Op::Flush); self.pending.insert(ticket, Pending::Flush); Ok(ticket) } @@ -356,7 +356,7 @@ impl Session { assert!(self.conn.is_none(), "blockd: reconnect while a session is open"); // Before the region goes to the new server: it must find this end's // two indices at zero, as it will set its own. - self.rings = layout::client(self.region.words()).expect("blockd: the region holds every ring word"); + self.rings = layout::client(self.region.words()); let (conn, opened) = handshake(&self.names, &self.service, self.guid, &self.region)?; if opened != self.opened { return Err(Error::Protocol); From 87bef932e3b7888e8184bc8bef50f20e8fd8ab2b Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 22:19:22 +0200 Subject: [PATCH 7/8] issues: a sysroot cloned while stage2 lacks cargo is finished and refused forever Found running this branch's guest gates: sysroot 5dc157f7fac727be, the key main's sources name, was cloned from the primary's stage2 while its bin/ had no cargo, then marked finished. `ensure` refuses it on every use and nothing rebuilds it, so no guest gate runs in a worktree naming that key. Co-Authored-By: Claude Opus 5.5 --- ...s-cargo-is-finished-and-refused-forever.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 issues/build/a-sysroot-cloned-while-stage2-lacks-cargo-is-finished-and-refused-forever.md diff --git a/issues/build/a-sysroot-cloned-while-stage2-lacks-cargo-is-finished-and-refused-forever.md b/issues/build/a-sysroot-cloned-while-stage2-lacks-cargo-is-finished-and-refused-forever.md new file mode 100644 index 00000000000..0ab1e5919d2 --- /dev/null +++ b/issues/build/a-sysroot-cloned-while-stage2-lacks-cargo-is-finished-and-refused-forever.md @@ -0,0 +1,30 @@ +--- +status: open +kind: tooling +opened: 2026-09-27 +--- + +# A sysroot cloned while the compiler's `bin/` lacks `cargo` is finished, and refused by every build that names it + +`src/sysroot.rs`'s `build` clones `compiler.stage2` into the partial sysroot, +places std and libc, writes `SOURCES` and renames it into place: finished, and +never written again. `ensure` runs `toolchain::assert_toolchain_is_honest` on +it only afterwards, on every use. A clone taken while the stage2 `bin/` has no +`cargo` is therefore a sysroot that every build naming its key refuses, and +that nothing rebuilds. + +Seen on 2026-09-27. `rust/build/sysroots/5dc157f7fac727be/` has `SOURCES` at +22:03:59, naming the fork `…/toyos-nokthread/rust`. Its `bin/` holds `rustc` +and `rustdoc` and no `cargo`. The primary's +`rust/build/aarch64-apple-darwin/stage2/` was re-made at 22:00, and its +`bin/cargo` link dates from 22:08. + +That key comes from `origin/main` c5518949's sources, so every worktree at main +with no ABI change names it. In `toyos-transport`, `cargo test --test +toyos-build -- --nightly blockd_survives_its_death` panicked at +`tests/toyos.rs:2972` before any guest ran: every C corpus case "no longer +links … the toyos toolchain at …/sysroots/5dc157f7fac727be/bin is missing +cargo". + +**Exit condition**: the honesty check runs on the partial sysroot before +`SOURCES` is written, so a sysroot without `cargo` is never finished. From 1a3fc0194036c85c66aee6ca1ecbde449ab10121 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 27 Sep 2026 22:39:18 +0200 Subject: [PATCH 8/8] issues: the cargo-less sysroot is the toolchain fix's defect, filed there Co-Authored-By: Claude Opus 5.5 --- ...s-cargo-is-finished-and-refused-forever.md | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 issues/build/a-sysroot-cloned-while-stage2-lacks-cargo-is-finished-and-refused-forever.md diff --git a/issues/build/a-sysroot-cloned-while-stage2-lacks-cargo-is-finished-and-refused-forever.md b/issues/build/a-sysroot-cloned-while-stage2-lacks-cargo-is-finished-and-refused-forever.md deleted file mode 100644 index 0ab1e5919d2..00000000000 --- a/issues/build/a-sysroot-cloned-while-stage2-lacks-cargo-is-finished-and-refused-forever.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-09-27 ---- - -# A sysroot cloned while the compiler's `bin/` lacks `cargo` is finished, and refused by every build that names it - -`src/sysroot.rs`'s `build` clones `compiler.stage2` into the partial sysroot, -places std and libc, writes `SOURCES` and renames it into place: finished, and -never written again. `ensure` runs `toolchain::assert_toolchain_is_honest` on -it only afterwards, on every use. A clone taken while the stage2 `bin/` has no -`cargo` is therefore a sysroot that every build naming its key refuses, and -that nothing rebuilds. - -Seen on 2026-09-27. `rust/build/sysroots/5dc157f7fac727be/` has `SOURCES` at -22:03:59, naming the fork `…/toyos-nokthread/rust`. Its `bin/` holds `rustc` -and `rustdoc` and no `cargo`. The primary's -`rust/build/aarch64-apple-darwin/stage2/` was re-made at 22:00, and its -`bin/cargo` link dates from 22:08. - -That key comes from `origin/main` c5518949's sources, so every worktree at main -with no ABI change names it. In `toyos-transport`, `cargo test --test -toyos-build -- --nightly blockd_survives_its_death` panicked at -`tests/toyos.rs:2972` before any guest ran: every C corpus case "no longer -links … the toyos toolchain at …/sysroots/5dc157f7fac727be/bin is missing -cargo". - -**Exit condition**: the honesty check runs on the partial sysroot before -`SOURCES` is written, so a sysroot without `cargo` is never finished.