diff --git a/crates/ffi/src/futex.rs b/crates/ffi/src/futex.rs index 6e04454..43910dd 100644 --- a/crates/ffi/src/futex.rs +++ b/crates/ffi/src/futex.rs @@ -61,7 +61,8 @@ pub extern "C" fn kore_futex_available() -> u32 { /// all waits/wakes. All concurrent accesses must be compatible 32-bit atomic /// operations (C atomic builtins or Zig atomics); never mix ordinary accesses /// or differently sized atomics. Managed-thread waits are cancellation points: -/// they may exit without stack unwinding, as specified by kore_thread_testcancel. +/// they may exit without stack unwinding, as specified by kore_thread_testcancel, +/// and a cancellation requested while the thread waits wakes it to act. /// Publish the condition with a sequentially consistent atomic store BEFORE waking. /// No address-space sharing is implied. #[unsafe(no_mangle)] @@ -77,32 +78,51 @@ pub unsafe extern "C" fn kore_futex_wait( if !crate::thread_local::cancellable() { return futex::wait(word, expected, timeout).into(); } - // A bounded native park makes cancellation observable even without a peer - // kick capability. Recompute against one deadline; slices never restart it. unsafe { crate::kore_thread_testcancel() }; - let start = match kore::time::uptime() { - Ok(time) => time.as_nanos(), - Err(_) => return KoreFutexOutcome::Unsupported, + let outcome = match crate::thread::slice() { + // A cancellation's wake reaches this thread (kore_thread_cancel), and + // the wait re-reads the request once it is registered: it parks until + // a wake, the caller's deadline or that request, and never polls. + None => futex::wait_unless(word, expected, timeout, crate::thread_local::cancel_pending), + Some(slice) => unsafe { sliced(word, expected, timeout_ns, slice) }, }; - let deadline = timeout.map(|_| start.saturating_add(timeout_ns)); + unsafe { crate::kore_thread_testcancel() }; + outcome.into() +} + +/// A cancellable wait on a thread the controller cannot wake: no wake would +/// announce a cancellation, so it parks one `slice` at a time and acts on a +/// request between slices, against one deadline that slices never restart. +/// # Safety +/// A managed thread's cancellation point, as kore_futex_wait. +unsafe fn sliced( + word: &AtomicU32, + expected: u32, + timeout_ns: u64, + slice: Duration, +) -> futex::WaitOutcome { + let Ok(start) = kore::time::uptime() else { + return futex::WaitOutcome::Unsupported; + }; + let deadline = + (timeout_ns != KORE_FUTEX_INFINITE).then(|| start.as_nanos().saturating_add(timeout_ns)); + let slice = u64::try_from(slice.as_nanos()).unwrap_or(u64::MAX); loop { unsafe { crate::kore_thread_testcancel() }; - let now = match kore::time::uptime() { - Ok(time) => time.as_nanos(), - Err(_) => return KoreFutexOutcome::Unsupported, + let Ok(now) = kore::time::uptime() else { + return futex::WaitOutcome::Unsupported; }; - let slice = crate::thread_local::CANCEL_SLICE_NS; - let slice = deadline.map_or(slice, |end| end.saturating_sub(now).min(slice)); - let outcome = futex::wait(word, expected, Some(Duration::from_nanos(slice))); + let bound = deadline.map_or(slice, |end| end.saturating_sub(now.as_nanos()).min(slice)); + let outcome = futex::wait(word, expected, Some(Duration::from_nanos(bound))); unsafe { crate::kore_thread_testcancel() }; if outcome != futex::WaitOutcome::TimedOut { - return outcome.into(); + return outcome; } if let Some(end) = deadline { match kore::time::uptime() { Ok(time) if time.as_nanos() < end => {} - Ok(_) => return KoreFutexOutcome::TimedOut, - Err(_) => return KoreFutexOutcome::Unsupported, + Ok(_) => return futex::WaitOutcome::TimedOut, + Err(_) => return futex::WaitOutcome::Unsupported, } } } diff --git a/crates/ffi/src/thread.rs b/crates/ffi/src/thread.rs index 14e7d1e..c6b474b 100644 --- a/crates/ffi/src/thread.rs +++ b/crates/ffi/src/thread.rs @@ -54,6 +54,10 @@ pub(crate) struct Control { outcome: AtomicU32, retired: AtomicU32, pub(crate) cancel: AtomicBool, + /// The native band the thread runs on, plus one; zero until it starts + /// (`futex::current_thread`). A canceller reads it after storing + /// `cancel`, to wake the thread out of a wait at a cancellation point. + band: AtomicU32, } impl Control { pub(crate) fn new() -> Self { @@ -62,13 +66,32 @@ impl Control { outcome: AtomicU32::new(0), retired: AtomicU32::new(0), cancel: AtomicBool::new(false), + band: AtomicU32::new(0), } } + /// Record the calling thread's band, before its callback can reach a + /// cancellation point. `SeqCst`, against the canceller's store of `cancel` + /// and then this load: a canceller that reads zero stored `cancel` before + /// the thread's first wait reads it. + fn start(&self) { + let band = kore::futex::current_thread().map_or(0, |band| band as u32 + 1); + self.band.store(band, Ordering::SeqCst); + } + fn band(&self) -> Option { + let band = self.band.load(Ordering::SeqCst); + band.checked_sub(1).map(|band| band as usize) + } pub(crate) fn finish(&self, value: *mut c_void, outcome: u32) { self.result.store(value, Ordering::Relaxed); self.outcome.store(outcome, Ordering::Release); } } +/// How long one wait of the calling thread may park ([`Controller::slice`]): +/// unbounded where the controller can wake it, and therefore where a +/// cancellation's wake reaches it; one bounded slice where nothing could. +pub(crate) fn slice() -> Option { + THREADS.slice() +} fn status(error: Error) -> i32 { match error { Error::Unavailable | Error::Full => KORE_THREAD_EAGAIN, @@ -95,6 +118,7 @@ fn invoke(argument: usize) -> usize { call_context(context) } fn call_context(context: &StartContext) -> usize { + context.control.start(); // SAFETY: forwarded foreign callback and argument lifetime contract. crate::thread_local::enter(&context.control); let value = unsafe { (context.call)(context.argument) }; @@ -217,12 +241,17 @@ pub unsafe extern "C" fn kore_thread_join( if control.retired.load(Ordering::SeqCst) != 0 { break; } - // A cancellation request is a flag, not a wake: a joiner that can be - // cancelled parks one slice at a time, as a cancellable futex wait - // does. Any other joiner parks until the controller retires the target. - let slice = crate::thread_local::cancellable() - .then(|| core::time::Duration::from_nanos(crate::thread_local::CANCEL_SLICE_NS)); - if kore::futex::wait(&control.retired, 0, slice) == kore::futex::WaitOutcome::Unsupported + // Park until the controller wakes `retired`, or this joiner's own + // cancellation is requested: its canceller stores the request, then + // wakes this thread (kore_thread_cancel), and the wait re-reads the + // request once it is registered. Only a joiner the controller cannot + // wake parks in bounded slices; nothing else would end its park. + if kore::futex::wait_unless( + &control.retired, + 0, + slice(), + crate::thread_local::cancel_pending, + ) == kore::futex::WaitOutcome::Unsupported // A native worker with no doorbell takes the deadline park, which // needs no wake capability at all. && !kore::runtime::pace::cede() @@ -279,7 +308,12 @@ pub unsafe extern "C" fn kore_thread_detach(thread: *mut KoreThread) -> i32 { } /// Request deferred cancellation. The target acts at testcancel, join or foreign -/// futex wait. A disabled target retains the request. No asynchronous kill occurs. +/// futex wait; a target waiting at one is woken to act. A disabled target +/// retains the request. No asynchronous kill occurs. EAGAIN when no wake the +/// calling thread sends can reach the target, because it neither started the +/// controller nor was created through it (create refuses it too). Then no +/// request is recorded, since a target parked at a cancellation point would +/// never see it. /// # Safety /// thread is a live handle. Borrowing cancel calls may run concurrently with one /// another, but must finish before a consuming join/detach can free the handle. @@ -288,12 +322,31 @@ pub unsafe extern "C" fn kore_thread_cancel(thread: *mut KoreThread) -> i32 { let Some(ticket) = NonNull::new(thread.cast::()) else { return KORE_THREAD_EINVAL; }; - let control = unsafe { &ticket.as_ref().control }; + // SAFETY: the caller supplies a live handle, borrowed for this call. + request_cancel(unsafe { &ticket.as_ref().control }) +} + +/// kore_thread_cancel's request against the target's shared control. +fn request_cancel(control: &Control) -> i32 { if control.outcome.load(Ordering::Acquire) != 0 || control.retired.load(Ordering::Acquire) != 0 { return KORE_THREAD_ESRCH; } - control.cancel.store(true, Ordering::Release); + // The request is a flag its wake announces: only the controller holds a + // send into a managed thread's doorbell, so the wake is relayed through + // it, and a thread that cannot relay through it is refused here rather + // than leaving a request behind a park nothing ends. + if !THREADS.wakes_threads() { + return KORE_THREAD_EAGAIN; + } + // Store, then wake (`futex::wake_thread`'s order). The wake reads the + // target's registration, and the target's wait reads the request once it + // is registered, so one of the two sees the other. A target that has not + // yet recorded its band reads the request at its first cancellation point. + control.cancel.store(true, Ordering::SeqCst); + if let Some(band) = control.band() { + let _ = kore::futex::wake_thread(band); + } KORE_THREAD_OK } @@ -322,6 +375,22 @@ mod tests { assert_eq!(Arc::strong_count(&control), 1); } #[test] + fn a_cancel_no_wake_can_announce_is_refused_and_never_recorded() { + // Off metal the controller never starts, so no thread's wake reaches + // its threads: the request is refused, and nothing a parked target + // would wait on unwoken is left behind. An ended target is gone first. + let control = Control::new(); + control.start(); + assert_eq!(control.band(), None, "off metal a thread has no band"); + assert_eq!(request_cancel(&control), KORE_THREAD_EAGAIN); + assert!(!control.cancel.load(Ordering::SeqCst)); + control.finish(ptr::null_mut(), 1); + assert_eq!(request_cancel(&control), KORE_THREAD_ESRCH); + assert!(!control.cancel.load(Ordering::SeqCst)); + // Every wait off metal is one the controller cannot end early. + assert!(slice().is_some()); + } + #[test] fn host_refusal_preserves_output_and_never_calls_foreign_code() { unsafe extern "C" fn never(_: *mut c_void) -> *mut c_void { panic!("host ran native callback") diff --git a/crates/ffi/src/thread_local.rs b/crates/ffi/src/thread_local.rs index 64bd577..21484c3 100644 --- a/crates/ffi/src/thread_local.rs +++ b/crates/ffi/src/thread_local.rs @@ -15,12 +15,6 @@ pub const KORE_THREAD_CANCEL_ENABLE: i32 = 0; pub const KORE_THREAD_CANCEL_DISABLE: i32 = 1; pub const KORE_THREAD_CANCEL_DEFERRED: i32 = 0; pub const KORE_THREAD_CANCEL_ASYNCHRONOUS: i32 = 1; -/// How long one park lasts at a cancellation point that waits: a request is -/// a flag no wake carries, so a cancellable wait re-checks it this often. -/// One millisecond is the cadence the native runtime parks an unreachable -/// waiter at (`kore-rt`'s `pace::CEDE_NS`): a cancellation is acted on within -/// one, and an idle cancellable waiter wakes no more often than that. -pub(crate) const CANCEL_SLICE_NS: u64 = 1_000_000; pub type KoreThreadKey = u64; pub type KoreThreadDestructor = Option; @@ -85,6 +79,16 @@ pub(crate) fn leave() { pub(crate) fn cancellable() -> bool { !CURRENT.get().is_null() && !DISABLED.get() && !FINISHING.get() } +/// An enabled cancellation request is pending for this managed thread: what +/// testcancel acts on, and the stop a cancellation point's wait re-reads once +/// it is registered (`futex::wait_unless`). `SeqCst`, as the canceller's store +/// is: the canceller stores, then reads this thread's registration before it +/// wakes it, so one of the two sees the other. +pub(crate) fn cancel_pending() -> bool { + // SAFETY: CURRENT is non-null while cancellable, and points at this + // managed thread's Control until `leave`. + cancellable() && unsafe { &*CURRENT.get() }.cancel.load(Ordering::SeqCst) +} fn active() -> bool { !CURRENT.get().is_null() } @@ -368,7 +372,7 @@ pub unsafe extern "C" fn kore_thread_setcanceltype(kind: i32, old_type: *mut i32 /// never cross live Rust RAII guards or executor callbacks at a cancellation point. #[unsafe(no_mangle)] pub unsafe extern "C" fn kore_thread_testcancel() { - if cancellable() && unsafe { &*CURRENT.get() }.cancel.load(Ordering::Acquire) { + if cancel_pending() { unsafe { kore_thread_exit(kore_thread_canceled()); } diff --git a/docs/provenance.md b/docs/provenance.md index 737162c..7f0bc5f 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -351,3 +351,135 @@ C program (`docs/languages.md`). It is the first libkore package a C program on Kore links without a Rust host. Its shared object is a distribution's one-line `cdylib` over it. `tools/check-program.py` links that object for the three Kore ISAs and checks its shape and its exact exported ABI. + +## A cancellation is a wake + +The export records Kore `a4ab7234e2fe60b43a1df42f12be8f636255fc7a`, one commit +past MicroPerceptron/kore#1494's branch head `76fddb9f`. Two of the six +commits since `0018956a` reach the closure, which keeps its 26 packages and +382 files: + +- `kore-rt`'s futex gains a wait another thread can end, and the wake that + ends it (Kore `a4ab7234`). `futex::wait_unless` is `wait` with a caller + condition read once the waiter is registered, as the word is, and always + parks on the doorbell. `futex::wake_thread` kicks one thread's doorbell + rather than a word's waiters, directly or relayed through the caller's hub, + and reads the target's registration before it kicks. A condition stored + with `SeqCst` before that wake is therefore never missed, and no slice is + needed to notice it. Arm, register and check are one function that every + doorbell wait shares, and the host suite drives it with the arm injected to + pin that order. `futex::current_thread` is public: the band a thread + records for its wakers. +- Its thread controller exposes the rule its own joins used: + `Controller::slice` is unbounded where the controller can wake the calling + thread, and `UNREACHED_SLICE` where nothing could. `Controller::wakes_threads` + says whether a wake the calling thread sends reaches the threads it founded. +- The services' `boot.toml` digests are #1494's reseal over the parking + controller (`76fddb9f`). + +No exported manifest changed, so `native/Cargo.lock` did not. + +`kore-ffi`'s cancellation points no longer poll. A managed thread records its +band when it starts; `kore_thread_cancel` stores the request and wakes that +band; `kore_thread_join` and a managed thread's `kore_futex_wait` wait with +`wait_unless`, with no timeout of their own. An idle cancellable waiter now +sleeps until something happens rather than waking every millisecond. Only a +thread the controller cannot wake keeps the bounded slice, now the +controller's `UNREACHED_SLICE`. A canceller whose wake cannot reach the +controller's threads is refused with EAGAIN, as create refuses it, and records +nothing. A host test drives `request_cancel` to that refusal. The metal +evidence is Kore's `keytest-tcc-x86`, whose libkore tour cancels a thread +parked in `pthread_join` and one parked in `kore_futex_wait` with no timeout. + +The export then moved to Kore `40de25825eb1ea7816f7e7df86e116b3ad20d1fc`, +which closes a child's founding window. The controller sets a child's reach +bit only after `spawn_with_arg` returns, and the child ran its callback at +once. Its first wait could therefore start before the controller had +recorded that it can kick the child. A cancellable `kore_futex_wait` in that +window took the unreached slice and kept it until its first real wake. +`child_entry` now settles on its record until the founding is published, +which follows the reach bit, so a thread the controller founds with a +doorbell is reached from its first wait. A host test holds a founding back +and checks that the callback waits for it. The closure keeps its 26 packages +and 382 files. Besides `thread_controller.rs`, it carries the +`services/swap/boot.toml` digest of Kore's reseal over the cancellation wake +(`bfecfd17`). + +The export then moved to Kore `618772900e088b8b45ebafa8101d066edf4ffb3b`, +the merge of MicroPerceptron/kore#1494's newer branch into this change. That +branch brings three changes, which its own exports carried here up to Kore +`2d4998ba`: + +- threads that found take a founder's arena; +- a founding's staged space waits for the reap; +- Kore's C library (`kore-c-shim`). + +The closure is now 27 packages and 401 files. Against that branch's export, +only kore-rt's `futex.rs` and `thread_controller.rs` and the record differ. +The two sides' controller changes compose: the controller and its children +are founded with `spawn_founder_with_arg`, and a child still waits for its +founding before its callback runs. + +It then moved to Kore `4a56f8279c79fd4d65918e581fe382f0775dfb51`, the merge +of #1494's next round: kore-rt names the link window's free span once, and C +programs gain descriptors and mmap. The closure stays at 27 packages and 401 +files. Against that branch's export at `10c7b400`, only kore-rt's `futex.rs` +and `thread_controller.rs`, the record, and five services' `boot.toml` +digests differ. The digests come from #1494's own reseal (`f2644e5b`), which +followed its export. + +It then moved to Kore `eb59823a29cc9b0cfe0ea27a386c3fac266534ef`, the merge +of #1494's merge of Kore main `f2dc8309`. Main reaches the closure in four +places: + +- the seat clipboard and Cmd chords (#1469): kore-abi's insert into a held + input line and a window's clipboard chunks, and kore-window's clipboard, + key split and shortcuts; +- Files' live folder and sidebar (#1450, #1456): a DagFS watch plane in the + `dagfs` crate and service, and kore-rt's owed service wakes + (`SystemService::take_owed_wakes`), which the serve host delivers after + each unit of work; +- DagFS's cwd anchor and `MatchView`, `.` and `..` in `resolve`, and a dead + view reaped by asking the kernel; +- five services' `boot.toml` digests, from #1494's reseal over the merge + (`e6b108d1`). + +The closure is now 27 packages and 406 files. Against Kore `e6b108d1`, only +kore-rt's `futex.rs` and `thread_controller.rs` differ. No exported manifest +changed, so `native/Cargo.lock` did not. + +It then moved to Kore `5c09f155b03682e95f24f924cdd65188fc9ebc60`, the merge +of #1494's next round into this change. That round lets the kernel observe a +PCI function's live state and its BARs' shapes, and puts the name a program +was run by in its process page. libkore#32 exported it first, at Kore +`dfece644`. The closure is now 27 packages and 407 files. Against Kore +`4b236fbe`, only kore-rt's `futex.rs` and `thread_controller.rs` differ. +Against libkore#32's export, five services' `boot.toml` digests differ as +well, from #1494's reseals that followed it. No exported manifest changed, +so `native/Cargo.lock` did not. + +It then moved to Kore `7c0be6e84ac2e8ee67591b44c0f42d46ab596f83`, the merge +of #1494's next round into this change. That round has the kernel offer a +function to drivers only on positive evidence and name each claim's holder, +which kore-abi's device state and its new device classes carry. The closure +is now 27 packages and 408 files. Against Kore `fe9603e3`, only kore-rt's +`futex.rs` and `thread_controller.rs` differ, and `services/swap/boot.toml`: +swap links kore-rt's futex, so its digests are this change's reseal over +the ioctl round (`b04325e9`). No exported manifest changed, so +`native/Cargo.lock` did not. + +It then moved to Kore `a07b4eae68b45ffd740e31a9cc69e708b21ad467`, the merge +of #1494's virtual-device round into this change, once libkore#32 had merged +into `alpha` with its export at Kore `cac17a02`. The closure is now 31 +packages and 442 files. Against Kore `625060b1`, only kore-rt's `futex.rs` +and `thread_controller.rs` differ. Against `alpha`'s export, eleven more +files differ, all from #1494's commits after `cac17a02`: + +- the C driver's protocol, rendered from the Rust that defines it + (`vdev-accel`'s `kore/vdev_accel.h`, its device and the header's test); +- kore-window's pointer acceleration, from main `bb1fe70d`; +- a command entering a task grant through one edge (the DagFS service's + authority, engine and service); +- the `boot.toml` digests of DagFS, swap and vdev, from #1494's reseals. + +No exported manifest changed, so `native/Cargo.lock` did not. diff --git a/docs/runtime.md b/docs/runtime.md index fd1d88b..0c80f91 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -131,10 +131,13 @@ names on Kore targets; it never interposes host pthreads. The first create starts one process-lifetime controller. It owns all native join capabilities; portable handles contain identifiers, never arena slot authority. -Calls use bounded shared records and paced parks, including a 1 ms controller -scan while idle. They must run on native threads, not executor callbacks that -owe other futures progress. Capacity depends on native bands and the controller's -capability arena; ordinary slot exhaustion is refused before spending a band. +Calls use bounded shared records and futex parks: the controller parks until a +request, a thread's end or a relayed wake moves its word, and a waiting caller +parks until the controller wakes it. Only a thread the controller cannot wake +parks in bounded 1 ms slices. Calls must run on native threads, not executor +callbacks that owe other futures progress. Capacity depends on native bands and +the controller's capability arena; ordinary slot exhaustion is refused before +spending a band. Create uses the installed process allocator for its handle and callback context. On error the output is unchanged and the argument is not retained. On success, @@ -193,8 +196,22 @@ this does not implement dynamic DSO unloading or Rust/C++ stack unwinding. `kore_thread_testcancel`, foreign join and foreign futex waits. Disabled targets retain pending requests. `setcancelstate` enables/disables delivery; `setcanceltype` accepts deferred and refuses asynchronous with ENOTSUP, preserving -output on failure. Cancellation-aware waits use bounded 1 ms native parks and -one monotonic deadline. A canceled joiner leaves its target joinable. +output on failure. A canceled joiner leaves its target joinable. + +A request wakes a target parked at a cancellation point; no cancellation-aware +wait polls for one. The canceller stores the request, then wakes the target's +thread (`futex::wake_thread`), relayed through the controller, the one owner of +a send into each managed thread's doorbell. The target's wait reads the request +again once it is registered (`futex::wait_unless`), and the wake reads that +registration, so one of the two always sees the other. Only a target the +controller cannot wake waits in bounded 1 ms slices against one monotonic +deadline. A thread the controller founds with a doorbell is never one: it +runs its callback only once its founding is published, after the controller +can kick it. A thread whose wake cannot reach the controller's threads, because +it neither started the controller nor was created through it, is refused with +EAGAIN and records no request, as create refuses it: Kore's capability model +grants no wake without a send, and a request a parked target never sees would +be a silent loss. On explicit `kore_thread_exit` or cancellation, caller-owned cleanup nodes run LIFO, then compiler TLS destructors, then pthread key destructors. Normal callback @@ -237,11 +254,14 @@ there is no second wait registry. Use `wait_async` in Rust executor tasks. C exposes `kore_futex_available`, `kore_futex_wait`, `kore_futex_wake` and `kore_futex_wake_all` in ``; Zig exposes `kore.futex.*`. The foreign wait blocks a native thread and must not run inside an executor callback. -For managed threads it is also a deferred cancellation point. Independent -controller-created threads are not endowed with doorbells, so their native futex -waits report `Unsupported`; this adapter does not create a peer wake mesh. -Foreign join uses the controller's deadline-park helper and works on these -threads. Cancellation is checked even when a foreign futex wait is unsupported. +For managed threads it is also a deferred cancellation point, which a +cancellation request wakes. Controller-created threads are founded with +doorbells when the thread that started the controller holds a channel-space +grant, and park; a wake one of them cannot deliver directly is relayed through +the controller, which holds a send into each. Without the grant they have none: +their native futex waits report `Unsupported`, and foreign join takes the +deadline park, which needs no wake. Cancellation is checked even when a foreign +futex wait is unsupported. Timeouts are relative nanoseconds; zero is a deadline of zero and `KORE_FUTEX_INFINITE` requests an unbounded wait. Retry, spurious wake, timeout, unsupported context and invalid pointer shape remain distinct outcomes. diff --git a/extern/c/include/kore.h b/extern/c/include/kore.h index a9089a0..ba27a6d 100644 --- a/extern/c/include/kore.h +++ b/extern/c/include/kore.h @@ -690,7 +690,8 @@ uint32_t kore_futex_available(void); * all waits/wakes. All concurrent accesses must be compatible 32-bit atomic * operations (C atomic builtins or Zig atomics); never mix ordinary accesses * or differently sized atomics. Managed-thread waits are cancellation points: - * they may exit without stack unwinding, as specified by kore_thread_testcancel. + * they may exit without stack unwinding, as specified by kore_thread_testcancel, + * and a cancellation requested while the thread waits wakes it to act. * Publish the condition with a sequentially consistent atomic store BEFORE waking. * No address-space sharing is implied. */ @@ -935,7 +936,12 @@ int32_t kore_thread_detach(struct KoreThread *thread); /** * Request deferred cancellation. The target acts at testcancel, join or foreign - * futex wait. A disabled target retains the request. No asynchronous kill occurs. + * futex wait; a target waiting at one is woken to act. A disabled target + * retains the request. No asynchronous kill occurs. EAGAIN when no wake the + * calling thread sends can reach the target, because it neither started the + * controller nor was created through it (create refuses it too). Then no + * request is recorded, since a target parked at a cancellation point would + * never see it. * # Safety * thread is a live handle. Borrowing cancel calls may run concurrently with one * another, but must finish before a consuming join/detach can free the handle. diff --git a/native/crates/rt/src/futex.rs b/native/crates/rt/src/futex.rs index bebb188..73d9168 100644 --- a/native/crates/rt/src/futex.rs +++ b/native/crates/rt/src/futex.rs @@ -122,6 +122,30 @@ //! simply that async code uses the async form, and it is a correctness rule //! rather than a stylistic one. //! +//! # Waking a thread out of its wait +//! +//! A futex wakes the threads that wait on a word. Some waits must also end +//! for a reason that is not the word: a C thread's cancellation has to end +//! the `pthread_join` or `kore_futex_wait` it is parked in. [`wait_unless`] +//! is that wait — [`wait`] with a caller condition, `stop`, read *after* the +//! waiter registers, as the word is — and [`wake_thread`] is its wake: a kick +//! into one thread's doorbell rather than to the waiters of one word. +//! +//! The condition's writer stores it (`SeqCst`), then wakes the thread. That +//! is mechanism 2 again, with the condition in the word's place. The writer +//! stores, then reads the waiter's registration — [`wake_thread`] reads it +//! before it kicks or relays, and a hub's sweep reads it again before it +//! kicks — and the waiter registers, then reads the condition, so one of +//! the two sees the other. A writer that misses the registration is one whose +//! store the waiter's read follows: the waiter does not park. A writer that +//! sees it kicks a post that already stands, since the waiter armed before it +//! registered, and that kick resolves the park the waiter is in or about to +//! enter. A wait that can be ended therefore carries no timeout it does not +//! need: nothing re-reads, on a slice, a condition its writer announces. +//! +//! A stoppable wait parks on its doorbell even for a word in a noted window: +//! the kernel's word wait ends only on its word, and a stop arrives as a kick. +//! //! # What a caller must still do //! //! **Re-check in a loop.** A return from [`wait`] is never a promise that the @@ -306,9 +330,9 @@ const NOT_WAITING: usize = 0; /// speed while reporting that it was politely waiting. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum WaitOutcome { - /// The word did not hold `expected` — the caller's condition had already - /// changed, and no park was attempted. The common fast path under - /// contention, and not a failure. + /// The word did not hold `expected`, or a [`wait_unless`] stop held — the + /// caller's condition had already changed, and no park was attempted. + /// The common fast path under contention, and not a failure. Retry, /// The thread parked and was woken. **Says nothing about the word**: it /// may be a matching [`wake`], an unrelated kick on the shared doorbell, @@ -399,7 +423,32 @@ pub(crate) const OWN_KICK: u16 = RELAY + 1; /// `std`'s locks will be built from, the same disposition /// [`block_on`](crate::block::block_on) takes. pub fn wait(word: &AtomicU32, expected: u32, timeout: Option) -> WaitOutcome { - wait_inner(word, expected, timeout, true, LOCK_SPIN) + wait_inner(word, expected, timeout, true, LOCK_SPIN, None) +} + +/// [`wait`] that another thread can also end by publishing `stop` and then +/// waking **this thread** ([`wake_thread`]) — the wait a cancellation point +/// is, whose request is a flag rather than a word. +/// +/// `stop` is read after this waiter registers, as the word is, and the park +/// follows only if the word still holds `expected` and `stop` is still +/// `false`; otherwise this returns [`Retry`](WaitOutcome::Retry), having +/// parked nothing. The writer of the condition stores it with `SeqCst` and +/// then calls [`wake_thread`] with this thread's band, so neither can miss +/// the other (module docs, *Waking a thread out of its wait*): no timeout is +/// needed to notice a stop. +/// +/// The park is always the doorbell's, even for a word in a noted window — +/// the kernel's word wait cannot be ended by a kick. Everything [`wait`] +/// says about spurious wakes holds: re-check the word *and* `stop` after any +/// return. Not from inside a future, for [`wait`]'s reasons. +pub fn wait_unless( + word: &AtomicU32, + expected: u32, + timeout: Option, + stop: impl Fn() -> bool, +) -> WaitOutcome { + wait_inner(word, expected, timeout, true, LOCK_SPIN, Some(&stop)) } /// [`wait`] with the **barrier's** longer pre-park monitored budget. @@ -422,7 +471,7 @@ pub(crate) fn wait_barrier( expected: u32, timeout: Option, ) -> WaitOutcome { - wait_inner(word, expected, timeout, true, BARRIER_SPIN) + wait_inner(word, expected, timeout, true, BARRIER_SPIN, None) } /// [`wait_barrier`] with **no** jitter-window spin: straight to the park. @@ -432,7 +481,7 @@ pub(crate) fn wait_barrier( /// Nothing co-resident is about to store the word, so the spin would only /// occupy the core; counted like the barrier wait it stands in for. pub(crate) fn wait_idle(word: &AtomicU32, expected: u32, timeout: Option) -> WaitOutcome { - wait_inner(word, expected, timeout, true, NO_SPIN) + wait_inner(word, expected, timeout, true, NO_SPIN, None) } /// [`wait`] without the [`stats`] accounting — for the star's hub, @@ -443,7 +492,7 @@ pub(crate) fn wait_quiet( expected: u32, timeout: Option, ) -> WaitOutcome { - wait_inner(word, expected, timeout, false, LOCK_SPIN) + wait_inner(word, expected, timeout, false, LOCK_SPIN, None) } fn wait_inner( @@ -452,6 +501,7 @@ fn wait_inner( timeout: Option, counted: bool, spin: SpinBudget, + stop: Option<&dyn Fn() -> bool>, ) -> WaitOutcome { let Some(me) = current_thread() else { return WaitOutcome::Unsupported; @@ -463,35 +513,19 @@ fn wait_inner( // named to the kernel by that authority and parks on the word itself // (the kernel address-wait); every other word keeps the doorbell park. // Decided once, before the registry entry, because the entry records - // the decision for wakers. - let route = crate::cap::mapped::locate(word as *const AtomicU32 as *const u8); - - // ── 1 · Arm ───────────────────────────────────────────────────────── - // The standing post must exist before this thread is discoverable as a - // waiter, or a notice sent in the window between registering and parking - // would find no post and be dropped (`WouldBlock`, on the floor). Arming - // first makes "too early" indistinguishable from "on time". A word park - // needs no post: the mapping is its wait authority, which is what lets a - // doorbell-less worker wait here at all. - if route.is_none() && !imp::arm() { - return WaitOutcome::Unsupported; - } - - // ── 2 · Register, then check ──────────────────────────────────────── - // This store must be visible before the load below, and `SeqCst` on both - // is what forbids the store-load reordering that would let this thread - // and a waker miss each other (module docs, mechanism 2). - let slot = match route { - Some(_) => claim_wait_on_word(me, addr), - None => claim_wait(me, addr), + // the decision for wakers. A wait with a `stop` keeps the doorbell + // whatever the word: its stop arrives as a kick, which a word park + // never sees (`wait_unless`). + let route = match stop { + Some(_) => None, + None => crate::cap::mapped::locate(word as *const AtomicU32 as *const u8), }; - let Some(slot) = slot else { - return WaitOutcome::Unsupported; + + // ── 1, 2 · Arm, register, then check ──────────────────────────────── + let slot = match stand(me, word, expected, route.is_some(), stop, imp::arm) { + Ok(slot) => slot, + Err(outcome) => return outcome, }; - let _slot = WaitSlot { thread: me, slot }; - if word.load(Ordering::SeqCst) != expected { - return WaitOutcome::Retry; - } // ── 3 · Spin on the line, briefly ─────────────────────────────────── if let Some(end) = spin_briefly(word, expected, spin) { @@ -499,7 +533,7 @@ fn wait_inner( } // ── 4 · Park ──────────────────────────────────────────────────────── - // `_slot` releases on the way out: a waiter that is awake must not stay + // `slot` releases on the way out: a waiter that is awake must not stay // findable, or the next `wake` spends its kick on nobody and reports // having woken someone. if counted { @@ -515,7 +549,7 @@ fn wait_inner( // stored — SeqCst on both sides, one of us sees the other), // then arm the doorbell. Without the demotion a word-mode // waiter parked on its doorbell is unreachable by any waker. - demote_to_bell(me, slot, addr) + demote_to_bell(me, slot.slot, addr) && word.load(Ordering::SeqCst) == expected && imp::arm() }) @@ -532,6 +566,57 @@ fn wait_inner( outcome } +/// Steps 1 and 2 of every wait — **arm, register, then check** — with the +/// arm injected, so the host suite holds the order to account against the +/// registry the target uses. +/// +/// `Ok` is a waiter that must park: registered on `word` (in word mode when +/// `on_word`), its post standing unless it parks on the word, and the word +/// still `expected` and `stop` still `false`, both read after the +/// registration. `Err` is what to return instead, having parked nothing; the +/// registration is released with it. +fn stand( + me: usize, + word: &AtomicU32, + expected: u32, + on_word: bool, + stop: Option<&dyn Fn() -> bool>, + arm: impl FnOnce() -> bool, +) -> Result { + let addr = address_of(word); + + // ── 1 · Arm ───────────────────────────────────────────────────────── + // The standing post must exist before this thread is discoverable as a + // waiter, or a notice sent in the window between registering and parking + // would find no post and be dropped (`WouldBlock`, on the floor). Arming + // first makes "too early" indistinguishable from "on time". A word park + // needs no post: the mapping is its wait authority, which is what lets a + // doorbell-less worker wait here at all. + if !on_word && !arm() { + return Err(WaitOutcome::Unsupported); + } + + // ── 2 · Register, then check ──────────────────────────────────────── + // This store must be visible before the loads below, and `SeqCst` on all + // of them is what forbids the store-load reordering that would let this + // thread and a waker miss each other (module docs, mechanism 2). `stop` + // is the same argument with the caller's condition in the word's place: + // its writer stores, then reads this registration before it kicks. + let slot = if on_word { + claim_wait_on_word(me, addr) + } else { + claim_wait(me, addr) + }; + let Some(slot) = slot else { + return Err(WaitOutcome::Unsupported); + }; + let slot = WaitSlot { thread: me, slot }; + if word.load(Ordering::SeqCst) != expected || stop.is_some_and(|stop| stop()) { + return Err(WaitOutcome::Retry); + } + Ok(slot) +} + /// Cumulative process-wide wait/wake accounting — the terminal's window /// into behavior no serial can show (`res/SPMD.md` §11's doctrine). /// Relaxed counters: telemetry, never synchronization. `rode` is the @@ -952,6 +1037,52 @@ pub fn wake_all(word: &AtomicU32) -> usize { woken } +/// Wake **thread `band`** rather than the waiters of a word: kick its +/// doorbell, so a park it is in or about to enter returns. The wake a +/// [`wait_unless`] waiter's `stop` is announced by. +/// +/// Store the condition first, with `SeqCst`. This reads the target's +/// registration before it kicks, so a target that registers after that read +/// reads the condition after it (module docs, *Waking a thread out of its +/// wait*); a target with no doorbell waiter registered is running, and is not +/// kicked. A thread waking itself kicks nothing either: it is not parked. +/// +/// The kick goes directly when this thread holds a send into the target's +/// doorbell, and otherwise through this thread's relay: the hub it adopted +/// (the [`thread_controller`](crate::thread_controller) of the threads it +/// founds), whose sweep kicks every thread with a doorbell waiter +/// registered, or the star's hub. `true` when a kick or a relay went out. +/// `false` when no kick was needed or this thread has no route to the +/// target, which a caller that must reach it checks before it publishes. +pub fn wake_thread(band: usize) -> bool { + wake_thread_with(current_thread(), band, imp::kick, relay_missed) +} + +/// [`wake_thread`]'s selection, with the kick and the relay injected so the +/// host suite can hold it to account. +fn wake_thread_with( + me: Option, + band: usize, + kick: impl FnOnce(usize) -> bool, + relay: impl FnOnce() -> bool, +) -> bool { + if band >= MAX_THREADS as usize || me == Some(band) || !waits_on_bell(band) { + return false; + } + if kick(band) { + STAT_KICKS.fetch_add(1, Ordering::Relaxed); + return true; + } + // Refused: no send into that doorbell, or its post already spent — the + // relay covers both, as it does for `wake`. + STAT_MISSED.fetch_add(1, Ordering::Relaxed); + if relay() { + STAT_RELAYS.fetch_add(1, Ordering::Relaxed); + return true; + } + false +} + /// A **relay hub that is also a waiter**: the thread that founds a group of /// threads and holds a send into each of their doorbells — the /// [`thread_controller`](crate::thread_controller) — parked on its @@ -1020,6 +1151,13 @@ pub(crate) fn adopt_hub(hub: &'static Hub) -> bool { imp::adopt_hub(hub) } +/// Whether the hub this thread relays through is `hub`: a wake it cannot +/// deliver itself goes there, and from there to every thread `hub`'s thread +/// can kick. +pub(crate) fn relays_through(hub: &Hub) -> bool { + imp::hub().is_some_and(|adopted| core::ptr::eq(adopted, hub)) +} + /// Record that this thread holds `send`, a send into band `peer`'s /// doorbell its founding did not stamp — the channel a controller mints for /// each child it founds, or a starter for the controller it founds. From @@ -1080,7 +1218,6 @@ fn sweep_with(me: usize, mut kick: impl FnMut(usize) -> bool) -> usize { } /// Whether `thread` has any waiter a kick reaches — a doorbell-mode entry. -#[cfg_attr(not(target_os = "none"), allow(dead_code))] fn waits_on_bell(thread: usize) -> bool { WAITING_ON[thread].iter().any(|cell| { let entry = cell.load(Ordering::SeqCst); @@ -1114,16 +1251,10 @@ pub(crate) fn wait_or_end( return WaitOutcome::Unsupported; }; // Arm, register, check — `wait`'s order and for its reasons. - if !imp::arm() { - return WaitOutcome::Unsupported; - } - let Some(slot) = claim_wait(me, address_of(word)) else { - return WaitOutcome::Unsupported; + let _slot = match stand(me, word, expected, false, None, imp::arm) { + Ok(slot) => slot, + Err(outcome) => return outcome, }; - let _slot = WaitSlot { thread: me, slot }; - if word.load(Ordering::SeqCst) != expected { - return WaitOutcome::Retry; - } imp::park_or_end(timeout, task, report) } @@ -1158,7 +1289,8 @@ fn address_of(word: &AtomicU32) -> usize { word as *const AtomicU32 as usize } -/// This thread's band index — its identity in [`WAITING_ON`]. +/// This thread's band index — its identity in the wait registry +/// (`WAITING_ON`). /// /// The allocator's shard index, deliberately: it is already stamped at /// anchor, already proved injective across live threads (the `MAX_THREADS <= @@ -1168,7 +1300,12 @@ fn address_of(word: &AtomicU32) -> usize { /// The bound is re-checked rather than assumed: an out-of-band index would /// index the registry out of range, and a thread that cannot name itself must /// refuse to wait rather than borrow a neighbour's entry. -pub(crate) fn current_thread() -> Option { +/// +/// Also the name [`wake_thread`] reaches this thread by, which a thread +/// records where its wakers can read it. An identity, not an authority: a +/// waker still needs a send into the band's doorbell, or a relay that holds +/// one. `None` off metal, and on a thread outside the band. +pub fn current_thread() -> Option { let index = imp::thread_index()?; (index < MAX_THREADS as usize).then_some(index) } @@ -2014,9 +2151,15 @@ mod tests { claim_wait(9, address_of(&a)).expect("a doorbell waiter"); claim_wait_on_word(10, address_of(&b)).expect("a word waiter"); let mut kicked = std::vec::Vec::new(); + // The sweep scans every row, and tests in parallel register rows of + // their own: only this test's rows are delivered and recorded, and + // another's is a refused kick, which is not counted. let n = sweep_with(8, |thread| { - kicked.push(thread); - true + let ours = (8..=11).contains(&thread); + if ours { + kicked.push(thread); + } + ours }); assert_eq!(kicked, [9], "the sweep kicked a thread no kick can wake"); assert_eq!(n, 1); @@ -2027,6 +2170,202 @@ mod tests { } } + #[test] + fn a_stop_is_read_only_once_the_waiter_is_registered() { + // The order `wait_unless` rests on: arm, register, and only then read + // the stop. Its writer stores the stop and then reads the registry, so + // it misses this waiter only when this waiter's read follows its store. + // Each step asserts that the one before it already happened. + use core::cell::Cell; + const ROW: usize = 12; + reset_thread(ROW); + let word = AtomicU32::new(5); + let addr = address_of(&word); + let armed = Cell::new(false); + let arm = || { + assert!( + !thread_waits_on(ROW, addr), + "registered before the post stood" + ); + armed.set(true); + true + }; + let holds = Cell::new(true); + let read = Cell::new(0); + let stop = || { + assert!( + armed.get() && thread_waits_on(ROW, addr), + "the stop was read before the waiter registered" + ); + read.set(read.get() + 1); + holds.get() + }; + + // A stop that holds: nothing parks, and the registration goes too. + let out = stand(ROW, &word, 5, false, Some(&stop), arm); + assert_eq!(out.err(), Some(WaitOutcome::Retry)); + assert_eq!(read.get(), 1); + assert!(!thread_waits_on(ROW, addr), "a stopped waiter stayed found"); + + // One that does not: the waiter parks registered, and leaves with + // its slot. + holds.set(false); + armed.set(false); + let slot = stand(ROW, &word, 5, false, Some(&stop), arm) + .expect("a waiter whose stop does not hold parks"); + assert_eq!(read.get(), 2); + assert!(thread_waits_on(ROW, addr), "a parking waiter is not found"); + drop(slot); + assert!(!thread_waits_on(ROW, addr)); + + // An arm the kernel refuses registers nothing and reads nothing. + let out = stand(ROW, &word, 5, false, Some(&stop), || false); + assert_eq!(out.err(), Some(WaitOutcome::Unsupported)); + assert_eq!(read.get(), 2); + assert!(!thread_waits_on(ROW, addr)); + reset_thread(ROW); + } + + #[test] + fn a_stop_published_before_its_wake_is_never_missed() { + // Every point at which the stop's writer — store, then + // `wake_thread` — can run against a waiter's arm, registration, stop + // read and park. The waiter either reads the stop and parks nothing, + // or parks on a post the writer's kick consumed, which the kernel + // resolves at once. The simulated post is the kernel's rule: arming + // stands it, a kick consumes a standing one and drops otherwise, and + // a park on a consumed post returns. + use core::sync::atomic::AtomicBool; + const ROW: usize = 13; + #[derive(Clone, Copy, Debug, PartialEq)] + enum At { + BeforeArm, + BeforeRegistering, + BeforeStopRead, + // Just after the waiter read `false`: the window a stop read + // before the registration would leave open. + AfterStopRead, + BeforePark, + } + for at in [ + At::BeforeArm, + At::BeforeRegistering, + At::BeforeStopRead, + At::AfterStopRead, + At::BeforePark, + ] { + reset_thread(ROW); + let word = AtomicU32::new(0); + let stopped = AtomicBool::new(false); + let post = AtomicBool::new(false); + let writer = || { + stopped.store(true, Ordering::SeqCst); + let _ = wake_thread_with( + None, + ROW, + |band| { + assert_eq!(band, ROW); + post.compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + }, + || false, + ); + }; + if at == At::BeforeArm { + writer(); + } + let arm = || { + post.store(true, Ordering::SeqCst); + if at == At::BeforeRegistering { + writer(); + } + true + }; + let stop = || { + if at == At::BeforeStopRead { + writer(); + } + let stop = stopped.load(Ordering::SeqCst); + if at == At::AfterStopRead { + writer(); + } + stop + }; + match stand(ROW, &word, 0, false, Some(&stop), arm) { + Err(WaitOutcome::Retry) => { + assert!( + !matches!(at, At::AfterStopRead | At::BeforePark), + "{at:?}: a stop not yet written was read" + ); + } + Ok(slot) => { + assert!( + matches!(at, At::AfterStopRead | At::BeforePark), + "{at:?}: the stop went unread" + ); + if at == At::BeforePark { + writer(); + } + assert!( + !post.load(Ordering::SeqCst), + "the waiter parks on a post its stop's wake never consumed" + ); + drop(slot); + } + Err(other) => panic!("{at:?}: {other:?}"), + } + } + reset_thread(ROW); + } + + #[test] + fn a_thread_wake_kicks_only_a_registered_waiter_and_relays_a_refusal() { + const ROW: usize = 14; + reset_thread(ROW); + let word = AtomicU32::new(0); + let never = |_: usize| -> bool { panic!("kicked a thread no kick can wake") }; + let no_relay = || -> bool { panic!("relayed a wake that needed none") }; + // A running thread (nothing registered), a band past the registry, + // and the caller itself are not kicked. + let past = MAX_THREADS as usize; + assert!(!wake_thread_with(None, ROW, never, no_relay)); + assert!(!wake_thread_with(None, past, never, no_relay)); + let slot = claim_wait(ROW, address_of(&word)).expect("a free wait slot"); + let waiter = WaitSlot { thread: ROW, slot }; + assert!(!wake_thread_with(Some(ROW), ROW, never, no_relay)); + // A registered doorbell waiter: kicked directly, relayed when the + // kick refuses, and `false` only when neither went out. + assert!(wake_thread_with( + None, + ROW, + |band| { + assert_eq!(band, ROW); + true + }, + no_relay + )); + assert!(wake_thread_with(None, ROW, |_| false, || true)); + assert!(!wake_thread_with(None, ROW, |_| false, || false)); + drop(waiter); + // A word-parked waiter is the kernel's address-wait's to end. + let slot = claim_wait_on_word(ROW, address_of(&word)).expect("a free wait slot"); + let waiter = WaitSlot { thread: ROW, slot }; + assert!(!wake_thread_with(None, ROW, never, no_relay)); + drop(waiter); + reset_thread(ROW); + } + + #[test] + fn off_metal_a_stoppable_wait_refuses_before_reading_its_stop() { + reset_thread(7); + let word = AtomicU32::new(1); + assert_eq!( + wait_unless(&word, 1, None, || panic!("read a stop with no identity")), + WaitOutcome::Unsupported + ); + assert!(!wake_thread(0), "off metal no kick reaches a thread"); + } + #[test] fn the_own_kick_is_named_past_every_peer_and_the_relay() { // No peer arithmetic (`PEER_BASE + band`) and no relay lookup may diff --git a/native/crates/rt/src/thread_controller.rs b/native/crates/rt/src/thread_controller.rs index dbee9b4..3e2e017 100644 --- a/native/crates/rt/src/thread_controller.rs +++ b/native/crates/rt/src/thread_controller.rs @@ -27,6 +27,8 @@ //! the controller mints a doorbell, keeps the send (an adopted peer, withdrawn //! before the child's band can be reused), and founds the child holding it, a //! send into the controller, the starter's kick, the host's wake and the heap. +//! The child runs its callback once its founding is published, which follows +//! the adopted send, so the controller reaches every wait the callback makes. //! Children and the starter relay through the controller's hub, so a wake //! between two threads neither can kick directly still arrives: the controller //! sweeps ([`futex::sweep`]). @@ -42,9 +44,12 @@ //! A starter with no channel space founds a controller without a doorbell: //! neither it nor its children can park, and each of their waits takes the //! deadline park every thread can ([`pace::cede`]). A client the controller -//! cannot kick waits in bounded slices ([`UNREACHED_SLICE`]). A thread that +//! cannot kick waits in bounded slices ([`Controller::slice`]). A thread that //! cannot wake a parking controller is refused a create, which would -//! otherwise wait on a request nothing announced. +//! otherwise wait on a request nothing announced; one that cannot relay +//! through it cannot announce a request to its threads either +//! ([`Controller::wakes_threads`]), so a parked child is never left waiting +//! on a request nothing woke it for. //! //! [`thread::exit`]: crate::thread::exit //! [`pace::cede`]: crate::pace::cede @@ -88,7 +93,8 @@ const POLLING: u32 = 3; /// at before the controller parked (`pace::CEDE_NS`): a client the wake /// fabric cannot reach is no worse off than it was. Shorter would wake an /// unreachable joiner more often for nothing; longer would delay it. -const UNREACHED_SLICE: Duration = Duration::from_nanos(crate::pace::CEDE_NS); +/// [`Controller::slice`] says which waits take it. +pub const UNREACHED_SLICE: Duration = Duration::from_nanos(crate::pace::CEDE_NS); /// How long the controller rides its doorbell after a reap the kernel did /// not complete — a refused lifecycle observation or reclamation, which no @@ -142,7 +148,8 @@ fn empty(_: usize) -> usize { struct Record { state: AtomicU64, /// Bumped by the controller after each transition a client can wait - /// for; clients park on it. An event count: the state is the truth. + /// for; clients park on it, the child too until it is founded. An event + /// count: the state is the truth. event: AtomicU32, /// Set by the child's exit hook: it has stopped running its own code /// and is committing its end. @@ -267,7 +274,8 @@ pub struct Controller { #[cfg_attr(not(target_os = "none"), allow(dead_code))] starter: AtomicU32, /// The bands whose doorbells the controller can kick, a bit each: the - /// clients its wakes reach. A child's bit is set once it is founded and + /// clients its wakes reach. A child's bit is set before its founding is + /// published, which the child waits for before its callback runs, and /// cleared before its band can be reused. reach: AtomicU32, /// What every founding here hands on ([`Handed`], packed; zero is @@ -432,6 +440,37 @@ impl Controller { .is_some_and(futex::reaches) } + /// Whether a wake this thread sends reaches the threads the controller + /// founded, so a request it announces to them by a wake — a cancellation, + /// stored and then [`futex::wake_thread`]ed — ends any park they are in. + /// + /// True when the controller parks and this thread both holds a send into + /// it and relays through its hub — the starter, or a thread it created — + /// since every thread the controller can kick is then one relay away. True + /// as well when it does not park, since neither do its threads, which need + /// no wake. False otherwise, and before the controller starts: a caller + /// refuses such a request, rather than leave it behind a park nothing ends. + pub fn wakes_threads(&self) -> bool { + match self.mode() { + None => false, + Some(PARKING) => self.wakes_controller() && futex::relays_through(&self.hub), + Some(_) => true, + } + } + + /// How long one wait of the calling thread may park: unbounded (`None`) + /// when the controller can kick it — it holds a send into the thread's + /// doorbell, so every wake it sends, and every wake relayed through it, + /// reaches the thread — and [`UNREACHED_SLICE`] when nothing would end + /// that park early. A thread the controller founds with a doorbell is + /// reached before its callback runs, so every wait of that callback is + /// unbounded from the first. + pub fn slice(&self) -> Option { + let reached = futex::current_thread() + .is_some_and(|band| self.reach.load(Ordering::SeqCst) & (1 << band) != 0); + (!reached).then_some(UNREACHED_SLICE) + } + /// Park until `record`'s state satisfies `done`, and return that state. fn settle(&self, record: &Record, done: impl Fn(u64) -> bool) -> u64 { loop { @@ -443,9 +482,7 @@ impl Controller { if done(state) { return state; } - let reached = futex::current_thread() - .is_some_and(|band| self.reach.load(Ordering::SeqCst) & (1 << band) != 0); - park(&record.event, seen, (!reached).then_some(UNREACHED_SLICE)); + park(&record.event, seen, self.slice()); } } @@ -582,7 +619,7 @@ fn pause() { } /// The managed thread's entry: announce its end however it comes, relay -/// through the controller, run the callback. +/// through the controller, wait to be founded, run the callback. fn child_entry(argument: usize) { // SAFETY: controller storage is static; record cannot be reused until join. let record = unsafe { &*(argument as *const Record) }; @@ -599,7 +636,14 @@ fn child_entry(argument: usize) { let _ = record.state.load(Ordering::Acquire); let start = unsafe { *record.start.get() }; // SAFETY: `home` names the static controller whose record this is. - let _ = futex::adopt_hub(&unsafe { &*(start.home as *const Controller) }.hub); + let home = unsafe { &*(start.home as *const Controller) }; + let _ = futex::adopt_hub(&home.hub); + // The callback runs once the founding is published, as `create` returns + // once it is. The controller publishes it only after it can kick this + // thread (`reach`), so no wait in the callback rides the unreached slice + // for want of a bit the controller had yet to set. A child that outran + // its founding parks here as its creator does, until the announcement. + home.settle(record, |state| state & MASK != REQUEST); record .value .store((start.call)(start.argument), Ordering::Release); @@ -1248,6 +1292,45 @@ mod tests { assert_eq!(record.state.load(Ordering::Acquire), 16); } #[test] + fn a_child_runs_its_callback_only_once_its_founding_is_published() { + // The child's half of its founding, with a host thread for the child + // and this one for the controller. The controller sets the child's + // reach bit before it publishes the founding, so a callback that ran + // first could find its own waits unreached, and bounded. + static SEEN: AtomicU64 = AtomicU64::new(u64::MAX); + fn callback(record: usize) -> usize { + // SAFETY: the record is leaked below, so it outlives this child. + let record = unsafe { &*(record as *const Record) }; + SEEN.store(record.state.load(Ordering::Acquire), Ordering::Release); + 7 + } + let controller: &'static Controller = + std::boxed::Box::leak(std::boxed::Box::new(Controller::new())); + let record = &controller.records[0]; + // SAFETY: no child runs yet; this thread is the BUILD claimant. + unsafe { + *record.start.get() = Start { + call: callback, + argument: record as *const Record as usize, + retire: None, + home: controller as *const Controller as usize, + }; + } + record.state.store(REQUEST, Ordering::Release); + std::thread::scope(|scope| { + let child = scope.spawn(|| child_entry(record as *const Record as usize)); + // Every chance for a child that does not wait to run first. + std::thread::sleep(Duration::from_millis(20)); + assert_eq!(record.returned.load(Ordering::Acquire), 0); + record.state.store(RUNNING, Ordering::Release); + record.announce(); + child.join().unwrap(); + }); + assert_eq!(SEEN.load(Ordering::Acquire), RUNNING); + assert_eq!(record.value.load(Ordering::Acquire), 7); + assert_eq!(record.returned.load(Ordering::Acquire), 1); + } + #[test] fn an_unstarted_controller_refuses_and_never_waits() { let controller: &'static Controller = std::boxed::Box::leak(std::boxed::Box::new(Controller::new())); @@ -1259,6 +1342,16 @@ mod tests { assert_eq!(controller.create(|_| 0, 0), Err(Error::Unavailable)); } #[test] + fn an_unstarted_controller_wakes_no_thread_and_bounds_every_wait() { + // Nothing relays through a controller that was never founded, so no + // request to its threads is announced by a wake; and a thread it + // cannot kick — off metal, every thread — parks one slice at a time. + let controller = Controller::new(); + assert!(!controller.wakes_threads()); + assert_eq!(controller.slice(), Some(UNREACHED_SLICE)); + assert_eq!(controller.started.load(Ordering::Acquire), IDLE); + } + #[test] fn records_own_their_interference_span() { assert_eq!(core::mem::align_of::(), 128); assert_eq!(core::mem::size_of::(), 128); diff --git a/native/crates/vdev-accel/include/kore/vdev_accel.h b/native/crates/vdev-accel/include/kore/vdev_accel.h new file mode 100644 index 0000000..9f66ebb --- /dev/null +++ b/native/crates/vdev-accel/include/kore/vdev_accel.h @@ -0,0 +1,308 @@ +/* : the virtual accelerator, for a driver in C. + * + * The accelerator is a virtio-accel 1.0 device with no hardware behind it + * (`/kore/device/virtual/accel`, ). A driver brings it up + * over its register block and then speaks the protocol to it: one command + * is one chain of its virtqueue, a request the device reads and a response + * it writes, each a header and a payload. + * + * It computes six functions of one FP8 (E4M3) value, elementwise over a + * buffer. A session: + * + * CREATE_CONTEXT; ALLOCATE_BUFFER twice (operands, results); + * WRITE_BUFFER the operands; CREATE_QUEUE; + * LOAD_PROGRAM naming a kernel; SUBMIT binding the operands to slot 0 + * for reading and the results to slot 1 for writing; + * POLL_EVENT until the event is complete; DESTROY_EVENT; READ_BUFFER. + * + * A buffer bound to a submission is the event's until the event is + * destroyed, so results are read after DESTROY_EVENT. A reset of the + * device gives every object a new namespace: an id from before it names + * nothing after it. + * + * Every word is little-endian, and no frame has padding: the structures + * are packed, and each asserts its size and its fields' offsets. + * + * Generated from the protocol's own definitions by kore-vdev-accel's + * `c_header` test. Do not edit. */ +#ifndef KORE_VDEV_ACCEL_H +#define KORE_VDEV_ACCEL_H + +#include +#include + +/* The protocol, and the device's name in the register block. */ +#define KORE_ACCEL_PROTOCOL_MAJOR 1 +#define KORE_ACCEL_PROTOCOL_MINOR 0 +#define KORE_ACCEL_DEVICE_ID 0x6c636361u /* "accl" */ + +/* Commands: a request's `opcode`. */ +#define KORE_ACCEL_OP_GET_DEVICE_INFO 0x0001 +#define KORE_ACCEL_OP_CREATE_CONTEXT 0x0100 +#define KORE_ACCEL_OP_DESTROY_CONTEXT 0x0101 +#define KORE_ACCEL_OP_ALLOCATE_BUFFER 0x0200 +#define KORE_ACCEL_OP_FREE_BUFFER 0x0201 +#define KORE_ACCEL_OP_WRITE_BUFFER 0x0202 +#define KORE_ACCEL_OP_READ_BUFFER 0x0203 +#define KORE_ACCEL_OP_LOAD_PROGRAM 0x0300 +#define KORE_ACCEL_OP_UNLOAD_PROGRAM 0x0301 +#define KORE_ACCEL_OP_CREATE_QUEUE 0x0400 +#define KORE_ACCEL_OP_DESTROY_QUEUE 0x0401 +#define KORE_ACCEL_OP_SUBMIT 0x0500 +#define KORE_ACCEL_OP_POLL_EVENT 0x0501 +#define KORE_ACCEL_OP_CANCEL_EVENT 0x0502 +#define KORE_ACCEL_OP_DESTROY_EVENT 0x0503 + +/* A response's `status`. */ +#define KORE_ACCEL_STATUS_OK 0 +#define KORE_ACCEL_STATUS_UNSUPPORTED 1 +#define KORE_ACCEL_STATUS_INCOMPATIBLE 2 +#define KORE_ACCEL_STATUS_INVALID_ARGUMENT 3 +#define KORE_ACCEL_STATUS_OUT_OF_BOUNDS 4 +#define KORE_ACCEL_STATUS_BUSY 5 +#define KORE_ACCEL_STATUS_OUT_OF_MEMORY 6 +#define KORE_ACCEL_STATUS_RESOURCE_LIMIT 7 +#define KORE_ACCEL_STATUS_DEADLINE_EXPIRED 8 +#define KORE_ACCEL_STATUS_DEVICE_LOST 9 +#define KORE_ACCEL_STATUS_PERMISSION_DENIED 10 +#define KORE_ACCEL_STATUS_STALE_OBJECT 11 +#define KORE_ACCEL_STATUS_INTERNAL_ERROR 0xffff + +/* An event's `state`. */ +#define KORE_ACCEL_EVENT_PENDING 0 +#define KORE_ACCEL_EVENT_COMPLETE 1 +#define KORE_ACCEL_EVENT_FAILED 2 +#define KORE_ACCEL_EVENT_CANCELLED 3 + +/* A buffer's `usage` bits, its `memory_domain`, and a binding's `access`. */ +#define KORE_ACCEL_BUFFER_TRANSFER_SOURCE 0x1u /* READ_BUFFER may read it */ +#define KORE_ACCEL_BUFFER_TRANSFER_DESTINATION 0x2u /* WRITE_BUFFER may write it */ +#define KORE_ACCEL_BUFFER_PROGRAM_INPUT 0x4u /* a program may read it */ +#define KORE_ACCEL_BUFFER_PROGRAM_OUTPUT 0x8u /* a program may write it */ +#define KORE_ACCEL_BUFFER_MUTABLE_STATE 0x10u +#define KORE_ACCEL_MEMORY_HOST 1 +#define KORE_ACCEL_ACCESS_READ 1 +#define KORE_ACCEL_ACCESS_WRITE 2 +#define KORE_ACCEL_ACCESS_READ_WRITE 3 + +/* A program of this device is a registered kernel: LOAD_PROGRAM with this + * format, this identity in `target[0]`, a kernel in `target[1]`, this + * artifact as the bytes after the request, and this resident charge. */ +#define KORE_ACCEL_PROGRAM_FORMAT 0x4d50534bu /* "KSPM" */ +#define KORE_ACCEL_PROGRAM_IDENTITY 0x4d50534bu /* "KSPM" */ +#define KORE_ACCEL_PROGRAM_ARTIFACT_BYTES 8 +#define KORE_ACCEL_PROGRAM_ARTIFACT { 0x4b, 0x53, 0x50, 0x4d, 0x01, 0x00, 0x00, 0x00 } /* a uint8_t array's initializer */ +#define KORE_ACCEL_PROGRAM_RESIDENT_BYTES 64 + +/* The kernels. Each reads slot 0 and writes slot 1, one E4M3 value a byte, + * and is defined by a 256-entry table: rounded to nearest, ties to even; + * saturating at +-448 (0x7e, 0xfe); a NaN operand or a result that is not + * real is the canonical NaN 0x7f. */ +#define KORE_ACCEL_KERNEL_SIN 0x0100u +#define KORE_ACCEL_KERNEL_COS 0x0101u +#define KORE_ACCEL_KERNEL_TAN 0x0102u +#define KORE_ACCEL_KERNEL_EXP 0x0103u +#define KORE_ACCEL_KERNEL_LOG 0x0104u +#define KORE_ACCEL_KERNEL_SQRT 0x0105u + +/* The device configuration, at the register block's `config`. */ +struct kore_accel_config { + uint16_t protocol_major; /* 1 */ + uint16_t protocol_minor; /* 0 */ + uint16_t command_queue_count; /* 1 */ + uint16_t max_chain_descriptors; /* the longest chain, and the smallest queue */ + uint32_t max_request_bytes; /* the largest request frame, header included */ + uint32_t max_response_bytes; /* the largest response frame, header included */ +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_config) == 16, "config"); +_Static_assert(offsetof(struct kore_accel_config, protocol_major) == 0, "config.protocol_major"); +_Static_assert(offsetof(struct kore_accel_config, protocol_minor) == 2, "config.protocol_minor"); +_Static_assert(offsetof(struct kore_accel_config, command_queue_count) == 4, "config.command_queue_count"); +_Static_assert(offsetof(struct kore_accel_config, max_chain_descriptors) == 6, "config.max_chain_descriptors"); +_Static_assert(offsetof(struct kore_accel_config, max_request_bytes) == 8, "config.max_request_bytes"); +_Static_assert(offsetof(struct kore_accel_config, max_response_bytes) == 12, "config.max_response_bytes"); + +/* The head of every request. Its payload follows it. */ +struct kore_accel_request { + uint16_t opcode; /* KORE_ACCEL_OP_* */ + uint16_t flags; /* 0 */ + uint32_t payload_bytes; /* bytes after this header */ + uint64_t request_id; /* the driver's own; the response repeats it */ +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_request) == 16, "request"); +_Static_assert(offsetof(struct kore_accel_request, opcode) == 0, "request.opcode"); +_Static_assert(offsetof(struct kore_accel_request, flags) == 2, "request.flags"); +_Static_assert(offsetof(struct kore_accel_request, payload_bytes) == 4, "request.payload_bytes"); +_Static_assert(offsetof(struct kore_accel_request, request_id) == 8, "request.request_id"); + +/* The head of every response. Its payload follows it. */ +struct kore_accel_response { + uint16_t status; /* KORE_ACCEL_STATUS_* */ + uint16_t flags; /* 0 */ + uint32_t payload_bytes; /* bytes after this header */ + uint64_t request_id; /* the request's */ +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_response) == 16, "response"); +_Static_assert(offsetof(struct kore_accel_response, status) == 0, "response.status"); +_Static_assert(offsetof(struct kore_accel_response, flags) == 2, "response.flags"); +_Static_assert(offsetof(struct kore_accel_response, payload_bytes) == 4, "response.payload_bytes"); +_Static_assert(offsetof(struct kore_accel_response, request_id) == 8, "response.request_id"); + +/* GET_DEVICE_INFO's response payload. */ +struct kore_accel_device_info { + uint8_t uuid[16]; /* the backend's name */ + uint16_t class; /* what kind of accelerator */ + uint16_t reserved; /* 0 */ + uint32_t vendor_id; + uint32_t device_id; + uint64_t capabilities; + uint32_t max_contexts; + uint32_t max_buffers_per_context; + uint32_t max_programs_per_context; + uint32_t max_queues_per_context; + uint32_t max_events_per_context; + uint32_t max_bindings_per_submission; + uint64_t max_buffer_bytes; + uint64_t max_artifact_bytes; +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_device_info) == 76, "device_info"); +_Static_assert(offsetof(struct kore_accel_device_info, uuid) == 0, "device_info.uuid"); +_Static_assert(offsetof(struct kore_accel_device_info, class) == 16, "device_info.class"); +_Static_assert(offsetof(struct kore_accel_device_info, reserved) == 18, "device_info.reserved"); +_Static_assert(offsetof(struct kore_accel_device_info, vendor_id) == 20, "device_info.vendor_id"); +_Static_assert(offsetof(struct kore_accel_device_info, device_id) == 24, "device_info.device_id"); +_Static_assert(offsetof(struct kore_accel_device_info, capabilities) == 28, "device_info.capabilities"); +_Static_assert(offsetof(struct kore_accel_device_info, max_contexts) == 36, "device_info.max_contexts"); +_Static_assert(offsetof(struct kore_accel_device_info, max_buffers_per_context) == 40, "device_info.max_buffers_per_context"); +_Static_assert(offsetof(struct kore_accel_device_info, max_programs_per_context) == 44, "device_info.max_programs_per_context"); +_Static_assert(offsetof(struct kore_accel_device_info, max_queues_per_context) == 48, "device_info.max_queues_per_context"); +_Static_assert(offsetof(struct kore_accel_device_info, max_events_per_context) == 52, "device_info.max_events_per_context"); +_Static_assert(offsetof(struct kore_accel_device_info, max_bindings_per_submission) == 56, "device_info.max_bindings_per_submission"); +_Static_assert(offsetof(struct kore_accel_device_info, max_buffer_bytes) == 60, "device_info.max_buffer_bytes"); +_Static_assert(offsetof(struct kore_accel_device_info, max_artifact_bytes) == 68, "device_info.max_artifact_bytes"); + +/* CREATE_CONTEXT's payload. The response is an object: the context. */ +struct kore_accel_create_context { + uint32_t flags; /* 0 */ + uint32_t reserved; /* 0 */ +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_create_context) == 8, "create_context"); +_Static_assert(offsetof(struct kore_accel_create_context, flags) == 0, "create_context.flags"); +_Static_assert(offsetof(struct kore_accel_create_context, reserved) == 4, "create_context.reserved"); + +/* One object's id: what creating answers and destroying takes. */ +struct kore_accel_object { + uint64_t object_id; +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_object) == 8, "object"); +_Static_assert(offsetof(struct kore_accel_object, object_id) == 0, "object.object_id"); + +/* ALLOCATE_BUFFER's payload. The response is an object: the buffer. */ +struct kore_accel_allocate_buffer { + uint64_t context_id; + uint64_t bytes; + uint64_t alignment; /* a power of two; 1 for bytes */ + uint8_t memory_domain; /* KORE_ACCEL_MEMORY_* */ + uint8_t reserved0[7]; /* 0 */ + uint32_t usage; /* KORE_ACCEL_BUFFER_* bits */ + uint32_t reserved1; /* 0 */ +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_allocate_buffer) == 40, "allocate_buffer"); +_Static_assert(offsetof(struct kore_accel_allocate_buffer, context_id) == 0, "allocate_buffer.context_id"); +_Static_assert(offsetof(struct kore_accel_allocate_buffer, bytes) == 8, "allocate_buffer.bytes"); +_Static_assert(offsetof(struct kore_accel_allocate_buffer, alignment) == 16, "allocate_buffer.alignment"); +_Static_assert(offsetof(struct kore_accel_allocate_buffer, memory_domain) == 24, "allocate_buffer.memory_domain"); +_Static_assert(offsetof(struct kore_accel_allocate_buffer, reserved0) == 25, "allocate_buffer.reserved0"); +_Static_assert(offsetof(struct kore_accel_allocate_buffer, usage) == 32, "allocate_buffer.usage"); +_Static_assert(offsetof(struct kore_accel_allocate_buffer, reserved1) == 36, "allocate_buffer.reserved1"); + +/* WRITE_BUFFER's payload, with the bytes after it, and READ_BUFFER's, whose response is the bytes. */ +struct kore_accel_transfer_buffer { + uint64_t buffer_id; + uint64_t offset; + uint64_t bytes; +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_transfer_buffer) == 24, "transfer_buffer"); +_Static_assert(offsetof(struct kore_accel_transfer_buffer, buffer_id) == 0, "transfer_buffer.buffer_id"); +_Static_assert(offsetof(struct kore_accel_transfer_buffer, offset) == 8, "transfer_buffer.offset"); +_Static_assert(offsetof(struct kore_accel_transfer_buffer, bytes) == 16, "transfer_buffer.bytes"); + +/* LOAD_PROGRAM's payload, with the artifact after it. The response is an object: the program. */ +struct kore_accel_load_program { + uint64_t context_id; + uint32_t format; /* KORE_ACCEL_PROGRAM_FORMAT */ + uint32_t flags; /* 0 */ + uint32_t target[12]; /* [0] KORE_ACCEL_PROGRAM_IDENTITY, [1] a kernel, the rest 0 */ + uint64_t payload_bytes; /* the artifact's bytes */ + uint64_t resident_bytes; /* KORE_ACCEL_PROGRAM_RESIDENT_BYTES */ +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_load_program) == 80, "load_program"); +_Static_assert(offsetof(struct kore_accel_load_program, context_id) == 0, "load_program.context_id"); +_Static_assert(offsetof(struct kore_accel_load_program, format) == 8, "load_program.format"); +_Static_assert(offsetof(struct kore_accel_load_program, flags) == 12, "load_program.flags"); +_Static_assert(offsetof(struct kore_accel_load_program, target) == 16, "load_program.target"); +_Static_assert(offsetof(struct kore_accel_load_program, payload_bytes) == 64, "load_program.payload_bytes"); +_Static_assert(offsetof(struct kore_accel_load_program, resident_bytes) == 72, "load_program.resident_bytes"); + +/* CREATE_QUEUE's payload. The response is an object: the execution queue. */ +struct kore_accel_create_queue { + uint64_t context_id; + uint32_t flags; /* 0 */ + uint32_t reserved; /* 0 */ +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_create_queue) == 16, "create_queue"); +_Static_assert(offsetof(struct kore_accel_create_queue, context_id) == 0, "create_queue.context_id"); +_Static_assert(offsetof(struct kore_accel_create_queue, flags) == 8, "create_queue.flags"); +_Static_assert(offsetof(struct kore_accel_create_queue, reserved) == 12, "create_queue.reserved"); + +/* SUBMIT's payload, with `binding_count` bindings after it. */ +struct kore_accel_submit { + uint64_t queue_id; + uint64_t program_id; + uint32_t binding_count; + uint32_t flags; /* 0 */ + uint64_t timeout_ns; /* from admission; 0 is none */ +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_submit) == 32, "submit"); +_Static_assert(offsetof(struct kore_accel_submit, queue_id) == 0, "submit.queue_id"); +_Static_assert(offsetof(struct kore_accel_submit, program_id) == 8, "submit.program_id"); +_Static_assert(offsetof(struct kore_accel_submit, binding_count) == 16, "submit.binding_count"); +_Static_assert(offsetof(struct kore_accel_submit, flags) == 20, "submit.flags"); +_Static_assert(offsetof(struct kore_accel_submit, timeout_ns) == 24, "submit.timeout_ns"); + +/* One buffer range a submission's program reads or writes. */ +struct kore_accel_binding { + uint64_t buffer_id; + uint64_t offset; + uint64_t bytes; + uint32_t slot; /* which of the program's operands */ + uint8_t access; /* KORE_ACCEL_ACCESS_* */ + uint8_t reserved[3]; /* 0 */ +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_binding) == 32, "binding"); +_Static_assert(offsetof(struct kore_accel_binding, buffer_id) == 0, "binding.buffer_id"); +_Static_assert(offsetof(struct kore_accel_binding, offset) == 8, "binding.offset"); +_Static_assert(offsetof(struct kore_accel_binding, bytes) == 16, "binding.bytes"); +_Static_assert(offsetof(struct kore_accel_binding, slot) == 24, "binding.slot"); +_Static_assert(offsetof(struct kore_accel_binding, access) == 28, "binding.access"); +_Static_assert(offsetof(struct kore_accel_binding, reserved) == 29, "binding.reserved"); + +/* SUBMIT's response payload: the event the submission ends as. */ +struct kore_accel_submit_response { + uint64_t event_id; +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_submit_response) == 8, "submit_response"); +_Static_assert(offsetof(struct kore_accel_submit_response, event_id) == 0, "submit_response.event_id"); + +/* POLL_EVENT's response payload. */ +struct kore_accel_event_state { + uint16_t state; /* KORE_ACCEL_EVENT_* */ + uint16_t error; /* a KORE_ACCEL_STATUS_* when the state is FAILED */ + uint32_t reserved; /* 0 */ +} __attribute__((packed)); +_Static_assert(sizeof(struct kore_accel_event_state) == 8, "event_state"); +_Static_assert(offsetof(struct kore_accel_event_state, state) == 0, "event_state.state"); +_Static_assert(offsetof(struct kore_accel_event_state, error) == 2, "event_state.error"); +_Static_assert(offsetof(struct kore_accel_event_state, reserved) == 4, "event_state.reserved"); + +#endif /* KORE_VDEV_ACCEL_H */ diff --git a/native/crates/vdev-accel/src/device.rs b/native/crates/vdev-accel/src/device.rs index ba3ca3a..2b3a0eb 100644 --- a/native/crates/vdev-accel/src/device.rs +++ b/native/crates/vdev-accel/src/device.rs @@ -31,7 +31,7 @@ //! complete; and a reset gives every object a new namespace, so an id a //! driver kept from before the reset names nothing afterwards. -use kore_svc_accel_spmd::SpmdBackend; +use kore_svc_accel_spmd::{REGISTERED_RESIDENT_BYTES, SpmdBackend}; use kore_vdev::device::{ReadableBytes, Unrecoverable, VirtualDevice, WritableBytes}; use virtio_accel_core::{ByteSink, ByteSource, TransportByteSink, TransportByteSource}; use virtio_accel_device::{ @@ -69,6 +69,17 @@ pub const MAX_BUFFER_BACKING_BYTES: u64 = 2 << 20; /// backend's own count limit allows many times over. pub const MAX_PROGRAM_RESIDENT_BYTES: u64 = 64 * 1024; +/// The resident charge a driver states when it loads one of this device's +/// programs (`LOAD_PROGRAM`'s `resident_bytes`): what the device will hold +/// for it. A registered kernel's handle is a few words, whose exact size is +/// the host's business; a driver states this round figure, which covers it +/// on every target, and a thousand programs fit the device's limit. +pub const PROGRAM_RESIDENT_BYTES: u64 = 64; +const _: () = { + assert!(PROGRAM_RESIDENT_BYTES >= REGISTERED_RESIDENT_BYTES); + assert!(PROGRAM_RESIDENT_BYTES <= MAX_PROGRAM_RESIDENT_BYTES); +}; + /// What a driver reads from the register block's device id: `"accl"`. The /// protocol assigns itself no Virtio device id, so the name is kore's. pub const DEVICE_ID: u32 = u32::from_le_bytes(*b"accl"); diff --git a/native/crates/vdev-accel/tests/c_header.rs b/native/crates/vdev-accel/tests/c_header.rs new file mode 100644 index 0000000..d156226 --- /dev/null +++ b/native/crates/vdev-accel/tests/c_header.rs @@ -0,0 +1,510 @@ +//! ``: what a C driver of the virtual accelerator needs, +//! written from the Rust that defines it. +//! +//! The header is not typed by hand. This test renders it from the +//! protocol's own types (`virtio-accel-proto`'s frames, with each field's +//! offset taken by `offset_of!`), the protocol's enumerations and this +//! device's registry, and requires the committed file to be exactly that. +//! A frame that moves, a status that is renumbered or a kernel that is added +//! fails here until the header is rendered again: +//! +//! ```text +//! KORE_BLESS=1 cargo test -p kore-vdev-accel --test c_header +//! ``` +//! +//! The header also carries its own check, a `_Static_assert` for every +//! frame's size and every field's offset, so each compiler that reads it +//! proves its layout is the protocol's. A second test hands the header to +//! the host's C compiler for that proof. + +use core::mem::{offset_of, size_of}; +use std::fmt::Write as _; +use std::path::PathBuf; + +use kore_svc_accel_spmd::{FORMAT_REGISTERED, IDENTITY_MAGIC, REGISTERED_MANIFEST}; +use kore_vdev_accel::device::{DEVICE_ID, PROGRAM_RESIDENT_BYTES}; +use kore_vdev_accel::fp8::Op; +use virtio_accel_core::{AccessMode, BufferUsage, MemoryDomain}; +use virtio_accel_proto::{ + AllocateBufferRequest, CreateContextRequest, CreateQueueRequest, KnownEventState, KnownOpcode, + LoadProgramRequest, ObjectPayload, PROTOCOL_MAJOR, PROTOCOL_MINOR, RequestHeader, + ResponseHeader, StatusCode, SubmitRequest, SubmitResponse, TransferBufferRequest, WireBinding, + WireConfig, WireDeviceInfo, WireEventState, +}; + +/// A field's C type. +#[derive(Clone, Copy)] +enum C { + U8, + U16, + U32, + U64, + /// `uint8_t name[n]`. + Bytes(usize), + /// `uint32_t name[n]`. + Words(usize), +} + +impl C { + fn bytes(self) -> usize { + match self { + Self::U8 => 1, + Self::U16 => 2, + Self::U32 => 4, + Self::U64 => 8, + Self::Bytes(n) => n, + Self::Words(n) => 4 * n, + } + } + + fn declare(self, name: &str) -> String { + match self { + Self::U8 => format!("uint8_t {name};"), + Self::U16 => format!("uint16_t {name};"), + Self::U32 => format!("uint32_t {name};"), + Self::U64 => format!("uint64_t {name};"), + Self::Bytes(n) => format!("uint8_t {name}[{n}];"), + Self::Words(n) => format!("uint32_t {name}[{n}];"), + } + } +} + +struct Field { + name: &'static str, + ty: C, + offset: usize, + note: &'static str, +} + +struct Frame { + name: &'static str, + about: &'static str, + bytes: usize, + fields: Vec, +} + +/// One frame: its C name, what it is, and each field with its C type. The +/// offsets and the size are the Rust type's. +macro_rules! frame { + ($ty:ty as $name:literal, $about:literal, { $($field:ident: $c:expr, $note:literal;)* }) => { + Frame { + name: $name, + about: $about, + bytes: size_of::<$ty>(), + fields: vec![$(Field { + name: stringify!($field), + ty: $c, + offset: offset_of!($ty, $field), + note: $note, + }),*], + } + }; +} + +fn frames() -> Vec { + vec![ + frame!(WireConfig as "config", + "The device configuration, at the register block's `config`.", { + protocol_major: C::U16, "1"; + protocol_minor: C::U16, "0"; + command_queue_count: C::U16, "1"; + max_chain_descriptors: C::U16, "the longest chain, and the smallest queue"; + max_request_bytes: C::U32, "the largest request frame, header included"; + max_response_bytes: C::U32, "the largest response frame, header included"; + }), + frame!(RequestHeader as "request", + "The head of every request. Its payload follows it.", { + opcode: C::U16, "KORE_ACCEL_OP_*"; + flags: C::U16, "0"; + payload_bytes: C::U32, "bytes after this header"; + request_id: C::U64, "the driver's own; the response repeats it"; + }), + frame!(ResponseHeader as "response", + "The head of every response. Its payload follows it.", { + status: C::U16, "KORE_ACCEL_STATUS_*"; + flags: C::U16, "0"; + payload_bytes: C::U32, "bytes after this header"; + request_id: C::U64, "the request's"; + }), + frame!(WireDeviceInfo as "device_info", + "GET_DEVICE_INFO's response payload.", { + uuid: C::Bytes(16), "the backend's name"; + class: C::U16, "what kind of accelerator"; + reserved: C::U16, "0"; + vendor_id: C::U32, ""; + device_id: C::U32, ""; + capabilities: C::U64, ""; + max_contexts: C::U32, ""; + max_buffers_per_context: C::U32, ""; + max_programs_per_context: C::U32, ""; + max_queues_per_context: C::U32, ""; + max_events_per_context: C::U32, ""; + max_bindings_per_submission: C::U32, ""; + max_buffer_bytes: C::U64, ""; + max_artifact_bytes: C::U64, ""; + }), + frame!(CreateContextRequest as "create_context", + "CREATE_CONTEXT's payload. The response is an object: the context.", { + flags: C::U32, "0"; + reserved: C::U32, "0"; + }), + frame!(ObjectPayload as "object", + "One object's id: what creating answers and destroying takes.", { + object_id: C::U64, ""; + }), + frame!(AllocateBufferRequest as "allocate_buffer", + "ALLOCATE_BUFFER's payload. The response is an object: the buffer.", { + context_id: C::U64, ""; + bytes: C::U64, ""; + alignment: C::U64, "a power of two; 1 for bytes"; + memory_domain: C::U8, "KORE_ACCEL_MEMORY_*"; + reserved0: C::Bytes(7), "0"; + usage: C::U32, "KORE_ACCEL_BUFFER_* bits"; + reserved1: C::U32, "0"; + }), + frame!(TransferBufferRequest as "transfer_buffer", + "WRITE_BUFFER's payload, with the bytes after it, and READ_BUFFER's, whose response is the bytes.", { + buffer_id: C::U64, ""; + offset: C::U64, ""; + bytes: C::U64, ""; + }), + frame!(LoadProgramRequest as "load_program", + "LOAD_PROGRAM's payload, with the artifact after it. The response is an object: the program.", { + context_id: C::U64, ""; + format: C::U32, "KORE_ACCEL_PROGRAM_FORMAT"; + flags: C::U32, "0"; + target: C::Words(12), "[0] KORE_ACCEL_PROGRAM_IDENTITY, [1] a kernel, the rest 0"; + payload_bytes: C::U64, "the artifact's bytes"; + resident_bytes: C::U64, "KORE_ACCEL_PROGRAM_RESIDENT_BYTES"; + }), + frame!(CreateQueueRequest as "create_queue", + "CREATE_QUEUE's payload. The response is an object: the execution queue.", { + context_id: C::U64, ""; + flags: C::U32, "0"; + reserved: C::U32, "0"; + }), + frame!(SubmitRequest as "submit", + "SUBMIT's payload, with `binding_count` bindings after it.", { + queue_id: C::U64, ""; + program_id: C::U64, ""; + binding_count: C::U32, ""; + flags: C::U32, "0"; + timeout_ns: C::U64, "from admission; 0 is none"; + }), + frame!(WireBinding as "binding", + "One buffer range a submission's program reads or writes.", { + buffer_id: C::U64, ""; + offset: C::U64, ""; + bytes: C::U64, ""; + slot: C::U32, "which of the program's operands"; + access: C::U8, "KORE_ACCEL_ACCESS_*"; + reserved: C::Bytes(3), "0"; + }), + frame!(SubmitResponse as "submit_response", + "SUBMIT's response payload: the event the submission ends as.", { + event_id: C::U64, ""; + }), + frame!(WireEventState as "event_state", + "POLL_EVENT's response payload.", { + state: C::U16, "KORE_ACCEL_EVENT_*"; + error: C::U16, "a KORE_ACCEL_STATUS_* when the state is FAILED"; + reserved: C::U32, "0"; + }), + ] +} + +fn define(out: &mut String, name: &str, value: impl core::fmt::Display, note: &str) { + let define = format!("#define KORE_ACCEL_{name} {value}"); + if note.is_empty() { + writeln!(out, "{define}").unwrap(); + } else { + writeln!(out, "{define} /* {note} */").unwrap(); + } +} + +/// A four-character code as a C unsigned literal, with its spelling. +fn fourcc(code: u32) -> (String, String) { + let spelled: String = code.to_le_bytes().iter().map(|&b| b as char).collect(); + (format!("0x{code:08x}u"), format!("\"{spelled}\"")) +} + +const BANNER: &str = r#"/* : the virtual accelerator, for a driver in C. + * + * The accelerator is a virtio-accel 1.0 device with no hardware behind it + * (`/kore/device/virtual/accel`, ). A driver brings it up + * over its register block and then speaks the protocol to it: one command + * is one chain of its virtqueue, a request the device reads and a response + * it writes, each a header and a payload. + * + * It computes six functions of one FP8 (E4M3) value, elementwise over a + * buffer. A session: + * + * CREATE_CONTEXT; ALLOCATE_BUFFER twice (operands, results); + * WRITE_BUFFER the operands; CREATE_QUEUE; + * LOAD_PROGRAM naming a kernel; SUBMIT binding the operands to slot 0 + * for reading and the results to slot 1 for writing; + * POLL_EVENT until the event is complete; DESTROY_EVENT; READ_BUFFER. + * + * A buffer bound to a submission is the event's until the event is + * destroyed, so results are read after DESTROY_EVENT. A reset of the + * device gives every object a new namespace: an id from before it names + * nothing after it. + * + * Every word is little-endian, and no frame has padding: the structures + * are packed, and each asserts its size and its fields' offsets. + * + * Generated from the protocol's own definitions by kore-vdev-accel's + * `c_header` test. Do not edit. */ +#ifndef KORE_VDEV_ACCEL_H +#define KORE_VDEV_ACCEL_H + +#include +#include + +"#; + +fn render() -> String { + let mut out = String::from(BANNER); + + out.push_str("/* The protocol, and the device's name in the register block. */\n"); + define(&mut out, "PROTOCOL_MAJOR", PROTOCOL_MAJOR, ""); + define(&mut out, "PROTOCOL_MINOR", PROTOCOL_MINOR, ""); + let (id, spelled) = fourcc(DEVICE_ID); + define(&mut out, "DEVICE_ID", id, &spelled); + + out.push_str("\n/* Commands: a request's `opcode`. */\n"); + for (name, opcode) in [ + ("GET_DEVICE_INFO", KnownOpcode::GetDeviceInfo), + ("CREATE_CONTEXT", KnownOpcode::CreateContext), + ("DESTROY_CONTEXT", KnownOpcode::DestroyContext), + ("ALLOCATE_BUFFER", KnownOpcode::AllocateBuffer), + ("FREE_BUFFER", KnownOpcode::FreeBuffer), + ("WRITE_BUFFER", KnownOpcode::WriteBuffer), + ("READ_BUFFER", KnownOpcode::ReadBuffer), + ("LOAD_PROGRAM", KnownOpcode::LoadProgram), + ("UNLOAD_PROGRAM", KnownOpcode::UnloadProgram), + ("CREATE_QUEUE", KnownOpcode::CreateQueue), + ("DESTROY_QUEUE", KnownOpcode::DestroyQueue), + ("SUBMIT", KnownOpcode::Submit), + ("POLL_EVENT", KnownOpcode::PollEvent), + ("CANCEL_EVENT", KnownOpcode::CancelEvent), + ("DESTROY_EVENT", KnownOpcode::DestroyEvent), + ] { + let value = format!("0x{:04x}", opcode as u16); + define(&mut out, &format!("OP_{name}"), value, ""); + } + + out.push_str("\n/* A response's `status`. */\n"); + for (name, status) in [ + ("OK", StatusCode::OK), + ("UNSUPPORTED", StatusCode::UNSUPPORTED), + ("INCOMPATIBLE", StatusCode::INCOMPATIBLE), + ("INVALID_ARGUMENT", StatusCode::INVALID_ARGUMENT), + ("OUT_OF_BOUNDS", StatusCode::OUT_OF_BOUNDS), + ("BUSY", StatusCode::BUSY), + ("OUT_OF_MEMORY", StatusCode::OUT_OF_MEMORY), + ("RESOURCE_LIMIT", StatusCode::RESOURCE_LIMIT), + ("DEADLINE_EXPIRED", StatusCode::DEADLINE_EXPIRED), + ("DEVICE_LOST", StatusCode::DEVICE_LOST), + ("PERMISSION_DENIED", StatusCode::PERMISSION_DENIED), + ("STALE_OBJECT", StatusCode::STALE_OBJECT), + ] { + define(&mut out, &format!("STATUS_{name}"), status.0, ""); + } + let internal = format!("0x{:04x}", StatusCode::INTERNAL_ERROR.0); + define(&mut out, "STATUS_INTERNAL_ERROR", internal, ""); + + out.push_str("\n/* An event's `state`. */\n"); + for (name, state) in [ + ("PENDING", KnownEventState::Pending), + ("COMPLETE", KnownEventState::Complete), + ("FAILED", KnownEventState::Failed), + ("CANCELLED", KnownEventState::Cancelled), + ] { + define(&mut out, &format!("EVENT_{name}"), state as u16, ""); + } + + out.push_str( + "\n/* A buffer's `usage` bits, its `memory_domain`, and a binding's `access`. */\n", + ); + for (name, usage, note) in [ + ( + "TRANSFER_SOURCE", + BufferUsage::TRANSFER_SOURCE, + "READ_BUFFER may read it", + ), + ( + "TRANSFER_DESTINATION", + BufferUsage::TRANSFER_DESTINATION, + "WRITE_BUFFER may write it", + ), + ( + "PROGRAM_INPUT", + BufferUsage::PROGRAM_INPUT, + "a program may read it", + ), + ( + "PROGRAM_OUTPUT", + BufferUsage::PROGRAM_OUTPUT, + "a program may write it", + ), + ("MUTABLE_STATE", BufferUsage::MUTABLE_STATE, ""), + ] { + let value = format!("0x{:x}u", usage.bits()); + define(&mut out, &format!("BUFFER_{name}"), value, note); + } + define(&mut out, "MEMORY_HOST", MemoryDomain::Host as u8, ""); + define(&mut out, "ACCESS_READ", AccessMode::Read as u8, ""); + define(&mut out, "ACCESS_WRITE", AccessMode::Write as u8, ""); + define( + &mut out, + "ACCESS_READ_WRITE", + AccessMode::ReadWrite as u8, + "", + ); + + out.push_str( + "\n/* A program of this device is a registered kernel: LOAD_PROGRAM with this\n \ + * format, this identity in `target[0]`, a kernel in `target[1]`, this\n \ + * artifact as the bytes after the request, and this resident charge. */\n", + ); + let (format, spelled) = fourcc(FORMAT_REGISTERED); + define(&mut out, "PROGRAM_FORMAT", format, &spelled); + let (identity, spelled) = fourcc(IDENTITY_MAGIC); + define(&mut out, "PROGRAM_IDENTITY", identity, &spelled); + define( + &mut out, + "PROGRAM_ARTIFACT_BYTES", + REGISTERED_MANIFEST.len(), + "", + ); + let artifact: Vec = REGISTERED_MANIFEST + .iter() + .map(|byte| format!("0x{byte:02x}")) + .collect(); + let artifact = format!("{{ {} }}", artifact.join(", ")); + define( + &mut out, + "PROGRAM_ARTIFACT", + artifact, + "a uint8_t array's initializer", + ); + define( + &mut out, + "PROGRAM_RESIDENT_BYTES", + PROGRAM_RESIDENT_BYTES, + "", + ); + + out.push_str( + "\n/* The kernels. Each reads slot 0 and writes slot 1, one E4M3 value a byte,\n \ + * and is defined by a 256-entry table: rounded to nearest, ties to even;\n \ + * saturating at +-448 (0x7e, 0xfe); a NaN operand or a result that is not\n \ + * real is the canonical NaN 0x7f. */\n", + ); + for op in Op::ALL { + let name = format!("KERNEL_{}", format!("{op:?}").to_uppercase()); + define(&mut out, &name, format!("0x{:04x}u", op.kernel_id()), ""); + } + + for frame in frames() { + writeln!(out, "\n/* {} */", frame.about).unwrap(); + writeln!(out, "struct kore_accel_{} {{", frame.name).unwrap(); + for field in &frame.fields { + let declared = field.ty.declare(field.name); + if field.note.is_empty() { + writeln!(out, " {declared}").unwrap(); + } else { + writeln!(out, " {declared} /* {} */", field.note).unwrap(); + } + } + out.push_str("} __attribute__((packed));\n"); + writeln!( + out, + "_Static_assert(sizeof(struct kore_accel_{}) == {}, \"{}\");", + frame.name, frame.bytes, frame.name + ) + .unwrap(); + for field in &frame.fields { + writeln!( + out, + "_Static_assert(offsetof(struct kore_accel_{}, {}) == {}, \"{}.{}\");", + frame.name, field.name, field.offset, frame.name, field.name + ) + .unwrap(); + } + } + + out.push_str("\n#endif /* KORE_VDEV_ACCEL_H */\n"); + out +} + +fn committed() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("include/kore/vdev_accel.h") +} + +/// The table above is the protocol's frames, whole: the fields are every +/// byte of each type, in order, with nothing between them. +#[test] +fn every_frame_is_its_fields_end_to_end() { + for frame in frames() { + let mut at = 0; + for field in &frame.fields { + assert_eq!(field.offset, at, "{}.{}", frame.name, field.name); + at += field.ty.bytes(); + } + assert_eq!(at, frame.bytes, "{}: fields cover the frame", frame.name); + } +} + +#[test] +fn the_committed_header_is_what_the_protocol_renders() { + let rendered = render(); + let path = committed(); + if std::env::var_os("KORE_BLESS").is_some() { + std::fs::create_dir_all(path.parent().expect("the header has a directory")).unwrap(); + std::fs::write(&path, &rendered).unwrap(); + } + let on_disk = std::fs::read_to_string(&path).unwrap_or_default(); + assert!( + on_disk == rendered, + "{} is not what the protocol renders; run\n KORE_BLESS=1 cargo test -p kore-vdev-accel --test c_header", + path.display() + ); +} + +/// The header's own assertions hold for a real C compiler, the host's: its +/// packed structures are laid out as the protocol's frames are. +#[test] +fn a_c_compiler_agrees_with_every_assertion() { + let cc = std::env::var_os("KORE_HOST_CC").unwrap_or_else(|| "cc".into()); + let dir = std::env::temp_dir().join(format!("kore-vdev-accel-header-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let source = dir.join("check.c"); + std::fs::write( + &source, + "#include \n\ + static const uint8_t artifact[KORE_ACCEL_PROGRAM_ARTIFACT_BYTES] = KORE_ACCEL_PROGRAM_ARTIFACT;\n\ + int kore_vdev_accel_header(void) { return artifact[0] + (int)KORE_ACCEL_KERNEL_SQRT; }\n", + ) + .unwrap(); + let output = std::process::Command::new(&cc) + .args([ + "-std=c11", + "-Wall", + "-Wextra", + "-Werror", + "-fsyntax-only", + "-I", + ]) + .arg(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("include")) + .arg(&source) + .output() + .unwrap_or_else(|error| panic!("the host C compiler {cc:?} did not run: {error}")); + let _ = std::fs::remove_dir_all(&dir); + assert!( + output.status.success(), + "the header does not compile:\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/native/crates/window/src/control.rs b/native/crates/window/src/control.rs index 21ccca6..1720497 100644 --- a/native/crates/window/src/control.rs +++ b/native/crates/window/src/control.rs @@ -17,7 +17,7 @@ use kore_rt::svc::{PeerBundlePart, PeerId}; use kore_svc_output::service::{OutputRequest, OutputResponse}; use crate::appearance::Density; -use crate::desktop::{DesktopRequest, DesktopState, Mode}; +use crate::desktop::{DesktopRequest, DesktopState, Mode, PointerPreference, PointerSpeed}; const PREPARE_SETTINGS: u8 = 0x52; const DESKTOP: u8 = 0x53; @@ -805,6 +805,12 @@ pub fn encode_req(request: &DisplayRequest) -> SurReqWire { w.subject = 3.into(); w.a = surface.into(); } + DesktopRequest::Pointer(preference) => { + w.subject = 5.into(); + w.a = u64::from(preference.acceleration).into(); + // The step is -2..=2; on the wire it is 0..=4. + w.b = ((preference.speed.step() - PointerSpeed::SLOWEST.step()) as u64).into(); + } } } DisplayRequest::Surface(request) => return surface::encode_req(&request), @@ -948,6 +954,18 @@ pub fn decode_req(w: &SurReqWire) -> Option { height: u16::try_from(w.b.get()).ok()?, }, 3 if w.a.get() != 0 => DesktopRequest::ApplyWallpaper { surface: w.a.get() }, + 5 => DesktopRequest::Pointer(PointerPreference { + acceleration: match w.a.get() { + 0 => false, + 1 => true, + _ => return None, + }, + speed: PointerSpeed::new( + i8::try_from(w.b.get()) + .ok()? + .checked_add(PointerSpeed::SLOWEST.step())?, + )?, + }), _ => return None, }), ADMIT => DisplayRequest::AdmitPeer { @@ -1256,7 +1274,11 @@ pub fn reply_matches(w: &SurReqWire, response: &DisplayResponse, grants: u8) -> DisplayResponse::Desktop(_), ) => grants == 0, ( - DisplayRequest::Desktop(DesktopRequest::ApplyWallpaper { .. } | DesktopRequest::RestoreComplete) + DisplayRequest::Desktop( + DesktopRequest::ApplyWallpaper { .. } + | DesktopRequest::RestoreComplete + | DesktopRequest::Pointer(_), + ) | DisplayRequest::PrepareSettings { .. }, DisplayResponse::Accepted, ) => grants == 0, @@ -1471,6 +1493,42 @@ mod tests { assert!(!reply_matches(&encode_req(&requests[4]), &reply, 0)); } + #[test] + fn a_pointer_preference_crosses_whole_and_nothing_else_does() { + use kore_rt::svc::SvcProtocol; + for acceleration in [false, true] { + for speed in PointerSpeed::ALL { + let request = DisplayRequest::Desktop(DesktopRequest::Pointer(PointerPreference { + acceleration, + speed, + })); + let wire = encode_req(&request); + assert_eq!(decode_req(&wire), Some(request)); + assert_eq!(WindowWire::request_grants(&wire), 0); + assert!(reply_matches(&wire, &DisplayResponse::Accepted, 0)); + assert!(!reply_matches(&wire, &DisplayResponse::Accepted, 1)); + } + } + let wire = encode_req(&DisplayRequest::Desktop(DesktopRequest::Pointer( + PointerPreference::default(), + ))); + assert_eq!(wire.b.get(), 2, "the default speed is the middle step"); + for (field, value) in [(0, 2), (1, 5), (1, u64::MAX), (2, 1), (3, 1)] { + let mut bad = wire; + match field { + 0 => bad.a = value.into(), + 1 => bad.b = value.into(), + 2 => bad.c = value.into(), + _ => bad.d = value.into(), + } + assert_eq!(decode_req(&bad), None, "field {field} = {value}"); + } + assert_eq!(PointerSpeed::new(3), None); + assert_eq!(PointerSpeed::new(-3), None); + assert_eq!(PointerSpeed::FASTEST.adjustment(), 1.0); + assert_eq!(PointerSpeed::SLOWEST.adjustment(), -1.0); + } + /// The capture vocabulary rides the controller face byte-exactly: a /// zero ticket, an unknown mode or refusal, and stray bits refuse. #[test] diff --git a/native/crates/window/src/desktop.rs b/native/crates/window/src/desktop.rs index 2a0a55f..9547f6e 100644 --- a/native/crates/window/src/desktop.rs +++ b/native/crates/window/src/desktop.rs @@ -18,6 +18,52 @@ impl Mode { } } } +/// How the pointer follows a mouse: Settings ▸ Keyboard & Pointer. A virtual +/// machine's emulated mouse follows its host's pointer instead, unchanged. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PointerPreference { + /// Faster motion moves the pointer further (libinput's adaptive profile). + /// Off, the pointer moves in proportion to the mouse at every speed. + pub acceleration: bool, + pub speed: PointerSpeed, +} +impl Default for PointerPreference { + fn default() -> Self { + Self { + acceleration: true, + speed: PointerSpeed::DEFAULT, + } + } +} +/// The pointer speed setting, in five steps from slowest to fastest. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct PointerSpeed(i8); +impl PointerSpeed { + pub const SLOWEST: Self = Self(-2); + pub const DEFAULT: Self = Self(0); + pub const FASTEST: Self = Self(2); + pub const ALL: [Self; 5] = [Self(-2), Self(-1), Self(0), Self(1), Self(2)]; + /// A step from -2 (slowest) through 0 (the default) to 2 (fastest). + pub const fn new(step: i8) -> Option { + if step < Self::SLOWEST.0 || step > Self::FASTEST.0 { + None + } else { + Some(Self(step)) + } + } + pub const fn step(self) -> i8 { + self.0 + } + /// libinput's speed adjustment for this step, in -1..=1. + pub fn adjustment(self) -> f64 { + f64::from(self.0) / 2.0 + } +} +impl Default for PointerSpeed { + fn default() -> Self { + Self::DEFAULT + } +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DesktopRequest { Query, @@ -36,6 +82,8 @@ pub enum DesktopRequest { ApplyWallpaper { surface: u64, }, + /// The seat's pointer preference, applied from every mouse's next report. + Pointer(PointerPreference), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct DesktopState { diff --git a/native/provenance.json b/native/provenance.json index 77c02e9..aeb1e1e 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,6 +1,6 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "cac17a02528e13f37a363ed4be71aca40f7adc70", + "revision": "a07b4eae68b45ffd740e31a9cc69e708b21ad467", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", @@ -234,7 +234,7 @@ "crates/rt/src/executor/time.rs": "62f9ca9e5e17b395ec2b19a52fc6ed1bb53fd4acc3554cfca81eaf20e48fc272", "crates/rt/src/exit.rs": "3bf7e8ce6dac63b4ce365bdf2e743a9fd171851f1c5e40f8afbb4548877cef1b", "crates/rt/src/fiber_stack.rs": "48bf4fedba07ea2826615368e22882301ae4e0bbce0d893d0d7754d86bd02ce6", - "crates/rt/src/futex.rs": "df5622a1bd811d7d97ec20dc44a568dba3a2c995254fc30405a60d41371e7390", + "crates/rt/src/futex.rs": "a3947ed8217f8c006ec8ba69d8d13a02dc01fd2afc8d1fe80af07342db246cd0", "crates/rt/src/heap/mod.rs": "a9c39d3de56187a14d4633a229cbbb6e7ddb2c438d6f9f154b3b2551031b9f81", "crates/rt/src/heap/pool.rs": "67d42c6ffc7c7e176d21e97dd3179a3e4ff331f46dd457ed4d29914b1a404de8", "crates/rt/src/heap/shard.rs": "d97f91b3ecc71e3ec56fde0017f9d98cb99c5fbeeffe6ff79ddd153f5c6690fa", @@ -272,7 +272,7 @@ "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", "crates/rt/src/thread.rs": "86278fc96aa71883905fbf1e851ea408c34256501b3712ba9459d98a16bc5131", - "crates/rt/src/thread_controller.rs": "298456742190090b14c718669b437906d1390037204e7d69a10877870029d63b", + "crates/rt/src/thread_controller.rs": "2f8e23fb8813bf5b17834ffdc6e4d3404d52e06f7a30ee3c92b4c4c76266d7dd", "crates/rt/src/tls.rs": "85bd95bafb3de73d249d7a2eb8779b5afc929077e27b4baa5f7c7428063063ab", "crates/rt/src/transport/mod.rs": "d5af890c800dd53abdbea8329383068cf43f0bf294ccb6b46dd7d8592ae5f321", "crates/rt/src/upcall.rs": "c0cbd2a59479dfba006850bb1bc5dd4f20f59317d178af776ba678f0e6404f02", @@ -292,12 +292,14 @@ "crates/uart/src/tx.rs": "532e0fef75563bf5294211502fc0da6534e51e96b4da0a71896fe629c3e5d7be", "crates/uart/src/uart16550.rs": "689427e689025653727db7bbc30a9eb4d0dbf12cc8202d4ae1aef1978f11555c", "crates/vdev-accel/Cargo.toml": "88d35110e153faa7dfe68089d901bddc07964c32b4f0767cb23c93497c0f018c", - "crates/vdev-accel/src/device.rs": "0e285a5c781dd515263e862fd7fd7689b991d9215734b8a88d38fb6613168da8", + "crates/vdev-accel/include/kore/vdev_accel.h": "a32f22733a794e39388c807c2d68499ba8b079a4f669f968f7aa7cbc085e22c9", + "crates/vdev-accel/src/device.rs": "547f83de4df91b7837b928710f556ddeb44d083778fa4ccb4ed0961295887843", "crates/vdev-accel/src/device/tests.rs": "43f1871d0965c75f9ff1be052e827c68e43b6c0ae6203a5451b2eedfce770e9e", "crates/vdev-accel/src/fp8.rs": "89d1a9e034706742ededf4fd7a89a56a3f64e9a9d03e53389914c959b760f088", "crates/vdev-accel/src/fp8/tables.rs": "d7f6f5d80428935cf5ff8a41c18cd5e20d477cca53a70a4ad711e10777ad4a14", "crates/vdev-accel/src/fp8/tests.rs": "2b4f879cada7a02ba986f00c54327d9c28a22199887e48e461c1d62d61cd7db4", "crates/vdev-accel/src/lib.rs": "3ee829e95ad07e54ebad67d6f0681c2406884ad5f946407f8f3ec3e8f3d5ed50", + "crates/vdev-accel/tests/c_header.rs": "ce59ebd4d70a5e76fd5d5ca4879b51e25f7ccae4e626fe15596dbc71505e8622", "crates/vdev-accel/tests/session.rs": "774c6a11da5c446d8685a7db3b3dea2aaf4f1139d7a70d4f90e581e2f6eca559", "crates/vdev/Cargo.toml": "ba7bf6f814d0b3a8aec36be489c95b33635ce267f7cb91185b1662e90098dbec", "crates/vdev/src/device.rs": "aee2413e3eb6710041c0c7b72fb943038625547a362dca43fb70c6e24eee1710", @@ -318,9 +320,9 @@ "crates/window/src/clipboard.rs": "c10c8d7d198b3562900e789077d16761db9cdb69e6425c02bb4a97991a44e55e", "crates/window/src/command.rs": "3ad4867ce62f3a430a68e4d105991229938e131e8cfde8afbbb0f522f99355d3", "crates/window/src/configure.rs": "5731662e919827aa6c9b189dce1d4ae06ebc7f7940c57fafd8662fc33bc46154", - "crates/window/src/control.rs": "3ab2400d32503f75b3419310049252167035179e76f8c838a89094d17946c9a5", + "crates/window/src/control.rs": "0d71431798232efc121bf706cc4aa1bb7a7aff9e0f3077410a5591cd1cbd03f1", "crates/window/src/cursor.rs": "2c320f4f19af4a5f9cd0d4059d1452661ac8cea23f0f6a44e0833cccd4d5f4bd", - "crates/window/src/desktop.rs": "2aca94383249fac9f770c4edae35f634ad89388c463b17e4484125190757532b", + "crates/window/src/desktop.rs": "49eb0f9bc0cad77b66b663b4e88dfc25cdced5d2a83c934197cb2b0c1b41d404", "crates/window/src/document.rs": "1c4fb5a4bd3ce170a8a79ea97e7efd9aa013b1d2678b5ca1dde06ea1508ddaca", "crates/window/src/event.rs": "cf6565ea882e6c77672f31f0b354b03c57f25591a558893bb9466070b3f503a4", "crates/window/src/face.rs": "1078d8664e6da529c9d4fd43a7e3df77f287d0061f418ed21859d289e86765a0", @@ -354,17 +356,17 @@ "services/block/src/service.rs": "191e9d887d3f3267d6e2aa96e81996f01321bb2febb85e68581159f6e098b4b5", "services/block/src/wire.rs": "a9d64aef9ec3d58d37ca55b841933fc51b76ab94e18b245278b4519e96c65939", "services/dagfs/Cargo.toml": "d229a126616b1d7638f0f5a6edb4ac37d9d116f32afaa1897ffe484fbb286e5e", - "services/dagfs/boot.toml": "8af7f22a788922f0fd91e39f701b2a8b36e209727a5feb340e649ef578468b86", + "services/dagfs/boot.toml": "170611236e40e25cc4b50cdc603a9c115380c9a9dd687f04802296d077ad5fab", "services/dagfs/examples/fsdemo.rs": "1666a14ef62fd4fc46f7f9efa2eaa5021ed79823f273a77f53758307f884e7d0", "services/dagfs/examples/fswire.rs": "994b97d699796b317741e2c87cbe812cc1776ddcd01173323e72a5f74c8ac267", "services/dagfs/examples/model_volume.rs": "0e3623e962da02b2126062a19d198b9e0a8f6e91896fd9a6f7d13f6f30ddc5f1", "services/dagfs/examples/volume_read.rs": "bd653b086745febd2b19ade1748207138f70d3165aebad0bc8ca66c1f5d8b9f3", - "services/dagfs/src/authority.rs": "3f924460db27d9d3632705261bd473a37b2e1ab3050c3d88c66d335897bd1e9f", + "services/dagfs/src/authority.rs": "0f0bca8a1ce15e9644ca927cfa7619b796bbe67320cee6bae84cedb2d1fec75f", "services/dagfs/src/bounds.rs": "c825fcab0f30777e772c21fcf68cd092b10cf57a741eac22747de87ec3742e87", "services/dagfs/src/client.rs": "3532931af9484f548f7ff60bfd58f56fe392d8a770779f766f4e0edb0580d7d4", "services/dagfs/src/digests.rs": "88a256596ab314714ef523a2bea44dd8d5265436b284534d25d4538535454f3a", "services/dagfs/src/durable.rs": "3caf0dc7bd6c7c13cde2c1a83f1db6d70169efb92f448b54ace80f07b6057456", - "services/dagfs/src/engine.rs": "e7cfda6e4f05a532cfc5bf15854b80254f64f40d25a6e7910bee38bfc0ba52c8", + "services/dagfs/src/engine.rs": "2bc5f89630479bb6f177c77726ce94eaf039ad3966bf48ea4b4fc9deb810bef6", "services/dagfs/src/error.rs": "0a0cac522b38baa34221adda4ccca010fd8119c5aaf1f7b6015ab7a5f602ac33", "services/dagfs/src/extraction.rs": "502c7ba6cc8a37016a6c4a43f46664c3f27fe4c01d2cac21f8eb2cc2591a531d", "services/dagfs/src/lib.rs": "34140702b546448b1b3e326b0112f1be23e1beeb13af836b7f5377b529bb4e76", @@ -381,7 +383,7 @@ "services/dagfs/src/router.rs": "a598a55029a034a95c14bcf242c1bcb86f3d80759dd797122d4f026af5bc0e26", "services/dagfs/src/search.rs": "98e3807b60e039082e2f16cd975d4637520642b380ea5ee3b7a25a4b843de448", "services/dagfs/src/service/contract.rs": "f978333c7da9cb7797e08b9c71690e09c5b1a58b7eedc3a48256671c7f7577d2", - "services/dagfs/src/service/mod.rs": "3403eaf8628cf0e0bc0ce6ff8d2d9ba7a80ea64d550e2822f680757a6d2672e2", + "services/dagfs/src/service/mod.rs": "ee433af7880efdae438fa922c8690a7c01e84415f575bc82924dfc361fe860ef", "services/dagfs/src/slot.rs": "e8f8751c09a4e6006bcaccd2d189cbb6762da1be6c4ea56bd331fbdc7f5cf827", "services/dagfs/src/watches.rs": "9ae8277cd4ea1eadbbc8cc992b53edb6e476f247cf208e43ea6e13af3702b74c", "services/dagfs/src/window.rs": "8be4bd5c8f1e51cfa5744801e12a9406e7dafb351d0340a625ca8b8bddeb3364", @@ -451,14 +453,14 @@ "services/output/src/service/mod.rs": "d80c3beb50979b3e1e500abdfa3c0b9d38e354c24e2e34aea443732abbead6f1", "services/output/src/wire.rs": "3752e994eb18714518facae1b78a9ee2b34f99aad8687653f0b89dc8fdeca621", "services/swap/Cargo.toml": "9c0af01403a6ad5a7f4ff69a02b18c8b1b2624a684814d6dc1ac07968c5bbe53", - "services/swap/boot.toml": "7bf8b67cf4b04c1fb1ce3e6c6c3bd4e6d0b180b5ab5ebff04d714809020c4389", + "services/swap/boot.toml": "a6f31118bac10c3d91bd836ec97c6e91cc02ad929f93bff53a9839e0c48c8e38", "services/swap/src/client.rs": "04f735d059b491765d5ba63a4f5f640332735bf336e62dcbae86c17647583e2c", "services/swap/src/extent.rs": "d6705f77951df1c42cda51fc4526ba6262cab6af4a0545405ce06d2d2e1db7b4", "services/swap/src/lib.rs": "883684df54a87ae66aa7895ea19a872cabc8ab7fd4aded4ea65b7e3dd51fe0cf", "services/swap/src/service/mod.rs": "34597aeb2af68085cf129ba6f6641fac52fd9d8fd9eca26dd264c9feadc77dd0", "services/swap/src/wire.rs": "ff1ab3e6b42214d0c009b8cb1bd07ffa5238348b63ca273337b427dd4637ca14", "services/vdev/Cargo.toml": "e2485be36449450884f91cb8f00741266ca5f2cab3eca9fa21b2bd3bc48bbc05", - "services/vdev/boot.toml": "21765495f72eb2f2950950dc6029028fcd61e061664dea3af503a8ce117c1231", + "services/vdev/boot.toml": "8cab4f0a2760b7f77b2e698b88952fab4d4b6173406adaae25f1e862c8a034ba", "services/vdev/src/client.rs": "1bcac2d3c2b8382afdb0d69ea51fbac9198be3c0f7cf71e2da76d331eb5e1002", "services/vdev/src/face.rs": "e9fc9cc2cf943d13ef9d9e792bee1e872f39dd6c7eb04c00a3cc4cac875fe589", "services/vdev/src/host.rs": "3c92e97f847393e88b62b2084ca43c8b3122733c5c8761ddd60dee014c087364", diff --git a/native/services/dagfs/boot.toml b/native/services/dagfs/boot.toml index 15b963a..e217395 100644 --- a/native/services/dagfs/boot.toml +++ b/native/services/dagfs/boot.toml @@ -47,5 +47,5 @@ exec_client = true # `dagfs_heap_supply_holds_every_session` holds this row to it; the rest is # room for the proofs open handles carry, allocated per open at their depth. heap_supply_bytes = 0x00D0_0000 -digest_x86_64 = "blake3:10d0cc3c987cd5303a7bdafe82961f3ce30ff9c067a21d96894a7c68fba5c291" -digest_aarch64 = "blake3:981d8318b9a9b415dcbc0e331aa984750134d35828ea499e0a4b53557aa4c548" +digest_x86_64 = "blake3:de7af3ff54ccb81f1ce36a5a0597e5ace873d6736b8bfccb99e9a55926a32f06" +digest_aarch64 = "blake3:3378f1908340b44c3f88420be7963589f3418069814c7e0b4b327ac6835c6498" diff --git a/native/services/dagfs/src/authority.rs b/native/services/dagfs/src/authority.rs index 82b38a5..ead1aa4 100644 --- a/native/services/dagfs/src/authority.rs +++ b/native/services/dagfs/src/authority.rs @@ -1370,6 +1370,54 @@ impl Authority { Ok(grants) } + /// The edge through which `holder` already holds `parent` whole: issued + /// by `issuer`, unnamed, with `parent`'s own root, rights and quota. + /// + /// A command enters a grant its task view holds through such an edge + /// (`Engine::open_task_grant`), and enters it again for every file it + /// reads beneath it. The edge says only "this session holds that grant"; + /// what one open may do is on its handle. So one edge serves every open, + /// and a session's edges follow the grants it enters, not how many times + /// it enters them. + /// + /// `parent` is a generational id, so an edge left by a grant since + /// retired and reissued cannot match; and that edge went with the + /// retired grant's closure anyway. + pub(crate) fn whole_projection( + &self, + holder: SessionId, + issuer: SessionId, + parent: GrantId, + source: &AuthorityEdge, + ) -> FsResult> { + let slot = self.session_slot(holder)?; + let Some(arena) = self.arena(slot) else { + return Ok(None); + }; + for index in 0..arena.edges.capacity() { + let Some(id) = arena.edges.live_id_at(index) else { + continue; + }; + let edge = arena.edges.try_read(id).map_err(|_| FsError::Busy)?; + if edge.holder() == holder + && edge.issuer() == issuer + && edge.parent() == Some(parent) + && edge.kind() == AuthorityKind::Grant + && edge.display_name().is_none() + && edge.root() == source.root() + && edge.rights() == source.rights() + && edge.quota() == source.quota() + { + return Ok(Some(GrantId::from_key(join( + slot, + id, + MAX_EDGES_PER_SESSION, + )))); + } + } + Ok(None) + } + /// `true` if `session` already sees an attachment named `name`. /// /// Used to keep a sender from overwriting a name in the recipient's view diff --git a/native/services/dagfs/src/engine.rs b/native/services/dagfs/src/engine.rs index 66a635d..84319ec 100644 --- a/native/services/dagfs/src/engine.rs +++ b/native/services/dagfs/src/engine.rs @@ -600,6 +600,24 @@ impl Engine

{ /// Open an attachment held by `view` into one transient command session. /// The command receives only a handle; the stable grant remains owned by /// the task view and therefore survives process teardown. + /// + /// The handle needs an edge the command holds to prove its route by, so + /// the grant is projected into the command's session: one edge, carrying + /// the grant whole, for as long as the session lives. **One per grant, + /// not one per open.** A command that reads a system view file by file — + /// a compiler and its headers — enters the same grant every time, and an + /// edge left behind by each entry spent the session's whole edge arena + /// (`MAX_EDGES_PER_SESSION`) on one view: the open after that refused + /// `OutOfEdges`, on a read. So an entry reuses the projection the session + /// already holds (`Authority::whole_projection`) and only the first + /// makes one. + /// + /// The projection carries the grant's own rights and the handle carries + /// what was asked for. That is not a widening: the request is attenuated + /// against the grant here, before any edge is touched, and again against + /// the projection when the handle is minted; a command could always ask + /// again for any right the grant conveys; and no request names a + /// projection, whose id never leaves the service. pub(crate) fn open_task_grant( &self, session: SessionId, @@ -613,11 +631,17 @@ impl Engine

{ } let root = edge.root(); let rights = edge.rights().attenuate(requested)?; - let quota = edge.quota(); + let standing = self.auth.whole_projection(session, view, grant, &edge)?; + let (whole, quota) = (edge.rights(), edge.quota()); drop(edge); + if let Some(projected) = standing { + // Other handles may hold this projection: a refused open leaves + // it where it was. + return self.open_authority_edge(session, projected, rights); + } let projected = self .auth - .insert_grant(view, session, grant, root, rights, quota, None)?; + .insert_grant(view, session, grant, root, whole, quota, None)?; self.ns.object_mut(root)?.grants_inc(); match self.open_authority_edge(session, projected, rights) { Ok(opened) => Ok(opened), diff --git a/native/services/dagfs/src/service/mod.rs b/native/services/dagfs/src/service/mod.rs index 51e04a5..10f0ee5 100644 --- a/native/services/dagfs/src/service/mod.rs +++ b/native/services/dagfs/src/service/mod.rs @@ -4447,6 +4447,103 @@ mod recyclable_session_tests { )); } + /// A command reads a system view file by file — a compiler and its + /// headers, each entered through the view's grant. Entering the grant + /// again must cost the session nothing it has not already spent: an edge + /// left behind per entry filled the session's edge arena, and the read + /// after that was refused `OutOfEdges`. + #[test] + fn entering_a_task_grant_again_adds_no_authority_edge() { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + assert!(service.establish_admin_share()); + assert!(service.establish_admin_workspace()); + let home = Name::new("terminal-0").unwrap(); + service + .register_private_invocation_view(0xA1, &home, Rights::HOME, None) + .unwrap(); + let baseline = service.engine.authority_edge_count(); + let command = admit_view(&mut service, 0x106, 0xA1, Rights::HOME); + let owner = service + .invocation_views + .iter() + .flatten() + .find(|view| view.scope == 0xA1) + .unwrap() + .task_session; + let share = service + .engine + .grants(owner) + .unwrap() + .into_iter() + .find(|grant| grant.display_name == Some(Name::new(SYSTEM_SHARE_VIEW).unwrap())) + .expect("the system share is a named grant of the task view"); + let attached = service.engine.authority_edge_count(); + + // A refused entry leaves nothing behind, before any projection exists. + assert!(matches!( + service + .engine + .open_task_grant(command, owner, share.grant, Rights::WRITE), + Err(FsError::AccessDenied) + )); + assert_eq!(service.engine.authority_edge_count(), attached); + + // The first entry projects the grant into the command's session… + let first = service + .engine + .open_task_grant(command, owner, share.grant, Rights::READ) + .unwrap(); + let projected = service.engine.authority_edge_count(); + assert_eq!(projected, attached + 1); + service.engine.close(command, first.handle).unwrap(); + + // …and every later one, far past a session's edge arena, reuses it. + for _ in 0..4 * crate::bounds::MAX_EDGES_PER_SESSION { + let opened = service + .engine + .open_task_grant(command, owner, share.grant, Rights::READ) + .expect("entering the same grant again"); + service.engine.close(command, opened.handle).unwrap(); + } + assert_eq!(service.engine.authority_edge_count(), projected); + + // Handles held together share it, each with the rights it asked for: + // the share is read-only, so neither can create beneath it. + let held = service + .engine + .open_task_grant(command, owner, share.grant, Rights::READ) + .unwrap(); + let beside = service + .engine + .open_task_grant(command, owner, share.grant, Rights::READ) + .unwrap(); + assert_ne!(held.handle, beside.handle); + assert_eq!(service.engine.authority_edge_count(), projected); + let note = Name::new("note").unwrap(); + assert!(matches!( + block_on( + service + .engine + .create(command, held.handle, ¬e, ObjectKind::File) + ), + Err(FsError::AccessDenied) + )); + // A refused entry beside live handles leaves their projection alone. + assert!(matches!( + service + .engine + .open_task_grant(command, owner, share.grant, Rights::WRITE), + Err(FsError::AccessDenied) + )); + assert!(service.engine.stat(command, held.handle).is_ok()); + assert!(service.engine.stat(command, beside.handle).is_ok()); + + // The command's end takes the projection with it. + assert!(service.peer_closed(0x106, PeerClose::Orderly)); + assert_eq!(service.engine.authority_edge_count(), baseline); + } + /// Recovery calls this on a volume that already carries the directory, so /// a second call must adopt the standing grant rather than mint a rival /// (or refuse on the taken display name) — otherwise the first reboot diff --git a/native/services/swap/boot.toml b/native/services/swap/boot.toml index 314360e..84d3cbb 100644 --- a/native/services/swap/boot.toml +++ b/native/services/swap/boot.toml @@ -9,5 +9,5 @@ image = "kore-svc-swap" endowment = "kore_svc_swap::ZramEndowment" protocol = "kore_svc_swap::ZramSwap" endows = ["MEM_SUPPLY@0"] -digest_x86_64 = "blake3:c6363f610b2e977482dca6d876e201b689667ba62f45052a6d2fc49ce425b754" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:7c9b860f6572ab954cebd6579ef63f38a9256e93206127f9cd0c1bfc93a72d7d" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:74c4de48188160a4554150b2ce1f04c0c7481aad6ddd932318db28cf8d15309f" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:f364cde3895e061a604739f9c4d951b2a6e42b956028be0e8043c26e61a3781a" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/vdev/boot.toml b/native/services/vdev/boot.toml index 9ff6c63..d91cbd9 100644 --- a/native/services/vdev/boot.toml +++ b/native/services/vdev/boot.toml @@ -26,5 +26,5 @@ protocol = "kore_svc_vdev::VdevService" endows = [] exec_client = true # pair 1 administers per-launch driver faces heap_supply_bytes = 0x00C0_0000 -digest_x86_64 = "blake3:33b4bd37c5e76b836ff3bb9f041b37184483f33ba7c37c4b04b09e11cc4b1b00" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:344e1f6da34c1ef515b0c0b9988b5e3b9e673cf08df6f6065ecd5bc661fe42de" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:845a6ad797b2f718a31345a4b89a7279be6a5e550dcd741e58dbe101e55f5717" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:5f28836ba4be218c10a700db505261c04d9801617a220520a8f3fbccb5e6df01" # res/aarch64/; make boot-digests-aarch64 diff --git a/tests/guest/thread_lifecycle.rs b/tests/guest/thread_lifecycle.rs index caad0e8..e447dfd 100644 --- a/tests/guest/thread_lifecycle.rs +++ b/tests/guest/thread_lifecycle.rs @@ -70,10 +70,12 @@ unsafe extern "C" fn worker(mode: *mut c_void) -> *mut c_void { 3 => { READY.store(1, Ordering::Release); loop { - // Every foreign wait entry checks cancellation. Independent - // controller workers have no doorbell: preserve Unsupported - // and park via the native deadline helper between retries. - // This witnesses cancellation delivery, not futex wake routing. + // Nothing wakes WORD: only the cancellation ends this wait. + // A worker with a doorbell parks until the request's wake, + // relayed through the controller, arrives. One founded + // without (a starter with no channel space) is refused the + // park: preserve Unsupported and take the native deadline + // park between retries, whose entry checks cancellation. if kore_futex_wait(WORD.as_ptr(), 0, KORE_FUTEX_INFINITE) == KoreFutexOutcome::Unsupported {