Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 36 additions & 16 deletions crates/ffi/src/futex.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ pub extern "C" fn kore_futex_available() -> u32 {
/// all waits/wakes. All concurrent accesses must be compatible 32-bit atomic
/// operations (C atomic builtins or Zig atomics); never mix ordinary accesses
/// or differently sized atomics. Managed-thread waits are cancellation points:
/// they may exit without stack unwinding, as specified by kore_thread_testcancel.
/// they may exit without stack unwinding, as specified by kore_thread_testcancel,
/// and a cancellation requested while the thread waits wakes it to act.
/// Publish the condition with a sequentially consistent atomic store BEFORE waking.
/// No address-space sharing is implied.
#[unsafe(no_mangle)]
Expand All @@ -77,32 +78,51 @@ pub unsafe extern "C" fn kore_futex_wait(
if !crate::thread_local::cancellable() {
return futex::wait(word, expected, timeout).into();
}
// A bounded native park makes cancellation observable even without a peer
// kick capability. Recompute against one deadline; slices never restart it.
unsafe { crate::kore_thread_testcancel() };
let start = match kore::time::uptime() {
Ok(time) => time.as_nanos(),
Err(_) => return KoreFutexOutcome::Unsupported,
let outcome = match crate::thread::slice() {
// A cancellation's wake reaches this thread (kore_thread_cancel), and
// the wait re-reads the request once it is registered: it parks until
// a wake, the caller's deadline or that request, and never polls.
None => futex::wait_unless(word, expected, timeout, crate::thread_local::cancel_pending),
Some(slice) => unsafe { sliced(word, expected, timeout_ns, slice) },
};
let deadline = timeout.map(|_| start.saturating_add(timeout_ns));
unsafe { crate::kore_thread_testcancel() };
outcome.into()
}

/// A cancellable wait on a thread the controller cannot wake: no wake would
/// announce a cancellation, so it parks one `slice` at a time and acts on a
/// request between slices, against one deadline that slices never restart.
/// # Safety
/// A managed thread's cancellation point, as kore_futex_wait.
unsafe fn sliced(
word: &AtomicU32,
expected: u32,
timeout_ns: u64,
slice: Duration,
) -> futex::WaitOutcome {
let Ok(start) = kore::time::uptime() else {
return futex::WaitOutcome::Unsupported;
};
let deadline =
(timeout_ns != KORE_FUTEX_INFINITE).then(|| start.as_nanos().saturating_add(timeout_ns));
let slice = u64::try_from(slice.as_nanos()).unwrap_or(u64::MAX);
loop {
unsafe { crate::kore_thread_testcancel() };
let now = match kore::time::uptime() {
Ok(time) => time.as_nanos(),
Err(_) => return KoreFutexOutcome::Unsupported,
let Ok(now) = kore::time::uptime() else {
return futex::WaitOutcome::Unsupported;
};
let slice = crate::thread_local::CANCEL_SLICE_NS;
let slice = deadline.map_or(slice, |end| end.saturating_sub(now).min(slice));
let outcome = futex::wait(word, expected, Some(Duration::from_nanos(slice)));
let bound = deadline.map_or(slice, |end| end.saturating_sub(now.as_nanos()).min(slice));
let outcome = futex::wait(word, expected, Some(Duration::from_nanos(bound)));
unsafe { crate::kore_thread_testcancel() };
if outcome != futex::WaitOutcome::TimedOut {
return outcome.into();
return outcome;
}
if let Some(end) = deadline {
match kore::time::uptime() {
Ok(time) if time.as_nanos() < end => {}
Ok(_) => return KoreFutexOutcome::TimedOut,
Err(_) => return KoreFutexOutcome::Unsupported,
Ok(_) => return futex::WaitOutcome::TimedOut,
Err(_) => return futex::WaitOutcome::Unsupported,
}
}
}
Expand Down
87 changes: 78 additions & 9 deletions crates/ffi/src/thread.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ pub(crate) struct Control {
outcome: AtomicU32,
retired: AtomicU32,
pub(crate) cancel: AtomicBool,
/// The native band the thread runs on, plus one; zero until it starts
/// (`futex::current_thread`). A canceller reads it after storing
/// `cancel`, to wake the thread out of a wait at a cancellation point.
band: AtomicU32,
}
impl Control {
pub(crate) fn new() -> Self {
Expand All @@ -62,13 +66,32 @@ impl Control {
outcome: AtomicU32::new(0),
retired: AtomicU32::new(0),
cancel: AtomicBool::new(false),
band: AtomicU32::new(0),
}
}
/// Record the calling thread's band, before its callback can reach a
/// cancellation point. `SeqCst`, against the canceller's store of `cancel`
/// and then this load: a canceller that reads zero stored `cancel` before
/// the thread's first wait reads it.
fn start(&self) {
let band = kore::futex::current_thread().map_or(0, |band| band as u32 + 1);
self.band.store(band, Ordering::SeqCst);
}
fn band(&self) -> Option<usize> {
let band = self.band.load(Ordering::SeqCst);
band.checked_sub(1).map(|band| band as usize)
}
pub(crate) fn finish(&self, value: *mut c_void, outcome: u32) {
self.result.store(value, Ordering::Relaxed);
self.outcome.store(outcome, Ordering::Release);
}
}
/// How long one wait of the calling thread may park ([`Controller::slice`]):
/// unbounded where the controller can wake it, and therefore where a
/// cancellation's wake reaches it; one bounded slice where nothing could.
pub(crate) fn slice() -> Option<core::time::Duration> {
THREADS.slice()
}
fn status(error: Error) -> i32 {
match error {
Error::Unavailable | Error::Full => KORE_THREAD_EAGAIN,
Expand All @@ -95,6 +118,7 @@ fn invoke(argument: usize) -> usize {
call_context(context)
}
fn call_context(context: &StartContext) -> usize {
context.control.start();
// SAFETY: forwarded foreign callback and argument lifetime contract.
crate::thread_local::enter(&context.control);
let value = unsafe { (context.call)(context.argument) };
Expand Down Expand Up @@ -217,12 +241,17 @@ pub unsafe extern "C" fn kore_thread_join(
if control.retired.load(Ordering::SeqCst) != 0 {
break;
}
// A cancellation request is a flag, not a wake: a joiner that can be
// cancelled parks one slice at a time, as a cancellable futex wait
// does. Any other joiner parks until the controller retires the target.
let slice = crate::thread_local::cancellable()
.then(|| core::time::Duration::from_nanos(crate::thread_local::CANCEL_SLICE_NS));
if kore::futex::wait(&control.retired, 0, slice) == kore::futex::WaitOutcome::Unsupported
// Park until the controller wakes `retired`, or this joiner's own
// cancellation is requested: its canceller stores the request, then
// wakes this thread (kore_thread_cancel), and the wait re-reads the
// request once it is registered. Only a joiner the controller cannot
// wake parks in bounded slices; nothing else would end its park.
if kore::futex::wait_unless(
&control.retired,
0,
slice(),
crate::thread_local::cancel_pending,
) == kore::futex::WaitOutcome::Unsupported
// A native worker with no doorbell takes the deadline park, which
// needs no wake capability at all.
&& !kore::runtime::pace::cede()
Expand Down Expand Up @@ -279,7 +308,12 @@ pub unsafe extern "C" fn kore_thread_detach(thread: *mut KoreThread) -> i32 {
}

/// Request deferred cancellation. The target acts at testcancel, join or foreign
/// futex wait. A disabled target retains the request. No asynchronous kill occurs.
/// futex wait; a target waiting at one is woken to act. A disabled target
/// retains the request. No asynchronous kill occurs. EAGAIN when no wake the
/// calling thread sends can reach the target, because it neither started the
/// controller nor was created through it (create refuses it too). Then no
/// request is recorded, since a target parked at a cancellation point would
/// never see it.
/// # Safety
/// thread is a live handle. Borrowing cancel calls may run concurrently with one
/// another, but must finish before a consuming join/detach can free the handle.
Expand All @@ -288,12 +322,31 @@ pub unsafe extern "C" fn kore_thread_cancel(thread: *mut KoreThread) -> i32 {
let Some(ticket) = NonNull::new(thread.cast::<Ticket>()) else {
return KORE_THREAD_EINVAL;
};
let control = unsafe { &ticket.as_ref().control };
// SAFETY: the caller supplies a live handle, borrowed for this call.
request_cancel(unsafe { &ticket.as_ref().control })
}

/// kore_thread_cancel's request against the target's shared control.
fn request_cancel(control: &Control) -> i32 {
if control.outcome.load(Ordering::Acquire) != 0 || control.retired.load(Ordering::Acquire) != 0
{
return KORE_THREAD_ESRCH;
}
control.cancel.store(true, Ordering::Release);
// The request is a flag its wake announces: only the controller holds a
// send into a managed thread's doorbell, so the wake is relayed through
// it, and a thread that cannot relay through it is refused here rather
// than leaving a request behind a park nothing ends.
if !THREADS.wakes_threads() {
return KORE_THREAD_EAGAIN;
}
// Store, then wake (`futex::wake_thread`'s order). The wake reads the
// target's registration, and the target's wait reads the request once it
// is registered, so one of the two sees the other. A target that has not
// yet recorded its band reads the request at its first cancellation point.
control.cancel.store(true, Ordering::SeqCst);
if let Some(band) = control.band() {
let _ = kore::futex::wake_thread(band);
}
KORE_THREAD_OK
}

Expand Down Expand Up @@ -322,6 +375,22 @@ mod tests {
assert_eq!(Arc::strong_count(&control), 1);
}
#[test]
fn a_cancel_no_wake_can_announce_is_refused_and_never_recorded() {
// Off metal the controller never starts, so no thread's wake reaches
// its threads: the request is refused, and nothing a parked target
// would wait on unwoken is left behind. An ended target is gone first.
let control = Control::new();
control.start();
assert_eq!(control.band(), None, "off metal a thread has no band");
assert_eq!(request_cancel(&control), KORE_THREAD_EAGAIN);
assert!(!control.cancel.load(Ordering::SeqCst));
control.finish(ptr::null_mut(), 1);
assert_eq!(request_cancel(&control), KORE_THREAD_ESRCH);
assert!(!control.cancel.load(Ordering::SeqCst));
// Every wait off metal is one the controller cannot end early.
assert!(slice().is_some());
}
#[test]
fn host_refusal_preserves_output_and_never_calls_foreign_code() {
unsafe extern "C" fn never(_: *mut c_void) -> *mut c_void {
panic!("host ran native callback")
Expand Down
18 changes: 11 additions & 7 deletions crates/ffi/src/thread_local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,6 @@ pub const KORE_THREAD_CANCEL_ENABLE: i32 = 0;
pub const KORE_THREAD_CANCEL_DISABLE: i32 = 1;
pub const KORE_THREAD_CANCEL_DEFERRED: i32 = 0;
pub const KORE_THREAD_CANCEL_ASYNCHRONOUS: i32 = 1;
/// How long one park lasts at a cancellation point that waits: a request is
/// a flag no wake carries, so a cancellable wait re-checks it this often.
/// One millisecond is the cadence the native runtime parks an unreachable
/// waiter at (`kore-rt`'s `pace::CEDE_NS`): a cancellation is acted on within
/// one, and an idle cancellable waiter wakes no more often than that.
pub(crate) const CANCEL_SLICE_NS: u64 = 1_000_000;
pub type KoreThreadKey = u64;
pub type KoreThreadDestructor = Option<unsafe extern "C" fn(*mut c_void)>;

Expand Down Expand Up @@ -85,6 +79,16 @@ pub(crate) fn leave() {
pub(crate) fn cancellable() -> bool {
!CURRENT.get().is_null() && !DISABLED.get() && !FINISHING.get()
}
/// An enabled cancellation request is pending for this managed thread: what
/// testcancel acts on, and the stop a cancellation point's wait re-reads once
/// it is registered (`futex::wait_unless`). `SeqCst`, as the canceller's store
/// is: the canceller stores, then reads this thread's registration before it
/// wakes it, so one of the two sees the other.
pub(crate) fn cancel_pending() -> bool {
// SAFETY: CURRENT is non-null while cancellable, and points at this
// managed thread's Control until `leave`.
cancellable() && unsafe { &*CURRENT.get() }.cancel.load(Ordering::SeqCst)
}
fn active() -> bool {
!CURRENT.get().is_null()
}
Expand Down Expand Up @@ -368,7 +372,7 @@ pub unsafe extern "C" fn kore_thread_setcanceltype(kind: i32, old_type: *mut i32
/// never cross live Rust RAII guards or executor callbacks at a cancellation point.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn kore_thread_testcancel() {
if cancellable() && unsafe { &*CURRENT.get() }.cancel.load(Ordering::Acquire) {
if cancel_pending() {
unsafe {
kore_thread_exit(kore_thread_canceled());
}
Expand Down
Loading
Loading