From 1d8248722a840fd5183653e3e8dbe6982ea93ada Mon Sep 17 00:00:00 2001 From: atsushi421 Date: Fri, 13 Nov 2026 14:20:12 +0900 Subject: [PATCH] refactor(sched): drop the clustered scheduler abstraction After the merge of GEDF into EDF, the EDF scheduler is the only scheduler whose tasks carry a CPU set. The clustered abstraction generalized over a family with one member, and its names called global EDF tasks "clustered". - get_next_task() calls edf::SCHEDULER directly and then tries PRIORITY_LIST[1..]. A compile-time assertion checks that the EDF scheduler is first in PRIORITY_LIST. This replaces is_clustered(), get_num_clustered_schedulers(), and the prefix assertion. - Scheduler::queued_cpu_mask() becomes an inherent method of EDFScheduler, and wake_workers() calls it directly. This removes clustered_queued_cpu_mask(). - ClusteredTask moves into edf.rs as the private QueuedTask, and NUM_CLUSTERED_TASKS_IN_QUEUE becomes NUM_EDF_TASKS_IN_QUEUE. - Comments and the mdbook use EDF terms. The behavior does not change. Signed-off-by: atsushi421 --- awkernel_async_lib/src/scheduler.rs | 135 +++--------------------- awkernel_async_lib/src/scheduler/edf.rs | 76 ++++++++++--- awkernel_async_lib/src/task.rs | 22 ++-- mdbook/src/internal/scheduler.md | 6 +- 4 files changed, 92 insertions(+), 147 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 064b3f88e..bb6816fa0 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -19,7 +19,7 @@ use awkernel_lib::{ #[cfg(not(feature = "std"))] use alloc::boxed::Box; -mod edf; +pub(super) mod edf; pub(super) mod panicked; mod prioritized_fifo; mod prioritized_rr; @@ -118,14 +118,6 @@ impl SchedulerType { _ => None, } } - - /// True if this is a clustered scheduler, i.e. one whose tasks carry a - /// CPU affinity set. Update this function when adding a new clustered - /// scheduler; it is the single source of truth for the clustered prefix - /// of `PRIORITY_LIST`. - pub const fn is_clustered(&self) -> bool { - matches!(self, SchedulerType::EDF(_, _)) - } } /// # Priority @@ -147,31 +139,12 @@ static PRIORITY_LIST: [SchedulerType; 4] = [ SchedulerType::Panicked, ]; -/// Return the number of clustered schedulers in `PRIORITY_LIST`. -const fn get_num_clustered_schedulers() -> usize { - let mut count = 0; - while count < PRIORITY_LIST.len() { - if PRIORITY_LIST[count].is_clustered() { - count += 1; - } else { - break; - } - } - count -} - -// `get_next_task` relies on the clustered schedulers forming a strict prefix -// of `PRIORITY_LIST`. Reject any reorder at compile time. -const _: () = { - let mut i = get_num_clustered_schedulers(); - while i < PRIORITY_LIST.len() { - assert!( - !PRIORITY_LIST[i].is_clustered(), - "clustered schedulers must form a strict prefix of PRIORITY_LIST" - ); - i += 1; - } -}; +// `get_next_task` tries the EDF scheduler before `PRIORITY_LIST[1..]`, so the +// EDF scheduler must have the highest priority. +const _: () = assert!( + matches!(PRIORITY_LIST[0], SchedulerType::EDF(_, _)), + "the EDF scheduler must be first in PRIORITY_LIST" +); /// When false, no scheduler requests preemption, and `wake_task` enqueues the woken task. /// `no_preempt` compiles out the IPI handler that runs preemption-pending tasks. @@ -183,17 +156,6 @@ pub(crate) const PREEMPTION_ENABLED: bool = cfg!(not(feature = "no_preempt")); /// the decision to preempt, dequeuing, enqueuing, and updating of RUNNING must be executed exclusively. static GLOBAL_WAKE_GET_MUTEX: Mutex<()> = Mutex::new(()); -/// # Implementing a clustered scheduler -/// -/// A clustered scheduler keeps tasks that may run only on a subset of CPUs. -/// Every clustered scheduler must: -/// -/// 1. Wrap each entry of its run queue in [`ClusteredTask`] so that -/// `NUM_CLUSTERED_TASKS_IN_QUEUE` is kept consistent. -/// 2. Implement [`Scheduler::queued_cpu_mask`] to report the exact set of -/// CPUs its queued tasks are runnable on (`wake_workers` relies on it). -/// 3. Be placed in the clustered prefix of `PRIORITY_LIST` and be matched by -/// `SchedulerType::is_clustered` (checked at compile time). pub(crate) trait Scheduler { /// Enqueue an executable task. /// The enqueued task will be taken by `get_next()`. @@ -207,23 +169,6 @@ pub(crate) trait Scheduler { #[allow(dead_code)] // TODO: to be removed fn priority(&self) -> u8; - - /// The set of CPUs that have at least one queued task eligible to run on - /// them. Non-clustered schedulers use the default (empty) implementation; - /// clustered schedulers must override it. - fn queued_cpu_mask(&self) -> CpuSet { - CpuSet::empty() - } -} - -/// Return the union of [`Scheduler::queued_cpu_mask`] over all clustered -/// schedulers: the exact set of CPUs for which a clustered task is queued. -pub(crate) fn clustered_queued_cpu_mask() -> CpuSet { - let mut mask = CpuSet::empty(); - for scheduler_type in &PRIORITY_LIST[..get_num_clustered_schedulers()] { - mask = mask.union(get_scheduler(scheduler_type).queued_cpu_mask()); - } - mask } /// Get the next executable task. @@ -232,28 +177,22 @@ pub(crate) fn get_next_task(execution_ensured: bool) -> Option> { let mut node = MCSNode::new(); let _guard = GLOBAL_WAKE_GET_MUTEX.lock(&mut node); - let num_clustered_tasks = crate::task::NUM_CLUSTERED_TASKS_IN_QUEUE.load(Ordering::Relaxed); - - // The counter is global: it may be non-zero even when no queued clustered - // task is runnable on this CPU. That is fine because the run queues fail - // fast (`pop_for_cpu` checks the root affinity mask in O(1)). - if num_clustered_tasks > 0 { - let task = PRIORITY_LIST[..get_num_clustered_schedulers()] - .iter() - .find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured)); - - // The counter is decremented by the ClusteredTask::take() call inside + // The counter may be non-zero even when no queued EDF task is runnable on + // this CPU. That is fine because the run queue fails fast (`pop_for_cpu` + // checks the root affinity mask in O(1)). + if crate::task::NUM_EDF_TASKS_IN_QUEUE.load(Ordering::Relaxed) > 0 { + // The counter is decremented by the QueuedTask::take() call inside // get_next() (Drop is a no-op once take() has run). - if task.is_some() { - return task; + if let Some(task) = edf::SCHEDULER.get_next(execution_ensured) { + return Some(task); } } - // Skip the clustered prefix: it was either just tried above and returned + // Skip the EDF scheduler: it was either just tried above and returned // None (the queue state cannot change in between because both `wake_task` // and `get_next_task` hold `GLOBAL_WAKE_GET_MUTEX`), or the counter was 0 - // and its queues are empty. - PRIORITY_LIST[get_num_clustered_schedulers()..] + // and its queue is empty. + PRIORITY_LIST[1..] .iter() .find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured)) } @@ -283,46 +222,6 @@ pub const fn get_priority(sched_type: &SchedulerType) -> u8 { panic!("Scheduler type not registered in PRIORITY_LIST or equals()") } -/// RAII wrapper representing one slot in `NUM_CLUSTERED_TASKS_IN_QUEUE`. -/// -/// Constructing a `ClusteredTask` increments the counter. The counter is -/// decremented exactly once — either via [`take`] (explicit ownership -/// transfer) or via `Drop` (e.g. when a terminated task is discarded). If -/// `take` has already been called, `Drop` is a no-op. -/// -/// [`take`]: ClusteredTask::take -pub(crate) struct ClusteredTask { - inner: Option, -} - -impl ClusteredTask { - pub(crate) fn new(inner: T) -> Self { - crate::task::NUM_CLUSTERED_TASKS_IN_QUEUE.fetch_add(1, Ordering::Relaxed); - Self { inner: Some(inner) } - } - - /// Take the inner value and decrement the counter. - /// - /// Returns `None` if the value has already been taken. - /// After this call `Drop` will be a no-op. - pub(crate) fn take(&mut self) -> Option { - let val = self.inner.take(); - if val.is_some() { - crate::task::NUM_CLUSTERED_TASKS_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); - } - val - } -} - -impl Drop for ClusteredTask { - fn drop(&mut self) { - // Decrement only if take() has not been called yet. - if self.inner.is_some() { - crate::task::NUM_CLUSTERED_TASKS_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); - } - } -} - /// Maintain sleeping tasks by a delta list. struct SleepingTasks { delta_list: DeltaList>, diff --git a/awkernel_async_lib/src/scheduler/edf.rs b/awkernel_async_lib/src/scheduler/edf.rs index d7811a52d..94223aa15 100644 --- a/awkernel_async_lib/src/scheduler/edf.rs +++ b/awkernel_async_lib/src/scheduler/edf.rs @@ -10,11 +10,12 @@ use super::{Scheduler, SchedulerType, Task}; use crate::{ dag::calculate_and_update_dag_deadline, scheduler::{ - get_priority, peek_preemption_pending, push_preemption_pending, ClusteredTask, - GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED, + get_priority, peek_preemption_pending, push_preemption_pending, GLOBAL_WAKE_GET_MUTEX, + PREEMPTION_ENABLED, }, task::{ - get_task, get_task_running, set_current_task, set_need_preemption, State, MAX_TASK_PRIORITY, + get_task, get_task_running, set_current_task, set_need_preemption, State, + MAX_TASK_PRIORITY, NUM_EDF_TASKS_IN_QUEUE, }, }; use affinity_btree_queue::{AffinityBTreeQueue, DEFAULT_MIN_DEGREE}; @@ -23,12 +24,13 @@ use awkernel_lib::{ cpu::{masked_workers, num_cpu, CpuSet, CPU_SET_WORDS}, sync::mutex::{MCSNode, Mutex}, }; +use core::sync::atomic::Ordering; /// The run queue. Priority is `(absolute_deadline, wake_time)`; smaller values /// dequeue first, and entries with fully equal keys are FIFO-ordered by the /// queue's internal sequence number. type EDFQueue = - AffinityBTreeQueue<(u64, u64), ClusteredTask>, DEFAULT_MIN_DEGREE, CPU_SET_WORDS>; + AffinityBTreeQueue<(u64, u64), QueuedTask>, DEFAULT_MIN_DEGREE, CPU_SET_WORDS>; pub struct EDFScheduler { // `AffinityBTreeQueue::new` is not a const fn, so the queue is lazily @@ -37,6 +39,46 @@ pub struct EDFScheduler { priority: u8, } +/// RAII wrapper representing one slot in `NUM_EDF_TASKS_IN_QUEUE`. +/// +/// Constructing a `QueuedTask` increments the counter. The counter is +/// decremented exactly once — either via [`take`] (explicit ownership +/// transfer) or via `Drop` (e.g. when a terminated task is discarded). If +/// `take` has already been called, `Drop` is a no-op. +/// +/// [`take`]: QueuedTask::take +struct QueuedTask { + inner: Option, +} + +impl QueuedTask { + fn new(inner: T) -> Self { + NUM_EDF_TASKS_IN_QUEUE.fetch_add(1, Ordering::Relaxed); + Self { inner: Some(inner) } + } + + /// Take the inner value and decrement the counter. + /// + /// Returns `None` if the value has already been taken. + /// After this call `Drop` will be a no-op. + fn take(&mut self) -> Option { + let val = self.inner.take(); + if val.is_some() { + NUM_EDF_TASKS_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); + } + val + } +} + +impl Drop for QueuedTask { + fn drop(&mut self) { + // Decrement only if take() has not been called yet. + if self.inner.is_some() { + NUM_EDF_TASKS_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); + } + } +} + impl Scheduler for EDFScheduler { fn wake_task(&self, task: Arc) { let (wake_time, absolute_deadline) = { @@ -83,7 +125,7 @@ impl Scheduler for EDFScheduler { .push( (absolute_deadline, wake_time), cpu_set, - ClusteredTask::new(task), + QueuedTask::new(task), ) .expect("EDF: failed to push a task"); drop(data); @@ -104,7 +146,7 @@ impl Scheduler for EDFScheduler { // Note: spawn removes CPU 0 from every set, so `pop_for_cpu(0)` always // returns `None` and the `get_next_task(false)` path executed on the - // primary CPU can never dequeue a clustered task. + // primary CPU can never dequeue an EDF task. loop { let (_, _, mut entry) = queue.pop_for_cpu(cpu_id)?; @@ -141,8 +183,17 @@ impl Scheduler for EDFScheduler { fn priority(&self) -> u8 { self.priority } +} - fn queued_cpu_mask(&self) -> CpuSet { +pub static SCHEDULER: EDFScheduler = EDFScheduler { + data: Mutex::new(None), + priority: get_priority(&SchedulerType::EDF(0, CpuSet::empty())), +}; + +impl EDFScheduler { + /// The set of CPUs that have at least one queued task eligible to run on + /// them. + pub(crate) fn queued_cpu_mask(&self) -> CpuSet { let mut node = MCSNode::new(); let data = self.data.lock(&mut node); @@ -153,14 +204,7 @@ impl Scheduler for EDFScheduler { None => CpuSet::empty(), } } -} -pub static SCHEDULER: EDFScheduler = EDFScheduler { - data: Mutex::new(None), - priority: get_priority(&SchedulerType::EDF(0, CpuSet::empty())), -}; - -impl EDFScheduler { fn invoke_preemption(&self, task: Arc) -> bool { if !PREEMPTION_ENABLED { return false; @@ -221,8 +265,8 @@ impl EDFScheduler { // NOTE: The task is now pinned to `victim_cpu`'s preemption-pending // heap and is not in the affinity queue, so it is invisible to the - // other CPUs in its `cpu_set` (neither `pop_for_cpu`, the clustered - // task counter, nor `queued_cpu_mask` sees it). If another CPU in + // other CPUs in its `cpu_set` (neither `pop_for_cpu`, + // `NUM_EDF_TASKS_IN_QUEUE`, nor `queued_cpu_mask` sees it). If another CPU in // the set is or becomes idle while the IPI is in flight, it cannot pick // the task up and may go to sleep — a latency window bounded by the // IPI delivery plus the victim's interrupt-disabled sections. This diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 7ac806ba5..332ff7443 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -51,16 +51,16 @@ static TASKS: Mutex = Mutex::new(Tasks::new()); // Set of tasks. static RUNNING: [AtomicU32; NUM_MAX_CPU] = array![_ => AtomicU32::new(0); NUM_MAX_CPU]; // IDs of running tasks. pub(crate) static MAX_TASK_PRIORITY: u64 = (1 << 56) - 1; // Maximum task priority. -/// Total number of tasks in the queues of the non-clustered schedulers. +/// Total number of tasks in the queues of the schedulers other than EDF. /// Each of these schedulers increments it when it pushes a task to its queue /// and decrements it when it pops one, so it counts every path that calls /// `wake_task`, including the re-wakes that bypass `ArcWake::wake`. pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); -/// Total number of tasks in the clustered schedulers' queues. +/// Total number of tasks in the queue of the EDF scheduler. /// Which CPUs those tasks are runnable on is tracked exactly by the run -/// queues themselves; see `scheduler::clustered_queued_cpu_mask`. -pub(crate) static NUM_CLUSTERED_TASKS_IN_QUEUE: AtomicU32 = AtomicU32::new(0); +/// queue itself; see `EDFScheduler::queued_cpu_mask`. +pub(crate) static NUM_EDF_TASKS_IN_QUEUE: AtomicU32 = AtomicU32::new(0); static PREEMPTION_REQUEST: [AtomicBool; NUM_MAX_CPU] = array![_ => AtomicBool::new(false); NUM_MAX_CPU]; @@ -73,7 +73,7 @@ pub struct Task { pub info: Mutex, scheduler: &'static dyn Scheduler, pub priority: PriorityInfo, - pub cpu_set: Option, // The set of cores on which the task may run in clustered scheduling. None if not assigned. + pub cpu_set: Option, // The set of cores on which an EDF task may run. None for the other schedulers. } impl Task { @@ -306,7 +306,7 @@ impl Tasks { // Validate and normalise the CPU set before creating TaskInfo so that // info.scheduler_type and task.cpu_set stay in sync. let cpu_set = if let SchedulerType::EDF(deadline, set) = scheduler_type { - // CPU 0 is the primary core and cannot run clustered tasks. + // CPU 0 is the primary core and cannot run EDF tasks. let masked = masked_workers(set, num_cpu()); let normalized = if masked.is_empty() { // `cpu::sanity_check()` guarantees num_cpu() >= 2 at boot, @@ -1663,18 +1663,18 @@ pub fn wake_workers() { let num_cpus = awkernel_lib::cpu::num_cpu(); let mut num_tasks = NUM_TASK_IN_QUEUE.load(Ordering::Relaxed); - // The exact set of CPUs for which a clustered task is queued, read from - // the clustered schedulers' run queues. - let clustered_mask = crate::scheduler::clustered_queued_cpu_mask(); + // The exact set of CPUs for which an EDF task is queued, read from the + // run queue of the EDF scheduler. + let edf_mask = crate::scheduler::edf::SCHEDULER.queued_cpu_mask(); for i in 1..num_cpus { - if clustered_mask.contains(i) { + if edf_mask.contains(i) { awkernel_lib::cpu::wake_cpu(i); continue; } // Even if there are no global tasks left, keep scanning: a - // higher-numbered CPU may still have clustered tasks waiting and + // higher-numbered CPU may still have EDF tasks waiting and // skipping the rest would leave it asleep (lost wakeup). if num_tasks == 0 { continue; diff --git a/mdbook/src/internal/scheduler.md b/mdbook/src/internal/scheduler.md index 9ce66e97d..79eed15ee 100644 --- a/mdbook/src/internal/scheduler.md +++ b/mdbook/src/internal/scheduler.md @@ -81,9 +81,11 @@ An idle core in the set does not stop the preemption. A task queued before or af `get_next()` pops the earliest-deadline task whose `cpu_set` contains the calling CPU (`pop_for_cpu`), so a task is only ever dequeued by a core within its set. This guarantees CPU affinity. CPU 0 (the primary core) is always excluded from `cpu_set` when a task is spawned. The `cpu_set` is normalized at spawn time by removing CPU 0 and out-of-range bits; if this leaves the set empty (for example an empty set, or one naming only CPU 0 or out-of-range cores), the task is not rejected but falls back to all worker cores (`1..num_cpu()`) with a warning. -Bookkeeping for sleeping workers uses two pieces of state. A single global counter (`NUM_CLUSTERED_TASKS_IN_QUEUE`) tracks how many tasks are queued across all clustered schedulers; it is maintained by the `ClusteredTask` RAII wrapper (incremented on enqueue, decremented exactly once on dequeue or drop) and lets `get_next_task()` skip the clustered schedulers entirely when it is zero. The per-CPU information — which CPUs actually have an eligible task queued — is not duplicated in counters: it is read directly from the run queue, whose B-tree root already maintains the OR of all queued affinities (`affinity_mask()`, O(1)). `wake_workers()` obtains this set via `Scheduler::queued_cpu_mask()` (unioned over all clustered schedulers by `clustered_queued_cpu_mask()`) and wakes exactly the cores that have an eligible clustered task. +Bookkeeping for sleeping workers uses two pieces of state. A single global counter (`NUM_EDF_TASKS_IN_QUEUE`) tracks how many tasks the EDF scheduler queues. The `QueuedTask` RAII wrapper increments it on enqueue and decrements it exactly once on dequeue or drop. If the counter is zero, `get_next_task()` skips the EDF scheduler. -A new clustered scheduler must (1) wrap its queue entries in `ClusteredTask`, (2) implement `Scheduler::queued_cpu_mask()`, and (3) be placed in the clustered prefix of `PRIORITY_LIST` and matched by `SchedulerType::is_clustered()`; the prefix requirement is enforced at compile time. +The counter does not record which CPUs have an eligible task queued. The B-tree root of the run queue maintains the OR of all queued affinities (`affinity_mask()`, O(1)). `wake_workers()` reads this set through `EDFScheduler::queued_cpu_mask()` and wakes exactly the cores that have an eligible EDF task. + +`get_next_task()` tries the EDF scheduler before the schedulers in `PRIORITY_LIST[1..]`. So the EDF scheduler must be first in `PRIORITY_LIST`, and a compile-time assertion checks it. ### PrioritizedFIFO Scheduler