From 9176f0f59f3f0cb9db3af6080449208580ee8444 Mon Sep 17 00:00:00 2001 From: atsushi421 Date: Fri, 13 Nov 2026 14:06:30 +0900 Subject: [PATCH 1/3] fix(sched): preempt when queued EDF tasks claim every idle core Before this change, the EDF scheduler did not preempt if one core in the set of the woken task was idle. An idle core takes only one task, and it stays idle until the primary CPU wakes it. If several tasks woke in that window, all of them were enqueued, and only one of them got the idle core. The others waited behind running tasks with later deadlines. Now the scheduler skips preemption only if an idle core takes the task. An idle core takes the earliest queued task runnable on it. So it takes the woken task if the task is earlier than every queued task runnable on that core, or if the queue holds fewer tasks than the set has idle cores. Otherwise, the scheduler chooses a victim among the busy cores of the set. Signed-off-by: atsushi421 --- awkernel_async_lib/src/scheduler/edf.rs | 113 +++++++++++++++++++++--- mdbook/src/internal/scheduler.md | 2 +- 2 files changed, 100 insertions(+), 15 deletions(-) diff --git a/awkernel_async_lib/src/scheduler/edf.rs b/awkernel_async_lib/src/scheduler/edf.rs index 065a880e8..f11af72e5 100644 --- a/awkernel_async_lib/src/scheduler/edf.rs +++ b/awkernel_async_lib/src/scheduler/edf.rs @@ -75,7 +75,7 @@ impl Scheduler for EDFScheduler { let mut node = MCSNode::new(); let _guard = GLOBAL_WAKE_GET_MUTEX.lock(&mut node); - if !self.invoke_preemption(task.clone()) { + if !self.invoke_preemption(task.clone(), (absolute_deadline, wake_time)) { let mut node_inner = MCSNode::new(); let mut data = self.data.lock(&mut node_inner); let queue = data.get_or_insert_with(|| EDFQueue::new(num_cpu())); @@ -161,31 +161,45 @@ pub static SCHEDULER: EDFScheduler = EDFScheduler { }; impl EDFScheduler { - fn invoke_preemption(&self, task: Arc) -> bool { + /// `key` is the queue priority of `task`, `(absolute_deadline, wake_time)`. + fn invoke_preemption(&self, task: Arc, key: (u64, u64)) -> bool { if !PREEMPTION_ENABLED { return false; } let cpu_set = task.cpu_set.expect("Task has no CPU set"); - // Find the CPU whose target (running or pending-preemption) task has - // the lowest priority among the set. - let mut victim: Option<(usize, Arc)> = None; - + let mut idle_cpus = CpuSet::empty(); for cpu_id in cpu_set.iter() { - let task_running = get_task_running(cpu_id); + match get_task_running(cpu_id).task_id { + // If the task has already been running, preempt is not required. + id if id == task.id => return false, + 0 => idle_cpus.insert(cpu_id), + _ => (), + } + } - // If the task has already been running, preempt is not required. - if task_running.task_id == task.id { + if !idle_cpus.is_empty() { + let mut node = MCSNode::new(); + let data = self.data.lock(&mut node); + if (*data) + .as_ref() + .is_none_or(|queue| idle_cpu_takes(queue, idle_cpus, key)) + { return false; } + } - // An idle CPU in the set will pick the task up; no preemption needed. - if task_running.task_id == 0 { - return false; + // Find the CPU whose target (running or pending-preemption) task has + // the lowest priority among the busy CPUs of the set. + let mut victim: Option<(usize, Arc)> = None; + + for cpu_id in cpu_set.iter() { + if idle_cpus.contains(cpu_id) { + continue; } - let task_running = get_task(task_running.task_id); + let task_running = get_task(get_task_running(cpu_id).task_id); let task_pending = peek_preemption_pending(cpu_id); let target_task = match (task_running, task_pending) { @@ -221,7 +235,7 @@ impl EDFScheduler { // is not a lost wakeup: the task is eventually run either by the // victim's preemption handler (`do_preemption`), or re-woken by the // victim's `run_main` if its running task finishes first, in which - // case the idle check above routes it to the queue. Future work: + // case this function decides again. Future work: // an enqueue-instead-of-pin strategy (push to the affinity queue // and treat the IPI as a hint) would close this window, but it // requires redesigning the duplicate-IPI suppression that the @@ -233,3 +247,74 @@ impl EDFScheduler { false } } + +/// Whether a CPU of `idle_cpus` takes a task of priority `key` if the task is +/// pushed to `queue` now. +/// +/// An idle CPU takes the earliest queued task runnable on it. So the task is +/// taken if it is earlier than every queued task runnable on one of +/// `idle_cpus`, or if `queue` holds fewer tasks than `idle_cpus` holds CPUs. +/// Otherwise, the queued tasks can take every idle CPU first, and the task +/// must preempt to run now. +fn idle_cpu_takes( + queue: &AffinityBTreeQueue<(u64, u64), V, DEFAULT_MIN_DEGREE, CPU_SET_WORDS>, + idle_cpus: CpuSet, + key: (u64, u64), +) -> bool { + queue.len() < idle_cpus.iter().count() + || idle_cpus.iter().any(|cpu_id| { + queue + .peek_key_for_cpu(cpu_id) + .is_none_or(|queued| queued.priority > key) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + type Queue = AffinityBTreeQueue<(u64, u64), (), DEFAULT_MIN_DEGREE, CPU_SET_WORDS>; + + const IDLE_2: CpuSet = CpuSet::empty().with(2); + const ALL: CpuSet = CpuSet::empty().with(1).with(2).with(3); + + #[test] + fn idle_cpu_takes_task_from_empty_queue() { + assert!(idle_cpu_takes(&Queue::new(4), IDLE_2, (10, 0))); + } + + #[test] + fn earlier_queued_task_claims_the_only_idle_cpu() { + let mut queue = Queue::new(4); + queue.push((5, 0), ALL, ()).unwrap(); + assert!(!idle_cpu_takes(&queue, IDLE_2, (10, 0))); + } + + #[test] + fn equal_key_is_fifo() { + let mut queue = Queue::new(4); + queue.push((10, 0), ALL, ()).unwrap(); + assert!(!idle_cpu_takes(&queue, IDLE_2, (10, 0))); + } + + #[test] + fn task_earlier_than_queue_head_is_taken() { + let mut queue = Queue::new(4); + queue.push((20, 0), ALL, ()).unwrap(); + assert!(idle_cpu_takes(&queue, IDLE_2, (10, 0))); + } + + #[test] + fn spare_idle_cpu_takes_task() { + let mut queue = Queue::new(4); + queue.push((5, 0), ALL, ()).unwrap(); + assert!(idle_cpu_takes(&queue, IDLE_2.with(3), (10, 0))); + } + + #[test] + fn queued_task_of_another_cpu_does_not_claim_idle_cpu() { + let mut queue = Queue::new(4); + queue.push((5, 0), CpuSet::empty().with(3), ()).unwrap(); + assert!(idle_cpu_takes(&queue, IDLE_2, (10, 0))); + } +} diff --git a/mdbook/src/internal/scheduler.md b/mdbook/src/internal/scheduler.md index c15853e4e..d39a2fc3d 100644 --- a/mdbook/src/internal/scheduler.md +++ b/mdbook/src/internal/scheduler.md @@ -75,7 +75,7 @@ The Earliest Deadline First (EDF) scheduler is implemented in [edf.rs](https://g The scheduler holds a single affinity-aware priority queue, [`AffinityBTreeQueue`](https://crates.io/crates/affinity_btree_queue), backed by an augmented B-tree. Each entry carries `(priority, affinity, task)`, where the priority is the pair `(absolute_deadline, wake_time)`; smaller values dequeue first, so tasks are ordered by earliest deadline with wake time as a tie-breaker. Every B-tree node stores the OR of the affinities in its subtree, which lets `pop_for_cpu(cpu)` find the earliest-deadline task runnable on `cpu` in logarithmic time while skipping subtrees with no eligible entry. -When a task is enqueued via `wake_task()`, the scheduler reads the `SchedulerType::EDF(relative_deadline, cpu_set)` attached to the task and calculates the absolute deadline as `uptime + relative_deadline`. If the task is part of a DAG, `calculate_and_update_dag_deadline()` is used instead to propagate deadlines through the DAG. The task priority (`PriorityInfo`) becomes `MAX_TASK_PRIORITY - absolute_deadline`, so an earlier deadline gives a higher priority. The task is then pushed into the queue with its `cpu_set` as the affinity mask. Preemption is handled via `invoke_preemption()`: if no core in the set is idle and the task is not already running, the core running the lowest-priority task among the set is chosen, and an IPI is sent to it when the newly enqueued task has an earlier deadline than the task currently running (or pending preemption) on that core. +When a task is enqueued via `wake_task()`, the scheduler reads the `SchedulerType::EDF(relative_deadline, cpu_set)` attached to the task and calculates the absolute deadline as `uptime + relative_deadline`. If the task is part of a DAG, `calculate_and_update_dag_deadline()` is used instead to propagate deadlines through the DAG. The task priority (`PriorityInfo`) becomes `MAX_TASK_PRIORITY - absolute_deadline`, so an earlier deadline gives a higher priority. The task is then pushed into the queue with its `cpu_set` as the affinity mask. Preemption is handled via `invoke_preemption()`. If the task is already running, the scheduler does nothing. If an idle core in the set will take the task, the scheduler only enqueues it. An idle core takes the earliest queued task runnable on it, so it takes the new task if the task is earlier than every queued task runnable on that core, or if the queue holds fewer tasks than the set has idle cores. Otherwise, the scheduler chooses the core running the lowest-priority task among the busy cores of the set. It sends an IPI to that core when the new task has a higher priority than the task running (or pending preemption) on it. Against another EDF task, a higher priority means an earlier deadline. `get_next()` pops the earliest-deadline task whose `cpu_set` contains the calling CPU (`pop_for_cpu`), so a task is only ever dequeued by a core within its set. This guarantees CPU affinity. CPU 0 (the primary core) is always excluded from `cpu_set` when a task is spawned. The `cpu_set` is normalized at spawn time by removing CPU 0 and out-of-range bits; if this leaves the set empty (for example an empty set, or one naming only CPU 0 or out-of-range cores), the task is not rejected but falls back to all worker cores (`1..num_cpu()`) with a warning. From d417ea85111c7ef70e4325ba10425992d107befc Mon Sep 17 00:00:00 2001 From: atsushi421 Date: Fri, 13 Nov 2026 14:24:58 +0900 Subject: [PATCH 2/3] fix(sched): count NUM_TASK_IN_QUEUE at queue push and pop NUM_TASK_IN_QUEUE underflowed to u32::MAX. ArcWake::wake incremented it, and get_next_task() decremented it when a non-clustered scheduler returned a task. Other paths did not match: - The re-wakes in do_preemption() and run_main(), and the re-wake of a preempted task, call wake_task() without ArcWake::wake, so their tasks were decremented but never incremented. - A task that invoke_preemption() pushed to a preemption-pending heap was incremented but runs without get_next_task(). - get_next() skips terminated and panicked tasks without a decrement. With the counter at u32::MAX, wake_workers() woke every sleeping worker on every loop of the primary CPU. Now the PrioritizedFIFO, PrioritizedRR, and Panicked schedulers increment the counter when they push a task to their queue and decrement it when they pop one. So the counter equals the number of queued non-clustered tasks on every path. These schedulers also wake the primary CPU after they enqueue a task, as the EDF scheduler does, because the re-wakes do not go through ArcWake::wake. The two target-specific definitions of the counter become one. Signed-off-by: atsushi421 --- awkernel_async_lib/src/scheduler.rs | 19 +++++------------ awkernel_async_lib/src/scheduler/panicked.rs | 21 ++++++++++++------- .../src/scheduler/prioritized_fifo.rs | 13 +++++++++++- .../src/scheduler/prioritized_rr.rs | 11 +++++++++- awkernel_async_lib/src/task.rs | 18 ++++++---------- 5 files changed, 46 insertions(+), 36 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index bcd4ebb5b..064b3f88e 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -160,9 +160,8 @@ const fn get_num_clustered_schedulers() -> usize { count } -// `get_next_task` and `NUM_TASK_IN_QUEUE` accounting rely on the clustered -// schedulers forming a strict prefix of `PRIORITY_LIST`. Reject any reorder -// at compile time. +// `get_next_task` relies on the clustered schedulers forming a strict prefix +// of `PRIORITY_LIST`. Reject any reorder at compile time. const _: () = { let mut i = get_num_clustered_schedulers(); while i < PRIORITY_LIST.len() { @@ -253,18 +252,10 @@ pub(crate) fn get_next_task(execution_ensured: bool) -> Option> { // Skip the clustered prefix: it was either just tried above and returned // None (the queue state cannot change in between because both `wake_task` // and `get_next_task` hold `GLOBAL_WAKE_GET_MUTEX`), or the counter was 0 - // and its queues are empty. This also structurally guarantees that the - // `NUM_TASK_IN_QUEUE` decrement below never applies to a clustered task, - // which is counted by `NUM_CLUSTERED_TASKS_IN_QUEUE` instead. - let task = PRIORITY_LIST[get_num_clustered_schedulers()..] + // and its queues are empty. + PRIORITY_LIST[get_num_clustered_schedulers()..] .iter() - .find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured)); - - if task.is_some() { - crate::task::NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); - } - - task + .find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured)) } /// Get a scheduler. diff --git a/awkernel_async_lib/src/scheduler/panicked.rs b/awkernel_async_lib/src/scheduler/panicked.rs index e25d907d3..2d2458ba9 100644 --- a/awkernel_async_lib/src/scheduler/panicked.rs +++ b/awkernel_async_lib/src/scheduler/panicked.rs @@ -4,10 +4,11 @@ use super::{Scheduler, SchedulerType, Task}; use crate::{ scheduler::get_priority, - task::{set_current_task, State}, + task::{set_current_task, State, NUM_TASK_IN_QUEUE}, }; use alloc::{collections::VecDeque, sync::Arc}; use awkernel_lib::sync::mutex::{MCSNode, Mutex}; +use core::sync::atomic::Ordering; pub struct PanickedScheduler { data: Mutex>, // Run queue. @@ -31,13 +32,16 @@ impl Scheduler for PanickedScheduler { let mut node = MCSNode::new(); let mut data = self.data.lock(&mut node); - if let Some(data) = data.as_mut() { - data.queue.push_back(task); - } else { - let mut panicked_data = PanickedData::new(); - panicked_data.queue.push_back(task); - *data = Some(panicked_data); - } + data.get_or_insert_with(PanickedData::new) + .queue + .push_back(task); + NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed); + drop(data); + + // `do_preemption` and `run_main` re-wake tasks through this function + // without `ArcWake::wake`, so notify the primary CPU here to wake an + // idle worker. + awkernel_lib::cpu::wake_cpu(0); } fn get_next(&self, execution_ensured: bool) -> Option> { @@ -54,6 +58,7 @@ impl Scheduler for PanickedScheduler { loop { let task = data.queue.pop_front()?; + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); // Make the state of the task Running. { diff --git a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs index 6c2091c23..c2942c212 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs @@ -6,12 +6,15 @@ use super::{Scheduler, SchedulerType, Task}; use crate::scheduler::{ peek_preemption_pending, push_preemption_pending, GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED, }; -use crate::task::{get_task, get_tasks_running, set_current_task, set_need_preemption}; +use crate::task::{ + get_task, get_tasks_running, set_current_task, set_need_preemption, NUM_TASK_IN_QUEUE, +}; use crate::{scheduler::get_priority, task::State}; use alloc::sync::Arc; use alloc::vec::Vec; use awkernel_lib::priority_queue::PriorityQueue; use awkernel_lib::sync::mutex::{MCSNode, Mutex}; +use core::sync::atomic::Ordering; pub struct PrioritizedFIFOScheduler { data: Mutex>, // Run queue. @@ -59,6 +62,13 @@ impl Scheduler for PrioritizedFIFOScheduler { _priority: priority, }, ); + NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed); + drop(data); + + // `do_preemption` and `run_main` re-wake tasks through this function + // without `ArcWake::wake`, so notify the primary CPU here to wake an + // idle worker. + awkernel_lib::cpu::wake_cpu(0); } } @@ -75,6 +85,7 @@ impl Scheduler for PrioritizedFIFOScheduler { loop { // Pop a task from the run queue. let task = data.queue.pop()?; + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); // Make the state of the task Running. { diff --git a/awkernel_async_lib/src/scheduler/prioritized_rr.rs b/awkernel_async_lib/src/scheduler/prioritized_rr.rs index 2b6c2e105..cab234118 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_rr.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_rr.rs @@ -10,12 +10,13 @@ use crate::{ }, task::{ get_last_executed_by_task_id, get_task, get_tasks_running, set_current_task, - set_need_preemption, State, + set_need_preemption, State, NUM_TASK_IN_QUEUE, }, }; use alloc::{sync::Arc, vec::Vec}; use awkernel_lib::priority_queue::PriorityQueue; use awkernel_lib::sync::mutex::{MCSNode, Mutex}; +use core::sync::atomic::Ordering; pub struct PrioritizedRRScheduler { // Time quantum @@ -65,6 +66,13 @@ impl Scheduler for PrioritizedRRScheduler { _priority: priority, }, ); + NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed); + drop(data); + + // `do_preemption` and `run_main` re-wake tasks through this function + // without `ArcWake::wake`, so notify the primary CPU here to wake an + // idle worker. + awkernel_lib::cpu::wake_cpu(0); } } @@ -79,6 +87,7 @@ impl Scheduler for PrioritizedRRScheduler { }; while let Some(rr_task) = data.queue.pop() { + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); { let mut node = MCSNode::new(); let mut task_info = rr_task.task.info.lock(&mut node); diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 346c0c890..bdd57026c 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -50,17 +50,18 @@ pub type TaskResult = Result<(), Cow<'static, str>>; static TASKS: Mutex = Mutex::new(Tasks::new()); // Set of tasks. static RUNNING: [AtomicU32; NUM_MAX_CPU] = array![_ => AtomicU32::new(0); NUM_MAX_CPU]; // IDs of running tasks. pub(crate) static MAX_TASK_PRIORITY: u64 = (1 << 56) - 1; // Maximum task priority. -#[cfg(target_pointer_width = "64")] -pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); // Number of tasks in the queue. + +/// Total number of tasks in the queues of the non-clustered schedulers. +/// Each of these schedulers increments it when it pushes a task to its queue +/// and decrements it when it pops one, so it counts every path that calls +/// `wake_task`, including the re-wakes that bypass `ArcWake::wake`. +pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); /// Total number of tasks in the clustered schedulers' queues. /// Which CPUs those tasks are runnable on is tracked exactly by the run /// queues themselves; see `scheduler::clustered_queued_cpu_mask`. pub(crate) static NUM_CLUSTERED_TASKS_IN_QUEUE: AtomicU32 = AtomicU32::new(0); -#[cfg(target_pointer_width = "32")] -pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); // Number of tasks in the queue. - static PREEMPTION_REQUEST: [AtomicBool; NUM_MAX_CPU] = array![_ => AtomicBool::new(false); NUM_MAX_CPU]; @@ -165,13 +166,6 @@ impl ArcWake for Task { panicked = info.panicked; } - // Panicked tasks go to the global panicked scheduler, so always use the global counter. - // Non-panicked clustered tasks: counters are managed inside edf::wake_task. - // Non-panicked non-clustered tasks: increment the global counter here. - if panicked || self.cpu_set.is_none() { - NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Release); - } - if panicked { scheduler::panicked::SCHEDULER.wake_task(self); } else { From 302181f754c4235e5890c85d6cb062e9a08dad91 Mon Sep 17 00:00:00 2001 From: atsushi421 Date: Fri, 13 Nov 2026 14:15:23 +0900 Subject: [PATCH 3/3] perf(sched): wake one worker per queued global EDF task The EDF scheduler kept every task in one affinity queue, and wake_workers() woke every core in the affinity mask of that queue. One queued task whose set holds every worker core put every worker in the mask, so wake_workers() woke every sleeping worker. The extra workers found no task and slept again. Now the EDF scheduler keeps two queues. The global queue holds the tasks whose set holds every worker core, and the restricted queue holds the other tasks. wake_workers() wakes every core in the mask of the restricted queue, and one sleeping worker per task of the global queue, as it does for the tasks of the other schedulers. Scheduler::queued_cpu_mask() becomes queued_workers(), which also returns the number of global tasks. The queue key gets a sequence number, so tasks with equal deadlines and wake times still dequeue in push order across both queues. Signed-off-by: atsushi421 --- awkernel_async_lib/src/scheduler.rs | 28 ++-- awkernel_async_lib/src/scheduler/edf.rs | 214 ++++++++++++++++++------ awkernel_async_lib/src/task.rs | 14 +- mdbook/src/internal/scheduler.md | 8 +- 4 files changed, 187 insertions(+), 77 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 064b3f88e..7d97f7620 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -190,8 +190,8 @@ static GLOBAL_WAKE_GET_MUTEX: Mutex<()> = Mutex::new(()); /// /// 1. Wrap each entry of its run queue in [`ClusteredTask`] so that /// `NUM_CLUSTERED_TASKS_IN_QUEUE` is kept consistent. -/// 2. Implement [`Scheduler::queued_cpu_mask`] to report the exact set of -/// CPUs its queued tasks are runnable on (`wake_workers` relies on it). +/// 2. Implement [`Scheduler::queued_workers`] to report which workers its +/// queued tasks need (`wake_workers` relies on it). /// 3. Be placed in the clustered prefix of `PRIORITY_LIST` and be matched by /// `SchedulerType::is_clustered` (checked at compile time). pub(crate) trait Scheduler { @@ -208,22 +208,28 @@ pub(crate) trait Scheduler { #[allow(dead_code)] // TODO: to be removed fn priority(&self) -> u8; - /// The set of CPUs that have at least one queued task eligible to run on - /// them. Non-clustered schedulers use the default (empty) implementation; + /// The workers that queued tasks need, as `(mask, num_global)`. + /// + /// `mask` holds every CPU that a queued task restricted to some of the + /// workers can run on. `num_global` is the number of queued tasks that + /// every worker can run, so any `num_global` workers can run them. + /// Non-clustered schedulers use the default (empty) implementation; /// clustered schedulers must override it. - fn queued_cpu_mask(&self) -> CpuSet { - CpuSet::empty() + fn queued_workers(&self) -> (CpuSet, usize) { + (CpuSet::empty(), 0) } } -/// Return the union of [`Scheduler::queued_cpu_mask`] over all clustered -/// schedulers: the exact set of CPUs for which a clustered task is queued. -pub(crate) fn clustered_queued_cpu_mask() -> CpuSet { +/// Return [`Scheduler::queued_workers`] summed over all clustered schedulers. +pub(crate) fn clustered_queued_workers() -> (CpuSet, usize) { let mut mask = CpuSet::empty(); + let mut num_global = 0; for scheduler_type in &PRIORITY_LIST[..get_num_clustered_schedulers()] { - mask = mask.union(get_scheduler(scheduler_type).queued_cpu_mask()); + let (m, n) = get_scheduler(scheduler_type).queued_workers(); + mask = mask.union(m); + num_global += n; } - mask + (mask, num_global) } /// Get the next executable task. diff --git a/awkernel_async_lib/src/scheduler/edf.rs b/awkernel_async_lib/src/scheduler/edf.rs index f11af72e5..303eb735d 100644 --- a/awkernel_async_lib/src/scheduler/edf.rs +++ b/awkernel_async_lib/src/scheduler/edf.rs @@ -2,9 +2,10 @@ //! //! Each task carries a [`CpuSet`] describing the CPU cores it may run on. //! A one-core set gives partitioned EDF, and `CpuSet::all()` gives global EDF. -//! Runnable tasks are kept in a single affinity-aware priority queue -//! ([`AffinityBTreeQueue`]); `get_next` on CPU `c` pops the earliest-deadline -//! task whose `cpu_set` contains `c`. +//! Runnable tasks are kept in two affinity-aware priority queues +//! ([`AffinityBTreeQueue`]), one for tasks that every worker core can run and +//! one for the other tasks; `get_next` on CPU `c` pops the earliest-deadline +//! task of either queue whose `cpu_set` contains `c`. use super::{Scheduler, SchedulerType, Task}; use crate::{ @@ -20,20 +21,86 @@ use crate::{ use affinity_btree_queue::{AffinityBTreeQueue, DEFAULT_MIN_DEGREE}; use alloc::sync::Arc; use awkernel_lib::{ - cpu::{masked_workers, num_cpu, CpuSet, CPU_SET_WORDS}, + cpu::{all_workers, masked_workers, num_cpu, CpuSet, CPU_SET_WORDS}, sync::mutex::{MCSNode, Mutex}, }; -/// The run queue. Priority is `(absolute_deadline, wake_time)`; smaller values -/// dequeue first, and entries with fully equal keys are FIFO-ordered by the -/// queue's internal sequence number. -type EDFQueue = - AffinityBTreeQueue<(u64, u64), ClusteredTask>, DEFAULT_MIN_DEGREE, CPU_SET_WORDS>; +/// Priority of a queued task, `(absolute_deadline, wake_time, seq)`. Smaller +/// values dequeue first. `seq` grows with each push, so entries with equal +/// deadlines and wake times are FIFO-ordered across both queues. +type Key = (u64, u64, u64); + +type Queue = AffinityBTreeQueue; + +/// The run queues of the EDF scheduler. +/// +/// `wake_workers` wakes every core in the affinity mask of `restricted`, but +/// only one core for each task of `global`. A single queue would put every +/// worker in the mask while one task that every worker can run is queued. +struct EDFQueues { + /// Tasks whose set holds every worker core. + global: Queue, + /// The other tasks. + restricted: Queue, + next_seq: u64, +} + +impl EDFQueues { + fn new(num_cpus: usize) -> Self { + Self { + global: Queue::new(num_cpus), + restricted: Queue::new(num_cpus), + next_seq: 0, + } + } + + /// `priority` is `(absolute_deadline, wake_time)`. + fn push(&mut self, priority: (u64, u64), cpu_set: CpuSet, value: V) { + let key = (priority.0, priority.1, self.next_seq); + self.next_seq += 1; + + let queue = if cpu_set == all_workers(self.global.num_cpus()) { + &mut self.global + } else { + &mut self.restricted + }; + queue + .push(key, cpu_set, value) + .expect("EDF: failed to push a task"); + } + + fn len(&self) -> usize { + self.global.len() + self.restricted.len() + } + + /// The key of the earliest queued task runnable on `cpu_id`. + fn peek_for_cpu(&self, cpu_id: usize) -> Option { + let global = self.global.peek_key_for_cpu(cpu_id).map(|k| k.priority); + let restricted = self.restricted.peek_key_for_cpu(cpu_id).map(|k| k.priority); + match (global, restricted) { + (Some(g), Some(r)) => Some(g.min(r)), + (g, r) => g.or(r), + } + } + + /// Remove and return the earliest queued task runnable on `cpu_id`. + fn pop_for_cpu(&mut self, cpu_id: usize) -> Option { + let global = self.global.peek_key_for_cpu(cpu_id).map(|k| k.priority); + let restricted = self.restricted.peek_key_for_cpu(cpu_id).map(|k| k.priority); + let queue = match (global, restricted) { + (None, None) => return None, + (Some(g), Some(r)) if r < g => &mut self.restricted, + (None, Some(_)) => &mut self.restricted, + _ => &mut self.global, + }; + queue.pop_for_cpu(cpu_id).map(|(_, _, value)| value) + } +} pub struct EDFScheduler { - // `AffinityBTreeQueue::new` is not a const fn, so the queue is lazily + // `AffinityBTreeQueue::new` is not a const fn, so the queues are lazily // initialized with `num_cpu()` on first use. - data: Mutex>, + data: Mutex>>>>, priority: u8, } @@ -78,14 +145,12 @@ impl Scheduler for EDFScheduler { if !self.invoke_preemption(task.clone(), (absolute_deadline, wake_time)) { let mut node_inner = MCSNode::new(); let mut data = self.data.lock(&mut node_inner); - let queue = data.get_or_insert_with(|| EDFQueue::new(num_cpu())); - queue - .push( - (absolute_deadline, wake_time), - cpu_set, - ClusteredTask::new(task), - ) - .expect("EDF: failed to push a task"); + let queues = data.get_or_insert_with(|| EDFQueues::new(num_cpu())); + queues.push( + (absolute_deadline, wake_time), + cpu_set, + ClusteredTask::new(task), + ); drop(data); // `do_preemption` and `run_main` re-wake tasks through this function @@ -100,13 +165,13 @@ impl Scheduler for EDFScheduler { let mut node = MCSNode::new(); let mut data = self.data.lock(&mut node); - let queue = (*data).as_mut()?; + let queues = (*data).as_mut()?; // Note: spawn removes CPU 0 from every set, so `pop_for_cpu(0)` always // returns `None` and the `get_next_task(false)` path executed on the // primary CPU can never dequeue a clustered task. loop { - let (_, _, mut entry) = queue.pop_for_cpu(cpu_id)?; + let mut entry = queues.pop_for_cpu(cpu_id)?; // take() decrements the counters; entry then drops with inner == None. let Some(task) = entry.take() else { @@ -142,15 +207,15 @@ impl Scheduler for EDFScheduler { self.priority } - fn queued_cpu_mask(&self) -> CpuSet { + fn queued_workers(&self) -> (CpuSet, usize) { let mut node = MCSNode::new(); let data = self.data.lock(&mut node); - // O(1): the queue maintains the OR of all queued affinities in its + // O(1): each queue maintains the OR of all queued affinities in its // B-tree root. match &*data { - Some(queue) => queue.affinity_mask(), - None => CpuSet::empty(), + Some(queues) => (queues.restricted.affinity_mask(), queues.global.len()), + None => (CpuSet::empty(), 0), } } } @@ -184,7 +249,7 @@ impl EDFScheduler { let data = self.data.lock(&mut node); if (*data) .as_ref() - .is_none_or(|queue| idle_cpu_takes(queue, idle_cpus, key)) + .is_none_or(|queues| idle_cpu_takes(queues, idle_cpus, key)) { return false; } @@ -228,7 +293,7 @@ impl EDFScheduler { // NOTE: The task is now pinned to `victim_cpu`'s preemption-pending // heap and is not in the affinity queue, so it is invisible to the // other CPUs in its `cpu_set` (neither `pop_for_cpu`, the clustered - // task counter, nor `queued_cpu_mask` sees it). If another CPU in + // task counter, nor `queued_workers` sees it). If another CPU in // the set becomes idle while the IPI is in flight, it cannot pick // the task up and may go to sleep — a latency window bounded by the // IPI delivery plus the victim's interrupt-disabled sections. This @@ -249,23 +314,19 @@ impl EDFScheduler { } /// Whether a CPU of `idle_cpus` takes a task of priority `key` if the task is -/// pushed to `queue` now. +/// pushed to `queues` now. /// /// An idle CPU takes the earliest queued task runnable on it. So the task is /// taken if it is earlier than every queued task runnable on one of -/// `idle_cpus`, or if `queue` holds fewer tasks than `idle_cpus` holds CPUs. +/// `idle_cpus`, or if `queues` hold fewer tasks than `idle_cpus` holds CPUs. /// Otherwise, the queued tasks can take every idle CPU first, and the task /// must preempt to run now. -fn idle_cpu_takes( - queue: &AffinityBTreeQueue<(u64, u64), V, DEFAULT_MIN_DEGREE, CPU_SET_WORDS>, - idle_cpus: CpuSet, - key: (u64, u64), -) -> bool { - queue.len() < idle_cpus.iter().count() +fn idle_cpu_takes(queues: &EDFQueues, idle_cpus: CpuSet, key: (u64, u64)) -> bool { + queues.len() < idle_cpus.iter().count() || idle_cpus.iter().any(|cpu_id| { - queue - .peek_key_for_cpu(cpu_id) - .is_none_or(|queued| queued.priority > key) + queues + .peek_for_cpu(cpu_id) + .is_none_or(|(deadline, wake_time, _)| (deadline, wake_time) > key) }) } @@ -273,48 +334,91 @@ fn idle_cpu_takes( mod tests { use super::*; - type Queue = AffinityBTreeQueue<(u64, u64), (), DEFAULT_MIN_DEGREE, CPU_SET_WORDS>; + type Queues = EDFQueues; const IDLE_2: CpuSet = CpuSet::empty().with(2); + /// Every worker core of a 4-CPU system. const ALL: CpuSet = CpuSet::empty().with(1).with(2).with(3); #[test] fn idle_cpu_takes_task_from_empty_queue() { - assert!(idle_cpu_takes(&Queue::new(4), IDLE_2, (10, 0))); + assert!(idle_cpu_takes(&Queues::new(4), IDLE_2, (10, 0))); } #[test] fn earlier_queued_task_claims_the_only_idle_cpu() { - let mut queue = Queue::new(4); - queue.push((5, 0), ALL, ()).unwrap(); - assert!(!idle_cpu_takes(&queue, IDLE_2, (10, 0))); + let mut queues = Queues::new(4); + queues.push((5, 0), ALL, 0); + assert!(!idle_cpu_takes(&queues, IDLE_2, (10, 0))); } #[test] fn equal_key_is_fifo() { - let mut queue = Queue::new(4); - queue.push((10, 0), ALL, ()).unwrap(); - assert!(!idle_cpu_takes(&queue, IDLE_2, (10, 0))); + let mut queues = Queues::new(4); + queues.push((10, 0), ALL, 0); + assert!(!idle_cpu_takes(&queues, IDLE_2, (10, 0))); } #[test] fn task_earlier_than_queue_head_is_taken() { - let mut queue = Queue::new(4); - queue.push((20, 0), ALL, ()).unwrap(); - assert!(idle_cpu_takes(&queue, IDLE_2, (10, 0))); + let mut queues = Queues::new(4); + queues.push((20, 0), ALL, 0); + assert!(idle_cpu_takes(&queues, IDLE_2, (10, 0))); } #[test] fn spare_idle_cpu_takes_task() { - let mut queue = Queue::new(4); - queue.push((5, 0), ALL, ()).unwrap(); - assert!(idle_cpu_takes(&queue, IDLE_2.with(3), (10, 0))); + let mut queues = Queues::new(4); + queues.push((5, 0), ALL, 0); + assert!(idle_cpu_takes(&queues, IDLE_2.with(3), (10, 0))); } #[test] fn queued_task_of_another_cpu_does_not_claim_idle_cpu() { - let mut queue = Queue::new(4); - queue.push((5, 0), CpuSet::empty().with(3), ()).unwrap(); - assert!(idle_cpu_takes(&queue, IDLE_2, (10, 0))); + let mut queues = Queues::new(4); + queues.push((5, 0), CpuSet::empty().with(3), 0); + assert!(idle_cpu_takes(&queues, IDLE_2, (10, 0))); + } + + #[test] + fn pop_takes_earliest_task_of_either_queue() { + let mut queues = Queues::new(4); + queues.push((30, 0), ALL, 1); + queues.push((10, 0), IDLE_2, 2); + queues.push((20, 0), ALL, 3); + assert_eq!(queues.pop_for_cpu(2), Some(2)); + assert_eq!(queues.pop_for_cpu(2), Some(3)); + assert_eq!(queues.pop_for_cpu(2), Some(1)); + assert_eq!(queues.pop_for_cpu(2), None); + } + + #[test] + fn equal_keys_are_fifo_across_queues() { + let mut queues = Queues::new(4); + queues.push((10, 0), ALL, 1); + queues.push((10, 0), IDLE_2, 2); + queues.push((10, 0), ALL, 3); + assert_eq!(queues.pop_for_cpu(2), Some(1)); + assert_eq!(queues.pop_for_cpu(2), Some(2)); + assert_eq!(queues.pop_for_cpu(2), Some(3)); + } + + #[test] + fn restricted_task_stays_off_other_cpus() { + let mut queues = Queues::new(4); + queues.push((10, 0), IDLE_2, 1); + queues.push((20, 0), ALL, 2); + assert_eq!(queues.pop_for_cpu(3), Some(2)); + assert_eq!(queues.pop_for_cpu(3), None); + assert_eq!(queues.pop_for_cpu(2), Some(1)); + } + + #[test] + fn global_task_stays_out_of_restricted_mask() { + let mut queues = Queues::new(4); + queues.push((10, 0), ALL, 1); + queues.push((20, 0), IDLE_2, 2); + assert_eq!(queues.restricted.affinity_mask(), IDLE_2); + assert_eq!(queues.global.len(), 1); } } diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index bdd57026c..a000ff94d 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -59,7 +59,7 @@ pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); /// Total number of tasks in the clustered schedulers' queues. /// Which CPUs those tasks are runnable on is tracked exactly by the run -/// queues themselves; see `scheduler::clustered_queued_cpu_mask`. +/// queues themselves; see `scheduler::clustered_queued_workers`. pub(crate) static NUM_CLUSTERED_TASKS_IN_QUEUE: AtomicU32 = AtomicU32::new(0); static PREEMPTION_REQUEST: [AtomicBool; NUM_MAX_CPU] = @@ -1654,11 +1654,11 @@ impl Ord for PriorityInfo { /// Wake workers up. pub fn wake_workers() { let num_cpus = awkernel_lib::cpu::num_cpu(); - let mut num_tasks = NUM_TASK_IN_QUEUE.load(Ordering::Relaxed); - - // The exact set of CPUs for which a clustered task is queued, read from - // the clustered schedulers' run queues. - let clustered_mask = crate::scheduler::clustered_queued_cpu_mask(); + // Every CPU in `clustered_mask` has a queued clustered task that only some + // workers can run, so wake each of them. Like a non-clustered task, a + // clustered task that every worker can run needs one worker of any kind. + let (clustered_mask, num_global) = crate::scheduler::clustered_queued_workers(); + let mut num_tasks = NUM_TASK_IN_QUEUE.load(Ordering::Relaxed) as usize + num_global; for i in 1..num_cpus { if clustered_mask.contains(i) { @@ -1667,7 +1667,7 @@ pub fn wake_workers() { } // Even if there are no global tasks left, keep scanning: a - // higher-numbered CPU may still have clustered tasks waiting and + // higher-numbered CPU may still have restricted clustered tasks waiting and // skipping the rest would leave it asleep (lost wakeup). if num_tasks == 0 { continue; diff --git a/mdbook/src/internal/scheduler.md b/mdbook/src/internal/scheduler.md index d39a2fc3d..74a96d1b4 100644 --- a/mdbook/src/internal/scheduler.md +++ b/mdbook/src/internal/scheduler.md @@ -73,15 +73,15 @@ Each scheduler must be registered in the following three locations. The Earliest Deadline First (EDF) scheduler is implemented in [edf.rs](https://github.com/tier4/awkernel/blob/main/awkernel_async_lib/src/scheduler/edf.rs). This scheduler restricts each task to a set of CPU cores (a cluster), specified as a `CpuSet` bitmask. A one-bit `CpuSet` pins a task to a single core (partitioned EDF), and `CpuSet::all()` lets it run on every worker core (global EDF). -The scheduler holds a single affinity-aware priority queue, [`AffinityBTreeQueue`](https://crates.io/crates/affinity_btree_queue), backed by an augmented B-tree. Each entry carries `(priority, affinity, task)`, where the priority is the pair `(absolute_deadline, wake_time)`; smaller values dequeue first, so tasks are ordered by earliest deadline with wake time as a tie-breaker. Every B-tree node stores the OR of the affinities in its subtree, which lets `pop_for_cpu(cpu)` find the earliest-deadline task runnable on `cpu` in logarithmic time while skipping subtrees with no eligible entry. +The scheduler holds two affinity-aware priority queues, [`AffinityBTreeQueue`](https://crates.io/crates/affinity_btree_queue), each backed by an augmented B-tree. The global queue holds the tasks whose set holds every worker core, and the restricted queue holds the other tasks. Each entry carries `(priority, affinity, task)`, where the priority is `(absolute_deadline, wake_time, seq)`. Smaller values dequeue first, so tasks are ordered by earliest deadline with wake time as a tie-breaker. `seq` grows with each push, so tasks with equal deadlines and wake times dequeue in push order across both queues. Every B-tree node stores the OR of the affinities in its subtree, which lets `pop_for_cpu(cpu)` find the earliest-deadline task runnable on `cpu` in logarithmic time while skipping subtrees with no eligible entry. When a task is enqueued via `wake_task()`, the scheduler reads the `SchedulerType::EDF(relative_deadline, cpu_set)` attached to the task and calculates the absolute deadline as `uptime + relative_deadline`. If the task is part of a DAG, `calculate_and_update_dag_deadline()` is used instead to propagate deadlines through the DAG. The task priority (`PriorityInfo`) becomes `MAX_TASK_PRIORITY - absolute_deadline`, so an earlier deadline gives a higher priority. The task is then pushed into the queue with its `cpu_set` as the affinity mask. Preemption is handled via `invoke_preemption()`. If the task is already running, the scheduler does nothing. If an idle core in the set will take the task, the scheduler only enqueues it. An idle core takes the earliest queued task runnable on it, so it takes the new task if the task is earlier than every queued task runnable on that core, or if the queue holds fewer tasks than the set has idle cores. Otherwise, the scheduler chooses the core running the lowest-priority task among the busy cores of the set. It sends an IPI to that core when the new task has a higher priority than the task running (or pending preemption) on it. Against another EDF task, a higher priority means an earlier deadline. -`get_next()` pops the earliest-deadline task whose `cpu_set` contains the calling CPU (`pop_for_cpu`), so a task is only ever dequeued by a core within its set. This guarantees CPU affinity. CPU 0 (the primary core) is always excluded from `cpu_set` when a task is spawned. The `cpu_set` is normalized at spawn time by removing CPU 0 and out-of-range bits; if this leaves the set empty (for example an empty set, or one naming only CPU 0 or out-of-range cores), the task is not rejected but falls back to all worker cores (`1..num_cpu()`) with a warning. +`get_next()` pops the earliest-deadline task of either queue whose `cpu_set` contains the calling CPU (`pop_for_cpu`), so a task is only ever dequeued by a core within its set. This guarantees CPU affinity. CPU 0 (the primary core) is always excluded from `cpu_set` when a task is spawned. The `cpu_set` is normalized at spawn time by removing CPU 0 and out-of-range bits; if this leaves the set empty (for example an empty set, or one naming only CPU 0 or out-of-range cores), the task is not rejected but falls back to all worker cores (`1..num_cpu()`) with a warning. -Bookkeeping for sleeping workers uses two pieces of state. A single global counter (`NUM_CLUSTERED_TASKS_IN_QUEUE`) tracks how many tasks are queued across all clustered schedulers; it is maintained by the `ClusteredTask` RAII wrapper (incremented on enqueue, decremented exactly once on dequeue or drop) and lets `get_next_task()` skip the clustered schedulers entirely when it is zero. The per-CPU information — which CPUs actually have an eligible task queued — is not duplicated in counters: it is read directly from the run queue, whose B-tree root already maintains the OR of all queued affinities (`affinity_mask()`, O(1)). `wake_workers()` obtains this set via `Scheduler::queued_cpu_mask()` (unioned over all clustered schedulers by `clustered_queued_cpu_mask()`) and wakes exactly the cores that have an eligible clustered task. +Bookkeeping for sleeping workers uses two pieces of state. A single global counter (`NUM_CLUSTERED_TASKS_IN_QUEUE`) tracks how many tasks are queued across all clustered schedulers; it is maintained by the `ClusteredTask` RAII wrapper (incremented on enqueue, decremented exactly once on dequeue or drop) and lets `get_next_task()` skip the clustered schedulers entirely when it is zero. `wake_workers()` reads the other information from the run queues through `Scheduler::queued_workers()` (summed over all clustered schedulers by `clustered_queued_workers()`). It wakes every core in the affinity mask of the restricted queue, which its B-tree root maintains (`affinity_mask()`, O(1)). For the global queue, it wakes only one sleeping core per queued task, as it does for the tasks of the other schedulers. With a single queue, one queued task that every worker can run would put every worker in the mask and wake all of them. -A new clustered scheduler must (1) wrap its queue entries in `ClusteredTask`, (2) implement `Scheduler::queued_cpu_mask()`, and (3) be placed in the clustered prefix of `PRIORITY_LIST` and matched by `SchedulerType::is_clustered()`; the prefix requirement is enforced at compile time. +A new clustered scheduler must (1) wrap its queue entries in `ClusteredTask`, (2) implement `Scheduler::queued_workers()`, and (3) be placed in the clustered prefix of `PRIORITY_LIST` and matched by `SchedulerType::is_clustered()`; the prefix requirement is enforced at compile time. ### PrioritizedFIFO Scheduler