From 47b6f5da9a23c3d7c1c9c27a58fc29a07ee89c4e Mon Sep 17 00:00:00 2001 From: atsushi421 Date: Fri, 13 Nov 2026 14:24:58 +0900 Subject: [PATCH 1/2] fix(sched): count NUM_TASK_IN_QUEUE at queue push and pop NUM_TASK_IN_QUEUE underflowed to u32::MAX. ArcWake::wake incremented it, and get_next_task() decremented it when a non-clustered scheduler returned a task. Other paths did not match: - The re-wakes in do_preemption() and run_main(), and the re-wake of a preempted task, call wake_task() without ArcWake::wake, so their tasks were decremented but never incremented. - A task that invoke_preemption() pushed to a preemption-pending heap was incremented but runs without get_next_task(). - get_next() skips terminated and panicked tasks without a decrement. With the counter at u32::MAX, wake_workers() woke every sleeping worker on every loop of the primary CPU. Now the PrioritizedFIFO, PrioritizedRR, and Panicked schedulers increment the counter when they push a task to their queue and decrement it when they pop one. So the counter equals the number of queued non-clustered tasks on every path. These schedulers also wake the primary CPU after they enqueue a task, as the EDF scheduler does, because the re-wakes do not go through ArcWake::wake. The two target-specific definitions of the counter become one. Signed-off-by: atsushi421 --- awkernel_async_lib/src/scheduler.rs | 19 +++++------------ awkernel_async_lib/src/scheduler/panicked.rs | 21 ++++++++++++------- .../src/scheduler/prioritized_fifo.rs | 13 +++++++++++- .../src/scheduler/prioritized_rr.rs | 11 +++++++++- awkernel_async_lib/src/task.rs | 18 ++++++---------- 5 files changed, 46 insertions(+), 36 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index bcd4ebb5b..064b3f88e 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -160,9 +160,8 @@ const fn get_num_clustered_schedulers() -> usize { count } -// `get_next_task` and `NUM_TASK_IN_QUEUE` accounting rely on the clustered -// schedulers forming a strict prefix of `PRIORITY_LIST`. Reject any reorder -// at compile time. +// `get_next_task` relies on the clustered schedulers forming a strict prefix +// of `PRIORITY_LIST`. Reject any reorder at compile time. const _: () = { let mut i = get_num_clustered_schedulers(); while i < PRIORITY_LIST.len() { @@ -253,18 +252,10 @@ pub(crate) fn get_next_task(execution_ensured: bool) -> Option> { // Skip the clustered prefix: it was either just tried above and returned // None (the queue state cannot change in between because both `wake_task` // and `get_next_task` hold `GLOBAL_WAKE_GET_MUTEX`), or the counter was 0 - // and its queues are empty. This also structurally guarantees that the - // `NUM_TASK_IN_QUEUE` decrement below never applies to a clustered task, - // which is counted by `NUM_CLUSTERED_TASKS_IN_QUEUE` instead. - let task = PRIORITY_LIST[get_num_clustered_schedulers()..] + // and its queues are empty. + PRIORITY_LIST[get_num_clustered_schedulers()..] .iter() - .find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured)); - - if task.is_some() { - crate::task::NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); - } - - task + .find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured)) } /// Get a scheduler. diff --git a/awkernel_async_lib/src/scheduler/panicked.rs b/awkernel_async_lib/src/scheduler/panicked.rs index e25d907d3..2d2458ba9 100644 --- a/awkernel_async_lib/src/scheduler/panicked.rs +++ b/awkernel_async_lib/src/scheduler/panicked.rs @@ -4,10 +4,11 @@ use super::{Scheduler, SchedulerType, Task}; use crate::{ scheduler::get_priority, - task::{set_current_task, State}, + task::{set_current_task, State, NUM_TASK_IN_QUEUE}, }; use alloc::{collections::VecDeque, sync::Arc}; use awkernel_lib::sync::mutex::{MCSNode, Mutex}; +use core::sync::atomic::Ordering; pub struct PanickedScheduler { data: Mutex>, // Run queue. @@ -31,13 +32,16 @@ impl Scheduler for PanickedScheduler { let mut node = MCSNode::new(); let mut data = self.data.lock(&mut node); - if let Some(data) = data.as_mut() { - data.queue.push_back(task); - } else { - let mut panicked_data = PanickedData::new(); - panicked_data.queue.push_back(task); - *data = Some(panicked_data); - } + data.get_or_insert_with(PanickedData::new) + .queue + .push_back(task); + NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed); + drop(data); + + // `do_preemption` and `run_main` re-wake tasks through this function + // without `ArcWake::wake`, so notify the primary CPU here to wake an + // idle worker. + awkernel_lib::cpu::wake_cpu(0); } fn get_next(&self, execution_ensured: bool) -> Option> { @@ -54,6 +58,7 @@ impl Scheduler for PanickedScheduler { loop { let task = data.queue.pop_front()?; + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); // Make the state of the task Running. { diff --git a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs index 6c2091c23..c2942c212 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs @@ -6,12 +6,15 @@ use super::{Scheduler, SchedulerType, Task}; use crate::scheduler::{ peek_preemption_pending, push_preemption_pending, GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED, }; -use crate::task::{get_task, get_tasks_running, set_current_task, set_need_preemption}; +use crate::task::{ + get_task, get_tasks_running, set_current_task, set_need_preemption, NUM_TASK_IN_QUEUE, +}; use crate::{scheduler::get_priority, task::State}; use alloc::sync::Arc; use alloc::vec::Vec; use awkernel_lib::priority_queue::PriorityQueue; use awkernel_lib::sync::mutex::{MCSNode, Mutex}; +use core::sync::atomic::Ordering; pub struct PrioritizedFIFOScheduler { data: Mutex>, // Run queue. @@ -59,6 +62,13 @@ impl Scheduler for PrioritizedFIFOScheduler { _priority: priority, }, ); + NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed); + drop(data); + + // `do_preemption` and `run_main` re-wake tasks through this function + // without `ArcWake::wake`, so notify the primary CPU here to wake an + // idle worker. + awkernel_lib::cpu::wake_cpu(0); } } @@ -75,6 +85,7 @@ impl Scheduler for PrioritizedFIFOScheduler { loop { // Pop a task from the run queue. let task = data.queue.pop()?; + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); // Make the state of the task Running. { diff --git a/awkernel_async_lib/src/scheduler/prioritized_rr.rs b/awkernel_async_lib/src/scheduler/prioritized_rr.rs index 2b6c2e105..cab234118 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_rr.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_rr.rs @@ -10,12 +10,13 @@ use crate::{ }, task::{ get_last_executed_by_task_id, get_task, get_tasks_running, set_current_task, - set_need_preemption, State, + set_need_preemption, State, NUM_TASK_IN_QUEUE, }, }; use alloc::{sync::Arc, vec::Vec}; use awkernel_lib::priority_queue::PriorityQueue; use awkernel_lib::sync::mutex::{MCSNode, Mutex}; +use core::sync::atomic::Ordering; pub struct PrioritizedRRScheduler { // Time quantum @@ -65,6 +66,13 @@ impl Scheduler for PrioritizedRRScheduler { _priority: priority, }, ); + NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed); + drop(data); + + // `do_preemption` and `run_main` re-wake tasks through this function + // without `ArcWake::wake`, so notify the primary CPU here to wake an + // idle worker. + awkernel_lib::cpu::wake_cpu(0); } } @@ -79,6 +87,7 @@ impl Scheduler for PrioritizedRRScheduler { }; while let Some(rr_task) = data.queue.pop() { + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); { let mut node = MCSNode::new(); let mut task_info = rr_task.task.info.lock(&mut node); diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index eee6a3a3f..bb8286927 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -50,17 +50,18 @@ pub type TaskResult = Result<(), Cow<'static, str>>; static TASKS: Mutex = Mutex::new(Tasks::new()); // Set of tasks. static RUNNING: [AtomicU32; NUM_MAX_CPU] = array![_ => AtomicU32::new(0); NUM_MAX_CPU]; // IDs of running tasks. pub(crate) static MAX_TASK_PRIORITY: u64 = (1 << 56) - 1; // Maximum task priority. -#[cfg(target_pointer_width = "64")] -pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); // Number of tasks in the queue. + +/// Total number of tasks in the queues of the non-clustered schedulers. +/// Each of these schedulers increments it when it pushes a task to its queue +/// and decrements it when it pops one, so it counts every path that calls +/// `wake_task`, including the re-wakes that bypass `ArcWake::wake`. +pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); /// Total number of tasks in the clustered schedulers' queues. /// Which CPUs those tasks are runnable on is tracked exactly by the run /// queues themselves; see `scheduler::clustered_queued_cpu_mask`. pub(crate) static NUM_CLUSTERED_TASKS_IN_QUEUE: AtomicU32 = AtomicU32::new(0); -#[cfg(target_pointer_width = "32")] -pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); // Number of tasks in the queue. - static PREEMPTION_REQUEST: [AtomicBool; NUM_MAX_CPU] = array![_ => AtomicBool::new(false); NUM_MAX_CPU]; @@ -165,13 +166,6 @@ impl ArcWake for Task { panicked = info.panicked; } - // Panicked tasks go to the global panicked scheduler, so always use the global counter. - // Non-panicked clustered tasks: counters are managed inside edf::wake_task. - // Non-panicked non-clustered tasks: increment the global counter here. - if panicked || self.cpu_set.is_none() { - NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Release); - } - if panicked { scheduler::panicked::SCHEDULER.wake_task(self); } else { From 202851cd99fa162e8983c31348db4de698dfb2e0 Mon Sep 17 00:00:00 2001 From: atsushi421 Date: Mon, 16 Nov 2026 17:13:54 +0900 Subject: [PATCH 2/2] fix: address review findings - Call wake_workers() before the primary CPU sleeps, so a task that a timer wakes on CPU 0 reaches a worker without waiting for the sleep to end - Remove the duplicate wake_cpu(0) in ArcWake::wake and reword the notify comments in the schedulers Signed-off-by: atsushi421 --- awkernel_async_lib/src/scheduler/edf.rs | 6 +++--- awkernel_async_lib/src/scheduler/panicked.rs | 6 +++--- awkernel_async_lib/src/scheduler/prioritized_fifo.rs | 6 +++--- awkernel_async_lib/src/scheduler/prioritized_rr.rs | 6 +++--- awkernel_async_lib/src/task.rs | 3 --- kernel/src/lib.rs | 8 +++++--- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/awkernel_async_lib/src/scheduler/edf.rs b/awkernel_async_lib/src/scheduler/edf.rs index 6539edc35..d7811a52d 100644 --- a/awkernel_async_lib/src/scheduler/edf.rs +++ b/awkernel_async_lib/src/scheduler/edf.rs @@ -88,9 +88,9 @@ impl Scheduler for EDFScheduler { .expect("EDF: failed to push a task"); drop(data); - // `do_preemption` and `run_main` re-wake tasks through this function - // without `ArcWake::wake`, so notify the primary CPU here to wake an - // idle worker in the set. + // Every enqueue goes through this function, including the re-wakes + // that bypass `ArcWake::wake`, so notify the primary CPU here to + // wake an idle worker in the set. awkernel_lib::cpu::wake_cpu(0); } } diff --git a/awkernel_async_lib/src/scheduler/panicked.rs b/awkernel_async_lib/src/scheduler/panicked.rs index 2d2458ba9..213d550e6 100644 --- a/awkernel_async_lib/src/scheduler/panicked.rs +++ b/awkernel_async_lib/src/scheduler/panicked.rs @@ -38,9 +38,9 @@ impl Scheduler for PanickedScheduler { NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed); drop(data); - // `do_preemption` and `run_main` re-wake tasks through this function - // without `ArcWake::wake`, so notify the primary CPU here to wake an - // idle worker. + // Every enqueue goes through this function, including the re-wakes + // that bypass `ArcWake::wake`, so notify the primary CPU here to + // wake an idle worker. awkernel_lib::cpu::wake_cpu(0); } diff --git a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs index c2942c212..a38b2f5a7 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs @@ -65,9 +65,9 @@ impl Scheduler for PrioritizedFIFOScheduler { NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed); drop(data); - // `do_preemption` and `run_main` re-wake tasks through this function - // without `ArcWake::wake`, so notify the primary CPU here to wake an - // idle worker. + // Every enqueue goes through this function, including the re-wakes + // that bypass `ArcWake::wake`, so notify the primary CPU here to + // wake an idle worker. awkernel_lib::cpu::wake_cpu(0); } } diff --git a/awkernel_async_lib/src/scheduler/prioritized_rr.rs b/awkernel_async_lib/src/scheduler/prioritized_rr.rs index cab234118..5c22e3d6b 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_rr.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_rr.rs @@ -69,9 +69,9 @@ impl Scheduler for PrioritizedRRScheduler { NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed); drop(data); - // `do_preemption` and `run_main` re-wake tasks through this function - // without `ArcWake::wake`, so notify the primary CPU here to wake an - // idle worker. + // Every enqueue goes through this function, including the re-wakes + // that bypass `ArcWake::wake`, so notify the primary CPU here to + // wake an idle worker. awkernel_lib::cpu::wake_cpu(0); } } diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index bb8286927..7ac806ba5 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -171,9 +171,6 @@ impl ArcWake for Task { } else { self.scheduler.wake_task(self); } - - // Notify the primary CPU to wake up workers. - awkernel_lib::cpu::wake_cpu(0); } } diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index fc18f23d2..d69498081 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -108,6 +108,11 @@ fn main(kernel_info: KernelInfo) { let dur = wake_task(); // Wake executable tasks periodically. + // Wake up other CPUs if there are any tasks to run. This runs before + // the sleep, because `wake_cpu(0)` on CPU 0 does nothing, so a task + // that `wake_task()` enqueued gets no other notification. + awkernel_async_lib::task::wake_workers(); + #[cfg(feature = "std")] { let dur = dur.unwrap_or(core::time::Duration::from_secs(1)); @@ -133,9 +138,6 @@ fn main(kernel_info: KernelInfo) { #[cfg(feature = "perf")] awkernel_async_lib::task::perf::start_kernel(); - - // Wake up other CPUs if there are any tasks to run. - awkernel_async_lib::task::wake_workers(); } }