diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index bcd4ebb5b..064b3f88e 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -160,9 +160,8 @@ const fn get_num_clustered_schedulers() -> usize { count } -// `get_next_task` and `NUM_TASK_IN_QUEUE` accounting rely on the clustered -// schedulers forming a strict prefix of `PRIORITY_LIST`. Reject any reorder -// at compile time. +// `get_next_task` relies on the clustered schedulers forming a strict prefix +// of `PRIORITY_LIST`. Reject any reorder at compile time. const _: () = { let mut i = get_num_clustered_schedulers(); while i < PRIORITY_LIST.len() { @@ -253,18 +252,10 @@ pub(crate) fn get_next_task(execution_ensured: bool) -> Option> { // Skip the clustered prefix: it was either just tried above and returned // None (the queue state cannot change in between because both `wake_task` // and `get_next_task` hold `GLOBAL_WAKE_GET_MUTEX`), or the counter was 0 - // and its queues are empty. This also structurally guarantees that the - // `NUM_TASK_IN_QUEUE` decrement below never applies to a clustered task, - // which is counted by `NUM_CLUSTERED_TASKS_IN_QUEUE` instead. - let task = PRIORITY_LIST[get_num_clustered_schedulers()..] + // and its queues are empty. + PRIORITY_LIST[get_num_clustered_schedulers()..] .iter() - .find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured)); - - if task.is_some() { - crate::task::NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); - } - - task + .find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured)) } /// Get a scheduler. diff --git a/awkernel_async_lib/src/scheduler/edf.rs b/awkernel_async_lib/src/scheduler/edf.rs index 6539edc35..d7811a52d 100644 --- a/awkernel_async_lib/src/scheduler/edf.rs +++ b/awkernel_async_lib/src/scheduler/edf.rs @@ -88,9 +88,9 @@ impl Scheduler for EDFScheduler { .expect("EDF: failed to push a task"); drop(data); - // `do_preemption` and `run_main` re-wake tasks through this function - // without `ArcWake::wake`, so notify the primary CPU here to wake an - // idle worker in the set. + // Every enqueue goes through this function, including the re-wakes + // that bypass `ArcWake::wake`, so notify the primary CPU here to + // wake an idle worker in the set. awkernel_lib::cpu::wake_cpu(0); } } diff --git a/awkernel_async_lib/src/scheduler/panicked.rs b/awkernel_async_lib/src/scheduler/panicked.rs index e25d907d3..213d550e6 100644 --- a/awkernel_async_lib/src/scheduler/panicked.rs +++ b/awkernel_async_lib/src/scheduler/panicked.rs @@ -4,10 +4,11 @@ use super::{Scheduler, SchedulerType, Task}; use crate::{ scheduler::get_priority, - task::{set_current_task, State}, + task::{set_current_task, State, NUM_TASK_IN_QUEUE}, }; use alloc::{collections::VecDeque, sync::Arc}; use awkernel_lib::sync::mutex::{MCSNode, Mutex}; +use core::sync::atomic::Ordering; pub struct PanickedScheduler { data: Mutex>, // Run queue. @@ -31,13 +32,16 @@ impl Scheduler for PanickedScheduler { let mut node = MCSNode::new(); let mut data = self.data.lock(&mut node); - if let Some(data) = data.as_mut() { - data.queue.push_back(task); - } else { - let mut panicked_data = PanickedData::new(); - panicked_data.queue.push_back(task); - *data = Some(panicked_data); - } + data.get_or_insert_with(PanickedData::new) + .queue + .push_back(task); + NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed); + drop(data); + + // Every enqueue goes through this function, including the re-wakes + // that bypass `ArcWake::wake`, so notify the primary CPU here to + // wake an idle worker. + awkernel_lib::cpu::wake_cpu(0); } fn get_next(&self, execution_ensured: bool) -> Option> { @@ -54,6 +58,7 @@ impl Scheduler for PanickedScheduler { loop { let task = data.queue.pop_front()?; + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); // Make the state of the task Running. { diff --git a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs index 6c2091c23..a38b2f5a7 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs @@ -6,12 +6,15 @@ use super::{Scheduler, SchedulerType, Task}; use crate::scheduler::{ peek_preemption_pending, push_preemption_pending, GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED, }; -use crate::task::{get_task, get_tasks_running, set_current_task, set_need_preemption}; +use crate::task::{ + get_task, get_tasks_running, set_current_task, set_need_preemption, NUM_TASK_IN_QUEUE, +}; use crate::{scheduler::get_priority, task::State}; use alloc::sync::Arc; use alloc::vec::Vec; use awkernel_lib::priority_queue::PriorityQueue; use awkernel_lib::sync::mutex::{MCSNode, Mutex}; +use core::sync::atomic::Ordering; pub struct PrioritizedFIFOScheduler { data: Mutex>, // Run queue. @@ -59,6 +62,13 @@ impl Scheduler for PrioritizedFIFOScheduler { _priority: priority, }, ); + NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed); + drop(data); + + // Every enqueue goes through this function, including the re-wakes + // that bypass `ArcWake::wake`, so notify the primary CPU here to + // wake an idle worker. + awkernel_lib::cpu::wake_cpu(0); } } @@ -75,6 +85,7 @@ impl Scheduler for PrioritizedFIFOScheduler { loop { // Pop a task from the run queue. let task = data.queue.pop()?; + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); // Make the state of the task Running. { diff --git a/awkernel_async_lib/src/scheduler/prioritized_rr.rs b/awkernel_async_lib/src/scheduler/prioritized_rr.rs index 2b6c2e105..5c22e3d6b 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_rr.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_rr.rs @@ -10,12 +10,13 @@ use crate::{ }, task::{ get_last_executed_by_task_id, get_task, get_tasks_running, set_current_task, - set_need_preemption, State, + set_need_preemption, State, NUM_TASK_IN_QUEUE, }, }; use alloc::{sync::Arc, vec::Vec}; use awkernel_lib::priority_queue::PriorityQueue; use awkernel_lib::sync::mutex::{MCSNode, Mutex}; +use core::sync::atomic::Ordering; pub struct PrioritizedRRScheduler { // Time quantum @@ -65,6 +66,13 @@ impl Scheduler for PrioritizedRRScheduler { _priority: priority, }, ); + NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed); + drop(data); + + // Every enqueue goes through this function, including the re-wakes + // that bypass `ArcWake::wake`, so notify the primary CPU here to + // wake an idle worker. + awkernel_lib::cpu::wake_cpu(0); } } @@ -79,6 +87,7 @@ impl Scheduler for PrioritizedRRScheduler { }; while let Some(rr_task) = data.queue.pop() { + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); { let mut node = MCSNode::new(); let mut task_info = rr_task.task.info.lock(&mut node); diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index eee6a3a3f..7ac806ba5 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -50,17 +50,18 @@ pub type TaskResult = Result<(), Cow<'static, str>>; static TASKS: Mutex = Mutex::new(Tasks::new()); // Set of tasks. static RUNNING: [AtomicU32; NUM_MAX_CPU] = array![_ => AtomicU32::new(0); NUM_MAX_CPU]; // IDs of running tasks. pub(crate) static MAX_TASK_PRIORITY: u64 = (1 << 56) - 1; // Maximum task priority. -#[cfg(target_pointer_width = "64")] -pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); // Number of tasks in the queue. + +/// Total number of tasks in the queues of the non-clustered schedulers. +/// Each of these schedulers increments it when it pushes a task to its queue +/// and decrements it when it pops one, so it counts every path that calls +/// `wake_task`, including the re-wakes that bypass `ArcWake::wake`. +pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); /// Total number of tasks in the clustered schedulers' queues. /// Which CPUs those tasks are runnable on is tracked exactly by the run /// queues themselves; see `scheduler::clustered_queued_cpu_mask`. pub(crate) static NUM_CLUSTERED_TASKS_IN_QUEUE: AtomicU32 = AtomicU32::new(0); -#[cfg(target_pointer_width = "32")] -pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); // Number of tasks in the queue. - static PREEMPTION_REQUEST: [AtomicBool; NUM_MAX_CPU] = array![_ => AtomicBool::new(false); NUM_MAX_CPU]; @@ -165,21 +166,11 @@ impl ArcWake for Task { panicked = info.panicked; } - // Panicked tasks go to the global panicked scheduler, so always use the global counter. - // Non-panicked clustered tasks: counters are managed inside edf::wake_task. - // Non-panicked non-clustered tasks: increment the global counter here. - if panicked || self.cpu_set.is_none() { - NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Release); - } - if panicked { scheduler::panicked::SCHEDULER.wake_task(self); } else { self.scheduler.wake_task(self); } - - // Notify the primary CPU to wake up workers. - awkernel_lib::cpu::wake_cpu(0); } } diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index fc18f23d2..d69498081 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -108,6 +108,11 @@ fn main(kernel_info: KernelInfo) { let dur = wake_task(); // Wake executable tasks periodically. + // Wake up other CPUs if there are any tasks to run. This runs before + // the sleep, because `wake_cpu(0)` on CPU 0 does nothing, so a task + // that `wake_task()` enqueued gets no other notification. + awkernel_async_lib::task::wake_workers(); + #[cfg(feature = "std")] { let dur = dur.unwrap_or(core::time::Duration::from_secs(1)); @@ -133,9 +138,6 @@ fn main(kernel_info: KernelInfo) { #[cfg(feature = "perf")] awkernel_async_lib::task::perf::start_kernel(); - - // Wake up other CPUs if there are any tasks to run. - awkernel_async_lib::task::wake_workers(); } }