diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 064b3f88e..93c7a494a 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -5,13 +5,14 @@ use core::sync::atomic::Ordering; use core::time::Duration; use crate::{ - task::{get_current_task, get_scheduler_type_by_task_id, Task}, + task::{get_current_task, get_scheduler_type_by_task_id, State, Task, NUM_TASK_IN_QUEUE}, time::Instant, }; use alloc::collections::{binary_heap::BinaryHeap, btree_map::BTreeMap}; use alloc::sync::Arc; use awkernel_lib::{ cpu::{num_cpu, CpuSet}, + priority_queue::PriorityQueue, sync::mutex::{MCSNode, Mutex}, verified::delta_list::DeltaList, }; @@ -202,6 +203,17 @@ pub(crate) trait Scheduler { /// Get the next executable task. fn get_next(&self, execution_ensured: bool) -> Option>; + /// Dequeue the task that `get_next(false)` would return, but only if + /// `pred` accepts it. A rejected task stays at the head of the run queue. + /// + /// The check and the dequeue must be done without releasing the lock of + /// the run queue, so that the dequeued task is the one `pred` accepted. + /// + /// This is used only by `get_next_task_if()`, which runs on the primary + /// CPU (CPU 0). A scheduler whose tasks must never be taken that way + /// returns [`ConditionalNext::Empty`]. + fn get_next_if(&self, pred: &dyn Fn(&Task) -> bool) -> ConditionalNext; + /// Get the scheduler name. fn scheduler_name(&self) -> SchedulerType; @@ -258,6 +270,87 @@ pub(crate) fn get_next_task(execution_ensured: bool) -> Option> { .find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured)) } +/// The result of [`Scheduler::get_next_if`]. +pub(crate) enum ConditionalNext { + /// The run queue has no executable task. + Empty, + + /// The head of the run queue was rejected and is left in the queue. + Rejected, + + /// The head of the run queue was accepted and dequeued. + Taken(Arc), +} + +/// Dequeue the next executable task only if `pred` accepts it. +/// +/// This examines the task that `get_next_task(false)` would return, except +/// that panicked tasks are never taken. If `pred` returns `false` for it, the +/// run queues are left untouched, so the task keeps its position in the +/// queue, and `None` is returned. +/// +/// `pred` is called with `GLOBAL_WAKE_GET_MUTEX`, the lock of the run queue, +/// and the lock of the task information held. +/// +/// This function must be called on the primary CPU, which can never dequeue +/// a clustered task (see `get_next_task`), so only the non-clustered +/// schedulers are examined. +pub(crate) fn get_next_task_if(pred: impl Fn(&Task) -> bool) -> Option> { + let mut node = MCSNode::new(); + let _guard = GLOBAL_WAKE_GET_MUTEX.lock(&mut node); + + for scheduler_type in &PRIORITY_LIST[get_num_clustered_schedulers()..] { + match get_scheduler(scheduler_type).get_next_if(&pred) { + ConditionalNext::Empty => (), + ConditionalNext::Rejected => return None, + ConditionalNext::Taken(task) => return Some(task), + } + } + + None +} + +/// The body of [`Scheduler::get_next_if`] for the schedulers whose run queue +/// is a [`PriorityQueue`]. `task_of` returns the task of a queue entry. +/// +/// Finished tasks at the head of the queue are discarded as `get_next()` +/// does. The lock of the run queue must be held by the caller. +fn take_head_if( + queue: &mut PriorityQueue, + task_of: impl Fn(&T) -> &Arc, + pred: &dyn Fn(&Task) -> bool, +) -> ConditionalNext { + loop { + let Some(task) = queue.peek().map(|entry| task_of(entry).clone()) else { + return ConditionalNext::Empty; + }; + + let mut node = MCSNode::new(); + let mut task_info = task.info.lock(&mut node); + + if matches!(task_info.state, State::Terminated | State::Panicked) { + drop(task_info); + queue.pop(); + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); + continue; + } + + if !pred(&task) { + return ConditionalNext::Rejected; + } + + queue.pop(); + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); + + if task_info.state == State::Preempted { + task_info.need_preemption = false; + } + + drop(task_info); + return ConditionalNext::Taken(task); + } +} + /// Get a scheduler. /// /// Takes `&SchedulerType` so the caller does not copy the full payload (the @@ -427,20 +520,37 @@ pub(crate) fn sleep_until_task(sleep_handler: Box, next: In /// If there are no sleeping tasks, this function returns `None`. pub fn wake_task() -> Option { // Check whether each running task exceeds the time quantum. + let mut quantum_wait: Option = None; + if PREEMPTION_ENABLED { for cpu_id in 1..num_cpu() { if let Some(task_id) = get_current_task(cpu_id) { if let Some(SchedulerType::PrioritizedRR(_)) = get_scheduler_type_by_task_id(task_id) { - prioritized_rr::SCHEDULER.invoke_preemption_tick(cpu_id, task_id) + let wait = prioritized_rr::SCHEDULER.invoke_preemption_tick(cpu_id, task_id); + quantum_wait = match (quantum_wait, wait) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }; } } } } + // A queued task may be waiting for a time quantum to expire. Nothing + // else wakes the primary CPU at that time, so do not sleep beyond it. + // If no task is queued, the next enqueue wakes the primary CPU. + if NUM_TASK_IN_QUEUE.load(Ordering::Relaxed) == 0 { + quantum_wait = None; + } + let mut node = MCSNode::new(); let mut guard = SLEEPING.lock(&mut node); guard.wake_task(); - guard.time_to_wait() + + match (guard.time_to_wait(), quantum_wait) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + } } diff --git a/awkernel_async_lib/src/scheduler/edf.rs b/awkernel_async_lib/src/scheduler/edf.rs index d7811a52d..fce33bb1d 100644 --- a/awkernel_async_lib/src/scheduler/edf.rs +++ b/awkernel_async_lib/src/scheduler/edf.rs @@ -6,7 +6,7 @@ //! ([`AffinityBTreeQueue`]); `get_next` on CPU `c` pops the earliest-deadline //! task whose `cpu_set` contains `c`. -use super::{Scheduler, SchedulerType, Task}; +use super::{ConditionalNext, Scheduler, SchedulerType, Task}; use crate::{ dag::calculate_and_update_dag_deadline, scheduler::{ @@ -134,6 +134,12 @@ impl Scheduler for EDFScheduler { } } + fn get_next_if(&self, _pred: &dyn Fn(&Task) -> bool) -> ConditionalNext { + // `get_next_if` runs on the primary CPU. Spawn removes CPU 0 from + // every set, so no queued task is runnable there (see `get_next`). + ConditionalNext::Empty + } + fn scheduler_name(&self) -> SchedulerType { SchedulerType::EDF(0, CpuSet::empty()) } diff --git a/awkernel_async_lib/src/scheduler/panicked.rs b/awkernel_async_lib/src/scheduler/panicked.rs index 213d550e6..4d40cb023 100644 --- a/awkernel_async_lib/src/scheduler/panicked.rs +++ b/awkernel_async_lib/src/scheduler/panicked.rs @@ -1,7 +1,7 @@ //! A scheduler for panicked tasks. //! Panicked tasks will be the lowest priority. -use super::{Scheduler, SchedulerType, Task}; +use super::{ConditionalNext, Scheduler, SchedulerType, Task}; use crate::{ scheduler::get_priority, task::{set_current_task, State, NUM_TASK_IN_QUEUE}, @@ -82,6 +82,13 @@ impl Scheduler for PanickedScheduler { } } + fn get_next_if(&self, _pred: &dyn Fn(&Task) -> bool) -> ConditionalNext { + // Panicked tasks have the lowest priority, but `Task::priority` still + // holds the priority a task had before it panicked, so `pred` cannot + // judge them. They are never taken ahead of a running task. + ConditionalNext::Empty + } + fn scheduler_name(&self) -> SchedulerType { SchedulerType::Panicked } diff --git a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs index a38b2f5a7..d699bdfe6 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs @@ -2,7 +2,7 @@ use core::cmp::max; -use super::{Scheduler, SchedulerType, Task}; +use super::{take_head_if, ConditionalNext, Scheduler, SchedulerType, Task}; use crate::scheduler::{ peek_preemption_pending, push_preemption_pending, GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED, }; @@ -109,6 +109,16 @@ impl Scheduler for PrioritizedFIFOScheduler { } } + fn get_next_if(&self, pred: &dyn Fn(&Task) -> bool) -> ConditionalNext { + let mut node = MCSNode::new(); + let mut data = self.data.lock(&mut node); + + match (*data).as_mut() { + Some(data) => take_head_if(&mut data.queue, |entry| &entry.task, pred), + None => ConditionalNext::Empty, + } + } + fn scheduler_name(&self) -> SchedulerType { SchedulerType::PrioritizedFIFO(0) } diff --git a/awkernel_async_lib/src/scheduler/prioritized_rr.rs b/awkernel_async_lib/src/scheduler/prioritized_rr.rs index 5c22e3d6b..4d553e5fc 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_rr.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_rr.rs @@ -2,21 +2,22 @@ use core::cmp::max; -use super::{Scheduler, SchedulerType, Task}; +use super::{take_head_if, ConditionalNext, Scheduler, SchedulerType, Task}; use crate::{ scheduler::{ - get_next_task, get_priority, peek_preemption_pending, push_preemption_pending, + get_next_task_if, get_priority, peek_preemption_pending, push_preemption_pending, GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED, }, task::{ - get_last_executed_by_task_id, get_task, get_tasks_running, set_current_task, - set_need_preemption, State, NUM_TASK_IN_QUEUE, + get_task, get_tasks_running, set_current_task, set_need_preemption, State, + NUM_TASK_IN_QUEUE, }, }; use alloc::{sync::Arc, vec::Vec}; use awkernel_lib::priority_queue::PriorityQueue; use awkernel_lib::sync::mutex::{MCSNode, Mutex}; use core::sync::atomic::Ordering; +use core::time::Duration; pub struct PrioritizedRRScheduler { // Time quantum @@ -111,6 +112,16 @@ impl Scheduler for PrioritizedRRScheduler { None } + fn get_next_if(&self, pred: &dyn Fn(&Task) -> bool) -> ConditionalNext { + let mut node = MCSNode::new(); + let mut data = self.data.lock(&mut node); + + match (*data).as_mut() { + Some(data) => take_head_if(&mut data.queue, |entry| &entry.task, pred), + None => ConditionalNext::Empty, + } + } + fn scheduler_name(&self) -> SchedulerType { SchedulerType::PrioritizedRR(0) } @@ -128,19 +139,68 @@ pub static SCHEDULER: PrioritizedRRScheduler = PrioritizedRRScheduler { }; impl PrioritizedRRScheduler { - // Invoke a preemption if the task exceeds the time quantum - pub fn invoke_preemption_tick(&self, cpu_id: usize, task_id: u32) { - if let Some(last_executed) = get_last_executed_by_task_id(task_id) { - let elapsed = last_executed.elapsed().as_micros() as u64; - if elapsed > self.interval { - if let Some(next_task) = get_next_task(false) { - push_preemption_pending(cpu_id, next_task); - let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); - set_need_preemption(task_id, cpu_id); - awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); - } - } + /// Rotate the PrioritizedRR task `task_id` running on `cpu_id` if it has + /// used up its time quantum. + /// + /// The next runnable task is dequeued and sent to `cpu_id` only if its + /// priority is at least that of the running task: an equal priority is + /// the quantum rotation, and a higher priority is a plain preemption. + /// A lower-priority task could not preempt the running task anyway, so + /// it is left in the run queue to keep its position, and the quantum of + /// the running task is restarted instead. + /// + /// Returns the time after which this function must be called again for + /// this CPU, or `None` if a preemption was requested: the preempted task + /// is enqueued again, and that wakes the primary CPU. + /// + /// This function must be called on the primary CPU. + pub fn invoke_preemption_tick(&self, cpu_id: usize, task_id: u32) -> Option { + let current_task = get_task(task_id)?; + let interval = Duration::from_micros(self.interval); + + let elapsed = { + let mut node = MCSNode::new(); + let info = current_task.info.lock(&mut node); + info.get_last_executed().elapsed() + }; + + if elapsed <= interval { + return Some(interval - elapsed); + } + + // A task is dequeued only while no preemption is pending on this + // CPU: `do_preemption` runs one pending task and re-queues the others + // at the tail of their level. The predicate runs under + // `GLOBAL_WAKE_GET_MUTEX`, so no wake path can push a pending task + // between this check and the dequeue. + let next_task = get_next_task_if(|next| { + peek_preemption_pending(cpu_id).is_none() && !current_task.preempts(next) + }); + + if let Some(next_task) = next_task { + push_preemption_pending(cpu_id, next_task); + let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); + set_need_preemption(task_id, cpu_id); + awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); + return None; } + + if peek_preemption_pending(cpu_id).is_some() { + // A preemption has already been requested on this CPU, but it + // has not been performed yet. Request it again in case the IPI + // was not handled. + let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); + set_need_preemption(task_id, cpu_id); + awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); + } else { + // There is no task to rotate with. Restart the quantum so that + // the run queues are not examined again on every tick. + let mut node = MCSNode::new(); + let mut info = current_task.info.lock(&mut node); + info.update_last_executed(); + } + + Some(interval) } fn invoke_preemption_wake(&self, task: Arc) -> bool { diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 7ac806ba5..598902f2b 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -93,10 +93,10 @@ impl Task { /// /// The one exception is the PrioritizedRR time quantum: /// `PrioritizedRRScheduler::invoke_preemption_tick` pushes the next - /// runnable task to the preemption-pending heap without a priority check, - /// and `preempt::do_preemption` lets such an equal-priority pending task - /// preempt a running PrioritizedRR task so that the quantum rotates tasks - /// of the same priority. + /// runnable task to the preemption-pending heap if its priority is at + /// least that of the running task, and `preempt::do_preemption` lets such + /// an equal-priority pending task preempt a running PrioritizedRR task so + /// that the quantum rotates tasks of the same priority. /// /// This is deliberately not `self > other`: [`Ord`] for `Task` breaks /// priority ties by task ID to give the preemption-pending heaps a total diff --git a/awkernel_async_lib/src/task/preempt.rs b/awkernel_async_lib/src/task/preempt.rs index df30685fc..c6119414d 100644 --- a/awkernel_async_lib/src/task/preempt.rs +++ b/awkernel_async_lib/src/task/preempt.rs @@ -164,11 +164,13 @@ unsafe fn do_preemption() { // path requires a strictly higher priority (`Task::preempts`), so a // pending task of equal priority can only come from // `PrioritizedRRScheduler::invoke_preemption_tick`, which pushes the - // next runnable task without a priority check when the quantum of - // the running task has expired. Let it through when the running task + // next runnable task when the quantum of the running task has + // expired and that task does not have a lower priority than the + // running one. Let it through when the running task // is PrioritizedRR so that the quantum rotates equal-priority tasks. - // `PriorityInfo` includes the scheduler priority, so equal priority - // implies that `next` is a PrioritizedRR task too. + // `PriorityInfo` includes the scheduler priority, and the tick never + // pushes a panicked task, so equal priority implies that `next` is a + // PrioritizedRR task too. let quantum_rotation = !current_task.preempts(&next) && matches!( current_task.scheduler_name(), @@ -206,7 +208,13 @@ unsafe fn do_preemption() { if let Some(next_thread) = { let mut node = MCSNode::new(); let mut task_info = next.info.lock(&mut node); - task_info.take_preempt_context() + let ctx = task_info.take_preempt_context(); + if ctx.is_some() { + // `next` resumes on its own thread without passing through + // `run_main`, so start its time quantum here. + task_info.update_last_executed(); + } + ctx } { // If the next task is a preempted task, yield to it. yield_preempted_and_wake_task(current_task, next_thread); diff --git a/awkernel_lib/src/priority_queue.rs b/awkernel_lib/src/priority_queue.rs index 92fd24b23..98009f627 100644 --- a/awkernel_lib/src/priority_queue.rs +++ b/awkernel_lib/src/priority_queue.rs @@ -42,6 +42,17 @@ impl PriorityQueue { next } + + /// Return a reference to the value that `pop` would return, without removing it + #[inline(always)] + pub fn peek(&self) -> Option<&T> { + if self.has_entry == 0 { + return None; + } + let next_priority = (u32::BITS - 1) - self.has_entry.leading_zeros(); + + self.queue[next_priority as usize].front() + } } impl Default for PriorityQueue { @@ -95,6 +106,44 @@ mod tests { // Queue should be empty assert_eq!(q.pop(), None); } + + #[test] + fn test_priority_queue_peek() { + let mut q = PriorityQueue::new(); + + // Nothing to peek in an empty queue + assert_eq!(q.peek(), None); + + // `peek` returns the head of the FIFO order and does not remove it + for id in 0..10 { + q.push(0, id); + } + + for expected in 0..10 { + assert_eq!(q.peek(), Some(&expected)); + assert_eq!(q.peek(), Some(&expected)); + assert_eq!(q.pop(), Some(expected)); + } + + assert_eq!(q.peek(), None); + + // `peek` returns the value with the highest priority + q.push(1, 10); + assert_eq!(q.peek(), Some(&10)); + q.push(3, 30); + assert_eq!(q.peek(), Some(&30)); + q.push(2, 20); + assert_eq!(q.peek(), Some(&30)); + q.push(3, 31); + assert_eq!(q.peek(), Some(&30)); + + // `peek` always agrees with the following `pop` + while let Some(expected) = q.peek().copied() { + assert_eq!(q.pop(), Some(expected)); + } + + assert_eq!(q.pop(), None); + } } // TODO: Verification diff --git a/mdbook/src/internal/scheduler.md b/mdbook/src/internal/scheduler.md index 9ce66e97d..90a8ae47f 100644 --- a/mdbook/src/internal/scheduler.md +++ b/mdbook/src/internal/scheduler.md @@ -99,7 +99,7 @@ The PrioritizedRR (Prioritized Round Robin) scheduler is implemented in [priorit The scheduler maintains a `PriorityQueue` similar to PrioritizedFIFO, but adds time quantum management with a default interval of 4ms (4,000 microseconds). The scheduler provides two preemption mechanisms: `invoke_preemption_wake()` for priority-based preemption when tasks are awakened, and `invoke_preemption_tick()` for time quantum-based preemption. -The `invoke_preemption_tick()` method is called periodically on primary CPU to check if the currently running task has exceeded its time quantum. It compares the elapsed execution time against the configured interval and triggers preemption by sending an IPI if the quantum is exceeded. The task pushed by the tick usually has the same priority as the running one; `do_preemption()` lets such an equal-priority pending task preempt a running PrioritizedRR task, which is the only case where equal priority preempts. +The `invoke_preemption_tick()` method is called periodically on primary CPU to check if the currently running task has exceeded its time quantum. It compares the elapsed execution time against the configured interval. If the quantum is exceeded, it peeks at the next runnable task and dequeues it only if its priority is at least that of the running task; it then triggers preemption by sending an IPI. The task pushed by the tick usually has the same priority as the running one; `do_preemption()` lets such an equal-priority pending task preempt a running PrioritizedRR task, which is the only case where equal priority preempts. A lower-priority task stays in the run queue and keeps its position, and the quantum of the running task is restarted, as it is when the run queues are empty. Panicked tasks are never taken by the tick. While a preemption requested earlier is still pending on that CPU, the tick dequeues nothing and only sends the IPI again. The tick returns the time until the quantum expires, and the primary CPU does not sleep beyond it while a task is queued. ### Panicked Scheduler