Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 113 additions & 3 deletions awkernel_async_lib/src/scheduler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@ use core::sync::atomic::Ordering;
use core::time::Duration;

use crate::{
task::{get_current_task, get_scheduler_type_by_task_id, Task},
task::{get_current_task, get_scheduler_type_by_task_id, State, Task, NUM_TASK_IN_QUEUE},
time::Instant,
};
use alloc::collections::{binary_heap::BinaryHeap, btree_map::BTreeMap};
use alloc::sync::Arc;
use awkernel_lib::{
cpu::{num_cpu, CpuSet},
priority_queue::PriorityQueue,
sync::mutex::{MCSNode, Mutex},
verified::delta_list::DeltaList,
};
Expand Down Expand Up @@ -202,6 +203,17 @@ pub(crate) trait Scheduler {
/// Get the next executable task.
fn get_next(&self, execution_ensured: bool) -> Option<Arc<Task>>;

/// Dequeue the task that `get_next(false)` would return, but only if

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After this change, only run_main calls get_next_task, and it always passes true, so the execution_ensured == false branches of the four schedulers are dead code. This doc, the doc of get_next_task_if, and the comment in edf.rs that the new EDF get_next_if points to still describe a get_next_task(false) path on the primary CPU. Rewording them, or removing the execution_ensured parameter, keeps them accurate.

/// `pred` accepts it. A rejected task stays at the head of the run queue.
///
/// The check and the dequeue must be done without releasing the lock of
/// the run queue, so that the dequeued task is the one `pred` accepted.
///
/// This is used only by `get_next_task_if()`, which runs on the primary
/// CPU (CPU 0). A scheduler whose tasks must never be taken that way
/// returns [`ConditionalNext::Empty`].
fn get_next_if(&self, pred: &dyn Fn(&Task) -> bool) -> ConditionalNext;

/// Get the scheduler name.
fn scheduler_name(&self) -> SchedulerType;

Expand Down Expand Up @@ -258,6 +270,87 @@ pub(crate) fn get_next_task(execution_ensured: bool) -> Option<Arc<Task>> {
.find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured))
}

/// The result of [`Scheduler::get_next_if`].
pub(crate) enum ConditionalNext {
/// The run queue has no executable task.
Empty,

/// The head of the run queue was rejected and is left in the queue.
Rejected,

/// The head of the run queue was accepted and dequeued.
Taken(Arc<Task>),
}

/// Dequeue the next executable task only if `pred` accepts it.
///
/// This examines the task that `get_next_task(false)` would return, except
/// that panicked tasks are never taken. If `pred` returns `false` for it, the
/// run queues are left untouched, so the task keeps its position in the
/// queue, and `None` is returned.
///
/// `pred` is called with `GLOBAL_WAKE_GET_MUTEX`, the lock of the run queue,
/// and the lock of the task information held.
///
/// This function must be called on the primary CPU, which can never dequeue
/// a clustered task (see `get_next_task`), so only the non-clustered
/// schedulers are examined.
pub(crate) fn get_next_task_if(pred: impl Fn(&Task) -> bool) -> Option<Arc<Task>> {
let mut node = MCSNode::new();
let _guard = GLOBAL_WAKE_GET_MUTEX.lock(&mut node);

for scheduler_type in &PRIORITY_LIST[get_num_clustered_schedulers()..] {
match get_scheduler(scheduler_type).get_next_if(&pred) {
ConditionalNext::Empty => (),
ConditionalNext::Rejected => return None,
ConditionalNext::Taken(task) => return Some(task),
}
}

None
}

/// The body of [`Scheduler::get_next_if`] for the schedulers whose run queue
/// is a [`PriorityQueue`]. `task_of` returns the task of a queue entry.
///
/// Finished tasks at the head of the queue are discarded as `get_next()`
/// does. The lock of the run queue must be held by the caller.
fn take_head_if<T>(
queue: &mut PriorityQueue<T>,
task_of: impl Fn(&T) -> &Arc<Task>,
pred: &dyn Fn(&Task) -> bool,
) -> ConditionalNext {
loop {
let Some(task) = queue.peek().map(|entry| task_of(entry).clone()) else {
return ConditionalNext::Empty;
};

let mut node = MCSNode::new();
let mut task_info = task.info.lock(&mut node);

if matches!(task_info.state, State::Terminated | State::Panicked) {
drop(task_info);
queue.pop();
NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed);
continue;
}

if !pred(&task) {
return ConditionalNext::Rejected;
}

queue.pop();
NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed);

if task_info.state == State::Preempted {
task_info.need_preemption = false;
}

drop(task_info);
return ConditionalNext::Taken(task);
}
}

/// Get a scheduler.
///
/// Takes `&SchedulerType` so the caller does not copy the full payload (the
Expand Down Expand Up @@ -427,20 +520,37 @@ pub(crate) fn sleep_until_task(sleep_handler: Box<dyn FnOnce() + Send>, next: In
/// If there are no sleeping tasks, this function returns `None`.
pub fn wake_task() -> Option<Duration> {
// Check whether each running task exceeds the time quantum.
let mut quantum_wait: Option<Duration> = None;

if PREEMPTION_ENABLED {
for cpu_id in 1..num_cpu() {
if let Some(task_id) = get_current_task(cpu_id) {
if let Some(SchedulerType::PrioritizedRR(_)) =
get_scheduler_type_by_task_id(task_id)
{
prioritized_rr::SCHEDULER.invoke_preemption_tick(cpu_id, task_id)
let wait = prioritized_rr::SCHEDULER.invoke_preemption_tick(cpu_id, task_id);
quantum_wait = match (quantum_wait, wait) {
(Some(a), Some(b)) => Some(a.min(b)),
(a, b) => a.or(b),
};
}
}
}
}

// A queued task may be waiting for a time quantum to expire. Nothing
// else wakes the primary CPU at that time, so do not sleep beyond it.
// If no task is queued, the next enqueue wakes the primary CPU.
if NUM_TASK_IN_QUEUE.load(Ordering::Relaxed) == 0 {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This check runs before guard.wake_task(), so a task that a sleep handler enqueues in this same call is not counted, and quantum_wait is already dropped. wake_cpu(0) does nothing on CPU 0, so a timer-woken task with the same priority as the running PrioritizedRR tasks then waits for the next sleeper, or up to 1 s. Reading NUM_TASK_IN_QUEUE after guard.wake_task() closes this case.

quantum_wait = None;
}

let mut node = MCSNode::new();
let mut guard = SLEEPING.lock(&mut node);
guard.wake_task();
guard.time_to_wait()

match (guard.time_to_wait(), quantum_wait) {
(Some(a), Some(b)) => Some(a.min(b)),
(a, b) => a.or(b),
}
}
8 changes: 7 additions & 1 deletion awkernel_async_lib/src/scheduler/edf.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
//! ([`AffinityBTreeQueue`]); `get_next` on CPU `c` pops the earliest-deadline
//! task whose `cpu_set` contains `c`.

use super::{Scheduler, SchedulerType, Task};
use super::{ConditionalNext, Scheduler, SchedulerType, Task};
use crate::{
dag::calculate_and_update_dag_deadline,
scheduler::{
Expand Down Expand Up @@ -134,6 +134,12 @@ impl Scheduler for EDFScheduler {
}
}

fn get_next_if(&self, _pred: &dyn Fn(&Task) -> bool) -> ConditionalNext {
// `get_next_if` runs on the primary CPU. Spawn removes CPU 0 from
// every set, so no queued task is runnable there (see `get_next`).
ConditionalNext::Empty
}

fn scheduler_name(&self) -> SchedulerType {
SchedulerType::EDF(0, CpuSet::empty())
}
Expand Down
9 changes: 8 additions & 1 deletion awkernel_async_lib/src/scheduler/panicked.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! A scheduler for panicked tasks.
//! Panicked tasks will be the lowest priority.

use super::{Scheduler, SchedulerType, Task};
use super::{ConditionalNext, Scheduler, SchedulerType, Task};
use crate::{
scheduler::get_priority,
task::{set_current_task, State, NUM_TASK_IN_QUEUE},
Expand Down Expand Up @@ -82,6 +82,13 @@ impl Scheduler for PanickedScheduler {
}
}

fn get_next_if(&self, _pred: &dyn Fn(&Task) -> bool) -> ConditionalNext {
// Panicked tasks have the lowest priority, but `Task::priority` still
// holds the priority a task had before it panicked, so `pred` cannot
// judge them. They are never taken ahead of a running task.
ConditionalNext::Empty
}

fn scheduler_name(&self) -> SchedulerType {
SchedulerType::Panicked
}
Expand Down
12 changes: 11 additions & 1 deletion awkernel_async_lib/src/scheduler/prioritized_fifo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

use core::cmp::max;

use super::{Scheduler, SchedulerType, Task};
use super::{take_head_if, ConditionalNext, Scheduler, SchedulerType, Task};
use crate::scheduler::{
peek_preemption_pending, push_preemption_pending, GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED,
};
Expand Down Expand Up @@ -109,6 +109,16 @@ impl Scheduler for PrioritizedFIFOScheduler {
}
}

fn get_next_if(&self, pred: &dyn Fn(&Task) -> bool) -> ConditionalNext {
let mut node = MCSNode::new();
let mut data = self.data.lock(&mut node);

match (*data).as_mut() {
Some(data) => take_head_if(&mut data.queue, |entry| &entry.task, pred),
None => ConditionalNext::Empty,
}
}

fn scheduler_name(&self) -> SchedulerType {
SchedulerType::PrioritizedFIFO(0)
}
Expand Down
92 changes: 76 additions & 16 deletions awkernel_async_lib/src/scheduler/prioritized_rr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,22 @@

use core::cmp::max;

use super::{Scheduler, SchedulerType, Task};
use super::{take_head_if, ConditionalNext, Scheduler, SchedulerType, Task};
use crate::{
scheduler::{
get_next_task, get_priority, peek_preemption_pending, push_preemption_pending,
get_next_task_if, get_priority, peek_preemption_pending, push_preemption_pending,
GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED,
},
task::{
get_last_executed_by_task_id, get_task, get_tasks_running, set_current_task,
set_need_preemption, State, NUM_TASK_IN_QUEUE,
get_task, get_tasks_running, set_current_task, set_need_preemption, State,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This removes the only use of task::get_last_executed_by_task_id. The function is pub, so the compiler does not warn about it, and it can go in this PR too.

NUM_TASK_IN_QUEUE,
},
};
use alloc::{sync::Arc, vec::Vec};
use awkernel_lib::priority_queue::PriorityQueue;
use awkernel_lib::sync::mutex::{MCSNode, Mutex};
use core::sync::atomic::Ordering;
use core::time::Duration;

pub struct PrioritizedRRScheduler {
// Time quantum
Expand Down Expand Up @@ -111,6 +112,16 @@ impl Scheduler for PrioritizedRRScheduler {
None
}

fn get_next_if(&self, pred: &dyn Fn(&Task) -> bool) -> ConditionalNext {
let mut node = MCSNode::new();
let mut data = self.data.lock(&mut node);

match (*data).as_mut() {
Some(data) => take_head_if(&mut data.queue, |entry| &entry.task, pred),
None => ConditionalNext::Empty,
}
}

fn scheduler_name(&self) -> SchedulerType {
SchedulerType::PrioritizedRR(0)
}
Expand All @@ -128,19 +139,68 @@ pub static SCHEDULER: PrioritizedRRScheduler = PrioritizedRRScheduler {
};

impl PrioritizedRRScheduler {
// Invoke a preemption if the task exceeds the time quantum
pub fn invoke_preemption_tick(&self, cpu_id: usize, task_id: u32) {
if let Some(last_executed) = get_last_executed_by_task_id(task_id) {
let elapsed = last_executed.elapsed().as_micros() as u64;
if elapsed > self.interval {
if let Some(next_task) = get_next_task(false) {
push_preemption_pending(cpu_id, next_task);
let preempt_irq = awkernel_lib::interrupt::get_preempt_irq();
set_need_preemption(task_id, cpu_id);
awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32);
}
}
/// Rotate the PrioritizedRR task `task_id` running on `cpu_id` if it has
/// used up its time quantum.
///
/// The next runnable task is dequeued and sent to `cpu_id` only if its
/// priority is at least that of the running task: an equal priority is
/// the quantum rotation, and a higher priority is a plain preemption.
/// A lower-priority task could not preempt the running task anyway, so
/// it is left in the run queue to keep its position, and the quantum of
/// the running task is restarted instead.
///
/// Returns the time after which this function must be called again for
/// this CPU, or `None` if a preemption was requested: the preempted task
/// is enqueued again, and that wakes the primary CPU.
///
/// This function must be called on the primary CPU.
pub fn invoke_preemption_tick(&self, cpu_id: usize, task_id: u32) -> Option<Duration> {
let current_task = get_task(task_id)?;
let interval = Duration::from_micros(self.interval);

let elapsed = {
let mut node = MCSNode::new();
let info = current_task.info.lock(&mut node);
info.get_last_executed().elapsed()
};

if elapsed <= interval {
return Some(interval - elapsed);
}

// A task is dequeued only while no preemption is pending on this
// CPU: `do_preemption` runs one pending task and re-queues the others
// at the tail of their level. The predicate runs under
// `GLOBAL_WAKE_GET_MUTEX`, so no wake path can push a pending task
// between this check and the dequeue.
let next_task = get_next_task_if(|next| {
peek_preemption_pending(cpu_id).is_none() && !current_task.preempts(next)
});

if let Some(next_task) = next_task {
push_preemption_pending(cpu_id, next_task);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

get_next_task_if releases GLOBAL_WAKE_GET_MUTEX before this push, and nothing checks again that task_id still runs on cpu_id. If that task finishes in this window, the worker can see an empty queue and no pending task, and go to sleep. The task pushed here then stays in the pending heap of an idle CPU, because the preempt IPI does nothing there and wake_workers() does not count pending tasks. main has the same window, but the idle check in run_main assumes that every pin is pushed under the mutex, as the wake paths do. Adding get_current_task(cpu_id) == Some(task_id) to the predicate and pushing before the mutex is released (for example in a callback on Taken) closes it.

let preempt_irq = awkernel_lib::interrupt::get_preempt_irq();
set_need_preemption(task_id, cpu_id);
awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32);
return None;
}

if peek_preemption_pending(cpu_id).is_some() {
// A preemption has already been requested on this CPU, but it
// has not been performed yet. Request it again in case the IPI
// was not handled.
let preempt_irq = awkernel_lib::interrupt::get_preempt_irq();
set_need_preemption(task_id, cpu_id);
awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32);
} else {
// There is no task to rotate with. Restart the quantum so that
// the run queues are not examined again on every tick.
let mut node = MCSNode::new();
let mut info = current_task.info.lock(&mut node);
info.update_last_executed();
}

Some(interval)
}

fn invoke_preemption_wake(&self, task: Arc<Task>) -> bool {
Expand Down
8 changes: 4 additions & 4 deletions awkernel_async_lib/src/task.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,10 @@ impl Task {
///
/// The one exception is the PrioritizedRR time quantum:
/// `PrioritizedRRScheduler::invoke_preemption_tick` pushes the next
/// runnable task to the preemption-pending heap without a priority check,
/// and `preempt::do_preemption` lets such an equal-priority pending task
/// preempt a running PrioritizedRR task so that the quantum rotates tasks
/// of the same priority.
/// runnable task to the preemption-pending heap if its priority is at
/// least that of the running task, and `preempt::do_preemption` lets such
/// an equal-priority pending task preempt a running PrioritizedRR task so
/// that the quantum rotates tasks of the same priority.
///
/// This is deliberately not `self > other`: [`Ord`] for `Task` breaks
/// priority ties by task ID to give the preemption-pending heaps a total
Expand Down
Loading
Loading