Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 17 additions & 118 deletions awkernel_async_lib/src/scheduler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use awkernel_lib::{
#[cfg(not(feature = "std"))]
use alloc::boxed::Box;

mod edf;
pub(super) mod edf;
pub(super) mod panicked;
mod prioritized_fifo;
mod prioritized_rr;
Expand Down Expand Up @@ -118,14 +118,6 @@ impl SchedulerType {
_ => None,
}
}

/// True if this is a clustered scheduler, i.e. one whose tasks carry a
/// CPU affinity set. Update this function when adding a new clustered
/// scheduler; it is the single source of truth for the clustered prefix
/// of `PRIORITY_LIST`.
pub const fn is_clustered(&self) -> bool {
matches!(self, SchedulerType::EDF(_, _))
}
}

/// # Priority
Expand All @@ -147,31 +139,12 @@ static PRIORITY_LIST: [SchedulerType; 4] = [
SchedulerType::Panicked,
];

/// Return the number of clustered schedulers in `PRIORITY_LIST`.
const fn get_num_clustered_schedulers() -> usize {
let mut count = 0;
while count < PRIORITY_LIST.len() {
if PRIORITY_LIST[count].is_clustered() {
count += 1;
} else {
break;
}
}
count
}

// `get_next_task` relies on the clustered schedulers forming a strict prefix
// of `PRIORITY_LIST`. Reject any reorder at compile time.
const _: () = {
let mut i = get_num_clustered_schedulers();
while i < PRIORITY_LIST.len() {
assert!(
!PRIORITY_LIST[i].is_clustered(),
"clustered schedulers must form a strict prefix of PRIORITY_LIST"
);
i += 1;
}
};
// `get_next_task` tries the EDF scheduler before `PRIORITY_LIST[1..]`, so the
// EDF scheduler must have the highest priority.
const _: () = assert!(
matches!(PRIORITY_LIST[0], SchedulerType::EDF(_, _)),
"the EDF scheduler must be first in PRIORITY_LIST"
);

/// When false, no scheduler requests preemption, and `wake_task` enqueues the woken task.
/// `no_preempt` compiles out the IPI handler that runs preemption-pending tasks.
Expand All @@ -183,17 +156,6 @@ pub(crate) const PREEMPTION_ENABLED: bool = cfg!(not(feature = "no_preempt"));
/// the decision to preempt, dequeuing, enqueuing, and updating of RUNNING must be executed exclusively.
static GLOBAL_WAKE_GET_MUTEX: Mutex<()> = Mutex::new(());

/// # Implementing a clustered scheduler
///
/// A clustered scheduler keeps tasks that may run only on a subset of CPUs.
/// Every clustered scheduler must:
///
/// 1. Wrap each entry of its run queue in [`ClusteredTask`] so that
/// `NUM_CLUSTERED_TASKS_IN_QUEUE` is kept consistent.
/// 2. Implement [`Scheduler::queued_cpu_mask`] to report the exact set of
/// CPUs its queued tasks are runnable on (`wake_workers` relies on it).
/// 3. Be placed in the clustered prefix of `PRIORITY_LIST` and be matched by
/// `SchedulerType::is_clustered` (checked at compile time).
pub(crate) trait Scheduler {
/// Enqueue an executable task.
/// The enqueued task will be taken by `get_next()`.
Expand All @@ -207,23 +169,6 @@ pub(crate) trait Scheduler {

#[allow(dead_code)] // TODO: to be removed
fn priority(&self) -> u8;

/// The set of CPUs that have at least one queued task eligible to run on
/// them. Non-clustered schedulers use the default (empty) implementation;
/// clustered schedulers must override it.
fn queued_cpu_mask(&self) -> CpuSet {
CpuSet::empty()
}
}

/// Return the union of [`Scheduler::queued_cpu_mask`] over all clustered
/// schedulers: the exact set of CPUs for which a clustered task is queued.
pub(crate) fn clustered_queued_cpu_mask() -> CpuSet {
let mut mask = CpuSet::empty();
for scheduler_type in &PRIORITY_LIST[..get_num_clustered_schedulers()] {
mask = mask.union(get_scheduler(scheduler_type).queued_cpu_mask());
}
mask
}

/// Get the next executable task.
Expand All @@ -232,28 +177,22 @@ pub(crate) fn get_next_task(execution_ensured: bool) -> Option<Arc<Task>> {
let mut node = MCSNode::new();
let _guard = GLOBAL_WAKE_GET_MUTEX.lock(&mut node);

let num_clustered_tasks = crate::task::NUM_CLUSTERED_TASKS_IN_QUEUE.load(Ordering::Relaxed);

// The counter is global: it may be non-zero even when no queued clustered
// task is runnable on this CPU. That is fine because the run queues fail
// fast (`pop_for_cpu` checks the root affinity mask in O(1)).
if num_clustered_tasks > 0 {
let task = PRIORITY_LIST[..get_num_clustered_schedulers()]
.iter()
.find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured));

// The counter is decremented by the ClusteredTask::take() call inside
// The counter may be non-zero even when no queued EDF task is runnable on
// this CPU. That is fine because the run queue fails fast (`pop_for_cpu`
// checks the root affinity mask in O(1)).
if crate::task::NUM_EDF_TASKS_IN_QUEUE.load(Ordering::Relaxed) > 0 {
// The counter is decremented by the QueuedTask::take() call inside
// get_next() (Drop is a no-op once take() has run).
if task.is_some() {
return task;
if let Some(task) = edf::SCHEDULER.get_next(execution_ensured) {
return Some(task);
}
}

// Skip the clustered prefix: it was either just tried above and returned
// Skip the EDF scheduler: it was either just tried above and returned
// None (the queue state cannot change in between because both `wake_task`
// and `get_next_task` hold `GLOBAL_WAKE_GET_MUTEX`), or the counter was 0
// and its queues are empty.
PRIORITY_LIST[get_num_clustered_schedulers()..]
// and its queue is empty.
PRIORITY_LIST[1..]
.iter()
.find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured))
}
Expand Down Expand Up @@ -283,46 +222,6 @@ pub const fn get_priority(sched_type: &SchedulerType) -> u8 {
panic!("Scheduler type not registered in PRIORITY_LIST or equals()")
}

/// RAII wrapper representing one slot in `NUM_CLUSTERED_TASKS_IN_QUEUE`.
///
/// Constructing a `ClusteredTask` increments the counter. The counter is
/// decremented exactly once — either via [`take`] (explicit ownership
/// transfer) or via `Drop` (e.g. when a terminated task is discarded). If
/// `take` has already been called, `Drop` is a no-op.
///
/// [`take`]: ClusteredTask::take
pub(crate) struct ClusteredTask<T> {
inner: Option<T>,
}

impl<T> ClusteredTask<T> {
pub(crate) fn new(inner: T) -> Self {
crate::task::NUM_CLUSTERED_TASKS_IN_QUEUE.fetch_add(1, Ordering::Relaxed);
Self { inner: Some(inner) }
}

/// Take the inner value and decrement the counter.
///
/// Returns `None` if the value has already been taken.
/// After this call `Drop` will be a no-op.
pub(crate) fn take(&mut self) -> Option<T> {
let val = self.inner.take();
if val.is_some() {
crate::task::NUM_CLUSTERED_TASKS_IN_QUEUE.fetch_sub(1, Ordering::Relaxed);
}
val
}
}

impl<T> Drop for ClusteredTask<T> {
fn drop(&mut self) {
// Decrement only if take() has not been called yet.
if self.inner.is_some() {
crate::task::NUM_CLUSTERED_TASKS_IN_QUEUE.fetch_sub(1, Ordering::Relaxed);
}
}
}

/// Maintain sleeping tasks by a delta list.
struct SleepingTasks {
delta_list: DeltaList<Box<dyn FnOnce() + Send>>,
Expand Down
76 changes: 60 additions & 16 deletions awkernel_async_lib/src/scheduler/edf.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,12 @@ use super::{Scheduler, SchedulerType, Task};
use crate::{
dag::calculate_and_update_dag_deadline,
scheduler::{
get_priority, peek_preemption_pending, push_preemption_pending, ClusteredTask,
GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED,
get_priority, peek_preemption_pending, push_preemption_pending, GLOBAL_WAKE_GET_MUTEX,
PREEMPTION_ENABLED,
},
task::{
get_task, get_task_running, set_current_task, set_need_preemption, State, MAX_TASK_PRIORITY,
get_task, get_task_running, set_current_task, set_need_preemption, State,
MAX_TASK_PRIORITY, NUM_EDF_TASKS_IN_QUEUE,
},
};
use affinity_btree_queue::{AffinityBTreeQueue, DEFAULT_MIN_DEGREE};
Expand All @@ -23,12 +24,13 @@ use awkernel_lib::{
cpu::{masked_workers, num_cpu, CpuSet, CPU_SET_WORDS},
sync::mutex::{MCSNode, Mutex},
};
use core::sync::atomic::Ordering;

/// The run queue. Priority is `(absolute_deadline, wake_time)`; smaller values
/// dequeue first, and entries with fully equal keys are FIFO-ordered by the
/// queue's internal sequence number.
type EDFQueue =
AffinityBTreeQueue<(u64, u64), ClusteredTask<Arc<Task>>, DEFAULT_MIN_DEGREE, CPU_SET_WORDS>;
AffinityBTreeQueue<(u64, u64), QueuedTask<Arc<Task>>, DEFAULT_MIN_DEGREE, CPU_SET_WORDS>;

pub struct EDFScheduler {
// `AffinityBTreeQueue::new` is not a const fn, so the queue is lazily
Expand All @@ -37,6 +39,46 @@ pub struct EDFScheduler {
priority: u8,
}

/// RAII wrapper representing one slot in `NUM_EDF_TASKS_IN_QUEUE`.
///
/// Constructing a `QueuedTask` increments the counter. The counter is
/// decremented exactly once — either via [`take`] (explicit ownership
/// transfer) or via `Drop` (e.g. when a terminated task is discarded). If
/// `take` has already been called, `Drop` is a no-op.
///
/// [`take`]: QueuedTask::take
struct QueuedTask<T> {
inner: Option<T>,
}

impl<T> QueuedTask<T> {
fn new(inner: T) -> Self {
NUM_EDF_TASKS_IN_QUEUE.fetch_add(1, Ordering::Relaxed);
Self { inner: Some(inner) }
}

/// Take the inner value and decrement the counter.
///
/// Returns `None` if the value has already been taken.
/// After this call `Drop` will be a no-op.
fn take(&mut self) -> Option<T> {
let val = self.inner.take();
if val.is_some() {
NUM_EDF_TASKS_IN_QUEUE.fetch_sub(1, Ordering::Relaxed);
}
val
}
}

impl<T> Drop for QueuedTask<T> {
fn drop(&mut self) {
// Decrement only if take() has not been called yet.
if self.inner.is_some() {
NUM_EDF_TASKS_IN_QUEUE.fetch_sub(1, Ordering::Relaxed);
}
}
}

impl Scheduler for EDFScheduler {
fn wake_task(&self, task: Arc<Task>) {
let (wake_time, absolute_deadline) = {
Expand Down Expand Up @@ -83,7 +125,7 @@ impl Scheduler for EDFScheduler {
.push(
(absolute_deadline, wake_time),
cpu_set,
ClusteredTask::new(task),
QueuedTask::new(task),
)
.expect("EDF: failed to push a task");
drop(data);
Expand All @@ -104,7 +146,7 @@ impl Scheduler for EDFScheduler {

// Note: spawn removes CPU 0 from every set, so `pop_for_cpu(0)` always
// returns `None` and the `get_next_task(false)` path executed on the
// primary CPU can never dequeue a clustered task.
// primary CPU can never dequeue an EDF task.
loop {
let (_, _, mut entry) = queue.pop_for_cpu(cpu_id)?;

Expand Down Expand Up @@ -141,8 +183,17 @@ impl Scheduler for EDFScheduler {
fn priority(&self) -> u8 {
self.priority
}
}

fn queued_cpu_mask(&self) -> CpuSet {
pub static SCHEDULER: EDFScheduler = EDFScheduler {
data: Mutex::new(None),
priority: get_priority(&SchedulerType::EDF(0, CpuSet::empty())),
};

impl EDFScheduler {
/// The set of CPUs that have at least one queued task eligible to run on
/// them.
pub(crate) fn queued_cpu_mask(&self) -> CpuSet {
let mut node = MCSNode::new();
let data = self.data.lock(&mut node);

Expand All @@ -153,14 +204,7 @@ impl Scheduler for EDFScheduler {
None => CpuSet::empty(),
}
}
}

pub static SCHEDULER: EDFScheduler = EDFScheduler {
data: Mutex::new(None),
priority: get_priority(&SchedulerType::EDF(0, CpuSet::empty())),
};

impl EDFScheduler {
fn invoke_preemption(&self, task: Arc<Task>) -> bool {
if !PREEMPTION_ENABLED {
return false;
Expand Down Expand Up @@ -221,8 +265,8 @@ impl EDFScheduler {

// NOTE: The task is now pinned to `victim_cpu`'s preemption-pending
// heap and is not in the affinity queue, so it is invisible to the
// other CPUs in its `cpu_set` (neither `pop_for_cpu`, the clustered
// task counter, nor `queued_cpu_mask` sees it). If another CPU in
// other CPUs in its `cpu_set` (neither `pop_for_cpu`,
// `NUM_EDF_TASKS_IN_QUEUE`, nor `queued_cpu_mask` sees it). If another CPU in
// the set is or becomes idle while the IPI is in flight, it cannot pick
// the task up and may go to sleep — a latency window bounded by the
// IPI delivery plus the victim's interrupt-disabled sections. This
Expand Down
22 changes: 11 additions & 11 deletions awkernel_async_lib/src/task.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,16 +51,16 @@ static TASKS: Mutex<Tasks> = Mutex::new(Tasks::new()); // Set of tasks.
static RUNNING: [AtomicU32; NUM_MAX_CPU] = array![_ => AtomicU32::new(0); NUM_MAX_CPU]; // IDs of running tasks.
pub(crate) static MAX_TASK_PRIORITY: u64 = (1 << 56) - 1; // Maximum task priority.

/// Total number of tasks in the queues of the non-clustered schedulers.
/// Total number of tasks in the queues of the schedulers other than EDF.
/// Each of these schedulers increments it when it pushes a task to its queue
/// and decrements it when it pops one, so it counts every path that calls
/// `wake_task`, including the re-wakes that bypass `ArcWake::wake`.
pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0);

/// Total number of tasks in the clustered schedulers' queues.
/// Total number of tasks in the queue of the EDF scheduler.
/// Which CPUs those tasks are runnable on is tracked exactly by the run
/// queues themselves; see `scheduler::clustered_queued_cpu_mask`.
pub(crate) static NUM_CLUSTERED_TASKS_IN_QUEUE: AtomicU32 = AtomicU32::new(0);
/// queue itself; see `EDFScheduler::queued_cpu_mask`.
pub(crate) static NUM_EDF_TASKS_IN_QUEUE: AtomicU32 = AtomicU32::new(0);

static PREEMPTION_REQUEST: [AtomicBool; NUM_MAX_CPU] =
array![_ => AtomicBool::new(false); NUM_MAX_CPU];
Expand All @@ -73,7 +73,7 @@ pub struct Task {
pub info: Mutex<TaskInfo>,
scheduler: &'static dyn Scheduler,
pub priority: PriorityInfo,
pub cpu_set: Option<CpuSet>, // The set of cores on which the task may run in clustered scheduling. None if not assigned.
pub cpu_set: Option<CpuSet>, // The set of cores on which an EDF task may run. None for the other schedulers.
}

impl Task {
Expand Down Expand Up @@ -306,7 +306,7 @@ impl Tasks {
// Validate and normalise the CPU set before creating TaskInfo so that
// info.scheduler_type and task.cpu_set stay in sync.
let cpu_set = if let SchedulerType::EDF(deadline, set) = scheduler_type {
// CPU 0 is the primary core and cannot run clustered tasks.
// CPU 0 is the primary core and cannot run EDF tasks.
let masked = masked_workers(set, num_cpu());
let normalized = if masked.is_empty() {
// `cpu::sanity_check()` guarantees num_cpu() >= 2 at boot,
Expand Down Expand Up @@ -1663,18 +1663,18 @@ pub fn wake_workers() {
let num_cpus = awkernel_lib::cpu::num_cpu();
let mut num_tasks = NUM_TASK_IN_QUEUE.load(Ordering::Relaxed);

// The exact set of CPUs for which a clustered task is queued, read from
// the clustered schedulers' run queues.
let clustered_mask = crate::scheduler::clustered_queued_cpu_mask();
// The exact set of CPUs for which an EDF task is queued, read from the
// run queue of the EDF scheduler.
let edf_mask = crate::scheduler::edf::SCHEDULER.queued_cpu_mask();

for i in 1..num_cpus {
if clustered_mask.contains(i) {
if edf_mask.contains(i) {
awkernel_lib::cpu::wake_cpu(i);
continue;
}

// Even if there are no global tasks left, keep scanning: a
// higher-numbered CPU may still have clustered tasks waiting and
// higher-numbered CPU may still have EDF tasks waiting and
// skipping the rest would leave it asleep (lost wakeup).
if num_tasks == 0 {
continue;
Expand Down
Loading
Loading