Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 5 additions & 14 deletions awkernel_async_lib/src/scheduler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,8 @@ const fn get_num_clustered_schedulers() -> usize {
count
}

// `get_next_task` and `NUM_TASK_IN_QUEUE` accounting rely on the clustered
// schedulers forming a strict prefix of `PRIORITY_LIST`. Reject any reorder
// at compile time.
// `get_next_task` relies on the clustered schedulers forming a strict prefix
// of `PRIORITY_LIST`. Reject any reorder at compile time.
const _: () = {
let mut i = get_num_clustered_schedulers();
while i < PRIORITY_LIST.len() {
Expand Down Expand Up @@ -253,18 +252,10 @@ pub(crate) fn get_next_task(execution_ensured: bool) -> Option<Arc<Task>> {
// Skip the clustered prefix: it was either just tried above and returned
// None (the queue state cannot change in between because both `wake_task`
// and `get_next_task` hold `GLOBAL_WAKE_GET_MUTEX`), or the counter was 0
// and its queues are empty. This also structurally guarantees that the
// `NUM_TASK_IN_QUEUE` decrement below never applies to a clustered task,
// which is counted by `NUM_CLUSTERED_TASKS_IN_QUEUE` instead.
let task = PRIORITY_LIST[get_num_clustered_schedulers()..]
// and its queues are empty.
PRIORITY_LIST[get_num_clustered_schedulers()..]
.iter()
.find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured));

if task.is_some() {
crate::task::NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed);
}

task
.find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured))
}

/// Get a scheduler.
Expand Down
6 changes: 3 additions & 3 deletions awkernel_async_lib/src/scheduler/edf.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,9 @@ impl Scheduler for EDFScheduler {
.expect("EDF: failed to push a task");
drop(data);

// `do_preemption` and `run_main` re-wake tasks through this function
// without `ArcWake::wake`, so notify the primary CPU here to wake an
// idle worker in the set.
// Every enqueue goes through this function, including the re-wakes
// that bypass `ArcWake::wake`, so notify the primary CPU here to
// wake an idle worker in the set.
awkernel_lib::cpu::wake_cpu(0);
}
}
Expand Down
21 changes: 13 additions & 8 deletions awkernel_async_lib/src/scheduler/panicked.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
use super::{Scheduler, SchedulerType, Task};
use crate::{
scheduler::get_priority,
task::{set_current_task, State},
task::{set_current_task, State, NUM_TASK_IN_QUEUE},
};
use alloc::{collections::VecDeque, sync::Arc};
use awkernel_lib::sync::mutex::{MCSNode, Mutex};
use core::sync::atomic::Ordering;

pub struct PanickedScheduler {
data: Mutex<Option<PanickedData>>, // Run queue.
Expand All @@ -31,13 +32,16 @@ impl Scheduler for PanickedScheduler {
let mut node = MCSNode::new();
let mut data = self.data.lock(&mut node);

if let Some(data) = data.as_mut() {
data.queue.push_back(task);
} else {
let mut panicked_data = PanickedData::new();
panicked_data.queue.push_back(task);
*data = Some(panicked_data);
}
data.get_or_insert_with(PanickedData::new)
.queue
.push_back(task);
NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed);
drop(data);

// Every enqueue goes through this function, including the re-wakes
// that bypass `ArcWake::wake`, so notify the primary CPU here to
// wake an idle worker.
awkernel_lib::cpu::wake_cpu(0);
}

fn get_next(&self, execution_ensured: bool) -> Option<Arc<Task>> {
Expand All @@ -54,6 +58,7 @@ impl Scheduler for PanickedScheduler {

loop {
let task = data.queue.pop_front()?;
NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed);

// Make the state of the task Running.
{
Expand Down
13 changes: 12 additions & 1 deletion awkernel_async_lib/src/scheduler/prioritized_fifo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,15 @@ use super::{Scheduler, SchedulerType, Task};
use crate::scheduler::{
peek_preemption_pending, push_preemption_pending, GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED,
};
use crate::task::{get_task, get_tasks_running, set_current_task, set_need_preemption};
use crate::task::{
get_task, get_tasks_running, set_current_task, set_need_preemption, NUM_TASK_IN_QUEUE,
};
use crate::{scheduler::get_priority, task::State};
use alloc::sync::Arc;
use alloc::vec::Vec;
use awkernel_lib::priority_queue::PriorityQueue;
use awkernel_lib::sync::mutex::{MCSNode, Mutex};
use core::sync::atomic::Ordering;

pub struct PrioritizedFIFOScheduler {
data: Mutex<Option<PrioritizedFIFOData>>, // Run queue.
Expand Down Expand Up @@ -59,6 +62,13 @@ impl Scheduler for PrioritizedFIFOScheduler {
_priority: priority,
},
);
NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed);
drop(data);

// Every enqueue goes through this function, including the re-wakes
// that bypass `ArcWake::wake`, so notify the primary CPU here to
// wake an idle worker.
awkernel_lib::cpu::wake_cpu(0);
}
}

Expand All @@ -75,6 +85,7 @@ impl Scheduler for PrioritizedFIFOScheduler {
loop {
// Pop a task from the run queue.
let task = data.queue.pop()?;
NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed);

// Make the state of the task Running.
{
Expand Down
11 changes: 10 additions & 1 deletion awkernel_async_lib/src/scheduler/prioritized_rr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,13 @@ use crate::{
},
task::{
get_last_executed_by_task_id, get_task, get_tasks_running, set_current_task,
set_need_preemption, State,
set_need_preemption, State, NUM_TASK_IN_QUEUE,
},
};
use alloc::{sync::Arc, vec::Vec};
use awkernel_lib::priority_queue::PriorityQueue;
use awkernel_lib::sync::mutex::{MCSNode, Mutex};
use core::sync::atomic::Ordering;

pub struct PrioritizedRRScheduler {
// Time quantum
Expand Down Expand Up @@ -65,6 +66,13 @@ impl Scheduler for PrioritizedRRScheduler {
_priority: priority,
},
);
NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Relaxed);
drop(data);

// Every enqueue goes through this function, including the re-wakes
// that bypass `ArcWake::wake`, so notify the primary CPU here to
// wake an idle worker.
awkernel_lib::cpu::wake_cpu(0);
}
}

Expand All @@ -79,6 +87,7 @@ impl Scheduler for PrioritizedRRScheduler {
};

while let Some(rr_task) = data.queue.pop() {
NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed);
{
let mut node = MCSNode::new();
let mut task_info = rr_task.task.info.lock(&mut node);
Expand Down
21 changes: 6 additions & 15 deletions awkernel_async_lib/src/task.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,17 +50,18 @@ pub type TaskResult = Result<(), Cow<'static, str>>;
static TASKS: Mutex<Tasks> = Mutex::new(Tasks::new()); // Set of tasks.
static RUNNING: [AtomicU32; NUM_MAX_CPU] = array![_ => AtomicU32::new(0); NUM_MAX_CPU]; // IDs of running tasks.
pub(crate) static MAX_TASK_PRIORITY: u64 = (1 << 56) - 1; // Maximum task priority.
#[cfg(target_pointer_width = "64")]
pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); // Number of tasks in the queue.

/// Total number of tasks in the queues of the non-clustered schedulers.
/// Each of these schedulers increments it when it pushes a task to its queue
/// and decrements it when it pops one, so it counts every path that calls
/// `wake_task`, including the re-wakes that bypass `ArcWake::wake`.
pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0);

/// Total number of tasks in the clustered schedulers' queues.
/// Which CPUs those tasks are runnable on is tracked exactly by the run
/// queues themselves; see `scheduler::clustered_queued_cpu_mask`.
pub(crate) static NUM_CLUSTERED_TASKS_IN_QUEUE: AtomicU32 = AtomicU32::new(0);

#[cfg(target_pointer_width = "32")]
pub(crate) static NUM_TASK_IN_QUEUE: AtomicU32 = AtomicU32::new(0); // Number of tasks in the queue.

static PREEMPTION_REQUEST: [AtomicBool; NUM_MAX_CPU] =
array![_ => AtomicBool::new(false); NUM_MAX_CPU];

Expand Down Expand Up @@ -165,21 +166,11 @@ impl ArcWake for Task {
panicked = info.panicked;
}

// Panicked tasks go to the global panicked scheduler, so always use the global counter.
// Non-panicked clustered tasks: counters are managed inside edf::wake_task.
// Non-panicked non-clustered tasks: increment the global counter here.
if panicked || self.cpu_set.is_none() {
NUM_TASK_IN_QUEUE.fetch_add(1, Ordering::Release);
}

if panicked {
scheduler::panicked::SCHEDULER.wake_task(self);
} else {
self.scheduler.wake_task(self);
}

// Notify the primary CPU to wake up workers.
awkernel_lib::cpu::wake_cpu(0);
}
}

Expand Down
8 changes: 5 additions & 3 deletions kernel/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,11 @@ fn main<Info: Debug>(kernel_info: KernelInfo<Info>) {

let dur = wake_task(); // Wake executable tasks periodically.

// Wake up other CPUs if there are any tasks to run. This runs before
// the sleep, because `wake_cpu(0)` on CPU 0 does nothing, so a task
// that `wake_task()` enqueued gets no other notification.
awkernel_async_lib::task::wake_workers();

#[cfg(feature = "std")]
{
let dur = dur.unwrap_or(core::time::Duration::from_secs(1));
Expand All @@ -133,9 +138,6 @@ fn main<Info: Debug>(kernel_info: KernelInfo<Info>) {

#[cfg(feature = "perf")]
awkernel_async_lib::task::perf::start_kernel();

// Wake up other CPUs if there are any tasks to run.
awkernel_async_lib::task::wake_workers();
}
}

Expand Down
Loading