diff --git a/src/ex/fireworks.zig b/src/ex/fireworks.zig index 26911db..7287dbe 100644 --- a/src/ex/fireworks.zig +++ b/src/ex/fireworks.zig @@ -170,7 +170,7 @@ fn make_thread(entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) *ke.Thread ret.init(@intFromPtr(stack), r.kib(16), entry, arg); ret.priority = ke.Thread.Priority.default; - ret.priority_class = .Batch; + ret.base_priority = ke.Thread.Priority.default; return ret; } diff --git a/src/kern/init.zig b/src/kern/init.zig index d6c734a..94009a4 100644 --- a/src/kern/init.zig +++ b/src/kern/init.zig @@ -17,7 +17,7 @@ pub fn init(boot_info: *pl.BootInfo) linksection(r.init) void { thread0.init(@intFromPtr(&stack), r.kib(16), ki.sched.idle, null); thread0.priority = 0; - thread0.priority_class = .Idle; + thread0.base_priority = 0; thread0.pinned = true; ki.sched.percpu.local().current_thread = &thread0; diff --git a/src/kern/sched.zig b/src/kern/sched.zig index 1a5b73f..3771293 100644 --- a/src/kern/sched.zig +++ b/src/kern/sched.zig @@ -60,8 +60,8 @@ //! always preempt batch and idle threads. //! //! Remote preemption (sending an IPI to another CPU) is done lock-free -//! by reading a packed `current_thread_status` field that encodes the -//! running thread's priority and interactivity in a single atomic byte. +//! by reading atomically a `current_thread_prio` field that stores the +//! running thread's priority. //! //! ## Load Balancing //! @@ -95,7 +95,7 @@ comptime { const runqueues_n = 64; const interactivity_threshold = 30; const scaling_factor = 50; -const preempt_threshold = ke.Thread.Priority.realtime; +const preempt_threshold = ke.Thread.Priority.low_realtime; const balance_interval = @as(u64, config.sched_balance_interval); const steal_threshold = 1; @@ -111,11 +111,6 @@ pub const RunQueue = struct { queues: [runqueues_n]rtl.List, }; -const ThreadStatus = packed struct(u8) { - interactive: bool, - priority: u7, -}; - pub const PreemptionReason = enum(u8) { None, /// A higher priority thread was readied. @@ -140,9 +135,8 @@ pub const PerCpu = struct { insidx: u8, /// Used to indicate to this CPU it can steal work steal_work: bool, - /// Atomic byte for getting the current thread's status locklessly. - /// 1 bit for thread interactivity, 7 for priority. - current_thread_status: std.atomic.Value(ThreadStatus), + /// Atomic byte for getting the current thread's priority locklessly. + current_thread_prio: std.atomic.Value(u8), /// DPC for rescheduling. resched_dpc: ke.Dpc, /// Timer for quantum expiration. @@ -182,6 +176,7 @@ pub fn handle_preemption(cpu: *PerCpu) void { // Put it back in the queue. cpu.queues_lock.acquire_no_ipl(); insert_in_queue(cpu, cur, false); + cur.cpu = ke.cpu.current(); cpu.queues_lock.release_no_ipl(); } @@ -212,7 +207,7 @@ pub fn clock(_: ?*anyopaque) void { cpu.insidx = (cpu.insidx + 1) % runqueues_n; } - if (curtd.priority_class == .Batch) { + if (curtd.priority_class() == .Batch) { // Update interactivity metrics. curtd.run_time += config.sched_timeslice; @@ -270,7 +265,7 @@ pub fn unblock(td: *ke.Thread) void { td.sleep_time = (ke.time.read_time() - td.sleep_start) / std.time.ns_per_ms; - if (td.sleep_time >= config.sched_timeslice and td.priority_class == .Batch) { + if (td.sleep_time >= config.sched_timeslice and td.priority_class() == .Batch) { // If we have slept for more than a tick, update interactivity. clamp_time(td); recompute_priority(td); @@ -283,6 +278,52 @@ pub fn unblock(td: *ke.Thread) void { td.lock.release(ipl); } +pub fn update_priority_locked(td: *ke.Thread, new_prio: u8) void { + std.debug.assert(td.lock.is_locked()); + + const old_prio = td.priority; + td.priority = new_prio; + + const c = td.cpu orelse return; + const cpu = percpu.remote(c); + + cpu.queues_lock.acquire_no_ipl(); + defer cpu.queues_lock.release_no_ipl(); + + if (td.state == .Ready) { + // Remove it from its queue and add it back. + remove_from_queue(cpu, td); + insert_in_queue(cpu, td, false); + } + + if (td.state == .Running) { + std.debug.assert(td.last_cpu == c); + + cpu.current_thread_prio.store(new_prio, .monotonic); + + // Do nothing else on promotion. + if (new_prio >= old_prio) return; + + // Demotion: try to preempt the thread if possible. + if (cpu.next_thread != null) return; + + // Only try picking from the realtime queue as batch threads don't preempt each other. + const next = pick_realtime_thread(cpu, new_prio + 1, false) orelse return; + _ = cpu.load.fetchSub(1, .monotonic); + + cpu.next_thread = next; + cpu.preemption_reason = .HigherPriority; + ki.ipl.set_softint_pending(c, .Dispatch); + + if (c != ke.cpu.current()) { + ki.impl.send_resched_ipi(c); + } + } + + // XXX: handle selected threads? + +} + export const sched_percpu_init linksection(r.percpu_init) = &init_cpu; /// Initialize a CPU for use by the scheduler. @@ -298,7 +339,7 @@ fn init_cpu() linksection(r.init) callconv(.c) void { .idle_queue = undefined, .steal_work = false, .load = .init(0), - .current_thread_status = .init(.{ .interactive = false, .priority = 0 }), + .current_thread_prio = .init(0), .resched_dpc = .init(ki.sched.clock), .resched_timer = undefined, .start_timer = false, @@ -360,6 +401,8 @@ fn pick_realtime_thread(cpu: *PerCpu, minprio: u8, migrate: bool) ?*ke.Thread { // Get the first thread of the queue. var td: *ke.Thread = @fieldParentPtr("runq_link", curr_queue.first()); + td.state = .Selected; + if (migrate) { // Find the first thread that is not pinned. while (td.pinned) { @@ -414,6 +457,8 @@ fn pick_batch_thread(cpu: *PerCpu, migrate: bool) ?*ke.Thread { // Get the first thread of the queue. var td: *ke.Thread = @fieldParentPtr("runq_link", curr_queue.first()); + td.state = .Selected; + if (migrate) { // Find the first thread that is not pinned. while (td.pinned) { @@ -455,6 +500,8 @@ fn pick_idle_thread(cpu: *PerCpu, migrate: bool) ?*ke.Thread { return null; } + td.state = .Selected; + // We can safely remove it. td.runq_link.remove(); @@ -469,7 +516,9 @@ fn insert_in_list(list: *rtl.List, link: *rtl.List.Entry, head: bool) void { fn insert_in_queue(cpu: *PerCpu, td: *ke.Thread, head: bool) void { std.debug.assert(cpu.queues_lock.is_locked()); - if (td.priority_class == .Realtime or td.interactive) { + td.cpu = null; + + if (td.priority_class() == .Realtime or td.is_interactive()) { // Insert in realtime queue. cpu.realtime_queue.status |= (@as(u64, 1) << @intCast(td.priority)); @@ -477,17 +526,17 @@ fn insert_in_queue(cpu: *PerCpu, td: *ke.Thread, head: bool) void { insert_in_list(&cpu.realtime_queue.queues[td.priority], &td.runq_link, head); td.runq = &cpu.realtime_queue; td.runq_idx = @intCast(td.priority); - } else if (td.priority_class == .Batch) { + } else if (td.priority_class() == .Batch) { // Insert in calendar queue. // The insertion index is determined by insidx and the the priority of the thread, // higher priority threads will be put closer to insidx, which ensures that they are ran more frequently. const idx = (cpu.insidx + (ke.Thread.Priority.high_batch - td.priority)) % runqueues_n; - td.runq_idx = @intCast(idx); cpu.calendar_queue.status |= (@as(u64, 1) << @intCast(idx)); insert_in_list(&cpu.calendar_queue.queues[idx], &td.runq_link, head); td.runq = &cpu.calendar_queue; + td.runq_idx = @intCast(idx); } else { // Insert in idle queue. insert_in_list(&cpu.idle_queue, &td.runq_link, head); @@ -592,9 +641,7 @@ fn recompute_priority(td: *ke.Thread) void { // Choose a priority based on score, the lower the score, // the higher the priority it will be. // This is a simple formula I came up with that is probably good enough. - td.priority = @intCast(ke.Thread.Priority.interactive + ((interactivity_threshold - score) / 4)); - td.interactive = true; - + td.priority = @intCast(ke.Thread.Priority.low_interactive + ((interactivity_threshold - score) / 2)); return; } @@ -625,18 +672,17 @@ fn recompute_priority(td: *ke.Thread) void { ke.Thread.Priority.low_batch, ke.Thread.Priority.high_batch, )); - td.interactive = false; } // Return whether or not a thread with given priority and interactivity should get preempted by `td`. -fn should_prio_preempt(td: *ke.Thread, other_prio: u8, other_interactive: bool) bool { +fn should_prio_preempt(td: *ke.Thread, other_prio: u8) bool { const class = ke.Thread.Priority.class_from_prio(other_prio); if (class == .Idle) { return true; } - if (class == .Batch and !other_interactive and td.interactive) { + if (class == .Batch and td.is_interactive() and other_prio < ke.Thread.Priority.low_interactive) { // Interactive threads always preempt batch non-interactive ones. return true; } @@ -644,7 +690,7 @@ fn should_prio_preempt(td: *ke.Thread, other_prio: u8, other_interactive: bool) // Preempt if the priority exceeds the preemption threshold (i.e the thread is real-time) // or if the thread is interactive. // Interactive threads may still preempt each other based on interactivity. - if ((td.priority >= preempt_threshold or td.interactive) and td.priority > other_prio) { + if ((td.priority >= preempt_threshold or td.is_interactive()) and td.priority > other_prio) { return true; } @@ -654,14 +700,14 @@ fn should_prio_preempt(td: *ke.Thread, other_prio: u8, other_interactive: bool) // Return whether or not `td` should preempt `cur`. // This is called with the current CPU's queue lock held. fn should_preempt(td: *ke.Thread, cur: *ke.Thread) bool { - return should_prio_preempt(td, cur.priority, cur.interactive); + return should_prio_preempt(td, cur.priority); } // Return whether or not `td` should preempt the thread running on `cpu`. // This is done in a lock-free manner. fn should_preempt_cpu(td: *ke.Thread, cpu: *PerCpu) bool { - const status = cpu.current_thread_status.load(.monotonic); - return should_prio_preempt(td, status.priority, status.interactive); + const prio = cpu.current_thread_prio.load(.monotonic); + return should_prio_preempt(td, prio); } // Enqueue a thread on a CPU. @@ -685,6 +731,7 @@ fn enqueue_on_cpu(c: u32, td: *ke.Thread) void { // This thread can preempt the next thread, // put the next thread back into a queue as the first element. insert_in_queue(cpu, n, true); + n.cpu = c; } else { // The next thread is higher priority, we can't preempt it. can_preempt = false; @@ -695,6 +742,7 @@ fn enqueue_on_cpu(c: u32, td: *ke.Thread) void { // Preempt the CPU. cpu.next_thread = td; cpu.preemption_reason = .HigherPriority; + td.cpu = c; ki.ipl.set_softint_pending(c, .Dispatch); @@ -712,6 +760,8 @@ fn enqueue_on_cpu(c: u32, td: *ke.Thread) void { // Add the thread to its appropriate queue. insert_in_queue(cpu, td, false); + td.cpu = c; + if (cpu.resched_timer.state.load(.monotonic) == .Stopped) { cpu.start_timer = true; ki.ipl.set_softint_pending(c, .Dispatch); @@ -764,7 +814,7 @@ fn select_thread(cur: ?*ke.Thread, cpu: *PerCpu, migrate: bool) ?*ke.Thread { } // Nothing lower can preempt this thread. - if (cur != null and (cur.?.priority_class == .Realtime or cur.?.interactive)) { + if (cur != null and (cur.?.priority_class() == .Realtime or cur.?.is_interactive())) { return null; } @@ -774,7 +824,7 @@ fn select_thread(cur: ?*ke.Thread, cpu: *PerCpu, migrate: bool) ?*ke.Thread { return td; } - if (cur != null and cur.?.priority_class == .Idle) { + if (cur != null and cur.?.priority_class() == .Idle) { // Idle threads don't preempt each other return null; } @@ -796,7 +846,7 @@ fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void { next.last_cpu = ke.cpu.current(); // Stash thread status. - cpu.current_thread_status.store(.{ .interactive = next.interactive, .priority = @intCast(next.priority) }, .monotonic); + cpu.current_thread_prio.store(next.priority, .monotonic); // Do machine-dependent switch. cur.context.switch_to(&next.context); diff --git a/src/kern/thread.zig b/src/kern/thread.zig index fe3a1d7..aadb947 100644 --- a/src/kern/thread.zig +++ b/src/kern/thread.zig @@ -19,19 +19,26 @@ pub const Thread = struct { }; pub const idle = 0; + + /// Batch threads have priorities 1-23 pub const low_batch = 1; - pub const high_batch = 32; + pub const high_batch = 23; pub const batch_range = high_batch - low_batch + 1; - pub const realtime = 33; + + /// Realtime threads have priorities 40-63 + pub const low_realtime = 40; pub const max = 63; + /// In the mid-range of batch threads - pub const default = 16; - /// Interactive threads have a slightly higher priority - pub const interactive = 20; + pub const default = 10; + + /// Interactive threads have priorities 24-39 + pub const low_interactive = 24; + pub const high_interactive = 39; pub fn class_from_prio(prio: u8) Class { - if (prio >= realtime and prio <= max) return .Realtime; - if (prio >= low_batch and prio <= high_batch) return .Batch; + if (prio >= low_realtime and prio <= max) return .Realtime; + if (prio >= low_batch and prio < low_realtime) return .Batch; if (prio == idle) return .Idle; unreachable; @@ -44,6 +51,8 @@ pub const Thread = struct { Ready, /// The thread is currently running. Running, + /// The thread was selected to run. + Selected, /// The thread is currently sleeping. Blocked, /// The thread has exited and is waiting to be reaped. @@ -57,10 +66,10 @@ pub const Thread = struct { lock: ke.SpinLock, /// Niceness value. nice: i8, - /// Priority value of the thread. + /// Effective priority value of the thread. priority: u8, - /// Priority class of the thread. - priority_class: Priority.Class, + /// Base priority value of the thread. + base_priority: u8, /// When the thread started sleeping. sleep_start: u64, /// Ticks spent voluntarily sleeping recently. @@ -78,6 +87,8 @@ pub const Thread = struct { runq_link: rtl.List.Entry, /// Last CPU this thread ran on. last_cpu: ?u32, + /// CPU this thread is enqueued on. + cpu: ?u32, /// Run queue this thread is in runq: ?*ki.sched.RunQueue, /// Index into the run queue this thread is currently in. @@ -101,7 +112,7 @@ pub const Thread = struct { .lock = .init(), .nice = 0, .priority = Thread.Priority.default, - .priority_class = Thread.Priority.Class.Batch, + .base_priority = Thread.Priority.default, .sleep_start = 0, .sleep_time = 0, .run_time = 0, @@ -110,15 +121,24 @@ pub const Thread = struct { .state = .Ready, .runq_link = .{}, .last_cpu = null, + .cpu = null, .runq = null, .runq_idx = 0, - .wait_status = .init(.InProgress), + .wait_status = .init(.Satisfied), .waitblocks = undefined, .timer = undefined, }; thread.timer.init(); } + + pub fn priority_class(self: *Thread) Priority.Class { + return Priority.class_from_prio(self.priority); + } + + pub fn is_interactive(self: *Thread) bool { + return self.priority >= Priority.low_interactive and self.priority <= Priority.high_interactive; + } }; comptime { diff --git a/src/platform/pc/smp.zig b/src/platform/pc/smp.zig index f2be00e..0708dc0 100644 --- a/src/platform/pc/smp.zig +++ b/src/platform/pc/smp.zig @@ -43,7 +43,7 @@ fn make_thread(entrypoint: *const fn (?*anyopaque) void, stack: usize) *ke.Threa td.init(stack, r.kib(32), entrypoint, null); td.priority = 0; - td.priority_class = .Idle; + td.base_priority = 0; td.pinned = true; return td; } diff --git a/src/platform/um/impl.zig b/src/platform/um/impl.zig index 0d32fa7..e0aee58 100644 --- a/src/platform/um/impl.zig +++ b/src/platform/um/impl.zig @@ -101,8 +101,8 @@ fn make_thread(entrypoint: *const fn (?*anyopaque) void, td: *ke.Thread) void { td.init(@intFromPtr(stack.ptr), 16384, entrypoint, null); - td.priority = 0; - td.priority_class = .Idle; + td.base_priority = ke.Thread.Priority.idle; + td.priority = td.base_priority; td.pinned = true; }