initial commit

src/kern/cpu.zig

@@ -0,0 +1,88 @@
+const std = @import("std");
+const rtl = @import("rtl");
+const b = @import("base");
+const ke = b.ke;
+const ki = b.ke.private;
+
+comptime {
+    if (!@hasDecl(ki.impl, "Cpu")) @compileError("impl must provide Cpu");
+}
+
+pub const ThreadStatus = packed struct(u8) {
+    interactive: bool,
+    priority: u7,
+};
+
+pub const PreemptionReason = enum(u8) {
+    None,
+    /// A higher priority thread was readied.
+    HigherPriority,
+};
+
+/// Per-CPU state
+pub const Cpu = struct {
+    /// Implementation-dependent CPU data.
+    impl: ki.impl.Cpu,
+    /// Unique ID of this CPU.
+    id: u32,
+    /// Current IPL on this CPU.
+    ipl: ke.Ipl,
+    /// Bitmask of pending software interrupts on this CPU.
+    pending_softints: std.atomic.Value(u8),
+    /// Currently running thread.
+    current_thread: ?*ke.Thread,
+    /// Atomic byte for getting the current thread's status locklessly.
+    /// 1 bit for thread interactivity, 7 for priority.
+    current_thread_status: std.atomic.Value(ThreadStatus),
+    /// Per-CPU idle thread.
+    idle_thread: ?*ke.Thread,
+    /// Thread selected for preemption.
+    next_thread: ?*ke.Thread,
+    /// The reason why the current thread was preempted.
+    preemption_reason: PreemptionReason,
+    /// Flag indicating the start of the scheduling timer.
+    start_timer: bool,
+    /// Queue of DPCs on this CPU.
+    dpc_queue: rtl.List,
+    /// Lock over this CPU's DPC queue.
+    dpc_lock: ke.SpinLock,
+    /// Heap of pending timers on this CPU.
+    timers_heap: rtl.PairingHeap(.min, ki.timer.cmp_timer),
+    /// Lock over the timer heap.
+    timers_lock: ke.SpinLock,
+    /// Per-CPU scheduler state.
+    sched_data: ki.sched.PerCpu,
+    /// DPCs
+    resched_dpc: ke.Dpc,
+    timer_dpc: ke.Dpc,
+    resched_timer: ke.Timer,
+
+    /// Initialize `cpu` for usage with its idle thread being `thread`.
+    pub fn init(cpu: *Cpu, thread: *ke.Thread) void {
+        cpu.* = .{
+            .id = cpu.id,
+            .ipl = .Zero,
+            .pending_softints = .init(0),
+            .idle_thread = thread,
+            .current_thread = thread,
+            .timer_dpc = .init(ki.timer.handle_expiry),
+            .resched_dpc = .init(ki.sched.clock),
+            .timers_heap = .init(),
+            .current_thread_status = .init(.{ .interactive = false, .priority = 0 }),
+            .next_thread = null,
+            .dpc_lock = .init(),
+            .preemption_reason = .None,
+            .start_timer = false,
+            .dpc_queue = undefined,
+            .timers_lock = .init(),
+            .sched_data = undefined,
+            .resched_timer = undefined,
+            .impl = cpu.impl,
+        };
+
+        cpu.dpc_queue.init();
+        cpu.resched_timer.init();
+
+        ki.sched.init_cpu(&cpu.sched_data);
+    }
+};

src/kern/dpc.zig

@@ -0,0 +1,133 @@
+const std = @import("std");
+const config = @import("config");
+const rtl = @import("rtl");
+const b = @import("base");
+const ke = b.ke;
+const ki = ke.private;
+
+/// Deferred Procedure Call (DPC) structure.
+/// Used for scheduling work to be done when IPL is lowered below `ke.Ipl.Dispatch`.
+pub const Dpc = struct {
+    /// Entry into per-CPU DPC queue
+    link: rtl.List.Entry,
+    /// Routine to call
+    func: *const fn (?*anyopaque) void,
+    /// Argument passed to the routine
+    arg: ?*anyopaque,
+    /// Whether or not the DPC is currently inserted
+    inserted: bool,
+
+    /// Initialize a DPC for `func`.
+    pub fn init(func: *const fn (?*anyopaque) void) Dpc {
+        return .{
+            .link = undefined,
+            .func = func,
+            .arg = undefined,
+            .inserted = false,
+        };
+    }
+};
+
+/// Enqueue a DPC.
+/// If the DPC is already enqueued, this function is a no-op.
+pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void {
+    const ipl = ke.ipl.raise(.High);
+    const cpu = ke.curcpu();
+
+    if (!dpc.inserted) {
+        dpc.arg = arg;
+
+        // Insert the DPC on this CPU's DPC queue
+        cpu.dpc_lock.acquire_no_ipl();
+
+        cpu.dpc_queue.insert_tail(&dpc.link);
+
+        // Mark the DPC as pending on this CPU
+        ki.ipl.set_softint_pending(cpu, .Dispatch);
+
+        dpc.inserted = true;
+
+        cpu.dpc_lock.release_no_ipl();
+    }
+
+    ke.ipl.lower(ipl);
+}
+
+fn dispatch_queue(cpu: *ke.Cpu) void {
+    cpu.ipl = .Dispatch;
+
+    // Mark as handled, this must be done before enabling interrupts or we will race.
+    ki.ipl.clear_softint_pending(cpu, .Dispatch);
+
+    _ = ki.impl.enable_interrupts();
+
+    while (true) {
+        const ipl = cpu.dpc_lock.acquire_at(.High);
+
+        var dpc: *Dpc = undefined;
+
+        if (cpu.dpc_queue.is_empty()) {
+            // No more DPCs to process.
+            cpu.dpc_lock.release(ipl);
+            break;
+        }
+
+        // Pop the head
+        var first_elem = cpu.dpc_queue.first();
+
+        first_elem.remove();
+
+        dpc = @fieldParentPtr("link", first_elem);
+        dpc.inserted = false;
+
+        // An interrupt which would enqueue this DPC could occur between loading the argument and calling the routine,
+        // which is why we capture it here to ensure that we get the intended context.
+        const arg = dpc.arg;
+
+        cpu.dpc_lock.release(ipl);
+
+        std.debug.assert(cpu.ipl == .Dispatch);
+
+        dpc.func(arg);
+    }
+
+    if (cpu.start_timer) {
+        ke.timer.set(&cpu.resched_timer, std.time.ns_per_ms * config.CONFIG_SCHED_TIMESLICE, &cpu.resched_dpc);
+    }
+
+    if (cpu.preemption_reason == .HigherPriority) {
+        // Reload the quantum for the new thread.
+        ke.timer.cancel(&cpu.resched_timer);
+
+        ke.timer.set(&cpu.resched_timer, std.time.ns_per_ms * config.CONFIG_SCHED_TIMESLICE, &cpu.resched_dpc);
+    }
+
+    cpu.start_timer = false;
+    cpu.preemption_reason = .None;
+
+    if (cpu.next_thread != null) {
+        ki.sched.handle_preemption(cpu);
+    }
+
+    _ = ki.impl.disable_interrupts();
+}
+
+/// Dispatch the DPC queue on `cpu`.
+pub fn dispatch(cpu: *ke.Cpu) void {
+    // DPC processing is done at Ipl.Dispatch
+    // We can't call lower/raise here because they might call us again recursively.
+    var mycpu = cpu;
+    const old_ipl = cpu.ipl;
+    const int_state = ki.impl.disable_interrupts();
+
+    while (ki.ipl.is_softint_pending(mycpu, .Dispatch)) {
+        dispatch_queue(mycpu);
+
+        // Our CPU might have changed.
+        mycpu = ke.curcpu();
+    }
+
+    cpu.ipl = old_ipl;
+
+    ki.impl.restore_interrupts(int_state);
+}

src/kern/init.zig

@@ -0,0 +1,54 @@
+const std = @import("std");
+const b = @import("base");
+const pl = b.pl;
+const ke = b.ke;
+const ki = ke.private;
+
+var thread0: ke.Thread = undefined;
+
+fn idle(_: ?*anyopaque) void {
+    while (true) {
+        std.atomic.spinLoopHint();
+    }
+}
+
+var stack: [8192]u8 align(16) = undefined;
+
+fn handler(_: ?*anyopaque) void {
+    std.log.info("Hi! CPU {}", .{ke.curcpu().id});
+
+    while (true) {}
+}
+
+fn make_thread(entry: *const fn (?*anyopaque) void, td: *ke.Thread) void {
+    const _stack = std.heap.page_allocator.alloc(u8, 16384) catch @panic("wtf");
+
+    td.init(@intFromPtr(_stack.ptr), 16384, entry, null);
+}
+
+var threads: [10]ke.Thread = undefined;
+
+pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void {
+    pl.early_init();
+    ki.bootstrap_cpu.init(&thread0);
+
+    thread0.init(@intFromPtr(&stack), 8192, idle, null);
+    thread0.priority = 0;
+    thread0.priority_class = .Idle;
+
+    std.log.info("hello, world", .{});
+    std.log.info("Zag for {s}, cmdline is {?s}", .{ pl.name, boot_info.cmdline });
+
+    pl.late_init();
+
+    _ = ke.ipl.raise(.Dispatch);
+
+    for (0..10) |i| {
+        make_thread(handler, &threads[i]);
+        ke.sched.enqueue(&threads[i]);
+    }
+
+    ke.ipl.lower(.Zero);
+
+    while (true) {}
+}

src/kern/ipl.zig

@@ -0,0 +1,96 @@
+const std = @import("std");
+const b = @import("base");
+const ke = b.ke;
+const ki = ke.private;
+
+/// Interrupt priority level (IPL)
+pub const Ipl = enum(u8) {
+    /// All interrupts enabled.
+    Zero = 0,
+    /// Preemption disabled.
+    Dispatch = 1,
+    /// Device interrupts disabled.
+    Device = 13,
+    /// All interrupts disabled.
+    High = 15,
+
+    pub fn get_max_software() Ipl {
+        return .Dispatch;
+    }
+};
+
+/// Raise IPL to `new`.
+/// Panics if the current IPL is higher than requested.
+pub fn raise(new: Ipl) Ipl {
+    var cpu = ke.curcpu();
+    const old = cpu.ipl;
+
+    if (@intFromEnum(new) < @intFromEnum(old)) {
+        ke.panic("ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})\n", .{ new, old });
+    }
+
+    if (@intFromEnum(new) > @intFromEnum(Ipl.get_max_software())) {
+        ki.impl.set_hardware_ipl(new);
+    }
+
+    cpu.ipl = new;
+
+    return old;
+}
+
+/// Lower IPL to `new`.
+/// Panics if the current IPL is lower than requested.
+pub fn lower(new: Ipl) void {
+    var cpu = ke.curcpu();
+    const old = cpu.ipl;
+
+    if (@intFromEnum(new) > @intFromEnum(old)) {
+        ke.panic("ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})\n", .{ new, old });
+    }
+
+    if (@intFromEnum(new) <= @intFromEnum(Ipl.get_max_software()) and @intFromEnum(old) > @intFromEnum(Ipl.get_max_software())) {
+        ki.impl.set_hardware_ipl(.Zero);
+    }
+
+    if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and is_softint_pending(cpu, .Dispatch)) {
+        ki.dpc.dispatch(cpu);
+    }
+
+    cpu.ipl = new;
+}
+
+comptime {
+    if (!@hasDecl(ki.impl, "set_hardware_ipl")) @compileError("impl must provide set_hardware_ipl()");
+}
+
+/// Set the hardware IPL to `new`
+pub fn set_hardware(new: Ipl) Ipl {
+    var cpu = ke.curcpu();
+    const old = cpu.ipl;
+
+    cpu.ipl = new;
+
+    ki.impl.set_hardware_ipl(new);
+
+    return old;
+}
+
+/// Mark a software interrupt of IPL `ipl` on `cpu` as pending
+pub fn set_softint_pending(cpu: *ke.Cpu, ipl: Ipl) void {
+    std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
+    _ = cpu.pending_softints.bitSet(@intCast(@intFromEnum(ipl)), .monotonic);
+}
+
+/// Mark a software interrupt of IPL `ipl` on `cpu` as handled
+pub fn clear_softint_pending(cpu: *ke.Cpu, ipl: Ipl) void {
+    std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
+    _ = cpu.pending_softints.bitReset(@intCast(@intFromEnum(ipl)), .monotonic);
+}
+
+/// Check whether a software interrupt of IPL `ipl` on `cpu` is pending
+pub fn is_softint_pending(cpu: *ke.Cpu, ipl: Ipl) bool {
+    std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
+    const shift: u3 = @intCast(@intFromEnum(ipl));
+    const bit: u8 = @as(u8, 1) << shift;
+    return cpu.pending_softints.load(.monotonic) & bit != 0;
+}

src/kern/log.zig

@@ -0,0 +1,57 @@
+const std = @import("std");
+const b = @import("base");
+const pl = b.pl;
+const ke = b.ke;
+
+pub var debug_lock = ke.SpinLock.init();
+
+pub fn print(bytes: []const u8) void {
+    for (bytes) |c| {
+        pl.debug_write(c);
+    }
+}
+
+const LogWriter = struct {
+    interface: std.Io.Writer,
+
+    pub fn init() LogWriter {
+        return .{
+            .interface = .{
+                .vtable = &.{ .drain = drain },
+                .buffer = &.{},
+                .end = 0,
+            },
+        };
+    }
+
+    fn drain(_: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize {
+        var total_written: usize = 0;
+        for (data) |slice| {
+            print(slice);
+            total_written += slice.len;
+        }
+
+        return total_written;
+    }
+};
+
+pub fn log(
+    comptime level: std.log.Level,
+    comptime scope: @EnumLiteral(),
+    comptime fmt: []const u8,
+    args: anytype,
+) void {
+    _ = level;
+    _ = scope;
+
+    const ipl = debug_lock.acquire();
+
+    var writer = LogWriter.init();
+
+    writer.interface.print(
+        fmt ++ "\r\n",
+        args,
+    ) catch unreachable;
+
+    debug_lock.release(ipl);
+}

src/kern/panic.zig

@@ -0,0 +1,19 @@
+const std = @import("std");
+const b = @import("base");
+const ke = b.ke;
+const ki = ke.private;
+
+/// Panics!
+pub fn panic(comptime fmt: []const u8, args: anytype) noreturn {
+    ki.log.debug_lock.acquire_no_ipl();
+
+    var buf: [1024]u8 = undefined;
+
+    const str = std.fmt.bufPrint(&buf, fmt, args) catch unreachable;
+
+    ki.log.print("Panic!\n");
+    ki.log.print(str);
+
+    ki.log.debug_lock.release_no_ipl();
+    while (true) {}
+}

src/kern/private.zig

@@ -0,0 +1,31 @@
+const std = @import("std");
+const config = @import("config");
+
+const b = @import("base");
+const ke = b.ke;
+const pl = b.pl;
+
+const init_mod = @import("init.zig");
+
+pub const impl = if (@hasDecl(config, "CONFIG_ARCH_AMD64"))
+    @import("amd64/impl.zig")
+else if (@hasDecl(config, "CONFIG_ARCH_UM"))
+    @import("um/impl.zig")
+else
+    @compileError("unsupported architecture");
+
+pub const init = init_mod.init;
+pub var bootstrap_cpu: ke.Cpu = undefined;
+
+// === Exported Modules ===
+pub const ipl = @import("ipl.zig");
+pub const panic = @import("panic.zig");
+pub const spinlock = @import("spinlock.zig");
+pub const log = @import("log.zig");
+pub const cpu = @import("cpu.zig");
+pub const thread = @import("thread.zig");
+pub const dpc = @import("dpc.zig");
+pub const timecounter = @import("timecounter.zig");
+pub const sched = @import("sched.zig");
+pub const wait = @import("wait.zig");
+pub const timer = @import("timer.zig");

src/kern/root.zig

@@ -0,0 +1,68 @@
+const config = @import("config");
+
+// === Exported Modules ===
+pub const private = @import("private.zig");
+
+const p = private;
+
+// === Exported types ===
+pub const Ipl = p.ipl.Ipl;
+pub const SpinLock = p.spinlock.SpinLock;
+pub const Thread = p.thread.Thread;
+pub const Dpc = p.dpc.Dpc;
+pub const TimeCounter = p.timecounter.TimeCounter;
+pub const Timer = p.timer.Timer;
+
+// === Exported Interfaces ===
+pub const log = p.log.log;
+pub const panic = p.panic.panic;
+
+pub const ipl = struct {
+    pub const raise = p.ipl.raise;
+    pub const lower = p.ipl.lower;
+    pub const set_hardware = p.ipl.set_hardware;
+};
+
+pub const thread = struct {
+    pub const init = p.thread.init;
+};
+
+pub const dpc = struct {
+    pub const enqueue = p.dpc.enqueue;
+};
+
+pub const timecounter = struct {
+    pub const register = p.timecounter.register;
+    pub const read_time_nano = p.timecounter.read_time_nano;
+};
+
+pub const sched = struct {
+    pub const enqueue = p.sched.enqueue;
+    pub const block = p.sched.block;
+    pub const unblock = p.sched.unblock;
+};
+
+pub const timer = struct {
+    pub const set = p.timer.set;
+    pub const cancel = p.timer.cancel;
+};
+
+pub const wait = struct {
+    pub const wait_one = p.wait.wait_one;
+    pub const wait_any = p.wait.wait_any;
+};
+
+pub const Cpu = p.cpu.Cpu;
+
+comptime {
+    if (!@hasDecl(p.impl, "curcpu")) @compileError("impl must provide curcpu()");
+}
+
+/// Returns the current CPU
+pub const curcpu: fn () *Cpu = p.impl.curcpu;
+
+/// Number of CPUs on the system
+pub var ncpus: usize = 0;
+
+/// Array of CPUs
+pub var cpus: []*Cpu = undefined;

src/kern/sched.zig

@@ -0,0 +1,764 @@
+//! Kernel scheduler inspired by FreeBSD's ULE scheduler.
+//!
+//! ## Overview
+//!
+//! Threads are divided into three priority classes:
+//!
+//! - `Realtime`: Always scheduled before batch and idle threads.
+//!   Picked from `realtime_runq` in strict priority order.
+//!   Interactive batch threads are promoted into this queue.
+//! - `Batch`: Scheduled via a circular calendar queue that rotates
+//!   on every tick, ensuring all threads get CPU time proportional
+//!   to their priority.
+//! - `Idle`: Only scheduled when both realtime and batch queues are empty.
+//!
+//! ## Interactivity Scoring
+//!
+//! Each batch thread is assigned an interactivity score based on the
+//! ratio of its recent sleep time to its recent CPU time:
+//!
+//! - If `sleep_time > run_time`: score = `run_time / (sleep_time / scaling_factor)`
+//!   — heavily sleeping threads score low (more interactive).
+//! - If `run_time > sleep_time`: score = `scaling_factor * 2 - (sleep_time / (run_time / scaling_factor))`
+//!   — CPU-bound threads score high (less interactive).
+//! - Niceness is added to the score, making low-nice threads easier to
+//!   qualify as interactive.
+//!
+//! Threads scoring below `interactivity_threshold` are considered
+//! interactive and promoted to the realtime queue with a priority derived
+//! from their score. This lets I/O-bound threads respond quickly without
+//! being assigned a static realtime priority.
+//!
+//! To prevent history from dominating forever, `sleep_time` and `run_time`
+//! are decayed periodically by `sched_clamp_time`:
+//! - Values are halved when their sum exceeds 120% of the 5-second cap.
+//! - Values are reduced by 20% when the sum exceeds 5 seconds.
+//! - Values are reset to (max, 1) when the sum exceeds 10 seconds.
+//!
+//! ## Calendar Queue
+//!
+//! The calendar queue is a circular array of `runqueues_n` (64) lists.
+//! Two indices drive it:
+//!
+//! - `runidx`: The queue currently being drained by the scheduler.
+//! - `insidx`: The queue where newly inserted threads land.
+//!   Always one slot ahead of `runidx` so fresh threads are never
+//!   picked in the same tick they were enqueued.
+//!
+//! Threads are inserted at `(insidx + prio_high_batch - thread.priority) % runqueues_n`,
+//! placing lower-priority threads further ahead in the rotation so
+//! they run less frequently than higher-priority ones.
+//!
+//! On every tick (10ms), `insidx` advances by one, giving all threads
+//! the chance to run over time regardless of priority.
+//!
+//! ## Preemption
+//!
+//! Timeshared (batch) threads do not preempt each other by default,
+//! mirroring FreeBSD's default preemption threshold. Interactive threads
+//! may still preempt non-interactive batch threads. Realtime threads
+//! always preempt batch and idle threads.
+//!
+//! Remote preemption (sending an IPI to another CPU) is done lock-free
+//! by reading a packed `current_thread_status` field that encodes the
+//! running thread's priority and interactivity in a single atomic byte.
+//!
+//! ## Load Balancing
+//!
+//! Two mechanisms keep load distributed across CPUs:
+//!
+//! - **Work stealing**: When a CPU goes idle, it immediately attempts to
+//!   steal a thread from the most loaded CPU.
+//! - **Periodic balancing**: Every ~1 second (randomized with an LCG to
+//!   avoid phase-locking with periodic threads), the bootstrap CPU moves
+//!   threads from the most loaded CPU to the least loaded one.
+//!
+//! ## CPU Selection
+//!
+//! When enqueuing a thread, the target CPU is chosen in priority order:
+//! 1. Last CPU the thread ran on, if it can be preempted or the thread is pinned.
+//! 2. Least loaded CPU on which the thread would run immediately (preemptible).
+//! 3. Least loaded CPU overall.
+//!
+//! See `sched_ule.c` in FreeBSD for the original implementation this is based on.
+const std = @import("std");
+const rtl = @import("rtl");
+const config = @import("config");
+const b = @import("base");
+const ke = b.ke;
+const ki = ke.private;
+
+comptime {
+    if (!@hasDecl(ki.impl, "send_resched_ipi")) @compileError("impl must provide send_resched_ipi");
+}
+
+const runqueues_n = 64;
+const interactivity_threshold = 30;
+const scaling_factor = 50;
+const preempt_threshold = ke.Thread.Priority.realtime;
+
+pub const RunQueue = struct {
+    /// Bitmap of non-empty queues. A set bit represents a non-empty queue.
+    status: u64,
+    queues: [runqueues_n]rtl.List,
+};
+
+/// Per-CPU scheduler data.
+pub const PerCpu = struct {
+    /// Current load on this CPU (number of ready threads)
+    load: std.atomic.Value(usize),
+    /// Lock over this CPU's queues.
+    queues_lock: ke.SpinLock,
+    /// Array of queues from which to pick realtime threads from.
+    realtime_queue: RunQueue,
+    /// Calendar queue.
+    calendar_queue: RunQueue,
+    /// Queue of idle threads.
+    idle_queue: rtl.List,
+    /// The index from which to pick the next thread.
+    runidx: u8,
+    /// The index on which the next thread is going to be inserted.
+    insidx: u8,
+    /// Used to indicate to this CPU it can steal work
+    steal_work: bool,
+};
+
+/// Handle preemption.
+/// This is called in DPC dispatch when it notices `next_thread` is set.
+pub fn handle_preemption(cpu: *ke.Cpu) void {
+    const sched_cpu = &cpu.sched_data;
+    sched_cpu.queues_lock.acquire_no_ipl();
+
+    const cur = cpu.current_thread.?;
+
+    const next = cpu.next_thread orelse {
+        sched_cpu.queues_lock.release_no_ipl();
+        return;
+    };
+
+    cpu.next_thread = null;
+
+    sched_cpu.queues_lock.release_no_ipl();
+    cur.lock.acquire_no_ipl();
+
+    if (cur != cpu.idle_thread and cur.state != .Blocked) {
+        // Put it back in the queue.
+        sched_cpu.queues_lock.acquire_no_ipl();
+        insert_in_queue(sched_cpu, cur, false);
+        sched_cpu.queues_lock.release_no_ipl();
+    }
+
+    do_switch(cpu, cur, next);
+
+    // cur.lock dropped
+}
+
+/// Called every time slice.
+pub fn clock(_: ?*anyopaque) void {
+    const cpu = ke.curcpu();
+    const sched_cpu = &cpu.sched_data;
+    std.debug.assert(cpu.ipl == .Dispatch);
+
+    const curtd = cpu.current_thread orelse return;
+
+    curtd.lock.acquire_no_ipl();
+    defer curtd.lock.release_no_ipl();
+
+    sched_cpu.queues_lock.acquire_no_ipl();
+    defer sched_cpu.queues_lock.release_no_ipl();
+
+    // Advance the insert index every tick, while keeping a separation of 1 with runidx.
+    // This ensures fairness.
+    if (sched_cpu.runidx == sched_cpu.insidx) {
+        sched_cpu.insidx = (sched_cpu.insidx) % runqueues_n;
+    }
+
+    if (curtd.priority_class == .Batch) {
+        // Update interactivity metrics.
+        curtd.run_time += config.CONFIG_SCHED_TIMESLICE;
+
+        clamp_time(curtd);
+        recompute_priority(curtd);
+    }
+
+    if (cpu.next_thread != null) {
+        // Another thread was already selected for preemption.
+        return;
+    }
+
+    // Pick a new thread to run.
+    const newtd = select_thread(curtd, cpu, false);
+
+    // Ensure the timer gets reloaded.
+    cpu.start_timer = true;
+    cpu.next_thread = newtd;
+}
+
+/// Enqueue a thread.
+pub fn enqueue(td: *ke.Thread) void {
+    const ipl = td.lock.acquire();
+    const cpu = pick_cpu(td);
+    enqueue_on_cpu(cpu, td);
+    td.lock.release(ipl);
+}
+
+/// Block the currently running thread.
+pub fn block() void {
+    const ipl = ke.ipl.raise(.Dispatch);
+    const td = ke.curcpu().current_thread.?;
+
+    td.lock.acquire_no_ipl();
+
+    td.state = .Blocked;
+    td.sleep_start = ke.timecounter.read_time_nano();
+    td.runq = null;
+
+    yield(ke.curcpu());
+
+    // td lock released
+    ke.ipl.lower(ipl);
+}
+
+/// Unblock a thread.
+pub fn unblock(td: *ke.Thread) void {
+    const ipl = td.lock.acquire();
+
+    td.sleep_time = (ke.timecounter.read_time_nano() - td.sleep_start) / std.time.ns_per_ms;
+
+    if (td.sleep_time >= config.CONFIG_SCHED_TIMESLICE and .priority_class == .Batch) {
+        // If we have slept for more than a tick, update interactivity.
+        clamp_time(td);
+        recompute_priority(td);
+    }
+
+    // Enqueue the thread
+    const cpu = pick_cpu(td);
+    enqueue_on_cpu(cpu, td);
+
+    td.lock.release(ipl);
+}
+
+/// Initialize a CPU for use by the scheduler.
+pub fn init_cpu(percpu: *PerCpu) linksection(b.init) void {
+    percpu.* = .{
+        .insidx = 0,
+        .runidx = 0,
+        .calendar_queue = .{ .status = 0, .queues = undefined },
+        .realtime_queue = .{ .status = 0, .queues = undefined },
+        .queues_lock = .init(),
+        .idle_queue = undefined,
+        .steal_work = false,
+        .load = .init(0),
+    };
+
+    percpu.idle_queue.init();
+
+    for (0..runqueues_n) |i| {
+        percpu.calendar_queue.queues[i].init();
+    }
+
+    for (0..runqueues_n) |i| {
+        percpu.realtime_queue.queues[i].init();
+    }
+}
+
+fn calendar_queue_increment(cpu: *PerCpu) void {
+    cpu.runidx = (cpu.runidx + 1) % runqueues_n;
+
+    // Ensure insidx is always one ahead of runidx
+    if (cpu.runidx == cpu.insidx) {
+        cpu.insidx = (cpu.insidx + 1) % runqueues_n;
+    }
+}
+
+fn pick_realtime_thread(cpu: *PerCpu, minprio: u8, migrate: bool) ?*ke.Thread {
+    std.debug.assert(cpu.queues_lock.is_locked());
+
+    const runq = &cpu.realtime_queue;
+
+    if (runq.status == 0) return null;
+
+    // Search higher priority queues first.
+    var i: usize = ke.Thread.Priority.max;
+
+    qloop: while (i >= minprio) : (i -= 1) {
+        const bit = (@as(u64, 1) << @intCast(i));
+
+        if (runq.status & bit == 0) {
+            // Queue is empty.
+            continue;
+        }
+
+        const curr_queue = &runq.queues[i];
+
+        std.debug.assert(!curr_queue.is_empty());
+
+        // Get the first thread of the queue.
+        var td: *ke.Thread = @fieldParentPtr("runq_link", curr_queue.first());
+
+        if (migrate) {
+            // Find the first thread that is not pinned.
+            while (td.pinned) {
+                const entry = td.runq_link.next;
+
+                td = @fieldParentPtr("runq_link", entry);
+
+                // Every thread is pinned, go through another queue
+                if (entry == &curr_queue.head) continue :qloop;
+            }
+        }
+
+        // Remove it from the queue.
+        td.runq_link.remove();
+
+        if (curr_queue.is_empty()) {
+            // Queue is now empty, clear its bit.
+            runq.status &= ~bit;
+        }
+    }
+
+    return null;
+}
+
+fn pick_batch_thread(cpu: *PerCpu, migrate: bool) ?*ke.Thread {
+    std.debug.assert(cpu.queues_lock.is_locked());
+    const runq = &cpu.calendar_queue;
+
+    if (runq.status == 0) return null;
+
+    // Search all queues, starting from runidx.
+    var i: usize = cpu.runidx;
+    var cnt: usize = 0;
+
+    qloop: while (cnt < runqueues_n) : ({
+        i = (i + 1) % runqueues_n;
+        cnt += 1;
+    }) {
+        const bit = (@as(u64, 1) << @intCast(i));
+
+        if (runq.status & bit == 0) {
+            // Queue is empty.
+            continue;
+        }
+
+        const curr_queue = &runq.queues[i];
+
+        std.debug.assert(!curr_queue.is_empty());
+
+        // Get the first thread of the queue.
+        var td: *ke.Thread = @fieldParentPtr("runq_link", curr_queue.first());
+
+        if (migrate) {
+            // Find the first thread that is not pinned.
+            while (td.pinned) {
+                const entry = td.runq_link.next;
+
+                td = @fieldParentPtr("runq_link", entry);
+
+                // Every thread is pinned, go through another queue
+                if (entry == &curr_queue.head) continue :qloop;
+            }
+        }
+
+        // Remove it from the queue.
+        td.runq_link.remove();
+
+        if (curr_queue.is_empty()) {
+            // Queue is now empty, clear its bit and increment runidx.
+            runq.status &= ~bit;
+
+            if (!migrate) {
+                calendar_queue_increment(cpu);
+            }
+        }
+
+        return td;
+    }
+
+    return null;
+}
+
+fn pick_idle_thread(cpu: *PerCpu, migrate: bool) ?*ke.Thread {
+    std.debug.assert(cpu.queues_lock.is_locked());
+    if (cpu.idle_queue.is_empty()) return null;
+
+    // Just get the first thread from the queue.
+    const td: *ke.Thread = @fieldParentPtr("runq_link", cpu.idle_queue.first());
+
+    if (!migrate and td.pinned) {
+        // We can safely remove it.
+        td.runq_link.remove();
+    }
+
+    return td;
+}
+
+fn insert_in_list(list: *rtl.List, link: *rtl.List.Entry, head: bool) void {
+    if (head) list.insert_head(link) else list.insert_tail(link);
+}
+
+// Insert a thread into its respective queue.
+fn insert_in_queue(cpu: *PerCpu, td: *ke.Thread, head: bool) void {
+    std.debug.assert(td.lock.is_locked());
+    std.debug.assert(cpu.queues_lock.is_locked());
+
+    if (td.priority_class == .Realtime or td.interactive) {
+        // Insert in realtime queue.
+        cpu.realtime_queue.status |=
+            (@as(u64, 1) << @intCast(td.priority));
+
+        insert_in_list(&cpu.realtime_queue.queues[td.priority], &td.runq_link, head);
+        td.runq = &cpu.realtime_queue;
+        td.runq_idx = @intCast(td.priority);
+    } else if (td.priority_class == .Batch) {
+        // Insert in calendar queue.
+        // The insertion index is determined by insidx and the the priority of the thread,
+        // higher priority threads will be put closer to insidx, which ensures that they are ran more frequently.
+        const idx = (cpu.insidx + (ke.Thread.Priority.high_batch - td.priority)) % runqueues_n;
+
+        td.runq_idx = @intCast(idx);
+        cpu.calendar_queue.status |= (@as(u64, 1) << @intCast(idx));
+
+        insert_in_list(&cpu.calendar_queue.queues[idx], &td.runq_link, head);
+        td.runq = &cpu.calendar_queue;
+    } else {
+        // Insert in idle queue.
+        insert_in_list(&cpu.idle_queue, &td.runq_link, head);
+        td.runq = null;
+        td.runq_idx = 0;
+    }
+
+    td.state = .Ready;
+    _ = cpu.load.fetchAdd(1, .monotonic);
+}
+
+// Remove a thread from a queue.
+fn remove_from_queue(cpu: *PerCpu, td: *ke.Thread) void {
+    std.debug.assert(td.lock.is_locked());
+    std.debug.assert(cpu.queues_lock.is_locked());
+
+    // Decrease load.
+    cpu.load.fetchSub(1, .monotonic);
+
+    // Remove the thread from its queue.
+    td.runq_link.remove();
+
+    if (td.runq) |rq| {
+        if (rq.queues[td.runq_idx].is_empty()) {
+            // Clear the bit.
+            rq.status &= ~(@as(u64, 1) << @intCast(td.runq_idx));
+        }
+    }
+}
+
+// Update recent history for a thread.
+fn clamp_time(td: *ke.Thread) void {
+    const max = 5 * std.time.ms_per_s; // 5 seconds
+    const sum = td.run_time + td.sleep_time;
+
+    if (sum < max) return;
+
+    if (sum > max * 2) {
+        // History is way out of range (>10s), hard reset.
+        // Preserve the dominant side to avoid flipping interactivity classification.
+        if (td.run_time > td.sleep_time) {
+            td.run_time = max;
+            td.sleep_time = 1;
+        } else {
+            td.sleep_time = max;
+            td.run_time = 1;
+        }
+
+        return;
+    }
+
+    if (sum > ((max / 5) * 6)) {
+        // History is moderately out of range (>6s), halve both values.
+        // Keeps the sleep/run ratio intact while pulling the sum back in range.
+        td.run_time /= 2;
+        td.sleep_time /= 2;
+        return;
+    }
+
+    // History is slightly out of range (5s-6s), gentle 20% decay.
+    // Gradually ages out old history without disturbing the ratio.
+    td.run_time = (td.run_time / 5) * 4;
+    td.sleep_time = (td.sleep_time / 5) * 4;
+}
+
+// Compute the interactivity score for a thread.
+fn interactive_score(td: *ke.Thread) usize {
+    var div: usize = undefined;
+    var score: usize = undefined;
+
+    // Calculate the interactivity penalty.
+    if (td.sleep_time > td.run_time) {
+        div = @max(1, td.sleep_time / scaling_factor);
+        score = td.run_time / div;
+    } else if (td.run_time > td.sleep_time) {
+        div = @max(1, td.run_time / scaling_factor);
+        score = (scaling_factor + scaling_factor - (td.sleep_time / div));
+    } else if (td.run_time != 0) {
+        score = scaling_factor;
+    } else return 0;
+
+    // Add niceness values to the penalty, this makes it easier for threads with
+    // lower nice values (higher priority) to be considered interactive.
+    var signed_score: isize = @intCast(score);
+    signed_score += td.nice;
+    score = @intCast(signed_score);
+
+    return score;
+}
+
+// Recompute the priority for a thread.
+fn recompute_priority(td: *ke.Thread) void {
+
+    // Priority is computed only for time-shared (batch) thread based on
+    // interactivity and nice. If the thread is determined interactive, it is
+    // effectively promoted to real-time with a lower priority than actual
+    // real-time threads. If the thread is determined non-interactive,
+    // priority is calculated based on recent CPU usage and nice.
+    const score = interactive_score(td);
+
+    if (score < interactivity_threshold) {
+        // Choose a priority based on score, the lower the score,
+        // the higher the priority it will be.
+        // This is a simple formula I came up with that is probably good enough.
+        td.priority = @intCast(ke.Thread.Priority.interactive + ((interactivity_threshold - score) / 4));
+        td.interactive = true;
+
+        return;
+    }
+
+    // Thread is not interactive, compute priority from recent CPU usage.
+    // cpu_pri_off is the fraction of the history window spent running,
+    // scaled to [0, batch_range). A thread that spent all its time running
+    // gets cpu_pri_off = batch_range - 1 (lowest priority).
+    // A thread that barely ran gets cpu_pri_off = 0 (highest batch priority).
+    // This is a pretty straightforward formula, which /should/ be good enough.
+    const cpu_range = ke.Thread.Priority.batch_range;
+    const window = if (td.run_time + td.sleep_time == 0) @as(u64, 1) else td.run_time + td.sleep_time;
+    const cpu_pri_off: u8 = @intCast((td.run_time * (cpu_range - 1)) / window);
+
+    // nice offset: negative nice increases priority value, positive decreases it.
+    // Clamped to half the range to avoid pushing priority out of bounds before clamping.
+    const nice_off = std.math.clamp(
+        -@as(i32, td.nice),
+        -@as(i32, cpu_range / 2),
+        @as(i32, cpu_range / 2),
+    );
+
+    const raw = @as(i32, ke.Thread.Priority.low_batch) +
+        @as(i32, cpu_pri_off) +
+        nice_off;
+
+    td.priority = @intCast(std.math.clamp(
+        raw,
+        ke.Thread.Priority.low_batch,
+        ke.Thread.Priority.high_batch,
+    ));
+    td.interactive = false;
+}
+
+// Return whether or not a thread with given priority and interactivity should get preempted by `td`.
+fn should_prio_preempt(td: *ke.Thread, other_prio: u8, other_interactive: bool) bool {
+    const class = ke.Thread.Priority.class_from_prio(other_prio);
+
+    if (class == .Idle) {
+        return true;
+    }
+
+    if (class == .Batch and !other_interactive and td.interactive) {
+        // Interactive threads always preempt batch non-interactive ones.
+        return true;
+    }
+
+    // Preempt if the priority exceeds the preemption threshold (i.e the thread is real-time)
+    // or if the thread is interactive.
+    // Interactive threads may still preempt each other based on interactivity.
+    if ((td.priority >= preempt_threshold or td.interactive) and td.priority > other_prio) {
+        return true;
+    }
+
+    return false;
+}
+
+// Return whether or not `td` should preempt `cur`.
+// This is called with the current CPU's queue lock held.
+fn should_preempt(td: *ke.Thread, cur: *ke.Thread) bool {
+    return should_prio_preempt(td, cur.priority, cur.interactive);
+}
+
+// Return whether or not `td` should preempt the thread running on `cpu`.
+// This is done in a lock-free manner.
+fn should_preempt_cpu(td: *ke.Thread, cpu: *ke.Cpu) bool {
+    const status = cpu.current_thread_status.load(.monotonic);
+    return should_prio_preempt(td, status.priority, status.interactive);
+}
+
+// Enqueue a thread on a CPU.
+fn enqueue_on_cpu(cpu: *ke.Cpu, td: *ke.Thread) void {
+    var c = &cpu.sched_data;
+
+    c.queues_lock.acquire_no_ipl();
+
+    defer c.queues_lock.release_no_ipl();
+
+    if (cpu.current_thread == null or should_preempt(td, cpu.current_thread.?)) {
+        // We can preempt the current thread, first check whether or not there
+        // is already a next thread selected, and if we can preempt it.
+
+        const next = cpu.next_thread;
+        var can_preempt = true;
+
+        if (next) |n| {
+            if (should_preempt(td, n)) {
+                // This thread can preempt the next thread,
+                // put the next thread back into a queue as the first element.
+                insert_in_queue(c, n, true);
+            } else {
+                // The next thread is higher priority, we can't preempt it.
+                can_preempt = false;
+            }
+        }
+
+        if (can_preempt) {
+            // Preempt the CPU.
+            cpu.next_thread = td;
+            cpu.preemption_reason = .HigherPriority;
+
+            ki.ipl.set_softint_pending(cpu, .Dispatch);
+
+            if (cpu != ke.curcpu()) {
+                // Send an IPI.
+                ki.impl.send_resched_ipi(cpu);
+            }
+
+            // queues_lock dropped
+            return;
+        }
+    }
+
+    // Normal enqueue path.
+    // Add the thread to its appropriate queue.
+    insert_in_queue(c, td, false);
+
+    if (cpu.resched_timer.state.load(.monotonic) == .Stopped) {
+        cpu.start_timer = true;
+        ki.ipl.set_softint_pending(cpu, .Dispatch);
+    }
+
+    // queues_lock dropped
+}
+
+// Find a suitable CPU to run the thread.
+fn pick_cpu(td: *ke.Thread) *ke.Cpu {
+    // CPU selection policy in order:
+    // 1. Pick the last CPU the thread ran on if it would run immediately or if it's pinned.
+    // 2. Pick the least loaded CPU on which the thread can run immediately.
+    // 3. Pick the least loaded CPU overall.
+
+    if (td.last_cpu) |cpu| {
+        if (should_preempt_cpu(td, cpu) or td.pinned) return cpu;
+    }
+
+    // Check all CPUs, keeping track of the least loaded one overall and the least loaded preemptible one.
+    var least: ?*ke.Cpu = null;
+    var least_preempt: ?*ke.Cpu = null;
+
+    for (0..ke.ncpus) |i| {
+        const cpu = ke.cpus[i];
+
+        if (least == null or cpu.sched_data.load.load(.monotonic) < least.?.sched_data.load.load(.monotonic)) {
+            least = cpu;
+        }
+
+        if (should_preempt_cpu(td, cpu)) {
+            if (least_preempt == null or cpu.sched_data.load.load(.monotonic) < least_preempt.?.sched_data.load.load(.monotonic)) {
+                least_preempt = cpu;
+            }
+        }
+    }
+
+    // If we found a preemptible CPU, pick that one. Otherwise pick the least loaded one.
+    return least_preempt orelse least.?;
+}
+
+// Find a thread to preempt `cur` with.
+fn select_thread(cur: ?*ke.Thread, cpu: *ke.Cpu, migrate: bool) ?*ke.Thread {
+    const sched_cpu = &cpu.sched_data;
+
+    // First try to pick from the realtime queues.
+    if (pick_realtime_thread(sched_cpu, if (cur) |c| c.priority else 0, migrate)) |td| {
+        _ = sched_cpu.load.fetchSub(1, .monotonic);
+        return td;
+    }
+
+    // Nothing lower can preempt this thread.
+    if (cur != null and (cur.?.priority_class == .Realtime or cur.?.interactive)) {
+        return null;
+    }
+
+    // Then try picking a batch thread.
+    if (pick_batch_thread(sched_cpu, migrate)) |td| {
+        _ = sched_cpu.load.fetchSub(1, .monotonic);
+        return td;
+    }
+
+    if (cur != null and cur.?.priority_class == .Idle) {
+        // Idle threads don't preempt each other
+        return null;
+    }
+
+    // Finally try picking from the idle queue.
+    if (pick_idle_thread(sched_cpu, migrate)) |td| {
+        _ = sched_cpu.load.fetchSub(1, .monotonic);
+        return td;
+    }
+
+    // Nothing to run.
+    return null;
+}
+
+// Execute a context switch on `cpu`.
+fn do_switch(cpu: *ke.Cpu, cur: *ke.Thread, next: *ke.Thread) void {
+    next.state = .Running;
+    cpu.current_thread = next;
+    next.last_cpu = cpu;
+
+    // Stash thread status.
+    cpu.current_thread_status.store(.{ .interactive = next.interactive, .priority = @intCast(next.priority) }, .monotonic);
+
+    // Do machine-dependent switch.
+    cur.context.switch_to(&next.context);
+}
+
+// Yield on `cpu`
+fn yield(cpu: *ke.Cpu) void {
+    const sched_cpu = &cpu.sched_data;
+
+    sched_cpu.queues_lock.acquire_no_ipl();
+
+    const cur = cpu.current_thread;
+    var next = cpu.next_thread;
+
+    if (next) {
+        cpu.next_thread = null;
+    } else {
+        // Pick a new thread to run.
+        next = select_thread(null, cpu, false);
+    }
+
+    if (next == null) {
+        // Nothing to run, go idle.
+        next = cpu.idle_thread;
+    }
+
+    sched_cpu.queues_lock.release_no_ipl();
+
+    // Switch into the thread.
+    do_switch(cpu, cur, next);
+
+    // cur lock dropped
+}

src/kern/spinlock.zig

@@ -0,0 +1,60 @@
+const std = @import("std");
+const b = @import("base");
+const ke = b.ke;
+
+/// Simple Spin lock implementation.
+pub const SpinLock = struct {
+    locked: u8,
+
+    pub fn init() SpinLock {
+        return .{
+            .locked = 0,
+        };
+    }
+
+    /// Acquire the lock at IPL `ipl`.
+    pub fn acquire_at(self: *SpinLock, ipl: ke.Ipl) ke.Ipl {
+        const old_ipl = ke.ipl.raise(ipl);
+
+        self.acquire_no_ipl();
+
+        return old_ipl;
+    }
+
+    /// Acquire the lock at IPL `.Dispatch`.
+    pub fn acquire(self: *SpinLock) ke.Ipl {
+        return self.acquire_at(.Dispatch);
+    }
+
+    /// Release the lock at IPL `ipl`.
+    pub fn release(self: *SpinLock, ipl: ke.Ipl) void {
+        ke.ipl.lower(ipl);
+        self.release_no_ipl();
+    }
+
+    /// Acquire the lock without changing the IPL.
+    pub fn acquire_no_ipl(self: *SpinLock) void {
+        while (true) {
+            if (@cmpxchgWeak(u8, &self.locked, 0, 1, .acquire, .monotonic) == null)
+                return;
+
+            while (@atomicLoad(u8, &self.locked, .monotonic) != 0) {
+                std.atomic.spinLoopHint();
+            }
+        }
+    }
+
+    /// Release the lock without changing the IPL.
+    pub fn release_no_ipl(self: *SpinLock) void {
+        @atomicStore(u8, &self.locked, 0, .release);
+    }
+
+    /// Try to acquire the lock. Return true if lock was acquired.
+    pub fn try_acquire(self: *SpinLock) bool {
+        return @cmpxchgStrong(u8, &self.locked, 0, 1, .acquire, .monotonic) == null;
+    }
+
+    pub fn is_locked(self: *SpinLock) bool {
+        return @atomicLoad(u8, &self.locked, .monotonic) == 1;
+    }
+};

src/kern/thread.zig

@@ -0,0 +1,128 @@
+const std = @import("std");
+const b = @import("base");
+const rtl = @import("rtl");
+
+const ke = b.ke;
+const ki = ke.private;
+
+/// Structure representing a kernel thread.
+pub const Thread = struct {
+    /// Thread priority classes.
+    /// Realtime: Highest priority, aggressively scheduled.
+    /// Batch: Normal priority.
+    /// Idle: For idle and background threads.
+    pub const Priority = struct {
+        pub const Class = enum(u8) {
+            Realtime,
+            Batch,
+            Idle,
+        };
+
+        pub const idle = 0;
+        pub const low_batch = 1;
+        pub const high_batch = 32;
+        pub const batch_range = high_batch - low_batch + 1;
+        pub const realtime = 33;
+        pub const max = 63;
+        /// In the mid-range of batch threads
+        pub const default = 16;
+        /// Interactive threads have a slightly higher priority
+        pub const interactive = 20;
+
+        pub fn class_from_prio(prio: u8) Class {
+            if (prio >= realtime and prio <= max) return .Realtime;
+            if (prio >= low_batch and prio <= high_batch) return .Batch;
+            if (prio == idle) return .Idle;
+
+            unreachable;
+        }
+    };
+
+    /// Thread state.
+    pub const State = enum(u8) {
+        /// The thread is ready to run.
+        Ready,
+        /// The thread is currently running.
+        Running,
+        /// The thread is currently sleeping.
+        Blocked,
+        /// The thread has exited and is waiting to be reaped.
+        Zombie,
+        /// The thread has exited.
+        Terminated,
+    };
+    /// Implementation-dependent context.
+    context: ki.impl.ThreadContext,
+    /// Thread lock.
+    lock: ke.SpinLock,
+    /// Niceness value.
+    nice: i8,
+    /// Priority value of the thread.
+    priority: u8,
+    /// Priority class of the thread.
+    priority_class: Priority.Class,
+    /// When the thread started sleeping.
+    sleep_start: u64,
+    /// Ticks spent voluntarily sleeping recently.
+    sleep_time: u64,
+    /// Ticks spent running recently.
+    run_time: u64,
+    /// Whether the thread is interactive or not.
+    interactive: bool,
+    /// Whether the thread is pinned to this CPU.
+    /// If it is pinned, then it can't be moved across another CPU.
+    pinned: bool,
+    /// Current state of the thread.
+    state: State,
+    /// Linkage into a scheduler run queue.
+    runq_link: rtl.List.Entry,
+    /// Last CPU this thread ran on.
+    last_cpu: ?*ke.Cpu,
+
+    /// Run queue this thread is in
+    runq: ?*ki.sched.RunQueue,
+    /// Index into the run queue this thread is currently in.
+    /// Only valid when `runq` is non-null.
+    runq_idx: u8,
+    /// Current wait status.
+    wait_status: std.atomic.Value(ki.wait.Status),
+    waitblocks: [4]ki.wait.WaitBlock,
+
+    /// Timer used for timeouts.
+    timer: ke.Timer,
+
+    /// Initialize a thread.
+    /// - `stack`: Address of the **base** of the stack on which the initial context for the thread is built
+    /// - `stack_size`: Size of the stack
+    /// - `entry`: Entry point of the thread
+    /// - `arg`: Extraneous argument to be passed to `entry`
+    pub fn init(thread: *Thread, stack: b.VAddr, stack_size: usize, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) void {
+        thread.* = .{
+            .context = .init(stack, stack_size, entry, arg),
+            .lock = ke.SpinLock.init(),
+            .nice = 0,
+            .priority = Thread.Priority.default,
+            .priority_class = Thread.Priority.Class.Batch,
+            .sleep_start = 0,
+            .sleep_time = 0,
+            .run_time = 0,
+            .interactive = false,
+            .pinned = false,
+            .state = .Ready,
+            .runq_link = .{},
+            .last_cpu = null,
+            .runq = null,
+            .runq_idx = 0,
+            .wait_status = .init(.InProgress),
+            .waitblocks = undefined,
+            .timer = undefined,
+        };
+
+        thread.timer.init();
+    }
+};
+
+comptime {
+    if (!@hasDecl(ki.impl, "ThreadContext")) @compileError("impl must provide ThreadContext");
+    if (!@hasDecl(ki.impl.ThreadContext, "init")) @compileError("ThreadContext must have init()");
+}

src/kern/timecounter.zig

@@ -0,0 +1,143 @@
+const std = @import("std");
+const rtl = @import("rtl");
+const b = @import("base");
+const ke = b.ke;
+const ki = ke.private;
+
+/// TimeCounter structure.
+/// Represents a time-keeping source upon which the kernel can rely.
+pub const TimeCounter = struct {
+    /// Callback that reads the current count.
+    read_count: *const fn () u64,
+    /// Frequency of the timer in Hz.
+    frequency: u64,
+    /// Name of the timer.
+    name: []const u8,
+    /// Quality of the timer.
+    /// Used to choose between two TimeCounters.
+    quality: i16,
+    /// Mask to apply to the counter value.
+    mask: u64,
+    /// Binary scaling exponent for fast ns conversion (0 if unused).
+    /// See `ticks_to_ns`.
+    p: u64,
+    /// Scaled nanoseconds-per-tick multiplier for fast ns conversion (0 if unused).
+    /// See `ticks_to_ns`.
+    n: u64,
+};
+
+var best_tc: ?*TimeCounter = null;
+
+const TimeState = struct {
+    // Counter value at the time the current TimeCounter was selected.
+    // Used as the baseline for elapsed time calculation.
+    initial_count: u64,
+    // Accumulated nanosecond offset from previous TimeCounters and overflow events.
+    // Added to the current counter's contribution in `read_time_nano`.
+    offset: u64,
+};
+
+// SeqLock-protected time state.
+// Avoids a race on `initial_count` and `offset` with `update_overflow`.
+var state = rtl.SeqLock(TimeState).init(.{
+    .initial_count = 0,
+    .offset = 0,
+});
+
+// Convert a TimeCounter's count to nanoseconds.
+// For 64-bit counters (`mask == maxInt(u64)`), uses a fast multiply-shift path
+// via precomputed `p` and `n` to avoid 128-bit division at runtime.
+// Falls back to a division-based conversion for narrower counters.
+fn ticks_to_ns(tc: *TimeCounter, count: u64) u64 {
+    if (tc.p != 0) {
+        const val: u128 = @as(u128, @intCast(count)) * tc.n;
+        return @intCast(val >> @intCast(tc.p));
+    }
+
+    return (count * std.time.ns_per_s) / tc.frequency;
+}
+
+/// Register a time-keeping source.
+/// The TimeCounter will be used if it is the highest quality one available.
+/// This function cannot be safely called when the hardware timer has been started, as data races on `best_tc` are not avoided.
+pub fn register(tc: *TimeCounter) void {
+    // Precompute scaling factors for the fast multiply-shift path.
+    // This is meant to avoid 128-bit division and only use multiplication for 64-bit counters.
+    if (tc.mask == std.math.maxInt(u64)) {
+        tc.p = std.math.log2_int_ceil(u64, tc.frequency);
+        tc.n = (@as(u64, std.time.ns_per_s) << @intCast(tc.p)) / tc.frequency;
+    } else {
+        tc.p = 0;
+        tc.n = 0;
+    }
+
+    // If the best TimeCounter changed, pass the baton:
+    // snapshot the old counter's elapsed time into `offset` and
+    // reset `initial_count` to the new counter's current value.
+    if (best_tc == null or tc.quality > best_tc.?.quality) {
+        const old_best = best_tc;
+        best_tc = tc;
+
+        if (old_best) |old| {
+            const old_count = old.read_count() & old.mask;
+            state.raw.offset = ticks_to_ns(old, old_count);
+        }
+
+        state.raw.initial_count = tc.read_count() & tc.mask;
+    }
+}
+
+/// Return the time elapsed since boot in nanoseconds.
+pub fn read_time_nano() b.Nanoseconds {
+    const tc = best_tc orelse return 0;
+
+    const s = state.load();
+    const curr_count = tc.read_count() & tc.mask;
+    const elapsed = (curr_count - s.initial_count) & tc.mask;
+    return s.offset + ticks_to_ns(tc, elapsed);
+}
+
+/// Return the best TimeCounter.
+pub fn best() ?*TimeCounter {
+    return best_tc;
+}
+
+/// Do a busy sleep of `ns` nanoseconds using TimeCounter.
+pub fn sleep(ns: b.Nanoseconds) void {
+    const start = read_time_nano();
+
+    while (true) {
+        var cur = read_time_nano();
+
+        if (cur < start) continue;
+
+        if (cur - start >= ns) {
+            cur = read_time_nano();
+
+            // Check again as we may have overflowed and read a bogus value..
+            // This is kinda hacky but oh well.
+            if (cur - start >= ns) {
+                break;
+            }
+        }
+
+        std.atomic.spinLoopHint();
+    }
+}
+
+/// Called periodically to handle counter overflows.
+pub fn update_overflow() void {
+    const tc = best_tc orelse return;
+    const s = state.load();
+    const cur_count = tc.read_count() & tc.mask;
+
+    if (s.initial_count > cur_count) {
+        // Counter has wrapped, use wrapping arithmetic to figure out by how much,
+        // then fold it into the offset and reset `initial_count`.
+        const overflowed_by = (cur_count -% s.initial_count) & tc.mask;
+        state.store(.{
+            .initial_count = cur_count,
+            .offset = s.offset + ticks_to_ns(tc, overflowed_by),
+        });
+    }
+}

src/kern/timer.zig

@@ -0,0 +1,175 @@
+//! Timer object implementation.
+//! Timer objects are useful when one wants to wait a fixed amount of time for an event to occur.
+const std = @import("std");
+const rtl = @import("rtl");
+const b = @import("base");
+const ke = b.ke;
+const ki = ke.private;
+const pl = b.pl;
+
+pub const Timer = struct {
+    pub const State = enum(u8) {
+        /// The timer is currently being handled.
+        Running,
+        /// The timer is currently enqueued.
+        Pending,
+        /// The timer was stopped.
+        Stopped,
+    };
+
+    hdr: ki.wait.DispatchHeader,
+
+    /// Timer state.
+    state: std.atomic.Value(State),
+    /// When the timer is bound to expire.
+    deadline: b.Nanoseconds,
+    /// Attached DPC.
+    dpc: ?*ke.Dpc,
+    /// Intrusive pairing heap node.
+    node: rtl.pairing_heap.Node,
+    /// CPU this timer is enqueued on.
+    cpu: ?*ke.Cpu,
+
+    pub fn init(self: *Timer) void {
+        self.* = .{
+            .hdr = undefined,
+            .state = .init(.Stopped),
+            .deadline = 0,
+            .dpc = null,
+            .node = .{},
+            .cpu = null,
+        };
+
+        self.hdr.init(.Notification);
+    }
+};
+
+/// Start a timer with an expiration time.
+/// A DPC that will be enqueued upon expiration can be passed.
+pub fn set(timer: *Timer, time: b.Nanoseconds, dpc: ?*ke.Dpc) void {
+    const ipl = timer.hdr.lock.acquire();
+    defer timer.hdr.lock.release(ipl);
+
+    const cpu = ke.curcpu();
+
+    if (timer.state.cmpxchgStrong(.Stopped, .Pending, .acquire, .monotonic) != null) {
+        // Could not set the timer.
+        return;
+    }
+
+    cpu.timers_lock.acquire_no_ipl();
+    defer cpu.timers_lock.release_no_ipl();
+
+    // Initialize the timer.
+    timer.deadline = ke.timecounter.read_time_nano() + time;
+
+    timer.cpu = cpu;
+    timer.dpc = dpc;
+    timer.hdr.signaled = 0;
+
+    cpu.timers_heap.insert(&timer.node);
+
+    if (cpu.timers_heap.root == &timer.node) {
+        // This is the earliest timer to expire, arm the hardware timer.
+        pl.arm_timer(time);
+    }
+
+    // Locks dropped
+}
+
+/// Cancel a timer.
+pub fn cancel(timer: *Timer) void {
+    const old_state = timer.state.load(.monotonic);
+
+    if (old_state == .Stopped or old_state == .Running) {
+        // Already stopped or currently executing.
+        return;
+    }
+
+    const ipl = timer.hdr.lock.acquire();
+    defer timer.hdr.lock.release(ipl);
+
+    if (timer.state.cmpxchgStrong(.Pending, .Stopped, .acquire, .monotonic) != null) {
+        // Something changed, it probably expired.
+        return;
+    }
+
+    if (timer.cpu) |cpu| {
+        // Dequeue the timer.
+        cpu.timers_lock.acquire_no_ipl();
+
+        cpu.timers_heap.remove(&timer.node);
+        timer.cpu = null;
+
+        cpu.timers_lock.release_no_ipl();
+    }
+
+    // Lock dropped
+}
+
+/// Compare two timers.
+pub fn cmp_timer(a: *rtl.pairing_heap.Node, b_: *rtl.pairing_heap.Node) std.math.Order {
+    const timer_a: *Timer = @fieldParentPtr("node", a);
+    const timer_b: *Timer = @fieldParentPtr("node", b_);
+
+    return std.math.order(timer_a.deadline, timer_b.deadline);
+}
+
+/// Called in a DPC when a timer has expired.
+pub fn handle_expiry(_: ?*anyopaque) void {
+    const cpu = ke.curcpu();
+    std.debug.assert(cpu.ipl == .Dispatch);
+
+    while (true) {
+        const curtime = ke.timecounter.read_time_nano();
+
+        cpu.timers_lock.acquire_no_ipl();
+
+        // Get the timer that expires the soonest.
+        const timer_node = cpu.timers_heap.root orelse {
+            cpu.timers_lock.release_no_ipl();
+            return;
+        };
+
+        const timer: *Timer = @fieldParentPtr("node", timer_node);
+
+        // If the timer expires more than 1ms in the future, consider it not yet due.
+        // Sub-millisecond differences are close enough to expire immediately.
+        if (timer.deadline > curtime and timer.deadline - curtime > std.time.ns_per_ms) {
+            const next = timer;
+            cpu.timers_lock.release_no_ipl();
+
+            // Re-check in case the timer expired while we were here.
+            const now = ke.timecounter.read_time_nano();
+            if (next.deadline <= now or next.deadline - now <= std.time.ns_per_ms) {
+                continue; // expired in the meantime, loop again
+            }
+
+            std.debug.assert(next.deadline > now);
+            pl.arm_timer(next.deadline - now);
+            return;
+        }
+
+        _ = cpu.timers_heap.pop();
+        cpu.timers_lock.release_no_ipl();
+
+        if (timer.state.cmpxchgStrong(.Pending, .Running, .acquire, .monotonic) != null) {
+            // Timer must've been canceled.
+            continue;
+        }
+
+        timer.hdr.lock.acquire_no_ipl();
+
+        // Set signaled.
+        timer.hdr.signaled = 1;
+        timer.cpu = null;
+
+        // Wake whomever was waiting on the timer.
+        ki.wait.satisfy_wait(&timer.hdr);
+
+        timer.state.store(.Stopped, .release);
+
+        if (timer.dpc) |dpc| ke.dpc.enqueue(dpc, null);
+        timer.hdr.lock.release_no_ipl();
+    }
+}

src/kern/um/impl.zig

@@ -0,0 +1,106 @@
+const std = @import("std");
+const b = @import("base");
+const c = b.pl.impl.c;
+const ke = b.ke;
+const ki = ke.private;
+
+pub const Cpu = struct { pthread: c.pthread_t };
+
+/// Return the current CPU.
+pub fn curcpu() *ke.Cpu {
+    return b.pl.impl.my_cpu;
+}
+
+fn set_signals(enabled: bool) bool {
+    var newmask: c.sigset_t = undefined;
+    var oldmask: c.sigset_t = undefined;
+
+    _ = c.sigfillset(&newmask);
+
+    const how = if (enabled) c.SIG_BLOCK else c.SIG_UNBLOCK;
+
+    if (c.pthread_sigmask(how, &newmask, &oldmask) != 0) {
+        ke.panic("pthread_sigmask failed", .{});
+    }
+
+    return std.mem.eql(u8, std.mem.asBytes(&newmask), std.mem.asBytes(&oldmask));
+}
+
+/// Set the hardware IPL.
+pub fn set_hardware_ipl(ipl: ke.Ipl) void {
+    _ = set_signals(@intFromEnum(ipl) > @intFromEnum(ke.Ipl.get_max_software()));
+}
+
+/// Disable interrupts and return the previous state.
+pub fn disable_interrupts() bool {
+    return set_signals(false);
+}
+
+/// Enable interrupts and return the previous state.
+pub fn enable_interrupts() bool {
+    return set_signals(true);
+}
+
+/// Restore an interrupt state.
+pub fn restore_interrupts(val: bool) void {
+    if (val) {
+        _ = enable_interrupts();
+    } else {
+        _ = disable_interrupts();
+    }
+}
+
+pub fn send_resched_ipi(cpu: *ke.Cpu) void {
+    _ = c.pthread_kill(cpu.impl.pthread, c.SIGUSR1);
+}
+
+pub const ThreadContext = struct {
+    ucontext: c.ucontext_t,
+
+    fn real_entry(entry_int: c_ulong, arg_int: c_ulong) callconv(.c) void {
+        const entry: *const fn (?*anyopaque) void = @ptrFromInt(entry_int);
+        const arg: ?*anyopaque = @ptrFromInt(arg_int);
+        ke.ipl.lower(.Zero);
+        entry(arg);
+    }
+
+    pub fn init(stack: b.VAddr, stack_size: usize, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) ThreadContext {
+        var new: ThreadContext = .{ .ucontext = undefined };
+
+        if (c.getcontext(&new.ucontext) == -1) {
+            ke.panic("getcontext() failed", .{});
+        }
+
+        new.ucontext.uc_stack.ss_sp = @ptrFromInt(stack);
+        new.ucontext.uc_stack.ss_size = stack_size;
+
+        // No successor
+        new.ucontext.uc_link = null;
+
+        const entry_int: usize = @intFromPtr(entry);
+        const arg_int: usize = @intFromPtr(arg);
+
+        c.makecontext(
+            &new.ucontext,
+            @ptrCast(&real_entry),
+            2,
+            @as(c_ulong, entry_int),
+            @as(c_ulong, arg_int),
+        );
+        return new;
+    }
+
+    pub fn load(self: *ThreadContext) void {
+        _ = c.setcontext(&self.ucontext);
+    }
+
+    pub fn switch_to(self: *ThreadContext, new: *ThreadContext) void {
+        const thread: *ke.Thread = @fieldParentPtr("context", self);
+
+        thread.lock.release_no_ipl();
+
+        if (c.swapcontext(&self.ucontext, &new.ucontext) == -1) {
+            ke.panic("swapcontext() failed", .{});
+        }
+    }
+};

src/kern/wait.zig

@@ -0,0 +1,235 @@
+//! Implementation of generic wait mechanisms.
+//! This is mostly based on the work by Arun Kishan on Windows 7.
+//! See more here: https://youtu.be/OAAiOEQhsK0
+const rtl = @import("rtl");
+const std = @import("std");
+const b = @import("base");
+const ke = b.ke;
+
+/// Header for waitable objects.
+/// This must be added to any structured which is considered waitable.
+pub const DispatchHeader = struct {
+    pub const Kind = enum {
+        /// When signaled, `signaled` is kept high and all waiters are woken up.
+        Notification,
+        /// When signaled, `signaled` is decreased until 0, and a single waiter is woken up.
+        Synchronization,
+    };
+
+    /// List of WaitBlocks.
+    waitblocks: rtl.List,
+    /// Kind of object.
+    kind: Kind,
+    /// Lock protecting the object.
+    lock: ke.SpinLock,
+    /// Signaled count.
+    signaled: u32,
+
+    /// Initialize a waitable object.
+    pub fn init(obj: *DispatchHeader, kind: Kind) void {
+        obj.kind = kind;
+        obj.lock = ke.SpinLock.init();
+        obj.signaled = 0;
+        obj.waitblocks.init();
+    }
+
+    fn consume(self: *DispatchHeader) void {
+        switch (self.kind) {
+            .Synchronization => self.signaled -= 1,
+            .Notification => {
+                // Object remains signaled.
+            },
+        }
+    }
+};
+
+pub const WaitBlock = struct {
+    pub const Status = enum(u8) {
+        /// WaitBlock is linked to an object as part of a thread that is waiting.
+        Active,
+        /// The wait associated with this WaitBlock has been satisfied (or timed out).
+        Inactive,
+        /// A signal was delivered to the wait (this could be before it was committed).
+        Signaled,
+    };
+    /// List linkage.
+    link: rtl.List.Entry,
+    /// Object being waited on.
+    object: *DispatchHeader,
+    /// Thread that owns the WaitBlock.
+    thread: *ke.Thread,
+    /// Status of the WaitBlock.
+    status: WaitBlock.Status,
+};
+
+/// Status of a wait operation.
+pub const Status = enum(u8) {
+    /// Wait is currently being processed.
+    InProgress,
+    /// Wait has been committed and is waiting for the object to be signaled.
+    Committed,
+    /// Wait has been satisfied (or timed out).
+    Satisfied,
+};
+
+/// Wait for the provided object to be signaled.
+pub fn wait_one(object: *DispatchHeader, timeout: ?b.Nanoseconds) !usize {
+    return wait_any(object, timeout, null);
+}
+
+/// Wait for any of the provided objects to be signaled.
+/// Returns the index of the object that was signaled, or an error.
+/// If `timeout` is not provided, it will wait indefinitely.
+/// If `waitblocks` is specified, then the wait will use those waitblocks for the operation.
+/// Note that if `timeout` is provided, then one additional waitblock must be allocated.
+pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks: ?[*]WaitBlock) !usize {
+    const ipl = ke.ipl.raise(.Dispatch);
+    defer ke.ipl.lower(ipl);
+    const curtd = ke.curcpu().current_thread;
+    const obj_count = objects.len;
+    const has_timeout = timeout != null;
+    const blocks = waitblocks orelse blk: {
+        std.debug.assert(obj_count <= curtd.waitblocks.len - @intFromBool(has_timeout));
+        break :blk &curtd.waitblocks;
+    };
+
+    const total_count = obj_count + @intFromBool(has_timeout);
+    const timer_i = obj_count;
+    const timer = &curtd.timer;
+    const timer_block = &curtd.waitblocks[3];
+
+    var satisfier: ?usize = null;
+    curtd.wait_status.store(.InProgress, .acquire);
+
+    for (0..total_count) |i| {
+        const is_timer = has_timeout and i == timer_i;
+        var obj = if (is_timer) &timer.hdr else objects[i];
+        var wb = if (is_timer) timer_block else &blocks[i];
+
+        obj.lock.acquire_no_ipl();
+        defer obj.lock.release_no_ipl();
+
+        if (obj.signaled > 0) {
+            // Object was already signaled. Try consuming it.
+            if (curtd.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acquire, .monotonic) == null) {
+                obj.consume();
+            }
+            // We have already been satisfied in the meantime, abort.
+            satisfier = i;
+            break;
+        }
+
+        wb.object = obj;
+        wb.thread = curtd;
+        wb.status = .Active;
+        // We are not satisfied yet, so add a waitblock to the object's waitblocks
+        obj.waitblocks.insert_tail(&wb.link);
+    }
+
+    // Wait was already satisfied, back out.
+    if (satisfier != null or (has_timeout and timeout.? == 0)) {
+        std.debug.assert(curtd.wait_status.load(.monotonic) == .Satisfied);
+
+        // Remove any wait block we might've installed
+        for (0..satisfier) |i| {
+            const is_timer = has_timeout and i == timer_i;
+            var obj = if (is_timer) &timer.hdr else objects[i];
+            var wb = if (is_timer) timer_block else &blocks[i];
+
+            obj.lock.acquire_no_ipl();
+            wb.status = .Inactive;
+            wb.link.remove();
+            obj.lock.release_no_ipl();
+        }
+
+        return satisfier orelse error.Timeout;
+    }
+
+    if (has_timeout) {
+        ke.timer.set(timer, timeout.?, null);
+    }
+
+    // Now try committing the wait.
+    // While we're trying to commit the wait, the object locks have been released, and the state could therefore change.
+    // We need to re-check the state of the wait before actually blocking.
+    if (curtd.wait_status.cmpxchgStrong(.InProgress, .Committed, .acquire, .monotonic) == null) {
+        // We're good, now actually block.
+        ke.sched.block();
+    }
+
+    // We're back!
+    // Stop the timer if it was set.
+    if (has_timeout) {
+        ke.timer.cancel(timer);
+    }
+
+    // Find the object that satisfied us.
+    for (0..total_count) |i| {
+        const is_timer = has_timeout and i == timer_i;
+        var obj = if (is_timer) &timer.hdr else objects[i];
+        var wb = if (is_timer) timer_block else &blocks[i];
+
+        obj.lock.acquire_no_ipl();
+        if (wb.status == .Active) {
+            // Waitblock is still active, remove it from the list.
+            wb.link.remove();
+        }
+        if (wb.status == .Signaled or wb.status == .SignaledInProgress) {
+            // This waitblock was signaled, it must be the one that satisfied the wait.
+            std.debug.assert(satisfier == null);
+            satisfier = i;
+        }
+        // Ignore inactive waitblocks
+        obj.lock.release_no_ipl();
+    }
+
+    const final_sat = satisfier orelse return error.Timeout;
+    return if (has_timeout and final_sat == timer_i) error.Timeout else final_sat;
+}
+
+/// Satisfy a wait on an object.
+pub fn satisfy_wait(obj: *DispatchHeader) void {
+    const all = obj.kind == .Notification;
+
+    // Go through (potentially) all wait blocks and try to satisfy them.
+    while (!obj.waitblocks.is_empty() and obj.signaled > 0) {
+        // Get the first waitblock from the list.
+        var wb: *WaitBlock = @fieldParentPtr("link", obj.waitblocks.first());
+        var td = wb.thread;
+
+        // Remove it.
+        wb.link.remove();
+
+        // Three cases may occur here:
+        // 1. The wait was still preparing (status == .InProgress) and we interrupted it.
+        // 2. The wait was already committed (status == .Committed) and we satisfied it.
+        // 3. The wait was already satisfied by another object (status == .Satisfied).
+
+        // 1.
+        if (td.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acquire, .monotonic) == null) {
+            // We interrupted the wait while it was being prepared.
+            wb.status = .Signaled;
+            obj.consume();
+        }
+
+        // 2.
+        else if (td.wait_status.cmpxchgStrong(.Committed, .Satisfied, .acquire, .monotonic) == null) {
+            // We interrupted the wait while it was committed.
+            // Wake the thread.
+            wb.status = .Signaled;
+            obj.consume();
+            ke.sched.unblock(td);
+        }
+
+        // 3.
+        else if (td.wait_status.load(.acquire) == .Satisfied) {
+            // Someone else satisfied the wait, deactivate the wait block.
+            wb.status = .Inactive;
+            continue;
+        }
+
+        if (!all) {
+            break;
+        }
+    }
+}