initial commit

src/kern/cpu.zig

@@ -0,0 +1,88 @@
+const std = @import("std");
+const rtl = @import("rtl");
+const b = @import("base");
+const ke = b.ke;
+const ki = b.ke.private;
+
+comptime {
+    if (!@hasDecl(ki.impl, "Cpu")) @compileError("impl must provide Cpu");
+}
+
+pub const ThreadStatus = packed struct(u8) {
+    interactive: bool,
+    priority: u7,
+};
+
+pub const PreemptionReason = enum(u8) {
+    None,
+    /// A higher priority thread was readied.
+    HigherPriority,
+};
+
+/// Per-CPU state
+pub const Cpu = struct {
+    /// Implementation-dependent CPU data.
+    impl: ki.impl.Cpu,
+    /// Unique ID of this CPU.
+    id: u32,
+    /// Current IPL on this CPU.
+    ipl: ke.Ipl,
+    /// Bitmask of pending software interrupts on this CPU.
+    pending_softints: std.atomic.Value(u8),
+    /// Currently running thread.
+    current_thread: ?*ke.Thread,
+    /// Atomic byte for getting the current thread's status locklessly.
+    /// 1 bit for thread interactivity, 7 for priority.
+    current_thread_status: std.atomic.Value(ThreadStatus),
+    /// Per-CPU idle thread.
+    idle_thread: ?*ke.Thread,
+    /// Thread selected for preemption.
+    next_thread: ?*ke.Thread,
+    /// The reason why the current thread was preempted.
+    preemption_reason: PreemptionReason,
+    /// Flag indicating the start of the scheduling timer.
+    start_timer: bool,
+    /// Queue of DPCs on this CPU.
+    dpc_queue: rtl.List,
+    /// Lock over this CPU's DPC queue.
+    dpc_lock: ke.SpinLock,
+    /// Heap of pending timers on this CPU.
+    timers_heap: rtl.PairingHeap(.min, ki.timer.cmp_timer),
+    /// Lock over the timer heap.
+    timers_lock: ke.SpinLock,
+    /// Per-CPU scheduler state.
+    sched_data: ki.sched.PerCpu,
+    /// DPCs
+    resched_dpc: ke.Dpc,
+    timer_dpc: ke.Dpc,
+    resched_timer: ke.Timer,
+
+    /// Initialize `cpu` for usage with its idle thread being `thread`.
+    pub fn init(cpu: *Cpu, thread: *ke.Thread) void {
+        cpu.* = .{
+            .id = cpu.id,
+            .ipl = .Zero,
+            .pending_softints = .init(0),
+            .idle_thread = thread,
+            .current_thread = thread,
+            .timer_dpc = .init(ki.timer.handle_expiry),
+            .resched_dpc = .init(ki.sched.clock),
+            .timers_heap = .init(),
+            .current_thread_status = .init(.{ .interactive = false, .priority = 0 }),
+            .next_thread = null,
+            .dpc_lock = .init(),
+            .preemption_reason = .None,
+            .start_timer = false,
+            .dpc_queue = undefined,
+            .timers_lock = .init(),
+            .sched_data = undefined,
+            .resched_timer = undefined,
+            .impl = cpu.impl,
+        };
+
+        cpu.dpc_queue.init();
+        cpu.resched_timer.init();
+
+        ki.sched.init_cpu(&cpu.sched_data);
+    }
+};

src/kern/dpc.zig

@@ -0,0 +1,133 @@
+const std = @import("std");
+const config = @import("config");
+const rtl = @import("rtl");
+const b = @import("base");
+const ke = b.ke;
+const ki = ke.private;
+
+/// Deferred Procedure Call (DPC) structure.
+/// Used for scheduling work to be done when IPL is lowered below `ke.Ipl.Dispatch`.
+pub const Dpc = struct {
+    /// Entry into per-CPU DPC queue
+    link: rtl.List.Entry,
+    /// Routine to call
+    func: *const fn (?*anyopaque) void,
+    /// Argument passed to the routine
+    arg: ?*anyopaque,
+    /// Whether or not the DPC is currently inserted
+    inserted: bool,
+
+    /// Initialize a DPC for `func`.
+    pub fn init(func: *const fn (?*anyopaque) void) Dpc {
+        return .{
+            .link = undefined,
+            .func = func,
+            .arg = undefined,
+            .inserted = false,
+        };
+    }
+};
+
+/// Enqueue a DPC.
+/// If the DPC is already enqueued, this function is a no-op.
+pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void {
+    const ipl = ke.ipl.raise(.High);
+    const cpu = ke.curcpu();
+
+    if (!dpc.inserted) {
+        dpc.arg = arg;
+
+        // Insert the DPC on this CPU's DPC queue
+        cpu.dpc_lock.acquire_no_ipl();
+
+        cpu.dpc_queue.insert_tail(&dpc.link);
+
+        // Mark the DPC as pending on this CPU
+        ki.ipl.set_softint_pending(cpu, .Dispatch);
+
+        dpc.inserted = true;
+
+        cpu.dpc_lock.release_no_ipl();
+    }
+
+    ke.ipl.lower(ipl);
+}
+
+fn dispatch_queue(cpu: *ke.Cpu) void {
+    cpu.ipl = .Dispatch;
+
+    // Mark as handled, this must be done before enabling interrupts or we will race.
+    ki.ipl.clear_softint_pending(cpu, .Dispatch);
+
+    _ = ki.impl.enable_interrupts();
+
+    while (true) {
+        const ipl = cpu.dpc_lock.acquire_at(.High);
+
+        var dpc: *Dpc = undefined;
+
+        if (cpu.dpc_queue.is_empty()) {
+            // No more DPCs to process.
+            cpu.dpc_lock.release(ipl);
+            break;
+        }
+
+        // Pop the head
+        var first_elem = cpu.dpc_queue.first();
+
+        first_elem.remove();
+
+        dpc = @fieldParentPtr("link", first_elem);
+        dpc.inserted = false;
+
+        // An interrupt which would enqueue this DPC could occur between loading the argument and calling the routine,
+        // which is why we capture it here to ensure that we get the intended context.
+        const arg = dpc.arg;
+
+        cpu.dpc_lock.release(ipl);
+
+        std.debug.assert(cpu.ipl == .Dispatch);
+
+        dpc.func(arg);
+    }
+
+    if (cpu.start_timer) {
+        ke.timer.set(&cpu.resched_timer, std.time.ns_per_ms * config.CONFIG_SCHED_TIMESLICE, &cpu.resched_dpc);
+    }
+
+    if (cpu.preemption_reason == .HigherPriority) {
+        // Reload the quantum for the new thread.
+        ke.timer.cancel(&cpu.resched_timer);
+
+        ke.timer.set(&cpu.resched_timer, std.time.ns_per_ms * config.CONFIG_SCHED_TIMESLICE, &cpu.resched_dpc);
+    }
+
+    cpu.start_timer = false;
+    cpu.preemption_reason = .None;
+
+    if (cpu.next_thread != null) {
+        ki.sched.handle_preemption(cpu);
+    }
+
+    _ = ki.impl.disable_interrupts();
+}
+
+/// Dispatch the DPC queue on `cpu`.
+pub fn dispatch(cpu: *ke.Cpu) void {
+    // DPC processing is done at Ipl.Dispatch
+    // We can't call lower/raise here because they might call us again recursively.
+    var mycpu = cpu;
+    const old_ipl = cpu.ipl;
+    const int_state = ki.impl.disable_interrupts();
+
+    while (ki.ipl.is_softint_pending(mycpu, .Dispatch)) {
+        dispatch_queue(mycpu);
+
+        // Our CPU might have changed.
+        mycpu = ke.curcpu();
+    }
+
+    cpu.ipl = old_ipl;
+
+    ki.impl.restore_interrupts(int_state);
+}

src/kern/init.zig

@@ -0,0 +1,54 @@
+const std = @import("std");
+const b = @import("base");
+const pl = b.pl;
+const ke = b.ke;
+const ki = ke.private;
+
+var thread0: ke.Thread = undefined;
+
+fn idle(_: ?*anyopaque) void {
+    while (true) {
+        std.atomic.spinLoopHint();
+    }
+}
+
+var stack: [8192]u8 align(16) = undefined;
+
+fn handler(_: ?*anyopaque) void {
+    std.log.info("Hi! CPU {}", .{ke.curcpu().id});
+
+    while (true) {}
+}
+
+fn make_thread(entry: *const fn (?*anyopaque) void, td: *ke.Thread) void {
+    const _stack = std.heap.page_allocator.alloc(u8, 16384) catch @panic("wtf");
+
+    td.init(@intFromPtr(_stack.ptr), 16384, entry, null);
+}
+
+var threads: [10]ke.Thread = undefined;
+
+pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void {
+    pl.early_init();
+    ki.bootstrap_cpu.init(&thread0);
+
+    thread0.init(@intFromPtr(&stack), 8192, idle, null);
+    thread0.priority = 0;
+    thread0.priority_class = .Idle;
+
+    std.log.info("hello, world", .{});
+    std.log.info("Zag for {s}, cmdline is {?s}", .{ pl.name, boot_info.cmdline });
+
+    pl.late_init();
+
+    _ = ke.ipl.raise(.Dispatch);
+
+    for (0..10) |i| {
+        make_thread(handler, &threads[i]);
+        ke.sched.enqueue(&threads[i]);
+    }
+
+    ke.ipl.lower(.Zero);
+
+    while (true) {}
+}

src/kern/ipl.zig

@@ -0,0 +1,96 @@
+const std = @import("std");
+const b = @import("base");
+const ke = b.ke;
+const ki = ke.private;
+
+/// Interrupt priority level (IPL)
+pub const Ipl = enum(u8) {
+    /// All interrupts enabled.
+    Zero = 0,
+    /// Preemption disabled.
+    Dispatch = 1,
+    /// Device interrupts disabled.
+    Device = 13,
+    /// All interrupts disabled.
+    High = 15,
+
+    pub fn get_max_software() Ipl {
+        return .Dispatch;
+    }
+};
+
+/// Raise IPL to `new`.
+/// Panics if the current IPL is higher than requested.
+pub fn raise(new: Ipl) Ipl {
+    var cpu = ke.curcpu();
+    const old = cpu.ipl;
+
+    if (@intFromEnum(new) < @intFromEnum(old)) {
+        ke.panic("ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})\n", .{ new, old });
+    }
+
+    if (@intFromEnum(new) > @intFromEnum(Ipl.get_max_software())) {
+        ki.impl.set_hardware_ipl(new);
+    }
+
+    cpu.ipl = new;
+
+    return old;
+}
+
+/// Lower IPL to `new`.
+/// Panics if the current IPL is lower than requested.
+pub fn lower(new: Ipl) void {
+    var cpu = ke.curcpu();
+    const old = cpu.ipl;
+
+    if (@intFromEnum(new) > @intFromEnum(old)) {
+        ke.panic("ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})\n", .{ new, old });
+    }
+
+    if (@intFromEnum(new) <= @intFromEnum(Ipl.get_max_software()) and @intFromEnum(old) > @intFromEnum(Ipl.get_max_software())) {
+        ki.impl.set_hardware_ipl(.Zero);
+    }
+
+    if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and is_softint_pending(cpu, .Dispatch)) {
+        ki.dpc.dispatch(cpu);
+    }
+
+    cpu.ipl = new;
+}
+
+comptime {
+    if (!@hasDecl(ki.impl, "set_hardware_ipl")) @compileError("impl must provide set_hardware_ipl()");
+}
+
+/// Set the hardware IPL to `new`
+pub fn set_hardware(new: Ipl) Ipl {
+    var cpu = ke.curcpu();
+    const old = cpu.ipl;
+
+    cpu.ipl = new;
+
+    ki.impl.set_hardware_ipl(new);
+
+    return old;
+}
+
+/// Mark a software interrupt of IPL `ipl` on `cpu` as pending
+pub fn set_softint_pending(cpu: *ke.Cpu, ipl: Ipl) void {
+    std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
+    _ = cpu.pending_softints.bitSet(@intCast(@intFromEnum(ipl)), .monotonic);
+}
+
+/// Mark a software interrupt of IPL `ipl` on `cpu` as handled
+pub fn clear_softint_pending(cpu: *ke.Cpu, ipl: Ipl) void {
+    std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
+    _ = cpu.pending_softints.bitReset(@intCast(@intFromEnum(ipl)), .monotonic);
+}
+
+/// Check whether a software interrupt of IPL `ipl` on `cpu` is pending
+pub fn is_softint_pending(cpu: *ke.Cpu, ipl: Ipl) bool {
+    std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
+    const shift: u3 = @intCast(@intFromEnum(ipl));
+    const bit: u8 = @as(u8, 1) << shift;
+    return cpu.pending_softints.load(.monotonic) & bit != 0;
+}

src/kern/log.zig

@@ -0,0 +1,57 @@
+const std = @import("std");
+const b = @import("base");
+const pl = b.pl;
+const ke = b.ke;
+
+pub var debug_lock = ke.SpinLock.init();
+
+pub fn print(bytes: []const u8) void {
+    for (bytes) |c| {
+        pl.debug_write(c);
+    }
+}
+
+const LogWriter = struct {
+    interface: std.Io.Writer,
+
+    pub fn init() LogWriter {
+        return .{
+            .interface = .{
+                .vtable = &.{ .drain = drain },
+                .buffer = &.{},
+                .end = 0,
+            },
+        };
+    }
+
+    fn drain(_: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize {
+        var total_written: usize = 0;
+        for (data) |slice| {
+            print(slice);
+            total_written += slice.len;
+        }
+
+        return total_written;
+    }
+};
+
+pub fn log(
+    comptime level: std.log.Level,
+    comptime scope: @EnumLiteral(),
+    comptime fmt: []const u8,
+    args: anytype,
+) void {
+    _ = level;
+    _ = scope;
+
+    const ipl = debug_lock.acquire();
+
+    var writer = LogWriter.init();
+
+    writer.interface.print(
+        fmt ++ "\r\n",
+        args,
+    ) catch unreachable;
+
+    debug_lock.release(ipl);
+}

src/kern/panic.zig

@@ -0,0 +1,19 @@
+const std = @import("std");
+const b = @import("base");
+const ke = b.ke;
+const ki = ke.private;
+
+/// Panics!
+pub fn panic(comptime fmt: []const u8, args: anytype) noreturn {
+    ki.log.debug_lock.acquire_no_ipl();
+
+    var buf: [1024]u8 = undefined;
+
+    const str = std.fmt.bufPrint(&buf, fmt, args) catch unreachable;
+
+    ki.log.print("Panic!\n");
+    ki.log.print(str);
+
+    ki.log.debug_lock.release_no_ipl();
+    while (true) {}
+}

src/kern/private.zig

@@ -0,0 +1,31 @@
+const std = @import("std");
+const config = @import("config");
+
+const b = @import("base");
+const ke = b.ke;
+const pl = b.pl;
+
+const init_mod = @import("init.zig");
+
+pub const impl = if (@hasDecl(config, "CONFIG_ARCH_AMD64"))
+    @import("amd64/impl.zig")
+else if (@hasDecl(config, "CONFIG_ARCH_UM"))
+    @import("um/impl.zig")
+else
+    @compileError("unsupported architecture");
+
+pub const init = init_mod.init;
+pub var bootstrap_cpu: ke.Cpu = undefined;
+
+// === Exported Modules ===
+pub const ipl = @import("ipl.zig");
+pub const panic = @import("panic.zig");
+pub const spinlock = @import("spinlock.zig");
+pub const log = @import("log.zig");
+pub const cpu = @import("cpu.zig");
+pub const thread = @import("thread.zig");
+pub const dpc = @import("dpc.zig");
+pub const timecounter = @import("timecounter.zig");
+pub const sched = @import("sched.zig");
+pub const wait = @import("wait.zig");
+pub const timer = @import("timer.zig");

src/kern/root.zig

@@ -0,0 +1,68 @@
+const config = @import("config");
+
+// === Exported Modules ===
+pub const private = @import("private.zig");
+
+const p = private;
+
+// === Exported types ===
+pub const Ipl = p.ipl.Ipl;
+pub const SpinLock = p.spinlock.SpinLock;
+pub const Thread = p.thread.Thread;
+pub const Dpc = p.dpc.Dpc;
+pub const TimeCounter = p.timecounter.TimeCounter;
+pub const Timer = p.timer.Timer;
+
+// === Exported Interfaces ===
+pub const log = p.log.log;
+pub const panic = p.panic.panic;
+
+pub const ipl = struct {
+    pub const raise = p.ipl.raise;
+    pub const lower = p.ipl.lower;
+    pub const set_hardware = p.ipl.set_hardware;
+};
+
+pub const thread = struct {
+    pub const init = p.thread.init;
+};
+
+pub const dpc = struct {
+    pub const enqueue = p.dpc.enqueue;
+};
+
+pub const timecounter = struct {
+    pub const register = p.timecounter.register;
+    pub const read_time_nano = p.timecounter.read_time_nano;
+};
+
+pub const sched = struct {
+    pub const enqueue = p.sched.enqueue;
+    pub const block = p.sched.block;
+    pub const unblock = p.sched.unblock;
+};
+
+pub const timer = struct {
+    pub const set = p.timer.set;
+    pub const cancel = p.timer.cancel;
+};
+
+pub const wait = struct {
+    pub const wait_one = p.wait.wait_one;
+    pub const wait_any = p.wait.wait_any;
+};
+
+pub const Cpu = p.cpu.Cpu;
+
+comptime {
+    if (!@hasDecl(p.impl, "curcpu")) @compileError("impl must provide curcpu()");
+}
+
+/// Returns the current CPU
+pub const curcpu: fn () *Cpu = p.impl.curcpu;
+
+/// Number of CPUs on the system
+pub var ncpus: usize = 0;
+
+/// Array of CPUs
+pub var cpus: []*Cpu = undefined;

src/kern/sched.zig

@@ -0,0 +1,764 @@
+//! Kernel scheduler inspired by FreeBSD's ULE scheduler.
+//!
+//! ## Overview
+//!
+//! Threads are divided into three priority classes:
+//!
+//! - `Realtime`: Always scheduled before batch and idle threads.
+//!   Picked from `realtime_runq` in strict priority order.
+//!   Interactive batch threads are promoted into this queue.
+//! - `Batch`: Scheduled via a circular calendar queue that rotates
+//!   on every tick, ensuring all threads get CPU time proportional
+//!   to their priority.
+//! - `Idle`: Only scheduled when both realtime and batch queues are empty.
+//!
+//! ## Interactivity Scoring
+//!
+//! Each batch thread is assigned an interactivity score based on the
+//! ratio of its recent sleep time to its recent CPU time:
+//!
+//! - If `sleep_time > run_time`: score = `run_time / (sleep_time / scaling_factor)`
+//!   — heavily sleeping threads score low (more interactive).
+//! - If `run_time > sleep_time`: score = `scaling_factor * 2 - (sleep_time / (run_time / scaling_factor))`
+//!   — CPU-bound threads score high (less interactive).
+//! - Niceness is added to the score, making low-nice threads easier to
+//!   qualify as interactive.
+//!
+//! Threads scoring below `interactivity_threshold` are considered
+//! interactive and promoted to the realtime queue with a priority derived
+//! from their score. This lets I/O-bound threads respond quickly without
+//! being assigned a static realtime priority.
+//!
+//! To prevent history from dominating forever, `sleep_time` and `run_time`
+//! are decayed periodically by `sched_clamp_time`:
+//! - Values are halved when their sum exceeds 120% of the 5-second cap.
+//! - Values are reduced by 20% when the sum exceeds 5 seconds.
+//! - Values are reset to (max, 1) when the sum exceeds 10 seconds.
+//!
+//! ## Calendar Queue
+//!
+//! The calendar queue is a circular array of `runqueues_n` (64) lists.
+//! Two indices drive it:
+//!
+//! - `runidx`: The queue currently being drained by the scheduler.
+//! - `insidx`: The queue where newly inserted threads land.
+//!   Always one slot ahead of `runidx` so fresh threads are never
+//!   picked in the same tick they were enqueued.
+//!
+//! Threads are inserted at `(insidx + prio_high_batch - thread.priority) % runqueues_n`,
+//! placing lower-priority threads further ahead in the rotation so
+//! they run less frequently than higher-priority ones.
+//!
+//! On every tick (10ms), `insidx` advances by one, giving all threads
+//! the chance to run over time regardless of priority.
+//!
+//! ## Preemption
+//!
+//! Timeshared (batch) threads do not preempt each other by default,
+//! mirroring FreeBSD's default preemption threshold. Interactive threads
+//! may still preempt non-interactive batch threads. Realtime threads
+//! always preempt batch and idle threads.
+//!
+//! Remote preemption (sending an IPI to another CPU) is done lock-free
+//! by reading a packed `current_thread_status` field that encodes the
+//! running thread's priority and interactivity in a single atomic byte.
+//!
+//! ## Load Balancing
+//!
+//! Two mechanisms keep load distributed across CPUs:
+//!
+//! - **Work stealing**: When a CPU goes idle, it immediately attempts to
+//!   steal a thread from the most loaded CPU.
+//! - **Periodic balancing**: Every ~1 second (randomized with an LCG to
+//!   avoid phase-locking with periodic threads), the bootstrap CPU moves
+//!   threads from the most loaded CPU to the least loaded one.
+//!
+//! ## CPU Selection
+//!
+//! When enqueuing a thread, the target CPU is chosen in priority order:
+//! 1. Last CPU the thread ran on, if it can be preempted or the thread is pinned.
+//! 2. Least loaded CPU on which the thread would run immediately (preemptible).
+//! 3. Least loaded CPU overall.
+//!
+//! See `sched_ule.c` in FreeBSD for the original implementation this is based on.
+const std = @import("std");
+const rtl = @import("rtl");
+const config = @import("config");
+const b = @import("base");
+const ke = b.ke;
+const ki = ke.private;
+
+comptime {
+    if (!@hasDecl(ki.impl, "send_resched_ipi")) @compileError("impl must provide send_resched_ipi");
+}
+
+const runqueues_n = 64;
+const interactivity_threshold = 30;
+const scaling_factor = 50;
+const preempt_threshold = ke.Thread.Priority.realtime;
+
+pub const RunQueue = struct {
+    /// Bitmap of non-empty queues. A set bit represents a non-empty queue.
+    status: u64,
+    queues: [runqueues_n]rtl.List,
+};
+
+/// Per-CPU scheduler data.
+pub const PerCpu = struct {
+    /// Current load on this CPU (number of ready threads)
+    load: std.atomic.Value(usize),
+    /// Lock over this CPU's queues.
+    queues_lock: ke.SpinLock,
+    /// Array of queues from which to pick realtime threads from.
+    realtime_queue: RunQueue,
+    /// Calendar queue.
+    calendar_queue: RunQueue,
+    /// Queue of idle threads.
+    idle_queue: rtl.List,
+    /// The index from which to pick the next thread.
+    runidx: u8,
+    /// The index on which the next thread is going to be inserted.
+    insidx: u8,
+    /// Used to indicate to this CPU it can steal work
+    steal_work: bool,
+};
+
+/// Handle preemption.
+/// This is called in DPC dispatch when it notices `next_thread` is set.
+pub fn handle_preemption(cpu: *ke.Cpu) void {
+    const sched_cpu = &cpu.sched_data;
+    sched_cpu.queues_lock.acquire_no_ipl();
+
+    const cur = cpu.current_thread.?;
+
+    const next = cpu.next_thread orelse {
+        sched_cpu.queues_lock.release_no_ipl();
+        return;
+    };
+
+    cpu.next_thread = null;
+
+    sched_cpu.queues_lock.release_no_ipl();
+    cur.lock.acquire_no_ipl();
+
+    if (cur != cpu.idle_thread and cur.state != .Blocked) {
+        // Put it back in the queue.
+        sched_cpu.queues_lock.acquire_no_ipl();
+        insert_in_queue(sched_cpu, cur, false);
+        sched_cpu.queues_lock.release_no_ipl();
+    }
+
+    do_switch(cpu, cur, next);
+
+    // cur.lock dropped
+}
+
+/// Called every time slice.
+pub fn clock(_: ?*anyopaque) void {
+    const cpu = ke.curcpu();
+    const sched_cpu = &cpu.sched_data;
+    std.debug.assert(cpu.ipl == .Dispatch);
+
+    const curtd = cpu.current_thread orelse return;
+
+    curtd.lock.acquire_no_ipl();
+    defer curtd.lock.release_no_ipl();
+
+    sched_cpu.queues_lock.acquire_no_ipl();
+    defer sched_cpu.queues_lock.release_no_ipl();
+
+    // Advance the insert index every tick, while keeping a separation of 1 with runidx.
+    // This ensures fairness.
+    if (sched_cpu.runidx == sched_cpu.insidx) {
+        sched_cpu.insidx = (sched_cpu.insidx) % runqueues_n;
+    }
+
+    if (curtd.priority_class == .Batch) {
+        // Update interactivity metrics.
+        curtd.run_time += config.CONFIG_SCHED_TIMESLICE;
+
+        clamp_time(curtd);
+        recompute_priority(curtd);
+    }
+
+    if (cpu.next_thread != null) {
+        // Another thread was already selected for preemption.
+        return;
+    }
+
+    // Pick a new thread to run.
+    const newtd = select_thread(curtd, cpu, false);
+
+    // Ensure the timer gets reloaded.
+    cpu.start_timer = true;
+    cpu.next_thread = newtd;
+}
+
+/// Enqueue a thread.
+pub fn enqueue(td: *ke.Thread) void {
+    const ipl = td.lock.acquire();
+    const cpu = pick_cpu(td);
+    enqueue_on_cpu(cpu, td);
+    td.lock.release(ipl);
+}
+
+/// Block the currently running thread.
+pub fn block() void {
+    const ipl = ke.ipl.raise(.Dispatch);
+    const td = ke.curcpu().current_thread.?;
+
+    td.lock.acquire_no_ipl();
+
+    td.state = .Blocked;
+    td.sleep_start = ke.timecounter.read_time_nano();
+    td.runq = null;
+
+    yield(ke.curcpu());
+
+    // td lock released
+    ke.ipl.lower(ipl);
+}
+
+/// Unblock a thread.
+pub fn unblock(td: *ke.Thread) void {
+    const ipl = td.lock.acquire();
+
+    td.sleep_time = (ke.timecounter.read_time_nano() - td.sleep_start) / std.time.ns_per_ms;
+
+    if (td.sleep_time >= config.CONFIG_SCHED_TIMESLICE and .priority_class == .Batch) {
+        // If we have slept for more than a tick, update interactivity.
+        clamp_time(td);
+        recompute_priority(td);
+    }
+
+    // Enqueue the thread
+    const cpu = pick_cpu(td);
+    enqueue_on_cpu(cpu, td);
+
+    td.lock.release(ipl);
+}
+
+/// Initialize a CPU for use by the scheduler.
+pub fn init_cpu(percpu: *PerCpu) linksection(b.init) void {
+    percpu.* = .{
+        .insidx = 0,
+        .runidx = 0,
+        .calendar_queue = .{ .status = 0, .queues = undefined },
+        .realtime_queue = .{ .status = 0, .queues = undefined },
+        .queues_lock = .init(),
+        .idle_queue = undefined,
+        .steal_work = false,
+        .load = .init(0),
+    };
+
+    percpu.idle_queue.init();
+
+    for (0..runqueues_n) |i| {
+        percpu.calendar_queue.queues[i].init();
+    }
+
+    for (0..runqueues_n) |i| {
+        percpu.realtime_queue.queues[i].init();
+    }
+}
+
+fn calendar_queue_increment(cpu: *PerCpu) void {
+    cpu.runidx = (cpu.runidx + 1) % runqueues_n;
+
+    // Ensure insidx is always one ahead of runidx
+    if (cpu.runidx == cpu.insidx) {
+        cpu.insidx = (cpu.insidx + 1) % runqueues_n;
+    }
+}
+
+fn pick_realtime_thread(cpu: *PerCpu, minprio: u8, migrate: bool) ?*ke.Thread {
+    std.debug.assert(cpu.queues_lock.is_locked());
+
+    const runq = &cpu.realtime_queue;
+
+    if (runq.status == 0) return null;
+
+    // Search higher priority queues first.
+    var i: usize = ke.Thread.Priority.max;
+
+    qloop: while (i >= minprio) : (i -= 1) {
+        const bit = (@as(u64, 1) << @intCast(i));
+
+        if (runq.status & bit == 0) {
+            // Queue is empty.
+            continue;
+        }
+
+        const curr_queue = &runq.queues[i];
+
+        std.debug.assert(!curr_queue.is_empty());
+
+        // Get the first thread of the queue.
+        var td: *ke.Thread = @fieldParentPtr("runq_link", curr_queue.first());
+
+        if (migrate) {
+            // Find the first thread that is not pinned.
+            while (td.pinned) {
+                const entry = td.runq_link.next;
+
+                td = @fieldParentPtr("runq_link", entry);
+
+                // Every thread is pinned, go through another queue
+                if (entry == &curr_queue.head) continue :qloop;
+            }
+        }
+
+        // Remove it from the queue.
+        td.runq_link.remove();
+
+        if (curr_queue.is_empty()) {
+            // Queue is now empty, clear its bit.
+            runq.status &= ~bit;
+        }
+    }
+
+    return null;
+}
+
+fn pick_batch_thread(cpu: *PerCpu, migrate: bool) ?*ke.Thread {
+    std.debug.assert(cpu.queues_lock.is_locked());
+    const runq = &cpu.calendar_queue;
+
+    if (runq.status == 0) return null;
+
+    // Search all queues, starting from runidx.
+    var i: usize = cpu.runidx;
+    var cnt: usize = 0;
+
+    qloop: while (cnt < runqueues_n) : ({
+        i = (i + 1) % runqueues_n;
+        cnt += 1;
+    }) {
+        const bit = (@as(u64, 1) << @intCast(i));
+
+        if (runq.status & bit == 0) {
+            // Queue is empty.
+            continue;
+        }
+
+        const curr_queue = &runq.queues[i];
+
+        std.debug.assert(!curr_queue.is_empty());
+
+        // Get the first thread of the queue.
+        var td: *ke.Thread = @fieldParentPtr("runq_link", curr_queue.first());
+
+        if (migrate) {
+            // Find the first thread that is not pinned.
+            while (td.pinned) {
+                const entry = td.runq_link.next;
+
+                td = @fieldParentPtr("runq_link", entry);
+
+                // Every thread is pinned, go through another queue
+                if (entry == &curr_queue.head) continue :qloop;
+            }
+        }
+
+        // Remove it from the queue.
+        td.runq_link.remove();
+
+        if (curr_queue.is_empty()) {
+            // Queue is now empty, clear its bit and increment runidx.
+            runq.status &= ~bit;
+
+            if (!migrate) {
+                calendar_queue_increment(cpu);
+            }
+        }
+
+        return td;
+    }
+
+    return null;
+}
+
+fn pick_idle_thread(cpu: *PerCpu, migrate: bool) ?*ke.Thread {
+    std.debug.assert(cpu.queues_lock.is_locked());
+    if (cpu.idle_queue.is_empty()) return null;
+
+    // Just get the first thread from the queue.
+    const td: *ke.Thread = @fieldParentPtr("runq_link", cpu.idle_queue.first());
+
+    if (!migrate and td.pinned) {
+        // We can safely remove it.
+        td.runq_link.remove();
+    }
+
+    return td;
+}
+
+fn insert_in_list(list: *rtl.List, link: *rtl.List.Entry, head: bool) void {
+    if (head) list.insert_head(link) else list.insert_tail(link);
+}
+
+// Insert a thread into its respective queue.
+fn insert_in_queue(cpu: *PerCpu, td: *ke.Thread, head: bool) void {
+    std.debug.assert(td.lock.is_locked());
+    std.debug.assert(cpu.queues_lock.is_locked());
+
+    if (td.priority_class == .Realtime or td.interactive) {
+        // Insert in realtime queue.
+        cpu.realtime_queue.status |=
+            (@as(u64, 1) << @intCast(td.priority));
+
+        insert_in_list(&cpu.realtime_queue.queues[td.priority], &td.runq_link, head);
+        td.runq = &cpu.realtime_queue;
+        td.runq_idx = @intCast(td.priority);
+    } else if (td.priority_class == .Batch) {
+        // Insert in calendar queue.
+        // The insertion index is determined by insidx and the the priority of the thread,
+        // higher priority threads will be put closer to insidx, which ensures that they are ran more frequently.
+        const idx = (cpu.insidx + (ke.Thread.Priority.high_batch - td.priority)) % runqueues_n;
+
+        td.runq_idx = @intCast(idx);
+        cpu.calendar_queue.status |= (@as(u64, 1) << @intCast(idx));
+
+        insert_in_list(&cpu.calendar_queue.queues[idx], &td.runq_link, head);
+        td.runq = &cpu.calendar_queue;
+    } else {
+        // Insert in idle queue.
+        insert_in_list(&cpu.idle_queue, &td.runq_link, head);
+        td.runq = null;
+        td.runq_idx = 0;
+    }
+
+    td.state = .Ready;
+    _ = cpu.load.fetchAdd(1, .monotonic);
+}
+
+// Remove a thread from a queue.
+fn remove_from_queue(cpu: *PerCpu, td: *ke.Thread) void {
+    std.debug.assert(td.lock.is_locked());
+    std.debug.assert(cpu.queues_lock.is_locked());
+
+    // Decrease load.
+    cpu.load.fetchSub(1, .monotonic);
+
+    // Remove the thread from its queue.
+    td.runq_link.remove();
+
+    if (td.runq) |rq| {
+        if (rq.queues[td.runq_idx].is_empty()) {
+            // Clear the bit.
+            rq.status &= ~(@as(u64, 1) << @intCast(td.runq_idx));
+        }
+    }
+}
+
+// Update recent history for a thread.
+fn clamp_time(td: *ke.Thread) void {
+    const max = 5 * std.time.ms_per_s; // 5 seconds
+    const sum = td.run_time + td.sleep_time;
+
+    if (sum < max) return;
+
+    if (sum > max * 2) {
+        // History is way out of range (>10s), hard reset.
+        // Preserve the dominant side to avoid flipping interactivity classification.
+        if (td.run_time > td.sleep_time) {
+            td.run_time = max;
+            td.sleep_time = 1;
+        } else {
+            td.sleep_time = max;
+            td.run_time = 1;
+        }
+
+        return;
+    }
+
+    if (sum > ((max / 5) * 6)) {
+        // History is moderately out of range (>6s), halve both values.
+        // Keeps the sleep/run ratio intact while pulling the sum back in range.
+        td.run_time /= 2;
+        td.sleep_time /= 2;
+        return;
+    }
+
+    // History is slightly out of range (5s-6s), gentle 20% decay.
+    // Gradually ages out old history without disturbing the ratio.
+    td.run_time = (td.run_time / 5) * 4;
+    td.sleep_time = (td.sleep_time / 5) * 4;
+}
+
+// Compute the interactivity score for a thread.
+fn interactive_score(td: *ke.Thread) usize {
+    var div: usize = undefined;
+    var score: usize = undefined;
+
+    // Calculate the interactivity penalty.
+    if (td.sleep_time > td.run_time) {
+        div = @max(1, td.sleep_time / scaling_factor);
+        score = td.run_time / div;
+    } else if (td.run_time > td.sleep_time) {
+        div = @max(1, td.run_time / scaling_factor);
+        score = (scaling_factor + scaling_factor - (td.sleep_time / div));
+    } else if (td.run_time != 0) {
+        score = scaling_factor;
+    } else return 0;
+
+    // Add niceness values to the penalty, this makes it easier for threads with
+    // lower nice values (higher priority) to be considered interactive.
+    var signed_score: isize = @intCast(score);
+    signed_score += td.nice;
+    score = @intCast(signed_score);
+
+    return score;
+}
+
+// Recompute the priority for a thread.
+fn recompute_priority(td: *ke.Thread) void {
+
+    // Priority is computed only for time-shared (batch) thread based on
+    // interactivity and nice. If the thread is determined interactive, it is
+    // effectively promoted to real-time with a lower priority than actual
+    // real-time threads. If the thread is determined non-interactive,
+    // priority is calculated based on recent CPU usage and nice.
+    const score = interactive_score(td);
+
+    if (score < interactivity_threshold) {
+        // Choose a priority based on score, the lower the score,
+        // the higher the priority it will be.
+        // This is a simple formula I came up with that is probably good enough.
+        td.priority = @intCast(ke.Thread.Priority.interactive + ((interactivity_threshold - score) / 4));
+        td.interactive = true;
+
+        return;
+    }
+
+    // Thread is not interactive, compute priority from recent CPU usage.
+    // cpu_pri_off is the fraction of the history window spent running,
+    // scaled to [0, batch_range). A thread that spent all its time running
+    // gets cpu_pri_off = batch_range - 1 (lowest priority).
+    // A thread that barely ran gets cpu_pri_off = 0 (highest batch priority).
+    // This is a pretty straightforward formula, which /should/ be good enough.
+    const cpu_range = ke.Thread.Priority.batch_range;
+    const window = if (td.run_time + td.sleep_time == 0) @as(u64, 1) else td.run_time + td.sleep_time;
+    const cpu_pri_off: u8 = @intCast((td.run_time * (cpu_range - 1)) / window);
+
+    // nice offset: negative nice increases priority value, positive decreases it.
+    // Clamped to half the range to avoid pushing priority out of bounds before clamping.
+    const nice_off = std.math.clamp(
+        -@as(i32, td.nice),
+        -@as(i32, cpu_range / 2),
+        @as(i32, cpu_range / 2),
+    );
+
+    const raw = @as(i32, ke.Thread.Priority.low_batch) +
+        @as(i32, cpu_pri_off) +
+        nice_off;
+
+    td.priority = @intCast(std.math.clamp(
+        raw,
+        ke.Thread.Priority.low_batch,
+        ke.Thread.Priority.high_batch,
+    ));
+    td.interactive = false;
+}
+
+// Return whether or not a thread with given priority and interactivity should get preempted by `td`.
+fn should_prio_preempt(td: *ke.Thread, other_prio: u8, other_interactive: bool) bool {
+    const class = ke.Thread.Priority.class_from_prio(other_prio);
+
+    if (class == .Idle) {
+        return true;
+    }
+
+    if (class == .Batch and !other_interactive and td.interactive) {
+        // Interactive threads always preempt batch non-interactive ones.
+        return true;
+    }
+
+    // Preempt if the priority exceeds the preemption threshold (i.e the thread is real-time)
+    // or if the thread is interactive.
+    // Interactive threads may still preempt each other based on interactivity.
+    if ((td.priority >= preempt_threshold or td.interactive) and td.priority > other_prio) {
+        return true;
+    }
+
+    return false;
+}
+
+// Return whether or not `td` should preempt `cur`.
+// This is called with the current CPU's queue lock held.
+fn should_preempt(td: *ke.Thread, cur: *ke.Thread) bool {
+    return should_prio_preempt(td, cur.priority, cur.interactive);
+}
+
+// Return whether or not `td` should preempt the thread running on `cpu`.
+// This is done in a lock-free manner.
+fn should_preempt_cpu(td: *ke.Thread, cpu: *ke.Cpu) bool {
+    const status = cpu.current_thread_status.load(.monotonic);
+    return should_prio_preempt(td, status.priority, status.interactive);
+}
+
+// Enqueue a thread on a CPU.
+fn enqueue_on_cpu(cpu: *ke.Cpu, td: *ke.Thread) void {
+    var c = &cpu.sched_data;
+
+    c.queues_lock.acquire_no_ipl();
+
+    defer c.queues_lock.release_no_ipl();
+
+    if (cpu.current_thread == null or should_preempt(td, cpu.current_thread.?)) {
+        // We can preempt the current thread, first check whether or not there
+        // is already a next thread selected, and if we can preempt it.
+
+        const next = cpu.next_thread;
+        var can_preempt = true;
+
+        if (next) |n| {
+            if (should_preempt(td, n)) {
+                // This thread can preempt the next thread,
+                // put the next thread back into a queue as the first element.
+                insert_in_queue(c, n, true);
+            } else {
+                // The next thread is higher priority, we can't preempt it.
+                can_preempt = false;
+            }
+        }
+
+        if (can_preempt) {
+            // Preempt the CPU.
+            cpu.next_thread = td;
+            cpu.preemption_reason = .HigherPriority;
+
+            ki.ipl.set_softint_pending(cpu, .Dispatch);
+
+            if (cpu != ke.curcpu()) {
+                // Send an IPI.
+                ki.impl.send_resched_ipi(cpu);
+            }
+
+            // queues_lock dropped
+            return;
+        }
+    }
+
+    // Normal enqueue path.
+    // Add the thread to its appropriate queue.
+    insert_in_queue(c, td, false);
+
+    if (cpu.resched_timer.state.load(.monotonic) == .Stopped) {
+        cpu.start_timer = true;
+        ki.ipl.set_softint_pending(cpu, .Dispatch);
+    }
+
+    // queues_lock dropped
+}
+
+// Find a suitable CPU to run the thread.
+fn pick_cpu(td: *ke.Thread) *ke.Cpu {
+    // CPU selection policy in order:
+    // 1. Pick the last CPU the thread ran on if it would run immediately or if it's pinned.
+    // 2. Pick the least loaded CPU on which the thread can run immediately.
+    // 3. Pick the least loaded CPU overall.
+
+    if (td.last_cpu) |cpu| {
+        if (should_preempt_cpu(td, cpu) or td.pinned) return cpu;
+    }
+
+    // Check all CPUs, keeping track of the least loaded one overall and the least loaded preemptible one.
+    var least: ?*ke.Cpu = null;
+    var least_preempt: ?*ke.Cpu = null;
+
+    for (0..ke.ncpus) |i| {
+        const cpu = ke.cpus[i];
+
+        if (least == null or cpu.sched_data.load.load(.monotonic) < least.?.sched_data.load.load(.monotonic)) {
+            least = cpu;
+        }
+
+        if (should_preempt_cpu(td, cpu)) {
+            if (least_preempt == null or cpu.sched_data.load.load(.monotonic) < least_preempt.?.sched_data.load.load(.monotonic)) {
+                least_preempt = cpu;
+            }
+        }
+    }
+
+    // If we found a preemptible CPU, pick that one. Otherwise pick the least loaded one.
+    return least_preempt orelse least.?;
+}
+
+// Find a thread to preempt `cur` with.
+fn select_thread(cur: ?*ke.Thread, cpu: *ke.Cpu, migrate: bool) ?*ke.Thread {
+    const sched_cpu = &cpu.sched_data;
+
+    // First try to pick from the realtime queues.
+    if (pick_realtime_thread(sched_cpu, if (cur) |c| c.priority else 0, migrate)) |td| {
+        _ = sched_cpu.load.fetchSub(1, .monotonic);
+        return td;
+    }
+
+    // Nothing lower can preempt this thread.
+    if (cur != null and (cur.?.priority_class == .Realtime or cur.?.interactive)) {
+        return null;
+    }
+
+    // Then try picking a batch thread.
+    if (pick_batch_thread(sched_cpu, migrate)) |td| {
+        _ = sched_cpu.load.fetchSub(1, .monotonic);
+        return td;
+    }
+
+    if (cur != null and cur.?.priority_class == .Idle) {
+        // Idle threads don't preempt each other
+        return null;
+    }
+
+    // Finally try picking from the idle queue.
+    if (pick_idle_thread(sched_cpu, migrate)) |td| {
+        _ = sched_cpu.load.fetchSub(1, .monotonic);
+        return td;
+    }
+
+    // Nothing to run.
+    return null;
+}
+
+// Execute a context switch on `cpu`.
+fn do_switch(cpu: *ke.Cpu, cur: *ke.Thread, next: *ke.Thread) void {
+    next.state = .Running;
+    cpu.current_thread = next;
+    next.last_cpu = cpu;
+
+    // Stash thread status.
+    cpu.current_thread_status.store(.{ .interactive = next.interactive, .priority = @intCast(next.priority) }, .monotonic);
+
+    // Do machine-dependent switch.
+    cur.context.switch_to(&next.context);
+}
+
+// Yield on `cpu`
+fn yield(cpu: *ke.Cpu) void {
+    const sched_cpu = &cpu.sched_data;
+
+    sched_cpu.queues_lock.acquire_no_ipl();
+
+    const cur = cpu.current_thread;
+    var next = cpu.next_thread;
+
+    if (next) {
+        cpu.next_thread = null;
+    } else {
+        // Pick a new thread to run.
+        next = select_thread(null, cpu, false);
+    }
+
+    if (next == null) {
+        // Nothing to run, go idle.
+        next = cpu.idle_thread;
+    }
+
+    sched_cpu.queues_lock.release_no_ipl();
+
+    // Switch into the thread.
+    do_switch(cpu, cur, next);
+
+    // cur lock dropped
+}

src/kern/spinlock.zig

@@ -0,0 +1,60 @@
+const std = @import("std");
+const b = @import("base");
+const ke = b.ke;
+
+/// Simple Spin lock implementation.
+pub const SpinLock = struct {
+    locked: u8,
+
+    pub fn init() SpinLock {
+        return .{
+            .locked = 0,
+        };
+    }
+
+    /// Acquire the lock at IPL `ipl`.
+    pub fn acquire_at(self: *SpinLock, ipl: ke.Ipl) ke.Ipl {
+        const old_ipl = ke.ipl.raise(ipl);
+
+        self.acquire_no_ipl();
+
+        return old_ipl;
+    }
+
+    /// Acquire the lock at IPL `.Dispatch`.
+    pub fn acquire(self: *SpinLock) ke.Ipl {
+        return self.acquire_at(.Dispatch);
+    }
+
+    /// Release the lock at IPL `ipl`.
+    pub fn release(self: *SpinLock, ipl: ke.Ipl) void {
+        ke.ipl.lower(ipl);
+        self.release_no_ipl();
+    }
+
+    /// Acquire the lock without changing the IPL.
+    pub fn acquire_no_ipl(self: *SpinLock) void {
+        while (true) {
+            if (@cmpxchgWeak(u8, &self.locked, 0, 1, .acquire, .monotonic) == null)
+                return;
+
+            while (@atomicLoad(u8, &self.locked, .monotonic) != 0) {
+                std.atomic.spinLoopHint();
+            }
+        }
+    }
+
+    /// Release the lock without changing the IPL.
+    pub fn release_no_ipl(self: *SpinLock) void {
+        @atomicStore(u8, &self.locked, 0, .release);
+    }
+
+    /// Try to acquire the lock. Return true if lock was acquired.
+    pub fn try_acquire(self: *SpinLock) bool {
+        return @cmpxchgStrong(u8, &self.locked, 0, 1, .acquire, .monotonic) == null;
+    }
+
+    pub fn is_locked(self: *SpinLock) bool {
+        return @atomicLoad(u8, &self.locked, .monotonic) == 1;
+    }
+};

src/kern/thread.zig

@@ -0,0 +1,128 @@
+const std = @import("std");
+const b = @import("base");
+const rtl = @import("rtl");
+
+const ke = b.ke;
+const ki = ke.private;
+
+/// Structure representing a kernel thread.
+pub const Thread = struct {
+    /// Thread priority classes.
+    /// Realtime: Highest priority, aggressively scheduled.
+    /// Batch: Normal priority.
+    /// Idle: For idle and background threads.
+    pub const Priority = struct {
+        pub const Class = enum(u8) {
+            Realtime,
+            Batch,
+            Idle,
+        };
+
+        pub const idle = 0;
+        pub const low_batch = 1;
+        pub const high_batch = 32;
+        pub const batch_range = high_batch - low_batch + 1;
+        pub const realtime = 33;
+        pub const max = 63;
+        /// In the mid-range of batch threads
+        pub const default = 16;
+        /// Interactive threads have a slightly higher priority
+        pub const interactive = 20;
+
+        pub fn class_from_prio(prio: u8) Class {
+            if (prio >= realtime and prio <= max) return .Realtime;
+            if (prio >= low_batch and prio <= high_batch) return .Batch;
+            if (prio == idle) return .Idle;
+
+            unreachable;
+        }
+    };
+
+    /// Thread state.
+    pub const State = enum(u8) {
+        /// The thread is ready to run.
+        Ready,
+        /// The thread is currently running.
+        Running,
+        /// The thread is currently sleeping.
+        Blocked,
+        /// The thread has exited and is waiting to be reaped.
+        Zombie,
+        /// The thread has exited.
+        Terminated,
+    };
+    /// Implementation-dependent context.
+    context: ki.impl.ThreadContext,
+    /// Thread lock.
+    lock: ke.SpinLock,
+    /// Niceness value.
+    nice: i8,
+    /// Priority value of the thread.
+    priority: u8,
+    /// Priority class of the thread.
+    priority_class: Priority.Class,
+    /// When the thread started sleeping.
+    sleep_start: u64,
+    /// Ticks spent voluntarily sleeping recently.
+    sleep_time: u64,
+    /// Ticks spent running recently.
+    run_time: u64,
+    /// Whether the thread is interactive or not.
+    interactive: bool,
+    /// Whether the thread is pinned to this CPU.
+    /// If it is pinned, then it can't be moved across another CPU.
+    pinned: bool,
+    /// Current state of the thread.
+    state: State,
+    /// Linkage into a scheduler run queue.
+    runq_link: rtl.List.Entry,
+    /// Last CPU this thread ran on.
+    last_cpu: ?*ke.Cpu,
+
+    /// Run queue this thread is in
+    runq: ?*ki.sched.RunQueue,
+    /// Index into the run queue this thread is currently in.
+    /// Only valid when `runq` is non-null.
+    runq_idx: u8,
+    /// Current wait status.
+    wait_status: std.atomic.Value(ki.wait.Status),
+    waitblocks: [4]ki.wait.WaitBlock,
+
+    /// Timer used for timeouts.
+    timer: ke.Timer,
+
+    /// Initialize a thread.
+    /// - `stack`: Address of the **base** of the stack on which the initial context for the thread is built
+    /// - `stack_size`: Size of the stack
+    /// - `entry`: Entry point of the thread
+    /// - `arg`: Extraneous argument to be passed to `entry`
+    pub fn init(thread: *Thread, stack: b.VAddr, stack_size: usize, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) void {
+        thread.* = .{
+            .context = .init(stack, stack_size, entry, arg),
+            .lock = ke.SpinLock.init(),
+            .nice = 0,
+            .priority = Thread.Priority.default,
+            .priority_class = Thread.Priority.Class.Batch,
+            .sleep_start = 0,
+            .sleep_time = 0,
+            .run_time = 0,
+            .interactive = false,
+            .pinned = false,
+            .state = .Ready,
+            .runq_link = .{},
+            .last_cpu = null,
+            .runq = null,
+            .runq_idx = 0,
+            .wait_status = .init(.InProgress),
+            .waitblocks = undefined,
+            .timer = undefined,
+        };
+
+        thread.timer.init();
+    }
+};
+
+comptime {
+    if (!@hasDecl(ki.impl, "ThreadContext")) @compileError("impl must provide ThreadContext");
+    if (!@hasDecl(ki.impl.ThreadContext, "init")) @compileError("ThreadContext must have init()");
+}

src/kern/timecounter.zig

@@ -0,0 +1,143 @@
+const std = @import("std");
+const rtl = @import("rtl");
+const b = @import("base");
+const ke = b.ke;
+const ki = ke.private;
+
+/// TimeCounter structure.
+/// Represents a time-keeping source upon which the kernel can rely.
+pub const TimeCounter = struct {
+    /// Callback that reads the current count.
+    read_count: *const fn () u64,
+    /// Frequency of the timer in Hz.
+    frequency: u64,
+    /// Name of the timer.
+    name: []const u8,
+    /// Quality of the timer.
+    /// Used to choose between two TimeCounters.
+    quality: i16,
+    /// Mask to apply to the counter value.
+    mask: u64,
+    /// Binary scaling exponent for fast ns conversion (0 if unused).
+    /// See `ticks_to_ns`.
+    p: u64,
+    /// Scaled nanoseconds-per-tick multiplier for fast ns conversion (0 if unused).
+    /// See `ticks_to_ns`.
+    n: u64,
+};
+
+var best_tc: ?*TimeCounter = null;
+
+const TimeState = struct {
+    // Counter value at the time the current TimeCounter was selected.
+    // Used as the baseline for elapsed time calculation.
+    initial_count: u64,
+    // Accumulated nanosecond offset from previous TimeCounters and overflow events.
+    // Added to the current counter's contribution in `read_time_nano`.
+    offset: u64,
+};
+
+// SeqLock-protected time state.
+// Avoids a race on `initial_count` and `offset` with `update_overflow`.
+var state = rtl.SeqLock(TimeState).init(.{
+    .initial_count = 0,
+    .offset = 0,
+});
+
+// Convert a TimeCounter's count to nanoseconds.
+// For 64-bit counters (`mask == maxInt(u64)`), uses a fast multiply-shift path
+// via precomputed `p` and `n` to avoid 128-bit division at runtime.
+// Falls back to a division-based conversion for narrower counters.
+fn ticks_to_ns(tc: *TimeCounter, count: u64) u64 {
+    if (tc.p != 0) {
+        const val: u128 = @as(u128, @intCast(count)) * tc.n;
+        return @intCast(val >> @intCast(tc.p));
+    }
+
+    return (count * std.time.ns_per_s) / tc.frequency;
+}
+
+/// Register a time-keeping source.
+/// The TimeCounter will be used if it is the highest quality one available.
+/// This function cannot be safely called when the hardware timer has been started, as data races on `best_tc` are not avoided.
+pub fn register(tc: *TimeCounter) void {
+    // Precompute scaling factors for the fast multiply-shift path.
+    // This is meant to avoid 128-bit division and only use multiplication for 64-bit counters.
+    if (tc.mask == std.math.maxInt(u64)) {
+        tc.p = std.math.log2_int_ceil(u64, tc.frequency);
+        tc.n = (@as(u64, std.time.ns_per_s) << @intCast(tc.p)) / tc.frequency;
+    } else {
+        tc.p = 0;
+        tc.n = 0;
+    }
+
+    // If the best TimeCounter changed, pass the baton:
+    // snapshot the old counter's elapsed time into `offset` and
+    // reset `initial_count` to the new counter's current value.
+    if (best_tc == null or tc.quality > best_tc.?.quality) {
+        const old_best = best_tc;
+        best_tc = tc;
+
+        if (old_best) |old| {
+            const old_count = old.read_count() & old.mask;
+            state.raw.offset = ticks_to_ns(old, old_count);
+        }
+
+        state.raw.initial_count = tc.read_count() & tc.mask;
+    }
+}
+
+/// Return the time elapsed since boot in nanoseconds.
+pub fn read_time_nano() b.Nanoseconds {
+    const tc = best_tc orelse return 0;
+
+    const s = state.load();
+    const curr_count = tc.read_count() & tc.mask;
+    const elapsed = (curr_count - s.initial_count) & tc.mask;
+    return s.offset + ticks_to_ns(tc, elapsed);
+}
+
+/// Return the best TimeCounter.
+pub fn best() ?*TimeCounter {
+    return best_tc;
+}
+
+/// Do a busy sleep of `ns` nanoseconds using TimeCounter.
+pub fn sleep(ns: b.Nanoseconds) void {
+    const start = read_time_nano();
+
+    while (true) {
+        var cur = read_time_nano();
+
+        if (cur < start) continue;
+
+        if (cur - start >= ns) {
+            cur = read_time_nano();
+
+            // Check again as we may have overflowed and read a bogus value..
+            // This is kinda hacky but oh well.
+            if (cur - start >= ns) {
+                break;
+            }
+        }
+
+        std.atomic.spinLoopHint();
+    }
+}
+
+/// Called periodically to handle counter overflows.
+pub fn update_overflow() void {
+    const tc = best_tc orelse return;
+    const s = state.load();
+    const cur_count = tc.read_count() & tc.mask;
+
+    if (s.initial_count > cur_count) {
+        // Counter has wrapped, use wrapping arithmetic to figure out by how much,
+        // then fold it into the offset and reset `initial_count`.
+        const overflowed_by = (cur_count -% s.initial_count) & tc.mask;
+        state.store(.{
+            .initial_count = cur_count,
+            .offset = s.offset + ticks_to_ns(tc, overflowed_by),
+        });
+    }
+}

src/kern/timer.zig

@@ -0,0 +1,175 @@
+//! Timer object implementation.
+//! Timer objects are useful when one wants to wait a fixed amount of time for an event to occur.
+const std = @import("std");
+const rtl = @import("rtl");
+const b = @import("base");
+const ke = b.ke;
+const ki = ke.private;
+const pl = b.pl;
+
+pub const Timer = struct {
+    pub const State = enum(u8) {
+        /// The timer is currently being handled.
+        Running,
+        /// The timer is currently enqueued.
+        Pending,
+        /// The timer was stopped.
+        Stopped,
+    };
+
+    hdr: ki.wait.DispatchHeader,
+
+    /// Timer state.
+    state: std.atomic.Value(State),
+    /// When the timer is bound to expire.
+    deadline: b.Nanoseconds,
+    /// Attached DPC.
+    dpc: ?*ke.Dpc,
+    /// Intrusive pairing heap node.
+    node: rtl.pairing_heap.Node,
+    /// CPU this timer is enqueued on.
+    cpu: ?*ke.Cpu,
+
+    pub fn init(self: *Timer) void {
+        self.* = .{
+            .hdr = undefined,
+            .state = .init(.Stopped),
+            .deadline = 0,
+            .dpc = null,
+            .node = .{},
+            .cpu = null,
+        };
+
+        self.hdr.init(.Notification);
+    }
+};
+
+/// Start a timer with an expiration time.
+/// A DPC that will be enqueued upon expiration can be passed.
+pub fn set(timer: *Timer, time: b.Nanoseconds, dpc: ?*ke.Dpc) void {
+    const ipl = timer.hdr.lock.acquire();
+    defer timer.hdr.lock.release(ipl);
+
+    const cpu = ke.curcpu();
+
+    if (timer.state.cmpxchgStrong(.Stopped, .Pending, .acquire, .monotonic) != null) {
+        // Could not set the timer.
+        return;
+    }
+
+    cpu.timers_lock.acquire_no_ipl();
+    defer cpu.timers_lock.release_no_ipl();
+
+    // Initialize the timer.
+    timer.deadline = ke.timecounter.read_time_nano() + time;
+
+    timer.cpu = cpu;
+    timer.dpc = dpc;
+    timer.hdr.signaled = 0;
+
+    cpu.timers_heap.insert(&timer.node);
+
+    if (cpu.timers_heap.root == &timer.node) {
+        // This is the earliest timer to expire, arm the hardware timer.
+        pl.arm_timer(time);
+    }
+
+    // Locks dropped
+}
+
+/// Cancel a timer.
+pub fn cancel(timer: *Timer) void {
+    const old_state = timer.state.load(.monotonic);
+
+    if (old_state == .Stopped or old_state == .Running) {
+        // Already stopped or currently executing.
+        return;
+    }
+
+    const ipl = timer.hdr.lock.acquire();
+    defer timer.hdr.lock.release(ipl);
+
+    if (timer.state.cmpxchgStrong(.Pending, .Stopped, .acquire, .monotonic) != null) {
+        // Something changed, it probably expired.
+        return;
+    }
+
+    if (timer.cpu) |cpu| {
+        // Dequeue the timer.
+        cpu.timers_lock.acquire_no_ipl();
+
+        cpu.timers_heap.remove(&timer.node);
+        timer.cpu = null;
+
+        cpu.timers_lock.release_no_ipl();
+    }
+
+    // Lock dropped
+}
+
+/// Compare two timers.
+pub fn cmp_timer(a: *rtl.pairing_heap.Node, b_: *rtl.pairing_heap.Node) std.math.Order {
+    const timer_a: *Timer = @fieldParentPtr("node", a);
+    const timer_b: *Timer = @fieldParentPtr("node", b_);
+
+    return std.math.order(timer_a.deadline, timer_b.deadline);
+}
+
+/// Called in a DPC when a timer has expired.
+pub fn handle_expiry(_: ?*anyopaque) void {
+    const cpu = ke.curcpu();
+    std.debug.assert(cpu.ipl == .Dispatch);
+
+    while (true) {
+        const curtime = ke.timecounter.read_time_nano();
+
+        cpu.timers_lock.acquire_no_ipl();
+
+        // Get the timer that expires the soonest.
+        const timer_node = cpu.timers_heap.root orelse {
+            cpu.timers_lock.release_no_ipl();
+            return;
+        };
+
+        const timer: *Timer = @fieldParentPtr("node", timer_node);
+
+        // If the timer expires more than 1ms in the future, consider it not yet due.
+        // Sub-millisecond differences are close enough to expire immediately.
+        if (timer.deadline > curtime and timer.deadline - curtime > std.time.ns_per_ms) {
+            const next = timer;
+            cpu.timers_lock.release_no_ipl();
+
+            // Re-check in case the timer expired while we were here.
+            const now = ke.timecounter.read_time_nano();
+            if (next.deadline <= now or next.deadline - now <= std.time.ns_per_ms) {
+                continue; // expired in the meantime, loop again
+            }
+
+            std.debug.assert(next.deadline > now);
+            pl.arm_timer(next.deadline - now);
+            return;
+        }
+
+        _ = cpu.timers_heap.pop();
+        cpu.timers_lock.release_no_ipl();
+
+        if (timer.state.cmpxchgStrong(.Pending, .Running, .acquire, .monotonic) != null) {
+            // Timer must've been canceled.
+            continue;
+        }
+
+        timer.hdr.lock.acquire_no_ipl();
+
+        // Set signaled.
+        timer.hdr.signaled = 1;
+        timer.cpu = null;
+
+        // Wake whomever was waiting on the timer.
+        ki.wait.satisfy_wait(&timer.hdr);
+
+        timer.state.store(.Stopped, .release);
+
+        if (timer.dpc) |dpc| ke.dpc.enqueue(dpc, null);
+        timer.hdr.lock.release_no_ipl();
+    }
+}

src/kern/um/impl.zig

@@ -0,0 +1,106 @@
+const std = @import("std");
+const b = @import("base");
+const c = b.pl.impl.c;
+const ke = b.ke;
+const ki = ke.private;
+
+pub const Cpu = struct { pthread: c.pthread_t };
+
+/// Return the current CPU.
+pub fn curcpu() *ke.Cpu {
+    return b.pl.impl.my_cpu;
+}
+
+fn set_signals(enabled: bool) bool {
+    var newmask: c.sigset_t = undefined;
+    var oldmask: c.sigset_t = undefined;
+
+    _ = c.sigfillset(&newmask);
+
+    const how = if (enabled) c.SIG_BLOCK else c.SIG_UNBLOCK;
+
+    if (c.pthread_sigmask(how, &newmask, &oldmask) != 0) {
+        ke.panic("pthread_sigmask failed", .{});
+    }
+
+    return std.mem.eql(u8, std.mem.asBytes(&newmask), std.mem.asBytes(&oldmask));
+}
+
+/// Set the hardware IPL.
+pub fn set_hardware_ipl(ipl: ke.Ipl) void {
+    _ = set_signals(@intFromEnum(ipl) > @intFromEnum(ke.Ipl.get_max_software()));
+}
+
+/// Disable interrupts and return the previous state.
+pub fn disable_interrupts() bool {
+    return set_signals(false);
+}
+
+/// Enable interrupts and return the previous state.
+pub fn enable_interrupts() bool {
+    return set_signals(true);
+}
+
+/// Restore an interrupt state.
+pub fn restore_interrupts(val: bool) void {
+    if (val) {
+        _ = enable_interrupts();
+    } else {
+        _ = disable_interrupts();
+    }
+}
+
+pub fn send_resched_ipi(cpu: *ke.Cpu) void {
+    _ = c.pthread_kill(cpu.impl.pthread, c.SIGUSR1);
+}
+
+pub const ThreadContext = struct {
+    ucontext: c.ucontext_t,
+
+    fn real_entry(entry_int: c_ulong, arg_int: c_ulong) callconv(.c) void {
+        const entry: *const fn (?*anyopaque) void = @ptrFromInt(entry_int);
+        const arg: ?*anyopaque = @ptrFromInt(arg_int);
+        ke.ipl.lower(.Zero);
+        entry(arg);
+    }
+
+    pub fn init(stack: b.VAddr, stack_size: usize, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) ThreadContext {
+        var new: ThreadContext = .{ .ucontext = undefined };
+
+        if (c.getcontext(&new.ucontext) == -1) {
+            ke.panic("getcontext() failed", .{});
+        }
+
+        new.ucontext.uc_stack.ss_sp = @ptrFromInt(stack);
+        new.ucontext.uc_stack.ss_size = stack_size;
+
+        // No successor
+        new.ucontext.uc_link = null;
+
+        const entry_int: usize = @intFromPtr(entry);
+        const arg_int: usize = @intFromPtr(arg);
+
+        c.makecontext(
+            &new.ucontext,
+            @ptrCast(&real_entry),
+            2,
+            @as(c_ulong, entry_int),
+            @as(c_ulong, arg_int),
+        );
+        return new;
+    }
+
+    pub fn load(self: *ThreadContext) void {
+        _ = c.setcontext(&self.ucontext);
+    }
+
+    pub fn switch_to(self: *ThreadContext, new: *ThreadContext) void {
+        const thread: *ke.Thread = @fieldParentPtr("context", self);
+
+        thread.lock.release_no_ipl();
+
+        if (c.swapcontext(&self.ucontext, &new.ucontext) == -1) {
+            ke.panic("swapcontext() failed", .{});
+        }
+    }
+};

src/kern/wait.zig

@@ -0,0 +1,235 @@
+//! Implementation of generic wait mechanisms.
+//! This is mostly based on the work by Arun Kishan on Windows 7.
+//! See more here: https://youtu.be/OAAiOEQhsK0
+const rtl = @import("rtl");
+const std = @import("std");
+const b = @import("base");
+const ke = b.ke;
+
+/// Header for waitable objects.
+/// This must be added to any structured which is considered waitable.
+pub const DispatchHeader = struct {
+    pub const Kind = enum {
+        /// When signaled, `signaled` is kept high and all waiters are woken up.
+        Notification,
+        /// When signaled, `signaled` is decreased until 0, and a single waiter is woken up.
+        Synchronization,
+    };
+
+    /// List of WaitBlocks.
+    waitblocks: rtl.List,
+    /// Kind of object.
+    kind: Kind,
+    /// Lock protecting the object.
+    lock: ke.SpinLock,
+    /// Signaled count.
+    signaled: u32,
+
+    /// Initialize a waitable object.
+    pub fn init(obj: *DispatchHeader, kind: Kind) void {
+        obj.kind = kind;
+        obj.lock = ke.SpinLock.init();
+        obj.signaled = 0;
+        obj.waitblocks.init();
+    }
+
+    fn consume(self: *DispatchHeader) void {
+        switch (self.kind) {
+            .Synchronization => self.signaled -= 1,
+            .Notification => {
+                // Object remains signaled.
+            },
+        }
+    }
+};
+
+pub const WaitBlock = struct {
+    pub const Status = enum(u8) {
+        /// WaitBlock is linked to an object as part of a thread that is waiting.
+        Active,
+        /// The wait associated with this WaitBlock has been satisfied (or timed out).
+        Inactive,
+        /// A signal was delivered to the wait (this could be before it was committed).
+        Signaled,
+    };
+    /// List linkage.
+    link: rtl.List.Entry,
+    /// Object being waited on.
+    object: *DispatchHeader,
+    /// Thread that owns the WaitBlock.
+    thread: *ke.Thread,
+    /// Status of the WaitBlock.
+    status: WaitBlock.Status,
+};
+
+/// Status of a wait operation.
+pub const Status = enum(u8) {
+    /// Wait is currently being processed.
+    InProgress,
+    /// Wait has been committed and is waiting for the object to be signaled.
+    Committed,
+    /// Wait has been satisfied (or timed out).
+    Satisfied,
+};
+
+/// Wait for the provided object to be signaled.
+pub fn wait_one(object: *DispatchHeader, timeout: ?b.Nanoseconds) !usize {
+    return wait_any(object, timeout, null);
+}
+
+/// Wait for any of the provided objects to be signaled.
+/// Returns the index of the object that was signaled, or an error.
+/// If `timeout` is not provided, it will wait indefinitely.
+/// If `waitblocks` is specified, then the wait will use those waitblocks for the operation.
+/// Note that if `timeout` is provided, then one additional waitblock must be allocated.
+pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks: ?[*]WaitBlock) !usize {
+    const ipl = ke.ipl.raise(.Dispatch);
+    defer ke.ipl.lower(ipl);
+    const curtd = ke.curcpu().current_thread;
+    const obj_count = objects.len;
+    const has_timeout = timeout != null;
+    const blocks = waitblocks orelse blk: {
+        std.debug.assert(obj_count <= curtd.waitblocks.len - @intFromBool(has_timeout));
+        break :blk &curtd.waitblocks;
+    };
+
+    const total_count = obj_count + @intFromBool(has_timeout);
+    const timer_i = obj_count;
+    const timer = &curtd.timer;
+    const timer_block = &curtd.waitblocks[3];
+
+    var satisfier: ?usize = null;
+    curtd.wait_status.store(.InProgress, .acquire);
+
+    for (0..total_count) |i| {
+        const is_timer = has_timeout and i == timer_i;
+        var obj = if (is_timer) &timer.hdr else objects[i];
+        var wb = if (is_timer) timer_block else &blocks[i];
+
+        obj.lock.acquire_no_ipl();
+        defer obj.lock.release_no_ipl();
+
+        if (obj.signaled > 0) {
+            // Object was already signaled. Try consuming it.
+            if (curtd.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acquire, .monotonic) == null) {
+                obj.consume();
+            }
+            // We have already been satisfied in the meantime, abort.
+            satisfier = i;
+            break;
+        }
+
+        wb.object = obj;
+        wb.thread = curtd;
+        wb.status = .Active;
+        // We are not satisfied yet, so add a waitblock to the object's waitblocks
+        obj.waitblocks.insert_tail(&wb.link);
+    }
+
+    // Wait was already satisfied, back out.
+    if (satisfier != null or (has_timeout and timeout.? == 0)) {
+        std.debug.assert(curtd.wait_status.load(.monotonic) == .Satisfied);
+
+        // Remove any wait block we might've installed
+        for (0..satisfier) |i| {
+            const is_timer = has_timeout and i == timer_i;
+            var obj = if (is_timer) &timer.hdr else objects[i];
+            var wb = if (is_timer) timer_block else &blocks[i];
+
+            obj.lock.acquire_no_ipl();
+            wb.status = .Inactive;
+            wb.link.remove();
+            obj.lock.release_no_ipl();
+        }
+
+        return satisfier orelse error.Timeout;
+    }
+
+    if (has_timeout) {
+        ke.timer.set(timer, timeout.?, null);
+    }
+
+    // Now try committing the wait.
+    // While we're trying to commit the wait, the object locks have been released, and the state could therefore change.
+    // We need to re-check the state of the wait before actually blocking.
+    if (curtd.wait_status.cmpxchgStrong(.InProgress, .Committed, .acquire, .monotonic) == null) {
+        // We're good, now actually block.
+        ke.sched.block();
+    }
+
+    // We're back!
+    // Stop the timer if it was set.
+    if (has_timeout) {
+        ke.timer.cancel(timer);
+    }
+
+    // Find the object that satisfied us.
+    for (0..total_count) |i| {
+        const is_timer = has_timeout and i == timer_i;
+        var obj = if (is_timer) &timer.hdr else objects[i];
+        var wb = if (is_timer) timer_block else &blocks[i];
+
+        obj.lock.acquire_no_ipl();
+        if (wb.status == .Active) {
+            // Waitblock is still active, remove it from the list.
+            wb.link.remove();
+        }
+        if (wb.status == .Signaled or wb.status == .SignaledInProgress) {
+            // This waitblock was signaled, it must be the one that satisfied the wait.
+            std.debug.assert(satisfier == null);
+            satisfier = i;
+        }
+        // Ignore inactive waitblocks
+        obj.lock.release_no_ipl();
+    }
+
+    const final_sat = satisfier orelse return error.Timeout;
+    return if (has_timeout and final_sat == timer_i) error.Timeout else final_sat;
+}
+
+/// Satisfy a wait on an object.
+pub fn satisfy_wait(obj: *DispatchHeader) void {
+    const all = obj.kind == .Notification;
+
+    // Go through (potentially) all wait blocks and try to satisfy them.
+    while (!obj.waitblocks.is_empty() and obj.signaled > 0) {
+        // Get the first waitblock from the list.
+        var wb: *WaitBlock = @fieldParentPtr("link", obj.waitblocks.first());
+        var td = wb.thread;
+
+        // Remove it.
+        wb.link.remove();
+
+        // Three cases may occur here:
+        // 1. The wait was still preparing (status == .InProgress) and we interrupted it.
+        // 2. The wait was already committed (status == .Committed) and we satisfied it.
+        // 3. The wait was already satisfied by another object (status == .Satisfied).
+
+        // 1.
+        if (td.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acquire, .monotonic) == null) {
+            // We interrupted the wait while it was being prepared.
+            wb.status = .Signaled;
+            obj.consume();
+        }
+
+        // 2.
+        else if (td.wait_status.cmpxchgStrong(.Committed, .Satisfied, .acquire, .monotonic) == null) {
+            // We interrupted the wait while it was committed.
+            // Wake the thread.
+            wb.status = .Signaled;
+            obj.consume();
+            ke.sched.unblock(td);
+        }
+
+        // 3.
+        else if (td.wait_status.load(.acquire) == .Satisfied) {
+            // Someone else satisfied the wait, deactivate the wait block.
+            wb.status = .Inactive;
+            continue;
+        }
+
+        if (!all) {
+            break;
+        }
+    }
+}

src/limine.zig

@@ -0,0 +1,70 @@
+pub const common_magic: [2]u64 = .{
+    0xc7b1dd30df4c8b88,
+    0x0a82e883a194f07b,
+};
+
+pub const requests_start_marker: [4]u64 = .{
+    0xf6b8f4b39de7d1ae,
+    0xfab91a6940fcb9cf,
+    0x785c6ed015d3e316,
+    0x181e920a7852b9d9,
+};
+
+pub const requests_end_marker: [2]u64 = .{
+    0xadc0e0531bb10d03,
+    0x9572709f31764c62,
+};
+
+pub fn baseRevision(rev: u64) [3]u64 {
+    return .{
+        0xf9562b2d5c95a6c8,
+        0x6a7b384944536bdc,
+        rev,
+    };
+}
+
+pub fn baseRevisionSupported(base_rev: *const [3]u64) bool {
+    // Limine will overwrite base_rev[2] with 0 if it supports the requested revision.
+    return base_rev[2] == 0;
+}
+
+pub const framebuffer_request_id: [4]u64 = .{
+    common_magic[0],
+    common_magic[1],
+    0x9d5827dcd881dd75,
+    0xa3148604f6fab11b,
+};
+
+pub const FramebufferRequest = extern struct {
+    id: [4]u64,
+    revision: u64,
+    response: ?*FramebufferResponse,
+};
+
+pub const FramebufferResponse = extern struct {
+    revision: u64,
+    framebuffer_count: u64,
+    framebuffers: ?[*]*Framebuffer,
+};
+
+pub const Framebuffer = extern struct {
+    address: ?*anyopaque,
+    width: u64,
+    height: u64,
+    pitch: u64,
+    bpp: u16,
+    memory_model: u8,
+    red_mask_size: u8,
+    red_mask_shift: u8,
+    green_mask_size: u8,
+    green_mask_shift: u8,
+    blue_mask_size: u8,
+    blue_mask_shift: u8,
+    unused: [7]u8,
+    edid_size: u64,
+    edid: ?*anyopaque,
+    // Response revision 1 fields (ignored by this template).
+    mode_count: u64,
+    modes: ?[*]*anyopaque,
+};
+

src/main.zig

@@ -0,0 +1,57 @@
+const std = @import("std");
+const ksyms = @import("ksyms");
+const pl = @import("base").pl;
+
+const DebugWriter = struct {
+    pub const Error = error{};
+    pub const Writer = std.io.GenericWriter(*DebugWriter, Error, write);
+
+    fn write(self: *DebugWriter, bytes: []const u8) Error!usize {
+        _ = self;
+        for (bytes) |c| pl.debug_write(c);
+        return bytes.len;
+    }
+
+    fn writer(self: *DebugWriter) Writer {
+        return .{ .context = self };
+    }
+};
+
+var debug_writer = DebugWriter{};
+
+export fn kmain() callconv(.c) void {
+    print("hello from kernel!\n", .{});
+
+    for (ksyms.ksyms) |sym| {
+        print("  0x{x:0>16} {s}\n", .{ sym.addr, sym.name });
+    }
+
+    pl.first_init();
+
+    while (true) {
+        asm volatile ("hlt");
+    }
+}
+
+export fn memset(dest: [*]u8, c: u8, n: usize) [*]u8 {
+    for (dest[0..n]) |*b| b.* = c;
+    return dest;
+}
+
+export fn memcpy(dest: [*]u8, src: [*]const u8, n: usize) [*]u8 {
+    for (0..n) |i| dest[i] = src[i];
+    return dest;
+}
+
+export fn memmove(dest: [*]u8, src: [*]const u8, n: usize) [*]u8 {
+    if (@intFromPtr(dest) < @intFromPtr(src)) {
+        for (0..n) |i| dest[i] = src[i];
+    } else {
+        var i = n;
+        while (i > 0) {
+            i -= 1;
+            dest[i] = src[i];
+        }
+    }
+    return dest;
+}

src/platform/amd64/entry.zig

@@ -0,0 +1,3 @@
+const b = @import("base");
+
+pub fn first_init() linksection(b.init) void {}

src/platform/amd64/impl.zig

@@ -0,0 +1,16 @@
+const entry = @import("entry.zig");
+
+pub const first_init = entry.first_init;
+
+pub fn devices_init() void {}
+
+pub fn debug_write(c: u8) void {
+    // Write to debugcon
+    asm volatile ("outb %[c], %[port]"
+        :
+        : [c] "{al}" (c),
+          [port] "N{dx}" (@as(u16, 0xe9)),
+    );
+}
+
+pub fn arm_timer(_: u64) void {}

src/platform/root.zig

@@ -0,0 +1,75 @@
+//! Platform interface: each port must implement these functions and variables.
+//! To add a new platform, create a new file and implement all required functions.
+
+const config = @import("config");
+
+pub const impl = if (@hasDecl(config, "CONFIG_ARCH_AMD64"))
+    @import("amd64/impl.zig")
+else if (@hasDecl(config, "CONFIG_ARCH_UM"))
+    @import("um/impl.zig")
+else
+    @compileError("unsupported architecture");
+
+/// Early platform-specific initialization, called before anything else.
+pub inline fn early_init() void {
+    return impl.early_init();
+}
+
+/// Late platform init. Memory allocator is available at this point.
+/// SMP bringup must be done after this returns.
+pub inline fn late_init() void {
+    return impl.late_init();
+}
+
+/// Write a single character to the debug console.
+pub inline fn debug_write(c: u8) void {
+    return impl.debug_write(c);
+}
+
+/// Arm a one-shot timer to fire in `ns` nanoseconds.
+/// The timer must call `ki_timer_expiry()` when it fires.
+pub inline fn arm_timer(ns: u64) void {
+    return impl.arm_timer(ns);
+}
+
+/// Pretty name for the platform
+pub const name = impl.name;
+
+/// A generic struct representing information passed from the bootloader.
+pub const BootInfo = struct {
+    /// Represents the system memory map.
+    pub const MemMap = struct {
+        /// A memory map entry.
+        pub const Entry = struct {
+            pub const Kind = enum {
+                Free,
+                Reserved,
+                LoaderReclaimable,
+                AcpiNvs,
+                AcpiReclaimable,
+                Kernel,
+            };
+
+            kind: Kind,
+            base: usize,
+            size: usize,
+        };
+
+        entry_count: usize,
+        entries: [256]Entry,
+    };
+
+    pub const KernelAddress = struct { physical_base: usize, virtual_base: usize };
+
+    /// RSDP on ACPI machines
+    rsdp: usize,
+
+    /// Kernel command-line arguments
+    cmdline: ?[]const u8,
+
+    /// Memory map
+    memory_map: MemMap,
+
+    /// Kernel address
+    kernel_address: KernelAddress,
+};

src/platform/um/entry.zig

@@ -0,0 +1,81 @@
+const b = @import("base");
+
+pub const std_options = b.std_options;
+
+const std = @import("std");
+const ke = b.ke;
+const pl = b.pl;
+
+const default_mem_size = b.gib(1);
+
+fn parse_mem_size(str: []const u8) !usize {
+    if (str.len == 0) return default_mem_size;
+
+    const last = str[str.len - 1];
+
+    const mul: usize = switch (std.ascii.toLower(last)) {
+        'k' => b.kib(1),
+        'm' => b.mib(1),
+        'g' => b.gib(1),
+        else => 1,
+    };
+
+    const digits = if (mul == 1) str else str[0 .. str.len - 1];
+    const size = try std.fmt.parseInt(usize, digits, 10);
+    return size * mul;
+}
+
+pub fn main(init: std.process.Init) !void {
+    var args = try init.minimal.args.iterateAllocator(init.gpa);
+    defer args.deinit();
+
+    const exe = args.next().?;
+
+    var mem_size: usize = default_mem_size;
+    var cmdline: ?[]const u8 = null;
+    var ncpus: usize = 1;
+
+    while (args.next()) |arg| {
+        if (std.mem.eql(u8, arg, "-m") or std.mem.eql(u8, arg, "--mem-size")) {
+            const val = args.next() orelse fatal("--mem-size requires a value", .{});
+            mem_size = parse_mem_size(val) catch fatal("invalid mem-size", .{});
+        } else if (std.mem.eql(u8, arg, "-c") or std.mem.eql(u8, arg, "--cmdline")) {
+            cmdline = args.next() orelse fatal("--cmdline requires a value", .{});
+        } else if (std.mem.eql(u8, arg, "-t") or std.mem.eql(u8, arg, "--threads")) {
+            const val = args.next() orelse fatal("--threads requires a value", .{});
+            ncpus = std.fmt.parseInt(usize, val, 10) catch fatal("invalid thread count", .{});
+        } else if (std.mem.eql(u8, arg, "--help") or std.mem.eql(u8, arg, "-h")) {
+            var stderr = std.Io.File.stderr().writerStreaming(init.io, &.{});
+            var w = &stderr.interface;
+
+            try w.print(
+                \\Usage: {s} [options]
+                \\  -m, --mem-size SIZE    Set the memory size
+                \\  -c, --cmdline STR      Set the kernel cmdline
+                \\  -t, --threads COUNT    Set the number of threads
+                \\
+            , .{exe});
+            std.process.exit(0);
+        } else {
+            fatal("unknown argument: {s}", .{arg});
+        }
+    }
+
+    pl.impl.global_state.params =
+        .{ .ncpus = ncpus, .cmdline = cmdline, .mem_size = mem_size };
+
+    pl.impl.global_state.boot_info.kernel_address.physical_base = 0;
+    pl.impl.global_state.boot_info.kernel_address.virtual_base = 0;
+
+    pl.impl.global_state.boot_info.rsdp = 0;
+    pl.impl.global_state.boot_info.cmdline = cmdline;
+
+    ke.ncpus = ncpus;
+
+    ke.private.init(&pl.impl.global_state.boot_info);
+}
+
+fn fatal(comptime fmt: []const u8, args: anytype) noreturn {
+    std.debug.print(fmt ++ "\n", args);
+    std.process.exit(1);
+}

src/platform/um/impl.zig

@@ -0,0 +1,200 @@
+const std = @import("std");
+const linux = std.os.linux;
+const b = @import("base");
+const pl = b.pl;
+const ke = b.ke;
+const ki = b.ke.private;
+const posix = std.posix;
+
+const timer = @import("timer.zig");
+
+pub const c = @cImport({
+    @cDefine("_GNU_SOURCE", "1");
+    @cInclude("pthread.h");
+    @cInclude("sys/mman.h");
+    @cInclude("ucontext.h");
+    @cInclude("signal.h");
+    @cInclude("unistd.h");
+    @cInclude("sys/time.h");
+    @cInclude("sched.h");
+});
+
+pub const name = "User-Mode";
+
+pub const GlobalState = struct {
+    const Params = struct {
+        mem_size: usize,
+        cmdline: ?[]const u8,
+        ncpus: usize,
+    };
+
+    params: Params,
+    boot_info: pl.BootInfo,
+    phys_memory_memfd: i32,
+    phys_base: usize,
+};
+
+pub var global_state: GlobalState = undefined;
+pub threadlocal var my_cpu: *ke.Cpu = undefined;
+
+var cpu_id: u32 = 0;
+
+pub fn devices_init() void {}
+
+pub fn debug_write(char: u8) void {
+    std.debug.print("{c}", .{char});
+}
+
+pub const arm_timer = timer.arm_timer;
+
+fn check_function(fn_name: []const u8, ret: usize) void {
+    const real_ret: i64 = @intCast(ret);
+
+    if (real_ret < 0) {
+        const errno_val = linux.errno(ret);
+
+        std.debug.print("error: {s}: {s}\n", .{ fn_name, @tagName(errno_val) });
+        std.process.exit(1);
+    }
+}
+
+pub fn early_init() linksection(b.init) void {
+    // Set current CPU to the bootstrap CPU
+    my_cpu = &ki.bootstrap_cpu;
+
+    my_cpu.impl.pthread = c.pthread_self();
+
+    // Allocate our ID
+    my_cpu.id = 0;
+    cpu_id += 1;
+
+    const memfd: i32 = @intCast(linux.memfd_create("phys_memory", 0));
+
+    check_function("ftruncate", linux.ftruncate(memfd, @intCast(global_state.params.mem_size)));
+
+    const phys_base = linux.mmap(null, global_state.params.mem_size, .{ .READ = true, .WRITE = true }, .{ .ANONYMOUS = true, .TYPE = .SHARED }, memfd, 0);
+
+    check_function("mmap", phys_base);
+
+    // Build memory map
+    global_state.boot_info.memory_map.entry_count = 1;
+    global_state.boot_info.memory_map.entries[0].base = phys_base;
+    global_state.boot_info.memory_map.entries[0].size = global_state.params.mem_size;
+    global_state.boot_info.memory_map.entries[0].kind = .Free;
+
+    global_state.phys_memory_memfd = memfd;
+    global_state.phys_base = phys_base;
+
+    const pfndb_base = linux.mmap(null, b.tib(1), .{}, .{ .ANONYMOUS = true, .TYPE = .PRIVATE }, -1, 0);
+
+    check_function("mmap", pfndb_base);
+
+    const heap_base = linux.mmap(null, b.gib(8), .{}, .{ .ANONYMOUS = true, .TYPE = .PRIVATE }, -1, 0);
+
+    check_function("mmap", heap_base);
+
+    b.kernel_pfndb_base = pfndb_base;
+    b.kernel_heap_base = heap_base;
+
+    std.log.info("um: kernel heap lives at {X}", .{heap_base});
+}
+
+fn sigusr1_handler(_: posix.SIG, _: *const posix.siginfo_t, _: ?*anyopaque) callconv(.c) void {
+    if (@intFromEnum(ke.curcpu().ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(ke.curcpu(), .Dispatch)) {
+        ki.dpc.dispatch(ke.curcpu());
+    }
+}
+
+fn make_thread(entry: *const fn (?*anyopaque) void, td: *ke.Thread) void {
+    const stack = std.heap.page_allocator.alloc(u8, 16384) catch @panic("wtf");
+
+    td.init(@intFromPtr(stack.ptr), 16384, entry, null);
+
+    td.priority = 0;
+    td.priority_class = .Idle;
+}
+
+var cpus_up = std.atomic.Value(usize).init(0);
+
+fn other_cpu_entry(arg: ?*anyopaque) callconv(.c) ?*anyopaque {
+    my_cpu = @ptrCast(@alignCast(arg.?));
+
+    my_cpu.id = @atomicRmw(u32, &cpu_id, .Add, 1, .monotonic);
+    my_cpu.impl.pthread = c.pthread_self();
+
+    my_cpu.init(my_cpu.idle_thread.?);
+
+    timer.init_cpu();
+
+    _ = cpus_up.fetchAdd(1, .monotonic);
+
+    while (true) {}
+
+    return null;
+}
+
+fn idle(_: ?*anyopaque) void {}
+
+pub fn late_init() linksection(b.init) void {
+    timer.init();
+    timer.init_cpu();
+
+    var sa: posix.Sigaction = std.mem.zeroes(posix.Sigaction);
+    sa.handler = .{
+        .sigaction = sigusr1_handler,
+    };
+
+    sa.flags = posix.SA.SIGINFO;
+
+    sa.mask = posix.sigemptyset();
+
+    _ = posix.sigaction(posix.SIG.USR1, &sa, null);
+
+    std.log.info("bum: starting {} thread{s}", .{
+        ke.ncpus - 1,
+        if ((ke.ncpus - 1) > 1) "s" else "",
+    });
+
+    const other_count = ke.ncpus - 1;
+
+    const allocator = std.heap.page_allocator;
+
+    var other_threads =
+        allocator.alloc(c.pthread_t, other_count) catch @panic("oom");
+
+    ke.cpus = allocator.alloc(*ke.Cpu, ke.ncpus) catch @panic("oom");
+    ke.cpus[0] = &ki.bootstrap_cpu;
+
+    for (1..ke.ncpus) |i| {
+        ke.cpus[i] = allocator.create(ke.Cpu) catch @panic("oom");
+
+        ke.cpus[i].idle_thread = allocator.create(ke.Thread) catch @panic("oom");
+
+        make_thread(idle, ke.cpus[i].idle_thread.?);
+
+        var attr: c.pthread_attr_t = undefined;
+
+        _ = c.pthread_attr_init(&attr);
+
+        var cpuset: c.cpu_set_t = std.mem.zeroes(c.cpu_set_t);
+        cpuset.__bits[i / 64] |= @as(c_ulong, 1) << @intCast(i % 64);
+        _ = c.pthread_attr_setaffinity_np(&attr, @sizeOf(c.cpu_set_t), &cpuset);
+
+        const r = c.pthread_create(
+            &other_threads[i - 1],
+            &attr,
+            other_cpu_entry,
+            ke.cpus[i],
+        );
+
+        if (r != 0) {
+            @panic("pthread_create() failed");
+        }
+    }
+
+    if (ke.ncpus > 1) {
+        while (cpus_up.load(.monotonic) < @as(u32, @intCast(ke.ncpus - 1))) {
+            std.atomic.spinLoopHint();
+        }
+    }
+}

src/platform/um/timer.zig

@@ -0,0 +1,73 @@
+const b = @import("base");
+const std = @import("std");
+const posix = std.posix;
+const linux = std.os.linux;
+const c = b.pl.impl.c;
+const ke = b.ke;
+const ki = ke.private;
+
+threadlocal var timerid: c.timer_t = undefined;
+
+fn timer_handler(_: posix.SIG, _: *const posix.siginfo_t, _: ?*anyopaque) callconv(.c) void {
+    const old_ipl = ke.ipl.set_hardware(.Device);
+
+    ke.dpc.enqueue(&ke.curcpu().timer_dpc, null);
+
+    _ = ke.ipl.set_hardware(old_ipl);
+
+    if (@intFromEnum(old_ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(ke.curcpu(), .Dispatch)) {
+        ki.dpc.dispatch(ke.curcpu());
+    }
+}
+
+pub fn arm_timer(ns: b.Nanoseconds) void {
+    const value: c.struct_itimerspec = .{ .it_value = .{
+        .tv_sec = @intCast(ns / std.time.ns_per_s),
+        .tv_nsec = @intCast(ns % std.time.ns_per_s),
+    }, .it_interval = .{ .tv_sec = 0, .tv_nsec = 0 } };
+
+    if (c.timer_settime(timerid, 0, &value, null) == -1) {
+        ke.panic("timer_settime failed", .{});
+    }
+}
+
+fn tc_read() u64 {
+    var ts: std.c.timespec = undefined;
+
+    _ = linux.clock_gettime(linux.CLOCK.MONOTONIC, &ts);
+
+    return @intCast(ts.sec * std.time.ns_per_s + ts.nsec);
+}
+
+var tc: ke.TimeCounter = .{
+    .read_count = tc_read,
+    .frequency = std.time.ns_per_s,
+    .name = "CLOCK_MONOTONIC",
+    .quality = 1000,
+    .mask = std.math.maxInt(u64),
+    .p = 0,
+    .n = 0,
+};
+
+pub fn init() void {
+    var act: posix.Sigaction = undefined;
+
+    act.handler.sigaction = timer_handler;
+    act.flags = posix.SA.RESTART | posix.SA.SIGINFO;
+
+    posix.sigaction(posix.SIG.ALRM, &act, null);
+
+    ke.timecounter.register(&tc);
+}
+
+pub fn init_cpu() void {
+    var sev: c.sigevent = std.mem.zeroes(c.sigevent);
+
+    sev.sigev_notify = c.SIGEV_THREAD_ID;
+    sev.sigev_signo = c.SIGALRM;
+    sev._sigev_un._tid = linux.gettid();
+
+    if (c.timer_create(c.CLOCK_MONOTONIC, &sev, &timerid) == -1) {
+        ke.panic("timer_create failed", .{});
+    }
+}

src/root.zig

@@ -0,0 +1,38 @@
+const std = @import("std");
+pub const ke = @import("kern/root.zig");
+pub const pl = @import("platform/root.zig");
+pub const rtl = @import("rtl");
+
+pub const init = ".init.text";
+
+pub const VAddr = usize;
+pub const PAddr = usize;
+
+pub const Nanoseconds = u64;
+
+pub var kernel_heap_base: usize = 0;
+pub var kernel_pfndb_base: usize = 0;
+
+/// Return N kibibytes in bytes.
+pub fn kib(comptime N: u32) usize {
+    return N * 1024;
+}
+
+/// Return N mebibytes in bytes.
+pub fn mib(comptime N: u32) usize {
+    return kib(N) * 1024;
+}
+
+/// Return N gibibytes in bytes.
+pub fn gib(comptime N: u32) usize {
+    return mib(N) * 1024;
+}
+
+/// Return N tibibytes in bytes.
+pub fn tib(comptime N: u32) usize {
+    return gib(N) * 1024;
+}
+
+pub const std_options = std.Options{
+    .logFn = ke.log,
+};

src/rtl/barrier.zig

@@ -0,0 +1,57 @@
+//! Atomic barrier implementations on various architectures.
+//! This is needed since Zig removed @fence.
+
+const builtin = @import("builtin");
+
+/// Read Memory Barrier (Load-Load)
+/// Ensures all loads before the barrier complete before any loads after it.
+pub inline fn rmb() void {
+    switch (builtin.cpu.arch) {
+        .x86, .x86_64 => {
+            asm volatile ("" ::: .{ .memory = true });
+        },
+        .aarch64 => {
+            asm volatile ("dmb ishld" ::: .{ .memory = true });
+        },
+        .arm => {
+            asm volatile ("dmb ish" ::: .{ .memory = true });
+        },
+        .riscv64, .riscv32 => {
+            asm volatile ("fence r,r" ::: .{ .memory = true });
+        },
+        else => @compileError("unsupported architecture"),
+    }
+}
+
+/// Write Memory Barrier (Store-Store)
+/// Ensures all stores before the barrier complete before any stores after it.
+pub inline fn wmb() void {
+    switch (builtin.cpu.arch) {
+        .x86, .x86_64 => {
+            asm volatile ("" ::: .{ .memory = true });
+        },
+        .aarch64, .arm => {
+            asm volatile ("dmb ish" ::: .{ .memory = true });
+        },
+        .riscv64, .riscv32 => {
+            asm volatile ("fence rw,w" ::: .{ .memory = true });
+        },
+        else => @compileError("unsupported architecture"),
+    }
+}
+
+/// Full Memory Barrier (Load/Store - Load/Store)
+pub inline fn mb() void {
+    switch (builtin.cpu.arch) {
+        .x86, .x86_64 => {
+            asm volatile ("mfence" ::: .{ .memory = true });
+        },
+        .aarch64, .arm => {
+            asm volatile ("dmb ish" ::: .{ .memory = true });
+        },
+        .riscv64, .riscv32 => {
+            asm volatile ("fence rw,rw" ::: .{ .memory = true });
+        },
+        else => @compileError("unsupported architecture"),
+    }
+}

src/rtl/list.zig

@@ -0,0 +1,204 @@
+//! Intrusive circular linked list module.
+/// Branchless intrusive circular linked list.
+/// Embed `List.Entry` in your struct and use `@fieldParentPtr` to recover it.
+pub const List = struct {
+    head: Entry,
+
+    /// A list entry
+    pub const Entry = struct {
+        /// Next element in the list
+        next: *Entry = undefined,
+        /// Previous element
+        prev: *Entry = undefined,
+
+        /// Removes an entry from whatever list it's in
+        pub fn remove(elem: *Entry) void {
+            elem.prev.next = elem.next;
+            elem.next.prev = elem.prev;
+        }
+
+        /// Inserts elem before `before`
+        pub fn insert_before(elem: *Entry, before: *Entry) void {
+            elem.prev = before.prev;
+            elem.next = before;
+            before.prev.next = elem;
+            before.prev = elem;
+        }
+    };
+
+    /// Initializes the list
+    pub fn init(self: *List) void {
+        self.head.next = &self.head;
+        self.head.prev = &self.head;
+    }
+
+    /// Returns whether the list is empty
+    pub fn is_empty(self: *List) bool {
+        return self.head.next == &self.head;
+    }
+
+    /// Inserts elem at the end of the list
+    pub fn insert_tail(self: *List, elem: *Entry) void {
+        self.head.prev.next = elem;
+        elem.next = &self.head;
+        elem.prev = self.head.prev;
+        self.head.prev = elem;
+    }
+
+    /// Inserts elem at the beginning of the list
+    pub fn insert_head(self: *List, elem: *Entry) void {
+        self.head.next.prev = elem;
+        elem.next = self.head.next;
+        elem.prev = &self.head;
+        self.head.next = elem;
+    }
+
+    /// Returns the first element of the list
+    pub fn first(self: *List) *Entry {
+        return self.head.next;
+    }
+
+    /// Returns the last element of the list
+    pub fn last(self: *List) *Entry {
+        return self.head.prev;
+    }
+
+    pub fn iterator(self: *List) Iterator {
+        return .{ .head = &self.head, .current = self.head.next };
+    }
+
+    pub const Iterator = struct {
+        head: *Entry,
+        current: *Entry,
+
+        pub fn next(self: *Iterator) bool {
+            return self.current != self.head;
+        }
+
+        pub fn get(self: *Iterator) *Entry {
+            return self.current;
+        }
+
+        pub fn advance(self: *Iterator) void {
+            self.current = self.current.next;
+        }
+    };
+};
+
+const std = @import("std");
+
+test "init creates empty list" {
+    var list: List = undefined;
+    list.init();
+    try std.testing.expect(list.is_empty());
+}
+
+test "insert_tail adds elements in order" {
+    var list: List = undefined;
+    list.init();
+
+    const Node = struct { value: u32, link: List.Entry = undefined };
+    var a = Node{ .value = 1 };
+    var b = Node{ .value = 2 };
+    var c = Node{ .value = 3 };
+
+    list.insert_tail(&a.link);
+    list.insert_tail(&b.link);
+    list.insert_tail(&c.link);
+
+    var it = list.iterator();
+    var i: u32 = 1;
+    while (it.next()) : (it.advance()) {
+        const node: *Node = @fieldParentPtr("link", it.get());
+        try std.testing.expectEqual(i, node.value);
+        i += 1;
+    }
+    try std.testing.expectEqual(@as(u32, 4), i);
+}
+
+test "insert_head adds elements in reverse order" {
+    var list: List = undefined;
+    list.init();
+
+    const Node = struct { value: u32, link: List.Entry = undefined };
+    var a = Node{ .value = 1 };
+    var b = Node{ .value = 2 };
+    var c = Node{ .value = 3 };
+
+    list.insert_head(&a.link);
+    list.insert_head(&b.link);
+    list.insert_head(&c.link);
+
+    var it = list.iterator();
+    var i: u32 = 3;
+    while (it.next()) : (it.advance()) {
+        const node: *Node = @fieldParentPtr("link", it.get());
+        try std.testing.expectEqual(i, node.value);
+        i -= 1;
+    }
+}
+
+test "remove detaches element" {
+    var list: List = undefined;
+    list.init();
+
+    const Node = struct { value: u32, link: List.Entry = undefined };
+    var a = Node{ .value = 1 };
+    var b = Node{ .value = 2 };
+    var c = Node{ .value = 3 };
+
+    list.insert_tail(&a.link);
+    list.insert_tail(&b.link);
+    list.insert_tail(&c.link);
+
+    b.link.remove();
+
+    var it = list.iterator();
+    const first: *Node = @fieldParentPtr("link", it.get());
+    try std.testing.expectEqual(@as(u32, 1), first.value);
+    it.advance();
+    const second: *Node = @fieldParentPtr("link", it.get());
+    try std.testing.expectEqual(@as(u32, 3), second.value);
+    it.advance();
+    try std.testing.expect(!it.next());
+}
+
+test "insert_before inserts in correct position" {
+    var list: List = undefined;
+    list.init();
+
+    const Node = struct { value: u32, link: List.Entry = undefined };
+    var a = Node{ .value = 1 };
+    var b = Node{ .value = 3 };
+    var mid = Node{ .value = 2 };
+
+    list.insert_tail(&a.link);
+    list.insert_tail(&b.link);
+
+    mid.link.insert_before(&b.link);
+
+    var it = list.iterator();
+    var i: u32 = 1;
+    while (it.next()) : (it.advance()) {
+        const node: *Node = @fieldParentPtr("link", it.get());
+        try std.testing.expectEqual(i, node.value);
+        i += 1;
+    }
+}
+
+test "first and last return correct elements" {
+    var list: List = undefined;
+    list.init();
+
+    const Node = struct { value: u32, link: List.Entry = undefined };
+    var a = Node{ .value = 1 };
+    var b = Node{ .value = 2 };
+
+    list.insert_tail(&a.link);
+    list.insert_tail(&b.link);
+
+    const first: *Node = @fieldParentPtr("link", list.first());
+    const last: *Node = @fieldParentPtr("link", list.last());
+    try std.testing.expectEqual(@as(u32, 1), first.value);
+    try std.testing.expectEqual(@as(u32, 2), last.value);
+}

src/rtl/pairing_heap.zig

@@ -0,0 +1,331 @@
+//! Intrusive Pairing Heap implementation.
+const std = @import("std");
+
+pub const Node = struct {
+    child: ?*Node = null,
+    next: ?*Node = null,
+    prev: ?*Node = null,
+};
+
+pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, *Node) std.math.Order) type {
+    return struct {
+        const Self = @This();
+
+        /// Number of elements in the heap
+        size: usize = 0,
+
+        /// Topmost element of the heap
+        root: ?*Node = null,
+
+        fn meld(a: *Node, b: *Node) *Node {
+            const a_wins = switch (order) {
+                .min => cmp(a, b) != .gt, // a <= b
+                .max => cmp(a, b) == .gt, // a > b
+            };
+
+            if (a_wins) {
+                if (a.child) |c| {
+                    c.prev = b;
+                }
+
+                b.next = a.child;
+                a.child = b;
+                b.prev = a;
+                return a;
+            }
+
+            if (b.child) |c| {
+                c.prev = a;
+            }
+
+            a.prev = b;
+            a.next = b.child;
+            b.child = a;
+            return b;
+        }
+
+        fn merge_pairs(node: *Node) *Node {
+            // First pass: pair up siblings left to right
+            var list: ?*Node = node;
+            var pairs: ?*Node = null;
+
+            while (list) |a| {
+                const b = a.next orelse {
+                    a.next = pairs;
+                    a.prev = null;
+                    pairs = a;
+                    break;
+                };
+                list = b.next;
+
+                a.next = null;
+                a.prev = null;
+                b.next = null;
+                b.prev = null;
+
+                const merged = meld(a, b);
+                merged.next = pairs;
+                merged.prev = null;
+                pairs = merged;
+            }
+
+            // Second pass: fold pairs right to left
+            var result = pairs.?;
+            var rest = result.next;
+            result.next = null;
+            while (rest) |p| {
+                rest = p.next;
+                p.next = null;
+                result = meld(result, p);
+            }
+            return result;
+        }
+
+        pub fn init() Self {
+            return .{
+                .size = 0,
+                .root = null,
+            };
+        }
+
+        /// Insert `node` into the heap.
+        pub fn insert(self: *Self, node: *Node) void {
+            node.* = .{
+                .child = null,
+                .next = null,
+                .prev = null,
+            };
+
+            self.size += 1;
+
+            if (self.root == null) {
+                self.root = node;
+                return;
+            }
+
+            self.root = meld(self.root.?, node);
+        }
+
+        /// Return and remove the topmost element.
+        pub fn pop(self: *Self) ?*Node {
+            const root = self.root orelse return null;
+            const child = root.child;
+
+            root.child = null;
+            self.size -= 1;
+
+            if (child) |c| {
+                c.prev = null;
+                self.root = merge_pairs(c);
+            } else {
+                self.root = null;
+            }
+
+            return root;
+        }
+
+        /// Remove `node` from the heap.
+        pub fn remove(self: *Self, node: *Node) void {
+            if (self.root == node) {
+                _ = self.pop();
+                return;
+            }
+
+            std.debug.assert(self.root != null);
+
+            self.size -= 1;
+
+            // Unlink the node
+            if (node.next) |n| {
+                n.prev = node.prev;
+            }
+
+            if (node.prev) |p| {
+                if (p.child == node) {
+                    // node was leftmost child
+                    p.child = node.next;
+                } else {
+                    // node was a sibling
+                    p.next = node.next;
+                }
+            }
+
+            node.prev = null;
+            node.next = null;
+
+            // Merge node's children back into the heap
+            if (node.child) |c| {
+                c.prev = null;
+                self.root = meld(self.root.?, merge_pairs(c));
+            }
+
+            node.child = null;
+        }
+    };
+}
+const TestNode = struct {
+    value: u32,
+    node: Node = .{},
+};
+
+fn my_cmp(a: *Node, b: *Node) std.math.Order {
+    const ta: *TestNode = @fieldParentPtr("node", a);
+    const tb: *TestNode = @fieldParentPtr("node", b);
+    return std.math.order(ta.value, tb.value);
+}
+
+const MinHeap = PairingHeap(.min, my_cmp);
+const MaxHeap = PairingHeap(.max, my_cmp);
+
+test "empty heap" {
+    var heap = MinHeap.init();
+    try std.testing.expect(heap.root == null);
+    try std.testing.expectEqual(0, heap.size);
+    try std.testing.expect(heap.pop() == null);
+}
+
+test "single element insert and pop" {
+    var heap = MinHeap.init();
+    var a = TestNode{ .value = 42 };
+
+    heap.insert(&a.node);
+    try std.testing.expectEqual(1, heap.size);
+
+    const top: *TestNode = @fieldParentPtr("node", heap.root.?);
+    try std.testing.expectEqual(42, top.value);
+
+    _ = heap.pop();
+    try std.testing.expect(heap.root == null);
+    try std.testing.expectEqual(0, heap.size);
+    try std.testing.expect(heap.pop() == null);
+}
+
+test "min heap ordering" {
+    var heap = MinHeap.init();
+    var a = TestNode{ .value = 3 };
+    var b = TestNode{ .value = 1 };
+    var c = TestNode{ .value = 2 };
+
+    heap.insert(&a.node);
+    heap.insert(&b.node);
+    heap.insert(&c.node);
+    try std.testing.expectEqual(3, heap.size);
+
+    const first: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(1, first.value);
+
+    const second: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(2, second.value);
+
+    const third: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(3, third.value);
+
+    try std.testing.expect(heap.pop() == null);
+}
+
+test "max heap ordering" {
+    var heap = MaxHeap.init();
+    var a = TestNode{ .value = 3 };
+    var b = TestNode{ .value = 1 };
+    var c = TestNode{ .value = 2 };
+
+    heap.insert(&a.node);
+    heap.insert(&b.node);
+    heap.insert(&c.node);
+
+    const first: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(3, first.value);
+
+    const second: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(2, second.value);
+
+    const third: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(1, third.value);
+}
+
+test "remove middle node" {
+    var heap = MinHeap.init();
+    var a = TestNode{ .value = 1 };
+    var b = TestNode{ .value = 2 };
+    var c = TestNode{ .value = 3 };
+
+    heap.insert(&a.node);
+    heap.insert(&b.node);
+    heap.insert(&c.node);
+
+    heap.remove(&b.node);
+    try std.testing.expectEqual(2, heap.size);
+
+    const first: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(1, first.value);
+
+    const second: *TestNode = @fieldParentPtr("node", heap.pop().?);
+    try std.testing.expectEqual(3, second.value);
+
+    try std.testing.expect(heap.pop() == null);
+}
+
+test "remove root" {
+    var heap = MinHeap.init();
+    var a = TestNode{ .value = 1 };
+    var b = TestNode{ .value = 2 };
+    var c = TestNode{ .value = 3 };
+
+    heap.insert(&a.node);
+    heap.insert(&b.node);
+    heap.insert(&c.node);
+
+    heap.remove(&a.node);
+    try std.testing.expectEqual(2, heap.size);
+
+    const top: *TestNode = @fieldParentPtr("node", heap.root.?);
+    try std.testing.expectEqual(2, top.value);
+}
+
+test "remove last element" {
+    var heap = MinHeap.init();
+    var a = TestNode{ .value = 1 };
+
+    heap.insert(&a.node);
+    heap.remove(&a.node);
+    try std.testing.expectEqual(0, heap.size);
+    try std.testing.expect(heap.root == null);
+}
+
+test "duplicate values" {
+    var heap = MinHeap.init();
+    var a = TestNode{ .value = 1 };
+    var b = TestNode{ .value = 1 };
+    var c = TestNode{ .value = 1 };
+
+    heap.insert(&a.node);
+    heap.insert(&b.node);
+    heap.insert(&c.node);
+    try std.testing.expectEqual(3, heap.size);
+
+    _ = heap.pop();
+    _ = heap.pop();
+    _ = heap.pop();
+    try std.testing.expectEqual(0, heap.size);
+    try std.testing.expect(heap.pop() == null);
+}
+
+test "large sequence" {
+    var heap = MinHeap.init();
+    var nodes: [16]TestNode = undefined;
+
+    for (&nodes, 0..) |*n, i| {
+        n.* = .{ .value = @intCast(16 - i) };
+        heap.insert(&n.node);
+    }
+
+    try std.testing.expectEqual(16, heap.size);
+
+    var prev: u32 = 0;
+    while (heap.pop()) |n| {
+        const t: *TestNode = @fieldParentPtr("node", n);
+        try std.testing.expect(t.value >= prev);
+        prev = t.value;
+    }
+    try std.testing.expectEqual(0, heap.size);
+}

src/rtl/root.zig

@@ -0,0 +1,11 @@
+const std = @import("std");
+
+pub const List = @import("list.zig").List;
+pub const SeqLock = @import("seqlock.zig").SeqLock;
+pub const barrier = @import("barrier.zig");
+pub const pairing_heap = @import("pairing_heap.zig");
+pub const PairingHeap = pairing_heap.PairingHeap;
+
+test {
+    std.testing.refAllDecls(@This());
+}

src/rtl/seqlock.zig

@@ -0,0 +1,52 @@
+const std = @import("std");
+const rtl = @import("rtl");
+
+pub fn SeqLock(comptime T: type) type {
+    return struct {
+        const Self = @This();
+
+        raw: T,
+        sequence: std.atomic.Value(usize),
+
+        pub fn init(val: T) Self {
+            return .{ .raw = val, .sequence = .init(0) };
+        }
+
+        pub fn load(self: *Self) T {
+            var ret: T = undefined;
+
+            while (true) {
+                const seq = self.sequence.load(.acquire);
+                if (seq & 1 != 0) {
+                    std.atomic.spinLoopHint();
+                    continue;
+                }
+
+                ret = self.raw;
+
+                // Ensure the sequence read happens *after* the data is fully loaded.
+                rtl.barrier.rmb();
+
+                if (self.sequence.load(.monotonic) == seq)
+                    break;
+
+                std.atomic.spinLoopHint();
+            }
+
+            return ret;
+        }
+
+        pub fn store(self: *Self, data: T) void {
+            const seq = self.sequence.load(.monotonic);
+
+            self.sequence.store(seq + 1, .monotonic);
+
+            // Ensure the data write happens *after* the sequence is incremented.
+            rtl.fence.wmb();
+
+            self.raw = data;
+
+            self.sequence.store(seq + 2, .release);
+        }
+    };
+}