feat: use per-cpu sections

build.zig

@@ -212,6 +212,7 @@ fn addKernel(b: *std.Build, plat: config.Platform, optimize: std.builtin.Optimiz
 
         kernel.linker_script = b.path(b.fmt("build/linker-scripts/{s}.lds", .{@tagName(plat.arch)}));
     } else {
+        kernel.linker_script = b.path("build/linker-scripts/uml.lds");
         kernel.root_module.link_libc = true;
     }
 

build/linker-scripts/uml.lds

@@ -0,0 +1,14 @@
+SECTIONS {
+    .data.percpu : {
+        __percpu_start = .;
+        *(.data.percpu)
+        __percpu_end = .;
+    }
+
+.percpu_init_array : {
+    __init_array_percpu_start = .;
+    KEEP(*(.percpu_init_array))
+    __init_array_percpu_end = .;
+	}
+	
+} INSERT AFTER .data;

src/kern/cpu.zig

@@ -8,18 +8,16 @@ comptime {
     if (!@hasDecl(ki.impl, "Cpu")) @compileError("impl must provide Cpu");
 }
 
-pub const ThreadStatus = packed struct(u8) {
-    interactive: bool,
-    priority: u7,
-};
-
 pub const PreemptionReason = enum(u8) {
     None,
     /// A higher priority thread was readied.
     HigherPriority,
 };
 
-/// Per-CPU state
+extern var __init_array_percpu_start: u8;
+extern var __init_array_percpu_end: u8;
+
+/// Global Per-CPU state.
 pub const Cpu = struct {
     /// Implementation-dependent CPU data.
     impl: ki.impl.Cpu,
@@ -27,13 +25,8 @@ pub const Cpu = struct {
     id: u32,
     /// Current IPL on this CPU.
     ipl: ke.Ipl,
-    /// Bitmask of pending software interrupts on this CPU.
-    pending_softints: std.atomic.Value(u8),
     /// Currently running thread.
     current_thread: ?*ke.Thread,
-    /// Atomic byte for getting the current thread's status locklessly.
-    /// 1 bit for thread interactivity, 7 for priority.
-    current_thread_status: std.atomic.Value(ThreadStatus),
     /// Per-CPU idle thread.
     idle_thread: ?*ke.Thread,
     /// Thread selected for preemption.
@@ -42,47 +35,57 @@ pub const Cpu = struct {
     preemption_reason: PreemptionReason,
     /// Flag indicating the start of the scheduling timer.
     start_timer: bool,
-    /// Queue of DPCs on this CPU.
-    dpc_queue: rtl.List,
-    /// Lock over this CPU's DPC queue.
-    dpc_lock: ke.SpinLock,
-    /// Heap of pending timers on this CPU.
-    timers_heap: rtl.PairingHeap(.min, ki.timer.cmp_timer),
-    /// Lock over the timer heap.
-    timers_lock: ke.SpinLock,
-    /// Per-CPU scheduler state.
-    sched_data: ki.sched.PerCpu,
     /// DPCs
     resched_dpc: ke.Dpc,
-    timer_dpc: ke.Dpc,
     resched_timer: ke.Timer,
 
     /// Initialize `cpu` for usage with its idle thread being `thread`.
     pub fn init(cpu: *Cpu, thread: *ke.Thread) void {
         cpu.* = .{
             .id = cpu.id,
-            .ipl = .Zero,
-            .pending_softints = .init(0),
+            .ipl = .Passive,
             .idle_thread = thread,
             .current_thread = thread,
-            .timer_dpc = .init(ki.timer.handle_expiry),
             .resched_dpc = .init(ki.sched.clock),
-            .timers_heap = .init(),
-            .current_thread_status = .init(.{ .interactive = false, .priority = 0 }),
             .next_thread = null,
-            .dpc_lock = .init(),
             .preemption_reason = .None,
             .start_timer = false,
-            .dpc_queue = undefined,
-            .timers_lock = .init(),
-            .sched_data = undefined,
             .resched_timer = undefined,
             .impl = cpu.impl,
         };
 
-        cpu.dpc_queue.init();
-        cpu.resched_timer.init();
+        // Call per-cpu init functions.
+        const start = @intFromPtr(&__init_array_percpu_start);
+        const end = @intFromPtr(&__init_array_percpu_end);
+        const count = (end - start) / @sizeOf(*const fn () void);
+        const funcs: [*]const *const fn () callconv(.c) void = @ptrFromInt(start);
 
-        ki.sched.init_cpu(&cpu.sched_data);
+        for (0..count) |i| {
+            funcs[i]();
+        }
+
+        cpu.resched_timer.init();
     }
 };
+
+comptime {
+    if (!@hasDecl(ki.impl, "percpu_ptr")) @compileError("impl must provide percpu_ptr()");
+    if (!@hasDecl(ki.impl, "percpu_ptr_other")) @compileError("impl must provide percpu_ptr_other()");
+}
+
+/// Wraps around CPU-local data.
+pub fn CpuLocal(comptime T: type, comptime init: T) type {
+    return struct {
+        var storage: T linksection(".data.percpu") = init;
+
+        /// Return a pointer to local CPU data.
+        pub fn local() @TypeOf(ki.impl.percpu_ptr(&storage)) {
+            return ki.impl.percpu_ptr(&storage);
+        }
+
+        /// Return a pointer to remote CPU data.
+        pub fn remote(cpu: *ke.Cpu) *T {
+            return ki.impl.percpu_ptr_other(&storage, cpu.id);
+        }
+    };
+}

src/kern/dpc.zig

@@ -28,26 +28,45 @@ pub const Dpc = struct {
     }
 };
 
+const PerCpu = struct {
+    /// Queue of DPCs on this CPU.
+    queue: rtl.List,
+    /// Lock over this CPU's DPC queue.
+    lock: ke.SpinLock,
+};
+
+const pcpu = ke.CpuLocal(PerCpu, .{
+    .lock = .init(),
+    .queue = undefined,
+});
+
+export const dpc_percpu_init linksection(b.percpu_init) = &pcpu_init;
+
+fn pcpu_init() linksection(b.init) callconv(.c) void {
+    pcpu.local().queue.init();
+}
+
 /// Enqueue a DPC.
 /// If the DPC is already enqueued, this function is a no-op.
 pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void {
     const ipl = ke.ipl.raise(.High);
     const cpu = ke.curcpu();
+    const dpc_cpu = pcpu.local();
 
     if (!dpc.inserted) {
         dpc.arg = arg;
 
         // Insert the DPC on this CPU's DPC queue
-        cpu.dpc_lock.acquire_no_ipl();
+        dpc_cpu.lock.acquire_no_ipl();
 
-        cpu.dpc_queue.insert_tail(&dpc.link);
+        dpc_cpu.queue.insert_tail(&dpc.link);
 
         // Mark the DPC as pending on this CPU
         ki.ipl.set_softint_pending(cpu, .Dispatch);
 
         dpc.inserted = true;
 
-        cpu.dpc_lock.release_no_ipl();
+        dpc_cpu.lock.release_no_ipl();
     }
 
     ke.ipl.lower(ipl);
@@ -55,6 +74,7 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void {
 
 fn dispatch_queue(cpu: *ke.Cpu) void {
     cpu.ipl = .Dispatch;
+    const dpc_cpu = pcpu.remote(cpu);
 
     // Mark as handled, this must be done before enabling interrupts or we will race.
     ki.ipl.clear_softint_pending(cpu, .Dispatch);
@@ -62,18 +82,18 @@ fn dispatch_queue(cpu: *ke.Cpu) void {
     _ = ki.impl.enable_interrupts();
 
     while (true) {
-        const ipl = cpu.dpc_lock.acquire_at(.High);
+        const ipl = dpc_cpu.lock.acquire_at(.High);
 
         var dpc: *Dpc = undefined;
 
-        if (cpu.dpc_queue.is_empty()) {
+        if (dpc_cpu.queue.is_empty()) {
             // No more DPCs to process.
-            cpu.dpc_lock.release(ipl);
+            dpc_cpu.lock.release(ipl);
             break;
         }
 
         // Pop the head
-        var first_elem = cpu.dpc_queue.first();
+        var first_elem = dpc_cpu.queue.first();
 
         first_elem.remove();
 
@@ -84,7 +104,7 @@ fn dispatch_queue(cpu: *ke.Cpu) void {
         // which is why we capture it here to ensure that we get the intended context.
         const arg = dpc.arg;
 
-        cpu.dpc_lock.release(ipl);
+        dpc_cpu.lock.release(ipl);
 
         std.debug.assert(cpu.ipl == .Dispatch);
 
@@ -120,7 +140,7 @@ pub fn dispatch(cpu: *ke.Cpu) void {
     const old_ipl = cpu.ipl;
     const int_state = ki.impl.disable_interrupts();
 
-    while (ki.ipl.is_softint_pending(mycpu, .Dispatch)) {
+    while (ki.ipl.is_softint_pending(.Dispatch)) {
         dispatch_queue(mycpu);
 
         // Our CPU might have changed.

src/kern/init.zig

@@ -43,12 +43,12 @@ pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void {
 
     _ = ke.ipl.raise(.Dispatch);
 
-    for (0..10) |i| {
+    for (0..threads.len) |i| {
         make_thread(handler, &threads[i]);
         ke.sched.enqueue(&threads[i]);
     }
 
-    ke.ipl.lower(.Zero);
+    ke.ipl.lower(.Passive);
 
     while (true) {}
 }

src/kern/ipl.zig

@@ -6,7 +6,7 @@ const ki = ke.private;
 /// Interrupt priority level (IPL)
 pub const Ipl = enum(u8) {
     /// All interrupts enabled.
-    Zero = 0,
+    Passive = 0,
     /// Preemption disabled.
     Dispatch = 1,
     /// Device interrupts disabled.
@@ -19,9 +19,14 @@ pub const Ipl = enum(u8) {
     }
 };
 
+/// Bitmask of pending software interrupts on this CPU.
+const pending_softints = ke.CpuLocal(std.atomic.Value(u8), .init(0));
+
 /// Raise IPL to `new`.
 /// Panics if the current IPL is higher than requested.
 pub fn raise(new: Ipl) Ipl {
+    const ints = ki.impl.disable_interrupts();
+
     var cpu = ke.curcpu();
     const old = cpu.ipl;
 
@@ -35,12 +40,15 @@ pub fn raise(new: Ipl) Ipl {
 
     cpu.ipl = new;
 
+    ki.impl.restore_interrupts(ints);
+
     return old;
 }
 
 /// Lower IPL to `new`.
 /// Panics if the current IPL is lower than requested.
 pub fn lower(new: Ipl) void {
+    const ints = ki.impl.disable_interrupts();
     var cpu = ke.curcpu();
     const old = cpu.ipl;
 
@@ -49,21 +57,23 @@ pub fn lower(new: Ipl) void {
     }
 
     if (@intFromEnum(new) <= @intFromEnum(Ipl.get_max_software()) and @intFromEnum(old) > @intFromEnum(Ipl.get_max_software())) {
-        ki.impl.set_hardware_ipl(.Zero);
-    }
-
-    if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and is_softint_pending(cpu, .Dispatch)) {
-        ki.dpc.dispatch(cpu);
+        ki.impl.set_hardware_ipl(.Passive);
     }
 
     cpu.ipl = new;
+
+    ki.impl.restore_interrupts(ints);
+
+    if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and is_softint_pending(.Dispatch)) {
+        ki.dpc.dispatch(cpu);
+    }
 }
 
 comptime {
     if (!@hasDecl(ki.impl, "set_hardware_ipl")) @compileError("impl must provide set_hardware_ipl()");
 }
 
-/// Set the hardware IPL to `new`
+/// Set the hardware IPL to `new`.
 pub fn set_hardware(new: Ipl) Ipl {
     var cpu = ke.curcpu();
     const old = cpu.ipl;
@@ -75,22 +85,22 @@ pub fn set_hardware(new: Ipl) Ipl {
     return old;
 }
 
-/// Mark a software interrupt of IPL `ipl` on `cpu` as pending
+/// Mark a software interrupt of IPL `ipl` on `cpu` as pending.
 pub fn set_softint_pending(cpu: *ke.Cpu, ipl: Ipl) void {
     std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
-    _ = cpu.pending_softints.bitSet(@intCast(@intFromEnum(ipl)), .monotonic);
+    _ = pending_softints.remote(cpu).bitSet(@intCast(@intFromEnum(ipl)), .monotonic);
 }
 
-/// Mark a software interrupt of IPL `ipl` on `cpu` as handled
+/// Mark a software interrupt of IPL `ipl` on `cpu` as handled.
 pub fn clear_softint_pending(cpu: *ke.Cpu, ipl: Ipl) void {
     std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
-    _ = cpu.pending_softints.bitReset(@intCast(@intFromEnum(ipl)), .monotonic);
+    _ = pending_softints.remote(cpu).bitReset(@intCast(@intFromEnum(ipl)), .monotonic);
 }
 
-/// Check whether a software interrupt of IPL `ipl` on `cpu` is pending
-pub fn is_softint_pending(cpu: *ke.Cpu, ipl: Ipl) bool {
+/// Check whether a software interrupt of IPL `ipl` on the current CPU is pending.
+pub fn is_softint_pending(ipl: Ipl) bool {
     std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
     const shift: u3 = @intCast(@intFromEnum(ipl));
     const bit: u8 = @as(u8, 1) << shift;
-    return cpu.pending_softints.load(.monotonic) & bit != 0;
+    return pending_softints.local().load(.monotonic) & bit != 0;
 }

src/kern/root.zig

@@ -53,12 +53,13 @@ pub const wait = struct {
 };
 
 pub const Cpu = p.cpu.Cpu;
+pub const CpuLocal = p.cpu.CpuLocal;
 
 comptime {
     if (!@hasDecl(p.impl, "curcpu")) @compileError("impl must provide curcpu()");
 }
 
-/// Returns the current CPU
+/// Return the current CPU
 pub const curcpu: fn () *Cpu = p.impl.curcpu;
 
 /// Number of CPUs on the system

src/kern/sched.zig

@@ -103,6 +103,11 @@ pub const RunQueue = struct {
     queues: [runqueues_n]rtl.List,
 };
 
+const ThreadStatus = packed struct(u8) {
+    interactive: bool,
+    priority: u7,
+};
+
 /// Per-CPU scheduler data.
 pub const PerCpu = struct {
     /// Current load on this CPU (number of ready threads)
@@ -121,12 +126,17 @@ pub const PerCpu = struct {
     insidx: u8,
     /// Used to indicate to this CPU it can steal work
     steal_work: bool,
+    /// Atomic byte for getting the current thread's status locklessly.
+    /// 1 bit for thread interactivity, 7 for priority.
+    current_thread_status: std.atomic.Value(ThreadStatus),
 };
 
+const percpu_data = ke.CpuLocal(PerCpu, undefined);
+
 /// Handle preemption.
 /// This is called in DPC dispatch when it notices `next_thread` is set.
 pub fn handle_preemption(cpu: *ke.Cpu) void {
-    const sched_cpu = &cpu.sched_data;
+    const sched_cpu = percpu_data.remote(cpu);
     sched_cpu.queues_lock.acquire_no_ipl();
 
     const cur = cpu.current_thread.?;
@@ -153,10 +163,10 @@ pub fn handle_preemption(cpu: *ke.Cpu) void {
     // cur.lock dropped
 }
 
-/// Called every time slice.
+/// Called every time slice in DPC context.
 pub fn clock(_: ?*anyopaque) void {
     const cpu = ke.curcpu();
-    const sched_cpu = &cpu.sched_data;
+    const sched_cpu = percpu_data.remote(cpu);
     std.debug.assert(cpu.ipl == .Dispatch);
 
     const curtd = cpu.current_thread orelse return;
@@ -238,8 +248,12 @@ pub fn unblock(td: *ke.Thread) void {
     td.lock.release(ipl);
 }
 
+export const sched_percpu_init linksection(b.percpu_init) = &init_cpu;
+
 /// Initialize a CPU for use by the scheduler.
-pub fn init_cpu(percpu: *PerCpu) linksection(b.init) void {
+fn init_cpu() linksection(b.init) callconv(.c) void {
+    var percpu = percpu_data.local();
+
     percpu.* = .{
         .insidx = 0,
         .runidx = 0,
@@ -249,6 +263,7 @@ pub fn init_cpu(percpu: *PerCpu) linksection(b.init) void {
         .idle_queue = undefined,
         .steal_work = false,
         .load = .init(0),
+        .current_thread_status = .init(.{ .interactive = false, .priority = 0 }),
     };
 
     percpu.idle_queue.init();
@@ -593,13 +608,13 @@ fn should_preempt(td: *ke.Thread, cur: *ke.Thread) bool {
 // Return whether or not `td` should preempt the thread running on `cpu`.
 // This is done in a lock-free manner.
 fn should_preempt_cpu(td: *ke.Thread, cpu: *ke.Cpu) bool {
-    const status = cpu.current_thread_status.load(.monotonic);
+    const status = percpu_data.remote(cpu).current_thread_status.load(.monotonic);
     return should_prio_preempt(td, status.priority, status.interactive);
 }
 
 // Enqueue a thread on a CPU.
 fn enqueue_on_cpu(cpu: *ke.Cpu, td: *ke.Thread) void {
-    var c = &cpu.sched_data;
+    var c = percpu_data.local();
 
     c.queues_lock.acquire_no_ipl();
 
@@ -669,13 +684,14 @@ fn pick_cpu(td: *ke.Thread) *ke.Cpu {
 
     for (0..ke.ncpus) |i| {
         const cpu = ke.cpus[i];
+        const data = percpu_data.remote(cpu);
 
-        if (least == null or cpu.sched_data.load.load(.monotonic) < least.?.sched_data.load.load(.monotonic)) {
+        if (least == null or data.load.load(.monotonic) < percpu_data.remote(least.?).load.load(.monotonic)) {
             least = cpu;
         }
 
         if (should_preempt_cpu(td, cpu)) {
-            if (least_preempt == null or cpu.sched_data.load.load(.monotonic) < least_preempt.?.sched_data.load.load(.monotonic)) {
+            if (least_preempt == null or data.load.load(.monotonic) < percpu_data.remote(least_preempt.?).load.load(.monotonic)) {
                 least_preempt = cpu;
             }
         }
@@ -687,7 +703,7 @@ fn pick_cpu(td: *ke.Thread) *ke.Cpu {
 
 // Find a thread to preempt `cur` with.
 fn select_thread(cur: ?*ke.Thread, cpu: *ke.Cpu, migrate: bool) ?*ke.Thread {
-    const sched_cpu = &cpu.sched_data;
+    const sched_cpu = percpu_data.remote(cpu);
 
     // First try to pick from the realtime queues.
     if (pick_realtime_thread(sched_cpu, if (cur) |c| c.priority else 0, migrate)) |td| {
@@ -728,7 +744,7 @@ fn do_switch(cpu: *ke.Cpu, cur: *ke.Thread, next: *ke.Thread) void {
     next.last_cpu = cpu;
 
     // Stash thread status.
-    cpu.current_thread_status.store(.{ .interactive = next.interactive, .priority = @intCast(next.priority) }, .monotonic);
+    percpu_data.remote(cpu).current_thread_status.store(.{ .interactive = next.interactive, .priority = @intCast(next.priority) }, .monotonic);
 
     // Do machine-dependent switch.
     cur.context.switch_to(&next.context);
@@ -736,14 +752,14 @@ fn do_switch(cpu: *ke.Cpu, cur: *ke.Thread, next: *ke.Thread) void {
 
 // Yield on `cpu`
 fn yield(cpu: *ke.Cpu) void {
-    const sched_cpu = &cpu.sched_data;
+    const sched_cpu = percpu_data.remote(cpu);
 
     sched_cpu.queues_lock.acquire_no_ipl();
 
     const cur = cpu.current_thread;
     var next = cpu.next_thread;
 
-    if (next) {
+    if (next != null) {
         cpu.next_thread = null;
     } else {
         // Pick a new thread to run.
@@ -758,7 +774,7 @@ fn yield(cpu: *ke.Cpu) void {
     sched_cpu.queues_lock.release_no_ipl();
 
     // Switch into the thread.
-    do_switch(cpu, cur, next);
+    do_switch(cpu, cur.?, next.?);
 
     // cur lock dropped
 }

src/kern/timer.zig

@@ -7,6 +7,14 @@ const ke = b.ke;
 const ki = ke.private;
 const pl = b.pl;
 
+const PerCpu = struct {
+    /// Heap of pending timers on this CPU.
+    timers: rtl.PairingHeap(.min, ki.timer.cmp_timer),
+    /// Lock over the timer heap.
+    lock: ke.SpinLock,
+    dpc: ke.Dpc,
+};
+
 pub const Timer = struct {
     pub const State = enum(u8) {
         /// The timer is currently being handled.
@@ -28,7 +36,7 @@ pub const Timer = struct {
     /// Intrusive pairing heap node.
     node: rtl.pairing_heap.Node,
     /// CPU this timer is enqueued on.
-    cpu: ?*ke.Cpu,
+    cpu: ?*PerCpu,
 
     pub fn init(self: *Timer) void {
         self.* = .{
@@ -44,21 +52,27 @@ pub const Timer = struct {
     }
 };
 
+const percpu = ke.CpuLocal(PerCpu, .{
+    .timers = .init(),
+    .lock = .init(),
+    .dpc = .init(handle_expiry),
+});
+
 /// Start a timer with an expiration time.
 /// A DPC that will be enqueued upon expiration can be passed.
 pub fn set(timer: *Timer, time: b.Nanoseconds, dpc: ?*ke.Dpc) void {
     const ipl = timer.hdr.lock.acquire();
     defer timer.hdr.lock.release(ipl);
 
-    const cpu = ke.curcpu();
+    const cpu = percpu.local();
 
     if (timer.state.cmpxchgStrong(.Stopped, .Pending, .acquire, .monotonic) != null) {
         // Could not set the timer.
         return;
     }
 
-    cpu.timers_lock.acquire_no_ipl();
-    defer cpu.timers_lock.release_no_ipl();
+    cpu.lock.acquire_no_ipl();
+    defer cpu.lock.release_no_ipl();
 
     // Initialize the timer.
     timer.deadline = ke.timecounter.read_time_nano() + time;
@@ -67,9 +81,9 @@ pub fn set(timer: *Timer, time: b.Nanoseconds, dpc: ?*ke.Dpc) void {
     timer.dpc = dpc;
     timer.hdr.signaled = 0;
 
-    cpu.timers_heap.insert(&timer.node);
+    cpu.timers.insert(&timer.node);
 
-    if (cpu.timers_heap.root == &timer.node) {
+    if (cpu.timers.root == &timer.node) {
         // This is the earliest timer to expire, arm the hardware timer.
         pl.arm_timer(time);
     }
@@ -96,12 +110,12 @@ pub fn cancel(timer: *Timer) void {
 
     if (timer.cpu) |cpu| {
         // Dequeue the timer.
-        cpu.timers_lock.acquire_no_ipl();
+        cpu.lock.acquire_no_ipl();
 
-        cpu.timers_heap.remove(&timer.node);
+        cpu.timers.remove(&timer.node);
         timer.cpu = null;
 
-        cpu.timers_lock.release_no_ipl();
+        cpu.lock.release_no_ipl();
     }
 
     // Lock dropped
@@ -115,19 +129,24 @@ pub fn cmp_timer(a: *rtl.pairing_heap.Node, b_: *rtl.pairing_heap.Node) std.math
     return std.math.order(timer_a.deadline, timer_b.deadline);
 }
 
-/// Called in a DPC when a timer has expired.
-pub fn handle_expiry(_: ?*anyopaque) void {
-    const cpu = ke.curcpu();
-    std.debug.assert(cpu.ipl == .Dispatch);
+/// Called by the platform on a clock interrupt.
+pub fn clock() void {
+    ke.dpc.enqueue(&percpu.local().dpc, null);
+}
+
+// Called in a DPC when a timer has expired.
+fn handle_expiry(_: ?*anyopaque) void {
+    std.debug.assert(ke.curcpu().ipl == .Dispatch);
+    const cpu = percpu.local();
 
     while (true) {
         const curtime = ke.timecounter.read_time_nano();
 
-        cpu.timers_lock.acquire_no_ipl();
+        cpu.lock.acquire_no_ipl();
 
         // Get the timer that expires the soonest.
-        const timer_node = cpu.timers_heap.root orelse {
-            cpu.timers_lock.release_no_ipl();
+        const timer_node = cpu.timers.root orelse {
+            cpu.lock.release_no_ipl();
             return;
         };
 
@@ -137,7 +156,7 @@ pub fn handle_expiry(_: ?*anyopaque) void {
         // Sub-millisecond differences are close enough to expire immediately.
         if (timer.deadline > curtime and timer.deadline - curtime > std.time.ns_per_ms) {
             const next = timer;
-            cpu.timers_lock.release_no_ipl();
+            cpu.lock.release_no_ipl();
 
             // Re-check in case the timer expired while we were here.
             const now = ke.timecounter.read_time_nano();
@@ -150,8 +169,8 @@ pub fn handle_expiry(_: ?*anyopaque) void {
             return;
         }
 
-        _ = cpu.timers_heap.pop();
-        cpu.timers_lock.release_no_ipl();
+        _ = cpu.timers.pop();
+        cpu.lock.release_no_ipl();
 
         if (timer.state.cmpxchgStrong(.Pending, .Running, .acquire, .monotonic) != null) {
             // Timer must've been canceled.

src/kern/um/impl.zig

@@ -60,7 +60,7 @@ pub const ThreadContext = struct {
     fn real_entry(entry_int: c_ulong, arg_int: c_ulong) callconv(.c) void {
         const entry: *const fn (?*anyopaque) void = @ptrFromInt(entry_int);
         const arg: ?*anyopaque = @ptrFromInt(arg_int);
-        ke.ipl.lower(.Zero);
+        ke.ipl.lower(.Passive);
         entry(arg);
     }
 
@@ -104,3 +104,11 @@ pub const ThreadContext = struct {
         }
     }
 };
+
+pub inline fn percpu_ptr_other(variable: anytype, cpu_id: usize) @TypeOf(variable) {
+    return @ptrFromInt(@intFromPtr(variable) +% b.pl.impl.cpu_offsets[cpu_id]);
+}
+
+pub inline fn percpu_ptr(variable: anytype) @TypeOf(variable) {
+    return @ptrFromInt(@intFromPtr(variable) +% b.pl.impl.cpu_offsets[b.pl.impl.my_cpu.id]);
+}

src/kern/wait.zig

@@ -74,7 +74,8 @@ pub const Status = enum(u8) {
 
 /// Wait for the provided object to be signaled.
 pub fn wait_one(object: *DispatchHeader, timeout: ?b.Nanoseconds) !usize {
-    return wait_any(object, timeout, null);
+    var objects = [_]*DispatchHeader{object};
+    return wait_any(&objects, timeout, null);
 }
 
 /// Wait for any of the provided objects to be signaled.
@@ -85,7 +86,7 @@ pub fn wait_one(object: *DispatchHeader, timeout: ?b.Nanoseconds) !usize {
 pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks: ?[*]WaitBlock) !usize {
     const ipl = ke.ipl.raise(.Dispatch);
     defer ke.ipl.lower(ipl);
-    const curtd = ke.curcpu().current_thread;
+    const curtd = ke.curcpu().current_thread.?;
     const obj_count = objects.len;
     const has_timeout = timeout != null;
     const blocks = waitblocks orelse blk: {
@@ -99,7 +100,7 @@ pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks:
     const timer_block = &curtd.waitblocks[3];
 
     var satisfier: ?usize = null;
-    curtd.wait_status.store(.InProgress, .acquire);
+    curtd.wait_status.store(.InProgress, .release);
 
     for (0..total_count) |i| {
         const is_timer = has_timeout and i == timer_i;
@@ -111,7 +112,7 @@ pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks:
 
         if (obj.signaled > 0) {
             // Object was already signaled. Try consuming it.
-            if (curtd.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acquire, .monotonic) == null) {
+            if (curtd.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acq_rel, .monotonic) == null) {
                 obj.consume();
             }
             // We have already been satisfied in the meantime, abort.
@@ -128,18 +129,20 @@ pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks:
 
     // Wait was already satisfied, back out.
     if (satisfier != null or (has_timeout and timeout.? == 0)) {
-        std.debug.assert(curtd.wait_status.load(.monotonic) == .Satisfied);
+        std.debug.assert(curtd.wait_status.load(.acquire) == .Satisfied);
 
-        // Remove any wait block we might've installed
-        for (0..satisfier) |i| {
-            const is_timer = has_timeout and i == timer_i;
-            var obj = if (is_timer) &timer.hdr else objects[i];
-            var wb = if (is_timer) timer_block else &blocks[i];
+        if (satisfier) |sat| {
+            // Remove any wait block we might've installed
+            for (0..sat) |i| {
+                const is_timer = has_timeout and i == timer_i;
+                var obj = if (is_timer) &timer.hdr else objects[i];
+                var wb = if (is_timer) timer_block else &blocks[i];
 
-            obj.lock.acquire_no_ipl();
-            wb.status = .Inactive;
-            wb.link.remove();
-            obj.lock.release_no_ipl();
+                obj.lock.acquire_no_ipl();
+                wb.status = .Inactive;
+                wb.link.remove();
+                obj.lock.release_no_ipl();
+            }
         }
 
         return satisfier orelse error.Timeout;
@@ -152,7 +155,7 @@ pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks:
     // Now try committing the wait.
     // While we're trying to commit the wait, the object locks have been released, and the state could therefore change.
     // We need to re-check the state of the wait before actually blocking.
-    if (curtd.wait_status.cmpxchgStrong(.InProgress, .Committed, .acquire, .monotonic) == null) {
+    if (curtd.wait_status.cmpxchgStrong(.InProgress, .Committed, .acq_rel, .monotonic) == null) {
         // We're good, now actually block.
         ke.sched.block();
     }
@@ -174,7 +177,7 @@ pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks:
             // Waitblock is still active, remove it from the list.
             wb.link.remove();
         }
-        if (wb.status == .Signaled or wb.status == .SignaledInProgress) {
+        if (wb.status == .Signaled) {
             // This waitblock was signaled, it must be the one that satisfied the wait.
             std.debug.assert(satisfier == null);
             satisfier = i;
@@ -206,14 +209,14 @@ pub fn satisfy_wait(obj: *DispatchHeader) void {
         // 3. The wait was already satisfied by another object (status == .Satisfied).
 
         // 1.
-        if (td.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acquire, .monotonic) == null) {
+        if (td.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acq_rel, .monotonic) == null) {
             // We interrupted the wait while it was being prepared.
             wb.status = .Signaled;
             obj.consume();
         }
 
         // 2.
-        else if (td.wait_status.cmpxchgStrong(.Committed, .Satisfied, .acquire, .monotonic) == null) {
+        else if (td.wait_status.cmpxchgStrong(.Committed, .Satisfied, .acq_rel, .monotonic) == null) {
             // We interrupted the wait while it was committed.
             // Wake the thread.
             wb.status = .Signaled;

src/platform/um/impl.zig

@@ -56,8 +56,15 @@ fn check_function(fn_name: []const u8, ret: usize) void {
     }
 }
 
+const bootstrap_offsets = [1]usize{0};
+
+pub var cpu_offsets: []usize = undefined;
+
 pub fn early_init() linksection(b.init) void {
-    // Set current CPU to the bootstrap CPU
+    // Ensure we can use per-cpu data.
+    cpu_offsets = @constCast(&bootstrap_offsets);
+
+    // Set current CPU to the bootstrap CPU.
     my_cpu = &ki.bootstrap_cpu;
 
     my_cpu.impl.pthread = c.pthread_self();
@@ -96,7 +103,7 @@ pub fn early_init() linksection(b.init) void {
 }
 
 fn sigusr1_handler(_: posix.SIG, _: *const posix.siginfo_t, _: ?*anyopaque) callconv(.c) void {
-    if (@intFromEnum(ke.curcpu().ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(ke.curcpu(), .Dispatch)) {
+    if (@intFromEnum(ke.curcpu().ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(.Dispatch)) {
         ki.dpc.dispatch(ke.curcpu());
     }
 }
@@ -128,8 +135,13 @@ fn other_cpu_entry(arg: ?*anyopaque) callconv(.c) ?*anyopaque {
     return null;
 }
 
+var cpu: ke.Cpu linksection(b.percpu) = undefined;
+
 fn idle(_: ?*anyopaque) void {}
 
+extern var __percpu_start: u8;
+extern var __percpu_end: u8;
+
 pub fn late_init() linksection(b.init) void {
     timer.init();
     timer.init_cpu();
@@ -158,10 +170,19 @@ pub fn late_init() linksection(b.init) void {
         allocator.alloc(c.pthread_t, other_count) catch @panic("oom");
 
     ke.cpus = allocator.alloc(*ke.Cpu, ke.ncpus) catch @panic("oom");
+    cpu_offsets = allocator.alloc(usize, ke.ncpus) catch @panic("oom");
     ke.cpus[0] = &ki.bootstrap_cpu;
+    cpu_offsets[0] = 0;
+
+    const size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start);
 
     for (1..ke.ncpus) |i| {
-        ke.cpus[i] = allocator.create(ke.Cpu) catch @panic("oom");
+        const block = allocator.alloc(u8, size) catch @panic("oom");
+        cpu_offsets[i] = @intFromPtr(block.ptr) -% @intFromPtr(&__percpu_start);
+
+        ke.cpus[i] = ki.impl.percpu_ptr_other(&cpu, i);
+
+        @memcpy(block, @as([*]u8, @ptrCast(&__percpu_start))[0..size]);
 
         ke.cpus[i].idle_thread = allocator.create(ke.Thread) catch @panic("oom");
         ke.cpus[i].id = @intCast(i);

src/platform/um/timer.zig

@@ -11,11 +11,11 @@ threadlocal var timerid: c.timer_t = undefined;
 fn timer_handler(_: posix.SIG, _: *const posix.siginfo_t, _: ?*anyopaque) callconv(.c) void {
     const old_ipl = ke.ipl.set_hardware(.Device);
 
-    ke.dpc.enqueue(&ke.curcpu().timer_dpc, null);
+    ki.timer.clock();
 
     _ = ke.ipl.set_hardware(old_ipl);
 
-    if (@intFromEnum(old_ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(ke.curcpu(), .Dispatch)) {
+    if (@intFromEnum(old_ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(.Dispatch)) {
         ki.dpc.dispatch(ke.curcpu());
     }
 }

src/root.zig

@@ -4,7 +4,9 @@ pub const pl = @import("platform/root.zig");
 pub const rtl = @import("rtl");
 pub const ksyms = @import("ksyms");
 
-pub const init = ".init.text";
+pub const init = ".text.init";
+pub const percpu_init = ".percpu_init_array";
+pub const percpu = ".data.percpu";
 
 pub const VAddr = usize;
 pub const PAddr = usize;