From 020e81f34b68f9ced12476377c767612cdbbee35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Gr=C3=A9goire?= Date: Mon, 23 Mar 2026 20:29:34 -0400 Subject: [PATCH] feat: use per-cpu sections --- build.zig | 1 + build/linker-scripts/uml.lds | 14 +++++++ src/kern/cpu.zig | 71 +++++++++++++++++++----------------- src/kern/dpc.zig | 38 ++++++++++++++----- src/kern/init.zig | 4 +- src/kern/ipl.zig | 38 ++++++++++++------- src/kern/root.zig | 3 +- src/kern/sched.zig | 42 ++++++++++++++------- src/kern/timer.zig | 57 +++++++++++++++++++---------- src/kern/um/impl.zig | 10 ++++- src/kern/wait.zig | 39 +++++++++++--------- src/platform/um/impl.zig | 27 ++++++++++++-- src/platform/um/timer.zig | 4 +- src/root.zig | 4 +- 14 files changed, 235 insertions(+), 117 deletions(-) create mode 100644 build/linker-scripts/uml.lds diff --git a/build.zig b/build.zig index 0b681c7..7f0e50f 100644 --- a/build.zig +++ b/build.zig @@ -212,6 +212,7 @@ fn addKernel(b: *std.Build, plat: config.Platform, optimize: std.builtin.Optimiz kernel.linker_script = b.path(b.fmt("build/linker-scripts/{s}.lds", .{@tagName(plat.arch)})); } else { + kernel.linker_script = b.path("build/linker-scripts/uml.lds"); kernel.root_module.link_libc = true; } diff --git a/build/linker-scripts/uml.lds b/build/linker-scripts/uml.lds new file mode 100644 index 0000000..49418fa --- /dev/null +++ b/build/linker-scripts/uml.lds @@ -0,0 +1,14 @@ +SECTIONS { + .data.percpu : { + __percpu_start = .; + *(.data.percpu) + __percpu_end = .; + } + +.percpu_init_array : { + __init_array_percpu_start = .; + KEEP(*(.percpu_init_array)) + __init_array_percpu_end = .; + } + +} INSERT AFTER .data; diff --git a/src/kern/cpu.zig b/src/kern/cpu.zig index 1d93076..1401ae9 100644 --- a/src/kern/cpu.zig +++ b/src/kern/cpu.zig @@ -8,18 +8,16 @@ comptime { if (!@hasDecl(ki.impl, "Cpu")) @compileError("impl must provide Cpu"); } -pub const ThreadStatus = packed struct(u8) { - interactive: bool, - priority: u7, -}; - pub const PreemptionReason = enum(u8) { None, /// A higher priority thread was readied. HigherPriority, }; -/// Per-CPU state +extern var __init_array_percpu_start: u8; +extern var __init_array_percpu_end: u8; + +/// Global Per-CPU state. pub const Cpu = struct { /// Implementation-dependent CPU data. impl: ki.impl.Cpu, @@ -27,13 +25,8 @@ pub const Cpu = struct { id: u32, /// Current IPL on this CPU. ipl: ke.Ipl, - /// Bitmask of pending software interrupts on this CPU. - pending_softints: std.atomic.Value(u8), /// Currently running thread. current_thread: ?*ke.Thread, - /// Atomic byte for getting the current thread's status locklessly. - /// 1 bit for thread interactivity, 7 for priority. - current_thread_status: std.atomic.Value(ThreadStatus), /// Per-CPU idle thread. idle_thread: ?*ke.Thread, /// Thread selected for preemption. @@ -42,47 +35,57 @@ pub const Cpu = struct { preemption_reason: PreemptionReason, /// Flag indicating the start of the scheduling timer. start_timer: bool, - /// Queue of DPCs on this CPU. - dpc_queue: rtl.List, - /// Lock over this CPU's DPC queue. - dpc_lock: ke.SpinLock, - /// Heap of pending timers on this CPU. - timers_heap: rtl.PairingHeap(.min, ki.timer.cmp_timer), - /// Lock over the timer heap. - timers_lock: ke.SpinLock, - /// Per-CPU scheduler state. - sched_data: ki.sched.PerCpu, /// DPCs resched_dpc: ke.Dpc, - timer_dpc: ke.Dpc, resched_timer: ke.Timer, /// Initialize `cpu` for usage with its idle thread being `thread`. pub fn init(cpu: *Cpu, thread: *ke.Thread) void { cpu.* = .{ .id = cpu.id, - .ipl = .Zero, - .pending_softints = .init(0), + .ipl = .Passive, .idle_thread = thread, .current_thread = thread, - .timer_dpc = .init(ki.timer.handle_expiry), .resched_dpc = .init(ki.sched.clock), - .timers_heap = .init(), - .current_thread_status = .init(.{ .interactive = false, .priority = 0 }), .next_thread = null, - .dpc_lock = .init(), .preemption_reason = .None, .start_timer = false, - .dpc_queue = undefined, - .timers_lock = .init(), - .sched_data = undefined, .resched_timer = undefined, .impl = cpu.impl, }; - cpu.dpc_queue.init(); - cpu.resched_timer.init(); + // Call per-cpu init functions. + const start = @intFromPtr(&__init_array_percpu_start); + const end = @intFromPtr(&__init_array_percpu_end); + const count = (end - start) / @sizeOf(*const fn () void); + const funcs: [*]const *const fn () callconv(.c) void = @ptrFromInt(start); - ki.sched.init_cpu(&cpu.sched_data); + for (0..count) |i| { + funcs[i](); + } + + cpu.resched_timer.init(); } }; + +comptime { + if (!@hasDecl(ki.impl, "percpu_ptr")) @compileError("impl must provide percpu_ptr()"); + if (!@hasDecl(ki.impl, "percpu_ptr_other")) @compileError("impl must provide percpu_ptr_other()"); +} + +/// Wraps around CPU-local data. +pub fn CpuLocal(comptime T: type, comptime init: T) type { + return struct { + var storage: T linksection(".data.percpu") = init; + + /// Return a pointer to local CPU data. + pub fn local() @TypeOf(ki.impl.percpu_ptr(&storage)) { + return ki.impl.percpu_ptr(&storage); + } + + /// Return a pointer to remote CPU data. + pub fn remote(cpu: *ke.Cpu) *T { + return ki.impl.percpu_ptr_other(&storage, cpu.id); + } + }; +} diff --git a/src/kern/dpc.zig b/src/kern/dpc.zig index a042491..bc010ab 100644 --- a/src/kern/dpc.zig +++ b/src/kern/dpc.zig @@ -28,26 +28,45 @@ pub const Dpc = struct { } }; +const PerCpu = struct { + /// Queue of DPCs on this CPU. + queue: rtl.List, + /// Lock over this CPU's DPC queue. + lock: ke.SpinLock, +}; + +const pcpu = ke.CpuLocal(PerCpu, .{ + .lock = .init(), + .queue = undefined, +}); + +export const dpc_percpu_init linksection(b.percpu_init) = &pcpu_init; + +fn pcpu_init() linksection(b.init) callconv(.c) void { + pcpu.local().queue.init(); +} + /// Enqueue a DPC. /// If the DPC is already enqueued, this function is a no-op. pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void { const ipl = ke.ipl.raise(.High); const cpu = ke.curcpu(); + const dpc_cpu = pcpu.local(); if (!dpc.inserted) { dpc.arg = arg; // Insert the DPC on this CPU's DPC queue - cpu.dpc_lock.acquire_no_ipl(); + dpc_cpu.lock.acquire_no_ipl(); - cpu.dpc_queue.insert_tail(&dpc.link); + dpc_cpu.queue.insert_tail(&dpc.link); // Mark the DPC as pending on this CPU ki.ipl.set_softint_pending(cpu, .Dispatch); dpc.inserted = true; - cpu.dpc_lock.release_no_ipl(); + dpc_cpu.lock.release_no_ipl(); } ke.ipl.lower(ipl); @@ -55,6 +74,7 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void { fn dispatch_queue(cpu: *ke.Cpu) void { cpu.ipl = .Dispatch; + const dpc_cpu = pcpu.remote(cpu); // Mark as handled, this must be done before enabling interrupts or we will race. ki.ipl.clear_softint_pending(cpu, .Dispatch); @@ -62,18 +82,18 @@ fn dispatch_queue(cpu: *ke.Cpu) void { _ = ki.impl.enable_interrupts(); while (true) { - const ipl = cpu.dpc_lock.acquire_at(.High); + const ipl = dpc_cpu.lock.acquire_at(.High); var dpc: *Dpc = undefined; - if (cpu.dpc_queue.is_empty()) { + if (dpc_cpu.queue.is_empty()) { // No more DPCs to process. - cpu.dpc_lock.release(ipl); + dpc_cpu.lock.release(ipl); break; } // Pop the head - var first_elem = cpu.dpc_queue.first(); + var first_elem = dpc_cpu.queue.first(); first_elem.remove(); @@ -84,7 +104,7 @@ fn dispatch_queue(cpu: *ke.Cpu) void { // which is why we capture it here to ensure that we get the intended context. const arg = dpc.arg; - cpu.dpc_lock.release(ipl); + dpc_cpu.lock.release(ipl); std.debug.assert(cpu.ipl == .Dispatch); @@ -120,7 +140,7 @@ pub fn dispatch(cpu: *ke.Cpu) void { const old_ipl = cpu.ipl; const int_state = ki.impl.disable_interrupts(); - while (ki.ipl.is_softint_pending(mycpu, .Dispatch)) { + while (ki.ipl.is_softint_pending(.Dispatch)) { dispatch_queue(mycpu); // Our CPU might have changed. diff --git a/src/kern/init.zig b/src/kern/init.zig index db2d908..61ce479 100644 --- a/src/kern/init.zig +++ b/src/kern/init.zig @@ -43,12 +43,12 @@ pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void { _ = ke.ipl.raise(.Dispatch); - for (0..10) |i| { + for (0..threads.len) |i| { make_thread(handler, &threads[i]); ke.sched.enqueue(&threads[i]); } - ke.ipl.lower(.Zero); + ke.ipl.lower(.Passive); while (true) {} } diff --git a/src/kern/ipl.zig b/src/kern/ipl.zig index 79584b5..d0bd87b 100644 --- a/src/kern/ipl.zig +++ b/src/kern/ipl.zig @@ -6,7 +6,7 @@ const ki = ke.private; /// Interrupt priority level (IPL) pub const Ipl = enum(u8) { /// All interrupts enabled. - Zero = 0, + Passive = 0, /// Preemption disabled. Dispatch = 1, /// Device interrupts disabled. @@ -19,9 +19,14 @@ pub const Ipl = enum(u8) { } }; +/// Bitmask of pending software interrupts on this CPU. +const pending_softints = ke.CpuLocal(std.atomic.Value(u8), .init(0)); + /// Raise IPL to `new`. /// Panics if the current IPL is higher than requested. pub fn raise(new: Ipl) Ipl { + const ints = ki.impl.disable_interrupts(); + var cpu = ke.curcpu(); const old = cpu.ipl; @@ -35,12 +40,15 @@ pub fn raise(new: Ipl) Ipl { cpu.ipl = new; + ki.impl.restore_interrupts(ints); + return old; } /// Lower IPL to `new`. /// Panics if the current IPL is lower than requested. pub fn lower(new: Ipl) void { + const ints = ki.impl.disable_interrupts(); var cpu = ke.curcpu(); const old = cpu.ipl; @@ -49,21 +57,23 @@ pub fn lower(new: Ipl) void { } if (@intFromEnum(new) <= @intFromEnum(Ipl.get_max_software()) and @intFromEnum(old) > @intFromEnum(Ipl.get_max_software())) { - ki.impl.set_hardware_ipl(.Zero); - } - - if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and is_softint_pending(cpu, .Dispatch)) { - ki.dpc.dispatch(cpu); + ki.impl.set_hardware_ipl(.Passive); } cpu.ipl = new; + + ki.impl.restore_interrupts(ints); + + if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and is_softint_pending(.Dispatch)) { + ki.dpc.dispatch(cpu); + } } comptime { if (!@hasDecl(ki.impl, "set_hardware_ipl")) @compileError("impl must provide set_hardware_ipl()"); } -/// Set the hardware IPL to `new` +/// Set the hardware IPL to `new`. pub fn set_hardware(new: Ipl) Ipl { var cpu = ke.curcpu(); const old = cpu.ipl; @@ -75,22 +85,22 @@ pub fn set_hardware(new: Ipl) Ipl { return old; } -/// Mark a software interrupt of IPL `ipl` on `cpu` as pending +/// Mark a software interrupt of IPL `ipl` on `cpu` as pending. pub fn set_softint_pending(cpu: *ke.Cpu, ipl: Ipl) void { std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch)); - _ = cpu.pending_softints.bitSet(@intCast(@intFromEnum(ipl)), .monotonic); + _ = pending_softints.remote(cpu).bitSet(@intCast(@intFromEnum(ipl)), .monotonic); } -/// Mark a software interrupt of IPL `ipl` on `cpu` as handled +/// Mark a software interrupt of IPL `ipl` on `cpu` as handled. pub fn clear_softint_pending(cpu: *ke.Cpu, ipl: Ipl) void { std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch)); - _ = cpu.pending_softints.bitReset(@intCast(@intFromEnum(ipl)), .monotonic); + _ = pending_softints.remote(cpu).bitReset(@intCast(@intFromEnum(ipl)), .monotonic); } -/// Check whether a software interrupt of IPL `ipl` on `cpu` is pending -pub fn is_softint_pending(cpu: *ke.Cpu, ipl: Ipl) bool { +/// Check whether a software interrupt of IPL `ipl` on the current CPU is pending. +pub fn is_softint_pending(ipl: Ipl) bool { std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch)); const shift: u3 = @intCast(@intFromEnum(ipl)); const bit: u8 = @as(u8, 1) << shift; - return cpu.pending_softints.load(.monotonic) & bit != 0; + return pending_softints.local().load(.monotonic) & bit != 0; } diff --git a/src/kern/root.zig b/src/kern/root.zig index 458cd53..3417576 100644 --- a/src/kern/root.zig +++ b/src/kern/root.zig @@ -53,12 +53,13 @@ pub const wait = struct { }; pub const Cpu = p.cpu.Cpu; +pub const CpuLocal = p.cpu.CpuLocal; comptime { if (!@hasDecl(p.impl, "curcpu")) @compileError("impl must provide curcpu()"); } -/// Returns the current CPU +/// Return the current CPU pub const curcpu: fn () *Cpu = p.impl.curcpu; /// Number of CPUs on the system diff --git a/src/kern/sched.zig b/src/kern/sched.zig index a3d7373..6b15b52 100644 --- a/src/kern/sched.zig +++ b/src/kern/sched.zig @@ -103,6 +103,11 @@ pub const RunQueue = struct { queues: [runqueues_n]rtl.List, }; +const ThreadStatus = packed struct(u8) { + interactive: bool, + priority: u7, +}; + /// Per-CPU scheduler data. pub const PerCpu = struct { /// Current load on this CPU (number of ready threads) @@ -121,12 +126,17 @@ pub const PerCpu = struct { insidx: u8, /// Used to indicate to this CPU it can steal work steal_work: bool, + /// Atomic byte for getting the current thread's status locklessly. + /// 1 bit for thread interactivity, 7 for priority. + current_thread_status: std.atomic.Value(ThreadStatus), }; +const percpu_data = ke.CpuLocal(PerCpu, undefined); + /// Handle preemption. /// This is called in DPC dispatch when it notices `next_thread` is set. pub fn handle_preemption(cpu: *ke.Cpu) void { - const sched_cpu = &cpu.sched_data; + const sched_cpu = percpu_data.remote(cpu); sched_cpu.queues_lock.acquire_no_ipl(); const cur = cpu.current_thread.?; @@ -153,10 +163,10 @@ pub fn handle_preemption(cpu: *ke.Cpu) void { // cur.lock dropped } -/// Called every time slice. +/// Called every time slice in DPC context. pub fn clock(_: ?*anyopaque) void { const cpu = ke.curcpu(); - const sched_cpu = &cpu.sched_data; + const sched_cpu = percpu_data.remote(cpu); std.debug.assert(cpu.ipl == .Dispatch); const curtd = cpu.current_thread orelse return; @@ -238,8 +248,12 @@ pub fn unblock(td: *ke.Thread) void { td.lock.release(ipl); } +export const sched_percpu_init linksection(b.percpu_init) = &init_cpu; + /// Initialize a CPU for use by the scheduler. -pub fn init_cpu(percpu: *PerCpu) linksection(b.init) void { +fn init_cpu() linksection(b.init) callconv(.c) void { + var percpu = percpu_data.local(); + percpu.* = .{ .insidx = 0, .runidx = 0, @@ -249,6 +263,7 @@ pub fn init_cpu(percpu: *PerCpu) linksection(b.init) void { .idle_queue = undefined, .steal_work = false, .load = .init(0), + .current_thread_status = .init(.{ .interactive = false, .priority = 0 }), }; percpu.idle_queue.init(); @@ -593,13 +608,13 @@ fn should_preempt(td: *ke.Thread, cur: *ke.Thread) bool { // Return whether or not `td` should preempt the thread running on `cpu`. // This is done in a lock-free manner. fn should_preempt_cpu(td: *ke.Thread, cpu: *ke.Cpu) bool { - const status = cpu.current_thread_status.load(.monotonic); + const status = percpu_data.remote(cpu).current_thread_status.load(.monotonic); return should_prio_preempt(td, status.priority, status.interactive); } // Enqueue a thread on a CPU. fn enqueue_on_cpu(cpu: *ke.Cpu, td: *ke.Thread) void { - var c = &cpu.sched_data; + var c = percpu_data.local(); c.queues_lock.acquire_no_ipl(); @@ -669,13 +684,14 @@ fn pick_cpu(td: *ke.Thread) *ke.Cpu { for (0..ke.ncpus) |i| { const cpu = ke.cpus[i]; + const data = percpu_data.remote(cpu); - if (least == null or cpu.sched_data.load.load(.monotonic) < least.?.sched_data.load.load(.monotonic)) { + if (least == null or data.load.load(.monotonic) < percpu_data.remote(least.?).load.load(.monotonic)) { least = cpu; } if (should_preempt_cpu(td, cpu)) { - if (least_preempt == null or cpu.sched_data.load.load(.monotonic) < least_preempt.?.sched_data.load.load(.monotonic)) { + if (least_preempt == null or data.load.load(.monotonic) < percpu_data.remote(least_preempt.?).load.load(.monotonic)) { least_preempt = cpu; } } @@ -687,7 +703,7 @@ fn pick_cpu(td: *ke.Thread) *ke.Cpu { // Find a thread to preempt `cur` with. fn select_thread(cur: ?*ke.Thread, cpu: *ke.Cpu, migrate: bool) ?*ke.Thread { - const sched_cpu = &cpu.sched_data; + const sched_cpu = percpu_data.remote(cpu); // First try to pick from the realtime queues. if (pick_realtime_thread(sched_cpu, if (cur) |c| c.priority else 0, migrate)) |td| { @@ -728,7 +744,7 @@ fn do_switch(cpu: *ke.Cpu, cur: *ke.Thread, next: *ke.Thread) void { next.last_cpu = cpu; // Stash thread status. - cpu.current_thread_status.store(.{ .interactive = next.interactive, .priority = @intCast(next.priority) }, .monotonic); + percpu_data.remote(cpu).current_thread_status.store(.{ .interactive = next.interactive, .priority = @intCast(next.priority) }, .monotonic); // Do machine-dependent switch. cur.context.switch_to(&next.context); @@ -736,14 +752,14 @@ fn do_switch(cpu: *ke.Cpu, cur: *ke.Thread, next: *ke.Thread) void { // Yield on `cpu` fn yield(cpu: *ke.Cpu) void { - const sched_cpu = &cpu.sched_data; + const sched_cpu = percpu_data.remote(cpu); sched_cpu.queues_lock.acquire_no_ipl(); const cur = cpu.current_thread; var next = cpu.next_thread; - if (next) { + if (next != null) { cpu.next_thread = null; } else { // Pick a new thread to run. @@ -758,7 +774,7 @@ fn yield(cpu: *ke.Cpu) void { sched_cpu.queues_lock.release_no_ipl(); // Switch into the thread. - do_switch(cpu, cur, next); + do_switch(cpu, cur.?, next.?); // cur lock dropped } diff --git a/src/kern/timer.zig b/src/kern/timer.zig index e703b1f..ca5550e 100644 --- a/src/kern/timer.zig +++ b/src/kern/timer.zig @@ -7,6 +7,14 @@ const ke = b.ke; const ki = ke.private; const pl = b.pl; +const PerCpu = struct { + /// Heap of pending timers on this CPU. + timers: rtl.PairingHeap(.min, ki.timer.cmp_timer), + /// Lock over the timer heap. + lock: ke.SpinLock, + dpc: ke.Dpc, +}; + pub const Timer = struct { pub const State = enum(u8) { /// The timer is currently being handled. @@ -28,7 +36,7 @@ pub const Timer = struct { /// Intrusive pairing heap node. node: rtl.pairing_heap.Node, /// CPU this timer is enqueued on. - cpu: ?*ke.Cpu, + cpu: ?*PerCpu, pub fn init(self: *Timer) void { self.* = .{ @@ -44,21 +52,27 @@ pub const Timer = struct { } }; +const percpu = ke.CpuLocal(PerCpu, .{ + .timers = .init(), + .lock = .init(), + .dpc = .init(handle_expiry), +}); + /// Start a timer with an expiration time. /// A DPC that will be enqueued upon expiration can be passed. pub fn set(timer: *Timer, time: b.Nanoseconds, dpc: ?*ke.Dpc) void { const ipl = timer.hdr.lock.acquire(); defer timer.hdr.lock.release(ipl); - const cpu = ke.curcpu(); + const cpu = percpu.local(); if (timer.state.cmpxchgStrong(.Stopped, .Pending, .acquire, .monotonic) != null) { // Could not set the timer. return; } - cpu.timers_lock.acquire_no_ipl(); - defer cpu.timers_lock.release_no_ipl(); + cpu.lock.acquire_no_ipl(); + defer cpu.lock.release_no_ipl(); // Initialize the timer. timer.deadline = ke.timecounter.read_time_nano() + time; @@ -67,9 +81,9 @@ pub fn set(timer: *Timer, time: b.Nanoseconds, dpc: ?*ke.Dpc) void { timer.dpc = dpc; timer.hdr.signaled = 0; - cpu.timers_heap.insert(&timer.node); + cpu.timers.insert(&timer.node); - if (cpu.timers_heap.root == &timer.node) { + if (cpu.timers.root == &timer.node) { // This is the earliest timer to expire, arm the hardware timer. pl.arm_timer(time); } @@ -96,12 +110,12 @@ pub fn cancel(timer: *Timer) void { if (timer.cpu) |cpu| { // Dequeue the timer. - cpu.timers_lock.acquire_no_ipl(); + cpu.lock.acquire_no_ipl(); - cpu.timers_heap.remove(&timer.node); + cpu.timers.remove(&timer.node); timer.cpu = null; - cpu.timers_lock.release_no_ipl(); + cpu.lock.release_no_ipl(); } // Lock dropped @@ -115,19 +129,24 @@ pub fn cmp_timer(a: *rtl.pairing_heap.Node, b_: *rtl.pairing_heap.Node) std.math return std.math.order(timer_a.deadline, timer_b.deadline); } -/// Called in a DPC when a timer has expired. -pub fn handle_expiry(_: ?*anyopaque) void { - const cpu = ke.curcpu(); - std.debug.assert(cpu.ipl == .Dispatch); +/// Called by the platform on a clock interrupt. +pub fn clock() void { + ke.dpc.enqueue(&percpu.local().dpc, null); +} + +// Called in a DPC when a timer has expired. +fn handle_expiry(_: ?*anyopaque) void { + std.debug.assert(ke.curcpu().ipl == .Dispatch); + const cpu = percpu.local(); while (true) { const curtime = ke.timecounter.read_time_nano(); - cpu.timers_lock.acquire_no_ipl(); + cpu.lock.acquire_no_ipl(); // Get the timer that expires the soonest. - const timer_node = cpu.timers_heap.root orelse { - cpu.timers_lock.release_no_ipl(); + const timer_node = cpu.timers.root orelse { + cpu.lock.release_no_ipl(); return; }; @@ -137,7 +156,7 @@ pub fn handle_expiry(_: ?*anyopaque) void { // Sub-millisecond differences are close enough to expire immediately. if (timer.deadline > curtime and timer.deadline - curtime > std.time.ns_per_ms) { const next = timer; - cpu.timers_lock.release_no_ipl(); + cpu.lock.release_no_ipl(); // Re-check in case the timer expired while we were here. const now = ke.timecounter.read_time_nano(); @@ -150,8 +169,8 @@ pub fn handle_expiry(_: ?*anyopaque) void { return; } - _ = cpu.timers_heap.pop(); - cpu.timers_lock.release_no_ipl(); + _ = cpu.timers.pop(); + cpu.lock.release_no_ipl(); if (timer.state.cmpxchgStrong(.Pending, .Running, .acquire, .monotonic) != null) { // Timer must've been canceled. diff --git a/src/kern/um/impl.zig b/src/kern/um/impl.zig index f9b6fd0..86c8b86 100644 --- a/src/kern/um/impl.zig +++ b/src/kern/um/impl.zig @@ -60,7 +60,7 @@ pub const ThreadContext = struct { fn real_entry(entry_int: c_ulong, arg_int: c_ulong) callconv(.c) void { const entry: *const fn (?*anyopaque) void = @ptrFromInt(entry_int); const arg: ?*anyopaque = @ptrFromInt(arg_int); - ke.ipl.lower(.Zero); + ke.ipl.lower(.Passive); entry(arg); } @@ -104,3 +104,11 @@ pub const ThreadContext = struct { } } }; + +pub inline fn percpu_ptr_other(variable: anytype, cpu_id: usize) @TypeOf(variable) { + return @ptrFromInt(@intFromPtr(variable) +% b.pl.impl.cpu_offsets[cpu_id]); +} + +pub inline fn percpu_ptr(variable: anytype) @TypeOf(variable) { + return @ptrFromInt(@intFromPtr(variable) +% b.pl.impl.cpu_offsets[b.pl.impl.my_cpu.id]); +} diff --git a/src/kern/wait.zig b/src/kern/wait.zig index 5a9968b..f83eddf 100644 --- a/src/kern/wait.zig +++ b/src/kern/wait.zig @@ -74,7 +74,8 @@ pub const Status = enum(u8) { /// Wait for the provided object to be signaled. pub fn wait_one(object: *DispatchHeader, timeout: ?b.Nanoseconds) !usize { - return wait_any(object, timeout, null); + var objects = [_]*DispatchHeader{object}; + return wait_any(&objects, timeout, null); } /// Wait for any of the provided objects to be signaled. @@ -85,7 +86,7 @@ pub fn wait_one(object: *DispatchHeader, timeout: ?b.Nanoseconds) !usize { pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks: ?[*]WaitBlock) !usize { const ipl = ke.ipl.raise(.Dispatch); defer ke.ipl.lower(ipl); - const curtd = ke.curcpu().current_thread; + const curtd = ke.curcpu().current_thread.?; const obj_count = objects.len; const has_timeout = timeout != null; const blocks = waitblocks orelse blk: { @@ -99,7 +100,7 @@ pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks: const timer_block = &curtd.waitblocks[3]; var satisfier: ?usize = null; - curtd.wait_status.store(.InProgress, .acquire); + curtd.wait_status.store(.InProgress, .release); for (0..total_count) |i| { const is_timer = has_timeout and i == timer_i; @@ -111,7 +112,7 @@ pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks: if (obj.signaled > 0) { // Object was already signaled. Try consuming it. - if (curtd.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acquire, .monotonic) == null) { + if (curtd.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acq_rel, .monotonic) == null) { obj.consume(); } // We have already been satisfied in the meantime, abort. @@ -128,18 +129,20 @@ pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks: // Wait was already satisfied, back out. if (satisfier != null or (has_timeout and timeout.? == 0)) { - std.debug.assert(curtd.wait_status.load(.monotonic) == .Satisfied); + std.debug.assert(curtd.wait_status.load(.acquire) == .Satisfied); - // Remove any wait block we might've installed - for (0..satisfier) |i| { - const is_timer = has_timeout and i == timer_i; - var obj = if (is_timer) &timer.hdr else objects[i]; - var wb = if (is_timer) timer_block else &blocks[i]; + if (satisfier) |sat| { + // Remove any wait block we might've installed + for (0..sat) |i| { + const is_timer = has_timeout and i == timer_i; + var obj = if (is_timer) &timer.hdr else objects[i]; + var wb = if (is_timer) timer_block else &blocks[i]; - obj.lock.acquire_no_ipl(); - wb.status = .Inactive; - wb.link.remove(); - obj.lock.release_no_ipl(); + obj.lock.acquire_no_ipl(); + wb.status = .Inactive; + wb.link.remove(); + obj.lock.release_no_ipl(); + } } return satisfier orelse error.Timeout; @@ -152,7 +155,7 @@ pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks: // Now try committing the wait. // While we're trying to commit the wait, the object locks have been released, and the state could therefore change. // We need to re-check the state of the wait before actually blocking. - if (curtd.wait_status.cmpxchgStrong(.InProgress, .Committed, .acquire, .monotonic) == null) { + if (curtd.wait_status.cmpxchgStrong(.InProgress, .Committed, .acq_rel, .monotonic) == null) { // We're good, now actually block. ke.sched.block(); } @@ -174,7 +177,7 @@ pub fn wait_any(objects: []*DispatchHeader, timeout: ?b.Nanoseconds, waitblocks: // Waitblock is still active, remove it from the list. wb.link.remove(); } - if (wb.status == .Signaled or wb.status == .SignaledInProgress) { + if (wb.status == .Signaled) { // This waitblock was signaled, it must be the one that satisfied the wait. std.debug.assert(satisfier == null); satisfier = i; @@ -206,14 +209,14 @@ pub fn satisfy_wait(obj: *DispatchHeader) void { // 3. The wait was already satisfied by another object (status == .Satisfied). // 1. - if (td.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acquire, .monotonic) == null) { + if (td.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acq_rel, .monotonic) == null) { // We interrupted the wait while it was being prepared. wb.status = .Signaled; obj.consume(); } // 2. - else if (td.wait_status.cmpxchgStrong(.Committed, .Satisfied, .acquire, .monotonic) == null) { + else if (td.wait_status.cmpxchgStrong(.Committed, .Satisfied, .acq_rel, .monotonic) == null) { // We interrupted the wait while it was committed. // Wake the thread. wb.status = .Signaled; diff --git a/src/platform/um/impl.zig b/src/platform/um/impl.zig index 70a1e33..764ec70 100644 --- a/src/platform/um/impl.zig +++ b/src/platform/um/impl.zig @@ -56,8 +56,15 @@ fn check_function(fn_name: []const u8, ret: usize) void { } } +const bootstrap_offsets = [1]usize{0}; + +pub var cpu_offsets: []usize = undefined; + pub fn early_init() linksection(b.init) void { - // Set current CPU to the bootstrap CPU + // Ensure we can use per-cpu data. + cpu_offsets = @constCast(&bootstrap_offsets); + + // Set current CPU to the bootstrap CPU. my_cpu = &ki.bootstrap_cpu; my_cpu.impl.pthread = c.pthread_self(); @@ -96,7 +103,7 @@ pub fn early_init() linksection(b.init) void { } fn sigusr1_handler(_: posix.SIG, _: *const posix.siginfo_t, _: ?*anyopaque) callconv(.c) void { - if (@intFromEnum(ke.curcpu().ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(ke.curcpu(), .Dispatch)) { + if (@intFromEnum(ke.curcpu().ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(.Dispatch)) { ki.dpc.dispatch(ke.curcpu()); } } @@ -128,8 +135,13 @@ fn other_cpu_entry(arg: ?*anyopaque) callconv(.c) ?*anyopaque { return null; } +var cpu: ke.Cpu linksection(b.percpu) = undefined; + fn idle(_: ?*anyopaque) void {} +extern var __percpu_start: u8; +extern var __percpu_end: u8; + pub fn late_init() linksection(b.init) void { timer.init(); timer.init_cpu(); @@ -158,10 +170,19 @@ pub fn late_init() linksection(b.init) void { allocator.alloc(c.pthread_t, other_count) catch @panic("oom"); ke.cpus = allocator.alloc(*ke.Cpu, ke.ncpus) catch @panic("oom"); + cpu_offsets = allocator.alloc(usize, ke.ncpus) catch @panic("oom"); ke.cpus[0] = &ki.bootstrap_cpu; + cpu_offsets[0] = 0; + + const size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start); for (1..ke.ncpus) |i| { - ke.cpus[i] = allocator.create(ke.Cpu) catch @panic("oom"); + const block = allocator.alloc(u8, size) catch @panic("oom"); + cpu_offsets[i] = @intFromPtr(block.ptr) -% @intFromPtr(&__percpu_start); + + ke.cpus[i] = ki.impl.percpu_ptr_other(&cpu, i); + + @memcpy(block, @as([*]u8, @ptrCast(&__percpu_start))[0..size]); ke.cpus[i].idle_thread = allocator.create(ke.Thread) catch @panic("oom"); ke.cpus[i].id = @intCast(i); diff --git a/src/platform/um/timer.zig b/src/platform/um/timer.zig index 96df68f..6b641ed 100644 --- a/src/platform/um/timer.zig +++ b/src/platform/um/timer.zig @@ -11,11 +11,11 @@ threadlocal var timerid: c.timer_t = undefined; fn timer_handler(_: posix.SIG, _: *const posix.siginfo_t, _: ?*anyopaque) callconv(.c) void { const old_ipl = ke.ipl.set_hardware(.Device); - ke.dpc.enqueue(&ke.curcpu().timer_dpc, null); + ki.timer.clock(); _ = ke.ipl.set_hardware(old_ipl); - if (@intFromEnum(old_ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(ke.curcpu(), .Dispatch)) { + if (@intFromEnum(old_ipl) < @intFromEnum(ke.Ipl.Dispatch) and ki.ipl.is_softint_pending(.Dispatch)) { ki.dpc.dispatch(ke.curcpu()); } } diff --git a/src/root.zig b/src/root.zig index fbe1bb6..6041910 100644 --- a/src/root.zig +++ b/src/root.zig @@ -4,7 +4,9 @@ pub const pl = @import("platform/root.zig"); pub const rtl = @import("rtl"); pub const ksyms = @import("ksyms"); -pub const init = ".init.text"; +pub const init = ".text.init"; +pub const percpu_init = ".percpu_init_array"; +pub const percpu = ".data.percpu"; pub const VAddr = usize; pub const PAddr = usize;