From 4aaae633494a8062c91d8724e7b90ab2f9a0a86b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Gr=C3=A9goire?= Date: Mon, 3 Aug 2026 20:00:45 -0400 Subject: [PATCH] ke: add generic IPI mechanism --- src/kern/amd64/impl.zig | 8 --- src/kern/amd64/int.zig | 5 +- src/kern/ipi.zig | 140 +++++++++++++++++++++++++++++++++++++++ src/kern/private.zig | 12 +--- src/kern/root.zig | 5 ++ src/kern/sched.zig | 5 +- src/kern/shootdown.zig | 36 +++------- src/platform/pc/impl.zig | 8 +-- src/platform/root.zig | 28 ++++++++ 9 files changed, 193 insertions(+), 54 deletions(-) create mode 100644 src/kern/ipi.zig diff --git a/src/kern/amd64/impl.zig b/src/kern/amd64/impl.zig index 2656131..7bd812c 100644 --- a/src/kern/amd64/impl.zig +++ b/src/kern/amd64/impl.zig @@ -113,14 +113,6 @@ pub inline fn restore_interrupts(state: bool) void { } } -pub inline fn send_resched_ipi(target: u32) void { - pl.impl.send_resched_ipi(target); -} - -pub inline fn send_tlb_ipi(target: u32) void { - pl.impl.send_tlb_ipi(target); -} - pub inline fn flush_full_tlb() void { const prev_cr3 = amd64.read_cr(3); // Reload CR3 to flush the entire TLB. diff --git a/src/kern/amd64/int.zig b/src/kern/amd64/int.zig index 33ba97a..7c8fdf5 100644 --- a/src/kern/amd64/int.zig +++ b/src/kern/amd64/int.zig @@ -85,8 +85,9 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys ki.timer.clock(); } - if (frame.intno == 241) { - ki.shootdown.ipi_handler(); + // Platform IPI number. + if (frame.intno == 240) { + ki.ipi.ipi_handler(); } if (frame.intno >= 32) { diff --git a/src/kern/ipi.zig b/src/kern/ipi.zig new file mode 100644 index 0000000..f7d524c --- /dev/null +++ b/src/kern/ipi.zig @@ -0,0 +1,140 @@ +//! Generic cross-core call mechanisms. + +const r = @import("root"); +const std = @import("std"); + +const ke = r.ke; +const ki = r.ki; +const pl = r.pl; + +const State = enum(u8) { + /// Not currently processing IPIs. + Idle = 0, + /// Has IPIs pending. + Pending = 1, + /// Is currently processing IPIs. + Running = 2, +}; + +const PerCpu = struct { + /// Function to be called on other CPUs. + func: *const fn (u32, ?*anyopaque) void, + /// Argument passed to the function. + arg: ?*anyopaque, + /// Pending work from other CPUs. + pending: ke.AtomicCpuMask, + /// Pending CPUs. + counter: std.atomic.Value(u16), + /// IPI state. + state: std.atomic.Value(State), +}; + +const percpu = ke.CpuLocal(PerCpu, undefined); + +/// Call a function on another CPU. +/// The function will be executed synchronously, i.e this will block until +/// the function finishes running. The callback is called with the sender cpu +/// and provided argument as arguments. +pub fn unicast(cpu: u32, func: *const fn (u32, ?*anyopaque) void, arg: ?*anyopaque) void { + const ipl = ke.ipl.raise(.Dispatch); + const curcpu = percpu.local(); + const remote = percpu.remote(cpu); + + curcpu.func = func; + curcpu.arg = arg; + curcpu.counter.store(1, .monotonic); + + remote.pending.set(ke.cpu.current(), .monotonic); + + const state = remote.state.swap(.Pending, .release); + + if (state == .Idle) { + std.log.info("Sent ipi", .{}); + pl.send_ipi(cpu); + } + + while (curcpu.counter.load(.acquire) != 0) { + std.atomic.spinLoopHint(); + } + + ke.ipl.lower(ipl); +} + +pub fn broadcast(func: *const fn (u32, ?*anyopaque) void, arg: ?*anyopaque) void { + const ipl = ke.ipl.raise(.High); + const curcpu = percpu.local(); + + curcpu.func = func; + curcpu.arg = arg; + curcpu.counter = .init(@truncate(ke.ncpus - 1)); + + for (0..ke.ncpus) |i| { + if (i == ke.cpu.current()) continue; + const remote = percpu.remote(@intCast(i)); + + remote.pending.set(ke.cpu.current(), .monotonic); + + const state = remote.state.swap(.Pending, .release); + + if (state == .Idle) { + pl.send_ipi(@intCast(i)); + } + } + + while (curcpu.counter.load(.acquire) != 0) { + std.atomic.spinLoopHint(); + } + + ke.ipl.lower(ipl); +} + +pub fn ipi_handler() void { + const curcpu = percpu.local(); + + while (true) { + // Mark the current CPU as processing IPIs, if the CAS fails here, + // then it's probably some kind of IPI that was sent to force us to check + // software interrupts (like a rescheduling IPI). + const val = curcpu.state.cmpxchgStrong( + .Pending, + .Running, + .acquire, + .monotonic, + ); + + if (val != null) { + // CAS failed. + return; + } + + const words = curcpu.pending.storage.len; + + for (0..words) |i| { + var word = curcpu.pending.storage[i].swap(0, .monotonic); + + // Go through every CPU in our pending mask. + while (word != 0) { + const bit = @ctz(word); + const cpu = @bitSizeOf(usize) * i + bit; + + // Call its function and decrement the counter. + const remote = percpu.remote(@intCast(cpu)); + remote.func(@intCast(cpu), remote.arg); + + // Ensure all previous memory operations complete. + _ = remote.counter.fetchSub(1, .release); + + word &= (word - 1); + } + } + + // Try to make the CPU state go from running to idle, if this fails + // then we still have work to do. + _ = curcpu.state.cmpxchgStrong( + .Running, + .Idle, + .monotonic, + .monotonic, + ) orelse return; + } +} diff --git a/src/kern/private.zig b/src/kern/private.zig index 5d28894..46385d3 100644 --- a/src/kern/private.zig +++ b/src/kern/private.zig @@ -28,17 +28,6 @@ const ImplSchema = struct { /// Read a per-cpu pointer on the current CPU. pub fn percpu_ptr(variable: anytype) @TypeOf(variable) {} - /// Send a rescheduling inter-processor interrupt (IPI) - /// on the target CPU. - pub fn send_resched_ipi(target: u32) void { - _ = target; - } - - /// Send a TLB shootdown IPI on the target CPU. - pub fn send_tlb_ipi(target: u32) void { - _ = target; - } - /// Flush the entire TLB on the current CPU. pub fn flush_full_tlb() void {} @@ -91,3 +80,4 @@ pub const queue = @import("queue.zig"); pub const shootdown = @import("shootdown.zig"); pub const tunable = @import("tunable.zig"); pub const smr = @import("smr.zig"); +pub const ipi = @import("ipi.zig"); diff --git a/src/kern/root.zig b/src/kern/root.zig index 94fdb54..e2a775d 100644 --- a/src/kern/root.zig +++ b/src/kern/root.zig @@ -103,5 +103,10 @@ pub const smr = struct { pub const Sequence = p.smr.Sequence; }; +pub const ipi = struct { + pub const unicast = p.ipi.unicast; + pub const broadcast = p.ipi.broadcast; +}; + /// Number of CPUs on the system pub var ncpus: usize = 0; diff --git a/src/kern/sched.zig b/src/kern/sched.zig index 5d33f72..142b88b 100644 --- a/src/kern/sched.zig +++ b/src/kern/sched.zig @@ -95,6 +95,7 @@ const config = @import("config"); const r = @import("root"); const ke = r.ke; const ki = ke.private; +const pl = r.pl; const runqueues_n = 64; const interactivity_threshold = 30; @@ -624,7 +625,7 @@ pub fn update_priority_locked(td: *ke.Thread, new_prio: u8) void { ki.ipl.set_softint_pending(c, .Dispatch); if (c != ke.cpu.current()) { - ki.impl.send_resched_ipi(c); + pl.send_ipi(c); } } @@ -1101,7 +1102,7 @@ fn enqueue_on_cpu(c: u32, td: *ke.Thread) void { if (c != ke.cpu.current()) { // Send an IPI. - ki.impl.send_resched_ipi(c); + pl.send_ipi(c); } // queues_lock dropped diff --git a/src/kern/shootdown.zig b/src/kern/shootdown.zig index dc7efe1..6317d99 100644 --- a/src/kern/shootdown.zig +++ b/src/kern/shootdown.zig @@ -7,6 +7,7 @@ const r = @import("root"); const ke = r.ke; const ki = ke.private; const mm = r.mm; +const pl = r.pl; /// Structure representing a single shootdown request. pub const ShootdownState = struct { @@ -36,6 +37,9 @@ const PerCpu = struct { valid_states: [config.ncpus]std.atomic.Value(u64), /// CPUs that have sent this CPU a shootdown. senders: ke.AtomicCpuMask, + + sync_addr: r.VAddr, + sync_count: usize, }; const slot_free: u16 = std.math.maxInt(u16); @@ -44,11 +48,6 @@ const percpu = ke.CpuLocal(PerCpu, undefined); pub var shootdowns: rtl.HandoffList = undefined; -var sync_counter: std.atomic.Value(usize) = .init(0); -var sync_addr: r.VAddr = 0; -var sync_count: usize = 0; -var shootdown_lock: ke.SpinLock = .init(); - fn pcpu_init() linksection(r.init) void { const local = percpu.local(); @@ -89,30 +88,17 @@ fn flush_range(va: r.VAddr, npages: usize) void { /// Synchronous shootdown path. fn do_synchronous_shootdown(state: ShootdownState) void { - shootdown_lock.acquire_no_ipl(); + const cpu = percpu.local(); - sync_addr = state.base; - sync_count = state.npages; - sync_counter.store(ke.ncpus - 1, .release); + cpu.sync_addr = state.base; + cpu.sync_count = state.npages; - const cur = ke.cpu.current(); - - for (0..ke.ncpus) |i| { - if (i == cur) continue; - ki.impl.send_tlb_ipi(@truncate(i)); - } - - while (sync_counter.load(.monotonic) != 0) { - std.atomic.spinLoopHint(); - } - - shootdown_lock.release_no_ipl(); + ke.ipi.broadcast(flush_tlb, null); } -/// Called by the TLB interrupt handler. -pub fn ipi_handler() void { - flush_range(sync_addr, sync_count); - _ = sync_counter.fetchSub(1, .release); +fn flush_tlb(sender: u32, _: ?*anyopaque) void { + const cpu = percpu.remote(sender); + flush_range(cpu.sync_addr, cpu.sync_count); } /// Called in a quiescent state to process the pending shootdowns. diff --git a/src/platform/pc/impl.zig b/src/platform/pc/impl.zig index 44f97c9..256000b 100644 --- a/src/platform/pc/impl.zig +++ b/src/platform/pc/impl.zig @@ -9,7 +9,7 @@ pub const smp = @import("smp.zig"); const ki = r.ke.private; -pub const name = "PC"; +pub const name: []const u8 = "PC"; const com1 = 0x3F8; @@ -113,10 +113,6 @@ pub fn arm_timer(deadline: r.Nanoseconds) void { apic.arm_timer(deadline); } -pub fn send_resched_ipi(target_cpu: u32) void { +pub fn send_ipi(target_cpu: u32) void { apic.send_ipi(smp.cpu_id_to_apic_id[target_cpu], 240, 0); } - -pub fn send_tlb_ipi(target_cpu: u32) void { - apic.send_ipi(smp.cpu_id_to_apic_id[target_cpu], 241, 0); -} diff --git a/src/platform/root.zig b/src/platform/root.zig index fef43d9..1384f7a 100644 --- a/src/platform/root.zig +++ b/src/platform/root.zig @@ -3,6 +3,7 @@ const config = @import("config"); const r = @import("root"); +const rtl = @import("rtl"); pub const impl = switch (config.arch) { .amd64 => @import("pc/impl.zig"), @@ -11,6 +12,28 @@ pub const impl = switch (config.arch) { pub const acpi = @import("acpi/root.zig"); +const ImplSchema = struct { + pub const name: []const u8 = undefined; + pub fn early_init() void {} + pub fn late_init() void {} + pub fn debug_write(c: u8) void { + _ = c; + } + pub fn debug_read() u8 { + unreachable; + } + pub fn send_ipi(cpu: u32) void { + _ = cpu; + } + pub fn arm_timer(ns: r.Nanoseconds) void { + _ = ns; + } +}; + +comptime { + rtl.assert_interface(impl, ImplSchema); +} + /// Early platform-specific initialization, called before anything else. pub inline fn early_init() void { return impl.early_init(); @@ -37,5 +60,10 @@ pub inline fn arm_timer(ns: r.Nanoseconds) void { return impl.arm_timer(ns); } +/// Send an inter-processor interrupt (IPI) to a target CPU. +pub inline fn send_ipi(cpu: u32) void { + return impl.send_ipi(cpu); +} + /// Pretty name for the platform. pub const name = impl.name;