From 52f62b8354684ae518cb3f392e7fec1d8bb6d2f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Gr=C3=A9goire?= Date: Fri, 7 Aug 2026 15:41:12 -0400 Subject: [PATCH] ke: optimize IPL operations --- src/kern/amd64/impl.zig | 27 ++++++++++++++ src/kern/amd64/int.zig | 6 ++- src/kern/dpc.zig | 19 +++++----- src/kern/ipl.zig | 83 ++++++++++++++++++++++++----------------- src/kern/private.zig | 16 ++++++++ 5 files changed, 105 insertions(+), 46 deletions(-) diff --git a/src/kern/amd64/impl.zig b/src/kern/amd64/impl.zig index 315e56c..a477ad7 100644 --- a/src/kern/amd64/impl.zig +++ b/src/kern/amd64/impl.zig @@ -146,3 +146,30 @@ pub inline fn flush_full_tlb() void { pub inline fn flush_tlb(va: r.VAddr) void { amd64.invlpg(va); } + +// Inline asm so the compiler has a chance to inline it. +pub inline fn raise_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool { + return asm volatile ( + \\movb %%gs:cpu_ipl, %%cl + \\movb %%cl, (%[old_ptr]) + \\movb %[new_val], %%gs:cpu_ipl + \\cmpb %%cl, %[new_val] + \\setl %[result] + : [result] "=r" (-> bool), + : [new_val] "r" (new), + [old_ptr] "r" (old), + : .{ .rcx = true, .memory = true }); +} + +pub inline fn lower_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool { + return asm volatile ( + \\movb %%gs:cpu_ipl, %%cl + \\movb %%cl, (%[old_ptr]) + \\movb %[new_val], %%gs:cpu_ipl + \\cmpb %[new_val], %%cl + \\setl %[result] + : [result] "=r" (-> bool), + : [new_val] "r" (new), + [old_ptr] "r" (old), + : .{ .rcx = true, .memory = true }); +} diff --git a/src/kern/amd64/int.zig b/src/kern/amd64/int.zig index 7c8fdf5..a23100a 100644 --- a/src/kern/amd64/int.zig +++ b/src/kern/amd64/int.zig @@ -96,8 +96,10 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys _ = ke.ipl.set_hardware(ipl); - if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and ki.ipl.is_softint_pending(.Dispatch)) { - ki.dpc.dispatch(ke.cpu.current()); + if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and + ki.ipl.is_softint_pending(.Dispatch)) + { + ki.dpc.dispatch(); } } diff --git a/src/kern/dpc.zig b/src/kern/dpc.zig index d4575ec..cf2cee4 100644 --- a/src/kern/dpc.zig +++ b/src/kern/dpc.zig @@ -72,11 +72,11 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void { } fn dispatch_queue(cpu: u32) void { - const ipl_cpu = ki.ipl.percpu.remote(cpu); - const dpc_cpu = pcpu.remote(cpu); - const sched_cpu = ki.sched.percpu.remote(cpu); + const ipl_cpu = ki.ipl.cpu_ipl.local(); + const dpc_cpu = pcpu.local(); + const sched_cpu = ki.sched.percpu.local(); - ipl_cpu.ipl = .Dispatch; + ipl_cpu.* = .Dispatch; // Mark as handled, this must be done before enabling interrupts or we will // race. @@ -148,13 +148,13 @@ fn dispatch_queue(cpu: u32) void { _ = ki.impl.disable_interrupts(); } -/// Dispatch the DPC queue on `cpu`. -pub fn dispatch(cpu: u32) void { +/// Dispatch the DPC queue on the current CPU. +pub fn dispatch() void { // DPC processing is done at IPL Dispatch. // Don't call lower/raise here because they might call us again recursively. - var mycpu = cpu; - const old_ipl = ki.ipl.percpu.remote(mycpu).ipl; const int_state = ki.impl.disable_interrupts(); + const old_ipl = ki.ipl.cpu_ipl.local().*; + var mycpu = ke.cpu.current(); while (ki.ipl.is_softint_pending(.Dispatch)) { dispatch_queue(mycpu); @@ -162,7 +162,6 @@ pub fn dispatch(cpu: u32) void { mycpu = ke.cpu.current(); } - ki.ipl.percpu.remote(mycpu).ipl = old_ipl; - + ki.ipl.cpu_ipl.local().* = old_ipl; ki.impl.restore_interrupts(int_state); } diff --git a/src/kern/ipl.zig b/src/kern/ipl.zig index f0e6767..ccd4863 100644 --- a/src/kern/ipl.zig +++ b/src/kern/ipl.zig @@ -20,77 +20,85 @@ pub const Ipl = enum(u8) { }; const PerCpu = struct { - /// Current IPL on this CPU. - ipl: Ipl, /// Bitmask of pending software interrupts on this CPU. pending_softints: std.atomic.Value(u8), }; +/// Current IPL on this CPU. +pub const cpu_ipl = ke.ExportedCpuLocal( + Ipl, + .Passive, + "cpu_ipl", +); + pub const percpu = ke.CpuLocal(PerCpu, .{ - .ipl = .Passive, - .pending_softints = std.atomic.Value(u8).init(0), + .pending_softints = .init(0), }); pub fn current() Ipl { - return percpu.local().ipl; + return cpu_ipl.local().*; } /// Raise IPL to `new`. /// Panics if the current IPL is higher than requested. pub fn raise(new: Ipl) Ipl { - const ints = ki.impl.disable_interrupts(); + var old: ke.Ipl = undefined; - var cpu = percpu.local(); - const old = cpu.ipl; - - if (@intFromEnum(new) < @intFromEnum(old)) { - std.debug.panic("ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})", .{ new, old }); + if (ki.impl.raise_software_ipl(new, &old)) { + @branchHint(.unlikely); + std.debug.panic( + "ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})", + .{ new, old }, + ); } if (@intFromEnum(new) > @intFromEnum(Ipl.get_max_software())) { + // Moving between hardware levels is rare. + @branchHint(.unlikely); ki.impl.set_hardware_ipl(new); } - cpu.ipl = new; - - ki.impl.restore_interrupts(ints); - return old; } /// Lower IPL to `new`. /// Panics if the current IPL is lower than requested. pub fn lower(new: Ipl) void { - const ints = ki.impl.disable_interrupts(); - var cpu = percpu.local(); - const cpu_id = ke.cpu.current(); - const old = cpu.ipl; + var old: ke.Ipl = undefined; - if (@intFromEnum(new) > @intFromEnum(old)) { - std.debug.panic("ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})", .{ new, old }); + if (ki.impl.lower_software_ipl(new, &old)) { + @branchHint(.unlikely); + std.debug.panic( + "ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})", + .{ new, old }, + ); } - if (@intFromEnum(new) <= @intFromEnum(Ipl.get_max_software()) and @intFromEnum(old) > @intFromEnum(Ipl.get_max_software())) { + if (@intFromEnum(new) <= @intFromEnum(Ipl.get_max_software()) and + @intFromEnum(old) > @intFromEnum(Ipl.get_max_software())) + { + @branchHint(.unlikely); ki.impl.set_hardware_ipl(.Passive); } - cpu.ipl = new; - - ki.impl.restore_interrupts(ints); - - if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and is_softint_pending(.Dispatch)) { - ki.dpc.dispatch(cpu_id); + if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and + is_softint_pending(.Dispatch)) + { + // Dispatch DPCs if necessary. + ki.dpc.dispatch(); } } /// Set the hardware IPL to `new`. pub fn set_hardware(new: Ipl) Ipl { - var cpu = percpu.local(); - const old = cpu.ipl; + const ints = ki.impl.disable_interrupts(); + const cpu = cpu_ipl.local(); + const old = cpu.*; - cpu.ipl = new; + cpu.* = new; ki.impl.set_hardware_ipl(new); + ki.impl.restore_interrupts(ints); return old; } @@ -98,16 +106,23 @@ pub fn set_hardware(new: Ipl) Ipl { /// Mark a software interrupt of IPL `ipl` on `cpu` as pending. pub fn set_softint_pending(cpu: u32, ipl: Ipl) void { std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch)); - _ = percpu.remote(cpu).pending_softints.bitSet(@intCast(@intFromEnum(ipl)), .monotonic); + _ = percpu.remote(cpu).pending_softints.bitSet( + @intCast(@intFromEnum(ipl)), + .monotonic, + ); } /// Mark a software interrupt of IPL `ipl` on `cpu` as handled. pub fn clear_softint_pending(cpu: u32, ipl: Ipl) void { std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch)); - _ = percpu.remote(cpu).pending_softints.bitReset(@intCast(@intFromEnum(ipl)), .monotonic); + _ = percpu.remote(cpu).pending_softints.bitReset( + @intCast(@intFromEnum(ipl)), + .monotonic, + ); } -/// Check whether a software interrupt of IPL `ipl` on the current CPU is pending. +/// Check whether a software interrupt of IPL `ipl` +/// is pending on the current CPU. pub fn is_softint_pending(ipl: Ipl) bool { std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch)); const shift: u3 = @intCast(@intFromEnum(ipl)); diff --git a/src/kern/private.zig b/src/kern/private.zig index 46385d3..c53f672 100644 --- a/src/kern/private.zig +++ b/src/kern/private.zig @@ -2,6 +2,7 @@ const config = @import("config"); const init_mod = @import("init.zig"); const r = @import("root"); const rtl = @import("rtl"); +const ke = r.ke; pub const impl = switch (config.arch) { .amd64 => @import("amd64/impl.zig"), @@ -52,6 +53,21 @@ const ImplSchema = struct { /// Implementation-defined early initialization. pub fn early_init() void {} + + /// Raise software IPL to the given value. + /// Returns true if the raise failed. + pub fn raise_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool { + _ = new; + _ = old; + unreachable; + } + /// Lower software IPL to the given value. + /// Returns true if the lower failed. + pub fn lower_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool { + _ = new; + _ = old; + unreachable; + } }; comptime {