ke: optimize IPL operations

src/kern/amd64/impl.zig

@@ -146,3 +146,30 @@ pub inline fn flush_full_tlb() void {
 pub inline fn flush_tlb(va: r.VAddr) void {
     amd64.invlpg(va);
 }
+
+// Inline asm so the compiler has a chance to inline it.
+pub inline fn raise_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool {
+    return asm volatile (
+        \\movb %%gs:cpu_ipl, %%cl
+        \\movb %%cl, (%[old_ptr])
+        \\movb %[new_val], %%gs:cpu_ipl
+        \\cmpb %%cl, %[new_val]
+        \\setl %[result]
+        : [result] "=r" (-> bool),
+        : [new_val] "r" (new),
+          [old_ptr] "r" (old),
+        : .{ .rcx = true, .memory = true });
+}
+
+pub inline fn lower_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool {
+    return asm volatile (
+        \\movb %%gs:cpu_ipl, %%cl
+        \\movb %%cl, (%[old_ptr])
+        \\movb %[new_val], %%gs:cpu_ipl
+        \\cmpb %[new_val], %%cl
+        \\setl %[result]
+        : [result] "=r" (-> bool),
+        : [new_val] "r" (new),
+          [old_ptr] "r" (old),
+        : .{ .rcx = true, .memory = true });
+}

src/kern/amd64/int.zig

@@ -96,8 +96,10 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
 
     _ = ke.ipl.set_hardware(ipl);
 
-    if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and ki.ipl.is_softint_pending(.Dispatch)) {
+    if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and
-        ki.dpc.dispatch(ke.cpu.current());
+        ki.ipl.is_softint_pending(.Dispatch))
+    {
+        ki.dpc.dispatch();
     }
 }
 

src/kern/dpc.zig

@@ -72,11 +72,11 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void {
 }
 
 fn dispatch_queue(cpu: u32) void {
-    const ipl_cpu = ki.ipl.percpu.remote(cpu);
+    const ipl_cpu = ki.ipl.cpu_ipl.local();
-    const dpc_cpu = pcpu.remote(cpu);
+    const dpc_cpu = pcpu.local();
-    const sched_cpu = ki.sched.percpu.remote(cpu);
+    const sched_cpu = ki.sched.percpu.local();
 
-    ipl_cpu.ipl = .Dispatch;
+    ipl_cpu.* = .Dispatch;
 
     // Mark as handled, this must be done before enabling interrupts or we will
     // race.
@@ -148,13 +148,13 @@ fn dispatch_queue(cpu: u32) void {
     _ = ki.impl.disable_interrupts();
 }
 
-/// Dispatch the DPC queue on `cpu`.
+/// Dispatch the DPC queue on the current CPU.
-pub fn dispatch(cpu: u32) void {
+pub fn dispatch() void {
     // DPC processing is done at IPL Dispatch.
     // Don't call lower/raise here because they might call us again recursively.
-    var mycpu = cpu;
-    const old_ipl = ki.ipl.percpu.remote(mycpu).ipl;
     const int_state = ki.impl.disable_interrupts();
+    const old_ipl = ki.ipl.cpu_ipl.local().*;
+    var mycpu = ke.cpu.current();
 
     while (ki.ipl.is_softint_pending(.Dispatch)) {
         dispatch_queue(mycpu);
@@ -162,7 +162,6 @@ pub fn dispatch(cpu: u32) void {
         mycpu = ke.cpu.current();
     }
 
-    ki.ipl.percpu.remote(mycpu).ipl = old_ipl;
+    ki.ipl.cpu_ipl.local().* = old_ipl;
-
     ki.impl.restore_interrupts(int_state);
 }

src/kern/ipl.zig

@@ -20,77 +20,85 @@ pub const Ipl = enum(u8) {
 };
 
 const PerCpu = struct {
-    /// Current IPL on this CPU.
-    ipl: Ipl,
     /// Bitmask of pending software interrupts on this CPU.
     pending_softints: std.atomic.Value(u8),
 };
 
+/// Current IPL on this CPU.
+pub const cpu_ipl = ke.ExportedCpuLocal(
+    Ipl,
+    .Passive,
+    "cpu_ipl",
+);
+
 pub const percpu = ke.CpuLocal(PerCpu, .{
-    .ipl = .Passive,
+    .pending_softints = .init(0),
-    .pending_softints = std.atomic.Value(u8).init(0),
 });
 
 pub fn current() Ipl {
-    return percpu.local().ipl;
+    return cpu_ipl.local().*;
 }
 
 /// Raise IPL to `new`.
 /// Panics if the current IPL is higher than requested.
 pub fn raise(new: Ipl) Ipl {
-    const ints = ki.impl.disable_interrupts();
+    var old: ke.Ipl = undefined;
 
-    var cpu = percpu.local();
+    if (ki.impl.raise_software_ipl(new, &old)) {
-    const old = cpu.ipl;
+        @branchHint(.unlikely);
-
+        std.debug.panic(
-    if (@intFromEnum(new) < @intFromEnum(old)) {
+            "ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})",
-        std.debug.panic("ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})", .{ new, old });
+            .{ new, old },
+        );
     }
 
     if (@intFromEnum(new) > @intFromEnum(Ipl.get_max_software())) {
+        // Moving between hardware levels is rare.
+        @branchHint(.unlikely);
         ki.impl.set_hardware_ipl(new);
     }
 
-    cpu.ipl = new;
-
-    ki.impl.restore_interrupts(ints);
-
     return old;
 }
 
 /// Lower IPL to `new`.
 /// Panics if the current IPL is lower than requested.
 pub fn lower(new: Ipl) void {
-    const ints = ki.impl.disable_interrupts();
+    var old: ke.Ipl = undefined;
-    var cpu = percpu.local();
-    const cpu_id = ke.cpu.current();
-    const old = cpu.ipl;
 
-    if (@intFromEnum(new) > @intFromEnum(old)) {
+    if (ki.impl.lower_software_ipl(new, &old)) {
-        std.debug.panic("ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})", .{ new, old });
+        @branchHint(.unlikely);
+        std.debug.panic(
+            "ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})",
+            .{ new, old },
+        );
     }
 
-    if (@intFromEnum(new) <= @intFromEnum(Ipl.get_max_software()) and @intFromEnum(old) > @intFromEnum(Ipl.get_max_software())) {
+    if (@intFromEnum(new) <= @intFromEnum(Ipl.get_max_software()) and
+        @intFromEnum(old) > @intFromEnum(Ipl.get_max_software()))
+    {
+        @branchHint(.unlikely);
         ki.impl.set_hardware_ipl(.Passive);
     }
 
-    cpu.ipl = new;
+    if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and
-
+        is_softint_pending(.Dispatch))
-    ki.impl.restore_interrupts(ints);
+    {
-
+        // Dispatch DPCs if necessary.
-    if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and is_softint_pending(.Dispatch)) {
+        ki.dpc.dispatch();
-        ki.dpc.dispatch(cpu_id);
     }
 }
 
 /// Set the hardware IPL to `new`.
 pub fn set_hardware(new: Ipl) Ipl {
-    var cpu = percpu.local();
+    const ints = ki.impl.disable_interrupts();
-    const old = cpu.ipl;
+    const cpu = cpu_ipl.local();
+    const old = cpu.*;
 
-    cpu.ipl = new;
+    cpu.* = new;
 
     ki.impl.set_hardware_ipl(new);
+    ki.impl.restore_interrupts(ints);
 
     return old;
 }
@@ -98,16 +106,23 @@ pub fn set_hardware(new: Ipl) Ipl {
 /// Mark a software interrupt of IPL `ipl` on `cpu` as pending.
 pub fn set_softint_pending(cpu: u32, ipl: Ipl) void {
     std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
-    _ = percpu.remote(cpu).pending_softints.bitSet(@intCast(@intFromEnum(ipl)), .monotonic);
+    _ = percpu.remote(cpu).pending_softints.bitSet(
+        @intCast(@intFromEnum(ipl)),
+        .monotonic,
+    );
 }
 
 /// Mark a software interrupt of IPL `ipl` on `cpu` as handled.
 pub fn clear_softint_pending(cpu: u32, ipl: Ipl) void {
     std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
-    _ = percpu.remote(cpu).pending_softints.bitReset(@intCast(@intFromEnum(ipl)), .monotonic);
+    _ = percpu.remote(cpu).pending_softints.bitReset(
+        @intCast(@intFromEnum(ipl)),
+        .monotonic,
+    );
 }
 
-/// Check whether a software interrupt of IPL `ipl` on the current CPU is pending.
+/// Check whether a software interrupt of IPL `ipl`
+/// is pending on the current CPU.
 pub fn is_softint_pending(ipl: Ipl) bool {
     std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
     const shift: u3 = @intCast(@intFromEnum(ipl));

src/kern/private.zig

@@ -2,6 +2,7 @@ const config = @import("config");
 const init_mod = @import("init.zig");
 const r = @import("root");
 const rtl = @import("rtl");
+const ke = r.ke;
 
 pub const impl = switch (config.arch) {
     .amd64 => @import("amd64/impl.zig"),
@@ -52,6 +53,21 @@ const ImplSchema = struct {
 
     /// Implementation-defined early initialization.
     pub fn early_init() void {}
+
+    /// Raise software IPL to the given value.
+    /// Returns true if the raise failed.
+    pub fn raise_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool {
+        _ = new;
+        _ = old;
+        unreachable;
+    }
+    /// Lower software IPL to the given value.
+    /// Returns true if the lower failed.
+    pub fn lower_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool {
+        _ = new;
+        _ = old;
+        unreachable;
+    }
 };
 
 comptime {