mirrors/Zag
1
0
Fork 0
mirror of https://github.com/rdmsr/zag.git synced 2026-10-27 13:58:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

ke: optimize IPL operations

Browse Source
This commit is contained in:
Loïc Grégoire
2026-08-07 15:41:12 -04:00
parent 5bb264a9ae
commit 52f62b8354
5 changed files with 105 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/kern/amd64/impl.zig
+27
View File
@@ -146,3 +146,30 @@ pub inline fn flush_full_tlb() void {
pub inline fn flush_tlb(va: r.VAddr) void {
amd64.invlpg(va);
}
// Inline asm so the compiler has a chance to inline it.
pub inline fn raise_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool {
return asm volatile (
\\movb %%gs:cpu_ipl, %%cl
\\movb %%cl, (%[old_ptr])
\\movb %[new_val], %%gs:cpu_ipl
\\cmpb %%cl, %[new_val]
\\setl %[result]
: [result] "=r" (-> bool),
: [new_val] "r" (new),
[old_ptr] "r" (old),
: .{ .rcx = true, .memory = true });
}
pub inline fn lower_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool {
return asm volatile (
\\movb %%gs:cpu_ipl, %%cl
\\movb %%cl, (%[old_ptr])
\\movb %[new_val], %%gs:cpu_ipl
\\cmpb %[new_val], %%cl
\\setl %[result]
: [result] "=r" (-> bool),
: [new_val] "r" (new),
[old_ptr] "r" (old),
: .{ .rcx = true, .memory = true });
}
src/kern/amd64/int.zig
+4 -2
View File
@@ -96,8 +96,10 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
_ = ke.ipl.set_hardware(ipl);
if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and ki.ipl.is_softint_pending(.Dispatch)) {
ki.dpc.dispatch(ke.cpu.current());
if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and
ki.ipl.is_softint_pending(.Dispatch))
{
ki.dpc.dispatch();
}
}
src/kern/dpc.zig
+9 -10
View File
@@ -72,11 +72,11 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void {
}
fn dispatch_queue(cpu: u32) void {
const ipl_cpu = ki.ipl.percpu.remote(cpu);
const dpc_cpu = pcpu.remote(cpu);
const sched_cpu = ki.sched.percpu.remote(cpu);
const ipl_cpu = ki.ipl.cpu_ipl.local();
const dpc_cpu = pcpu.local();
const sched_cpu = ki.sched.percpu.local();
ipl_cpu.ipl = .Dispatch;
ipl_cpu.* = .Dispatch;
// Mark as handled, this must be done before enabling interrupts or we will
// race.
@@ -148,13 +148,13 @@ fn dispatch_queue(cpu: u32) void {
_ = ki.impl.disable_interrupts();
}
/// Dispatch the DPC queue on `cpu`.
pub fn dispatch(cpu: u32) void {
/// Dispatch the DPC queue on the current CPU.
pub fn dispatch() void {
// DPC processing is done at IPL Dispatch.
// Don't call lower/raise here because they might call us again recursively.
var mycpu = cpu;
const old_ipl = ki.ipl.percpu.remote(mycpu).ipl;
const int_state = ki.impl.disable_interrupts();
const old_ipl = ki.ipl.cpu_ipl.local().*;
var mycpu = ke.cpu.current();
while (ki.ipl.is_softint_pending(.Dispatch)) {
dispatch_queue(mycpu);
@@ -162,7 +162,6 @@ pub fn dispatch(cpu: u32) void {
mycpu = ke.cpu.current();
}
ki.ipl.percpu.remote(mycpu).ipl = old_ipl;
ki.ipl.cpu_ipl.local().* = old_ipl;
ki.impl.restore_interrupts(int_state);
}
src/kern/ipl.zig
+49 -34
View File
@@ -20,77 +20,85 @@ pub const Ipl = enum(u8) {
};
const PerCpu = struct {
/// Current IPL on this CPU.
ipl: Ipl,
/// Bitmask of pending software interrupts on this CPU.
pending_softints: std.atomic.Value(u8),
};
/// Current IPL on this CPU.
pub const cpu_ipl = ke.ExportedCpuLocal(
Ipl,
.Passive,
"cpu_ipl",
);
pub const percpu = ke.CpuLocal(PerCpu, .{
.ipl = .Passive,
.pending_softints = std.atomic.Value(u8).init(0),
.pending_softints = .init(0),
});
pub fn current() Ipl {
return percpu.local().ipl;
return cpu_ipl.local().*;
}
/// Raise IPL to `new`.
/// Panics if the current IPL is higher than requested.
pub fn raise(new: Ipl) Ipl {
const ints = ki.impl.disable_interrupts();
var old: ke.Ipl = undefined;
var cpu = percpu.local();
const old = cpu.ipl;
if (@intFromEnum(new) < @intFromEnum(old)) {
std.debug.panic("ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})", .{ new, old });
if (ki.impl.raise_software_ipl(new, &old)) {
@branchHint(.unlikely);
std.debug.panic(
"ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})",
.{ new, old },
);
}
if (@intFromEnum(new) > @intFromEnum(Ipl.get_max_software())) {
// Moving between hardware levels is rare.
@branchHint(.unlikely);
ki.impl.set_hardware_ipl(new);
}
cpu.ipl = new;
ki.impl.restore_interrupts(ints);
return old;
}
/// Lower IPL to `new`.
/// Panics if the current IPL is lower than requested.
pub fn lower(new: Ipl) void {
const ints = ki.impl.disable_interrupts();
var cpu = percpu.local();
const cpu_id = ke.cpu.current();
const old = cpu.ipl;
var old: ke.Ipl = undefined;
if (@intFromEnum(new) > @intFromEnum(old)) {
std.debug.panic("ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})", .{ new, old });
if (ki.impl.lower_software_ipl(new, &old)) {
@branchHint(.unlikely);
std.debug.panic(
"ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})",
.{ new, old },
);
}
if (@intFromEnum(new) <= @intFromEnum(Ipl.get_max_software()) and @intFromEnum(old) > @intFromEnum(Ipl.get_max_software())) {
if (@intFromEnum(new) <= @intFromEnum(Ipl.get_max_software()) and
@intFromEnum(old) > @intFromEnum(Ipl.get_max_software()))
{
@branchHint(.unlikely);
ki.impl.set_hardware_ipl(.Passive);
}
cpu.ipl = new;
ki.impl.restore_interrupts(ints);
if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and is_softint_pending(.Dispatch)) {
ki.dpc.dispatch(cpu_id);
if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and
is_softint_pending(.Dispatch))
{
// Dispatch DPCs if necessary.
ki.dpc.dispatch();
}
}
/// Set the hardware IPL to `new`.
pub fn set_hardware(new: Ipl) Ipl {
var cpu = percpu.local();
const old = cpu.ipl;
const ints = ki.impl.disable_interrupts();
const cpu = cpu_ipl.local();
const old = cpu.*;
cpu.ipl = new;
cpu.* = new;
ki.impl.set_hardware_ipl(new);
ki.impl.restore_interrupts(ints);
return old;
}
@@ -98,16 +106,23 @@ pub fn set_hardware(new: Ipl) Ipl {
/// Mark a software interrupt of IPL `ipl` on `cpu` as pending.
pub fn set_softint_pending(cpu: u32, ipl: Ipl) void {
std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
_ = percpu.remote(cpu).pending_softints.bitSet(@intCast(@intFromEnum(ipl)), .monotonic);
_ = percpu.remote(cpu).pending_softints.bitSet(
@intCast(@intFromEnum(ipl)),
.monotonic,
);
}
/// Mark a software interrupt of IPL `ipl` on `cpu` as handled.
pub fn clear_softint_pending(cpu: u32, ipl: Ipl) void {
std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
_ = percpu.remote(cpu).pending_softints.bitReset(@intCast(@intFromEnum(ipl)), .monotonic);
_ = percpu.remote(cpu).pending_softints.bitReset(
@intCast(@intFromEnum(ipl)),
.monotonic,
);
}
/// Check whether a software interrupt of IPL `ipl` on the current CPU is pending.
/// Check whether a software interrupt of IPL `ipl`
/// is pending on the current CPU.
pub fn is_softint_pending(ipl: Ipl) bool {
std.debug.assert(@intFromEnum(ipl) <= @intFromEnum(Ipl.Dispatch));
const shift: u3 = @intCast(@intFromEnum(ipl));
src/kern/private.zig
+16
View File
@@ -2,6 +2,7 @@ const config = @import("config");
const init_mod = @import("init.zig");
const r = @import("root");
const rtl = @import("rtl");
const ke = r.ke;
pub const impl = switch (config.arch) {
.amd64 => @import("amd64/impl.zig"),
@@ -52,6 +53,21 @@ const ImplSchema = struct {
/// Implementation-defined early initialization.
pub fn early_init() void {}
/// Raise software IPL to the given value.
/// Returns true if the raise failed.
pub fn raise_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool {
_ = new;
_ = old;
unreachable;
}
/// Lower software IPL to the given value.
/// Returns true if the lower failed.
pub fn lower_software_ipl(new: ke.Ipl, old: *ke.Ipl) bool {
_ = new;
_ = old;
unreachable;
}
};
comptime {