ke: add lock warden

config.zig.zon

@@ -54,6 +54,14 @@
                     .default = .{ .int = 14 },
                     .help = "This option sets the size of the kernel log buffer as a power of two. The actual size is calculated as 2 raised to the power of this value (i.e., 2^LOG_BUFFER_SHIFT bytes).",
                 },
+                .{
+                    .kind = .bool,
+                    .key = "warden",
+                    .label = "Enable the kernel warden",
+                    .default = .{ .bool = false },
+                    .help = "This option enables the kernel warden, which checks for correctness of lock acquisitions. This has a substantial performance overhead.",
+                },
+
                 .{
                     .kind = .menu,
                     .label = "Memory Management Options",

src/ex/vmem.zig

@@ -168,7 +168,7 @@ pub const Arena = struct {
             }
         }
 
-        self.lock = .init();
+        self.lock = ke.Mutex.init("arena");
 
         self.allocated_segments = .init();
         self.rotor = null;

src/kern/amd64/int.zig

@@ -49,7 +49,6 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
     const ipl = ke.ipl.set_hardware(.Device);
 
     if (frame.intno < 32) {
-        kep.panic.panic_lock.acquire_no_ipl();
         std.log.err("Unhandled exception: 0x{x} ({s}), err=0x{x}, pc=0x{x}", .{ frame.intno, exception_msg[frame.intno], frame.errcode, frame.rip });
         std.log.err("RAX=0x{x:0>16} RBX=0x{x:0>16} RCX=0x{x:0>16} RDX=0x{x:0>16}", .{ frame.rax, frame.rbx, frame.rcx, frame.rdx });
         std.log.err("RSI=0x{x:0>16} RDI=0x{x:0>16} RBP=0x{x:0>16} RSP=0x{x:0>16}", .{ frame.rsi, frame.rdi, frame.rbp, frame.rsp });

src/kern/dpc.zig

@@ -36,11 +36,12 @@ const PerCpu = struct {
 };
 
 const pcpu = ke.CpuLocal(PerCpu, .{
-    .lock = .init(),
+    .lock = undefined,
     .queue = undefined,
 });
 
 fn pcpu_init() linksection(r.init) void {
+    pcpu.local().lock = ke.SpinLock.init("dpc");
     pcpu.local().queue.init();
 }
 

src/kern/init.zig

@@ -6,12 +6,19 @@ const arch = r.arch;
 const kep = ke.private;
 const ex = r.ex;
 const bv = r.bv;
+const config = @import("config");
 
 var thread0: ke.Thread = undefined;
 
 export fn kmain(boot_info: *r.BootInfo) callconv(.c) void {
     r.boot_info = boot_info;
 
+    if (config.warden) {
+        // Try to do this as early as possible so we register as many classes
+        // as we can.
+        kep.warden.init();
+    }
+
     // Immediately initialize the context of what will eventually become
     // our idle thread. At this point we're already running on the kernel stack.
     thread0.init(.{
@@ -32,10 +39,12 @@ fn init() linksection(r.init) void {
     ke.ncpus = 1;
     kep.impl.early_init();
     kep.tunable.init();
+
     bv.init(r.boot_info);
+    kep.log.init();
     std.log.info("Welcome to the machine", .{});
     pl.early_init();
-    kep.log.init();
+    kep.thread.init();
     kep.cpu.init_cpu(0);
     kep.turnstile.init_turnstiles();
 

src/kern/log.zig

@@ -7,7 +7,7 @@ const ke = r.ke;
 const kep = ke.private;
 const bv = r.bv;
 
-var out_lock: ke.SpinLock = .init();
+var out_lock: ke.SpinLock = undefined;
 
 const DebugWriter = struct {
     interface: std.Io.Writer,
@@ -48,25 +48,17 @@ pub var ringbuffer = kep.log_ring.RingBuffer(
     avg_msg_size_bits,
 ).init();
 
-// Initialize manually because we need this ASAP.
+pub var event: ke.Event = undefined;
-pub var event: ke.Event = .{
-    .hdr = .{
-        .lock = .init(),
-        .signaled = 0,
-        .type = .Notification,
-        .waitblocks = .{
-            .head = .{
-                .next = &event.hdr.waitblocks.head,
-                .prev = &event.hdr.waitblocks.head,
-            },
-        },
-    },
-};
 
 pub fn init() void {
+    out_lock = ke.SpinLock.init("");
     event.init(.Notification);
 }
 
+pub fn force_release_lock() void {
+    out_lock.release_no_ipl();
+}
+
 pub fn log(
     comptime level: std.log.Level,
     comptime scope: @EnumLiteral(),

src/kern/mutex.zig

@@ -1,7 +1,9 @@
 const std = @import("std");
+const rtl = @import("rtl");
+const config = @import("config");
+
 const ke = @import("root").ke;
 const kep = ke.private;
-const rtl = @import("rtl");
 
 /// Number of spins before blocking.
 const optimistic_spins = 100;
@@ -9,8 +11,17 @@ const optimistic_spins = 100;
 pub const Mutex = struct {
     /// The thread currently holding the mutex, null if unlocked.
     owner: std.atomic.Value(?*ke.Thread),
+    wd: kep.warden.LockData,
+
+    pub fn init(class: []const u8) Mutex {
+        if (config.warden) {
+            const cl = kep.warden.find_or_create_lock_class(class);
+            return .{
+                .owner = std.atomic.Value(?*ke.Thread).init(null),
+                .wd = cl,
+            };
+        }
 
-    pub fn init() Mutex {
         return .{ .owner = .init(null) };
     }
 
@@ -22,6 +33,14 @@ pub const Mutex = struct {
         const ipl = ke.ipl.raise(.Dispatch);
         const curtd = kep.sched.percpu.local().current_thread.?;
 
+        if (config.warden) {
+            kep.warden.check(m.wd, .Lock);
+        }
+
+        defer if (config.warden) {
+            kep.warden.acquired(m, m.wd, .Lock);
+        };
+
         // Very fast path: the lock is uncontended and we can acquire it
         //immediately.
         const cur_owner = m.owner.cmpxchgStrong(
@@ -99,11 +118,14 @@ pub const Mutex = struct {
 
     pub fn release(m: *Mutex) void {
         const ipl = ke.ipl.raise(.Dispatch);
-
         const ts = kep.turnstile.lookup(m);
 
         m.owner.store(null, .release);
 
+        defer if (config.warden) {
+            kep.warden.released(m, .Lock);
+        };
+
         if (ts == null) {
             kep.turnstile.exit(m, ts);
             ke.ipl.lower(ipl);
@@ -126,9 +148,7 @@ pub const Mutex = struct {
         );
 
         kep.turnstile.exit(m, ts);
-
         kep.turnstile.wakeup(&waiters);
-
         ke.ipl.lower(ipl);
     }
 };

src/kern/panic.zig

@@ -74,7 +74,6 @@ fn walk_stack_frame(base: usize) void {
     }
 }
 
-pub var panic_lock: ke.SpinLock = .init();
 var crash_count: std.atomic.Value(u8) = .init(0);
 
 pub fn panic_with_frame(
@@ -98,8 +97,6 @@ pub fn panic_with_frame(
     // doing the freeze.
     kep.ipi.freeze_cpus();
 
-    _ = ke.ipl.raise(.High);
-
     std.log.err("KERNEL PANIC: {s} on CPU {}", .{
         msg,
         ke.cpu.current(),

src/kern/private.zig

@@ -101,3 +101,4 @@ pub const shootdown = @import("shootdown.zig");
 pub const tunable = @import("tunable.zig");
 pub const smr = @import("smr.zig");
 pub const ipi = @import("ipi.zig");
+pub const warden = @import("warden.zig");

src/kern/sched.zig

@@ -260,7 +260,7 @@ fn init_cpu() linksection(r.init) void {
             .status = 0,
             .queues = undefined,
         },
-        .queues_lock = .init(),
+        .queues_lock = .init("runq"),
         .idle_queue = undefined,
         .steal_work = false,
         .load = .init(0),
@@ -1125,6 +1125,12 @@ fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void {
         kep.impl.call_continuation(&next.context, continuation);
     }
 
+    if (config.warden) {
+        // The lock will get released through the context switch assembly code,
+        // start pretending we don't hold it right now.
+        kep.warden.released(&cur.lock, .Spin);
+    }
+
     if (relinquish) {
         kep.impl.switch_cont_to_normal(&cur.context, &next.context);
     } else {

src/kern/smr.zig

@@ -41,7 +41,7 @@ pub const Domain = struct {
         for (0..ke.ncpus) |i| {
             self.cpus[i] = .{
                 .current_seq = .init(seq_invalid),
-                .stall_lock = .init(),
+                .stall_lock = ke.SpinLock.init("stall_lock"),
                 .stall_seq = .init(seq_invalid),
                 .stalled = undefined,
                 .stall_goal = seq_invalid,

src/kern/spinlock.zig

@@ -1,7 +1,10 @@
 const std = @import("std");
-const r = @import("root");
 const rtl = @import("rtl");
+const config = @import("config");
+
+const r = @import("root");
 const ke = r.ke;
+const kep = ke.private;
 
 fn LockTemplate(comptime T: type) type {
     comptime {
@@ -13,17 +16,25 @@ fn LockTemplate(comptime T: type) type {
 
     return struct {
         inner: T,
+        wd: kep.warden.LockData = undefined,
 
         const Self = @This();
 
-        pub fn init() Self {
+        pub fn init(comptime class: []const u8) Self {
+            if (config.warden) {
+                const cl = kep.warden.find_or_create_lock_class(class);
+                return .{
+                    .inner = T.init(),
+                    .wd = cl,
+                };
+            }
             return .{ .inner = T.init() };
         }
 
         /// Acquire the lock, raising IPL to `ipl`. Returns the previous IPL.
         pub fn acquire_at(self: *Self, ipl: ke.Ipl) ke.Ipl {
             const old_ipl = ke.ipl.raise(ipl);
-            self.inner.acquire_no_ipl();
+            self.acquire_no_ipl();
             return old_ipl;
         }
 
@@ -34,7 +45,7 @@ fn LockTemplate(comptime T: type) type {
 
         /// Release the lock and restore IPL to `ipl`.
         pub fn release(self: *Self, ipl: ke.Ipl) void {
-            self.inner.release_no_ipl();
+            self.release_no_ipl();
             ke.ipl.lower(ipl);
         }
 
@@ -43,6 +54,10 @@ fn LockTemplate(comptime T: type) type {
         pub fn try_acquire(self: *Self) ?ke.Ipl {
             const old_ipl = ke.ipl.raise(.Dispatch);
             if (self.inner.try_acquire_no_ipl()) {
+                if (config.warden) {
+                    kep.warden.acquired(self, self.wd, .Spin);
+                }
+
                 return old_ipl;
             } else {
                 ke.ipl.lower(old_ipl);
@@ -52,18 +67,36 @@ fn LockTemplate(comptime T: type) type {
 
         /// Acquire the lock without changing the IPL.
         pub fn acquire_no_ipl(self: *Self) void {
+            if (config.warden) {
+                kep.warden.check(self.wd, .Spin);
+            }
+
             self.inner.acquire_no_ipl();
+
+            if (config.warden) {
+                kep.warden.acquired(self, self.wd, .Spin);
+            }
         }
 
         /// Release the lock without changing the IPL.
         pub fn release_no_ipl(self: *Self) void {
+            if (config.warden) {
+                kep.warden.released(self, .Spin);
+            }
+
             self.inner.release_no_ipl();
         }
 
         /// Try to acquire the lock without changing the IPL.
         /// Returns true if the lock was acquired.
         pub fn try_acquire_no_ipl(self: *Self) bool {
-            return self.inner.try_acquire_no_ipl();
+            const ret = self.inner.try_acquire_no_ipl();
+
+            if (ret and config.warden) {
+                kep.warden.acquired(self, self.wd, .Spin);
+            }
+
+            return ret;
         }
 
         /// Returns true if the lock is currently held.

src/kern/thread.zig

@@ -1,7 +1,9 @@
 const std = @import("std");
 const r = @import("root");
 const rtl = @import("rtl");
+const config = @import("config");
 
+const r = @import("root");
 const ke = r.ke;
 const kep = ke.private;
 
@@ -178,6 +180,7 @@ pub const Thread = struct {
         thread.* = .{
             .context = .init_with_stack(opts.stack),
             .lock = .init(),
+            .lock = ke.SpinLock.init("thread"),
             .nice = 0,
             .priority = @intFromEnum(opts.priority),
             .base_priority = @intFromEnum(opts.priority),
@@ -205,6 +208,7 @@ pub const Thread = struct {
             .acct = .{},
             .smr_sections = undefined,
             .hard_affinity = .init(true),
+            .warden_data = .{},
         };
 
         thread.turnstiles_owned.init();
@@ -269,9 +273,26 @@ pub fn exit() void {
     curtd.lock.acquire_no_ipl();
     curtd.state.store(.Zombie, .monotonic);
 
+    if (config.warden) {
+        // There is normally a dispatch object -> thread ordering, but here we
+        // have a thread -> dispatch object ordering because we insert into a
+        // kernel queue (ex.enqueue is called as an activation to the reaper
+        // list) while holding the current thread's lock. This would normally be
+        // a deadlock, and the warden rightfully flags it; but this is safe
+        // because the thread that's currently exiting won't ever block or get
+        // woken up by this queue. Get sneaky under the warden because it can't
+        // understand this.
+        kep.warden.released(&curtd.lock, .Spin);
+    }
+
     // Reuse the runq linkage to put on reaper list.
     reaper_list.insert(@ptrCast(&curtd.runq_link.next));
 
+    if (config.warden) {
+        // Now we're good.
+        kep.warden.acquired(&curtd.lock, curtd.lock.wd, .Spin);
+    }
+
     kep.sched.detach_load_avg(kep.sched.percpu.local(), curtd);
     kep.sched.yield_locked(Continuation.dummy);
 }
@@ -316,7 +337,7 @@ const Cpu = struct {
 /// Keep this low for minimal memory overhead, but potentially increased latency.
 const excess_stacks = ke.Tunable(u32, 1, "ke.thread.excess_stacks");
 
-var global_depot_lock: ke.SpinLock = .init();
+var global_depot_lock: ke.SpinLock = undefined;
 var global_depot: Depot = .{};
 
 const percpu = ke.CpuLocal(Cpu, undefined);
@@ -343,6 +364,10 @@ comptime {
     _ = r.percpu_init_set.insert(&init_cpu);
 }
 
+pub fn init() void {
+    global_depot_lock = ke.SpinLock.init("stack_cache");
+}
+
 pub fn call_continuation(ptr: ?*anyopaque) noreturn {
     const td: *ke.Thread = @ptrCast(@alignCast(ptr));
     std.debug.assert(td.continuation != null);

src/kern/timer.zig

@@ -55,10 +55,18 @@ pub const Timer = struct {
 
 const percpu = ke.CpuLocal(PerCpu, .{
     .timers = .init(),
-    .lock = .init(),
+    .lock = undefined,
     .dpc = .init(handle_expiry),
 });
 
+fn pcpu_init() linksection(r.init) void {
+    percpu.local().lock = ke.SpinLock.init("timers");
+}
+
+comptime {
+    _ = r.percpu_init_set.insert(&pcpu_init);
+}
+
 const Options = struct {
     dpc: ?*ke.Dpc = null,
 };

src/kern/turnstile.zig

@@ -377,7 +377,7 @@ fn propagate(curtd: *ke.Thread) void {
 pub fn init_turnstiles() void {
     for (&chains) |*chain| {
         chain.list.init();
-        chain.lock = .init();
+        chain.lock = ke.SpinLock.init("turnstile_chain");
     }
 }
 

src/kern/wait.zig

@@ -33,6 +33,7 @@ pub const DispatchHeader = struct {
     pub fn init(obj: *DispatchHeader, kind: Type) void {
         obj.type = kind;
         obj.lock = ke.SpinLock.init();
+        obj.lock = ke.SpinLock.init("DispatchObject");
         obj.signaled = 0;
         obj.waitblocks.init();
     }

src/kern/warden.zig

@@ -0,0 +1,344 @@
+//! Lock ordering verifier.
+
+const config = @import("config");
+const rtl = @import("rtl");
+const r = @import("root");
+const ke = r.ke;
+
+const std = @import("std");
+const assert = std.debug.assert;
+
+/// One relation state for a matrix entry.
+const Relation = struct {
+    /// bitmap[L] is set iff L is reachable from this lock.
+    bitmap: [num_classes / @bitSizeOf(usize)]usize = @splat(0),
+
+    fn index_for(idx: usize) struct { usize, u6 } {
+        return .{
+            idx / @bitSizeOf(usize),
+            @intCast(idx % @bitSizeOf(usize)),
+        };
+    }
+
+    pub fn contains(self: Relation, idx: usize) bool {
+        const word, const bit = index_for(idx);
+        return (self.bitmap[word] & (@as(usize, 1) << bit)) != 0;
+    }
+
+    pub fn set(self: *Relation, idx: usize) void {
+        const word, const bit = index_for(idx);
+        self.bitmap[word] |= (@as(usize, 1) << bit);
+    }
+
+    pub fn merge(self: *Relation, other: Relation) void {
+        for (0.., other.bitmap) |i, word| {
+            self.bitmap[i] |= word;
+        }
+    }
+};
+
+/// A lock class, represents logically a single lock type, e.g thread lock.
+pub const Class = struct {
+    name: []const u8,
+
+    /// Index into the relationship matrix.
+    idx: u32,
+
+    entry: rtl.bst.Node,
+};
+
+const LockInstance = struct {
+    class: ?*const Class,
+    lock: *anyopaque,
+};
+
+const held_locks_max = 5;
+
+const HeldLocks = struct {
+    next: ?*HeldLocks = null,
+    locks: [held_locks_max]LockInstance = undefined,
+    num: u8 = 0,
+};
+
+pub const LockData =
+    if (config.warden) ?*Class else void;
+
+pub const HolderData = if (config.warden) HeldLocks else struct {};
+
+const num_classes = 128;
+const num_buckets = 128;
+
+const cpu_held_spinlocks = ke.CpuLocal(HeldLocks, .{});
+
+/// Matrix of reachability between classes
+var reachable: [num_classes]Relation = @splat(.{});
+var graph_lock: ke.SpinLock = undefined;
+
+var classes: [num_classes]Class = @splat(undefined);
+var class_buckets: [num_classes]rtl.AVLTree(nodes_cmp) = @splat(.init());
+var started = false;
+var num_active_classes: u32 = 0;
+
+fn nodes_cmp(a: *const rtl.bst.Node, b: *const rtl.bst.Node) std.math.Order {
+    const aclass: *const Class = @fieldParentPtr("entry", a);
+    const bclass: *const Class = @fieldParentPtr("entry", b);
+
+    return std.mem.order(u8, aclass.name, bclass.name);
+}
+
+/// Find a lock class by its name.
+fn find_lock_class(name: []const u8) struct {
+    ?*Class,
+    ?*rtl.AVLTree(nodes_cmp),
+} {
+    if (name.len == 0) {
+        // This lock shouldn't be checked.
+        return .{ null, null };
+    }
+
+    const hash = std.hash.Murmur2_32.hash(name);
+    const bucket = &class_buckets[hash & (num_classes - 1)];
+
+    var search_for: Class = .{
+        .entry = undefined,
+        .idx = 0,
+        .name = name,
+    };
+
+    const res = bucket.tree.search(&search_for.entry);
+
+    const class_res: ?*Class = if (res) |n|
+        @fieldParentPtr("entry", n)
+    else
+        null;
+
+    return .{ class_res, bucket };
+}
+
+/// Find a lock class by its name or create it if it doesn't exist.
+pub fn find_or_create_lock_class(name: []const u8) ?*Class {
+    if (name.len == 0) {
+        return null;
+    }
+
+    const ipl = graph_lock.acquire_at(.High);
+    defer graph_lock.release(ipl);
+
+    var class, const bucket = find_lock_class(name);
+
+    if (class != null) {
+        return class;
+    }
+
+    assert(num_active_classes < num_classes);
+
+    // Get a new spot for our class.
+    class = &classes[num_active_classes];
+    class.?.name = name;
+
+    // Insert it so that we can find it by name.
+    bucket.?.insert(&class.?.entry) catch unreachable;
+
+    num_active_classes += 1;
+
+    return class;
+}
+
+/// Add an edge from parent to child.
+fn add_edge(
+    parent: *const Class,
+    child: *const Class,
+) error{LockOrderReversal}!void {
+    if (reachable[child.idx].contains(parent.idx)) {
+        @branchHint(.unlikely);
+        return error.LockOrderReversal;
+    }
+
+    if (child.idx == parent.idx) {
+        return;
+    }
+
+    if (reachable[parent.idx].contains(child.idx)) {
+        // We already know about it!
+        return;
+    }
+
+    var mask = reachable[child.idx];
+    mask.set(child.idx);
+
+    // Now try to match up transitive dependencies, i.e
+    // A -> B -> C, if we already knew A -> B and now we want to link B->C,
+    // make a direct link from A -> C.
+    for (0..num_active_classes) |i| {
+        // Find each ancestor of parent.
+        if (!reachable[i].contains(parent.idx) and i != parent.idx)
+            continue;
+
+        // Merge all reachable nodes from C to A.
+        reachable[i].merge(mask);
+    }
+}
+
+/// Check that a lock of class `class` acquisition is safe.
+fn check_impl(class: ?*const Class, held_locks: *HeldLocks) void {
+    if (started == false) {
+        return;
+    }
+
+    const c = class orelse return;
+    const num = held_locks.num;
+
+    if (num == 0) {
+        // First lock, nothing to check.
+        return;
+    }
+
+    var fast = true;
+
+    // First try doing it lock-free as to avoid contention on the graph lock.
+    // This is *technically* UB against the memory model, but don't bother
+    // trying to use atomic reads when this is much nicer & simpler to do.
+    for (0..held_locks.num) |i| {
+        const held = held_locks.locks[i];
+        const hc = held.class orelse continue;
+
+        if (!reachable[hc.idx].contains(c.idx)) {
+            fast = false;
+        }
+    }
+
+    if (fast)
+        return;
+
+    // We couldn't determine that this is safe, so grab the lock and re-check.
+    // Also add new edges if needed.
+    const ipl = graph_lock.acquire_at(.High);
+
+    for (0..held_locks.num) |i| {
+        const held = held_locks.locks[i];
+        const hc = held.class orelse continue;
+
+        add_edge(hc, c) catch {
+            graph_lock.release(ipl);
+
+            std.debug.panic(
+                "Lock ordering inversion at between {s} and {s}",
+                .{
+                    c.name,
+                    hc.name,
+                },
+            );
+        };
+    }
+
+    graph_lock.release(ipl);
+}
+
+/// Called when a lock got acquired.
+fn acquired_impl(
+    lock: *anyopaque,
+    class: ?*const Class,
+    held_locks: *HeldLocks,
+) void {
+    const num = held_locks.num;
+
+    assert(num < held_locks_max);
+
+    held_locks.locks[num].class = class;
+    held_locks.locks[num].lock = lock;
+
+    held_locks.num += 1;
+}
+
+/// Called when a lock gets released.
+fn released_impl(
+    lock: *anyopaque,
+    held_locks: *HeldLocks,
+) void {
+    const num = held_locks.num;
+    assert(num > 0);
+
+    var i: u32 = 0;
+
+    // Find the lock we just released.
+    for (held_locks.locks[0..num]) |l| {
+        if (l.lock == lock) break;
+        i += 1;
+    }
+
+    assert(i < num);
+
+    held_locks.num = num - 1;
+
+    // Shift all other locks down.
+    for (i..held_locks.num) |j| {
+        held_locks.locks[j] = held_locks.locks[j + 1];
+    }
+}
+
+const LockKind = enum {
+    Spin,
+    Lock,
+};
+
+/// Called when a spinlock gets acquired.
+pub fn acquired(
+    lock: *anyopaque,
+    class: ?*const Class,
+    comptime kind: LockKind,
+) void {
+    if (started == false) return;
+
+    if (kind == .Spin) {
+        const ipl = ke.ipl.raise(.High);
+        const cpu = cpu_held_spinlocks.local();
+
+        acquired_impl(lock, class, cpu);
+
+        ke.ipl.lower(ipl);
+    } else {
+        const curtd = ke.thread.current();
+        acquired_impl(lock, class, &curtd.warden_data);
+    }
+}
+
+/// Called when a lock gets released.
+pub fn released(lock: *anyopaque, comptime kind: LockKind) void {
+    if (kind == .Spin) {
+        const ipl = ke.ipl.raise(.High);
+        const cpu = cpu_held_spinlocks.local();
+        released_impl(lock, cpu);
+        ke.ipl.lower(ipl);
+    } else {
+        const curtd = ke.thread.current();
+        released_impl(lock, &curtd.warden_data);
+    }
+}
+
+/// Check that a lock acquisition is safe.
+pub fn check(class: ?*const Class, comptime kind: LockKind) void {
+    if (class == null) return;
+
+    if (kind == .Spin) {
+        const ipl = ke.ipl.raise(.High);
+        const cpu = cpu_held_spinlocks.local();
+
+        check_impl(class, cpu);
+
+        ke.ipl.lower(ipl);
+    } else {
+        const curtd = ke.thread.current();
+        check_impl(class, &curtd.warden_data);
+    }
+}
+
+/// Initialize the kernel warden.
+pub fn init() void {
+    graph_lock = ke.SpinLock.init("");
+
+    for (0.., &classes) |i, *class| {
+        class.idx = @truncate(i);
+    }
+
+    started = true;
+}

src/mm/phys.zig

@@ -22,7 +22,7 @@ pub var low_memory_event: ke.Event = undefined;
 
 var memory_map: *r.BootInfo.MemMap = undefined;
 var early_allocs: usize = 0;
-var list_lock: ke.SpinLock = .init();
+var list_lock: ke.SpinLock = undefined;
 var free_list: rtl.List = undefined;
 
 var free_page_event: ke.Event = undefined;
@@ -155,6 +155,7 @@ pub fn init() linksection(r.init) void {
     free_list.init();
     free_page_event.init(.Notification);
     low_memory_event.init(.Notification);
+    list_lock = ke.SpinLock.init("PageList");
 
     var total_usable_memory: usize = memory_map.loader_memory_used;
     log.info("physical memory map:", .{});

src/mm/private.zig

@@ -37,5 +37,5 @@ pub const impl = switch (config.arch) {
 pub var kernel_space: Space = .{
     .pmap = undefined,
     .arena = undefined,
-    .lock = .init(),
+    .lock = undefined,
 };

src/mm/zone.zig

@@ -193,7 +193,7 @@ var generic_zones: [generic_zones_num]Zone = undefined;
 
 var magazines_initialized = false;
 var all_zones: ?*Zone = null;
-var zone_list_lock: ke.Mutex = .init();
+var zone_list_lock: ke.Mutex = undefined;
 
 var magazine_zone: Zone = undefined;
 var cpu_zone: Zone = undefined;
@@ -610,8 +610,8 @@ pub const Zone = struct {
         self.chunk_size = chunk_size;
         self.ctor = options.ctor;
         self.dtor = options.dtor;
-        self.lock = .init();
+        self.lock = ke.Mutex.init("zone");
-        self.depot_lock = .init();
+        self.depot_lock = ke.QSpinLock.init("zone_depot");
         self.use_magazines = options.magazines;
 
         self.depot = std.mem.zeroes(Depot);
@@ -659,7 +659,7 @@ pub const Zone = struct {
         self.cpus = cpus_alloc();
 
         for (0..ke.ncpus) |i| {
-            self.cpus[i].value.lock = .init();
+            self.cpus[i].value.lock = ke.SpinLock.init("Depot");
             self.cpus[i].value.alloc = null;
             self.cpus[i].value.free = null;
             self.cpus[i].value.alloc_rounds = 0;
@@ -1755,6 +1755,8 @@ pub fn TypedZone(comptime T: type) type {
 }
 
 pub fn early_init() linksection(r.init) void {
+    zone_list_lock = ke.Mutex.init("zone_list");
+
     for (0..generic_zones.len, &generic_zones) |i, *zone| {
         zone.init(generic_zone_names[i], @as(usize, 1) << @intCast(i + 3), .{});
     }
@@ -1786,7 +1788,7 @@ pub fn late_init() linksection(r.init) void {
 
             for (0..ke.ncpus) |i| {
                 z.cpus[i] = .init(.{
-                    .lock = .init(),
+                    .lock = ke.SpinLock.init("Magazine"),
                     .alloc = null,
                     .free = null,
                     .alloc_rounds = 0,

src/proc/private.zig

@@ -15,7 +15,7 @@ fn turnstile_ctor(ts: *kep.turnstile.Turnstile) void {
         .queues = undefined,
         .waiters = 0,
         .owners = .none,
-        .lock = .init(),
+        .lock = .init("turnstile"),
     };
     ts.reset();
 }

src/tests/mutex.zig

@@ -157,7 +157,7 @@ fn quiesce_verify() u64 {
 
 pub fn start(_: ?*anyopaque) void {
     for (&locks) |*l| {
-        l.mutex = .init();
+        l.mutex = ke.Mutex.init("test");
         l.guard = 0;
         l.count = 0;
     }

src/tests/turnstile.zig

@@ -111,8 +111,8 @@ fn spawn(prio: u8, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) *ps.Th
 }
 
 fn chain_test() void {
-    m1 = .init();
+    m1 = .init("");
-    m2 = .init();
+    m2 = .init("");
 
     low_td = spawn(prio_low, &chain_low, null);
     wait_flag(&low_locked);
@@ -209,7 +209,7 @@ fn quiesce_check() void {
 pub fn start(_: ?*anyopaque) void {
     chain_test();
 
-    for (&locks) |*l| l.* = .init();
+    for (&locks) |*l| l.* = .init("");
 
     nstress = @max(4, @min(ke.ncpus * 2, max_stress));