From b802919fc23c2efec8fed70031b1f86763d952d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Gr=C3=A9goire?= Date: Sun, 4 Oct 2026 11:10:35 -0400 Subject: [PATCH] ke: add lock warden --- config.zig.zon | 8 + src/ex/vmem.zig | 2 +- src/kern/amd64/int.zig | 1 - src/kern/dpc.zig | 3 +- src/kern/init.zig | 11 +- src/kern/log.zig | 22 +-- src/kern/mutex.zig | 30 +++- src/kern/panic.zig | 3 - src/kern/private.zig | 1 + src/kern/sched.zig | 8 +- src/kern/smr.zig | 2 +- src/kern/spinlock.zig | 43 ++++- src/kern/thread.zig | 27 +++- src/kern/timer.zig | 10 +- src/kern/turnstile.zig | 2 +- src/kern/wait.zig | 1 + src/kern/warden.zig | 344 ++++++++++++++++++++++++++++++++++++++++ src/mm/phys.zig | 3 +- src/mm/private.zig | 2 +- src/mm/zone.zig | 12 +- src/proc/private.zig | 2 +- src/tests/mutex.zig | 2 +- src/tests/turnstile.zig | 6 +- 23 files changed, 496 insertions(+), 49 deletions(-) create mode 100644 src/kern/warden.zig diff --git a/config.zig.zon b/config.zig.zon index ce51691..a3324b8 100644 --- a/config.zig.zon +++ b/config.zig.zon @@ -54,6 +54,14 @@ .default = .{ .int = 14 }, .help = "This option sets the size of the kernel log buffer as a power of two. The actual size is calculated as 2 raised to the power of this value (i.e., 2^LOG_BUFFER_SHIFT bytes).", }, + .{ + .kind = .bool, + .key = "warden", + .label = "Enable the kernel warden", + .default = .{ .bool = false }, + .help = "This option enables the kernel warden, which checks for correctness of lock acquisitions. This has a substantial performance overhead.", + }, + .{ .kind = .menu, .label = "Memory Management Options", diff --git a/src/ex/vmem.zig b/src/ex/vmem.zig index 9ce0a70..0bee8e7 100644 --- a/src/ex/vmem.zig +++ b/src/ex/vmem.zig @@ -168,7 +168,7 @@ pub const Arena = struct { } } - self.lock = .init(); + self.lock = ke.Mutex.init("arena"); self.allocated_segments = .init(); self.rotor = null; diff --git a/src/kern/amd64/int.zig b/src/kern/amd64/int.zig index 7dba84d..2f82334 100644 --- a/src/kern/amd64/int.zig +++ b/src/kern/amd64/int.zig @@ -49,7 +49,6 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys const ipl = ke.ipl.set_hardware(.Device); if (frame.intno < 32) { - kep.panic.panic_lock.acquire_no_ipl(); std.log.err("Unhandled exception: 0x{x} ({s}), err=0x{x}, pc=0x{x}", .{ frame.intno, exception_msg[frame.intno], frame.errcode, frame.rip }); std.log.err("RAX=0x{x:0>16} RBX=0x{x:0>16} RCX=0x{x:0>16} RDX=0x{x:0>16}", .{ frame.rax, frame.rbx, frame.rcx, frame.rdx }); std.log.err("RSI=0x{x:0>16} RDI=0x{x:0>16} RBP=0x{x:0>16} RSP=0x{x:0>16}", .{ frame.rsi, frame.rdi, frame.rbp, frame.rsp }); diff --git a/src/kern/dpc.zig b/src/kern/dpc.zig index 928583a..56c01a1 100644 --- a/src/kern/dpc.zig +++ b/src/kern/dpc.zig @@ -36,11 +36,12 @@ const PerCpu = struct { }; const pcpu = ke.CpuLocal(PerCpu, .{ - .lock = .init(), + .lock = undefined, .queue = undefined, }); fn pcpu_init() linksection(r.init) void { + pcpu.local().lock = ke.SpinLock.init("dpc"); pcpu.local().queue.init(); } diff --git a/src/kern/init.zig b/src/kern/init.zig index 6eceb75..de7ceba 100644 --- a/src/kern/init.zig +++ b/src/kern/init.zig @@ -6,12 +6,19 @@ const arch = r.arch; const kep = ke.private; const ex = r.ex; const bv = r.bv; +const config = @import("config"); var thread0: ke.Thread = undefined; export fn kmain(boot_info: *r.BootInfo) callconv(.c) void { r.boot_info = boot_info; + if (config.warden) { + // Try to do this as early as possible so we register as many classes + // as we can. + kep.warden.init(); + } + // Immediately initialize the context of what will eventually become // our idle thread. At this point we're already running on the kernel stack. thread0.init(.{ @@ -32,10 +39,12 @@ fn init() linksection(r.init) void { ke.ncpus = 1; kep.impl.early_init(); kep.tunable.init(); + bv.init(r.boot_info); + kep.log.init(); std.log.info("Welcome to the machine", .{}); pl.early_init(); - kep.log.init(); + kep.thread.init(); kep.cpu.init_cpu(0); kep.turnstile.init_turnstiles(); diff --git a/src/kern/log.zig b/src/kern/log.zig index 6ff337b..8c00236 100644 --- a/src/kern/log.zig +++ b/src/kern/log.zig @@ -7,7 +7,7 @@ const ke = r.ke; const kep = ke.private; const bv = r.bv; -var out_lock: ke.SpinLock = .init(); +var out_lock: ke.SpinLock = undefined; const DebugWriter = struct { interface: std.Io.Writer, @@ -48,25 +48,17 @@ pub var ringbuffer = kep.log_ring.RingBuffer( avg_msg_size_bits, ).init(); -// Initialize manually because we need this ASAP. -pub var event: ke.Event = .{ - .hdr = .{ - .lock = .init(), - .signaled = 0, - .type = .Notification, - .waitblocks = .{ - .head = .{ - .next = &event.hdr.waitblocks.head, - .prev = &event.hdr.waitblocks.head, - }, - }, - }, -}; +pub var event: ke.Event = undefined; pub fn init() void { + out_lock = ke.SpinLock.init(""); event.init(.Notification); } +pub fn force_release_lock() void { + out_lock.release_no_ipl(); +} + pub fn log( comptime level: std.log.Level, comptime scope: @EnumLiteral(), diff --git a/src/kern/mutex.zig b/src/kern/mutex.zig index 7c41c47..70f63ef 100644 --- a/src/kern/mutex.zig +++ b/src/kern/mutex.zig @@ -1,7 +1,9 @@ const std = @import("std"); +const rtl = @import("rtl"); +const config = @import("config"); + const ke = @import("root").ke; const kep = ke.private; -const rtl = @import("rtl"); /// Number of spins before blocking. const optimistic_spins = 100; @@ -9,8 +11,17 @@ const optimistic_spins = 100; pub const Mutex = struct { /// The thread currently holding the mutex, null if unlocked. owner: std.atomic.Value(?*ke.Thread), + wd: kep.warden.LockData, + + pub fn init(class: []const u8) Mutex { + if (config.warden) { + const cl = kep.warden.find_or_create_lock_class(class); + return .{ + .owner = std.atomic.Value(?*ke.Thread).init(null), + .wd = cl, + }; + } - pub fn init() Mutex { return .{ .owner = .init(null) }; } @@ -22,6 +33,14 @@ pub const Mutex = struct { const ipl = ke.ipl.raise(.Dispatch); const curtd = kep.sched.percpu.local().current_thread.?; + if (config.warden) { + kep.warden.check(m.wd, .Lock); + } + + defer if (config.warden) { + kep.warden.acquired(m, m.wd, .Lock); + }; + // Very fast path: the lock is uncontended and we can acquire it //immediately. const cur_owner = m.owner.cmpxchgStrong( @@ -99,11 +118,14 @@ pub const Mutex = struct { pub fn release(m: *Mutex) void { const ipl = ke.ipl.raise(.Dispatch); - const ts = kep.turnstile.lookup(m); m.owner.store(null, .release); + defer if (config.warden) { + kep.warden.released(m, .Lock); + }; + if (ts == null) { kep.turnstile.exit(m, ts); ke.ipl.lower(ipl); @@ -126,9 +148,7 @@ pub const Mutex = struct { ); kep.turnstile.exit(m, ts); - kep.turnstile.wakeup(&waiters); - ke.ipl.lower(ipl); } }; diff --git a/src/kern/panic.zig b/src/kern/panic.zig index 26ca0c6..acc2fc9 100644 --- a/src/kern/panic.zig +++ b/src/kern/panic.zig @@ -74,7 +74,6 @@ fn walk_stack_frame(base: usize) void { } } -pub var panic_lock: ke.SpinLock = .init(); var crash_count: std.atomic.Value(u8) = .init(0); pub fn panic_with_frame( @@ -98,8 +97,6 @@ pub fn panic_with_frame( // doing the freeze. kep.ipi.freeze_cpus(); - _ = ke.ipl.raise(.High); - std.log.err("KERNEL PANIC: {s} on CPU {}", .{ msg, ke.cpu.current(), diff --git a/src/kern/private.zig b/src/kern/private.zig index cd8be19..e5a9ac1 100644 --- a/src/kern/private.zig +++ b/src/kern/private.zig @@ -101,3 +101,4 @@ pub const shootdown = @import("shootdown.zig"); pub const tunable = @import("tunable.zig"); pub const smr = @import("smr.zig"); pub const ipi = @import("ipi.zig"); +pub const warden = @import("warden.zig"); diff --git a/src/kern/sched.zig b/src/kern/sched.zig index ba64ccd..e7414eb 100644 --- a/src/kern/sched.zig +++ b/src/kern/sched.zig @@ -260,7 +260,7 @@ fn init_cpu() linksection(r.init) void { .status = 0, .queues = undefined, }, - .queues_lock = .init(), + .queues_lock = .init("runq"), .idle_queue = undefined, .steal_work = false, .load = .init(0), @@ -1125,6 +1125,12 @@ fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void { kep.impl.call_continuation(&next.context, continuation); } + if (config.warden) { + // The lock will get released through the context switch assembly code, + // start pretending we don't hold it right now. + kep.warden.released(&cur.lock, .Spin); + } + if (relinquish) { kep.impl.switch_cont_to_normal(&cur.context, &next.context); } else { diff --git a/src/kern/smr.zig b/src/kern/smr.zig index e4f8991..3bf4abe 100644 --- a/src/kern/smr.zig +++ b/src/kern/smr.zig @@ -41,7 +41,7 @@ pub const Domain = struct { for (0..ke.ncpus) |i| { self.cpus[i] = .{ .current_seq = .init(seq_invalid), - .stall_lock = .init(), + .stall_lock = ke.SpinLock.init("stall_lock"), .stall_seq = .init(seq_invalid), .stalled = undefined, .stall_goal = seq_invalid, diff --git a/src/kern/spinlock.zig b/src/kern/spinlock.zig index 861b95d..39e5396 100644 --- a/src/kern/spinlock.zig +++ b/src/kern/spinlock.zig @@ -1,7 +1,10 @@ const std = @import("std"); -const r = @import("root"); const rtl = @import("rtl"); +const config = @import("config"); + +const r = @import("root"); const ke = r.ke; +const kep = ke.private; fn LockTemplate(comptime T: type) type { comptime { @@ -13,17 +16,25 @@ fn LockTemplate(comptime T: type) type { return struct { inner: T, + wd: kep.warden.LockData = undefined, const Self = @This(); - pub fn init() Self { + pub fn init(comptime class: []const u8) Self { + if (config.warden) { + const cl = kep.warden.find_or_create_lock_class(class); + return .{ + .inner = T.init(), + .wd = cl, + }; + } return .{ .inner = T.init() }; } /// Acquire the lock, raising IPL to `ipl`. Returns the previous IPL. pub fn acquire_at(self: *Self, ipl: ke.Ipl) ke.Ipl { const old_ipl = ke.ipl.raise(ipl); - self.inner.acquire_no_ipl(); + self.acquire_no_ipl(); return old_ipl; } @@ -34,7 +45,7 @@ fn LockTemplate(comptime T: type) type { /// Release the lock and restore IPL to `ipl`. pub fn release(self: *Self, ipl: ke.Ipl) void { - self.inner.release_no_ipl(); + self.release_no_ipl(); ke.ipl.lower(ipl); } @@ -43,6 +54,10 @@ fn LockTemplate(comptime T: type) type { pub fn try_acquire(self: *Self) ?ke.Ipl { const old_ipl = ke.ipl.raise(.Dispatch); if (self.inner.try_acquire_no_ipl()) { + if (config.warden) { + kep.warden.acquired(self, self.wd, .Spin); + } + return old_ipl; } else { ke.ipl.lower(old_ipl); @@ -52,18 +67,36 @@ fn LockTemplate(comptime T: type) type { /// Acquire the lock without changing the IPL. pub fn acquire_no_ipl(self: *Self) void { + if (config.warden) { + kep.warden.check(self.wd, .Spin); + } + self.inner.acquire_no_ipl(); + + if (config.warden) { + kep.warden.acquired(self, self.wd, .Spin); + } } /// Release the lock without changing the IPL. pub fn release_no_ipl(self: *Self) void { + if (config.warden) { + kep.warden.released(self, .Spin); + } + self.inner.release_no_ipl(); } /// Try to acquire the lock without changing the IPL. /// Returns true if the lock was acquired. pub fn try_acquire_no_ipl(self: *Self) bool { - return self.inner.try_acquire_no_ipl(); + const ret = self.inner.try_acquire_no_ipl(); + + if (ret and config.warden) { + kep.warden.acquired(self, self.wd, .Spin); + } + + return ret; } /// Returns true if the lock is currently held. diff --git a/src/kern/thread.zig b/src/kern/thread.zig index 8091712..fa9c3d2 100644 --- a/src/kern/thread.zig +++ b/src/kern/thread.zig @@ -1,7 +1,9 @@ const std = @import("std"); const r = @import("root"); const rtl = @import("rtl"); +const config = @import("config"); +const r = @import("root"); const ke = r.ke; const kep = ke.private; @@ -178,6 +180,7 @@ pub const Thread = struct { thread.* = .{ .context = .init_with_stack(opts.stack), .lock = .init(), + .lock = ke.SpinLock.init("thread"), .nice = 0, .priority = @intFromEnum(opts.priority), .base_priority = @intFromEnum(opts.priority), @@ -205,6 +208,7 @@ pub const Thread = struct { .acct = .{}, .smr_sections = undefined, .hard_affinity = .init(true), + .warden_data = .{}, }; thread.turnstiles_owned.init(); @@ -269,9 +273,26 @@ pub fn exit() void { curtd.lock.acquire_no_ipl(); curtd.state.store(.Zombie, .monotonic); + if (config.warden) { + // There is normally a dispatch object -> thread ordering, but here we + // have a thread -> dispatch object ordering because we insert into a + // kernel queue (ex.enqueue is called as an activation to the reaper + // list) while holding the current thread's lock. This would normally be + // a deadlock, and the warden rightfully flags it; but this is safe + // because the thread that's currently exiting won't ever block or get + // woken up by this queue. Get sneaky under the warden because it can't + // understand this. + kep.warden.released(&curtd.lock, .Spin); + } + // Reuse the runq linkage to put on reaper list. reaper_list.insert(@ptrCast(&curtd.runq_link.next)); + if (config.warden) { + // Now we're good. + kep.warden.acquired(&curtd.lock, curtd.lock.wd, .Spin); + } + kep.sched.detach_load_avg(kep.sched.percpu.local(), curtd); kep.sched.yield_locked(Continuation.dummy); } @@ -316,7 +337,7 @@ const Cpu = struct { /// Keep this low for minimal memory overhead, but potentially increased latency. const excess_stacks = ke.Tunable(u32, 1, "ke.thread.excess_stacks"); -var global_depot_lock: ke.SpinLock = .init(); +var global_depot_lock: ke.SpinLock = undefined; var global_depot: Depot = .{}; const percpu = ke.CpuLocal(Cpu, undefined); @@ -343,6 +364,10 @@ comptime { _ = r.percpu_init_set.insert(&init_cpu); } +pub fn init() void { + global_depot_lock = ke.SpinLock.init("stack_cache"); +} + pub fn call_continuation(ptr: ?*anyopaque) noreturn { const td: *ke.Thread = @ptrCast(@alignCast(ptr)); std.debug.assert(td.continuation != null); diff --git a/src/kern/timer.zig b/src/kern/timer.zig index 128c106..d33eeaa 100644 --- a/src/kern/timer.zig +++ b/src/kern/timer.zig @@ -55,10 +55,18 @@ pub const Timer = struct { const percpu = ke.CpuLocal(PerCpu, .{ .timers = .init(), - .lock = .init(), + .lock = undefined, .dpc = .init(handle_expiry), }); +fn pcpu_init() linksection(r.init) void { + percpu.local().lock = ke.SpinLock.init("timers"); +} + +comptime { + _ = r.percpu_init_set.insert(&pcpu_init); +} + const Options = struct { dpc: ?*ke.Dpc = null, }; diff --git a/src/kern/turnstile.zig b/src/kern/turnstile.zig index 8882ca6..71d1043 100644 --- a/src/kern/turnstile.zig +++ b/src/kern/turnstile.zig @@ -377,7 +377,7 @@ fn propagate(curtd: *ke.Thread) void { pub fn init_turnstiles() void { for (&chains) |*chain| { chain.list.init(); - chain.lock = .init(); + chain.lock = ke.SpinLock.init("turnstile_chain"); } } diff --git a/src/kern/wait.zig b/src/kern/wait.zig index 33c973d..7751043 100644 --- a/src/kern/wait.zig +++ b/src/kern/wait.zig @@ -33,6 +33,7 @@ pub const DispatchHeader = struct { pub fn init(obj: *DispatchHeader, kind: Type) void { obj.type = kind; obj.lock = ke.SpinLock.init(); + obj.lock = ke.SpinLock.init("DispatchObject"); obj.signaled = 0; obj.waitblocks.init(); } diff --git a/src/kern/warden.zig b/src/kern/warden.zig new file mode 100644 index 0000000..92521cc --- /dev/null +++ b/src/kern/warden.zig @@ -0,0 +1,344 @@ +//! Lock ordering verifier. + +const config = @import("config"); +const rtl = @import("rtl"); +const r = @import("root"); +const ke = r.ke; + +const std = @import("std"); +const assert = std.debug.assert; + +/// One relation state for a matrix entry. +const Relation = struct { + /// bitmap[L] is set iff L is reachable from this lock. + bitmap: [num_classes / @bitSizeOf(usize)]usize = @splat(0), + + fn index_for(idx: usize) struct { usize, u6 } { + return .{ + idx / @bitSizeOf(usize), + @intCast(idx % @bitSizeOf(usize)), + }; + } + + pub fn contains(self: Relation, idx: usize) bool { + const word, const bit = index_for(idx); + return (self.bitmap[word] & (@as(usize, 1) << bit)) != 0; + } + + pub fn set(self: *Relation, idx: usize) void { + const word, const bit = index_for(idx); + self.bitmap[word] |= (@as(usize, 1) << bit); + } + + pub fn merge(self: *Relation, other: Relation) void { + for (0.., other.bitmap) |i, word| { + self.bitmap[i] |= word; + } + } +}; + +/// A lock class, represents logically a single lock type, e.g thread lock. +pub const Class = struct { + name: []const u8, + + /// Index into the relationship matrix. + idx: u32, + + entry: rtl.bst.Node, +}; + +const LockInstance = struct { + class: ?*const Class, + lock: *anyopaque, +}; + +const held_locks_max = 5; + +const HeldLocks = struct { + next: ?*HeldLocks = null, + locks: [held_locks_max]LockInstance = undefined, + num: u8 = 0, +}; + +pub const LockData = + if (config.warden) ?*Class else void; + +pub const HolderData = if (config.warden) HeldLocks else struct {}; + +const num_classes = 128; +const num_buckets = 128; + +const cpu_held_spinlocks = ke.CpuLocal(HeldLocks, .{}); + +/// Matrix of reachability between classes +var reachable: [num_classes]Relation = @splat(.{}); +var graph_lock: ke.SpinLock = undefined; + +var classes: [num_classes]Class = @splat(undefined); +var class_buckets: [num_classes]rtl.AVLTree(nodes_cmp) = @splat(.init()); +var started = false; +var num_active_classes: u32 = 0; + +fn nodes_cmp(a: *const rtl.bst.Node, b: *const rtl.bst.Node) std.math.Order { + const aclass: *const Class = @fieldParentPtr("entry", a); + const bclass: *const Class = @fieldParentPtr("entry", b); + + return std.mem.order(u8, aclass.name, bclass.name); +} + +/// Find a lock class by its name. +fn find_lock_class(name: []const u8) struct { + ?*Class, + ?*rtl.AVLTree(nodes_cmp), +} { + if (name.len == 0) { + // This lock shouldn't be checked. + return .{ null, null }; + } + + const hash = std.hash.Murmur2_32.hash(name); + const bucket = &class_buckets[hash & (num_classes - 1)]; + + var search_for: Class = .{ + .entry = undefined, + .idx = 0, + .name = name, + }; + + const res = bucket.tree.search(&search_for.entry); + + const class_res: ?*Class = if (res) |n| + @fieldParentPtr("entry", n) + else + null; + + return .{ class_res, bucket }; +} + +/// Find a lock class by its name or create it if it doesn't exist. +pub fn find_or_create_lock_class(name: []const u8) ?*Class { + if (name.len == 0) { + return null; + } + + const ipl = graph_lock.acquire_at(.High); + defer graph_lock.release(ipl); + + var class, const bucket = find_lock_class(name); + + if (class != null) { + return class; + } + + assert(num_active_classes < num_classes); + + // Get a new spot for our class. + class = &classes[num_active_classes]; + class.?.name = name; + + // Insert it so that we can find it by name. + bucket.?.insert(&class.?.entry) catch unreachable; + + num_active_classes += 1; + + return class; +} + +/// Add an edge from parent to child. +fn add_edge( + parent: *const Class, + child: *const Class, +) error{LockOrderReversal}!void { + if (reachable[child.idx].contains(parent.idx)) { + @branchHint(.unlikely); + return error.LockOrderReversal; + } + + if (child.idx == parent.idx) { + return; + } + + if (reachable[parent.idx].contains(child.idx)) { + // We already know about it! + return; + } + + var mask = reachable[child.idx]; + mask.set(child.idx); + + // Now try to match up transitive dependencies, i.e + // A -> B -> C, if we already knew A -> B and now we want to link B->C, + // make a direct link from A -> C. + for (0..num_active_classes) |i| { + // Find each ancestor of parent. + if (!reachable[i].contains(parent.idx) and i != parent.idx) + continue; + + // Merge all reachable nodes from C to A. + reachable[i].merge(mask); + } +} + +/// Check that a lock of class `class` acquisition is safe. +fn check_impl(class: ?*const Class, held_locks: *HeldLocks) void { + if (started == false) { + return; + } + + const c = class orelse return; + const num = held_locks.num; + + if (num == 0) { + // First lock, nothing to check. + return; + } + + var fast = true; + + // First try doing it lock-free as to avoid contention on the graph lock. + // This is *technically* UB against the memory model, but don't bother + // trying to use atomic reads when this is much nicer & simpler to do. + for (0..held_locks.num) |i| { + const held = held_locks.locks[i]; + const hc = held.class orelse continue; + + if (!reachable[hc.idx].contains(c.idx)) { + fast = false; + } + } + + if (fast) + return; + + // We couldn't determine that this is safe, so grab the lock and re-check. + // Also add new edges if needed. + const ipl = graph_lock.acquire_at(.High); + + for (0..held_locks.num) |i| { + const held = held_locks.locks[i]; + const hc = held.class orelse continue; + + add_edge(hc, c) catch { + graph_lock.release(ipl); + + std.debug.panic( + "Lock ordering inversion at between {s} and {s}", + .{ + c.name, + hc.name, + }, + ); + }; + } + + graph_lock.release(ipl); +} + +/// Called when a lock got acquired. +fn acquired_impl( + lock: *anyopaque, + class: ?*const Class, + held_locks: *HeldLocks, +) void { + const num = held_locks.num; + + assert(num < held_locks_max); + + held_locks.locks[num].class = class; + held_locks.locks[num].lock = lock; + + held_locks.num += 1; +} + +/// Called when a lock gets released. +fn released_impl( + lock: *anyopaque, + held_locks: *HeldLocks, +) void { + const num = held_locks.num; + assert(num > 0); + + var i: u32 = 0; + + // Find the lock we just released. + for (held_locks.locks[0..num]) |l| { + if (l.lock == lock) break; + i += 1; + } + + assert(i < num); + + held_locks.num = num - 1; + + // Shift all other locks down. + for (i..held_locks.num) |j| { + held_locks.locks[j] = held_locks.locks[j + 1]; + } +} + +const LockKind = enum { + Spin, + Lock, +}; + +/// Called when a spinlock gets acquired. +pub fn acquired( + lock: *anyopaque, + class: ?*const Class, + comptime kind: LockKind, +) void { + if (started == false) return; + + if (kind == .Spin) { + const ipl = ke.ipl.raise(.High); + const cpu = cpu_held_spinlocks.local(); + + acquired_impl(lock, class, cpu); + + ke.ipl.lower(ipl); + } else { + const curtd = ke.thread.current(); + acquired_impl(lock, class, &curtd.warden_data); + } +} + +/// Called when a lock gets released. +pub fn released(lock: *anyopaque, comptime kind: LockKind) void { + if (kind == .Spin) { + const ipl = ke.ipl.raise(.High); + const cpu = cpu_held_spinlocks.local(); + released_impl(lock, cpu); + ke.ipl.lower(ipl); + } else { + const curtd = ke.thread.current(); + released_impl(lock, &curtd.warden_data); + } +} + +/// Check that a lock acquisition is safe. +pub fn check(class: ?*const Class, comptime kind: LockKind) void { + if (class == null) return; + + if (kind == .Spin) { + const ipl = ke.ipl.raise(.High); + const cpu = cpu_held_spinlocks.local(); + + check_impl(class, cpu); + + ke.ipl.lower(ipl); + } else { + const curtd = ke.thread.current(); + check_impl(class, &curtd.warden_data); + } +} + +/// Initialize the kernel warden. +pub fn init() void { + graph_lock = ke.SpinLock.init(""); + + for (0.., &classes) |i, *class| { + class.idx = @truncate(i); + } + + started = true; +} diff --git a/src/mm/phys.zig b/src/mm/phys.zig index 73fd0ee..02878c6 100644 --- a/src/mm/phys.zig +++ b/src/mm/phys.zig @@ -22,7 +22,7 @@ pub var low_memory_event: ke.Event = undefined; var memory_map: *r.BootInfo.MemMap = undefined; var early_allocs: usize = 0; -var list_lock: ke.SpinLock = .init(); +var list_lock: ke.SpinLock = undefined; var free_list: rtl.List = undefined; var free_page_event: ke.Event = undefined; @@ -155,6 +155,7 @@ pub fn init() linksection(r.init) void { free_list.init(); free_page_event.init(.Notification); low_memory_event.init(.Notification); + list_lock = ke.SpinLock.init("PageList"); var total_usable_memory: usize = memory_map.loader_memory_used; log.info("physical memory map:", .{}); diff --git a/src/mm/private.zig b/src/mm/private.zig index 2e505fa..2dcde26 100644 --- a/src/mm/private.zig +++ b/src/mm/private.zig @@ -37,5 +37,5 @@ pub const impl = switch (config.arch) { pub var kernel_space: Space = .{ .pmap = undefined, .arena = undefined, - .lock = .init(), + .lock = undefined, }; diff --git a/src/mm/zone.zig b/src/mm/zone.zig index c023e2a..c3a2a06 100644 --- a/src/mm/zone.zig +++ b/src/mm/zone.zig @@ -193,7 +193,7 @@ var generic_zones: [generic_zones_num]Zone = undefined; var magazines_initialized = false; var all_zones: ?*Zone = null; -var zone_list_lock: ke.Mutex = .init(); +var zone_list_lock: ke.Mutex = undefined; var magazine_zone: Zone = undefined; var cpu_zone: Zone = undefined; @@ -610,8 +610,8 @@ pub const Zone = struct { self.chunk_size = chunk_size; self.ctor = options.ctor; self.dtor = options.dtor; - self.lock = .init(); - self.depot_lock = .init(); + self.lock = ke.Mutex.init("zone"); + self.depot_lock = ke.QSpinLock.init("zone_depot"); self.use_magazines = options.magazines; self.depot = std.mem.zeroes(Depot); @@ -659,7 +659,7 @@ pub const Zone = struct { self.cpus = cpus_alloc(); for (0..ke.ncpus) |i| { - self.cpus[i].value.lock = .init(); + self.cpus[i].value.lock = ke.SpinLock.init("Depot"); self.cpus[i].value.alloc = null; self.cpus[i].value.free = null; self.cpus[i].value.alloc_rounds = 0; @@ -1755,6 +1755,8 @@ pub fn TypedZone(comptime T: type) type { } pub fn early_init() linksection(r.init) void { + zone_list_lock = ke.Mutex.init("zone_list"); + for (0..generic_zones.len, &generic_zones) |i, *zone| { zone.init(generic_zone_names[i], @as(usize, 1) << @intCast(i + 3), .{}); } @@ -1786,7 +1788,7 @@ pub fn late_init() linksection(r.init) void { for (0..ke.ncpus) |i| { z.cpus[i] = .init(.{ - .lock = .init(), + .lock = ke.SpinLock.init("Magazine"), .alloc = null, .free = null, .alloc_rounds = 0, diff --git a/src/proc/private.zig b/src/proc/private.zig index b3af54d..01c5858 100644 --- a/src/proc/private.zig +++ b/src/proc/private.zig @@ -15,7 +15,7 @@ fn turnstile_ctor(ts: *kep.turnstile.Turnstile) void { .queues = undefined, .waiters = 0, .owners = .none, - .lock = .init(), + .lock = .init("turnstile"), }; ts.reset(); } diff --git a/src/tests/mutex.zig b/src/tests/mutex.zig index b8471ee..9cbe8c4 100644 --- a/src/tests/mutex.zig +++ b/src/tests/mutex.zig @@ -157,7 +157,7 @@ fn quiesce_verify() u64 { pub fn start(_: ?*anyopaque) void { for (&locks) |*l| { - l.mutex = .init(); + l.mutex = ke.Mutex.init("test"); l.guard = 0; l.count = 0; } diff --git a/src/tests/turnstile.zig b/src/tests/turnstile.zig index 1fd640a..0ab5577 100644 --- a/src/tests/turnstile.zig +++ b/src/tests/turnstile.zig @@ -111,8 +111,8 @@ fn spawn(prio: u8, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) *ps.Th } fn chain_test() void { - m1 = .init(); - m2 = .init(); + m1 = .init(""); + m2 = .init(""); low_td = spawn(prio_low, &chain_low, null); wait_flag(&low_locked); @@ -209,7 +209,7 @@ fn quiesce_check() void { pub fn start(_: ?*anyopaque) void { chain_test(); - for (&locks) |*l| l.* = .init(); + for (&locks) |*l| l.* = .init(""); nstress = @max(4, @min(ke.ncpus * 2, max_stress));