all: tidy

build.zig.zon

@@ -13,8 +13,8 @@
             .hash = "N-V-__8AAAAAjAwsopZIlbp4U6NN1Eh573uVdLTwl1mkFUug",
         },
         .zonfig = .{
-            .url = "git+https://github.com/rdmsr/zonfig?ref=HEAD#7cb4e030077acbef88458aef2d3fec4331b3b6da",
-            .hash = "zonfig-0.1.0-V0D8-aRMFAAQ3Yb-0u8JNpdceK6lUOWUP7EY5BRXz12C",
+            .url = "git+https://github.com/rdmsr/zonfig?ref=HEAD#d02cbb9262c5e617676a252116c9cee3560b6e59",
+            .hash = "zonfig-0.1.0-V0D8-YtLFAAKh8PQgGaoCiGG1WUxWdqcY-iTysVPllQV",
         },
     },
 

build/config.zig

@@ -28,8 +28,24 @@ pub const Config = struct {
 };
 
 pub fn parseConfig(b: *std.Build) !Config {
+    const gpa = b.allocator;
+    var arena_allocator: std.heap.ArenaAllocator = .init(gpa);
+    defer arena_allocator.deinit();
+    const arena = arena_allocator.allocator();
+
+    var diag: std.zon.parse.Diagnostics = undefined;
+
     const file = try std.Io.Dir.cwd().readFileAllocOptions(b.graph.io, ".config.zig.zon", b.allocator, std.Io.Limit.unlimited, .@"1", 0);
-    return std.zon.parse.fromSlice(Config, b.allocator, file, null, .{ .ignore_unknown_fields = true });
+    return std.zon.parse.fromSlice(
+        Config,
+        .{
+            .gpa = gpa,
+            .source = file,
+            .arena = arena,
+            .diagnostics = &diag,
+            .ignore_unknown_fields = true,
+        },
+    );
 }
 
 pub fn getPlatform(config: Config) !Platform {

build/limine.conf

@@ -11,4 +11,4 @@ timeout: 3
 
 	module_path: boot():/boot/kernel
 	module_string: kernel
-
+	cmdline: test=fireworks

build/run.zig

@@ -58,7 +58,7 @@ pub fn addRun(b: *std.Build, kernel: *std.Build.Step.Compile, loader: *std.Build
 
         switch (plat.arch) {
             .x86_64 => {
-                qemu.addArgs(&.{ "-serial", "stdio", "-m", "1G", "-smp", "4" });
+                qemu.addArgs(&.{ "-serial", "stdio", "-m", "1G", "-smp", "1" });
 
                 if (!debug and builtin.target.os.tag == .linux and builtin.cpu.arch == .x86_64) qemu.addArgs(&.{ "-enable-kvm", "-cpu", "host,+invtsc" });
             },

src/arch/amd64/paravirt/kvm.zig

@@ -38,10 +38,6 @@ const Msr = enum(u32) {
     _,
 };
 
-fn read_msr(msr: Msr) u64 {
-    return amd64.rdmsr(@intFromEnum(msr));
-}
-
 fn write_msr(msr: Msr, value: u64) void {
     amd64.wrmsr(@intFromEnum(msr), value);
 }

src/arch/amd64/root.zig

@@ -1,5 +1,6 @@
 //! Wrappers over amd64 CPU definitions and CPUID.
 const std = @import("std");
+const assert = std.debug.assert;
 
 pub const hypervisor = @import("hypervisor.zig");
 
@@ -286,14 +287,14 @@ pub const Cr4 = packed struct(u64) {
 };
 
 pub inline fn read_cr(comptime n: u8) u64 {
-    comptime std.debug.assert(n != 1);
+    comptime assert(n != 1);
     return asm volatile (std.fmt.comptimePrint("mov %%cr{}, %[ret]", .{n})
         : [ret] "=r" (-> u64),
     );
 }
 
 pub inline fn write_cr(comptime n: u8, value: u64) void {
-    comptime std.debug.assert(n != 1);
+    comptime assert(n != 1);
     asm volatile (std.fmt.comptimePrint("mov %[value], %%cr{}", .{n})
         :
         : [value] "r" (value),
@@ -519,23 +520,23 @@ const known_vendors = [_]struct {
 };
 
 pub const CpuFeatures = struct {
-    x2apic: bool,
-    five_level_paging: bool,
-    gib_pages: bool,
-    tsc_deadline: bool,
-    fxsave: bool,
-    xsave: bool,
-    invariant_tsc: bool,
-    nx: bool,
-    pcid: bool,
-    smap: bool,
-    smep: bool,
-    pge: bool,
-    syscall: bool,
-    umip: bool,
-    vendor: CpuVendor,
-    family: u8,
-    brand_string: [48]u8,
+    x2apic: bool = false,
+    five_level_paging: bool = false,
+    gib_pages: bool = false,
+    tsc_deadline: bool = false,
+    fxsave: bool = false,
+    xsave: bool = false,
+    invariant_tsc: bool = false,
+    nx: bool = false,
+    pcid: bool = false,
+    smap: bool = false,
+    smep: bool = false,
+    pge: bool = false,
+    syscall: bool = false,
+    umip: bool = false,
+    vendor: CpuVendor = undefined,
+    family: u8 = 0,
+    brand_string: [48]u8 = undefined,
 };
 
 pub const HypervisorVendor = enum(u32) {
@@ -563,71 +564,59 @@ fn detect_vendor(string: [12]u8) CpuVendor {
 pub fn detect_cpu_features() void {
     const vendor_info = CpuidRequest.execute(.VendorInfo);
     const max_ext = CpuidRequest.execute(.HighestExtendedFunction).eax;
-
     const max_basic = vendor_info.eax;
 
-    var x2apic = false;
-    var five_level_paging = false;
-    var gib_pages = false;
-    var tsc_deadline = false;
-    var xsave = false;
-    var fxsave = false;
-    var invtsc = false;
-    var nx = false;
-    var pcid = false;
-    var smap = false;
-    var smep = false;
-    var pge = false;
     var hypervisor_flag = false;
-    var syscall = false;
-    var family: u8 = 0;
-    var umip = false;
-
     var vendor_string: [12]u8 = undefined;
-    var brand_string: [48]u8 = undefined;
+
+    cpu_features = .{};
 
     std.mem.writeInt(u32, vendor_string[0..4], vendor_info.ebx, .little);
     std.mem.writeInt(u32, vendor_string[4..8], vendor_info.edx, .little);
     std.mem.writeInt(u32, vendor_string[8..12], vendor_info.ecx, .little);
 
-    const vendor = detect_vendor(vendor_string);
+    cpu_features.vendor = detect_vendor(vendor_string);
 
     if (max_basic >= 0x1) {
         const r = CpuidRequest.execute(.FeatureInfo);
         const ecx: FeatureInfoEcx = @bitCast(r.ecx);
         const edx: FeatureInfoEdx = @bitCast(r.edx);
 
-        x2apic = ecx.x2apic;
-        tsc_deadline = ecx.tsc_deadline;
-        xsave = ecx.xsave;
-        pcid = ecx.pcid;
-        fxsave = edx.fxsave;
-        pge = edx.pge;
+        cpu_features.x2apic = ecx.x2apic;
+        cpu_features.tsc_deadline = ecx.tsc_deadline;
+        cpu_features.xsave = ecx.xsave;
+        cpu_features.pcid = ecx.pcid;
+        cpu_features.fxsave = edx.fxsave;
+        cpu_features.pge = edx.pge;
+
         hypervisor_flag = ecx.hypervisor;
 
         const sig = r.eax;
-        family = @truncate((sig >> 8) & 0xf);
+        var family: u8 = @truncate((sig >> 8) & 0xf);
+
         if (family == 0xf) {
             family +%= @truncate((sig >> 20) & 0xff);
         }
+
+        cpu_features.family = family;
     }
 
     if (max_basic >= 0x7) {
         const r = CpuidRequest.execute(.{ .ExtendedFeatures = .First });
         const ecx: ExtendedFeaturesEcx = @bitCast(r.ecx);
         const ebx: ExtendedFeaturesEbx = @bitCast(r.ebx);
-        smap = ebx.smap;
-        smep = ebx.smep;
-        five_level_paging = ecx.la57;
-        umip = ecx.umip;
+        cpu_features.smap = ebx.smap;
+        cpu_features.smep = ebx.smep;
+        cpu_features.five_level_paging = ecx.la57;
+        cpu_features.umip = ecx.umip;
     }
 
     if (max_ext >= 0x80000001) {
         const r = CpuidRequest.execute(.ExtendedInfo);
         const edx: ExtendedProcessorInfoEdx = @bitCast(r.edx);
-        gib_pages = edx.pdpe1gb;
-        nx = edx.nx;
-        syscall = edx.syscall_sysret;
+        cpu_features.gib_pages = edx.pdpe1gb;
+        cpu_features.nx = edx.nx;
+        cpu_features.syscall = edx.syscall_sysret;
     }
 
     if (max_ext >= 0x80000004) {
@@ -638,6 +627,8 @@ pub fn detect_cpu_features() void {
         };
 
         var off: usize = 0;
+        const brand_string = &cpu_features.brand_string;
+
         for (parts) |p| {
             for ([_]u32{ p.eax, p.ebx, p.ecx, p.edx }) |reg| {
                 const dst: *[4]u8 = @ptrCast(brand_string[off..].ptr);
@@ -650,32 +641,12 @@ pub fn detect_cpu_features() void {
     if (max_ext >= 0x80000007) {
         const r = CpuidRequest.execute(.PowerManagementInfo);
         const edx: PowerManagementInfoEdx = @bitCast(r.edx);
-        invtsc = edx.invtsc;
+        cpu_features.invariant_tsc = edx.invtsc;
     }
 
     if (hypervisor_flag) {
         hypervisor.detect();
     }
-
-    cpu_features = .{
-        .x2apic = x2apic,
-        .five_level_paging = five_level_paging,
-        .gib_pages = gib_pages,
-        .tsc_deadline = tsc_deadline,
-        .fxsave = fxsave,
-        .xsave = xsave,
-        .invariant_tsc = invtsc,
-        .nx = nx,
-        .pcid = pcid,
-        .smap = smap,
-        .smep = smep,
-        .vendor = vendor,
-        .brand_string = brand_string,
-        .pge = pge,
-        .family = family,
-        .umip = umip,
-        .syscall = syscall,
-    };
 }
 
 pub const TssDescriptor = extern struct {

src/arch/riscv64/root.zig

@@ -1,3 +1,5 @@
+//! Wrappers over riscv64 CPU definitions.
+
 pub const name: []const u8 = "riscv64";
 pub const BootInfo = struct {};
 

src/bootvid/bootvid.zig

@@ -39,7 +39,7 @@ const Cursor = struct {
 
 const font_file = @embedFile("sun12x22-ascii.psf");
 const text_color: u32 = 0xFFFFFFFF;
-const bg_color: u32 = 0xFF000000;
+const bg_color: u32 = 0xFF574c8f;
 const crash_bg_color: u32 = 0xFF0000FF;
 const window_padding: usize = 10;
 
@@ -93,8 +93,8 @@ fn load_font() void {
 
 fn plot_character(x: usize, y: usize, c: u8) void {
     const bytes_per_row = font.glyph_size / font.height;
-    const glyph_offset = font.header_size + (@as(usize, c) * font.glyph_size);
-    const glyph_data = font_file[glyph_offset..(glyph_offset + font.glyph_size)];
+    const offset = font.header_size + (@as(usize, c) * font.glyph_size);
+    const glyph_data = font_file[offset .. offset + font.glyph_size];
     const start_x = window_padding + x;
     const start_y = window_padding + y;
 
@@ -102,14 +102,19 @@ fn plot_character(x: usize, y: usize, c: u8) void {
         var row_data: u32 = 0;
 
         for (0..bytes_per_row) |b| {
-            row_data = (row_data << 8) | @as(u32, glyph_data[row * bytes_per_row + b]);
+            row_data = (row_data << 8) |
+                @as(u32, glyph_data[row * bytes_per_row + b]);
         }
 
         const shift_base = bytes_per_row * 8;
 
         for (0..font.width) |col| {
-            if ((row_data & (@as(u32, 1) << @intCast(shift_base - 1 - col))) != 0) {
-                framebuffer[(start_y + row) * framebuffer_width + (start_x + col)] = text_color;
+            const mask = @as(u32, 1) << @intCast(shift_base - 1 - col);
+
+            if (row_data & mask != 0) {
+                framebuffer[
+                    (start_y + row) * framebuffer_width + (start_x + col)
+                ] = text_color;
             }
         }
     }
@@ -134,7 +139,9 @@ fn scroll() void {
         const dst_y = window_padding + (console_height - 1) * font.height + row;
 
         for (0..content_w) |col| {
-            framebuffer[dst_y * framebuffer_width + window_padding + col] = bg_color;
+            framebuffer[
+                dst_y * framebuffer_width + window_padding + col
+            ] = bg_color;
         }
     }
 }
@@ -187,11 +194,10 @@ pub fn init(boot_info: *r.BootInfo) void {
     content_h = framebuffer_height - (window_padding * 2);
 
     console_width = @intCast(content_w / font.width);
-    console_height = std.math.divCeil(
-        u16,
-        @intCast(content_h),
+    console_height = @divFloor(
+        @as(u16, @intCast(content_h)),
         font.height,
-    ) catch unreachable;
+    );
 
     for (0..framebuffer_width * framebuffer_height) |i| {
         framebuffer[i] = bg_color;

src/ex/init.zig

@@ -1,3 +1,5 @@
+//! Initialization of the executive.
+
 const std = @import("std");
 const r = @import("root");
 const pl = r.pl;
@@ -24,7 +26,11 @@ pub fn init() void {
         if (rtl.cmdline.get_string(cline, "test")) |tst| {
             // Run a test if specified.
             if (r.tests.tests.get(tst)) |func| {
-                const t = ps.thread.create_kernel(.Default, .{ .func = func, .arg = r.boot_info }, true) catch
+                const t = ps.thread.create_kernel(
+                    .Default,
+                    .{ .func = func, .arg = r.boot_info },
+                    true,
+                ) catch
                     @panic("oom");
                 ke.sched.enqueue(&t.kern);
             } else {

src/ex/private.zig

@@ -1,3 +1,4 @@
+//! Private executive definitions.
 pub const init_mod = @import("init.zig");
 pub const work = @import("work.zig");
 pub const init = init_mod.init;

src/ex/root.zig

@@ -1,3 +1,5 @@
+//! Public executive definitions.
+
 pub const private = @import("private.zig");
 
 const p = private;

src/ex/string.zig

@@ -44,18 +44,16 @@ const Context = struct {
 pub const InternedString = packed struct(usize) {
     bits: usize,
 
-    const Self = @This();
-
-    fn from_ptr(e: *Entry) Self {
+    fn from_ptr(e: *Entry) InternedString {
         return .{ .bits = @intFromPtr(e) };
     }
 
-    fn as_entry(self: *const Self) ?*Entry {
+    fn as_entry(self: *const InternedString) ?*Entry {
         if (self.bits & 1 != 0) return null;
         return @ptrFromInt(self.bits);
     }
 
-    fn from_inline(s: []const u8) Self {
+    fn from_inline(s: []const u8) InternedString {
         var buf: [@sizeOf(usize)]u8 = @splat(0);
 
         // byte 0: tag (bit 0) | len (bits 1-7)
@@ -71,7 +69,7 @@ pub const InternedString = packed struct(usize) {
         return @sizeOf(usize) > 4 and len <= 7;
     }
 
-    pub fn slice(self: *const Self) []const u8 {
+    pub fn slice(self: *const InternedString) []const u8 {
         if (self.bits & 1 != 0) {
             const buf: *const [@sizeOf(usize)]u8 = @ptrCast(&self.bits);
             const len: usize = buf[0] >> 1;
@@ -97,7 +95,7 @@ pub fn init() void {
     table.init(.{
         .smr = domain,
         .policy = .Balanced,
-    }) catch unreachable;
+    }) catch @panic("Failed to initialize string interning");
 
     entry_zone.init("str entry", .{
         .smr = domain,
@@ -114,6 +112,7 @@ pub fn retain(string: []const u8) InternedString {
         // Fast path where the string already exists.
         const ipl = ke.smr.enter(domain);
         defer ke.smr.exit(domain, ipl);
+
         if (table.get(string)) |e| {
             _ = e.ref.fetchAdd(1, .monotonic);
             return .from_ptr(e);
@@ -121,10 +120,17 @@ pub fn retain(string: []const u8) InternedString {
     }
 
     // The string (probably) doesn't exist, create a new one.
-    var new_entry: *Entry = entry_zone.create() catch unreachable;
+    var new_entry: *Entry = entry_zone.create() catch {
+        // This can't error since the allocation will wait for free memory.
+        unreachable;
+    };
 
-    new_entry.string = mm.zone.gpa.dupe(u8, string) catch unreachable;
-    new_entry.ref = .init(1);
+    new_entry.string = mm.zone.gpa.dupe(u8, string) catch {
+        // This can't error since the allocation will wait for free memory.
+        unreachable;
+    };
+
+    new_entry.ref = std.atomic.Value(u32).init(1);
 
     const ipl = ke.smr.enter(domain);
     const entry = table.get_or_insert(string, &new_entry.link);

src/ex/vmem.zig

@@ -1,13 +1,17 @@
 //! Generic resource allocator, based on the vmem allocator described in:
 //! Jeff Bonwick and Jonathan Adams'
-//! "Magazines and Vmem: Extending the Slab Allocator to Many CPUs and Arbitrary Resources"
-const std = @import("std");
+//! "Magazines and Vmem:  Extending the Slab Allocator to Many CPUs and
+//! Arbitrary Resources"
+
 const rtl = @import("rtl");
 const r = @import("root");
 const mm = r.mm;
 const mmp = mm.private;
 const ke = r.ke;
 
+const std = @import("std");
+const assert = std.debug.assert;
+
 pub const Policy = enum {
     /// Use the smallest free segment that can satisfy the request.
     /// This tends to minimize fragmentation.
@@ -37,9 +41,9 @@ const Segment = struct {
     };
 
     const Linkage = union {
-        /// Linkage into a freelist, valid when type == .Free
+        /// Linkage into a freelist, valid when .Free
         free_link: rtl.List.Entry,
-        /// Linkage into the allocated segments tree, valid when type == .Allocated
+        /// Linkage into the allocated segments tree, valid when .Allocated
         tree_link: rtl.bst.Node,
     };
     /// The type of the segment.
@@ -80,6 +84,8 @@ const max_qcaches = 16;
 
 var seg_zone: mmp.zone.TypedZone(Segment) = undefined;
 
+const SegmentTree = rtl.RBTree(Segment.cmp);
+
 pub const Arena = struct {
     /// Name of the arena, for debugging purposes.
     name: []const u8,
@@ -90,7 +96,7 @@ pub const Arena = struct {
     /// Power of two freelists for free segments, indexed by log2(size).
     freelists: [freelist_count]rtl.List,
     /// Tree for allocated segments, indexed by base address.
-    allocated_segments: rtl.RBTree(Segment.cmp),
+    allocated_segments: SegmentTree,
     /// Last segment allocated from, for NextFit policy.
     rotor: ?*Segment,
     /// Maximum size (**in bytes**) to do quantum caching on.
@@ -105,14 +111,12 @@ pub const Arena = struct {
 
     lock: ke.Mutex,
 
-    const Self = @This();
-
     const Span = struct { addr: usize, size: usize };
 
     const Source = struct {
-        arena: *Self,
-        import: *const fn (*Self, usize, opts: AllocOptions) mm.Error!usize,
-        release: *const fn (*Self, addr: usize, size: usize) void,
+        arena: *Arena,
+        import: *const fn (*Arena, usize, opts: AllocOptions) mm.Error!usize,
+        release: *const fn (*Arena, addr: usize, size: usize) void,
     };
 
     const InitOptions = struct {
@@ -124,7 +128,7 @@ pub const Arena = struct {
 
     /// Initialize the area for use.
     pub fn init(
-        self: *Self,
+        self: *Arena,
         name: []const u8,
         opts: InitOptions,
     ) !void {
@@ -142,7 +146,11 @@ pub const Arena = struct {
         self.qshift = std.math.log2_int(usize, self.quantum);
 
         if (opts.span == null) {
-            rtl.assert(opts.source != null, "Arena init: null span without source", .{});
+            rtl.assert(
+                opts.source != null,
+                "Arena init: null span without source",
+                .{},
+            );
         }
 
         self.list.init();
@@ -170,7 +178,7 @@ pub const Arena = struct {
 
         self.lock = ke.Mutex.init("arena");
 
-        self.allocated_segments = .init();
+        self.allocated_segments = SegmentTree.init();
         self.rotor = null;
 
         // Add initial span.
@@ -179,7 +187,7 @@ pub const Arena = struct {
         }
     }
 
-    pub fn deinit(self: *Self) void {
+    pub fn deinit(self: *Arena) void {
         // Free all segments in the segment list
         var entry = self.list.first();
         while (entry != &self.list.head) {
@@ -191,12 +199,12 @@ pub const Arena = struct {
     }
 
     /// Add a span of memory to the arena.
-    pub fn add(self: *Self, span: Span) !void {
+    pub fn add(self: *Arena, span: Span) !void {
         return self.add_opts(span, .Span);
     }
 
     /// Add a span of memory to the arena.
-    fn add_opts(self: *Self, span: Span, kind: Segment.Type) mm.Error!void {
+    fn add_opts(self: *Arena, span: Span, kind: Segment.Type) mm.Error!void {
         const new_free_seg = try self.alloc_segment();
 
         new_free_seg.* = Segment{
@@ -226,12 +234,15 @@ pub const Arena = struct {
     /// Allocate a segment of memory from the arena.
     /// Options:
     /// - policy: allocation policy to use. Default is .InstantFit.
-    /// - alignment: if specified, the returned segment will be aligned to this boundary.
-    /// - min: if specified, the returned segment will be at least on this address.
-    /// - max: if specified, the returned segment will be at most on this address.
+    /// - alignment: if specified, the returned segment will be aligned to
+    /// this boundary.
+    /// - min: if specified, the returned segment will be at least on this
+    /// address.
+    /// - max: if specified, the returned segment will be at most on this
+    /// address.
     /// Returns the base address of the allocated segment.
     pub fn alloc(
-        self: *Self,
+        self: *Arena,
         size: usize,
         options: AllocOptions,
     ) mm.Error!usize {
@@ -252,7 +263,7 @@ pub const Arena = struct {
     }
 
     fn alloc_impl(
-        self: *Self,
+        self: *Arena,
         size: usize,
         options: AllocOptions,
     ) mm.Error!usize {
@@ -263,20 +274,20 @@ pub const Arena = struct {
                 .NextFit => self.next_fit(size, options),
             };
 
-            if (ret != null) break :blk ret.?;
+            if (ret != null) break :blk ret orelse unreachable;
 
             if (ret == null) {
                 try self.import(size, options);
             }
 
-            // This is guaranteed to terminate eventually, either when the import
-            // brings in new free memory, or when it runs out.
+            // This is guaranteed to terminate eventually, either when
+            // the import brings in new free memory, or when it runs out.
         };
 
         const start, var seg = result;
 
-        std.debug.assert(seg.type == .Free);
-        std.debug.assert(seg.size >= size);
+        assert(seg.type == .Free);
+        assert(seg.size >= size);
 
         // Remove the segment from the freelist.
         self.remove_segment_from_freelist(seg);
@@ -303,11 +314,13 @@ pub const Arena = struct {
             self.add_segment_to_freelist(left_seg);
         }
 
-        // right split: seg is larger than needed, so split into allocated + free remainder.
+        // right split: seg is larger than needed, so split into alloc + free
+        // remainder.
         // e.g. seg=[0x100, 0x10000], size=0x1000:
         //   before: [0x100, 0x10000] being processed
         //   after:  [0x100, 0x1100] allocated, [0x1100, 0x10000] free
-        // the quantum check avoids creating a remainder too small to ever be useful.
+        // the quantum check avoids creating a remainder too small to ever be
+        // useful.
         if (seg.size != size and (seg.size - size) > self.quantum - 1) {
             const new_seg = try self.alloc_segment();
             new_seg.* = .{
@@ -345,7 +358,7 @@ pub const Arena = struct {
     }
 
     /// Try to import memory from the parent arena.
-    fn import(self: *Self, size: usize, options: AllocOptions) mm.Error!void {
+    fn import(self: *Arena, size: usize, options: AllocOptions) mm.Error!void {
         const src = self.source orelse return error.OutOfMemory;
 
         // Ensure no funny stuff happens wrt the lock.
@@ -362,7 +375,7 @@ pub const Arena = struct {
         );
     }
 
-    pub fn free(self: *Self, addr: usize, size: usize) void {
+    pub fn free(self: *Arena, addr: usize, size: usize) void {
         if (size <= self.qcache_max) {
             const i = size >> self.qshift;
             self.qcaches[i - 1].free(@ptrFromInt(addr));
@@ -375,7 +388,7 @@ pub const Arena = struct {
         return self.free_impl(addr, size);
     }
 
-    fn free_impl(self: *Self, addr: usize, size: usize) void {
+    fn free_impl(self: *Arena, addr: usize, size: usize) void {
 
         // Find the allocated segment containing addr.
         var search_node = Segment{
@@ -449,7 +462,7 @@ pub const Arena = struct {
     }
 
     /// Try to release a segment back to its source.
-    fn release(self: *Self, seg: *Segment) bool {
+    fn release(self: *Arena, seg: *Segment) bool {
         const prev_entry = seg.link.prev;
 
         if (prev_entry == &self.list.head) {
@@ -479,7 +492,7 @@ pub const Arena = struct {
 
     // Implementation for the instant fit policy.
     fn instant_fit(
-        self: *Self,
+        self: *Arena,
         size: usize,
         options: AllocOptions,
     ) ?struct { usize, *Segment } {
@@ -506,7 +519,7 @@ pub const Arena = struct {
         // something in the lower freelist, so check there.
         // This is O(n) but should happen very rarely.
         if (!is_pow2) {
-            std.debug.assert(orig_idx != 0);
+            assert(orig_idx != 0);
 
             var it = self.freelists[orig_idx].iterator();
             while (it.next()) : (it.advance()) {
@@ -522,7 +535,7 @@ pub const Arena = struct {
     // Implementation for the best fit policy.
     // Find the smallest segment that can satisfy the request.
     fn best_fit(
-        self: *Self,
+        self: *Arena,
         size: usize,
         options: AllocOptions,
     ) ?struct { usize, *Segment } {
@@ -552,7 +565,7 @@ pub const Arena = struct {
     // Continue searching from the last allocated segment, wrapping around at
     // the end of the arena.
     fn next_fit(
-        self: *Self,
+        self: *Arena,
         size: usize,
         options: AllocOptions,
     ) ?struct { usize, *Segment } {
@@ -588,7 +601,7 @@ pub const Arena = struct {
     // Try to fit the given segment to the request with the given options.
     // Returns the fitted address if successful.
     fn try_to_fit(
-        self: *Self,
+        self: *Arena,
         segment: *const Segment,
         size: usize,
         options: AllocOptions,
@@ -610,22 +623,22 @@ pub const Arena = struct {
         return null;
     }
 
-    fn alloc_segment(self: *Self) !*Segment {
+    fn alloc_segment(self: *Arena) !*Segment {
         _ = self;
         return seg_zone.create();
     }
 
-    fn free_segment(self: *Self, segment: *Segment) void {
+    fn free_segment(self: *Arena, segment: *Segment) void {
         seg_zone.destroy(segment);
         _ = self;
     }
 
-    fn add_segment_to_freelist(self: *Self, segment: *Segment) void {
+    fn add_segment_to_freelist(self: *Arena, segment: *Segment) void {
         const freelist = self.freelist_for_size(segment.size);
         freelist.insert_tail(&segment.linkage.free_link);
     }
 
-    fn remove_segment_from_freelist(self: *Self, segment: *Segment) void {
+    fn remove_segment_from_freelist(self: *Arena, segment: *Segment) void {
         _ = self;
         segment.linkage.free_link.remove();
     }
@@ -634,7 +647,7 @@ pub const Arena = struct {
         return std.math.log2_int(usize, size);
     }
 
-    inline fn freelist_for_size(self: *Self, size: usize) *rtl.List {
+    inline fn freelist_for_size(self: *Arena, size: usize) *rtl.List {
         return &self.freelists[freelist_index(size)];
     }
 };

src/ex/work.zig

@@ -6,8 +6,8 @@
 //! - High
 //! - Normal
 //! - Low
-//! High and Normal worker pools are per-cpu, and there is one global worker pool
-//! for low-priority work.
+//! High and Normal worker pools are per-cpu,
+//! and there is one global worker pool for low-priority work.
 //!
 //! Each CPU worker pool comprises of at least 2 threads, and the low pool
 //! has 4 threads by default. An additional thread will get spawned by the
@@ -69,7 +69,7 @@ pub const WorkItem = struct {
             .func = func,
             .arg = arg,
             .link = undefined,
-            .enqueued = .init(false),
+            .enqueued = std.atomic.Value(bool).init(false),
             .priority = priority,
         };
     }
@@ -89,7 +89,7 @@ pub const DelayedWorkItem = struct {
     ) void {
         self.* = .{
             .item = undefined,
-            .dpc = .init(work_dpc),
+            .dpc = ke.Dpc.init(work_dpc),
             .timer = undefined,
         };
 
@@ -117,14 +117,17 @@ const Pool = struct {
 
         const how_many = min_threads_for_prio(prio);
 
-        self.total_threads = .init(how_many);
+        self.total_threads = std.atomic.Value(usize).init(how_many);
 
         for (0..how_many) |_| {
-            const ctx: Context = .init(self, 0);
+            const ctx = Context.init(self, 0);
 
             var td = try ps.thread.create_kernel(
                 WorkItem.Priority.to_sched_prio(self.prio),
-                .{ .func = work_loop, .arg = @ptrFromInt(ctx.value) },
+                .{
+                    .func = work_loop,
+                    .arg = @ptrFromInt(ctx.value),
+                },
                 true,
             );
 
@@ -139,7 +142,7 @@ const Pool = struct {
         }
     }
 
-    /// Return whether or not a new worker thread should be spawned on this pool.
+    /// Return whether or not a new worker thread should be added to this pool.
     fn should_grow(self: *Pool) bool {
         // NOTE: lockless racy check whether or not the queue is empty
         // and its active count.
@@ -166,20 +169,23 @@ const Pool = struct {
     }
 
     /// Spawn a dynamic worker thread on this pool.
-    fn grow(self: *Pool, cpu: ?u32) !void {
+    fn grow(self: *Pool, cpu: ?u32) void {
         if (self.total_threads.load(.monotonic) >=
             max_threads_for_prio(self.prio))
             return;
 
         _ = self.total_threads.fetchAdd(1, .monotonic);
 
-        const ctx: Context = .init(self, 1);
+        const ctx = Context.init(self, 1);
 
-        var td = try ps.thread.create_kernel(
+        var td = ps.thread.create_kernel(
             WorkItem.Priority.to_sched_prio(self.prio),
             .{ .func = work_loop, .arg = @ptrFromInt(ctx.value) },
             true,
-        );
+        ) catch {
+            // This will wait for free memory.
+            unreachable;
+        };
 
         if (cpu != null) {
             td.kern.last_cpu = cpu;
@@ -205,15 +211,15 @@ var low: Pool = undefined;
 
 fn work_loop(p: ?*anyopaque) void {
     const ctx: Context = .{ .value = @intFromPtr(p) };
-    const dynamic = ctx.get_tag() == 1;
+    const dynamic = ctx.tag() == 1;
 
     while (true) {
         const timeout: ?r.Nanoseconds = if (dynamic)
             .from(r.Seconds.init(5))
         else
             null;
-        const item = ctx.get_ptr().queue.remove(timeout) catch {
-            _ = ctx.get_ptr().total_threads.fetchSub(1, .monotonic);
+        const item = ctx.ptr().queue.remove(timeout) catch {
+            _ = ctx.ptr().total_threads.fetchSub(1, .monotonic);
 
             // We haven't had work in a while, exit.
             ps.thread.exit();
@@ -244,7 +250,12 @@ pub fn enqueue(item: *WorkItem) void {
     const ipl = ke.ipl.raise(.Dispatch);
     defer ke.ipl.lower(ipl);
 
-    if (item.enqueued.cmpxchgStrong(false, true, .acquire, .monotonic) != null) {
+    if (item.enqueued.cmpxchgStrong(
+        false,
+        true,
+        .acquire,
+        .monotonic,
+    ) != null) {
         return;
     }
 
@@ -286,16 +297,16 @@ fn pool_manager(_: ?*anyopaque) void {
             const c = percpu.remote(cpu);
 
             if (c.high.should_grow()) {
-                _ = c.high.grow(cpu) catch {};
+                _ = c.high.grow(cpu);
             }
 
             if (c.normal.should_grow()) {
-                _ = c.normal.grow(cpu) catch {};
+                _ = c.normal.grow(cpu);
             }
         }
 
         if (low.should_grow()) {
-            _ = low.grow(null) catch {};
+            _ = low.grow(null);
         }
     }
 }

src/kern/amd64/impl.zig

@@ -1,3 +1,5 @@
+//! amd64-specific kernel code.
+
 const r = @import("root");
 const ke = r.ke;
 const kep = ke.private;
@@ -139,7 +141,10 @@ pub fn switch_cont_to_normal(
     );
 }
 
-pub fn call_continuation(ctx: *Context, continuation: ke.Continuation) noreturn {
+pub fn call_continuation(
+    ctx: *Context,
+    continuation: ke.Continuation,
+) noreturn {
     const td: *ke.Thread = @alignCast(@fieldParentPtr("context", ctx));
 
     amd64_call_continuation(

src/kern/amd64/init.zig

@@ -1,3 +1,5 @@
+//! amd64-specific kernel initialization code.
+
 const amd64 = @import("arch");
 const int = @import("int.zig");
 const r = @import("root");
@@ -26,7 +28,7 @@ var gdt = extern struct {
     .tss = undefined,
 };
 
-extern fn gdt_load(gdtr: *const amd64.Gdtr) callconv(.{ .x86_64_sysv = .{} }) void;
+extern fn gdt_load(gdtr: *const amd64.Gdtr) callconv(.c) void;
 
 fn early_cpu_init() linksection(r.init) void {
     const gdtr: amd64.Gdtr = .{

src/kern/amd64/int.zig

@@ -1,3 +1,5 @@
+//! amd64 interrupt handling.
+
 const amd64 = @import("arch");
 const std = @import("std");
 const r = @import("root");
@@ -45,15 +47,36 @@ const exception_msg = [32][]const u8{
     "Security Exception",
 };
 
-export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sysv = .{} }) void {
+fn ignore(_: std.Io.Writer.Error!void) void {}
+
+export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.c) void {
     const ipl = ke.ipl.set_hardware(.Device);
 
     if (frame.intno < 32) {
-        std.log.err("Unhandled exception: 0x{x} ({s}), err=0x{x}, pc=0x{x}", .{ frame.intno, exception_msg[frame.intno], frame.errcode, frame.rip });
-        std.log.err("RAX=0x{x:0>16} RBX=0x{x:0>16} RCX=0x{x:0>16} RDX=0x{x:0>16}", .{ frame.rax, frame.rbx, frame.rcx, frame.rdx });
-        std.log.err("RSI=0x{x:0>16} RDI=0x{x:0>16} RBP=0x{x:0>16} RSP=0x{x:0>16}", .{ frame.rsi, frame.rdi, frame.rbp, frame.rsp });
-        std.log.err("R8= 0x{x:0>16}  R9=0x{x:0>16} R10=0x{x:0>16} R11=0x{x:0>16}", .{ frame.r8, frame.r9, frame.r10, frame.r11 });
-        std.log.err("R12=0x{x:0>16} R13=0x{x:0>16} R14=0x{x:0>16} R15=0x{x:0>16}", .{ frame.r12, frame.r13, frame.r14, frame.r15 });
+        std.log.err("Unhandled exception: 0x{x} ({s}), err=0x{x}, pc=0x{x}", .{
+            frame.intno,
+            exception_msg[frame.intno],
+            frame.errcode,
+            frame.rip,
+        });
+
+        std.log.err(
+            "RAX=0x{x:0>16} RBX=0x{x:0>16} RCX=0x{x:0>16} RDX=0x{x:0>16}",
+            .{ frame.rax, frame.rbx, frame.rcx, frame.rdx },
+        );
+        std.log.err(
+            "RSI=0x{x:0>16} RDI=0x{x:0>16} RBP=0x{x:0>16} RSP=0x{x:0>16}",
+            .{ frame.rsi, frame.rdi, frame.rbp, frame.rsp },
+        );
+        std.log.err(
+            "R8= 0x{x:0>16}  R9=0x{x:0>16} R10=0x{x:0>16} R11=0x{x:0>16}",
+            .{ frame.r8, frame.r9, frame.r10, frame.r11 },
+        );
+        std.log.err(
+            "R12=0x{x:0>16} R13=0x{x:0>16} R14=0x{x:0>16} R15=0x{x:0>16}",
+            .{ frame.r12, frame.r13, frame.r14, frame.r15 },
+        );
+
         const cr2 = amd64.read_cr(2);
         const cr3 = amd64.read_cr(3);
         const rflags = amd64.rflags();
@@ -61,21 +84,27 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
         var buf: [128]u8 = undefined;
         var writer = std.Io.Writer.fixed(&buf);
 
-        writer.print("0x{x:0>8} [", .{@as(u64, @bitCast(rflags))}) catch {};
-        if (rflags.carry) writer.writeAll(" CF") catch {};
-        if (rflags.parity) writer.writeAll(" PF") catch {};
-        if (rflags.auxiliary) writer.writeAll(" AF") catch {};
-        if (rflags.zero) writer.writeAll(" ZF") catch {};
-        if (rflags.sign) writer.writeAll(" SF") catch {};
-        if (rflags.trap) writer.writeAll(" TF") catch {};
-        if (rflags.interrupt_enable) writer.writeAll(" IF") catch {};
-        if (rflags.direction) writer.writeAll(" DF") catch {};
-        if (rflags.overflow) writer.writeAll(" OF") catch {};
-        if (rflags.resume_) writer.writeAll(" RF") catch {};
-        if (rflags.virtual_8086_mode) writer.writeAll(" VM") catch {};
-        writer.print(" IOPL={d}]", .{rflags.iopl}) catch {};
+        writer.print("0x{x:0>8} [", .{@as(u64, @bitCast(rflags))}) catch {
+            // Shouldn't fail.
+        };
 
-        std.log.err("CR2=0x{x:0>16} CR3=0x{x:0>16} RFLAGS={s}", .{ cr2, cr3, buf[0..writer.end] });
+        if (rflags.carry) ignore(writer.writeAll(" CF"));
+        if (rflags.parity) ignore(writer.writeAll(" PF"));
+        if (rflags.auxiliary) ignore(writer.writeAll(" AF"));
+        if (rflags.zero) ignore(writer.writeAll(" ZF"));
+        if (rflags.sign) ignore(writer.writeAll(" SF"));
+        if (rflags.trap) ignore(writer.writeAll(" TF"));
+        if (rflags.interrupt_enable) ignore(writer.writeAll(" IF"));
+        if (rflags.direction) ignore(writer.writeAll(" DF"));
+        if (rflags.overflow) ignore(writer.writeAll(" OF"));
+        if (rflags.resume_) ignore(writer.writeAll(" RF"));
+        if (rflags.virtual_8086_mode) ignore(writer.writeAll(" VM"));
+        ignore(writer.print(" IOPL={d}]", .{rflags.iopl}));
+
+        std.log.err(
+            "CR2=0x{x:0>16} CR3=0x{x:0>16} RFLAGS={s}",
+            .{ cr2, cr3, buf[0..writer.end] },
+        );
 
         kep.panic.panic_with_frame("Unhandled exception", frame.rbp);
     }
@@ -102,7 +131,7 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
     }
 }
 
-extern fn idt_load(idt_ptr: *const amd64.Idtr) callconv(.{ .x86_64_sysv = .{} }) void;
+extern fn idt_load(idt_ptr: *const amd64.Idtr) callconv(.c) void;
 
 pub fn init() linksection(r.init) void {
     const idtr: amd64.Idtr = .{
@@ -111,7 +140,12 @@ pub fn init() linksection(r.init) void {
     };
 
     for (&idt, 0..256) |*entry, i| {
-        entry.* = .init(0x28, 0, .InterruptGate, __interrupt_vectors[i]);
+        entry.* = amd64.IdtEntry.init(
+            0x28,
+            0,
+            .InterruptGate,
+            __interrupt_vectors[i],
+        );
     }
 
     idt_load(&idtr);

src/kern/cpu.zig

@@ -1,3 +1,5 @@
+//! CPU-local data.
+
 const config = @import("config");
 const rtl = @import("rtl");
 const r = @import("root");

src/kern/dpc.zig

@@ -1,10 +1,14 @@
-const std = @import("std");
+//! Deferred procedure calls (DPC).
+
 const config = @import("config");
 const rtl = @import("rtl");
 const r = @import("root");
 const ke = r.ke;
 const kep = ke.private;
 
+const std = @import("std");
+const assert = std.debug.assert;
+
 /// Deferred Procedure Call (DPC) structure.
 /// Used for scheduling work to be done when IPL is lowered below `.Dispatch`.
 pub const Dpc = struct {
@@ -23,7 +27,7 @@ pub const Dpc = struct {
             .link = undefined,
             .func = func,
             .arg = undefined,
-            .inserted = .init(false),
+            .inserted = std.atomic.Value(bool).init(false),
         };
     }
 };
@@ -108,7 +112,7 @@ fn dispatch_queue(cpu: u32) void {
         const arg = dpc.arg;
 
         dpc_cpu.lock.release(ipl);
-        std.debug.assert(ke.ipl.current() == .Dispatch);
+        assert(ke.ipl.current() == .Dispatch);
 
         dpc.func(dpc, arg);
     }
@@ -137,7 +141,7 @@ fn dispatch_queue(cpu: u32) void {
     kep.shootdown.process_shootdowns();
 
     if (sched_cpu.next_thread != null) {
-        const curtd = sched_cpu.current_thread.?;
+        const curtd = sched_cpu.current_thread orelse unreachable;
 
         if (!curtd.smr_sections.is_empty()) {
             kep.smr.mark_thread_stalled(curtd);

src/kern/event.zig

@@ -1,3 +1,5 @@
+//! Kernel events.
+
 const ke = @import("root").ke;
 const kep = ke.private;
 
@@ -5,8 +7,6 @@ const kep = ke.private;
 pub const Event = struct {
     hdr: kep.wait.DispatchHeader,
 
-    const Self = @This();
-
     pub const Type = enum {
         /// Wake only one waiter and decrease signaled.
         Synchronization,
@@ -15,7 +15,7 @@ pub const Event = struct {
     };
 
     /// Initialize an event with a specified type.
-    pub fn init(self: *Self, @"type": Type) void {
+    pub fn init(self: *Event, @"type": Type) void {
         self.hdr.init(switch (@"type") {
             .Synchronization => .Synchronization,
             .Notification => .Notification,
@@ -23,7 +23,7 @@ pub const Event = struct {
     }
 
     /// Signal an event, waking its waiter(s).
-    pub fn signal(self: *Self) void {
+    pub fn signal(self: *Event) void {
         const ipl = self.hdr.lock.acquire();
         self.hdr.signaled = 1;
         kep.wait.satisfy_wait(&self.hdr);
@@ -31,7 +31,7 @@ pub const Event = struct {
     }
 
     /// Reset an event.
-    pub fn reset(self: *Self) void {
+    pub fn reset(self: *Event) void {
         const ipl = self.hdr.lock.acquire();
         self.hdr.signaled = 0;
         self.hdr.lock.release(ipl);

src/kern/ipi.zig

@@ -31,7 +31,7 @@ const PerCpu = struct {
 
 const percpu = ke.CpuLocal(PerCpu, undefined);
 
-var frozen_cpus: std.atomic.Value(u32) = .init(0);
+var frozen_cpus = std.atomic.Value(u32).init(0);
 
 /// Call a function on another CPU.
 /// The function will be executed synchronously, i.e this will block until
@@ -69,13 +69,16 @@ pub fn unicast(
 /// The function will be executed synchronously, i.e this will block until
 /// the function finishes running. The callback is called with the sender cpu
 /// and provided argument as arguments.
-pub fn broadcast(func: *const fn (u32, ?*anyopaque) void, arg: ?*anyopaque) void {
+pub fn broadcast(
+    func: *const fn (u32, ?*anyopaque) void,
+    arg: ?*anyopaque,
+) void {
     const ipl = ke.ipl.raise(.Dispatch);
     const curcpu = percpu.local();
 
     curcpu.func = func;
     curcpu.arg = arg;
-    curcpu.counter = .init(@truncate(ke.ncpus - 1));
+    curcpu.counter = std.atomic.Value(u16).init(@truncate(ke.ncpus - 1));
 
     for (0..ke.ncpus) |i| {
         if (i == ke.cpu.current()) continue;

src/kern/ipl.zig

@@ -1,8 +1,12 @@
-const std = @import("std");
+//! Interrupt priority levels.
+
 const r = @import("root");
 const ke = r.ke;
 const kep = ke.private;
 
+const std = @import("std");
+const assert = std.debug.assert;
+
 /// Interrupt priority level (IPL)
 pub const Ipl = enum(u8) {
     /// All interrupts enabled.
@@ -19,7 +23,7 @@ pub const Ipl = enum(u8) {
     }
 
     pub fn value(self: Ipl) u8 {
-        return @intFromEnum(self);
+        return @backingInt(self);
     }
 };
 
@@ -36,7 +40,7 @@ pub const cpu_ipl = ke.ExportedCpuLocal(
 );
 
 pub const percpu = ke.CpuLocal(PerCpu, .{
-    .pending_softints = .init(0),
+    .pending_softints = std.atomic.Value(u8).init(0),
 });
 
 pub fn current() Ipl {
@@ -109,7 +113,7 @@ pub fn set_hardware(new: Ipl) Ipl {
 
 /// Mark a software interrupt of IPL `ipl` on `cpu` as pending.
 pub fn set_softint_pending(cpu: u32, ipl: Ipl) void {
-    std.debug.assert(ipl.value() <= Ipl.Dispatch.value());
+    assert(ipl.value() <= Ipl.Dispatch.value());
     _ = percpu.remote(cpu).pending_softints.bitSet(
         @intCast(ipl.value()),
         .monotonic,
@@ -118,7 +122,7 @@ pub fn set_softint_pending(cpu: u32, ipl: Ipl) void {
 
 /// Mark a software interrupt of IPL `ipl` on `cpu` as handled.
 pub fn clear_softint_pending(cpu: u32, ipl: Ipl) void {
-    std.debug.assert(ipl.value() <= Ipl.Dispatch.value());
+    assert(ipl.value() <= Ipl.Dispatch.value());
     _ = percpu.remote(cpu).pending_softints.bitReset(
         @intCast(ipl.value()),
         .monotonic,
@@ -128,7 +132,7 @@ pub fn clear_softint_pending(cpu: u32, ipl: Ipl) void {
 /// Check whether a software interrupt of IPL `ipl`
 /// is pending on the current CPU.
 pub fn is_softint_pending(ipl: Ipl) bool {
-    std.debug.assert(ipl.value() <= Ipl.Dispatch.value());
+    assert(ipl.value() <= Ipl.Dispatch.value());
     const shift: u3 = @intCast(ipl.value());
     const bit: u8 = @as(u8, 1) << shift;
     return percpu.local().pending_softints.load(.monotonic) & bit != 0;

src/kern/log.zig

@@ -1,3 +1,5 @@
+//! Kernel logging.
+
 const std = @import("std");
 const config = @import("config");
 const r = @import("root");

src/kern/log_ring.zig

@@ -62,10 +62,10 @@
 //! # Wrap handling
 //!
 //! Data blocks are never split across the physical boundary of the data array.
-//! If an allocation would cross the boundary, the block is placed at offset 0 of
-//! the next generation and a stub (containing only the descriptor ID) is left at
-//! the original position so the data ring can be walked linearly during eviction.
-//!
+//! If an allocation would cross the boundary, the block is placed at offset 0
+//! of the next generation and a stub (containing only the descriptor ID) is
+//! left at the original position so the data ring can be walked linearly
+//! during eviction.
 
 const std = @import("std");
 const r = @import("root");
@@ -154,10 +154,10 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
         /// Beginning of the oldest data block.
         tail_lpos: std.atomic.Value(usize),
 
-        const Self = @This();
+        const Ring = @This();
 
-        // Return the exact Id of the descriptor that occupied this physical slot
-        // one generation ago.
+        // Return the exact Id of the descriptor that occupied this physical
+        // slot one generation ago.
         fn prev_wrap(id: Id) Id {
             return id -% desc_count;
         }
@@ -178,15 +178,15 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
             return sz;
         }
 
-        fn to_desc(self: *Self, id: Id) *Desc {
+        fn to_desc(self: *Ring, id: Id) *Desc {
             return &self.descs[id & desc_mask];
         }
 
-        fn to_info(self: *Self, id: Id) *Info {
+        fn to_info(self: *Ring, id: Id) *Info {
             return &self.infos[id & desc_mask];
         }
 
-        fn to_block(self: *Self, lpos: usize) *DataBlock {
+        fn to_block(self: *Ring, lpos: usize) *DataBlock {
             return @ptrCast(@alignCast(&self.data[lpos & data_mask]));
         }
 
@@ -198,12 +198,12 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
 
         // Return true if `lpos_current` has not yet reached `lpos_next`.
         // Uses wrapping subtraction so that:
-        //   - if current > next (another CPU already advanced past the target),
-        //   returns false.
-        //   - if current == next (already at target), the -1 wraps to usize max,
-        //   returns false.
-        //   - if the distance exceeds data_size (should never happen),
-        //   returns false.
+        //  - if current > next (another CPU already advanced past the target),
+        //  returns false.
+        //  - if current == next (already at target), the -1 wraps to usize_max,
+        //  returns false.
+        //  - if the distance exceeds data_size (should never happen),
+        //  returns false.
         fn need_more_space(lpos_current: usize, lpos_next: usize) bool {
             return lpos_next -% lpos_current -% 1 < data_size;
         }
@@ -214,7 +214,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
             desc: Desc = undefined,
         };
 
-        fn read_desc(self: *Self, id: Id) ReadDescResult {
+        fn read_desc(self: *Ring, id: Id) ReadDescResult {
             const desc = self.to_desc(id);
             const info = self.to_info(id);
             var state_var = desc.state.load(.acquire);
@@ -245,7 +245,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
         }
 
         fn read_desc_finalized(
-            self: *Self,
+            self: *Ring,
             id: Id,
             seq: u64,
             desc_out: *Desc,
@@ -263,7 +263,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
                 return error.DataLost;
         }
 
-        fn free_desc(self: *Self, id: Id) void {
+        fn free_desc(self: *Ring, id: Id) void {
             var desc = self.to_desc(id);
             const expected_state: State = .{
                 .state = .Published,
@@ -282,7 +282,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
             );
         }
 
-        fn free_data(self: *Self, begin: usize, end: usize) ?usize {
+        fn free_data(self: *Ring, begin: usize, end: usize) ?usize {
             var desc: Desc = undefined;
             var cur_begin = begin;
 
@@ -330,7 +330,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
             return cur_begin;
         }
 
-        fn data_advance_tail(self: *Self, new_tail_lpos: usize) !void {
+        fn data_advance_tail(self: *Ring, new_tail_lpos: usize) !void {
             if (new_tail_lpos & 1 != 0) {
                 // No data.
                 return;
@@ -355,8 +355,8 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
                         break;
                     }
                 } else {
-                    // Failed to free the data, which means we either lost a race
-                    // or the descriptor is still in use.
+                    // Failed to free the data, which means we either lost a
+                    // race or the descriptor is still in use.
                     // Reload the tail position to try again.
                     const new_tail = self.tail_lpos.load(.acquire);
                     if (new_tail == cur_tail) {
@@ -372,7 +372,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
         }
 
         // Try advancing the tail.
-        fn advance_tail(self: *Self, tail_id: Id) !void {
+        fn advance_tail(self: *Ring, tail_id: Id) !void {
             const res = self.read_desc(tail_id);
             const tail_desc = res.desc;
 
@@ -387,7 +387,8 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
                     return error.ReservationFailed;
                 },
                 .Miss => {
-                    if (tail_desc.state.load(.monotonic).id == prev_wrap(tail_id)) {
+                    const tail_desc_id = tail_desc.state.load(.monotonic).id;
+                    if (tail_desc_id == prev_wrap(tail_id)) {
                         // This must mean that the descriptor is currently
                         // getting reserved by another writer.
                         return error.ReservationFailed;
@@ -411,8 +412,9 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
 
             if (next_res.state == .Published or next_res.state == .Free) {
                 // Advance the tail to the next descriptor.
-                // release: push all previous state changes before the tail update,
-                // guaranteeing the new tail is visible after all evictions.
+                // release: push all previous state changes before the tail
+                // update,  guaranteeing the new tail is visible after all
+                // evictions.
                 _ = self.tail_id.cmpxchgStrong(
                     @as(usize, tail_id),
                     @as(usize, tail_id +% 1),
@@ -433,7 +435,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
         }
 
         // Reserve a new descriptor and make space for it if needed.
-        fn reserve_desc(self: *Self) !Id {
+        fn reserve_desc(self: *Ring) !Id {
             var head_id: Id = @truncate(self.head_id.load(.acquire));
             var new_id: Id = undefined;
             var prev_id: Id = undefined;
@@ -520,7 +522,12 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
             return next_lpos;
         }
 
-        fn alloc_data(self: *Self, size: usize, id: Id, out_position: *BlkPos) ReserveError![]u8 {
+        fn alloc_data(
+            self: *Ring,
+            size: usize,
+            id: Id,
+            out_position: *BlkPos,
+        ) ReserveError![]u8 {
             const blk_size = to_block_size(size);
             var head_lpos = self.head_lpos.load(.monotonic);
             var new_head_lpos = head_lpos;
@@ -534,7 +541,8 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
             while (true) {
                 new_head_lpos = get_next_lpos(head_lpos, blk_size);
 
-                // Make room if needed, try advancing the tail until we have enough space.
+                // Make room if needed, try advancing the tail until we have
+                // enough space.
                 self.data_advance_tail(new_head_lpos -% data_size) catch |e| {
                     out_position.begin = lpos_no_data;
                     out_position.end = lpos_no_data;
@@ -570,7 +578,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
             return @as([*]u8, @ptrCast(&blk.data))[0..size];
         }
 
-        fn read_internal(self: *Self, seq: u64, buf: ?[]u8) !Info {
+        fn read_internal(self: *Ring, seq: u64, buf: ?[]u8) !Info {
             const rdesc = self.to_desc(@truncate(seq));
             const info = self.to_info(@truncate(seq));
             var desc: Desc = undefined;
@@ -601,14 +609,14 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
         }
 
         /// Initialize and return a new ring buffer instance.
-        pub fn init() Self {
+        pub fn init() Ring {
             // Start near usize max so that the first wrap of the ID/lpos space
             // happens almost immediately, testing overflow handling early
             // rather than after billions of records.
             const dummy_id: Id = 0 -% @as(Id, desc_count + 1);
             const blk0: usize = -%@as(usize, data_size);
 
-            var self: Self = std.mem.zeroes(Self);
+            var self: Ring = std.mem.zeroes(Ring);
 
             // Place a dummy descriptor in the last slot. head_id and tail_id
             // both start here so that the first reservation (head_id + 1) lands
@@ -638,7 +646,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
         }
 
         /// Reserve space for a record and return a reservation.
-        pub fn reserve(self: *Self, size: usize) ReserveError!Reservation {
+        pub fn reserve(self: *Ring, size: usize) ReserveError!Reservation {
             if (!check_size(size)) {
                 return error.NoSpace;
             }
@@ -657,15 +665,15 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
 
             if (seq == 0 and (id & desc_mask) != 0) {
                 // First time this slot has been used.
-                // Slot 0 never hits this branch, its seq is set to -(desc_count)
-                // in init() so that the else branch below produces seq=0 on
-                // its first use.
+                // Slot 0 never hits this branch, its seq is set
+                // to -(desc_count) in init() so that the else branch
+                // below produces seq=0 on its first use.
                 info.sequence = @as(u64, id & desc_mask);
             } else {
                 // Slot is being recycled (or is slot 0 on first use).
-                // Adding desc_count advances the sequence by one full ring wrap,
-                // keeping sequence numbers monotonically increasing across
-                // generations.
+                // Adding desc_count advances the sequence by one full
+                // ring wrap, keeping sequence numbers monotonically increasing
+                // across generations.
                 info.sequence = seq +% desc_count;
             }
 
@@ -686,7 +694,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
         }
 
         /// Publish a previously reserved record.
-        pub fn publish(self: *Self, res: Reservation) void {
+        pub fn publish(self: *Ring, res: Reservation) void {
             const desc = self.to_desc(res.id);
 
             _ = desc.state.cmpxchgStrong(
@@ -707,7 +715,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
         }
 
         /// Return the first readable sequence currently retained by the ring.
-        pub fn first_seq(self: *Self) u64 {
+        pub fn first_seq(self: *Ring) u64 {
             var seq: u64 = undefined;
 
             while (true) {
@@ -723,30 +731,31 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
         }
 
         /// Read a record at `seq` into `buf`.
-        pub fn read(self: *Self, seq: u64, buf: ?[]u8) ReadError!Info {
+        pub fn read(self: *Ring, seq: u64, buf: ?[]u8) ReadError!Info {
             var cur_seq = seq;
 
             while (true) {
-                return self.read_internal(cur_seq, buf) catch |err| switch (err) {
-                    error.Invalid => {
-                        const first = self.first_seq();
-                        if (cur_seq < first) {
-                            // Behind the tail, catch up and try again.
-                            cur_seq = first;
+                return self.read_internal(cur_seq, buf) catch |err|
+                    switch (err) {
+                        error.Invalid => {
+                            const first = self.first_seq();
+                            if (cur_seq < first) {
+                                // Behind the tail, catch up and try again.
+                                cur_seq = first;
+                                continue;
+                            }
+                            return error.NotYetAvailable;
+                        },
+                        error.DataLost => {
+                            // Record at cur_seq was overwritten, skip forward.
+                            const first = self.first_seq();
+                            if (cur_seq < first)
+                                cur_seq = first
+                            else
+                                cur_seq += 1;
                             continue;
-                        }
-                        return error.NotYetAvailable;
-                    },
-                    error.DataLost => {
-                        // Record at cur_seq was overwritten, skip forward.
-                        const first = self.first_seq();
-                        if (cur_seq < first)
-                            cur_seq = first
-                        else
-                            cur_seq += 1;
-                        continue;
-                    },
-                };
+                        },
+                    };
             }
         }
     };

src/kern/mutex.zig

@@ -1,3 +1,5 @@
+//! Adaptive lock implementation.
+
 const std = @import("std");
 const rtl = @import("rtl");
 const config = @import("config");
@@ -22,7 +24,7 @@ pub const Mutex = struct {
             };
         }
 
-        return .{ .owner = .init(null) };
+        return .{ .owner = std.atomic.Value(?*ke.Thread).init(null) };
     }
 
     pub fn is_locked(self: *Mutex) bool {
@@ -31,7 +33,8 @@ pub const Mutex = struct {
 
     pub fn acquire(m: *Mutex) void {
         const ipl = ke.ipl.raise(.Dispatch);
-        const curtd = kep.sched.percpu.local().current_thread.?;
+        const curtd = kep.sched.percpu.local().current_thread orelse
+            unreachable;
 
         if (config.warden) {
             kep.warden.check(m.wd, .Lock);
@@ -98,7 +101,7 @@ pub const Mutex = struct {
             kep.turnstile.block(
                 ts,
                 m,
-                .{ .single = owner.? },
+                .{ .single = owner orelse unreachable },
                 .Exclusive,
             );
 
@@ -133,6 +136,7 @@ pub const Mutex = struct {
         }
 
         var waiters: rtl.List = undefined;
+        const turn = ts orelse unreachable;
 
         // Note: wake up all waiters.
         // This so-called "lock barging" (name from WTF::ParkingLot) has been
@@ -140,9 +144,9 @@ pub const Mutex = struct {
         // see this mysterious 70s paper:
         // https://dl.acm.org/doi/pdf/10.1145/850657.850659
         kep.turnstile.signal(
-            ts.?,
+            turn,
             .Exclusive,
-            ts.?.waiters,
+            turn.waiters,
             null,
             &waiters,
         );

src/kern/panic.zig

@@ -1,3 +1,5 @@
+//! Kernel panic handling code.
+
 const std = @import("std");
 const config = @import("config");
 const r = @import("root");
@@ -43,7 +45,10 @@ fn get_symbol_name(addr: usize) ?Symbol {
         const sym = symbols[left - 1];
 
         if (sym.addr <= addr) {
-            return .{ .name = sym.name_ptr[0..sym.name_len], .offset = addr - sym.addr };
+            return .{
+                .name = sym.name_ptr[0..sym.name_len],
+                .offset = addr - sym.addr,
+            };
         }
     }
 
@@ -57,7 +62,11 @@ fn is_kernel_text(addr: usize) bool {
 }
 
 fn walk_stack_frame(base: usize) void {
-    var frame: ?*StackFrame = @ptrFromInt(if (config.arch == .riscv64) base - 16 else base);
+    var frame: ?*StackFrame = @ptrFromInt(if (config.arch == .riscv64)
+        base - 16
+    else
+        base);
+
     var depth: usize = 0;
     while (frame) |f| : (depth += 1) {
         if (depth > 64) break;
@@ -65,16 +74,27 @@ fn walk_stack_frame(base: usize) void {
         if (!is_kernel_text(ret_addr)) {
             break;
         }
-        const sym = get_symbol_name(ret_addr) orelse Symbol{ .name = "???", .offset = 0 };
-        std.log.err("  #{d} {s}+0x{x} - 0x{x}", .{ depth, sym.name, sym.offset, ret_addr });
-        frame = if (config.arch == .riscv64 and f.prev != null)
-            @ptrFromInt(@intFromPtr(f.prev.?) - 16)
-        else
-            f.prev;
+        const sym = get_symbol_name(ret_addr) orelse
+            Symbol{ .name = "???", .offset = 0 };
+
+        std.log.err("  #{d} {s}+0x{x} - 0x{x}", .{
+            depth,
+            sym.name,
+            sym.offset,
+            ret_addr,
+        });
+
+        if (config.arch == .riscv64) {
+            if (f.prev) |fr| {
+                frame = @ptrFromInt(@intFromPtr(fr) - 16);
+            }
+        } else {
+            frame = f.prev;
+        }
     }
 }
 
-var crash_count: std.atomic.Value(u8) = .init(0);
+var crash_count = std.atomic.Value(u8).init(0);
 
 pub fn panic_with_frame(
     msg: []const u8,

src/kern/private.zig

@@ -1,3 +1,5 @@
+//! Private kernel definitions.
+
 const config = @import("config");
 const init_mod = @import("init.zig");
 const r = @import("root");

src/kern/queue.zig

@@ -9,7 +9,9 @@ const r = @import("root");
 const rtl = @import("rtl");
 const ke = r.ke;
 const kep = ke.private;
+
 const std = @import("std");
+const assert = std.debug.assert;
 
 pub const Queue = struct {
     hdr: kep.wait.DispatchHeader,
@@ -56,11 +58,11 @@ pub const Queue = struct {
     /// This blocks until an item is actually popped.
     pub fn remove(self: *Queue, timeout: ?r.Nanoseconds) !*rtl.List.Entry {
         const ipl = self.hdr.lock.acquire();
-        const td = kep.sched.percpu.local().current_thread.?;
+        const td = kep.sched.percpu.local().current_thread orelse unreachable;
 
         if (td.queue) |q| {
-            std.debug.assert(q == self);
-            std.debug.assert(self.active > 0);
+            assert(q == self);
+            assert(self.active > 0);
 
             self.active -= 1;
             if (self.hdr.signaled > 0 and self.active < self.max_active) {
@@ -90,7 +92,7 @@ pub const Queue = struct {
 /// than the queue.
 pub fn signal_wait(queue: *Queue) void {
     const ipl = queue.hdr.lock.acquire();
-    std.debug.assert(queue.active > 0);
+    assert(queue.active > 0);
 
     queue.active -= 1;
 

src/kern/riscv64/impl.zig

@@ -1,3 +1,5 @@
+//! Kernel riscv64-specific code.
+
 const r = @import("root");
 const ke = r.ke;
 const rv64 = @import("arch");
@@ -20,7 +22,12 @@ pub const ThreadContext = extern struct {
         return .{ .sp = stack };
     }
 
-    pub fn reset(self: *@This(), stack_top: r.VAddr, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) @This() {
+    pub fn reset(
+        self: *@This(),
+        stack_top: r.VAddr,
+        entry: *const fn (?*anyopaque) void,
+        arg: ?*anyopaque,
+    ) @This() {
         self.sp = stack_top & ~@as(usize, 15);
         self.ra = @intFromPtr(&thread_start);
         self.s = @splat(0);
@@ -35,11 +42,27 @@ pub const ThreadContext = extern struct {
 };
 
 extern fn riscv_context_load(ctx: *ThreadContext) callconv(.c) noreturn;
-extern fn riscv_context_switch(old: *ThreadContext, new: *ThreadContext, lock: *u8, switching: *bool) callconv(.c) void;
-extern fn riscv_context_switch_cont(old: *ke.Thread.Context, new: *ThreadContext, lock: *u8, switching: *bool) callconv(.c) void;
+
+extern fn riscv_context_switch(
+    old: *ThreadContext,
+    new: *ThreadContext,
+    lock: *u8,
+    switching: *bool,
+) callconv(.c) void;
+
+extern fn riscv_context_switch_cont(
+    old: *ke.Thread.Context,
+    new: *ThreadContext,
+    lock: *u8,
+    switching: *bool,
+) callconv(.c) void;
+
 extern fn thread_start() callconv(.c) noreturn;
 
-export fn riscv_thread_entry(entry_addr: usize, arg_addr: usize) callconv(.c) noreturn {
+export fn riscv_thread_entry(
+    entry_addr: usize,
+    arg_addr: usize,
+) callconv(.c) noreturn {
     const entry: *const fn (?*anyopaque) void = @ptrFromInt(entry_addr);
     entry(@ptrFromInt(arg_addr));
     @panic("Thread entry returned");
@@ -49,24 +72,46 @@ export fn riscv_free_old_stack(stack: usize) callconv(.c) void {
     ke.private.thread.stack_cache_free(stack);
 }
 
-export fn riscv_run_continuation(func_addr: usize, arg: ?*anyopaque) callconv(.c) noreturn {
+export fn riscv_run_continuation(
+    func_addr: usize,
+    arg: ?*anyopaque,
+) callconv(.c) noreturn {
     ke.ipl.lower(.Passive);
     const func: *const fn (?*anyopaque) void = @ptrFromInt(func_addr);
     func(arg);
     @panic("Continuation returned");
 }
 
-pub fn switch_normal_to_normal(old: *ke.Thread.Context, new: *ke.Thread.Context) void {
+pub fn switch_normal_to_normal(
+    old: *ke.Thread.Context,
+    new: *ke.Thread.Context,
+) void {
     const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", old));
-    riscv_context_switch(&old.impl, &new.impl, &thread.lock.inner.locked.raw, &thread.switching.raw);
+    riscv_context_switch(
+        &old.impl,
+        &new.impl,
+        &thread.lock.inner.locked.raw,
+        &thread.switching.raw,
+    );
 }
 
-pub fn switch_cont_to_normal(old: *ke.Thread.Context, new: *ke.Thread.Context) void {
+pub fn switch_cont_to_normal(
+    old: *ke.Thread.Context,
+    new: *ke.Thread.Context,
+) void {
     const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", old));
-    riscv_context_switch_cont(old, &new.impl, &thread.lock.inner.locked.raw, &thread.switching.raw);
+    riscv_context_switch_cont(
+        old,
+        &new.impl,
+        &thread.lock.inner.locked.raw,
+        &thread.switching.raw,
+    );
 }
 
-pub fn call_continuation(ctx: *ke.Thread.Context, continuation: ke.Continuation) noreturn {
+pub fn call_continuation(
+    ctx: *ke.Thread.Context,
+    continuation: ke.Continuation,
+) noreturn {
     const sp = ctx.stack_top & ~@as(usize, 15);
     asm volatile (
         \\mv sp, %[stack]

src/kern/riscv64/int.zig

@@ -1,3 +1,5 @@
+//! riscv64 interrupt handling.
+
 const r = @import("root");
 const ke = r.ke;
 const std = @import("std");
@@ -13,5 +15,9 @@ export fn riscv_trap_handler(cause: usize, pc: usize) callconv(.c) void {
         }
         return;
     }
-    std.debug.panic("trap: cause = 0x{x}, pc = 0x{x}, tval = 0x{x}", .{ cause, pc, rv64.read_csr("stval") });
+    std.debug.panic("trap: cause = 0x{x}, pc = 0x{x}, tval = 0x{x}", .{
+        cause,
+        pc,
+        rv64.read_csr("stval"),
+    });
 }

src/kern/root.zig

@@ -1,3 +1,5 @@
+//! Public kernel interfaces.
+
 // === Exported Modules ===
 pub const private = @import("private.zig");
 

src/kern/sched.zig

@@ -91,9 +91,9 @@
 //! balancing, the load is preferred as to prioritize latency
 //! (i.e. you want to run a thread ASAP).
 //!
-//! See `sched_ule.c` in FreeBSD for the original implementation this is based on.
+//! See `sched_ule.c` in FreeBSD for the original implementation this is based
+//! on.
 
-const std = @import("std");
 const rtl = @import("rtl");
 const config = @import("config");
 const r = @import("root");
@@ -101,10 +101,13 @@ const ke = r.ke;
 const kep = ke.private;
 const pl = r.pl;
 
+const std = @import("std");
+const assert = std.debug.assert;
+
 const runqueues_n = 64;
 const interactivity_threshold = 30;
 const scaling_factor = 50;
-const preempt_threshold = @intFromEnum(ke.Priority.LowRealtime);
+const preempt_threshold = @backingInt(ke.Priority.LowRealtime);
 const balance_interval = @as(u64, config.sched_balance_interval);
 const sched_migration_cost = ke.Tunable(
     u64,
@@ -122,8 +125,9 @@ const pct_window_max = 11 * std.time.us_per_s;
 // Each thread carries a decaying exponential weighted moving average (EWMA)
 // of the time it has spent runnable (running or in a queue). Each CPU maintains
 // the sum of its threads' values, which is used in load balancing decisions to
-// estimate the *actual* load of a CPU (and not just its thread count). Each value
-// is updated using periods of 1024us (~1ms) with a half-life of 32ms, i.e y^32 = 0.5.
+// estimate the *actual* load of a CPU (and not just its thread count).
+// Each value  is updated using periods of 1024us (~1ms) with a half-life of
+// 32ms, i.e y^32 = 0.5.
 // Math goes more in depth in comments below.
 const pelt_scale = 1024;
 const pelt_decay = 32;
@@ -146,7 +150,7 @@ const pelt_halflife = pelt_scale * pelt_decay;
 pub const pelt_load_avg_max = 47742;
 
 // pelt_inv_table[k] = floor(2^32 * 2^(-k/pelt_decay))
-// NOTE: Linux uses 2^32 - 1 as base but this practically yields the same values.
+// NOTE: Linux uses 2^32-1 as base but this practically yields the same values.
 const pelt_inv_table: [pelt_decay]u32 = .{
     0xffffffff, 0xfa83b2db, 0xf5257d15, 0xefe4b99b,
     0xeac0c6e7, 0xe5b906e7, 0xe0ccdeec, 0xdbfbb797,
@@ -223,7 +227,7 @@ pub const Average = struct {
     // Amount of the period that has been accounted for
     period_contrib: usize = 0,
     /// Timestamp (us) up to which history has been accounted.
-    last_update: r.Microseconds = .init(0),
+    last_update: r.Microseconds = r.Microseconds.init(0),
 };
 
 pub const Accounting = struct {
@@ -240,7 +244,7 @@ pub const Accounting = struct {
 pub const percpu = ke.CpuLocal(PerCpu, undefined);
 
 var balance_timer: ke.Timer = undefined;
-var balance_dpc: ke.Dpc = .init(balance);
+var balance_dpc = ke.Dpc.init(balance);
 
 // LCG values taken from FreeBSD.
 var lcg = std.Random.lcg.Wrapping(u32).init(0, 69069, 5);
@@ -260,15 +264,15 @@ fn init_cpu() linksection(r.init) void {
             .status = 0,
             .queues = undefined,
         },
-        .queues_lock = .init("runq"),
+        .queues_lock = ke.SpinLock.init("runq"),
         .idle_queue = undefined,
         .steal_work = false,
-        .load = .init(0),
-        .load_avg = .init(0),
-        .est_load_avg = .init(0),
-        .migratable = .init(0),
-        .current_thread_prio = .init(0),
-        .resched_dpc = .init(kep.sched.clock),
+        .load = std.atomic.Value(usize).init(0),
+        .load_avg = std.atomic.Value(usize).init(0),
+        .est_load_avg = std.atomic.Value(usize).init(0),
+        .migratable = std.atomic.Value(usize).init(0),
+        .current_thread_prio = std.atomic.Value(u8).init(0),
+        .resched_dpc = ke.Dpc.init(kep.sched.clock),
         .resched_timer = undefined,
         .start_timer = false,
         .preemption_reason = .None,
@@ -318,7 +322,8 @@ fn pelt_do_decay(val: u64, n: u64) u64 {
 
     if (rem != 0) {
         // Fractional remainder, index into the table.
-        // The table gives us 2^32 * 2^(-k/pelt_decay), shift by 32 to remove the 2^32.
+        // The table gives us 2^32 * 2^(-k/pelt_decay), shift by 32 to remove
+        // the 2^32.
         v = (v * pelt_inv_table[(rem * pelt_decay) / pelt_halflife]) >> 32;
     }
 
@@ -360,10 +365,12 @@ fn pelt_update(avg: *Average, runnable: bool) void {
         // d1 * y^p
         const c1 = pelt_do_decay(d1, decay_time);
 
-        // For a full period, its contribution is 1024y + 1024y^2 + ... + 1024y^(p-1)
-        //                            inf
+        // For a full period, its contribution is:
+        //     1024y + 1024y^2 + ... + 1024y^(p-1)
+        //
         // We know load_avg_max = p * Sum y^n
         //                            n=0
+        //
         // So, load_avg_max - load_avg_max * y^p yields:
         //     p-1                                                    p-1
         // p * Sum y^n, subtracting the first term (n=0) gives us p * Sum
@@ -541,7 +548,7 @@ fn recompute_priority(td: *ke.Thread) void {
         // Choose a priority based on score, the lower the score,
         // the higher the priority it will be.
         // This is a simple formula I came up with that is probably good enough.
-        @intCast(@intFromEnum(ke.Priority.LowInteractive) +
+        @intCast(@backingInt(ke.Priority.LowInteractive) +
             ((interactivity_threshold - score) / 2))
     else blk: {
         // Thread is not interactive, compute priority from recent CPU usage.
@@ -560,14 +567,14 @@ fn recompute_priority(td: *ke.Thread) void {
         // positive decreases it.
         const nice_off: i16 = @as(i16, td.nice) + ke.Priority.nice_max;
 
-        const raw: i16 = @as(i16, @intFromEnum(ke.Priority.HighBatch)) -
+        const raw: i16 = @as(i16, @backingInt(ke.Priority.HighBatch)) -
             @as(i16, cpu_pri_off) -
             nice_off;
 
         break :blk @intCast(std.math.clamp(
             raw,
-            @intFromEnum(ke.Priority.LowBatch),
-            @intFromEnum(ke.Priority.HighBatch),
+            @backingInt(ke.Priority.LowBatch),
+            @backingInt(ke.Priority.HighBatch),
         ));
     };
 
@@ -589,8 +596,12 @@ fn calendar_queue_increment(cpu: *PerCpu) void {
     }
 }
 
-fn pick_realtime_thread(cpu: *PerCpu, minimum_prio: u8, target: ?u32) ?*ke.Thread {
-    std.debug.assert(cpu.queues_lock.is_locked());
+fn pick_realtime_thread(
+    cpu: *PerCpu,
+    minimum_prio: u8,
+    target: ?u32,
+) ?*ke.Thread {
+    assert(cpu.queues_lock.is_locked());
 
     const runq = &cpu.realtime_queue;
     const minprio = to_sched_prio(minimum_prio);
@@ -610,7 +621,7 @@ fn pick_realtime_thread(cpu: *PerCpu, minimum_prio: u8, target: ?u32) ?*ke.Threa
 
         const curr_queue = &runq.queues[i];
 
-        std.debug.assert(!curr_queue.is_empty());
+        assert(!curr_queue.is_empty());
 
         // Get the first thread of the queue.
         var td: *ke.Thread = @fieldParentPtr(
@@ -647,7 +658,7 @@ fn pick_realtime_thread(cpu: *PerCpu, minimum_prio: u8, target: ?u32) ?*ke.Threa
 }
 
 fn pick_batch_thread(cpu: *PerCpu, target: ?u32) ?*ke.Thread {
-    std.debug.assert(cpu.queues_lock.is_locked());
+    assert(cpu.queues_lock.is_locked());
     const runq = &cpu.calendar_queue;
 
     if (runq.status == 0) return null;
@@ -669,7 +680,7 @@ fn pick_batch_thread(cpu: *PerCpu, target: ?u32) ?*ke.Thread {
 
         const curr_queue = &runq.queues[i];
 
-        std.debug.assert(!curr_queue.is_empty());
+        assert(!curr_queue.is_empty());
 
         // Get the first thread of the queue.
         var td: *ke.Thread = @fieldParentPtr(
@@ -710,7 +721,7 @@ fn pick_batch_thread(cpu: *PerCpu, target: ?u32) ?*ke.Thread {
 }
 
 fn pick_idle_thread(cpu: *PerCpu, target: ?u32) ?*ke.Thread {
-    std.debug.assert(cpu.queues_lock.is_locked());
+    assert(cpu.queues_lock.is_locked());
     if (cpu.idle_queue.is_empty()) return null;
 
     // Just get the first thread from the queue that can run.
@@ -743,7 +754,7 @@ fn insert_in_list(list: *rtl.List, link: *rtl.List.Entry, head: bool) void {
 
 // Insert a thread into its respective queue.
 fn insert_in_queue(cpu: *PerCpu, td: *ke.Thread, preempted: bool) void {
-    std.debug.assert(cpu.queues_lock.is_locked());
+    assert(cpu.queues_lock.is_locked());
 
     td.cpu = null;
 
@@ -766,7 +777,7 @@ fn insert_in_queue(cpu: *PerCpu, td: *ke.Thread, preempted: bool) void {
         // The insertion index is determined by insidx and the the priority of
         // the thread, higher priority threads will be put closer to insidx,
         // which ensures that they are ran more frequently in proportion.
-        var idx = (cpu.insidx + (@intFromEnum(ke.Priority.HighBatch) -
+        var idx = (cpu.insidx + (@backingInt(ke.Priority.HighBatch) -
             td.priority)) % runqueues_n;
 
         if (preempted) {
@@ -800,8 +811,8 @@ fn insert_in_queue(cpu: *PerCpu, td: *ke.Thread, preempted: bool) void {
 
 // Remove a thread from a queue.
 fn remove_from_queue(cpu: *PerCpu, td: *ke.Thread) void {
-    std.debug.assert(td.lock.is_locked());
-    std.debug.assert(cpu.queues_lock.is_locked());
+    assert(td.lock.is_locked());
+    assert(cpu.queues_lock.is_locked());
 
     // Decrease load.
     _ = cpu.load.fetchSub(1, .monotonic);
@@ -840,11 +851,11 @@ fn select_thread(cur: ?*ke.Thread, cpu: *PerCpu, target: ?u32) ?*ke.Thread {
         return td;
     }
 
-    // Nothing lower can preempt this thread.
-    if (cur != null and (cur.?.priority_class() == .Realtime or
-        cur.?.is_interactive()))
-    {
-        return null;
+    if (cur) |curtd| {
+        if (curtd.priority_class() == .Realtime or curtd.is_interactive()) {
+            // Nothing lower can preempt this thread.
+            return null;
+        }
     }
 
     // Then try picking a batch thread.
@@ -856,9 +867,11 @@ fn select_thread(cur: ?*ke.Thread, cpu: *PerCpu, target: ?u32) ?*ke.Thread {
         return td;
     }
 
-    if (cur != null and cur.?.priority_class() == .Idle) {
-        // Idle threads don't preempt each other
-        return null;
+    if (cur) |curtd| {
+        if (curtd.priority_class() == .Idle) {
+            // Idle threads don't preempt each other.
+            return null;
+        }
     }
 
     // Finally try picking from the idle queue.
@@ -890,7 +903,7 @@ fn should_prio_preempt(td: *ke.Thread, other_prio: u8, remote: bool) bool {
     }
 
     if (class == .Timeshare and td.is_interactive() and
-        other_prio < @intFromEnum(ke.Priority.LowInteractive) and remote)
+        other_prio < @backingInt(ke.Priority.LowInteractive) and remote)
     {
         // Interactive threads always preempt batch non-interactive ones
         // on remote CPUs.
@@ -924,7 +937,9 @@ fn should_preempt_cpu(td: *ke.Thread, cpu: *PerCpu, remote: bool) bool {
 // Find a suitable CPU to run the thread.
 fn pick_cpu(td: *ke.Thread, slept: u64) u32 {
     // CPU selection policy in order:
-    // 1. Pick the last CPU the thread ran on if it would run immediately or if it's pinned.
+    // 1. Pick the last CPU the thread ran on if it would run immediately or
+    // if it's pinned.
+    //
     // 2. Pick the least loaded CPU on which the thread can run immediately.
     // 3. Pick the least loaded CPU overall.
 
@@ -993,12 +1008,12 @@ fn pick_cpu(td: *ke.Thread, slept: u64) u32 {
 
     // If we found a preemptible CPU, pick that one.
     // Otherwise pick the least loaded one.
-    return least_preempt orelse least.?;
+    return least_preempt orelse least orelse unreachable;
 }
 
 // Enqueue a thread on a CPU.
 fn enqueue_on_cpu(c: u32, td: *ke.Thread) void {
-    std.debug.assert(td.lock.is_locked());
+    assert(td.lock.is_locked());
 
     const cpu = percpu.remote(c);
     cpu.queues_lock.acquire_no_ipl();
@@ -1097,7 +1112,7 @@ fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void {
         next.can_receive_stack();
 
     if (next.continuation != null and !handoff) {
-        std.debug.assert(next.context.stack_top != 0);
+        assert(next.context.stack_top != 0);
 
         next.context.reset(
             next.context.stack_top,
@@ -1107,7 +1122,7 @@ fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void {
     }
 
     if (next.context.stack_top == 0) {
-        std.debug.assert(handoff);
+        assert(handoff);
     }
 
     next.state.store(.Running, .monotonic);
@@ -1138,7 +1153,7 @@ fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void {
 /// Handle preemption.
 /// This is called in DPC dispatch when it notices `next_thread` is set.
 pub fn handle_preemption(cpu: *PerCpu) void {
-    const cur = cpu.current_thread.?;
+    const cur = cpu.current_thread orelse unreachable;
     var next: *ke.Thread = undefined;
 
     while (true) {
@@ -1157,7 +1172,7 @@ pub fn handle_preemption(cpu: *PerCpu) void {
         }
 
         if (next.context.stack_top == 0 and next.continuation != null) {
-            // This thread has no stack, try allocating one from the stack cache.
+            // This thread has no stack, try getting one from the stack cache.
             // Otherwise, the thread will be waiting for its stack to get
             // allocated and we pick another one in the meantime.
             const new_stack = allocate_stack(next);
@@ -1170,14 +1185,18 @@ pub fn handle_preemption(cpu: *PerCpu) void {
             detach_load_avg(cpu, next);
             continue;
         } else if (next.continuation != null) {
-            next.context.reset(next.context.stack_top, ke.thread.call_continuation, next);
+            next.context.reset(
+                next.context.stack_top,
+                ke.thread.call_continuation,
+                next,
+            );
         }
 
         // Found a thread with a stack.
         break;
     }
 
-    std.debug.assert(cur != next);
+    assert(cur != next);
 
     cpu.current_thread_prio.store(next.priority, .monotonic);
 
@@ -1209,7 +1228,7 @@ pub fn handle_preemption(cpu: *PerCpu) void {
 /// Called every time slice in DPC context.
 pub fn clock(_: *ke.Dpc, _: ?*anyopaque) void {
     const cpu = percpu.local();
-    std.debug.assert(kep.ipl.current() == .Dispatch);
+    assert(kep.ipl.current() == .Dispatch);
 
     const curtd = cpu.current_thread orelse return;
 
@@ -1222,8 +1241,8 @@ pub fn clock(_: *ke.Dpc, _: ?*anyopaque) void {
     cpu.queues_lock.acquire_no_ipl();
     defer cpu.queues_lock.release_no_ipl();
 
-    // Advance the insert index every tick, while keeping a separation of 1 with runidx.
-    // This ensures fairness.
+    // Advance the insert index every tick, while keeping a separation of 1
+    // with runidx. This ensures fairness.
     if (cpu.runidx == cpu.insidx) {
         cpu.insidx = (cpu.insidx + 1) % runqueues_n;
 
@@ -1273,7 +1292,7 @@ pub fn enqueue(td: *ke.Thread) void {
 /// Yield on the current CPU.
 pub fn yield(continuation: ?ke.Continuation) void {
     const ipl = ke.ipl.raise(.Dispatch);
-    const td = percpu.local().current_thread.?;
+    const td = percpu.local().current_thread orelse unreachable;
 
     td.lock.acquire_no_ipl();
     td.switching.store(true, .monotonic);
@@ -1292,7 +1311,7 @@ pub fn yield(continuation: ?ke.Continuation) void {
     ke.ipl.lower(ipl);
 }
 
-/// Yield the current CPU with the current thread already locked at Dispatch IPL.
+/// Yield on the current CPU with the thread already locked at Dispatch IPL.
 pub fn yield_locked(continuation: ?ke.Continuation) void {
     const sched_cpu = percpu.local();
 
@@ -1321,38 +1340,38 @@ pub fn yield_locked(continuation: ?ke.Continuation) void {
             break;
         }
 
-        if (next == cur) {
+        const n = next orelse unreachable;
+
+        if (n == cur) {
             // We just selected the same thread again.
             break;
         }
 
-        while (next.?.switching.load(.monotonic) == true) {
+        while (n.switching.load(.monotonic) == true) {
             std.atomic.spinLoopHint();
         }
 
-        if (next) |n| {
-            const curstack = n.context.stack_top;
+        const curstack = n.context.stack_top;
 
-            const can_handoff =
-                cur.continuation != null and
-                cur.can_relinquish_stack() and
-                n.continuation != null and
-                n.context.stack_top == 0 and
-                n.can_receive_stack();
+        const can_handoff =
+            cur.continuation != null and
+            cur.can_relinquish_stack() and
+            n.continuation != null and
+            n.context.stack_top == 0 and
+            n.can_receive_stack();
 
-            if (curstack != 0 or can_handoff) {
+        if (curstack != 0 or can_handoff) {
+            break;
+        }
+
+        if (curstack == 0) {
+            if (allocate_stack(n)) |new| {
+                n.context.stack_top = new;
                 break;
             }
 
-            if (curstack == 0) {
-                if (allocate_stack(n)) |new| {
-                    n.context.stack_top = new;
-                    break;
-                }
-
-                detach_load_avg(sched_cpu, n);
-                next = null;
-            }
+            detach_load_avg(sched_cpu, n);
+            next = null;
         }
 
         sched_cpu.queues_lock.acquire_no_ipl();
@@ -1385,7 +1404,7 @@ pub fn yield_locked(continuation: ?ke.Continuation) void {
 /// Block the currently running thread.
 pub fn block(cont: ?ke.Continuation) void {
     const ipl = ke.ipl.raise(.Dispatch);
-    const td = percpu.local().current_thread.?;
+    const td = ke.thread.current();
 
     td.lock.acquire_no_ipl();
     block_locked(td, cont);
@@ -1424,8 +1443,8 @@ pub fn unblock(td: *ke.Thread) void {
 }
 
 pub fn unblock_locked(td: *ke.Thread) void {
-    std.debug.assert(td.lock.is_locked());
-    std.debug.assert(td.state.load(.monotonic) == .Blocked);
+    assert(td.lock.is_locked());
+    assert(td.state.load(.monotonic) == .Blocked);
 
     const now = ke.time.read_time();
 
@@ -1450,7 +1469,7 @@ pub fn unblock_locked(td: *ke.Thread) void {
 }
 
 pub fn update_priority_locked(td: *ke.Thread, new_prio: u8) void {
-    std.debug.assert(td.lock.is_locked());
+    assert(td.lock.is_locked());
 
     const old_prio = td.priority;
 
@@ -1476,7 +1495,7 @@ pub fn update_priority_locked(td: *ke.Thread, new_prio: u8) void {
     }
 
     if (state == .Running) {
-        std.debug.assert(td.last_cpu == c);
+        assert(td.last_cpu == c);
 
         cpu.current_thread_prio.store(new_prio, .monotonic);
 
@@ -1625,12 +1644,18 @@ fn find_least_loaded(exclude: ?*ke.CpuMask) ?u32 {
     return least;
 }
 
-fn steal_thread_from_cpu(cpu: *PerCpu, curcpu: ?*PerCpu, target: ?u32) ?*ke.Thread {
+fn steal_thread_from_cpu(
+    cpu: *PerCpu,
+    curcpu: ?*PerCpu,
+    target: ?u32,
+) ?*ke.Thread {
     cpu.queues_lock.acquire_no_ipl();
     defer cpu.queues_lock.release_no_ipl();
 
-    if (curcpu != null and curcpu.?.load.load(.monotonic) > 0)
-        return null;
+    if (curcpu) |c| {
+        if (c.load.load(.monotonic) > 0)
+            return null;
+    }
 
     const td = select_thread(null, cpu, target);
 

src/kern/shootdown.zig

@@ -1,4 +1,5 @@
-//! Kernel-side mechanism for asynchronous TLB shootdown and deferred memory reclamation.
+//! Kernel-side mechanism for asynchronous TLB shootdowns and deferred
+//! memory reclamation.
 
 const std = @import("std");
 const config = @import("config");
@@ -32,8 +33,9 @@ const PerCpu = struct {
     /// States attached to this CPU.
     states: [64]ShootdownState,
     /// States relevant to each CPU.
-    /// If CPU0 sent a shootdown to CPU1, then CPU0 will have `valid_states[CPU1]`
-    /// pointing to that shootdown, which will sit in `states`.
+    /// If CPU0 sent a shootdown to CPU1, then CPU0 will have
+    /// `valid_states[CPU1]` pointing to that shootdown,
+    /// which will sit in `states`.
     valid_states: [config.ncpus]std.atomic.Value(u64),
     /// CPUs that have sent this CPU a shootdown.
     senders: ke.AtomicCpuMask,
@@ -52,17 +54,17 @@ pub var shootdowns: rtl.HandoffList = undefined;
 fn pcpu_init() linksection(r.init) void {
     const local = percpu.local();
 
-    local.valid_states = @splat(.init(0));
-    local.senders = .init(false);
+    local.valid_states = @splat(std.atomic.Value(u64).init(0));
+    local.senders = ke.AtomicCpuMask.init(false);
     local.states = @splat(.{
         .base = 0,
         .npages = 0,
-        .state = .init(slot_free),
+        .state = std.atomic.Value(u16).init(slot_free),
         .payload = undefined,
         .link = undefined,
     });
 
-    local.npages = .init(0);
+    local.npages = std.atomic.Value(u32).init(0);
 }
 
 comptime {
@@ -178,10 +180,6 @@ pub fn process_shootdowns() void {
     }
 }
 
-fn process_shootdowns_kick(_: u32, _: ?*anyopaque) void {
-    process_shootdowns();
-}
-
 /// Submit a shootdown to occur asynchronously on `target_mask`.
 /// Returns an error if a synchronous shootdown was done instead.
 pub fn submit(state: ShootdownState, target_mask: ke.CpuMask) !void {

src/kern/smr.zig

@@ -1,14 +1,16 @@
 //! Safe memory reclamation implementation.
 //! based on FreeBSD's "Global Unbounded Sequences".
 
-const r = @import("root");
 const rtl = @import("rtl");
-const std = @import("std");
 const config = @import("config");
 
+const r = @import("root");
 const ke = r.ke;
 const kep = ke.private;
 
+const std = @import("std");
+const assert = std.debug.assert;
+
 pub const Sequence = u64;
 
 pub const seq_invalid: Sequence = 0;
@@ -34,15 +36,15 @@ pub const Domain = struct {
     }
 
     pub fn init(self: *Domain, preempt: bool) void {
-        self.clock.read_seq = .init(seq_init);
-        self.clock.write_seq = .init(seq_init);
+        self.clock.read_seq = std.atomic.Value(Sequence).init(seq_init);
+        self.clock.write_seq = std.atomic.Value(Sequence).init(seq_init);
         self.preempt = preempt;
 
         for (0..ke.ncpus) |i| {
             self.cpus[i] = .{
-                .current_seq = .init(seq_invalid),
+                .current_seq = std.atomic.Value(Sequence).init(seq_invalid),
                 .stall_lock = ke.SpinLock.init("stall_lock"),
-                .stall_seq = .init(seq_invalid),
+                .stall_seq = std.atomic.Value(Sequence).init(seq_invalid),
                 .stalled = undefined,
                 .stall_goal = seq_invalid,
             };
@@ -74,12 +76,17 @@ pub const Tracker = struct {
 };
 
 /// Used by tests.
-pub var full_scans: std.atomic.Value(usize) = .init(0);
+pub var full_scans = std.atomic.Value(usize).init(0);
 
 /// Scan all CPUs and return the minimum observed value.
 /// If `should_wait` is true, this will spinloop (or block)
 /// until all CPUs have reached the given goal.
-fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence {
+fn scan(
+    dom: *Domain,
+    goal: Sequence,
+    clock: Clock,
+    should_wait: bool,
+) Sequence {
     rtl.barrier.fence(.seq_cst);
 
     const clk_write = clock.write_seq.raw;
@@ -92,7 +99,6 @@ fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence
 
     for (0..ke.ncpus) |i| {
         const cpu = &dom.cpus[i];
-
         var seq = cpu.current_seq.load(.monotonic);
 
         while (seq != seq_invalid) {
@@ -169,8 +175,8 @@ fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence
                     .{ .shared = &cpu.stalled },
                     .Exclusive,
                 );
-                ke.ipl.lower(ipl);
 
+                ke.ipl.lower(ipl);
                 seq = cpu.stall_seq.load(.monotonic);
             }
 
@@ -260,7 +266,9 @@ pub fn advance(dom: *Domain) Sequence {
     return dom.clock.write_seq.fetchAdd(seq_incr, .release) + seq_incr;
 }
 
-/// Pretend-advance the write sequence and return the value for use as a wait goal.
+/// Pretend-advance the write sequence and return the value for use as a wait
+/// goal.
+///
 /// It is guaranteed that all previous memory writes made by
 /// the calling thread are visible.
 /// The global clock isn't advanced unlike `advance`, it is only when commit()
@@ -303,7 +311,7 @@ fn enter_internal(dom: *Domain) Sequence {
     // on a full memory barrier, as it is guaranteed that the CPU was treated as
     // inactive and cannot possibly hold a reference to anything the poll
     // declared reclaimable. See scan().
-    std.debug.assert(cpu.current_seq.load(.monotonic) == seq_invalid);
+    assert(cpu.current_seq.load(.monotonic) == seq_invalid);
 
     const wr_seq = dom.clock.write_seq.load(.monotonic);
 
@@ -420,7 +428,7 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void {
         return;
     }
 
-    const cpu = tracker.cpu.?;
+    const cpu = tracker.cpu orelse unreachable;
 
     // We got preempted.
     cpu.stall_lock.acquire_no_ipl();
@@ -449,7 +457,7 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void {
     }
 
     if (wake and turnstile != null) {
-        const ts = turnstile.?;
+        const ts = turnstile orelse unreachable;
         var waiters: rtl.List = undefined;
         kep.turnstile.signal(ts, .Exclusive, ts.waiters, null, &waiters);
         kep.turnstile.exit(cpu, turnstile);

src/kern/spinlock.zig

@@ -1,3 +1,5 @@
+//! Kernel spinlocks.
+
 const std = @import("std");
 const rtl = @import("rtl");
 const config = @import("config");
@@ -8,19 +10,23 @@ const kep = ke.private;
 
 fn LockTemplate(comptime T: type) type {
     comptime {
-        if (!@hasDecl(T, "acquire_no_ipl")) @compileError("lock backend must implement acquire_no_ipl");
-        if (!@hasDecl(T, "release_no_ipl")) @compileError("lock backend must implement release_no_ipl");
-        if (!@hasDecl(T, "try_acquire_no_ipl")) @compileError("lock backend must implement try_acquire_no_ipl");
-        if (!@hasDecl(T, "is_locked")) @compileError("lock backend must implement is_locked");
+        if (!@hasDecl(T, "acquire_no_ipl"))
+            @compileError("lock backend must implement acquire_no_ipl");
+        if (!@hasDecl(T, "release_no_ipl"))
+            @compileError("lock backend must implement release_no_ipl");
+        if (!@hasDecl(T, "try_acquire_no_ipl"))
+            @compileError("lock backend must implement try_acquire_no_ipl");
+        if (!@hasDecl(T, "is_locked"))
+            @compileError("lock backend must implement is_locked");
     }
 
     return struct {
         inner: T,
         wd: kep.warden.LockData = undefined,
 
-        const Self = @This();
+        const Lock = @This();
 
-        pub fn init(comptime class: []const u8) Self {
+        pub fn init(comptime class: []const u8) Lock {
             if (config.warden) {
                 const cl = kep.warden.find_or_create_lock_class(class);
                 return .{
@@ -32,26 +38,28 @@ fn LockTemplate(comptime T: type) type {
         }
 
         /// Acquire the lock, raising IPL to `ipl`. Returns the previous IPL.
-        pub fn acquire_at(self: *Self, ipl: ke.Ipl) ke.Ipl {
+        pub fn acquire_at(self: *Lock, ipl: ke.Ipl) ke.Ipl {
             const old_ipl = ke.ipl.raise(ipl);
             self.acquire_no_ipl();
             return old_ipl;
         }
 
-        /// Acquire the lock, raising IPL to `.Dispatch`. Returns the previous IPL.
-        pub fn acquire(self: *Self) ke.Ipl {
+        /// Acquire the lock, raising IPL to `.Dispatch`.
+        /// Returns the previous IPL.
+        pub fn acquire(self: *Lock) ke.Ipl {
             return self.acquire_at(.Dispatch);
         }
 
         /// Release the lock and restore IPL to `ipl`.
-        pub fn release(self: *Self, ipl: ke.Ipl) void {
+        pub fn release(self: *Lock, ipl: ke.Ipl) void {
             self.release_no_ipl();
             ke.ipl.lower(ipl);
         }
 
         /// Try to acquire the lock at IPL `.Dispatch`.
-        /// Returns the previous IPL on success, null if the lock is already held.
-        pub fn try_acquire(self: *Self) ?ke.Ipl {
+        /// Returns the previous IPL on success,
+        /// null if the lock is already held.
+        pub fn try_acquire(self: *Lock) ?ke.Ipl {
             const old_ipl = ke.ipl.raise(.Dispatch);
             if (self.inner.try_acquire_no_ipl()) {
                 if (config.warden) {
@@ -66,7 +74,7 @@ fn LockTemplate(comptime T: type) type {
         }
 
         /// Acquire the lock without changing the IPL.
-        pub fn acquire_no_ipl(self: *Self) void {
+        pub fn acquire_no_ipl(self: *Lock) void {
             if (config.warden) {
                 kep.warden.check(self.wd, .Spin);
             }
@@ -79,7 +87,7 @@ fn LockTemplate(comptime T: type) type {
         }
 
         /// Release the lock without changing the IPL.
-        pub fn release_no_ipl(self: *Self) void {
+        pub fn release_no_ipl(self: *Lock) void {
             if (config.warden) {
                 kep.warden.released(self, .Spin);
             }
@@ -89,7 +97,7 @@ fn LockTemplate(comptime T: type) type {
 
         /// Try to acquire the lock without changing the IPL.
         /// Returns true if the lock was acquired.
-        pub fn try_acquire_no_ipl(self: *Self) bool {
+        pub fn try_acquire_no_ipl(self: *Lock) bool {
             const ret = self.inner.try_acquire_no_ipl();
 
             if (ret and config.warden) {
@@ -100,7 +108,7 @@ fn LockTemplate(comptime T: type) type {
         }
 
         /// Returns true if the lock is currently held.
-        pub fn is_locked(self: *Self) bool {
+        pub fn is_locked(self: *Lock) bool {
             return self.inner.is_locked();
         }
     };
@@ -110,16 +118,16 @@ fn LockTemplate(comptime T: type) type {
 pub const SpinLock = LockTemplate(struct {
     locked: std.atomic.Value(u8),
 
-    const Self = @This();
+    const Lock = @This();
 
-    pub fn init() Self {
+    pub fn init() Lock {
         return .{
-            .locked = .init(0),
+            .locked = std.atomic.Value(u8).init(0),
         };
     }
 
     /// Acquire the lock without changing the IPL.
-    pub fn acquire_no_ipl(self: *Self) void {
+    pub fn acquire_no_ipl(self: *Lock) void {
         while (true) {
             if (self.locked.cmpxchgWeak(0, 1, .acquire, .monotonic) == null)
                 return;
@@ -131,16 +139,16 @@ pub const SpinLock = LockTemplate(struct {
     }
 
     /// Release the lock without changing the IPL.
-    pub fn release_no_ipl(self: *Self) void {
+    pub fn release_no_ipl(self: *Lock) void {
         self.locked.store(0, .release);
     }
 
     /// Try to acquire the lock. Return true if lock was acquired.
-    pub fn try_acquire_no_ipl(self: *Self) bool {
+    pub fn try_acquire_no_ipl(self: *Lock) bool {
         return self.locked.cmpxchgStrong(0, 1, .acquire, .monotonic) == null;
     }
 
-    pub fn is_locked(self: *Self) bool {
+    pub fn is_locked(self: *Lock) bool {
         return self.locked.load(.monotonic) == 1;
     }
 });
@@ -199,15 +207,15 @@ pub const QSpinLock = LockTemplate(struct {
     const pending_val: u32 = 0x00000100;
     const pending_loops = 512;
 
-    const Self = @This();
+    const Lock = @This();
 
     data: Data,
 
-    pub fn init() Self {
+    pub fn init() Lock {
         return .{ .data = .{ .val = std.atomic.Value(u32).init(0) } };
     }
 
-    pub fn acquire_no_ipl(self: *Self) void {
+    pub fn acquire_no_ipl(self: *Lock) void {
         // Fast path: try to acquire the lock.
         var v = self.data.val.cmpxchgStrong(
             0,
@@ -262,11 +270,11 @@ pub const QSpinLock = LockTemplate(struct {
         self.data.split.locked_pending.store(1, .release);
     }
 
-    pub fn release_no_ipl(self: *Self) void {
+    pub fn release_no_ipl(self: *Lock) void {
         self.data.low_word.locked.store(0, .release);
     }
 
-    pub fn try_acquire_no_ipl(self: *Self) bool {
+    pub fn try_acquire_no_ipl(self: *Lock) bool {
         return self.data.val.cmpxchgStrong(
             0,
             locked_val,
@@ -275,11 +283,11 @@ pub const QSpinLock = LockTemplate(struct {
         ) == null;
     }
 
-    pub fn is_locked(self: *Self) bool {
+    pub fn is_locked(self: *Lock) bool {
         return self.data.low_word.locked.load(.monotonic) != 0;
     }
 
-    fn queue(self: *Self) void {
+    fn queue(self: *Lock) void {
         // Claim an index from our CPU.
         const idx = pcpu.local().curr_idx;
         pcpu.local().curr_idx += 1;
@@ -369,6 +377,8 @@ pub const QSpinLock = LockTemplate(struct {
             std.atomic.spinLoopHint();
         }
 
-        next.?.locked.store(1, .release);
+        const n = next orelse unreachable;
+
+        n.locked.store(1, .release);
     }
 });

src/kern/thread.zig

@@ -1,4 +1,5 @@
-const std = @import("std");
+//! Kernel thread management.
+
 const rtl = @import("rtl");
 const config = @import("config");
 
@@ -6,9 +7,10 @@ const r = @import("root");
 const ke = r.ke;
 const kep = ke.private;
 
-pub const Priority = enum(u8) {
-    const Self = @This();
+const std = @import("std");
+const assert = std.debug.assert;
 
+pub const Priority = enum(u8) {
     /// Reserved for the idle thread.
     IdleThread = 0,
     /// Background kernel work.
@@ -42,18 +44,18 @@ pub const Priority = enum(u8) {
 
     // The low and top 20 priorities from the batch range are reserved
     // for nice.
-    pub const cpu_range = @intFromEnum(Self.HighBatch) - (nice_max * 2) - 1;
+    pub const cpu_range = @backingInt(Priority.HighBatch) - (nice_max * 2) - 1;
 
     pub fn class_from_prio(prio: u8) Class {
-        if (prio >= @intFromEnum(Self.LowRealtime) and
-            prio <= @intFromEnum(Self.HighRealtime))
+        if (prio >= @backingInt(Priority.LowRealtime) and
+            prio <= @backingInt(Priority.HighRealtime))
             return .Realtime;
 
-        if (prio >= @intFromEnum(Self.LowBatch) and
-            prio < @intFromEnum(Self.LowRealtime))
+        if (prio >= @backingInt(Priority.LowBatch) and
+            prio < @backingInt(Priority.LowRealtime))
             return .Timeshare;
 
-        if (prio <= @intFromEnum(Self.Idle))
+        if (prio <= @backingInt(Priority.Idle))
             return .Idle;
 
         unreachable;
@@ -110,7 +112,7 @@ pub const Thread = struct {
     lock: ke.SpinLock,
     /// Niceness value.
     nice: i8,
-    /// Effective priority value of the thread: `max(base_priority, inherited_prio)`.
+    /// Effective priority value of the thread: `max(base, inherited)`.
     priority: u8,
     /// Base priority value of the thread.
     base_priority: u8,
@@ -188,17 +190,17 @@ pub const Thread = struct {
             .context = .init_with_stack(opts.stack),
             .lock = ke.SpinLock.init("thread"),
             .nice = 0,
-            .priority = @intFromEnum(opts.priority),
-            .base_priority = @intFromEnum(opts.priority),
+            .priority = @backingInt(opts.priority),
+            .base_priority = @backingInt(opts.priority),
             .inherited_prio = 0,
             .pinned = false,
-            .state = .init(.Ready),
+            .state = std.atomic.Value(State).init(.Ready),
             .runq_link = .{},
             .last_cpu = null,
             .cpu = null,
             .runq = null,
             .runq_idx = 0,
-            .wait_status = .init(.Satisfied),
+            .wait_status = std.atomic.Value(kep.wait.Status).init(.Satisfied),
             .inner_waitblocks = undefined,
             .waitblocks = &thread.inner_waitblocks,
             .wait_reason = null,
@@ -213,11 +215,11 @@ pub const Thread = struct {
             .queue = null,
             .queue_item = null,
             .continuation = opts.entry,
-            .switching = .init(false),
+            .switching = std.atomic.Value(bool).init(false),
             .avg = .{},
             .acct = .{},
             .smr_sections = undefined,
-            .hard_affinity = .init(true),
+            .hard_affinity = ke.CpuMask.init(true),
             .warden_data = .{},
         };
 
@@ -249,8 +251,8 @@ pub const Thread = struct {
     }
 
     pub fn is_interactive(self: *Thread) bool {
-        return self.priority >= @intFromEnum(Priority.LowInteractive) and
-            self.priority <= @intFromEnum(Priority.HighInteractive);
+        return self.priority >= @backingInt(Priority.LowInteractive) and
+            self.priority <= @backingInt(Priority.HighInteractive);
     }
 
     pub fn can_relinquish_stack(self: *Thread) bool {
@@ -309,7 +311,7 @@ pub fn exit() void {
 
 /// Return the currently running thread.
 pub fn current() *ke.Thread {
-    return kep.sched.percpu.local().current_thread.?;
+    return kep.sched.percpu.local().current_thread orelse unreachable;
 }
 
 // -- Stack allocation and continuations ---------------------------------------
@@ -344,7 +346,8 @@ const Cpu = struct {
 
 /// Number of excess stacks allowed in the global stack depot before trimming.
 /// Note that this is in addition to the global minimum of `ncpus` stacks.
-/// Keep this low for minimal memory overhead, but potentially increased latency.
+/// Keep this low for minimal memory overhead, but potentially
+/// increased latency.
 const excess_stacks = ke.Tunable(u32, 1, "ke.thread.excess_stacks");
 
 var global_depot_lock: ke.SpinLock = undefined;
@@ -355,7 +358,7 @@ const percpu_count_max = 2;
 
 const wma_unit = 256;
 
-pub var thread_stack_queue: rtl.HandoffList = .init(stack_activation);
+pub var thread_stack_queue = rtl.HandoffList.init(stack_activation);
 
 inline fn wma_mix(old: u32, new: u32) u32 {
     // Keep 75% of old sample and 25% of new.
@@ -380,14 +383,14 @@ pub fn init() void {
 
 pub fn call_continuation(ptr: ?*anyopaque) noreturn {
     const td: *ke.Thread = @ptrCast(@alignCast(ptr));
-    std.debug.assert(td.continuation != null);
+    assert(td.continuation != null);
 
     // Clean things that were set up by the wait (e.g timeout).
     _ = kep.wait.post_wait(td) catch {
         // Don't care about the status.
     };
 
-    const cont = td.continuation.?;
+    const cont = td.continuation orelse unreachable;
     td.continuation = null;
 
     kep.impl.call_continuation(&td.context, cont);
@@ -466,7 +469,7 @@ pub fn stack_cache_update() ?*Stack {
     // If we have more than `excess_stacks` sitting unused, then trim.
     // Ensure there are still `ncpus` stacks sitting in the depot.
     if (excess > excess_stacks.load()) {
-        std.debug.assert(global_depot.count >= excess);
+        assert(global_depot.count >= excess);
 
         // We want every CPU to have at least one stack sitting in the depot.
         const floor: u32 = @truncate(ke.ncpus);

src/kern/time.zig

@@ -43,9 +43,6 @@ var state = rtl.SeqLock(State).init(.{
 });
 
 // Convert a TimeCounter's count to nanoseconds.
-// For 64-bit counters (`mask == maxInt(u64)`), uses a fast multiply-shift path
-// via precomputed `p` and `n` to avoid 128-bit division at runtime.
-// Falls back to a division-based conversion for narrower counters.
 fn ticks_to_ns(tc: *TimeCounter, count: u64) u64 {
     if (tc.p != 0) {
         const val: u128 = @as(u128, @intCast(count)) * tc.n;
@@ -132,8 +129,8 @@ pub fn update_overflow() void {
     const cur_count = tc.read_count() & tc.mask;
 
     if (s.initial_count > cur_count) {
-        // Counter has wrapped, use wrapping arithmetic to figure out by how much,
-        // then fold it into the offset and reset `initial_count`.
+        // Counter has wrapped, use wrapping arithmetic to figure out by how
+        // much, then fold it into the offset and reset `initial_count`.
         const overflowed_by = (cur_count -% s.initial_count) & tc.mask;
         state.store(.{
             .initial_count = cur_count,

src/kern/timer.zig

@@ -10,9 +10,11 @@ const pl = r.pl;
 const std = @import("std");
 const assert = std.debug.assert;
 
+const TimerHeap = rtl.PairingHeap(.Min, kep.timer.cmp_timer);
+
 const PerCpu = struct {
     /// Heap of pending timers on this CPU.
-    timers: rtl.PairingHeap(.min, kep.timer.cmp_timer),
+    timers: TimerHeap,
     /// Lock over the timer heap.
     lock: ke.SpinLock,
     dpc: ke.Dpc,
@@ -44,8 +46,8 @@ pub const Timer = struct {
     pub fn init(self: *Timer) void {
         self.* = .{
             .hdr = undefined,
-            .state = .init(.Stopped),
-            .deadline = .init(0),
+            .state = std.atomic.Value(State).init(.Stopped),
+            .deadline = r.Nanoseconds.init(0),
             .dpc = null,
             .node = .{},
             .cpu = null,
@@ -56,7 +58,7 @@ pub const Timer = struct {
 };
 
 const percpu = ke.CpuLocal(PerCpu, .{
-    .timers = .init(),
+    .timers = TimerHeap.init(),
     .lock = undefined,
     .dpc = .init(handle_expiry),
 });
@@ -127,7 +129,7 @@ pub fn cancel(timer: *Timer) void {
                 }
             },
             .Pending => {
-                const cpu = timer.cpu.?;
+                const cpu = timer.cpu orelse unreachable;
                 cpu.lock.acquire_no_ipl();
 
                 // Re-check under the lock.
@@ -170,7 +172,7 @@ pub fn clock() void {
 
 // Called in a DPC when a timer has expired.
 fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void {
-    std.debug.assert(kep.ipl.current() == .Dispatch);
+    assert(kep.ipl.current() == .Dispatch);
     const cpu = percpu.local();
 
     while (true) {

src/kern/tunable.zig

@@ -53,12 +53,12 @@ fn store_int(comptime T: type, addr: *anyopaque, value: anytype) bool {
 }
 
 fn apply_num(
-    comptime ValueT: type,
+    comptime VT: type,
     cmdline: []const u8,
     entry: *const Entry,
     info: std.builtin.Type.Int,
 ) void {
-    const value = rtl.cmdline.get_number(ValueT, cmdline, entry.name) catch |e| {
+    const value = rtl.cmdline.get_number(VT, cmdline, entry.name) catch |e| {
         if (e == error.Format) {
             std.log.warn(
                 "Invalid number value for '{s}', falling back.",
@@ -68,7 +68,7 @@ fn apply_num(
         return;
     };
 
-    const sign = @typeInfo(ValueT).int.signedness;
+    const sign = @typeInfo(VT).int.signedness;
 
     _ = switch (info.bits) {
         8 => store_int(@Int(sign, 8), entry.address, value),

src/kern/turnstile.zig

@@ -22,9 +22,10 @@
 //! a lock, which might get preempted by another medium-priority thread.
 //!
 //! Priority inheritance is achieved through turnstiles by walking a
-//! given turnstile's owner chain and willing the priority of the highest priority
-//! waiter, i.e it will go through turnstile.owner.turnstile.owner...
-//! until the end of the chain, inheriting the waiter's priority along the way.
+//! given turnstile's owner chain and willing the priority of the
+//! highest priority waiter, i.e it will go through
+//! turnstile.owner.turnstile.owner...  until the end of the chain,
+//! inheriting the waiter's priority along the way.
 //!
 //! ## Locking
 //! ----------
@@ -41,15 +42,17 @@
 //!
 //! ## Resources
 //! ------------
-//! See the original Illumos implementation, the Solaris book or FreeBSD book for
-//! a more in-depth description.
+//! See the original Illumos implementation, the Solaris book or FreeBSD book
+//! for a more in-depth description.
 //! I have also written more about turnstiles here:
-//! https://rdmsr.github.io/writing/turnstiles/.
+//! https://gregoire.dev/writing/turnstiles/.
 
 const ke = @import("root").ke;
 const kep = ke.private;
 const rtl = @import("rtl");
+
 const std = @import("std");
+const assert = std.debug.assert;
 
 const num_chains = 128;
 const hash_mask = num_chains - 1;
@@ -229,7 +232,8 @@ fn update_prio(td: *ke.Thread) void {
 /// `to.lock` and the owning turnstile's lock.
 fn donate_to(boost: *Boost, to: *ke.Thread, pri: u8) void {
     // A lend only ever raises.
-    if (boost.donated != null and pri <= boost.donated.?) return;
+    if (boost.donated != null and pri <= boost.donated orelse unreachable)
+        return;
 
     if (boost.donated == null) to.turnstiles_owned.insert_head(&boost.link);
     boost.donated = pri;
@@ -306,7 +310,7 @@ fn propagate(curtd: *ke.Thread) void {
         // We hold `thread.lock` and `root.lock`.
         const obj = thread.waiting_on orelse break;
 
-        std.debug.assert(thread.waiting_on == obj);
+        assert(thread.waiting_on == obj);
 
         const ts = thread.turnstile;
 
@@ -319,8 +323,8 @@ fn propagate(curtd: *ke.Thread) void {
             // on the same turnstile.
             if (!ts.lock.try_acquire_no_ipl()) {
                 // The turnstile could not be acquired, drop everything and
-                // restart the walk from the start. Priority inheritance is idem-
-                // potent so there is no issue in applying it many times.
+                // restart the walk from the start. Priority inheritance is
+                // idempotent so there is no issue in applying it many times.
                 thread.lock.release_no_ipl();
 
                 if (root) |ro| ro.lock.release_no_ipl();
@@ -443,9 +447,9 @@ pub fn block(
     const ipl = ke.ipl.current();
 
     var ts = turnstile;
-    const queue_idx = @intFromEnum(queue);
+    const queue_idx = @backingInt(queue);
 
-    std.debug.assert(chain.lock.is_locked());
+    assert(chain.lock.is_locked());
 
     if (ts) |turn| {
         // Another thread already donated its turnstile,
@@ -500,7 +504,8 @@ pub fn block(
     _ = ke.ipl.raise(ipl);
 }
 
-/// Signal the end of a turnstile-backed wait and return a list of waiters to wake.
+/// Signal the end of a turnstile-backed wait and return a list of waiters
+/// to wake.
 /// Do hand-off to new_owner if specified.
 /// The locks acquired by `lookup` are still held on return.
 pub fn signal(
@@ -510,7 +515,7 @@ pub fn signal(
     new_owner: ?*ke.Thread,
     waiters: *rtl.List,
 ) void {
-    const queue_idx = @intFromEnum(queue);
+    const queue_idx = @backingInt(queue);
 
     // Revoke any priority we gave to the owner(s).
     revoke(ts);
@@ -546,8 +551,9 @@ pub fn signal(
 pub fn wakeup(waiters: *rtl.List) void {
     // Now wake all the waiters, this ensures that the turnstile and chain lock
     // hold times stay low.
-    // Also, if we did this *before* unlocking `ts`, there is no guarantee that it
-    // is still alive, as the last waiter could've woken up, exited, and freed it.
+    // Also, if we did this *before* unlocking `ts`, there is no guarantee that
+    // it is still alive, as the last waiter could've woken up, exited,
+    // and freed it.
     // Instead of special casing it, let's just do the wakeup in a nicer
     // environment here.
     var it = waiters.iterator();
@@ -562,9 +568,9 @@ pub fn wakeup(waiters: *rtl.List) void {
 fn dequeue(ts: *Turnstile, td: *ke.Thread) *Waiter {
     td.lock.acquire_no_ipl();
 
-    std.debug.assert(td.turnstile == ts);
-    std.debug.assert(td.turnstile_waiter != null);
-    std.debug.assert(!ts.queues[0].is_empty() or !ts.queues[1].is_empty());
+    assert(td.turnstile == ts);
+    assert(td.turnstile_waiter != null);
+    assert(!ts.queues[0].is_empty() or !ts.queues[1].is_empty());
 
     const waiter = td.turnstile_waiter.?;
     waiter.link.remove();

src/kern/wait.zig

@@ -133,7 +133,7 @@ pub fn wait_any(
     const ipl = ke.ipl.raise(.Dispatch);
     defer ke.ipl.lower(ipl);
 
-    const curtd = kep.sched.percpu.local().current_thread.?;
+    const curtd = ke.thread.current();
     const obj_count = objects.len;
     const has_timeout = opts.timeout != null;
     const total_count = obj_count + @intFromBool(has_timeout);
@@ -264,7 +264,7 @@ pub fn wait_any(
 
 /// Satisfy a wait on an object.
 pub fn satisfy_wait(obj: *DispatchHeader) void {
-    std.debug.assert(obj.lock.is_locked());
+    assert(obj.lock.is_locked());
 
     const all = obj.type == .Notification;
 
@@ -274,8 +274,8 @@ pub fn satisfy_wait(obj: *DispatchHeader) void {
         var wb: *WaitBlock = @fieldParentPtr("link", obj.waitblocks.first());
         var td = wb.thread;
 
-        std.debug.assert(wb.status == .Active);
-        std.debug.assert(wb.object == obj);
+        assert(wb.status == .Active);
+        assert(wb.object == obj);
 
         // Remove it.
         wb.link.remove();

src/kern/warden.zig

@@ -124,20 +124,24 @@ pub fn find_or_create_lock_class(name: []const u8) ?*Class {
     const ipl = graph_lock.acquire_at(.High);
     defer graph_lock.release(ipl);
 
-    var class, const bucket = find_lock_class(name);
+    const found, const bucket = find_lock_class(name);
 
-    if (class != null) {
-        return class;
+    if (found != null) {
+        return found;
     }
 
     assert(num_active_classes < num_classes);
 
     // Get a new spot for our class.
-    class = &classes[num_active_classes];
-    class.?.name = name;
+    const class = &classes[num_active_classes];
+    class.name = name;
 
     // Insert it so that we can find it by name.
-    bucket.?.insert(&class.?.entry) catch unreachable;
+    bucket.?.insert(&class.entry) catch {
+        // This would only fail if the lock class already existed.
+        // Since this was accounted for earlier, the error can never happen.
+        unreachable;
+    };
 
     num_active_classes += 1;
 

src/loader/amd64/impl.zig

@@ -1,3 +1,5 @@
+//! amd64-specific loader code.
+
 const amd64 = @import("arch");
 const r = @import("root");
 const pmap = @import("../pmap.zig");
@@ -75,7 +77,11 @@ pub inline fn make_table_pte(pa: usize) Pte {
     };
 }
 
-pub inline fn make_leaf_pte(pa: usize, flags: r.mem.MapFlags, level: usize) Pte {
+pub inline fn make_leaf_pte(
+    pa: usize,
+    flags: r.mem.MapFlags,
+    level: usize,
+) Pte {
     return Pte{
         .present = true,
         .writable = flags.write,

src/loader/limine/impl.zig

@@ -1,56 +1,70 @@
+//! Limine-specific code for the loader.
+
 const std = @import("std");
 const limine = @import("limine.zig");
 const r = @import("root");
 
-pub export var limine_base_revision: [3]u64 linksection(".limine_requests") = limine.base_revision(6);
+pub export var base_revision: [3]u64 linksection(".limine_requests") =
+    limine.base_revision(6);
 
-pub export var framebuffer_request: limine.FramebufferRequest linksection(".limine_requests") = .{
-    .id = limine.framebuffer_request_id,
-    .revision = 0,
-    .response = null,
-};
+pub export var framebuffer_request linksection(".limine_requests") =
+    limine.FramebufferRequest{
+        .id = limine.framebuffer_request_id,
+        .revision = 0,
+        .response = null,
+    };
 
-pub export var limine_requests_start_marker: [4]u64 linksection(".limine_requests_start") = limine.requests_start_marker;
-pub export var limine_requests_end_marker: [2]u64 linksection(".limine_requests_end") = limine.requests_end_marker;
+pub export var start_marker: [4]u64 linksection(".limine_requests_start") =
+    limine.requests_start_marker;
 
-pub export var memmap_request: limine.MemmapRequest linksection(".limine_requests") = .{
-    .id = limine.memmap_request_id,
-    .revision = 0,
-    .response = null,
-};
+pub export var end_marker: [2]u64 linksection(".limine_requests_end") =
+    limine.requests_end_marker;
 
-pub export var rsdp_request: limine.RsdpRequest linksection(".limine_requests") = .{
-    .id = limine.rsdp_request_id,
-    .revision = 0,
-    .response = null,
-};
+pub export var memmap_request linksection(".limine_requests") =
+    limine.MemmapRequest{
+        .id = limine.memmap_request_id,
+        .revision = 0,
+        .response = null,
+    };
 
-pub export var cmdline_request: limine.CmdlineRequest linksection(".limine_requests") = .{
-    .id = limine.cmdline_request_id,
-    .revision = 0,
-    .response = null,
-};
+pub export var rsdp_request linksection(".limine_requests") =
+    limine.RsdpRequest{
+        .id = limine.rsdp_request_id,
+        .revision = 0,
+        .response = null,
+    };
 
-pub export var hhdm_request: limine.HHDMRequest linksection(".limine_requests") = .{
-    .id = limine.hhdm_request_id,
-    .revision = 0,
-    .response = null,
-};
+pub export var cmdline_request linksection(".limine_requests") =
+    limine.CmdlineRequest{
+        .id = limine.cmdline_request_id,
+        .revision = 0,
+        .response = null,
+    };
 
-pub export var kernel_request: limine.ExecutableAddressRequest linksection(".limine_requests") = .{
-    .id = limine.executable_address_request_id,
-    .revision = 0,
-    .response = null,
-};
+pub export var hhdm_request linksection(".limine_requests") =
+    limine.HHDMRequest{
+        .id = limine.hhdm_request_id,
+        .revision = 0,
+        .response = null,
+    };
 
-pub export var module_request: limine.ModuleRequest linksection(".limine_requests") = .{
-    .id = limine.module_request_id,
-    .revision = 0,
-    .response = null,
-};
+pub export var kernel_request linksection(".limine_requests") =
+    limine.ExecutableAddressRequest{
+        .id = limine.executable_address_request_id,
+        .revision = 0,
+        .response = null,
+    };
 
+pub export var module_request linksection(".limine_requests") =
+    limine.ModuleRequest{
+        .id = limine.module_request_id,
+        .revision = 0,
+        .response = null,
+    };
+
+var hhdm_offset: usize = 0;
 pub fn p2v(pa: usize) usize {
-    return hhdm_request.response.?.offset + pa;
+    return hhdm_offset + pa;
 }
 
 pub fn get_image_layout() r.ImageLayout {
@@ -64,36 +78,12 @@ pub fn get_image_layout() r.ImageLayout {
 
 var cmdline: [256]u8 = undefined;
 
-export fn loader_entry() callconv(.c) void {
-    var kernel: ?*anyopaque = null;
-    var module_start: usize = 0;
-    var module_end: usize = 0;
-
-    if (module_request.response) |resp| {
-        const modules = resp.modules.?[0..resp.module_count];
-
-        for (modules) |mod| {
-            if (std.mem.eql(u8, std.mem.span(mod.string), "kernel")) {
-                kernel = mod.address;
-                const phys = @intFromPtr(mod.address) -
-                    hhdm_request.response.?.offset;
-                const size: usize = @intCast(mod.size);
-
-                module_start = std.mem.alignForward(usize, phys, r.page_size);
-                module_end = std.mem.alignBackward(usize, phys + size, r.page_size);
-                break;
-            }
-        }
-    }
-
-    if (kernel == null) {
-        @panic("loader: kernel not found");
-    }
-
-    const mmap = memmap_request.response.?;
+fn build_mmap(module_start: usize, module_end: usize) void {
+    const mmap = memmap_request.response orelse unreachable;
+    const entries = mmap.entries orelse unreachable;
 
     for (0..mmap.entry_count) |i| {
-        const entry = mmap.entries.?[i];
+        const entry = entries[i];
 
         if (entry.type == .KernelAndModules) {
             const end = entry.base + entry.length;
@@ -112,10 +102,18 @@ export fn loader_entry() callconv(.c) void {
             // anyway.
             if (reclaim_start < reclaim_end) {
                 if (entry.base < reclaim_start) {
-                    r.mem.add_entry(entry.base, reclaim_start - entry.base, .Reserved);
+                    r.mem.add_entry(
+                        entry.base,
+                        reclaim_start - entry.base,
+                        .Reserved,
+                    );
                 }
 
-                r.mem.add_entry(reclaim_start, reclaim_end - reclaim_start, .LoaderReclaimable);
+                r.mem.add_entry(
+                    reclaim_start,
+                    reclaim_end - reclaim_start,
+                    .LoaderReclaimable,
+                );
 
                 if (reclaim_end < end) {
                     r.mem.add_entry(reclaim_end, end - reclaim_end, .Reserved);
@@ -132,6 +130,46 @@ export fn loader_entry() callconv(.c) void {
             else => .Reserved,
         });
     }
+}
+
+export fn loader_entry() callconv(.c) void {
+    var kernel: ?*anyopaque = null;
+    var module_start: usize = 0;
+    var module_end: usize = 0;
+
+    const hhdm_resp = hhdm_request.response orelse unreachable;
+    hhdm_offset = hhdm_resp.offset;
+
+    if (module_request.response) |resp| {
+        const limine_mods = resp.modules orelse unreachable;
+        const modules = limine_mods[0..resp.module_count];
+
+        for (modules) |mod| {
+            if (std.mem.eql(u8, std.mem.span(mod.string), "kernel")) {
+                kernel = mod.address;
+                const phys = @intFromPtr(mod.address) - hhdm_offset;
+                const size: usize = @intCast(mod.size);
+
+                module_start = std.mem.alignForward(
+                    usize,
+                    phys,
+                    r.page_size,
+                );
+                module_end = std.mem.alignBackward(
+                    usize,
+                    phys + size,
+                    r.page_size,
+                );
+                break;
+            }
+        }
+    }
+
+    if (kernel == null) {
+        @panic("loader: kernel not found");
+    }
+
+    build_mmap(module_start, module_end);
 
     if (framebuffer_request.response) |resp| {
         const fb = resp.framebuffers[0];

src/loader/main.zig

@@ -4,7 +4,11 @@ const std = @import("std");
 const Entry = r.BootInfo.MemMap.Entry;
 const stack_size = 1024 * 16; // should be enough
 
-extern fn jump_to_kernel(loader_info: usize, entry: usize, stack_top: usize) void;
+extern fn jump_to_kernel(
+    loader_info: usize,
+    entry: usize,
+    stack_top: usize,
+) void;
 
 fn load_kernel(kernel: *anyopaque) usize {
     const elf: *std.elf.Elf64.Ehdr = @ptrCast(@alignCast(kernel));
@@ -22,12 +26,18 @@ fn load_kernel(kernel: *anyopaque) usize {
         @panic("loader: invalid ELF type (expected ET_EXEC)");
     }
 
-    var phdr: *std.elf.Elf64.Phdr = @ptrFromInt(@intFromPtr(kernel) + elf.phoff);
+    var phdr: *std.elf.Elf64.Phdr = @ptrFromInt(
+        @intFromPtr(kernel) + elf.phoff,
+    );
 
     for (0..elf.phnum) |_| {
         switch (phdr.type) {
             .LOAD => {
-                const as_pages = std.mem.alignForward(usize, phdr.memsz, r.page_size);
+                const as_pages = std.mem.alignForward(
+                    usize,
+                    phdr.memsz,
+                    r.page_size,
+                );
                 const npages = as_pages / r.page_size;
                 var filesz_rem = phdr.filesz;
 
@@ -43,7 +53,9 @@ fn load_kernel(kernel: *anyopaque) usize {
 
                     const file_remaining = @min(filesz_rem, r.page_size);
 
-                    const off: [*]u8 = @ptrFromInt(@intFromPtr(kernel) + phdr.offset + i * r.page_size);
+                    const off: [*]u8 = @ptrFromInt(
+                        @intFromPtr(kernel) + phdr.offset + i * r.page_size,
+                    );
                     const source = off[0..file_remaining];
                     const va: [*]u8 = @ptrFromInt(addr);
                     const dest = va[0..file_remaining];

src/loader/mem.zig

@@ -100,7 +100,11 @@ pub fn alloc(size: usize) usize {
     return va;
 }
 
-pub fn add_entry(base: usize, size: usize, @"type": r.BootInfo.MemMap.Entry.Type) void {
+pub fn add_entry(
+    base: usize,
+    size: usize,
+    @"type": r.BootInfo.MemMap.Entry.Type,
+) void {
     if (memory_map.entry_count == memory_map.entries.len) {
         @panic("loader: memory map is full");
     }
@@ -190,13 +194,17 @@ fn map_self() void {
         },
     );
 
-    // Now go through every usable entry and map to the HHDM every part that isnt covered by [0, hhdm_minimum_max_address).
-    // We can't blindly map until the maximum usable physical address because on some CPUs this might cause MCEs.
-    // See https://github.com/torvalds/linux/commit/66520ebc2df3fe52eb4792f8101fac573b766baf
+    // Now go through every usable entry and map to the HHDM every part that
+    // isnt covered by [0, hhdm_minimum_max_address).
+    // We can't blindly map until the maximum usable physical address because
+    // on some CPUs this might cause MCEs.
+    // See Linux commit 66520ebc2df3fe52eb4792f8101fac573b766baf
     for (0..memory_map.entry_count) |i| {
         const entry = memory_map.entries[i];
 
-        if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < r.page_size) {
+        if ((entry.type != .Free and entry.type != .LoaderReclaimable) or
+            entry.size < r.page_size)
+        {
             continue;
         }
 
@@ -233,20 +241,33 @@ fn map_pfndb() void {
     for (0..memory_map.entry_count) |i| {
         const entry = memory_map.entries[i];
 
-        if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < r.page_size) {
+        if ((entry.type != .Free and entry.type != .LoaderReclaimable) or
+            entry.size < r.page_size)
+        {
             continue;
         }
 
-        const npages = std.math.divCeil(usize, entry.size, r.page_size) catch unreachable;
+        const npages = std.math.divCeil(usize, entry.size, r.page_size) catch
+            unreachable;
 
-        // 1. Calculate the exact virtual address range needed for this region's page structs.
+        // 1. Calculate the exact virtual address range needed for this
+        // region's page structs.
         const start_pfn: usize = entry.base / r.page_size;
         const exact_start = memory_layout.pfndb + (start_pfn * size);
         const exact_end = exact_start + (npages * size);
 
         // 2. Ensure the addresses are aligned on page boundaries.
-        const map_start = std.mem.alignBackward(usize, exact_start, r.page_size);
-        const map_end = std.mem.alignForward(usize, exact_end, r.page_size);
+        const map_start = std.mem.alignBackward(
+            usize,
+            exact_start,
+            r.page_size,
+        );
+
+        const map_end = std.mem.alignForward(
+            usize,
+            exact_end,
+            r.page_size,
+        );
 
         // 3. Map the virtual pages.
         pagemap.map_range_allocating(

src/loader/pmap.zig

@@ -34,8 +34,9 @@ const ContiguousSource = struct {
     }
 };
 
-/// A mapping source that allocates physical pages on demand. Each call to `next`
-/// returns a single page, so this is intended to be consumed by `map_from` in a loop until the entire range is mapped.
+/// A mapping source that allocates physical pages on demand. Each call to
+/// `next` returns a single page, so this is intended to be consumed by
+/// `map_from` in a loop until the entire range is mapped.
 const AllocatingSource = struct {
     flags: mem.MapFlags,
 
@@ -62,8 +63,6 @@ fn index_for_level(va: usize, level: usize) usize {
 }
 
 pub const PMap = struct {
-    const Self = @This();
-
     const num_levels = r.arch.levels.len;
     const entries_per_table = 512;
     const index_mask = entries_per_table - 1;
@@ -78,7 +77,7 @@ pub const PMap = struct {
         return is_leaf_level(level) and r.arch.is_leaf_level_enabled(level);
     }
 
-    fn cursor(self: *Self, va: usize) Cursor {
+    fn cursor(self: *PMap, va: usize) Cursor {
         var c = Cursor{
             .pmap = self,
             .va = va,
@@ -90,11 +89,11 @@ pub const PMap = struct {
         return c;
     }
 
-    pub fn activate(self: *Self) void {
+    pub fn activate(self: *PMap) void {
         r.arch.activate(self.root_pa);
     }
 
-    pub fn map_from(self: *Self, va: usize, size: usize, source: anytype) void {
+    pub fn map_from(self: *PMap, va: usize, size: usize, source: anytype) void {
         var c = self.cursor(va);
         var remain = size;
 
@@ -105,8 +104,15 @@ pub const PMap = struct {
         }
     }
 
-    /// Map a contiguous virtual address range to a contiguous physical address range.
-    pub fn map_contiguous_range(self: *Self, va: usize, pa: usize, size: usize, flags: r.mem.MapFlags) void {
+    /// Map a contiguous virtual address range to a contiguous physical address
+    /// range.
+    pub fn map_contiguous_range(
+        self: *PMap,
+        va: usize,
+        pa: usize,
+        size: usize,
+        flags: r.mem.MapFlags,
+    ) void {
         const src = ContiguousSource{
             .flags = flags,
             .size = size,
@@ -118,12 +124,22 @@ pub const PMap = struct {
     }
 
     /// Map a single virtual page to a physical page.
-    pub fn map_page(self: *Self, va: usize, pa: usize, flags: r.mem.MapFlags) void {
+    pub fn map_page(
+        self: *PMap,
+        va: usize,
+        pa: usize,
+        flags: r.mem.MapFlags,
+    ) void {
         self.map_contiguous_range(va, pa, r.page_size, flags);
     }
 
     /// Map a virtual address range to physical addresses
-    pub fn map_range_allocating(self: *Self, va: usize, size: usize, flags: mem.MapFlags) void {
+    pub fn map_range_allocating(
+        self: *PMap,
+        va: usize,
+        size: usize,
+        flags: mem.MapFlags,
+    ) void {
         const src = AllocatingSource{
             .flags = flags,
         };
@@ -132,7 +148,7 @@ pub const PMap = struct {
     }
 
     pub const Cursor = struct {
-        pmap: *Self,
+        pmap: *PMap,
         va: usize,
         /// Cached pointers to each level's page table, filled as the cursor
         /// descends. Only indices in `[0, top_level]` are valid.
@@ -142,7 +158,11 @@ pub const PMap = struct {
         /// Reset upward in `advance` when the VA crosses a level boundary.
         top_level: usize,
 
-        fn walk_down(self: *Cursor, target_level: usize, allocate: bool) error{PageNotMapped}!void {
+        fn walk_down(
+            self: *Cursor,
+            target_level: usize,
+            allocate: bool,
+        ) error{PageNotMapped}!void {
             var current_level = self.top_level;
 
             while (current_level > target_level) {
@@ -156,7 +176,9 @@ pub const PMap = struct {
 
                     const new_table_pa = mem.alloc_page();
 
-                    const table_ptr: [*]r.arch.Pte = @ptrFromInt(r.mem.p2v(new_table_pa));
+                    const table_ptr: [*]r.arch.Pte = @ptrFromInt(
+                        r.mem.p2v(new_table_pa),
+                    );
                     @memset(table_ptr[0..entries_per_table], r.arch.Pte.zero());
 
                     pte = r.arch.make_table_pte(new_table_pa);
@@ -193,14 +215,23 @@ pub const PMap = struct {
             @panic("no leaf level can map current alignment/size");
         }
 
-        pub fn map_range(self: *Cursor, pa: usize, size: usize, flags: mem.MapFlags) void {
+        pub fn map_range(
+            self: *Cursor,
+            pa: usize,
+            size: usize,
+            flags: mem.MapFlags,
+        ) void {
             var remain = size;
             var current_pa = pa;
 
             while (remain > 0) {
-                const target_level = self.choose_target_level(current_pa, remain);
+                const target_level = self.choose_target_level(
+                    current_pa,
+                    remain,
+                );
 
-                self.walk_down(target_level, true) catch @panic("walk_down failed during map_range");
+                self.walk_down(target_level, true) catch
+                    @panic("walk_down failed during map_range");
 
                 const table = self.tables[target_level];
                 table[index_for_level(self.va, target_level)] =

src/loader/riscv64/impl.zig

@@ -75,7 +75,11 @@ pub inline fn make_table_pte(pa: usize) Pte {
     };
 }
 
-pub inline fn make_leaf_pte(pa: usize, flags: r.mem.MapFlags, level: usize) Pte {
+pub inline fn make_leaf_pte(
+    pa: usize,
+    flags: r.mem.MapFlags,
+    level: usize,
+) Pte {
     _ = level;
     return Pte{
         .present = true,

src/loader/root.zig

@@ -21,7 +21,11 @@ const DebugWriter = struct {
         };
     }
 
-    fn drain(_: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize {
+    fn drain(
+        _: *std.Io.Writer,
+        data: []const []const u8,
+        _: usize,
+    ) std.Io.Writer.Error!usize {
         var total_written: usize = 0;
         for (data) |slice| {
             for (slice) |byte| {
@@ -47,7 +51,8 @@ pub fn log(
     // Calculate the length required.
     var buf: [256]u8 = undefined;
 
-    const written = std.fmt.bufPrint(&buf, scope_str ++ fmt ++ "\n", args) catch return;
+    const written = std.fmt.bufPrint(&buf, scope_str ++ fmt ++ "\n", args) catch
+        return;
 
     var writer = DebugWriter.init();
 

src/mm/balance.zig

@@ -35,10 +35,10 @@ fn reap_stacks() void {
         // Reap all the excess stacks.
         var entry: ?*ke.Stack = lst;
 
-        while (entry != null) {
-            const next = entry.?.next;
+        while (entry) |e| {
+            const next = e.next;
             mm.heap.free(
-                (@intFromPtr(entry) + @sizeOf(ke.Stack)) -
+                (@intFromPtr(e) + @sizeOf(ke.Stack)) -
                     ps.thread.kernel_thread_stack_size,
                 ps.thread.kernel_thread_stack_size,
             );
@@ -86,7 +86,6 @@ fn balance_manager(_: ?*anyopaque) void {
 
             if (stack_reap_time == 0) {
                 stack_reap_time = stack_reap_interval;
-
                 reap_stacks();
             }
         }

src/mm/heap.zig

@@ -1,3 +1,5 @@
+//! Kernel heap.
+
 const r = @import("root");
 const ke = r.ke;
 const mm = r.mm;
@@ -43,7 +45,11 @@ pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque {
 
         mmp.kernel_space.lock.acquire();
 
-        pte.* = mmp.impl.make_leaf_pte(page, .{ .read = true, .write = true }, 0);
+        pte.* = mmp.impl.make_leaf_pte(
+            page,
+            .{ .read = true, .write = true },
+            0,
+        );
     }
 
     mmp.kernel_space.lock.release();

src/mm/init.zig

@@ -1,3 +1,5 @@
+//! Memory manager initialization.
+
 const r = @import("root");
 const pl = r.pl;
 const mm = r.mm;

src/mm/phys.zig

@@ -1,17 +1,19 @@
 const r = @import("root");
-const std = @import("std");
 const rtl = @import("rtl");
 const pl = r.pl;
 const mm = r.mm;
 const mmp = mm.private;
 const ke = r.ke;
 
+const std = @import("std");
+const assert = std.debug.assert;
 const log = std.log.scoped(.@"mm/phys");
 
 // How aggressive is memory reclamation.
 const threshold_scale_factor = 10;
 
-/// Threshold at which most memory allocations block, only high priority allocations are allowed.
+/// Threshold at which most memory allocations block, only high priority
+/// allocations are allowed.
 pub var min_memory_threshold: usize = 0;
 /// Threshold at which reclamation starts.
 pub var low_memory_threshold: usize = 0;
@@ -29,14 +31,14 @@ var free_page_event: ke.Event = undefined;
 
 var free_pages: usize = 0;
 
-pub var usable_memory: std.atomic.Value(usize) = .init(0);
+pub var usable_memory = std.atomic.Value(usize).init(0);
 
 // Wait for pages to be available.
 // List lock is held.
 fn wait_for_pages(old_ipl: ke.Ipl) void {
     var ipl = old_ipl;
 
-    std.debug.assert(@intFromEnum(ipl) < @intFromEnum(ke.Ipl.Dispatch));
+    assert(@backingInt(ipl) < @backingInt(ke.Ipl.Dispatch));
 
     if (free_pages >= min_memory_threshold) {
         return;
@@ -65,7 +67,7 @@ pub fn alloc() r.PAddr {
 
     wait_for_pages(ipl);
 
-    std.debug.assert(!free_list.is_empty());
+    assert(!free_list.is_empty());
 
     const head = free_list.first();
     head.remove();
@@ -94,8 +96,8 @@ pub fn alloc_opts(opts: struct { policy: mm.WaitPolicy }) ?r.PAddr {
         wait_for_pages(ipl);
     }
 
-    std.debug.assert(free_pages > 0);
-    std.debug.assert(!free_list.is_empty());
+    assert(free_pages > 0);
+    assert(!free_list.is_empty());
 
     const head = free_list.first();
     head.remove();

src/mm/pmap.zig

@@ -2,10 +2,12 @@
 
 const r = @import("root");
 const rtl = @import("rtl");
-const std = @import("std");
 const mm = r.mm;
 const mmp = mm.private;
 
+const std = @import("std");
+const assert = std.debug.assert;
+
 /// Describes a contiguous region of physical memory to be mapped
 /// at a given virtual address.
 const MapItem = struct {
@@ -45,8 +47,6 @@ fn index_for_level(va: usize, level: usize) usize {
 }
 
 pub const PMap = struct {
-    const Self = @This();
-
     const num_levels = mmp.impl.levels.len;
     const entries_per_table = 512;
     const index_mask = entries_per_table - 1;
@@ -61,7 +61,7 @@ pub const PMap = struct {
         return is_leaf_level(level) and mmp.impl.is_leaf_level_enabled(level);
     }
 
-    fn cursor(self: *Self, va: r.VAddr) Cursor {
+    fn cursor(self: *PMap, va: r.VAddr) Cursor {
         var c = Cursor{
             .pmap = self,
             .va = va,
@@ -73,11 +73,16 @@ pub const PMap = struct {
         return c;
     }
 
-    pub fn activate(self: *Self) void {
+    pub fn activate(self: *PMap) void {
         mmp.impl.activate(self.root_pa);
     }
 
-    pub fn map_from(self: *Self, va: r.VAddr, size: usize, source: anytype) void {
+    pub fn map_from(
+        self: *PMap,
+        va: r.VAddr,
+        size: usize,
+        source: anytype,
+    ) void {
         var c = self.cursor(va);
         var remain = size;
 
@@ -91,7 +96,7 @@ pub const PMap = struct {
     /// Map a contiguous virtual address range to a
     /// contiguous physical address range.
     pub fn map_contiguous_range(
-        self: *Self,
+        self: *PMap,
         va: r.VAddr,
         pa: r.PAddr,
         size: usize,
@@ -107,7 +112,7 @@ pub const PMap = struct {
         self.map_from(va, size, src);
     }
 
-    pub fn query(self: *Self, va: r.VAddr) ?r.PAddr {
+    pub fn query(self: *PMap, va: r.VAddr) ?r.PAddr {
         var c = self.cursor(va);
 
         var level: usize = num_levels - 1;
@@ -129,8 +134,8 @@ pub const PMap = struct {
     /// Unmap a contiguous range of virtual pages.
     /// Only small pages are supported.
     /// Return a pfn that points to the physical pages that were unmapped.
-    pub fn unmap(self: *Self, va: r.VAddr, size: usize) ?mmp.PfnList {
-        std.debug.assert(std.mem.isAligned(va, mm.page_size));
+    pub fn unmap(self: *PMap, va: r.VAddr, size: usize) ?mmp.PfnList {
+        assert(std.mem.isAligned(va, mm.page_size));
 
         var c = self.cursor(va);
 
@@ -178,7 +183,7 @@ pub const PMap = struct {
     }
 
     pub const Cursor = struct {
-        pmap: *Self,
+        pmap: *PMap,
         va: r.VAddr,
         /// Cached pointers to each level's page table, filled as the cursor
         /// descends. Only indices in `[0, top_level]` are valid.
@@ -283,9 +288,12 @@ pub fn wire_pte(
     va: r.VAddr,
     policy: mm.WaitPolicy,
 ) mm.Error!*mmp.impl.Pte {
-    std.debug.assert(space.lock.is_locked());
+    assert(space.lock.is_locked());
+
+    var table: [*]mmp.impl.Pte = @ptrFromInt(
+        mm.p2v(mmp.kernel_space.pmap.root_pa),
+    );
 
-    var table: [*]mmp.impl.Pte = @ptrFromInt(mm.p2v(mmp.kernel_space.pmap.root_pa));
     var level = mmp.impl.levels.len - 1;
 
     while (true) {
@@ -293,7 +301,7 @@ pub fn wire_pte(
         var entry = table[idx];
 
         if (level == 0) {
-            std.debug.assert(!entry.present);
+            assert(!entry.present);
             return &table[idx];
         }
 
@@ -302,10 +310,13 @@ pub fn wire_pte(
             // since that may block.
             space.lock.release();
 
-            const new_table_pa = mmp.phys.alloc_opts(.{ .policy = policy }) orelse {
-                space.lock.acquire();
-                return mm.Error.OutOfMemory;
-            };
+            const new_table_pa = mmp.phys.alloc_opts(
+                .{ .policy = policy },
+            ) orelse
+                {
+                    space.lock.acquire();
+                    return mm.Error.OutOfMemory;
+                };
 
             // Re-acquire the lock and re-validate the entry,
             // since another thread may have raced to allocate it.

src/mm/riscv64/impl.zig

@@ -88,7 +88,11 @@ pub inline fn make_table_pte(pa: r.PAddr) Pte {
     };
 }
 
-pub inline fn make_leaf_pte(pa: r.PAddr, flags: mm.MapFlags, level: usize) Pte {
+pub inline fn make_leaf_pte(
+    pa: r.PAddr,
+    flags: mm.MapFlags,
+    level: usize,
+) Pte {
     _ = level;
     return Pte{
         .present = true,
@@ -118,5 +122,6 @@ pub fn is_leaf_level_enabled(level: usize) bool {
 }
 
 pub fn init_kernel() void {
-    mmp.kernel_space.pmap.root_pa = (rv64.read_csr("satp") & ((@as(u64, 1) << 44) - 1)) << 12;
+    mmp.kernel_space.pmap.root_pa = (rv64.read_csr("satp") &
+        ((@as(u64, 1) << 44) - 1)) << 12;
 }

src/mm/root.zig

@@ -35,7 +35,7 @@ comptime {
     if (@sizeOf(Page) != r.BootInfo.page_struct_size) {
         @compileError(
             std.fmt.comptimePrint(
-                "'Page' size mismatch between loader and kernel: kernel={} loader={}",
+                "'Page' size mismatch between loader and kernel: k={} l={}",
                 .{ @sizeOf(Page), r.BootInfo.page_struct_size },
             ),
         );
@@ -68,11 +68,13 @@ pub inline fn pfn_to_page(pfn: Pfn) usize {
 }
 
 pub inline fn struct_page_to_pfn(page: *Page) Pfn {
-    return @intCast((@intFromPtr(page) - private.impl.pfndb_base) / @sizeOf(Page));
+    return @intCast((@intFromPtr(page) - private.impl.pfndb_base) /
+        @sizeOf(Page));
 }
 
 pub inline fn pfn_to_struct_page(pfn: Pfn) *Page {
-    return @ptrFromInt(private.impl.pfndb_base + (@as(usize, @intCast(pfn)) * @sizeOf(Page)));
+    return @ptrFromInt(private.impl.pfndb_base +
+        (@as(usize, pfn) * @sizeOf(Page)));
 }
 
 pub var pfndb: [*]Page = @ptrFromInt(private.impl.pfndb_base);

src/mm/tlb.zig

@@ -1,7 +1,6 @@
 //! MM-level TLB shootdown support
 
 const r = @import("root");
-const std = @import("std");
 const rtl = @import("rtl");
 
 const ke = r.ke;
@@ -10,9 +9,12 @@ const mmp = mm.private;
 const ps = r.ps;
 const ex = r.ex;
 
-pub var sync_shootdowns: std.atomic.Value(usize) = .init(0);
-pub var async_shootdowns: std.atomic.Value(usize) = .init(0);
-pub var sent_shootdowns: std.atomic.Value(usize) = .init(0);
+const std = @import("std");
+const assert = std.debug.assert;
+
+pub var sync_shootdowns = std.atomic.Value(usize).init(0);
+pub var async_shootdowns = std.atomic.Value(usize).init(0);
+pub var sent_shootdowns = std.atomic.Value(usize).init(0);
 
 var work_item: ex.WorkItem = undefined;
 
@@ -54,7 +56,7 @@ fn reap_pages(_: ?*anyopaque) void {
 /// Unmap and flush a virtual range on the given space.
 /// Space lock is held on entry and IPL is raised to Dispatch.
 pub fn reclaim_range(space: *mm.Space, va: r.VAddr, size: usize) void {
-    std.debug.assert(std.mem.isAligned(va, mm.page_size));
+    assert(std.mem.isAligned(va, mm.page_size));
 
     // Unmap the virtual addresses and get the backing physical pages.
     const list = space.pmap.unmap(va, size) orelse {
@@ -67,7 +69,7 @@ pub fn reclaim_range(space: *mm.Space, va: r.VAddr, size: usize) void {
         .base = va,
         .npages = @truncate(size / mm.page_size),
         .link = undefined,
-        .state = .init(0),
+        .state = std.atomic.Value(u16).init(0),
         .payload = .{
             @intFromPtr(space),
             @as(*const usize, @ptrCast(@alignCast(&list))).*,
@@ -77,7 +79,7 @@ pub fn reclaim_range(space: *mm.Space, va: r.VAddr, size: usize) void {
     space.lock.release();
 
     const ipl = ke.ipl.raise(.Dispatch);
-    var mask: ke.CpuMask = .init(false);
+    var mask = ke.CpuMask.init(false);
 
     for (0..ke.ncpus) |i| {
         if (i != ke.cpu.current()) {
@@ -110,6 +112,6 @@ fn activation(_: *rtl.HandoffList) void {
 }
 
 pub fn init() void {
-    ke.shootdown.shootdowns.* = .init(activation);
+    ke.shootdown.shootdowns.* = rtl.HandoffList.init(activation);
     work_item.init(.High, reap_pages, null);
 }

src/mm/zone.zig

@@ -114,16 +114,18 @@
 //! an object leaves the slab layer and destructors run when it returns to it,
 //! slabs hold raw objects while magazines hold constructed ones.
 
-const rtl = @import("rtl");
-const r = @import("root");
-const std = @import("std");
 const config = @import("config");
+const rtl = @import("rtl");
 
+const r = @import("root");
 const ke = r.ke;
 const mm = r.mm;
 const ex = r.ex;
 const mmp = mm.private;
 
+const std = @import("std");
+const assert = std.debug.assert;
+
 /// Interval at which we do housekeeping (working set update, reaping, etc.)
 pub const update_interval_s = 15;
 
@@ -276,7 +278,7 @@ const Slab = struct {
         const bm = self.bitmap();
 
         const bit = @as(u64, 1) << @intCast(idx % 64);
-        std.debug.assert(bm[idx / 64] & (bit) == 0);
+        assert(bm[idx / 64] & (bit) == 0);
 
         bm[idx / 64] |= bit;
     }
@@ -558,8 +560,6 @@ pub const Zone = struct {
 
     arg: ?*anyopaque,
 
-    const Self = @This();
-
     const InitOptions = struct {
         alignment: u16 = slab_align,
         ctor: ?*const fn (*anyopaque) void = null,
@@ -586,7 +586,7 @@ pub const Zone = struct {
 
     /// Initialize a zone.
     pub fn init(
-        self: *Self,
+        self: *Zone,
         name: []const u8,
         size: usize,
         options: InitOptions,
@@ -634,9 +634,9 @@ pub const Zone = struct {
         self.smr_reclaim = options.smr_reclaim;
 
         if (options.import) |i| {
-            std.debug.assert(options.release != null);
+            assert(options.release != null);
             self.import = i;
-            self.release = options.release.?;
+            self.release = options.release orelse unreachable;
             self.arg = options.arg;
         } else {
             self.import = zone_import;
@@ -669,7 +669,7 @@ pub const Zone = struct {
     }
 
     /// Allocate an object from the zone.
-    pub fn alloc(self: *Self, opts: AllocOpts) mm.Error!*anyopaque {
+    pub fn alloc(self: *Zone, opts: AllocOpts) mm.Error!*anyopaque {
         // Try grabbing an object from the magazine layer.
         if (magazines_initialized and self.use_magazines) {
             @branchHint(.likely);
@@ -682,12 +682,13 @@ pub const Zone = struct {
             while (true) {
                 if (cpu.alloc_rounds > 0) {
                     // Fast path: just pop from the alloc magazine.
+                    const alloc_mag = cpu.alloc orelse unreachable;
                     cpu.alloc_rounds -= 1;
-                    buf = cpu.alloc.?.rounds_ptr()[cpu.alloc_rounds];
+                    buf = alloc_mag.rounds_ptr()[cpu.alloc_rounds];
 
                     if (is_poison_enabled and !self.dont_touch) {
                         if (!check_poison(buf, self.obj_size, free_poison)) {
-                            @panic("slab poison mismatch: object corrupted after free");
+                            @panic("poison mismatch: possible use-after-free");
                         }
 
                         fill_with_poison(buf, self.obj_size, alloc_poison);
@@ -702,7 +703,7 @@ pub const Zone = struct {
                 }
 
                 if (cpu.free_rounds > 0 and self.smr == null) {
-                    std.debug.assert(cpu.free != null);
+                    assert(cpu.free != null);
 
                     // Alloc magazine is empty. If the free magazine has items,
                     // swap them. The free magazine now becomes our alloc
@@ -715,7 +716,8 @@ pub const Zone = struct {
                 // Both magazines are empty.
                 // Try to get a full magazine from the CPU-local depot.
                 if (self.full_mags_ready(&cpu.depot.full_mags)) {
-                    const full_mag = Depot.alloc(&cpu.depot.full_mags).?;
+                    const full_mag = Depot.alloc(&cpu.depot.full_mags) orelse
+                        unreachable;
 
                     // Discard our empty alloc magazine to the depot.
                     if (cpu.alloc) |empty_mag| {
@@ -741,7 +743,8 @@ pub const Zone = struct {
 
                 // Try getting a full magazine from the CPU-local depot again.
                 if (self.full_mags_ready(&cpu.depot.full_mags)) {
-                    const full_mag = Depot.alloc(&cpu.depot.full_mags).?;
+                    const full_mag = Depot.alloc(&cpu.depot.full_mags) orelse
+                        unreachable;
 
                     // Discard our empty alloc magazine to the depot.
                     if (cpu.alloc) |empty_mag| {
@@ -758,14 +761,15 @@ pub const Zone = struct {
                     continue;
                 }
 
-                // Nothing was found. Allocate a new magazine and fill it from the
-                // import layer, then replace the CPU's alloc magazine with it.
-                // It is preferable (maybe even required?) to drop the CPU lock
-                // while we do this work; when we do re-acquire the lock we then
-                // might notice that someone has already done the work for us and
-                // put a magazine in the CPU's alloc storage. In that case, it would
-                // be better to take our fresh magazine and insert it into the depot
-                // instead, since there is a higher chance the impostor magazine
+                // Nothing was found. Allocate a new magazine and fill it from
+                // the import layer, then replace the CPU's alloc magazine with
+                // it.  It is preferable (maybe even required?) to drop the CPU
+                // lock  while we do this work; when we do re-acquire the lock
+                // we then might notice that someone has already done the work
+                // for us and  put a magazine in the CPU's alloc storage.
+                // In that case, it would  be better to take our fresh magazine
+                // and insert it into the depot instead,
+                // since there is a higher chance the  impostor magazine
                 // contains objects that are hotter in cache.
                 cpu.lock.release(ipl);
 
@@ -783,7 +787,8 @@ pub const Zone = struct {
                         opts.policy,
                     );
 
-                    // We dropped the lock and IPL, so we might've been migrated.
+                    // We dropped the lock and IPL, so we might've been
+                    // migrated.
                     ipl = ke.ipl.raise(.Dispatch);
                     cpu = &self.cpus[ke.cpu.current()].value;
                     cpu.lock.acquire_no_ipl();
@@ -793,14 +798,19 @@ pub const Zone = struct {
                         cpu.alloc_rounds = cnt;
                         continue;
                     } else {
-                        // We lost the race. If the magazine we got was fully filled,
-                        // then put it in the depot. Otherwise, dispose of it.
+                        // We lost the race. If the magazine we got was fully
+                        // filled,  then put it in the depot.
+                        // Otherwise, dispose of it.
                         if (cnt == magazine_size.load()) {
-                            // Ensure this magazine isn't counted as a freed one,
-                            // it is instantly reclaimable.
+                            // Ensure this magazine isn't counted as a freed
+                            // one,  it is instantly reclaimable.
                             nm.seq = ke.smr.seq_invalid;
 
-                            Depot.free(nm, &cpu.depot.full_mags, self.reuse_policy);
+                            Depot.free(
+                                nm,
+                                &cpu.depot.full_mags,
+                                self.reuse_policy,
+                            );
                             continue;
                         }
 
@@ -828,8 +838,8 @@ pub const Zone = struct {
             }
         }
 
-        // Fallback in case this is a zone with magazines disabled or we got into trouble above,
-        // just fetch an object from the import layer.
+        // Fallback in case this is a zone with magazines disabled or we got
+        // into trouble above, just fetch an object from the import layer.
         var elems: [1]*anyopaque = undefined;
 
         _ = try self.import(
@@ -843,7 +853,7 @@ pub const Zone = struct {
     }
 
     /// Free an object back to the zone.
-    pub fn free(self: *Self, obj: *anyopaque) void {
+    pub fn free(self: *Zone, obj: *anyopaque) void {
         if (magazines_initialized and self.use_magazines) {
             @branchHint(.likely);
 
@@ -857,8 +867,12 @@ pub const Zone = struct {
                 {
                     // Fast path: push directly to free magazine.
                     // For SMR zones the dtor and poisoning are deferred until
-                    // the magazine is reused, as readers may still hold the object.
-                    if (is_poison_enabled and self.smr == null and !self.dont_touch) {
+                    // the magazine is reused, as readers may still hold
+                    // the object.
+                    const free_mag = cpu.free orelse unreachable;
+                    if (is_poison_enabled and self.smr == null and
+                        !self.dont_touch)
+                    {
                         if (self.dtor) |dtor| {
                             dtor(obj);
                         }
@@ -866,12 +880,16 @@ pub const Zone = struct {
                         fill_with_poison(obj, self.obj_size, free_poison);
                     }
 
-                    cpu.free.?.rounds_ptr()[cpu.free_rounds] = obj;
+                    free_mag.rounds_ptr()[cpu.free_rounds] = obj;
                     cpu.free_rounds += 1;
 
-                    if (self.smr != null and cpu.free_rounds == magazine_size.load()) {
+                    if (self.smr != null and
+                        cpu.free_rounds == magazine_size.load())
+                    {
+                        const smr = self.smr orelse unreachable;
+
                         // Stamp the now-full magazine.
-                        cpu.free.?.seq = ke.smr.deferred_advance(self.smr.?);
+                        free_mag.seq = ke.smr.deferred_advance(smr);
                     }
 
                     break true;
@@ -929,8 +947,8 @@ pub const Zone = struct {
                 }
 
                 // No empty magazine in the depot, try to allocate a new one.
-                // We need to drop IPL and the CPU lock here because we need to acquire
-                // the magazine's allocator's blocking lock.
+                // We need to drop IPL and the CPU lock here because we need to
+                // acquire the magazine's allocator's blocking lock.
                 cpu.lock.release(ipl);
 
                 const raw_mag = magazine_zone.alloc(
@@ -970,7 +988,7 @@ pub const Zone = struct {
         self.release(self.arg, &slice, self.chunk_size);
     }
 
-    fn slab_free(self: *Self, obj: *anyopaque, poison: bool) void {
+    fn slab_free(self: *Zone, obj: *anyopaque, poison: bool) void {
         if (!is_poison_enabled or poison and !self.dont_touch) {
             if (self.dtor) |dtor| {
                 dtor(obj);
@@ -988,7 +1006,8 @@ pub const Zone = struct {
                 @intFromPtr(obj),
                 mm.page_size,
             );
-            const phys = mmp.kernel_space.pmap.query(page_va).?;
+            const phys = mmp.kernel_space.pmap.query(page_va) orelse
+                unreachable;
             const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys));
             break :blk page.alloced.slab_data.slab;
         } else blk: {
@@ -1023,7 +1042,7 @@ pub const Zone = struct {
         elem_size: usize,
         policy: mm.WaitPolicy,
     ) mm.Error!usize {
-        const self: *Self = @ptrCast(@alignCast(arg));
+        const self: *Zone = @ptrCast(@alignCast(arg));
 
         _ = elem_size;
 
@@ -1046,18 +1065,21 @@ pub const Zone = struct {
 
                 self.lock.acquire();
 
-                // Note: this is racy and could lead to two threads creating a slab
-                // at the same time. Worst case scenario, we have one slab too many,
-                // so we don't care; we already did the work required for slab
-                // construction and this ideally should not happen very often.
+                // Note: this is racy and could lead to two threads creating a
+                // slab at the same time. Worst case scenario, we have one slab
+                // too many,  so we don't care; we already did the work
+                // required for slab  construction and this ideally should not
+                // happen very often.
                 self.partial_slabs.insert_tail(&slab.link);
             } else {
                 slab = @fieldParentPtr("link", self.partial_slabs.first());
             }
 
             while (slab.refcount != slab.capacity and i < requested) {
-                const idx = slab.bitmap_alloc().?;
-                const buf: *anyopaque = @ptrFromInt(slab.base + idx * self.chunk_size);
+                const idx = slab.bitmap_alloc() orelse unreachable;
+                const buf: *anyopaque = @ptrFromInt(
+                    slab.base + idx * self.chunk_size,
+                );
 
                 slab.refcount += 1;
 
@@ -1068,7 +1090,7 @@ pub const Zone = struct {
 
                 if (is_poison_enabled) {
                     if (!check_poison(buf, self.obj_size, free_poison)) {
-                        @panic("slab poison mismatch: object corrupted after free");
+                        @panic("poison mismatch: possible use-after-free");
                     }
 
                     fill_with_poison(buf, self.obj_size, alloc_poison);
@@ -1087,8 +1109,12 @@ pub const Zone = struct {
         return i;
     }
 
-    fn zone_release(arg: ?*anyopaque, elems: []*anyopaque, elem_size: usize) void {
-        const self: *Self = @ptrCast(@alignCast(arg));
+    fn zone_release(
+        arg: ?*anyopaque,
+        elems: []*anyopaque,
+        elem_size: usize,
+    ) void {
+        const self: *Zone = @ptrCast(@alignCast(arg));
 
         _ = elem_size;
 
@@ -1112,7 +1138,7 @@ pub const Zone = struct {
 
     /// Periodic updates on the zone.
     /// Updates the magazine working set and trims or grows them if needed.
-    fn update(self: *Self) void {
+    fn update(self: *Zone) void {
         const ipl = self.depot_lock.acquire();
 
         // Update the working set size.
@@ -1187,7 +1213,7 @@ pub const Zone = struct {
     }
 
     /// Returns whether or not we should trim excess magazines.
-    fn trim_needed(self: *Self) bool {
+    fn trim_needed(self: *Zone) bool {
         if (!self.use_magazines) return false;
         if (self.trim_depot) return true;
 
@@ -1221,8 +1247,9 @@ pub const Zone = struct {
 
     /// Called periodically to trim our magazines.
     /// This frees the excess magazines.
-    fn trim(self: *Self, trim_depot: bool) void {
-        // Detach magazines under the locks, then release their objects unlocked.
+    fn trim(self: *Zone, trim_depot: bool) void {
+        // Detach magazines under the locks, then release their objects
+        // unlocked.
         self.lock.acquire();
 
         const empty_mags = self.trim_maglist(&self.depot.empty_mags);
@@ -1262,7 +1289,7 @@ pub const Zone = struct {
 
     /// Called when memory is low and we need to make memory ASAP.
     /// Drains all magazines from the depot.
-    fn drain(self: *Self) void {
+    fn drain(self: *Zone) void {
         if (!self.use_magazines) return;
 
         // Detach magazines before invoking release callbacks.
@@ -1304,7 +1331,7 @@ pub const Zone = struct {
     }
 
     /// Detach excess magazines for destruction.
-    fn trim_maglist(self: *Self, maglist: *MagazineList) ?*Magazine {
+    fn trim_maglist(self: *Zone, maglist: *MagazineList) ?*Magazine {
         const ipl = self.depot_lock.acquire();
 
         // Grab the magazines and WSS under the depot lock.
@@ -1345,7 +1372,7 @@ pub const Zone = struct {
     }
 
     /// Trim a CPU's depot to `target`.
-    fn trim_cpu(self: *Self, cpu: *Cpu, depot: *Depot, target: usize) void {
+    fn trim_cpu(self: *Zone, cpu: *Cpu, depot: *Depot, target: usize) void {
         const ipl = cpu.lock.acquire();
 
         // Split the target (cpu_depot_size) in 2 for each magazine type.
@@ -1361,7 +1388,7 @@ pub const Zone = struct {
             // Move them to our depot.
             for (0..n) |_| {
                 Depot.free(
-                    Depot.alloc(&cpu.depot.full_mags).?,
+                    Depot.alloc(&cpu.depot.full_mags) orelse unreachable,
                     &depot.full_mags,
                     self.reuse_policy,
                 );
@@ -1375,7 +1402,7 @@ pub const Zone = struct {
             // Move them to our depot.
             for (0..n) |_| {
                 Depot.free(
-                    Depot.alloc(&cpu.depot.empty_mags).?,
+                    Depot.alloc(&cpu.depot.empty_mags) orelse unreachable,
                     &depot.empty_mags,
                     self.reuse_policy,
                 );
@@ -1385,7 +1412,7 @@ pub const Zone = struct {
         cpu.lock.release(ipl);
     }
 
-    fn maglist_destroy(self: *Self, head: ?*Magazine, n: usize) void {
+    fn maglist_destroy(self: *Zone, head: ?*Magazine, n: usize) void {
         var elems = head;
         while (elems) |mag| {
             elems = mag.next;
@@ -1394,7 +1421,7 @@ pub const Zone = struct {
     }
 
     /// Destroy a magazine.
-    fn magazine_destroy(self: *Self, magazine: *Magazine, rounds: usize) void {
+    fn magazine_destroy(self: *Zone, magazine: *Magazine, rounds: usize) void {
         if (rounds != 0) {
             if (self.smr) |smr| {
                 // The stamp may still be uncommitted if the magazine never
@@ -1439,7 +1466,7 @@ pub const Zone = struct {
         mmp.phys.free(phys);
     }
 
-    fn slab_create_small(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab {
+    fn slab_create_small(self: *Zone, policy: mm.WaitPolicy) mm.Error!*Slab {
         const buf = try alloc_page(policy);
 
         const capacity = Slab.calc_capacity(self.chunk_size, self.alignment);
@@ -1463,7 +1490,9 @@ pub const Zone = struct {
         }
 
         for (0..capacity) |i| {
-            const obj: *anyopaque = @ptrFromInt(slab.base + i * self.chunk_size);
+            const obj: *anyopaque = @ptrFromInt(
+                slab.base + i * self.chunk_size,
+            );
             if (is_poison_enabled) {
                 fill_with_poison(obj, self.obj_size, free_poison);
             }
@@ -1472,7 +1501,7 @@ pub const Zone = struct {
         return slab;
     }
 
-    fn slab_create_large(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab {
+    fn slab_create_large(self: *Zone, policy: mm.WaitPolicy) mm.Error!*Slab {
         const capacity = (self.slab_size) / self.chunk_size;
         const buf = try mmp.heap.alloc(self.slab_size, policy);
 
@@ -1515,7 +1544,7 @@ pub const Zone = struct {
         return ret;
     }
 
-    fn slab_create(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab {
+    fn slab_create(self: *Zone, policy: mm.WaitPolicy) mm.Error!*Slab {
         if (self.obj_size <= small_slab_size) {
             return self.slab_create_small(policy);
         } else {
@@ -1523,7 +1552,7 @@ pub const Zone = struct {
         }
     }
 
-    fn slab_destroy(self: *Self, slab: *Slab) void {
+    fn slab_destroy(self: *Zone, slab: *Slab) void {
         if (self.obj_size > small_slab_size) {
             const base = slab.base;
             const alloc_size = @sizeOf(Slab) + Slab.bitmap_bytes(slab.capacity);
@@ -1538,7 +1567,7 @@ pub const Zone = struct {
     }
 
     /// Check that a full magazine can be taken from the list head.
-    fn full_mags_ready(self: *Self, list: *MagazineList) bool {
+    fn full_mags_ready(self: *Zone, list: *MagazineList) bool {
         const head = list.head orelse return false;
         const smr = self.smr orelse return true;
 
@@ -1547,7 +1576,7 @@ pub const Zone = struct {
 
     /// Deferred free-time work for SMR zones, ran when a full magazine
     /// is reused after its grace period expired.
-    fn magazine_reuse(self: *Self, mag: *Magazine) void {
+    fn magazine_reuse(self: *Zone, mag: *Magazine) void {
         if (self.smr == null) return;
 
         for (0..magazine_size.load()) |i| {
@@ -1564,7 +1593,7 @@ pub const Zone = struct {
 
     /// Move all local full magazines to the zone depot so their sequences
     /// can age, committing their deferred advances. Depot lock is held.
-    fn smr_rotate_full(self: *Self, smr: *ke.smr.Domain, cpu: *Cpu) void {
+    fn smr_rotate_full(self: *Zone, smr: *ke.smr.Domain, cpu: *Cpu) void {
         const src = &cpu.depot.full_mags;
         const dst = &self.depot.full_mags;
         const tail = src.tail orelse return;
@@ -1584,7 +1613,8 @@ pub const Zone = struct {
         src.num = 0;
         src.min = 0;
 
-        // The last magazine has the newest stamp, committing it covers them all.
+        // The last magazine has the newest stamp, committing it covers them
+        // all.
         ke.smr.deferred_advance_commit(smr, tail.seq);
     }
 
@@ -1592,7 +1622,7 @@ pub const Zone = struct {
     /// after an allocation. The CPU depot moves its extra full magazines to the
     /// zone depot, and tries to get full magazines from the zone depot up to
     /// `target`. CPU lock is held.
-    fn alloc_depot_rebalance(self: *Self, target: usize, cpu: *Cpu) void {
+    fn alloc_depot_rebalance(self: *Zone, target: usize, cpu: *Cpu) void {
         if (!self.depot_lock.try_acquire_no_ipl()) {
             // Register the contention.
             self.depot_lock.acquire_no_ipl();
@@ -1605,7 +1635,7 @@ pub const Zone = struct {
 
             for (0..n) |_| {
                 Depot.free(
-                    Depot.alloc(&cpu.depot.empty_mags).?,
+                    Depot.alloc(&cpu.depot.empty_mags) orelse unreachable,
                     &self.depot.empty_mags,
                     self.reuse_policy,
                 );
@@ -1631,7 +1661,7 @@ pub const Zone = struct {
         if (n != 0 and self.full_mags_ready(&self.depot.full_mags)) {
             for (0..n) |_| {
                 Depot.free(
-                    Depot.alloc(&self.depot.full_mags).?,
+                    Depot.alloc(&self.depot.full_mags) orelse unreachable,
                     &cpu.depot.full_mags,
                     self.reuse_policy,
                 );
@@ -1645,7 +1675,7 @@ pub const Zone = struct {
     /// after a free.The CPU depot moves its extra full magazines to the
     /// zone depot, and tries to get empty magazines from the zone depot up to
     /// `target`. CPU lock is held.
-    fn free_depot_rebalance(self: *Self, target: usize, cpu: *Cpu) void {
+    fn free_depot_rebalance(self: *Zone, target: usize, cpu: *Cpu) void {
         if (!self.depot_lock.try_acquire_no_ipl()) {
             // Register the contention.
             self.depot_lock.acquire_no_ipl();
@@ -1663,7 +1693,8 @@ pub const Zone = struct {
                 if (n != 0 and self.full_mags_ready(&self.depot.full_mags)) {
                     for (0..n) |_| {
                         Depot.free(
-                            Depot.alloc(&self.depot.full_mags).?,
+                            Depot.alloc(&self.depot.full_mags) orelse
+                                unreachable,
                             &cpu.depot.full_mags,
                             self.reuse_policy,
                         );
@@ -1675,7 +1706,7 @@ pub const Zone = struct {
 
                 for (0..n) |_| {
                     Depot.free(
-                        Depot.alloc(&cpu.depot.full_mags).?,
+                        Depot.alloc(&cpu.depot.full_mags) orelse unreachable,
                         &self.depot.full_mags,
                         self.reuse_policy,
                     );
@@ -1691,7 +1722,7 @@ pub const Zone = struct {
         );
         for (0..n) |_| {
             Depot.free(
-                Depot.alloc(&self.depot.empty_mags).?,
+                Depot.alloc(&self.depot.empty_mags) orelse unreachable,
                 &cpu.depot.empty_mags,
                 self.reuse_policy,
             );
@@ -1719,7 +1750,7 @@ pub fn TypedZone(comptime T: type) type {
     return struct {
         zone: Zone,
 
-        const Self = @This();
+        const TZone = @This();
 
         pub const InitOptions = struct {
             ctor: ?*const fn (*T) void = null,
@@ -1728,7 +1759,11 @@ pub fn TypedZone(comptime T: type) type {
             smr_reclaim: ?*const fn (*T) void = null,
         };
 
-        pub fn init(self: *@This(), name: []const u8, options: InitOptions) void {
+        pub fn init(
+            self: *TZone,
+            name: []const u8,
+            options: InitOptions,
+        ) void {
             self.zone.init(name, @sizeOf(T), .{
                 .alignment = @alignOf(T),
                 .ctor = @ptrCast(options.ctor),
@@ -1738,17 +1773,17 @@ pub fn TypedZone(comptime T: type) type {
             });
         }
 
-        pub fn create(self: *Self) mm.Error!*T {
+        pub fn create(self: *TZone) mm.Error!*T {
             const ret = try self.zone.alloc(.{});
             return @ptrCast(@alignCast(ret));
         }
 
-        pub fn create_opts(self: *Self, opts: AllocOpts) mm.Error!*T {
+        pub fn create_opts(self: *TZone, opts: AllocOpts) mm.Error!*T {
             const ret = try self.zone.alloc(opts);
             return @ptrCast(@alignCast(ret));
         }
 
-        pub fn destroy(self: *Self, obj: *T) void {
+        pub fn destroy(self: *TZone, obj: *T) void {
             self.zone.free(obj);
         }
     };
@@ -1787,7 +1822,7 @@ pub fn late_init() linksection(r.init) void {
             z.cpus = cpus_alloc();
 
             for (0..ke.ncpus) |i| {
-                z.cpus[i] = .init(.{
+                z.cpus[i] = rtl.CachePadded(Cpu).init(.{
                     .lock = ke.SpinLock.init("Magazine"),
                     .alloc = null,
                     .free = null,
@@ -1889,7 +1924,8 @@ fn gpa_remap(
     new_len: usize,
     ret_addr: usize,
 ) ?[*]u8 {
-    if (gpa_resize(undefined, buf, alignment, new_len, ret_addr)) return buf.ptr;
+    if (gpa_resize(undefined, buf, alignment, new_len, ret_addr))
+        return buf.ptr;
 
     const new_ptr = gpa_alloc(undefined, new_len, alignment, ret_addr) orelse
         return null;

src/platform/acpi/root.zig

@@ -276,7 +276,8 @@ pub var rsdt: ?*Rsdt = null;
 
 pub fn find_table(signature: []const u8) ?*SdtHeader {
     if (xsdt) |x| {
-        const entry_count = (x.header.length - @sizeOf(SdtHeader)) / @sizeOf(u64);
+        const entry_count = (x.header.length - @sizeOf(SdtHeader)) /
+            @sizeOf(u64);
         const entries_ptr: [*]align(1) u64 = @ptrCast(&x.entries);
 
         for (0..entry_count) |i| {
@@ -286,7 +287,8 @@ pub fn find_table(signature: []const u8) ?*SdtHeader {
             }
         }
     } else if (rsdt) |rsdt_table| {
-        const entry_count = (rsdt_table.header.length - @sizeOf(SdtHeader)) / @sizeOf(u32);
+        const entry_count = (rsdt_table.header.length - @sizeOf(SdtHeader)) /
+            @sizeOf(u32);
         const entries_ptr: [*]align(1) u32 = @ptrCast(&rsdt_table.entries);
 
         for (0..entry_count) |i| {
@@ -316,7 +318,8 @@ fn format_table(hdr: *SdtHeader, phys_addr: u64) void {
 
 fn enumerate_tables() void {
     if (xsdt) |x| {
-        const entry_count = (x.header.length - @sizeOf(SdtHeader)) / @sizeOf(u64);
+        const entry_count = (x.header.length - @sizeOf(SdtHeader)) /
+            @sizeOf(u64);
         const entries_ptr: [*]align(1) u64 = @ptrCast(&x.entries);
 
         for (0..entry_count) |i| {
@@ -325,7 +328,8 @@ fn enumerate_tables() void {
             format_table(hdr, phys);
         }
     } else if (rsdt) |rsdt_table| {
-        const entry_count = (rsdt_table.header.length - @sizeOf(SdtHeader)) / @sizeOf(u32);
+        const entry_count = (rsdt_table.header.length - @sizeOf(SdtHeader)) /
+            @sizeOf(u32);
         const entries_ptr: [*]align(1) u32 = @ptrCast(&rsdt_table.entries);
 
         for (0..entry_count) |i| {

src/platform/pc/smp.zig

@@ -23,7 +23,7 @@ const start_stack = ke.ExportedCpuLocal(usize, 0, "ap_start_stack");
 
 pub const start_thread = ke.CpuLocal(*ke.Thread, undefined);
 
-var aps_booted: std.atomic.Value(usize) = .init(0);
+var aps_booted = std.atomic.Value(usize).init(0);
 
 const ApData = extern struct {
     entry: usize align(1),
@@ -77,15 +77,18 @@ pub fn init() linksection(r.init) void {
         .Unknown => false,
     };
 
-    const init_delay: r.Nanoseconds = .init(
+    const init_delay = r.Nanoseconds.init(
         if (skip_delay) 0 else std.time.ns_per_ms * 10,
     );
-    const sipi_delay: r.Nanoseconds = .init(
+    const sipi_delay = r.Nanoseconds.init(
         if (skip_delay) std.time.ns_per_us * 10 else std.time.ns_per_us * 300,
     );
 
     const page: [*]u8 = @ptrFromInt(mm.p2v(0x8000));
-    @memcpy(page[0..trampoline_size], @as([*]u8, @ptrFromInt(trampoline_start)));
+    @memcpy(
+        page[0..trampoline_size],
+        @as([*]u8, @ptrFromInt(trampoline_start)),
+    );
 
     const data_offset = @intFromPtr(&AP_TRAMPOLINE_DATA) - trampoline_start;
     const data_phys: *ApData = @ptrFromInt(mm.p2v(0x8000 + data_offset));
@@ -97,7 +100,8 @@ pub fn init() linksection(r.init) void {
 
     // Allocate per-cpu offsets for CPU-local data.
     kep.impl.cpu_offsets = @ptrCast(offsets);
-    const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start);
+    const percpu_size = @intFromPtr(&__percpu_end) -
+        @intFromPtr(&__percpu_start);
 
     log.info("per-CPU data size: {} bytes", .{percpu_size});
 

src/platform/root.zig

@@ -1,5 +1,6 @@
 //! Platform interface: each port must implement these functions and variables.
-//! To add a new platform, create a new file and implement all required functions.
+//! To add a new platform, create a new file and implement all required
+//! functions.
 
 const config = @import("config");
 const r = @import("root");

src/proc/private.zig

@@ -15,7 +15,7 @@ fn turnstile_ctor(ts: *kep.turnstile.Turnstile) void {
         .queues = undefined,
         .waiters = 0,
         .owners = .none,
-        .lock = .init("turnstile"),
+        .lock = ke.SpinLock.init("turnstile"),
     };
     ts.reset();
 }

src/proc/thread.zig

@@ -51,7 +51,7 @@ pub fn stack_activation() void {
 
 pub fn init() void {
     thread_zone.init("thread", .{});
-    ke.thread.reaper_list.* = .init(reaper_activation);
+    ke.thread.reaper_list.* = rtl.HandoffList.init(reaper_activation);
     reaper_item.init(.High, reaper_fn, null);
     stack_item.init(.High, stack_fn, null);
 }
@@ -68,7 +68,9 @@ pub fn create_kernel(
     var td = try thread_zone.create();
 
     const stack = if (start_with_stack)
-        @intFromPtr(try mm.heap.alloc(kernel_thread_stack_size, .WaitForMemory)) +
+        @intFromPtr(
+            try mm.heap.alloc(kernel_thread_stack_size, .WaitForMemory),
+        ) +
             kernel_thread_stack_size
     else
         0;

src/rcu/hashtable.zig

@@ -3,14 +3,17 @@
 //! the buckets for write operations.
 //! The general design is inspired by XNU's smr_shash.
 
-const std = @import("std");
 const rtl = @import("rtl");
+
 const r = @import("root");
 const mm = r.mm;
 const ex = r.ex;
 const ke = r.ke;
 const rcu = r.rcu;
 
+const std = @import("std");
+const assert = std.debug.assert;
+
 /// Atomic state for the table
 const State = packed struct(u32) {
     /// The current index.
@@ -30,8 +33,6 @@ const State = packed struct(u32) {
 const Bucket = struct {
     word: std.atomic.Value(usize),
 
-    const Self = @This();
-
     const lock_tag = 0b01;
     const stop_tag = 0b10;
     const mask = lock_tag | stop_tag;
@@ -46,12 +47,12 @@ const Bucket = struct {
         return @intFromPtr(head) | stop_tag;
     }
 
-    fn load(self: *const Self) usize {
+    fn load(self: *const Bucket) usize {
         return self.word.load(.acquire) & ~@as(usize, lock_tag);
     }
 
     /// Lock the bucket and return the address.
-    fn lock(self: *Self) usize {
+    fn lock(self: *Bucket) usize {
         while (true) {
             const v = self.word.fetchOr(lock_tag, .acquire);
             if (v & lock_tag == 0) return v;
@@ -62,17 +63,17 @@ const Bucket = struct {
     }
 
     /// Unlock the bucket and store `val`.
-    fn unlock(self: *Self, val: usize) void {
-        std.debug.assert(val & lock_tag == 0);
+    fn unlock(self: *Bucket, val: usize) void {
+        assert(val & lock_tag == 0);
         self.word.store(val, .release);
     }
 
-    fn store_locked(self: *Self, val: usize) void {
-        std.debug.assert(val & lock_tag == 0);
+    fn store_locked(self: *Bucket, val: usize) void {
+        assert(val & lock_tag == 0);
         self.word.store(val | lock_tag, .release);
     }
 
-    fn unlock_no_changes(self: *Self) void {
+    fn unlock_no_changes(self: *Bucket) void {
         _ = self.word.fetchAnd(~@as(usize, lock_tag), .release);
     }
 };
@@ -145,10 +146,8 @@ const Core = struct {
     smr: *ke.smr.Domain,
     hash_link: *const fn (link: *rcu.SList.Entry, seed: u32) u32,
 
-    const Self = @This();
-
     fn init(
-        self: *Self,
+        self: *Core,
         smr: *ke.smr.Domain,
         policy: ResizePolicy,
         min_size: u32,
@@ -168,29 +167,29 @@ const Core = struct {
         const arr = try mm.zone.gpa.alloc(Bucket, size);
 
         for (arr) |*elem| {
-            elem.word = .init(Bucket.stop_for(elem));
+            elem.word = std.atomic.Value(usize).init(Bucket.stop_for(elem));
         }
 
         self.min_shift = @intCast(std.math.log2_int_ceil(usize, min));
 
-        self.state = .init(.{
+        self.state = std.atomic.Value(State).init(.{
             .current_shift = shift,
             .new_shift = shift,
             .current_index = 0,
             .new_index = 0,
         });
 
-        self.rehashing_state = .init(rehashing_none);
-        self.elems = .init(0);
+        self.rehashing_state = std.atomic.Value(u8).init(rehashing_none);
+        self.elems = std.atomic.Value(u32).init(0);
         self.smr = smr;
 
-        self.buckets[0] = .init(@ptrCast(arr));
-        self.seeds[0] = .init(0);
+        self.buckets[0] = std.atomic.Value([*]Bucket).init(@ptrCast(arr));
+        self.seeds[0] = std.atomic.Value(u32).init(0);
         self.rehash_work.init(.Normal, rehash, self);
     }
 
     fn migrate_bucket(
-        self: *Self,
+        self: *Core,
         old: *Bucket,
         new_array: [*]Bucket,
         new_seed: u32,
@@ -244,7 +243,7 @@ const Core = struct {
         old.unlock(Bucket.moved);
     }
 
-    fn enqueue_rehash(self: *Self, reason: u8) void {
+    fn enqueue_rehash(self: *Core, reason: u8) void {
         const prev = self.rehashing_state.fetchOr(reason, .monotonic);
 
         if (prev == rehashing_none) {
@@ -254,10 +253,14 @@ const Core = struct {
 
     /// Called in worker thread context when the table needs rehashing.
     fn rehash(ptr: ?*anyopaque) void {
-        const self: *Self = @ptrCast(@alignCast(ptr.?));
+        const ctx = ptr orelse unreachable;
+        const self: *Core = @ptrCast(@alignCast(ctx));
 
         while (true) {
-            const reason = self.rehashing_state.swap(rehashing_running, .monotonic);
+            const reason = self.rehashing_state.swap(
+                rehashing_running,
+                .monotonic,
+            );
 
             const elems = self.elems.load(.monotonic);
             const state = self.state.load(.monotonic);
@@ -288,7 +291,7 @@ const Core = struct {
     }
 
     /// Resize (and re-seed) the hash table to a given target shift.
-    fn resize(self: *Self, new_shift: u5) void {
+    fn resize(self: *Core, new_shift: u5) void {
         var state = self.state.load(.monotonic);
         state.new_shift = new_shift;
 
@@ -307,7 +310,7 @@ const Core = struct {
 
         // 1. Prepare the new array and publish it.
         for (new_array) |*elem| {
-            elem.word = .init(Bucket.stop_for(elem));
+            elem.word = std.atomic.Value(usize).init(Bucket.stop_for(elem));
         }
 
         self.buckets[state.new_index].store(new_array.ptr, .monotonic);
@@ -338,7 +341,7 @@ const Core = struct {
     }
 
     /// Remove a given element from the table, SMR must be entered.
-    fn remove(self: *Self, link: *rcu.SList.Entry) void {
+    fn remove(self: *Core, link: *rcu.SList.Entry) void {
         var state = self.state.load(.acquire);
         var h = self.hash_link(
             link,
@@ -394,7 +397,7 @@ const Core = struct {
         }
     }
 
-    fn bucket(self: *Self, state: State, which: Which, h: u32) *Bucket {
+    fn bucket(self: *Core, state: State, which: Which, h: u32) *Bucket {
         const idx: usize, const shift: u5 = switch (which) {
             .Cur => .{ state.current_index, @intCast(state.current_shift) },
             .New => .{ state.new_index, @intCast(state.new_shift) },
@@ -408,12 +411,12 @@ const Core = struct {
         return @as(u32, 1) << shift;
     }
 
-    fn should_grow(self: *Self, elems: u32, size: u32) bool {
+    fn should_grow(self: *Core, elems: u32, size: u32) bool {
         const target_depth = ResizePolicy.depth_for(self.policy);
         return elems > target_depth * size;
     }
 
-    fn should_shrink(self: *Self, elems: u32, size: u32) bool {
+    fn should_shrink(self: *Core, elems: u32, size: u32) bool {
         const min_size = size_for_shift(self.min_shift);
 
         return switch (self.policy) {
@@ -429,9 +432,9 @@ const Core = struct {
     /// This avoids going through multiple steps when resizing a busy table,
     /// i.e instead of doubling the size and doing 16 -> 32 -> 64,
     /// we can just do 16 -> 64 in a single resize.
-    fn target_shift_for(self: *Self, elems: u32) u5 {
+    fn target_shift_for(self: *Core, elems: u32) u5 {
         const depth = ResizePolicy.depth_for(self.policy);
-        const needed = std.math.divCeil(usize, elems, depth) catch unreachable;
+        const needed = @divCeil(elems, depth);
 
         var shift = self.min_shift;
         while (size_for_shift(shift) < needed) : (shift += 1) {}
@@ -470,7 +473,8 @@ pub fn Table(
 
     return struct {
         core: Core,
-        const Self = @This();
+
+        const Map = @This();
 
         const Options = struct {
             smr: *ke.smr.Domain,
@@ -481,13 +485,13 @@ pub fn Table(
         /// Initialize a new table, `min_size` provides the minimum count of
         /// buckets. If `min_size` is 0, it will get clamped to sane defaults
         /// depending on the chosen policy.
-        pub fn init(self: *Self, opts: Options) !void {
+        pub fn init(self: *Map, opts: Options) !void {
             try self.core.init(opts.smr, opts.policy, opts.min_size);
             self.core.hash_link = hash_link;
         }
 
         /// Get an element given a key, SMR must be entered.
-        pub fn get(self: *Self, key: Context.Key) ?*T {
+        pub fn get(self: *Map, key: Context.Key) ?*T {
             const state = self.core.state.load(.acquire);
             const h = Context.hash(
                 key,
@@ -521,7 +525,7 @@ pub fn Table(
 
         /// Find an element or insert it given a key, SMR must be entered.
         pub fn get_or_insert(
-            self: *Self,
+            self: *Map,
             key: Context.Key,
             value: *rcu.SList.Entry,
         ) ?*T {
@@ -590,11 +594,11 @@ pub fn Table(
         }
 
         /// Remove a given element from the table, SMR must be entered.
-        pub fn remove(self: *Self, link: *rcu.SList.Entry) void {
+        pub fn remove(self: *Map, link: *rcu.SList.Entry) void {
             self.core.remove(link);
         }
 
-        fn get_slow(self: *Self, key: Context.Key, bucket: *Bucket) ?*T {
+        fn get_slow(self: *Map, key: Context.Key, bucket: *Bucket) ?*T {
             // Wait until the bucket finishes moving.
             while (bucket.word.load(.monotonic) != Bucket.moved) {
                 std.atomic.spinLoopHint();

src/rtl/avl.zig

@@ -1,4 +1,4 @@
-//! AVL tree implementation
+//! AVL tree impLementation
 //! Invariant: For every node, the difference in height between its left
 //! and right subtrees is strictly limited to -1, 0 or 1.
 const bst = @import("bst.zig");
@@ -12,26 +12,28 @@ const zero: u2 = 0;
 const one: u2 = 2;
 
 fn get_bf(node: *bst.Node) u2 {
-    return node.parent.get_tag();
+    return node.parent.tag();
 }
 
 fn set_bf(node: *bst.Node, bf: u2) void {
     node.parent.set_tag(bf);
 }
 
-pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order) type {
+pub fn AVLTree(
+    comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order,
+) type {
     return struct {
         tree: bst.BST(cmp),
 
-        const Self = @This();
+        const Tree = @This();
 
-        pub fn init() Self {
-            return Self{
+        pub fn init() Tree {
+            return Tree{
                 .tree = bst.BST(cmp).init(),
             };
         }
 
-        fn rotate_left(self: *Self, x: *bst.Node, z: *bst.Node) *bst.Node {
+        fn rotate_left(self: *Tree, x: *bst.Node, z: *bst.Node) *bst.Node {
             x.right = z.left;
 
             if (!self.tree.is_nil(z.left)) {
@@ -53,7 +55,7 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde
             return z;
         }
 
-        fn rotate_right(self: *Self, x: *bst.Node, z: *bst.Node) *bst.Node {
+        fn rotate_right(self: *Tree, x: *bst.Node, z: *bst.Node) *bst.Node {
             x.left = z.right;
 
             if (!self.tree.is_nil(z.right)) {
@@ -75,7 +77,11 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde
             return z;
         }
 
-        fn rotate_right_left(self: *Self, x: *bst.Node, z: *bst.Node) *bst.Node {
+        fn rotate_right_left(
+            self: *Tree,
+            x: *bst.Node,
+            z: *bst.Node,
+        ) *bst.Node {
             var y = z.left;
             var t2 = y.right;
 
@@ -123,7 +129,11 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde
             return y;
         }
 
-        fn rotate_left_right(self: *Self, x: *bst.Node, z: *bst.Node) *bst.Node {
+        fn rotate_left_right(
+            self: *Tree,
+            x: *bst.Node,
+            z: *bst.Node,
+        ) *bst.Node {
             var y = z.right;
             var t2 = y.left;
 
@@ -172,9 +182,14 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde
         }
 
         /// Insert the node `elem` into the tree.
-        pub fn insert(self: *Self, elem: *bst.Node) !void {
+        pub fn insert(self: *Tree, elem: *bst.Node) !void {
             if (self.tree.is_empty()) {
-                self.tree.insert(elem) catch unreachable;
+                self.tree.insert(elem) catch {
+                    // This can't ever error since the tree is empty, so the
+                    // element can't already exist.
+                    unreachable;
+                };
+
                 set_bf(elem, zero);
                 return;
             }
@@ -188,17 +203,18 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde
 
             // Now the tree may be unbalanced, rebalance it.
             var node = elem;
-            var parent = elem.parent.get_ptr();
+            var parent = elem.parent.ptr();
             var n = &bst.nil;
 
-            while (!self.tree.is_nil(parent)) : (parent = parent.parent.get_ptr()) {
-                const orig_parent = parent.parent.get_ptr();
+            while (!self.tree.is_nil(parent)) : (parent = parent.parent.ptr()) {
+                const orig_parent = parent.parent.ptr();
 
                 if (node == parent.right) {
                     if (get_bf(parent) == one) {
-                        // This node has been inserted on the right of the parent,
-                        // and the parent already has a balance factor of 1 (right-heavy),
-                        // the parent's BF would then become +2, which breaks the invariant.
+                        // This node has been inserted on the right of the
+                        // parent,  and the parent already has a balance factor
+                        // of 1 (right-heavy),  the parent's BF would then
+                        // become +2, which breaks the invariant.
                         // We need to rebalance.
 
                         if (get_bf(node) == minus_one) {
@@ -214,13 +230,16 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde
                             //      2
                             //     / \
                             //    1  3
-                            // In the code, `parent` would be 1 and `node` would be 2
+                            // In the code, `parent` would be 1 and `node` would
+                            // be 2
                             n = self.rotate_left(parent, node);
                         }
                     } else {
                         // The balance factor is either 0 or -1, increase it.
-                        // If it was -1, now it is 0 so the subtree is perfectly balanced, we can stop.
-                        // If it was 0, now it is +1, so we may need to check higher up for imbalances.
+                        // If it was -1, now it is 0 so the subtree is
+                        // perfectly balanced, we can stop.
+                        // If it was 0, now it is +1, so we may need to check
+                        // higher up for imbalances.
 
                         if (get_bf(parent) == minus_one) {
                             set_bf(parent, zero);
@@ -236,9 +255,10 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde
                 // The logic is the same as the other case but inverted.
                 else {
                     if (get_bf(parent) == minus_one) {
-                        // This node has been inserted on the left of the parent,
-                        // and the parent already has a balance factor of -1 (left-heavy),
-                        // the parent's balance factor would then become -2, which breaks the invariant.
+                        // This node has been inserted on the left of the
+                        // parent, and the parent already has a balance factor
+                        // of -1 (left-heavy), the parent's balance factor would
+                        // then become -2, which breaks the invariant.
                         // We need to rebalance.
 
                         if (get_bf(node) == one) {
@@ -276,10 +296,10 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde
         }
 
         /// Delete the node `elem` from the tree.
-        pub fn delete(self: *Self, elem: *bst.Node) void {
+        pub fn delete(self: *Tree, elem: *bst.Node) void {
             var node = elem;
             var original_parent = &bst.nil;
-            var parent = node.parent.get_ptr();
+            var parent = node.parent.ptr();
             var was_on_left = !self.tree.is_nil(parent) and parent.left == node;
             var b = zero;
 
@@ -289,7 +309,7 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde
                 self.tree.transplant(elem, elem.left);
             } else {
                 var succ = self.tree.successor(node);
-                parent = succ.parent.get_ptr();
+                parent = succ.parent.ptr();
                 was_on_left = !self.tree.is_nil(parent) and succ == parent.left;
 
                 if (parent != elem) {
@@ -297,7 +317,8 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde
                     succ.right = elem.right;
                     succ.right.parent.set_ptr(succ);
                 } else {
-                    // The successor is the direct child, so start rebalancing from it.
+                    // The successor is the direct child, so start rebalancing
+                    // from it.
                     parent = succ;
                 }
 
@@ -311,7 +332,7 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde
 
             // Rebalance the tree.
             while (!self.tree.is_nil(parent)) {
-                original_parent = parent.parent.get_ptr();
+                original_parent = parent.parent.ptr();
 
                 if (node == parent.left or was_on_left) {
                     was_on_left = false;

src/rtl/barrier.zig

@@ -1,9 +1,11 @@
 //! Atomic barrier implementations on various architectures.
 //! This is needed since Zig removed @fence.
 
-const std = @import("std");
 const builtin = @import("builtin");
 
+const std = @import("std");
+const assert = std.debug.assert;
+
 pub inline fn fence(comptime ordering: std.builtin.AtomicOrder) void {
     switch (ordering) {
         .seq_cst, .acq_rel => mb(),
@@ -67,7 +69,11 @@ inline fn rel() void {
 
 /// Copy to dest from src with atomic loads.
 /// Both addresses need to be aligned.
-pub fn atomic_load_memcpy(dest: anytype, src: anytype, comptime ordering: std.builtin.AtomicOrder) void {
+pub fn atomic_load_memcpy(
+    dest: anytype,
+    src: anytype,
+    comptime ordering: std.builtin.AtomicOrder,
+) void {
     const T = @TypeOf(dest.*);
     const Word = switch (@alignOf(T)) {
         1 => u8,
@@ -78,7 +84,7 @@ pub fn atomic_load_memcpy(dest: anytype, src: anytype, comptime ordering: std.bu
 
     const word_size = @sizeOf(Word);
     const n = @sizeOf(T);
-    comptime std.debug.assert(n % word_size == 0);
+    comptime assert(n % word_size == 0);
 
     const src_words: [*]const volatile Word = @ptrCast(src);
     const dest_words: [*]Word = @ptrCast(dest);
@@ -91,7 +97,11 @@ pub fn atomic_load_memcpy(dest: anytype, src: anytype, comptime ordering: std.bu
 
 /// Copy to dest from src with atomic stores.
 /// Both addresses need to be aligned.
-pub fn atomic_store_memcpy(dest: anytype, src: anytype, comptime ordering: std.builtin.AtomicOrder) void {
+pub fn atomic_store_memcpy(
+    dest: anytype,
+    src: anytype,
+    comptime ordering: std.builtin.AtomicOrder,
+) void {
     const T = @TypeOf(dest.*);
     const Word = switch (@alignOf(T)) {
         1 => u8,
@@ -102,7 +112,7 @@ pub fn atomic_store_memcpy(dest: anytype, src: anytype, comptime ordering: std.b
 
     const word_size = @sizeOf(Word);
     const n = @sizeOf(T);
-    comptime std.debug.assert(n % word_size == 0);
+    comptime assert(n % word_size == 0);
 
     const src_words: [*]const volatile Word = @ptrCast(src);
     const dest_words: [*]Word = @ptrCast(dest);

src/rtl/bitmap.zig

@@ -1,7 +1,13 @@
 const std = @import("std");
+const AtomicOrder = std.builtin.AtomicOrder;
+const assert = std.debug.assert;
+
+fn ClosestType(comptime N: usize) type {
+    const bits = std.math.ceilPowerOfTwo(usize, N) catch {
+        // Shouldn't happen.
+        unreachable;
+    };
 
-fn get_closest_type(comptime N: usize) type {
-    const bits = std.math.ceilPowerOfTwo(usize, N) catch unreachable;
     return @Int(.unsigned, bits);
 }
 
@@ -12,15 +18,15 @@ fn bit_mask(comptime Word: type, bit_index: usize) Word {
 /// Static bitmap that uses the most efficient underlying word size for N < 64
 pub fn BitMap(comptime N: usize) type {
     return struct {
-        pub const Word = if (N > 32) u64 else get_closest_type(N);
-        const Self = @This();
+        pub const Word = if (N > 32) u64 else ClosestType(N);
+        const Bitmap = @This();
         const bits_per_word = @bitSizeOf(Word);
         const arr_len = (N + bits_per_word - 1) / bits_per_word;
 
         storage: [arr_len]Word,
 
         fn valid_word_mask(word_index: usize) Word {
-            std.debug.assert(word_index < arr_len);
+            assert(word_index < arr_len);
 
             if (word_index != arr_len - 1) return std.math.maxInt(Word);
 
@@ -30,39 +36,39 @@ pub fn BitMap(comptime N: usize) type {
         }
 
         /// Initialize a bitmap, setting all bits to `val`.
-        pub fn init(val: bool) Self {
+        pub fn init(val: bool) Bitmap {
             return .{
-                .storage = @splat(if (val) std.math.maxInt(Self.Word) else 0),
+                .storage = @splat(if (val) std.math.maxInt(Bitmap.Word) else 0),
             };
         }
 
         /// Set a given bit.
-        pub fn set(self: *Self, bit: usize) void {
-            std.debug.assert(bit < N);
-            const word_index = bit / Self.bits_per_word;
-            const bit_index = bit % Self.bits_per_word;
+        pub fn set(self: *Bitmap, bit: usize) void {
+            assert(bit < N);
+            const word_index = bit / Bitmap.bits_per_word;
+            const bit_index = bit % Bitmap.bits_per_word;
 
             self.storage[word_index] |= bit_mask(Word, bit_index);
         }
 
         /// Clear a given bit.
-        pub fn clear(self: *Self, bit: usize) void {
-            std.debug.assert(bit < N);
-            const word_index = bit / Self.bits_per_word;
-            const bit_index = bit % Self.bits_per_word;
+        pub fn clear(self: *Bitmap, bit: usize) void {
+            assert(bit < N);
+            const word_index = bit / Bitmap.bits_per_word;
+            const bit_index = bit % Bitmap.bits_per_word;
             self.storage[word_index] &= ~bit_mask(Word, bit_index);
         }
 
         /// Get a given bit.
-        pub fn get(self: *const Self, bit: usize) bool {
-            std.debug.assert(bit < N);
-            const word_index = bit / Self.bits_per_word;
-            const bit_index = bit % Self.bits_per_word;
+        pub fn get(self: *const Bitmap, bit: usize) bool {
+            assert(bit < N);
+            const word_index = bit / Bitmap.bits_per_word;
+            const bit_index = bit % Bitmap.bits_per_word;
             return self.storage[word_index] & bit_mask(Word, bit_index) != 0;
         }
 
         /// Count the number of set bits in the bitmap.
-        pub fn count(self: *const Self) usize {
+        pub fn count(self: *const Bitmap) usize {
             var total: usize = 0;
             for (self.storage, 0..) |word, i| {
                 total += @intCast(@popCount(word & valid_word_mask(i)));
@@ -71,7 +77,7 @@ pub fn BitMap(comptime N: usize) type {
         }
 
         pub const Iterator = struct {
-            bitmap: *const Self,
+            bitmap: *const Bitmap,
             word_index: usize = 0,
             word: Word = 0,
 
@@ -81,25 +87,29 @@ pub fn BitMap(comptime N: usize) type {
                     if (self.word != 0) {
                         const bit_index: usize = @intCast(@ctz(self.word));
                         self.word &= self.word - 1;
-                        return ((self.word_index - 1) * bits_per_word) + bit_index;
+                        return ((self.word_index - 1) * bits_per_word) +
+                            bit_index;
                     }
 
                     if (self.word_index >= arr_len) return null;
 
-                    self.word = self.bitmap.storage[self.word_index] & valid_word_mask(self.word_index);
+                    self.word = self.bitmap.storage[self.word_index] &
+                        valid_word_mask(self.word_index);
                     self.word_index += 1;
                 }
             }
         };
 
         /// Iterate over the indexes of set bits in ascending order.
-        pub fn iter(self: *const Self) Iterator {
+        pub fn iter(self: *const Bitmap) Iterator {
             return .{ .bitmap = self };
         }
 
         /// Check whether all bits in the bitmap are equal to a given value.
-        pub fn is_all(self: *const Self, value: bool) bool {
-            const v: Self.Word = if (value) std.math.maxInt(Self.Word) else 0;
+        pub fn is_all(self: *const Bitmap, value: bool) bool {
+            const v: Bitmap.Word = if (value) std.math.maxInt(Bitmap.Word)
+            else
+                0;
             for (self.storage) |word| {
                 if (word != v) return false;
             }
@@ -108,18 +118,19 @@ pub fn BitMap(comptime N: usize) type {
     };
 }
 
-/// Atomic static bitmap with the same API as `BitMap`, but each operation accepts a memory ordering.
+/// Atomic static bitmap with the same API as `BitMap`, but each operation
+/// accepts a memory ordering.
 pub fn AtomicBitMap(comptime N: usize) type {
     return struct {
-        pub const Word = if (N > 32) u64 else get_closest_type(N);
-        const Self = @This();
+        pub const Word = if (N > 32) u64 else ClosestType(N);
+        const Bitmap = @This();
         const bits_per_word = @bitSizeOf(Word);
         const arr_len = (N + bits_per_word - 1) / bits_per_word;
 
         storage: [arr_len]std.atomic.Value(Word),
 
         fn valid_word_mask(word_index: usize) Word {
-            std.debug.assert(word_index < arr_len);
+            assert(word_index < arr_len);
 
             if (word_index != arr_len - 1) return std.math.maxInt(Word);
 
@@ -129,9 +140,9 @@ pub fn AtomicBitMap(comptime N: usize) type {
         }
 
         /// Initialize a bitmap, setting all bits to `val`.
-        pub fn init(val: bool) Self {
+        pub fn init(val: bool) Bitmap {
             var storage: [arr_len]std.atomic.Value(Word) = undefined;
-            const word: Word = if (val) std.math.maxInt(Self.Word) else 0;
+            const word: Word = if (val) std.math.maxInt(Bitmap.Word) else 0;
             inline for (&storage) |*elem| {
                 elem.* = std.atomic.Value(Word).init(word);
             }
@@ -139,58 +150,86 @@ pub fn AtomicBitMap(comptime N: usize) type {
         }
 
         /// Set a given bit.
-        pub fn set(self: *Self, bit: usize, comptime ordering: std.builtin.AtomicOrder) void {
-            std.debug.assert(bit < N);
-            const word_index = bit / Self.bits_per_word;
-            const bit_index = bit % Self.bits_per_word;
+        pub fn set(
+            self: *Bitmap,
+            bit: usize,
+            comptime order: AtomicOrder,
+        ) void {
+            assert(bit < N);
+            const word_index = bit / Bitmap.bits_per_word;
+            const bit_index = bit % Bitmap.bits_per_word;
 
-            _ = self.storage[word_index].fetchOr(bit_mask(Word, bit_index), ordering);
+            _ = self.storage[word_index].fetchOr(
+                bit_mask(Word, bit_index),
+                order,
+            );
         }
 
         /// Clear a given bit.
-        pub fn clear(self: *Self, bit: usize, comptime ordering: std.builtin.AtomicOrder) void {
-            std.debug.assert(bit < N);
-            const word_index = bit / Self.bits_per_word;
-            const bit_index = bit % Self.bits_per_word;
+        pub fn clear(
+            self: *Bitmap,
+            bit: usize,
+            comptime order: AtomicOrder,
+        ) void {
+            assert(bit < N);
+            const word_index = bit / Bitmap.bits_per_word;
+            const bit_index = bit % Bitmap.bits_per_word;
 
-            _ = self.storage[word_index].fetchAnd(~bit_mask(Word, bit_index), ordering);
+            _ = self.storage[word_index].fetchAnd(
+                ~bit_mask(Word, bit_index),
+                order,
+            );
         }
 
         /// Get a given bit.
-        pub fn get(self: *const Self, bit: usize, comptime ordering: std.builtin.AtomicOrder) bool {
-            std.debug.assert(bit < N);
-            const word_index = bit / Self.bits_per_word;
-            const bit_index = bit % Self.bits_per_word;
-            return self.storage[word_index].load(ordering) & bit_mask(Word, bit_index) != 0;
+        pub fn get(
+            self: *const Bitmap,
+            bit: usize,
+            comptime order: AtomicOrder,
+        ) bool {
+            assert(bit < N);
+            const word_index = bit / Bitmap.bits_per_word;
+            const bit_index = bit % Bitmap.bits_per_word;
+            return self.storage[word_index].load(order) &
+                bit_mask(Word, bit_index) != 0;
         }
 
         /// Count the number of set bits in the bitmap.
-        pub fn count(self: *const Self, comptime ordering: std.builtin.AtomicOrder) usize {
+        pub fn count(
+            self: *const Bitmap,
+            comptime order: AtomicOrder,
+        ) usize {
             var total: usize = 0;
             for (self.storage, 0..) |word, i| {
-                total += @intCast(@popCount(word.load(ordering) & valid_word_mask(i)));
+                total += @intCast(@popCount(word.load(order) &
+                    valid_word_mask(i)));
             }
             return total;
         }
 
-        pub fn Iterator(comptime ordering: std.builtin.AtomicOrder) type {
+        pub fn Iterator(comptime order: AtomicOrder) type {
             return struct {
-                bitmap: *const Self,
+                bitmap: *const Bitmap,
                 word_index: usize = 0,
                 word: Word = 0,
 
-                /// Return the index of the next set bit, or null when exhausted.
+                /// Return the index of the next set bit, or null when
+                /// exhausted.
                 pub fn next(self: *@This()) ?usize {
                     while (true) {
                         if (self.word != 0) {
                             const bit_index: usize = @intCast(@ctz(self.word));
                             self.word &= self.word - 1;
-                            return ((self.word_index - 1) * bits_per_word) + bit_index;
+                            return ((self.word_index - 1) * bits_per_word) +
+                                bit_index;
                         }
 
                         if (self.word_index >= arr_len) return null;
 
-                        self.word = self.bitmap.storage[self.word_index].load(ordering) & valid_word_mask(self.word_index);
+                        const word = &self.bitmap.storage[self.word_index];
+
+                        self.word = word.load(order) &
+                            valid_word_mask(self.word_index);
                         self.word_index += 1;
                     }
                 }
@@ -198,15 +237,23 @@ pub fn AtomicBitMap(comptime N: usize) type {
         }
 
         /// Iterate over the indexes of set bits in ascending order.
-        pub fn iter(self: *const Self, comptime ordering: std.builtin.AtomicOrder) Iterator(ordering) {
+        pub fn iter(
+            self: *const Bitmap,
+            comptime order: AtomicOrder,
+        ) Iterator(order) {
             return .{ .bitmap = self };
         }
 
         /// Check whether all bits in the bitmap are equal to a given value.
-        pub fn is_all(self: *const Self, value: bool, comptime ordering: std.builtin.AtomicOrder) bool {
-            const v: Self.Word = if (value) std.math.maxInt(Self.Word) else 0;
+        pub fn is_all(
+            self: *const Bitmap,
+            value: bool,
+            comptime order: AtomicOrder,
+        ) bool {
+            const v: Bitmap.Word = if (value) std.math.maxInt(Bitmap.Word) else
+                0;
             for (self.storage) |word| {
-                if (word.load(ordering) != v) return false;
+                if (word.load(order) != v) return false;
             }
             return true;
         }
@@ -214,37 +261,37 @@ pub fn AtomicBitMap(comptime N: usize) type {
 }
 
 test "BitMap has proper underlying type" {
-    std.debug.assert(BitMap(8).Word == u8);
-    std.debug.assert(BitMap(10).Word == u16);
-    std.debug.assert(BitMap(16).Word == u16);
-    std.debug.assert(BitMap(21).Word == u32);
-    std.debug.assert(BitMap(32).Word == u32);
-    std.debug.assert(BitMap(64).Word == u64);
-    std.debug.assert(BitMap(128).Word == u64);
-    std.debug.assert(BitMap(1024).Word == u64);
+    assert(BitMap(8).Word == u8);
+    assert(BitMap(10).Word == u16);
+    assert(BitMap(16).Word == u16);
+    assert(BitMap(21).Word == u32);
+    assert(BitMap(32).Word == u32);
+    assert(BitMap(64).Word == u64);
+    assert(BitMap(128).Word == u64);
+    assert(BitMap(1024).Word == u64);
 }
 
 test BitMap {
     var bitmap = BitMap(10).init(false);
 
-    std.debug.assert(@TypeOf(bitmap).Word == u16);
-    std.debug.assert(bitmap.get(0) == false);
+    assert(@TypeOf(bitmap).Word == u16);
+    assert(bitmap.get(0) == false);
     bitmap.set(0);
 
-    std.debug.assert(bitmap.get(0) == true);
+    assert(bitmap.get(0) == true);
 
     bitmap.clear(0);
 
-    std.debug.assert(bitmap.get(0) == false);
+    assert(bitmap.get(0) == false);
 }
 
 test "BitMap count and iter" {
     var bitmap = BitMap(70).init(false);
 
-    std.debug.assert(bitmap.count() == 0);
+    assert(bitmap.count() == 0);
     {
         var it = bitmap.iter();
-        std.debug.assert(it.next() == null);
+        assert(it.next() == null);
     }
 
     bitmap.set(0);
@@ -253,54 +300,54 @@ test "BitMap count and iter" {
     bitmap.set(64);
     bitmap.set(69);
 
-    std.debug.assert(bitmap.count() == 5);
+    assert(bitmap.count() == 5);
 
     var it = bitmap.iter();
-    std.debug.assert(it.next().? == 0);
-    std.debug.assert(it.next().? == 3);
-    std.debug.assert(it.next().? == 63);
-    std.debug.assert(it.next().? == 64);
-    std.debug.assert(it.next().? == 69);
-    std.debug.assert(it.next() == null);
+    assert(it.next().? == 0);
+    assert(it.next().? == 3);
+    assert(it.next().? == 63);
+    assert(it.next().? == 64);
+    assert(it.next().? == 69);
+    assert(it.next() == null);
 
     bitmap.clear(63);
 
-    std.debug.assert(bitmap.count() == 4);
+    assert(bitmap.count() == 4);
 
     var all = BitMap(70).init(true);
-    std.debug.assert(all.count() == 70);
+    assert(all.count() == 70);
 
     var all_it = all.iter();
     var expected: usize = 0;
     while (all_it.next()) |bit| : (expected += 1) {
-        std.debug.assert(bit == expected);
+        assert(bit == expected);
     }
-    std.debug.assert(expected == 70);
+    assert(expected == 70);
 }
 
 test AtomicBitMap {
     var bitmap = AtomicBitMap(10).init(false);
 
-    std.debug.assert(@TypeOf(bitmap).Word == u16);
-    std.debug.assert(bitmap.get(0, .seq_cst) == false);
+    assert(@TypeOf(bitmap).Word == u16);
+    assert(bitmap.get(0, .seq_cst) == false);
     bitmap.set(0, .seq_cst);
 
-    std.debug.assert(bitmap.get(0, .seq_cst) == true);
-    std.debug.assert(bitmap.is_all(false, .seq_cst) == false);
+    assert(bitmap.get(0, .seq_cst) == true);
+    assert(bitmap.is_all(false, .seq_cst) == false);
 
     bitmap.clear(0, .seq_cst);
 
-    std.debug.assert(bitmap.get(0, .seq_cst) == false);
-    std.debug.assert(bitmap.is_all(false, .seq_cst) == true);
+    assert(bitmap.get(0, .seq_cst) == false);
+    assert(bitmap.is_all(false, .seq_cst) == true);
 }
 
 test "AtomicBitMap count and iter" {
     var bitmap = AtomicBitMap(70).init(false);
 
-    std.debug.assert(bitmap.count(.seq_cst) == 0);
+    assert(bitmap.count(.seq_cst) == 0);
     {
         var it = bitmap.iter(.seq_cst);
-        std.debug.assert(it.next() == null);
+        assert(it.next() == null);
     }
 
     bitmap.set(0, .seq_cst);
@@ -309,27 +356,27 @@ test "AtomicBitMap count and iter" {
     bitmap.set(64, .seq_cst);
     bitmap.set(69, .seq_cst);
 
-    std.debug.assert(bitmap.count(.seq_cst) == 5);
+    assert(bitmap.count(.seq_cst) == 5);
 
     var it = bitmap.iter(.seq_cst);
-    std.debug.assert(it.next().? == 0);
-    std.debug.assert(it.next().? == 3);
-    std.debug.assert(it.next().? == 63);
-    std.debug.assert(it.next().? == 64);
-    std.debug.assert(it.next().? == 69);
-    std.debug.assert(it.next() == null);
+    assert(it.next().? == 0);
+    assert(it.next().? == 3);
+    assert(it.next().? == 63);
+    assert(it.next().? == 64);
+    assert(it.next().? == 69);
+    assert(it.next() == null);
 
     bitmap.clear(63, .seq_cst);
 
-    std.debug.assert(bitmap.count(.seq_cst) == 4);
+    assert(bitmap.count(.seq_cst) == 4);
 
     var all = AtomicBitMap(70).init(true);
-    std.debug.assert(all.count(.seq_cst) == 70);
+    assert(all.count(.seq_cst) == 70);
 
     var all_it = all.iter(.seq_cst);
     var expected: usize = 0;
     while (all_it.next()) |bit| : (expected += 1) {
-        std.debug.assert(bit == expected);
+        assert(bit == expected);
     }
-    std.debug.assert(expected == 70);
+    assert(expected == 70);
 }

src/rtl/bst.zig

@@ -14,27 +14,29 @@ pub var nil: Node = .{
 };
 
 /// Base struct for binary search trees.
-/// This should be used as a field in other types that are based on binary search trees, such as red-black trees or AVL trees.
+/// This should be used as a field in other types that are based on
+/// binary search trees, such as red-black trees or AVL trees.
 pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type {
     return struct {
-        const Self = @This();
+        const Tree = @This();
 
         root: *Node = &nil,
 
-        pub fn init() Self {
-            return Self{
+        pub fn init() Tree {
+            return Tree{
                 .root = &nil,
             };
         }
 
         /// Return whether or not the given node is the sentinel nil node.
-        pub fn is_nil(self: *Self, node: *Node) bool {
+        pub fn is_nil(self: *Tree, node: *Node) bool {
             _ = self;
             return node == &nil;
         }
 
-        /// Search for a node in the BST that matches `elem`, and return a pointer to it if found.
-        pub fn search(self: *Self, elem: *Node) ?*Node {
+        /// Search for a node in the BST that matches `elem`.
+        /// Return a pointer to it if found.
+        pub fn search(self: *Tree, elem: *Node) ?*Node {
             var node = self.root;
             while (!self.is_nil(node)) {
                 const order = cmp(elem, node);
@@ -49,8 +51,9 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type {
             return null;
         }
 
-        /// Return a pointer to the minimum node in the subtree rooted at `node`.
-        pub fn minimum(self: *Self, node: *Node) *Node {
+        /// Return a pointer to the minimum node in the subtree
+        /// rooted at `node`.
+        pub fn minimum(self: *Tree, node: *Node) *Node {
             var current = node;
             while (!self.is_nil(current.left)) {
                 current = current.left;
@@ -58,8 +61,9 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type {
             return current;
         }
 
-        /// Return a pointer to the maximum node in the subtree rooted at `node`.
-        pub fn maximum(self: *Self, node: *Node) *Node {
+        /// Return a pointer to the maximum node in the subtree
+        /// rooted at `node`.
+        pub fn maximum(self: *Tree, node: *Node) *Node {
             var current = node;
             while (!self.is_nil(current.right)) {
                 current = current.right;
@@ -67,8 +71,9 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type {
             return current;
         }
 
-        /// Return a pointer to the successor of `node`, or nil if `node` is the maximum element in the tree.
-        pub fn successor(self: *Self, node: *Node) *Node {
+        /// Return a pointer to the successor of `node`, or nil if `node`
+        /// is the maximum element in the tree.
+        pub fn successor(self: *Tree, node: *Node) *Node {
             if (!self.is_nil(node.right)) {
                 // Minimum of right subtree
                 return self.minimum(node.right);
@@ -76,10 +81,10 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type {
 
             // Go up and find it.
             var current = node;
-            var parent = current.parent.get_ptr();
+            var parent = current.parent.ptr();
             while (!self.is_nil(parent) and current == parent.right) {
                 current = parent;
-                parent = parent.parent.get_ptr();
+                parent = parent.parent.ptr();
             }
 
             if (parent == self.root) {
@@ -89,9 +94,9 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type {
             return parent;
         }
 
-        /// Transplant the subtree rooted at `u` with the subtree rooted at `v`. This is used as a helper function.
-        pub fn transplant(self: *Self, u: *Node, v: *Node) void {
-            const u_parent = u.parent.get_ptr();
+        /// Transplant the subtree rooted at `u` with the subtree rooted at `v`.
+        pub fn transplant(self: *Tree, u: *Node, v: *Node) void {
+            const u_parent = u.parent.ptr();
             if (self.is_nil(u_parent)) {
                 self.root = v;
             } else if (u == u_parent.left) {
@@ -104,7 +109,7 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type {
 
         // Insert `elem` into the BST, if already inserted, do nothing.
         // This does not perform any balancing.
-        pub fn insert(self: *Self, elem: *Node) !void {
+        pub fn insert(self: *Tree, elem: *Node) !void {
             var current = self.root;
 
             if (self.is_nil(current)) {
@@ -142,7 +147,7 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type {
         }
 
         /// Return whether or not the tree is empty.
-        pub fn is_empty(self: *Self) bool {
+        pub fn is_empty(self: *Tree) bool {
             return self.is_nil(self.root);
         }
     };
@@ -162,7 +167,7 @@ fn node_cmp(a: *const Node, b: *const Node) std.math.Order {
 }
 
 test BST {
-    var tree: BST(node_cmp) = .init();
+    var tree = BST(node_cmp).init();
 
     var nodes = [_]MyNode{
         .{ .value = 5, .node = undefined },
@@ -181,6 +186,11 @@ test BST {
     for (&nodes) |*node| {
         const found = tree.search(&node.node);
         try std.testing.expect(found != null);
-        try std.testing.expectEqual(node.value, @as(*MyNode, @fieldParentPtr("node", found.?)).value);
+        const f = found orelse unreachable;
+
+        try std.testing.expectEqual(
+            node.value,
+            @as(*MyNode, @fieldParentPtr("node", f)).value,
+        );
     }
 }

src/rtl/cmdline.zig

@@ -51,7 +51,7 @@ test "get_number() should return the integer value of a key" {
     try std.testing.expectEqual(100, get_number(cmdline, "baz").?);
 }
 
-test "get_number() should return null if the key is not found or value is not an integer" {
+test "get_number() returns null if the key not found or value not an integer" {
     const cmdline = "foo=42 baz=qux";
     try std.testing.expect(get_number(cmdline, "quux") == null);
     try std.testing.expect(get_number(cmdline, "baz") == null);

src/rtl/handoff.zig

@@ -1,15 +1,15 @@
-//! Lock-free singly linked list with an activation callback when items are first inserted.
-//! This is meant for use where there is a set of objects being worked upon in an
-//! asynchronous context.
+//! Lock-free singly linked list with an activation callback invoked when items
+//! are first inserted.
+//! This is meant for use where there is a set of objects being worked upon in
+//! an asynchronous context.
 //! Data structure generalized from the NT kernel reaper lists, and inspired by
-//! MINTIA's custody list (https://github.com/xrarch/mintia2/blob/main/OS/Executive/Ke/KeCustodyList.jkl)
+//! MINTIA's custody list (https://github.com/xrarch/mintia2).
 
 const std = @import("std");
 
 pub const HandoffList = struct {
-    const ActivationFn = *const fn (*Self) void;
-
-    const Self = @This();
+    const ActivationFn = *const fn (*HandoffList) void;
+    const Callback = *const fn (obj: *anyopaque, ctx: ?*anyopaque) void;
 
     /// Non-null value indicating processing is in progress.
     const processing_tag: *anyopaque = @ptrFromInt(1);
@@ -17,21 +17,26 @@ pub const HandoffList = struct {
     activation: ActivationFn,
     head: std.atomic.Value(?*anyopaque),
 
-    pub fn init(activation: ActivationFn) Self {
+    pub fn init(activation: ActivationFn) HandoffList {
         return .{
             .activation = activation,
-            .head = .init(null),
+            .head = std.atomic.Value(?*anyopaque).init(null),
         };
     }
 
     /// Insert an element at the head of the list.
     /// If the list was previously empty, the activation routine will be called.
-    pub fn insert(self: *Self, link: *?*anyopaque) void {
+    pub fn insert(self: *HandoffList, link: *?*anyopaque) void {
         var head = self.head.load(.monotonic);
 
         while (true) {
             link.* = head;
-            head = self.head.cmpxchgWeak(head, @ptrCast(link), .release, .monotonic) orelse break;
+            head = self.head.cmpxchgWeak(
+                head,
+                @ptrCast(link),
+                .release,
+                .monotonic,
+            ) orelse break;
         }
 
         if (head == null) {
@@ -43,17 +48,27 @@ pub const HandoffList = struct {
     /// Pop the entire list and process it.
     /// The callback takes the object pointer as its first parameter
     /// and ctx as its second.
-    pub fn process(self: *Self, callback: *const fn (*anyopaque, ?*anyopaque) void, ctx: ?*anyopaque) void {
+    pub fn process(
+        self: *HandoffList,
+        callback: Callback,
+        ctx: ?*anyopaque,
+    ) void {
         while (true) {
             const head = self.head.swap(processing_tag, .acquire);
 
-            // Someone else is already processing it (or we set it previously and got nothing).
+            // Someone else is already processing it, or we set it previously
+            // and got nothing.
             if (head == processing_tag) return;
 
-            // List is empty; this shouldn't happen, but it'll just return on the next
-            // iteration if it is truly empty.
+            // List is empty; this shouldn't happen, but it'll just return
+            // on the next iteration if it is truly empty.
             if (head == null) {
-                _ = self.head.cmpxchgStrong(processing_tag, null, .release, .monotonic) orelse return;
+                _ = self.head.cmpxchgStrong(
+                    processing_tag,
+                    null,
+                    .release,
+                    .monotonic,
+                ) orelse return;
                 continue;
             }
 
@@ -68,7 +83,12 @@ pub const HandoffList = struct {
             }
 
             // If the cmpxchg fails, a new item was added; also process it.
-            _ = self.head.cmpxchgStrong(processing_tag, null, .release, .monotonic) orelse return;
+            _ = self.head.cmpxchgStrong(
+                processing_tag,
+                null,
+                .release,
+                .monotonic,
+            ) orelse return;
         }
     }
 };
@@ -99,7 +119,7 @@ fn test_activation(list: *HandoffList) void {
 
 fn record_node(node_ptr: *anyopaque, ctx_ptr: ?*anyopaque) void {
     const node: *TestNode = @ptrCast(@alignCast(node_ptr));
-    const ctx: *TestContext = @ptrCast(@alignCast(ctx_ptr.?));
+    const ctx: *TestContext = @ptrCast(@alignCast(ctx_ptr orelse unreachable));
 
     ctx.processed[ctx.processed_len] = node.value;
     ctx.processed_len += 1;
@@ -107,7 +127,7 @@ fn record_node(node_ptr: *anyopaque, ctx_ptr: ?*anyopaque) void {
 
 fn record_node_and_insert(node_ptr: *anyopaque, ctx_ptr: ?*anyopaque) void {
     const node: *TestNode = @ptrCast(@alignCast(node_ptr));
-    const ctx: *TestContext = @ptrCast(@alignCast(ctx_ptr.?));
+    const ctx: *TestContext = @ptrCast(@alignCast(ctx_ptr orelse unreachable));
 
     ctx.processed[ctx.processed_len] = node.value;
     ctx.processed_len += 1;

src/rtl/linker_set.zig

@@ -1,3 +1,5 @@
+//! Linker sets, inspired by FreeBSD.
+
 const std = @import("std");
 
 /// Used for declaring global objects that get collected by the linker
@@ -13,8 +15,6 @@ pub fn LinkerSet(name: []const u8, comptime T: type) type {
         const start = @extern(*u8, .{ .name = "__start_set_" ++ name });
         const end = @extern(*u8, .{ .name = "__stop_set_" ++ name });
 
-        const Self = @This();
-
         pub fn count() usize {
             return (@intFromPtr(end) - @intFromPtr(start)) / @sizeOf(T);
         }
@@ -41,7 +41,8 @@ pub fn LinkerSet(name: []const u8, comptime T: type) type {
                     @export(
                         @as(*const usize, @ptrCast(&storage)),
                         .{
-                            .name = "_set_elem_" ++ name ++ "_" ++ @typeName(Tag(elem)),
+                            .name = "_set_elem_" ++ name ++ "_" ++
+                                @typeName(Tag(elem)),
                             .section = "set_" ++ name,
                         },
                     );

src/rtl/pairing_heap.zig

@@ -1,5 +1,6 @@
 //! Intrusive Pairing Heap implementation.
 const std = @import("std");
+const assert = std.debug.assert;
 
 pub const Node = struct {
     child: ?*Node = null,
@@ -7,9 +8,12 @@ pub const Node = struct {
     prev: ?*Node = null,
 };
 
-pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, *Node) std.math.Order) type {
+pub fn PairingHeap(
+    comptime order: enum { Min, Max },
+    comptime cmp: fn (*Node, *Node) std.math.Order,
+) type {
     return struct {
-        const Self = @This();
+        const Heap = @This();
 
         /// Number of elements in the heap
         size: usize = 0,
@@ -19,8 +23,8 @@ pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, *
 
         fn meld(a: *Node, b: *Node) *Node {
             const a_wins = switch (order) {
-                .min => cmp(a, b) != .gt, // a <= b
-                .max => cmp(a, b) == .gt, // a > b
+                .Min => cmp(a, b) != .gt, // a <= b
+                .Max => cmp(a, b) == .gt, // a > b
             };
 
             if (a_wins) {
@@ -70,7 +74,7 @@ pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, *
             }
 
             // Second pass: fold pairs right to left
-            var result = pairs.?;
+            var result = pairs orelse unreachable;
             var rest = result.next;
             result.next = null;
             while (rest) |p| {
@@ -81,7 +85,7 @@ pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, *
             return result;
         }
 
-        pub fn init() Self {
+        pub fn init() Heap {
             return .{
                 .size = 0,
                 .root = null,
@@ -89,7 +93,7 @@ pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, *
         }
 
         /// Insert `node` into the heap.
-        pub fn insert(self: *Self, node: *Node) void {
+        pub fn insert(self: *Heap, node: *Node) void {
             node.* = .{
                 .child = null,
                 .next = null,
@@ -107,7 +111,7 @@ pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, *
         }
 
         /// Return and remove the topmost element.
-        pub fn pop(self: *Self) ?*Node {
+        pub fn pop(self: *Heap) ?*Node {
             const root = self.root orelse return null;
             const child = root.child;
 
@@ -125,13 +129,13 @@ pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, *
         }
 
         /// Remove `node` from the heap.
-        pub fn remove(self: *Self, node: *Node) void {
+        pub fn remove(self: *Heap, node: *Node) void {
             if (self.root == node) {
                 _ = self.pop();
                 return;
             }
 
-            std.debug.assert(self.root != null);
+            assert(self.root != null);
 
             self.size -= 1;
 
@@ -174,8 +178,8 @@ fn my_cmp(a: *Node, b: *Node) std.math.Order {
     return std.math.order(ta.value, tb.value);
 }
 
-const MinHeap = PairingHeap(.min, my_cmp);
-const MaxHeap = PairingHeap(.max, my_cmp);
+const MinHeap = PairingHeap(.Min, my_cmp);
+const MaxHeap = PairingHeap(.Max, my_cmp);
 
 test "empty heap" {
     var heap = MinHeap.init();

src/rtl/rbtree.zig

@@ -1,5 +1,6 @@
 //! Red-Black Tree implementation mostly based on pseudocode from CLRS.
-//! Refer to the book for details on the algorithms and properties of red-black trees.
+//! Refer to the book for details on the algorithms and properties of red-black
+//! trees.
 const bst = @import("bst.zig");
 const std = @import("std");
 
@@ -7,22 +8,24 @@ const black: u2 = 0;
 const red: u2 = 1;
 
 fn get_color(node: *bst.Node) u2 {
-    return node.parent.get_tag();
+    return node.parent.tag();
 }
 
 fn set_color(node: *bst.Node, color: u2) void {
     node.parent.set_tag(color);
 }
 
-pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order) type {
+pub fn RBTree(
+    comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order,
+) type {
     return struct {
         tree: bst.BST(cmp),
 
-        const Self = @This();
+        const Tree = @This();
 
-        fn rotate_right(self: *Self, node: *bst.Node) void {
+        fn rotate_right(self: *Tree, node: *bst.Node) void {
             var y = node.left;
-            const parent = node.parent.get_ptr();
+            const parent = node.parent.ptr();
 
             node.left = y.right;
 
@@ -44,9 +47,9 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
             node.parent.set_ptr(y);
         }
 
-        fn rotate_left(self: *Self, node: *bst.Node) void {
+        fn rotate_left(self: *Tree, node: *bst.Node) void {
             var y = node.right;
-            const parent = node.parent.get_ptr();
+            const parent = node.parent.ptr();
 
             node.right = y.left;
 
@@ -68,16 +71,19 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
             node.parent.set_ptr(y);
         }
 
-        pub fn init() Self {
-            return Self{
+        pub fn init() Tree {
+            return Tree{
                 .tree = bst.BST(cmp).init(),
             };
         }
 
         /// Insert the node `elem` into the tree.
-        pub fn insert(self: *Self, elem: *bst.Node) !void {
+        pub fn insert(self: *Tree, elem: *bst.Node) !void {
             if (self.tree.is_empty()) {
-                self.tree.insert(elem) catch unreachable;
+                self.tree.insert(elem) catch {
+                    // This can't happen as the tree is empty.
+                    unreachable;
+                };
                 set_color(elem, black);
                 return;
             }
@@ -90,11 +96,11 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
             set_color(elem, red);
 
             var node = elem;
-            var parent = node.parent.get_ptr();
+            var parent = node.parent.ptr();
 
             // The tree might be unbalanced, rebalance it.
             while (get_color(parent) == red) {
-                var grandparent = parent.parent.get_ptr();
+                var grandparent = parent.parent.ptr();
 
                 // The parent of the node is a left child
                 if (parent == grandparent.left) {
@@ -108,19 +114,19 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
                         set_color(uncle, black);
                         set_color(grandparent, red);
                         node = grandparent;
-                        parent = node.parent.get_ptr();
+                        parent = node.parent.ptr();
                     } else {
                         // Otherwise, if the node is a right child, rotate left
                         // to make it a left child.
                         if (node == parent.right) {
                             node = parent;
                             self.rotate_left(node);
-                            parent = node.parent.get_ptr();
-                            grandparent = parent.parent.get_ptr();
+                            parent = node.parent.ptr();
+                            grandparent = parent.parent.ptr();
                         }
 
-                        // Then, recolor the parent and grandparent and rotate right
-                        // to maintain the properties of the red-black tree.
+                        // Then, recolor the parent and grandparent and rotate
+                        // right to maintain the invariants.
                         set_color(parent, black);
                         set_color(grandparent, red);
                         self.rotate_right(grandparent);
@@ -134,13 +140,13 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
                         set_color(uncle, black);
                         set_color(grandparent, red);
                         node = grandparent;
-                        parent = node.parent.get_ptr();
+                        parent = node.parent.ptr();
                     } else {
                         if (node == parent.left) {
                             node = parent;
                             self.rotate_right(node);
-                            parent = node.parent.get_ptr();
-                            grandparent = parent.parent.get_ptr();
+                            parent = node.parent.ptr();
+                            grandparent = parent.parent.ptr();
                         }
 
                         set_color(parent, black);
@@ -148,7 +154,7 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
                         self.rotate_left(grandparent);
                     }
 
-                    parent = node.parent.get_ptr();
+                    parent = node.parent.ptr();
                 }
             }
 
@@ -156,7 +162,7 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
         }
 
         /// Delete the node `elem` from the tree.
-        pub fn delete(self: *Self, elem: *bst.Node) void {
+        pub fn delete(self: *Tree, elem: *bst.Node) void {
             var node = elem;
             var child = &bst.nil;
             var orig_color = get_color(node);
@@ -175,7 +181,7 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
                 orig_color = get_color(node);
                 child = node.right;
 
-                if (node.parent.get_ptr() == elem) {
+                if (node.parent.ptr() == elem) {
                     child.parent.set_ptr(node);
                 } else {
                     self.tree.transplant(node, node.right);
@@ -198,7 +204,7 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
 
             // Fix the tree
             while (node != self.tree.root and get_color(node) == black) {
-                var parent = node.parent.get_ptr();
+                var parent = node.parent.ptr();
 
                 if (node == parent.left) {
                     var sibling = parent.right;
@@ -208,11 +214,13 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
                         set_color(parent, red);
                         self.rotate_left(parent);
 
-                        parent = node.parent.get_ptr();
+                        parent = node.parent.ptr();
                         sibling = parent.right;
                     }
 
-                    if (get_color(sibling.left) == black and get_color(sibling.right) == black) {
+                    if (get_color(sibling.left) == black and
+                        get_color(sibling.right) == black)
+                    {
                         set_color(sibling, red);
                         node = parent;
                     } else {
@@ -220,7 +228,7 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
                             set_color(sibling.left, black);
                             set_color(sibling, red);
                             self.rotate_right(sibling);
-                            parent = node.parent.get_ptr();
+                            parent = node.parent.ptr();
                             sibling = parent.right;
                         }
 
@@ -238,11 +246,13 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
                         set_color(parent, red);
                         self.rotate_right(parent);
 
-                        parent = node.parent.get_ptr();
+                        parent = node.parent.ptr();
                         sibling = parent.left;
                     }
 
-                    if (get_color(sibling.left) == black and get_color(sibling.right) == black) {
+                    if (get_color(sibling.left) == black and
+                        get_color(sibling.right) == black)
+                    {
                         set_color(sibling, red);
                         node = parent;
                     } else {
@@ -250,7 +260,7 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order
                             set_color(sibling.right, black);
                             set_color(sibling, red);
                             self.rotate_left(sibling);
-                            parent = node.parent.get_ptr();
+                            parent = node.parent.ptr();
                             sibling = parent.left;
                         }
 

src/rtl/root.zig

@@ -22,6 +22,45 @@ pub fn comptime_error(comptime msg: []const u8, args: anytype) void {
     @compileError(std.fmt.comptimePrint(msg, args));
 }
 
+const InterfaceError = enum {
+    MissingField,
+    FieldType,
+    MissingMethod,
+    ParameterCount,
+    ReturnType,
+    ParameterType,
+    MissingType,
+    TypeDeclaration,
+    MissingVariable,
+    VariableType,
+};
+
+fn interface_error(comptime err: InterfaceError, args: anytype) void {
+    const msg = switch (err) {
+        .MissingField => "Expected field '{s}' of type '{s}' in type '{s}'" ++
+            "required by '{s}'",
+        .FieldType => "Expected field '{s}' of type '{s}' in type '{s}'" ++
+            "required by '{s}', got '{s}'",
+        .MissingMethod => "Expected method '{s}' in type '{s}' " ++
+            "required by '{s}'",
+        .ParameterCount => "Parameter count mismatch in method '{s}' " ++
+            "in type '{s}'" ++ "required by '{s}'",
+        .ReturnType => "Return type mismatch in method '{s}' in type '{s}'" ++
+            "required by '{s}'",
+        .ParameterType => "Parameter type mismatch in method '{s}' in " ++
+            "type '{s}' required by '{s}'",
+        .MissingType => "Expected type declaration '{s}' in type '{s}'" ++
+            "required by '{s}'",
+        .TypeDeclaration => "Declaration '{s}' in type '{s}' " ++
+            "is required to be a" ++ "type by '{s}', but got '{s}'",
+        .MissingVariable => "Expected variable declaration '{s}' " ++
+            "in type '{s}'" ++ "required by '{s}'",
+        .VariableType => "Declaration '{s}' in type '{s}' is required to be" ++
+            "a variable of type '{s}' by '{s}', but got '{s}'",
+    };
+    comptime_error(msg, args);
+}
+
 /// Asserts that a given type `T` matches the schema declared by `I`.
 /// This includes public methods and fields.
 pub fn assert_interface(T: type, I: type) void {
@@ -31,19 +70,14 @@ pub fn assert_interface(T: type, I: type) void {
 
     inline for (info.field_names, info.field_types) |name, t| {
         if (!@hasField(T, name)) {
-            comptime_error(
-                "Expected field '{s}' of type '{s}' in type '{s}' required by '{s}'",
-                .{
-                    name,
-                    @typeName(t),
-                    @typeName(T),
-                    @typeName(I),
-                },
+            interface_error(
+                .MissingField,
+                .{ name, @typeName(t), @typeName(T), @typeName(I) },
             );
         } else {
             if (@FieldType(T, name) != t) {
-                comptime_error(
-                    "Expected field '{s}' of type '{s}' in type '{s}' required by '{s}', got '{s}'",
+                interface_error(
+                    .FieldType,
                     .{
                         name,
                         @typeName(t),
@@ -61,8 +95,8 @@ pub fn assert_interface(T: type, I: type) void {
 
         if (@typeInfo(@TypeOf(member)) == .@"fn") {
             if (!@hasDecl(T, decl)) {
-                comptime_error(
-                    "Expected method '{s}' in type '{s}' required by '{s}'",
+                interface_error(
+                    .MissingMethod,
                     .{
                         decl,
                         @typeName(T),
@@ -80,55 +114,39 @@ pub fn assert_interface(T: type, I: type) void {
                 const impl_fn = @typeInfo(ImplFnType).@"fn";
 
                 if (iface_fn.param_types.len != impl_fn.param_types.len) {
-                    comptime_error(
-                        "Parameter count mismatch in method '{s}' in type '{s}' required by '{s}'",
-                        .{
-                            decl,
-                            @typeName(T),
-                            @typeName(I),
-                        },
+                    interface_error(
+                        .ParameterCount,
+                        .{ decl, @typeName(T), @typeName(I) },
                     );
                 }
                 if (iface_fn.return_type != impl_fn.return_type) {
-                    comptime_error(
-                        "Return type mismatch in method '{s}' in type '{s}' required by '{s}'",
-                        .{
-                            decl,
-                            @typeName(T),
-                            @typeName(I),
-                        },
+                    interface_error(
+                        .ReturnType,
+                        .{ decl, @typeName(T), @typeName(I) },
                     );
                 }
 
                 for (iface_fn.param_types, impl_fn.param_types) |a, b| {
                     if (a != b) {
-                        comptime_error(
-                            "Parameter type mismatch in method '{s}' in type '{s}' required by '{s}'",
-                            .{
-                                decl,
-                                @typeName(T),
-                                @typeName(I),
-                            },
+                        interface_error(
+                            .ParameterType,
+                            .{ decl, @typeName(T), @typeName(I) },
                         );
                     }
                 }
             }
         } else if (@TypeOf(member) == type) {
             if (!@hasDecl(T, decl)) {
-                comptime_error(
-                    "Expected type declaration '{s}' in type '{s}' required by '{s}'",
-                    .{
-                        decl,
-                        @typeName(T),
-                        @typeName(I),
-                    },
+                interface_error(
+                    .MissingType,
+                    .{ decl, @typeName(T), @typeName(I) },
                 );
             }
 
             const impl_member = @field(T, decl);
             if (@TypeOf(impl_member) != type) {
-                comptime_error(
-                    "Declaration '{s}' in type '{s}' is required to be a type by '{s}', but got '{s}'",
+                interface_error(
+                    .TypeDeclaration,
                     .{
                         decl,
                         @typeName(T),
@@ -139,20 +157,16 @@ pub fn assert_interface(T: type, I: type) void {
             }
         } else {
             if (!@hasDecl(T, decl)) {
-                comptime_error(
-                    "Expected variable declaration '{s}' in type '{s}' required by '{s}'",
-                    .{
-                        decl,
-                        @typeName(T),
-                        @typeName(I),
-                    },
+                interface_error(
+                    .MissingVariable,
+                    .{ decl, @typeName(T), @typeName(I) },
                 );
             }
 
             const impl_member = @field(T, decl);
             if (@TypeOf(impl_member) != @TypeOf(member)) {
-                comptime_error(
-                    "Declaration '{s}' in type '{s}' is required to be a variable of type '{s}' by '{s}', but got '{s}'",
+                interface_error(
+                    .VariableType,
                     .{
                         decl,
                         @typeName(T),

src/rtl/seqlock.zig

@@ -1,18 +1,20 @@
+//! Sequence locks.
+
 const std = @import("std");
 const rtl = @import("rtl");
 
 pub fn SeqLock(comptime T: type) type {
     return struct {
-        const Self = @This();
+        const Lock = @This();
 
         raw: T,
         sequence: std.atomic.Value(usize),
 
-        pub fn init(val: T) Self {
-            return .{ .raw = val, .sequence = .init(0) };
+        pub fn init(val: T) Lock {
+            return .{ .raw = val, .sequence = std.atomic.Value(usize).init(0) };
         }
 
-        pub fn load(self: *Self) T {
+        pub fn load(self: *Lock) T {
             var ret: T = undefined;
 
             while (true) {
@@ -25,7 +27,8 @@ pub fn SeqLock(comptime T: type) type {
                 // Relaxed load memcpy is needed here since data races are UB.
                 rtl.barrier.atomic_load_memcpy(&ret, &self.raw, .monotonic);
 
-                // Ensure the sequence read happens *after* the data is fully loaded.
+                // Ensure the sequence read happens *after* the data is fully
+                // loaded.
                 rtl.barrier.fence(.acquire);
 
                 if (self.sequence.load(.monotonic) == seq)
@@ -37,12 +40,13 @@ pub fn SeqLock(comptime T: type) type {
             return ret;
         }
 
-        pub fn store(self: *Self, data: T) void {
+        pub fn store(self: *Lock, data: T) void {
             const seq = self.sequence.load(.monotonic);
 
             self.sequence.store(seq + 1, .monotonic);
 
-            // Ensure the data write happens *after* the sequence is incremented.
+            // Ensure the data write happens *after* the sequence
+            // is incremented.
             rtl.barrier.fence(.release);
 
             // Relaxed store memcpy is needed here since data races are UB.

src/rtl/tagged_ptr.zig

@@ -4,32 +4,35 @@ pub fn TaggedPtr(comptime T: type) type {
     return struct {
         value: usize,
 
-        const Self = @This();
+        const Ptr = @This();
 
         const mask: usize = 0x3;
 
-        pub fn init(ptr: *T, tag: u2) Self {
-            return Self{ .value = (@intFromPtr(ptr) & ~mask) | (@as(usize, tag) & mask) };
+        pub fn init(ptr_val: *T, tag_val: u2) Ptr {
+            return Ptr{
+                .value = (@intFromPtr(ptr_val) & ~mask) |
+                    (@as(usize, tag_val) & mask),
+            };
         }
 
         /// Return the pointer with the tag bits masked out.
-        pub fn get_ptr(self: *const Self) *T {
+        pub fn ptr(self: *const Ptr) *T {
             return @ptrFromInt(self.value & ~mask);
         }
 
         /// Set the tag bits to `tag`, while preserving the pointer.
-        pub fn set_tag(self: *Self, tag: u2) void {
-            self.value = (self.value & ~mask) | (@as(usize, tag) & mask);
+        pub fn set_tag(self: *Ptr, val: u2) void {
+            self.value = (self.value & ~mask) | (@as(usize, val) & mask);
         }
 
         /// Return the tag bits.
-        pub fn get_tag(self: *const Self) u2 {
+        pub fn tag(self: *const Ptr) u2 {
             return @as(u2, @truncate(self.value & mask));
         }
 
         /// Set the pointer to `ptr`, while preserving the tag bits.
-        pub fn set_ptr(self: *Self, ptr: *T) void {
-            self.value = (@intFromPtr(ptr) & ~mask) | (self.value & mask);
+        pub fn set_ptr(self: *Ptr, val: *T) void {
+            self.value = (@intFromPtr(val) & ~mask) | (self.value & mask);
         }
     };
 }
@@ -40,13 +43,13 @@ test TaggedPtr {
     var x: u32 = 0;
     var tagged = TaggedPtr(u32).init(&x, 2);
 
-    try std.testing.expectEqual(&x, tagged.get_ptr());
-    try std.testing.expectEqual(2, tagged.get_tag());
+    try std.testing.expectEqual(&x, tagged.ptr());
+    try std.testing.expectEqual(2, tagged.tag());
 
     tagged.set_tag(1);
-    try std.testing.expectEqual(1, tagged.get_tag());
+    try std.testing.expectEqual(1, tagged.tag());
 
     var y: u32 = 0;
     tagged.set_ptr(&y);
-    try std.testing.expectEqual(&y, tagged.get_ptr());
+    try std.testing.expectEqual(&y, tagged.ptr());
 }

src/rtl/units.zig

@@ -3,7 +3,11 @@ const std = @import("std");
 
 const Dimension = enum { Time };
 
-fn scale_value(comptime from_scale: u64, comptime to_scale: u64, value: u64) u64 {
+fn scale_value(
+    comptime from_scale: u64,
+    comptime to_scale: u64,
+    value: u64,
+) u64 {
     if (comptime from_scale >= to_scale) {
         return value * comptime (from_scale / to_scale);
     } else {
@@ -17,7 +21,7 @@ pub fn Unit(
     comptime unit_suffix: []const u8,
 ) type {
     return struct {
-        const Self = @This();
+        const UnitType = @This();
         pub const dimension = dim;
         pub const base_per_unit = scale;
 
@@ -28,27 +32,30 @@ pub fn Unit(
                 if (!(@typeInfo(T) == .@"struct" and @hasDecl(T, "dimension")))
                     @compileError(@typeName(T) ++ " is not a unit");
                 if (T.dimension != dim)
-                    @compileError("cannot convert between " ++ @tagName(T.dimension) ++
+                    @compileError("cannot convert between " ++
+                        @tagName(T.dimension) ++
                         " and " ++ @tagName(dim));
             }
         }
 
-        pub fn init(value: u64) Self {
+        pub fn init(value: u64) UnitType {
             return .{ .value = value };
         }
 
-        pub fn to(self: Self, comptime T: type) T {
+        pub fn to(self: UnitType, comptime T: type) T {
             check(T);
             return T.init(scale_value(scale, T.base_per_unit, self.value));
         }
 
-        pub fn from(other: anytype) Self {
+        pub fn from(other: anytype) UnitType {
             const From = @TypeOf(other);
             check(From);
-            return Self.init(scale_value(From.base_per_unit, scale, other.value));
+            return UnitType.init(
+                scale_value(From.base_per_unit, scale, other.value),
+            );
         }
 
-        pub inline fn format(self: Self, writer: *std.Io.Writer) !void {
+        pub inline fn format(self: UnitType, writer: *std.Io.Writer) !void {
             try writer.print("{} {s}", .{ self.value, unit_suffix });
         }
     };

src/tidy.zig

@@ -117,17 +117,6 @@ const Errors = struct {
         );
     }
 
-    pub fn add_orelse_unreachable(
-        errors: *Errors,
-        file: SourceFile,
-        line_num: usize,
-    ) void {
-        errors.emit(
-            "{s}:{d}: error: use 'orelse unreachable' instead of .?\n",
-            .{ file.path, line_num },
-        );
-    }
-
     pub fn add_function_too_long(
         errors: *Errors,
         name: []const u8,
@@ -226,6 +215,12 @@ const SourceFile = struct {
 
 fn tidy_file(gpa: Allocator, file: SourceFile, errors: *Errors) !void {
     if (!file.has_extension(".zig")) return;
+    if (std.mem.eql(u8, file.path, "build.zig")) return;
+    if (std.mem.eql(u8, file.path, "build/config.zig")) return;
+    if (std.mem.eql(u8, file.path, "build/image.zig")) return;
+    if (std.mem.eql(u8, file.path, "build/run.zig")) return;
+    if (std.mem.eql(u8, file.path, "build/ksyms.zig")) return;
+    if (std.mem.eql(u8, file.path, "src/tidy.zig")) return;
 
     // Tidy lines.
     var line_iterator = std.mem.splitScalar(u8, file.text, '\n');
@@ -264,7 +259,6 @@ fn tidy_line(
 
 fn tidy_ast(file: SourceFile, tree: std.zig.Ast, errors: *Errors) void {
     tidy_defer(file, tree, errors);
-    tidy_optional_unwrap(file, tree, errors);
 
     for (tree.rootDecls()) |decl| {
         if (tree.nodeTag(decl) != .fn_decl) continue;
@@ -358,27 +352,14 @@ fn function_has_reference(
     return false;
 }
 
-fn tidy_optional_unwrap(file: SourceFile, tree: std.zig.Ast, errors: *Errors) void {
-    for (0..tree.nodes.len) |index| {
-        const node: std.zig.Ast.Node.Index = @enumFromInt(index);
-        if (tree.nodeTag(node) != .@"orelse") continue;
-
-        const rhs = tree.nodeData(node).node_and_node[1];
-        if (tree.nodeTag(rhs) != .unreachable_literal) continue;
-
-        const line_num = tree.tokenLocation(0, tree.nodeMainToken(node)).line +
-            1;
-        errors.add_orelse_unreachable(file, line_num);
-    }
-}
-
 fn tidy_banned(file: SourceFile, errors: *Errors) void {
     const ban_list: []const struct { []const u8, []const u8 } = &.{
         .{ "debug.assert(", "unqualified assert" },
         .{ "Self = @This()", "proper type name" },
         .{ "catch unreachable", "proper error handling or documentation" },
         .{ "catch {}", "proper error handling or documentation" },
-        .{ "= .init", "full type name" },
+        .{ "= .init(", "full type name" },
+        .{ ".?", "orelse unreachable" },
     };
 
     for (ban_list) |ban_item| {
@@ -392,7 +373,7 @@ fn tidy_banned(file: SourceFile, errors: *Errors) void {
 
 fn tidy_defer(file: SourceFile, tree: std.zig.Ast, errors: *Errors) void {
     for (0..tree.nodes.len) |index| {
-        const node: std.zig.Ast.Node.Index = @enumFromInt(index);
+        const node: std.zig.Ast.Node.Index = @fromBackingInt(@intCast(index));
         if (tree.nodeTag(node) != .@"defer" and
             tree.nodeTag(node) != .@"errdefer") continue;