diff --git a/build.zig.zon b/build.zig.zon index d3fbdce..cef3b3d 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -13,8 +13,8 @@ .hash = "N-V-__8AAAAAjAwsopZIlbp4U6NN1Eh573uVdLTwl1mkFUug", }, .zonfig = .{ - .url = "git+https://github.com/rdmsr/zonfig?ref=HEAD#7cb4e030077acbef88458aef2d3fec4331b3b6da", - .hash = "zonfig-0.1.0-V0D8-aRMFAAQ3Yb-0u8JNpdceK6lUOWUP7EY5BRXz12C", + .url = "git+https://github.com/rdmsr/zonfig?ref=HEAD#d02cbb9262c5e617676a252116c9cee3560b6e59", + .hash = "zonfig-0.1.0-V0D8-YtLFAAKh8PQgGaoCiGG1WUxWdqcY-iTysVPllQV", }, }, diff --git a/build/config.zig b/build/config.zig index 8759bf4..da6397e 100644 --- a/build/config.zig +++ b/build/config.zig @@ -28,8 +28,24 @@ pub const Config = struct { }; pub fn parseConfig(b: *std.Build) !Config { + const gpa = b.allocator; + var arena_allocator: std.heap.ArenaAllocator = .init(gpa); + defer arena_allocator.deinit(); + const arena = arena_allocator.allocator(); + + var diag: std.zon.parse.Diagnostics = undefined; + const file = try std.Io.Dir.cwd().readFileAllocOptions(b.graph.io, ".config.zig.zon", b.allocator, std.Io.Limit.unlimited, .@"1", 0); - return std.zon.parse.fromSlice(Config, b.allocator, file, null, .{ .ignore_unknown_fields = true }); + return std.zon.parse.fromSlice( + Config, + .{ + .gpa = gpa, + .source = file, + .arena = arena, + .diagnostics = &diag, + .ignore_unknown_fields = true, + }, + ); } pub fn getPlatform(config: Config) !Platform { diff --git a/build/limine.conf b/build/limine.conf index 8385c74..56fb951 100644 --- a/build/limine.conf +++ b/build/limine.conf @@ -11,4 +11,4 @@ timeout: 3 module_path: boot():/boot/kernel module_string: kernel - + cmdline: test=fireworks diff --git a/build/run.zig b/build/run.zig index 7f3e6f6..7b326ff 100644 --- a/build/run.zig +++ b/build/run.zig @@ -58,7 +58,7 @@ pub fn addRun(b: *std.Build, kernel: *std.Build.Step.Compile, loader: *std.Build switch (plat.arch) { .x86_64 => { - qemu.addArgs(&.{ "-serial", "stdio", "-m", "1G", "-smp", "4" }); + qemu.addArgs(&.{ "-serial", "stdio", "-m", "1G", "-smp", "1" }); if (!debug and builtin.target.os.tag == .linux and builtin.cpu.arch == .x86_64) qemu.addArgs(&.{ "-enable-kvm", "-cpu", "host,+invtsc" }); }, diff --git a/src/arch/amd64/paravirt/kvm.zig b/src/arch/amd64/paravirt/kvm.zig index 30735f0..5c51300 100644 --- a/src/arch/amd64/paravirt/kvm.zig +++ b/src/arch/amd64/paravirt/kvm.zig @@ -38,10 +38,6 @@ const Msr = enum(u32) { _, }; -fn read_msr(msr: Msr) u64 { - return amd64.rdmsr(@intFromEnum(msr)); -} - fn write_msr(msr: Msr, value: u64) void { amd64.wrmsr(@intFromEnum(msr), value); } diff --git a/src/arch/amd64/root.zig b/src/arch/amd64/root.zig index 34bee02..ae36435 100644 --- a/src/arch/amd64/root.zig +++ b/src/arch/amd64/root.zig @@ -1,5 +1,6 @@ //! Wrappers over amd64 CPU definitions and CPUID. const std = @import("std"); +const assert = std.debug.assert; pub const hypervisor = @import("hypervisor.zig"); @@ -286,14 +287,14 @@ pub const Cr4 = packed struct(u64) { }; pub inline fn read_cr(comptime n: u8) u64 { - comptime std.debug.assert(n != 1); + comptime assert(n != 1); return asm volatile (std.fmt.comptimePrint("mov %%cr{}, %[ret]", .{n}) : [ret] "=r" (-> u64), ); } pub inline fn write_cr(comptime n: u8, value: u64) void { - comptime std.debug.assert(n != 1); + comptime assert(n != 1); asm volatile (std.fmt.comptimePrint("mov %[value], %%cr{}", .{n}) : : [value] "r" (value), @@ -519,23 +520,23 @@ const known_vendors = [_]struct { }; pub const CpuFeatures = struct { - x2apic: bool, - five_level_paging: bool, - gib_pages: bool, - tsc_deadline: bool, - fxsave: bool, - xsave: bool, - invariant_tsc: bool, - nx: bool, - pcid: bool, - smap: bool, - smep: bool, - pge: bool, - syscall: bool, - umip: bool, - vendor: CpuVendor, - family: u8, - brand_string: [48]u8, + x2apic: bool = false, + five_level_paging: bool = false, + gib_pages: bool = false, + tsc_deadline: bool = false, + fxsave: bool = false, + xsave: bool = false, + invariant_tsc: bool = false, + nx: bool = false, + pcid: bool = false, + smap: bool = false, + smep: bool = false, + pge: bool = false, + syscall: bool = false, + umip: bool = false, + vendor: CpuVendor = undefined, + family: u8 = 0, + brand_string: [48]u8 = undefined, }; pub const HypervisorVendor = enum(u32) { @@ -563,71 +564,59 @@ fn detect_vendor(string: [12]u8) CpuVendor { pub fn detect_cpu_features() void { const vendor_info = CpuidRequest.execute(.VendorInfo); const max_ext = CpuidRequest.execute(.HighestExtendedFunction).eax; - const max_basic = vendor_info.eax; - var x2apic = false; - var five_level_paging = false; - var gib_pages = false; - var tsc_deadline = false; - var xsave = false; - var fxsave = false; - var invtsc = false; - var nx = false; - var pcid = false; - var smap = false; - var smep = false; - var pge = false; var hypervisor_flag = false; - var syscall = false; - var family: u8 = 0; - var umip = false; - var vendor_string: [12]u8 = undefined; - var brand_string: [48]u8 = undefined; + + cpu_features = .{}; std.mem.writeInt(u32, vendor_string[0..4], vendor_info.ebx, .little); std.mem.writeInt(u32, vendor_string[4..8], vendor_info.edx, .little); std.mem.writeInt(u32, vendor_string[8..12], vendor_info.ecx, .little); - const vendor = detect_vendor(vendor_string); + cpu_features.vendor = detect_vendor(vendor_string); if (max_basic >= 0x1) { const r = CpuidRequest.execute(.FeatureInfo); const ecx: FeatureInfoEcx = @bitCast(r.ecx); const edx: FeatureInfoEdx = @bitCast(r.edx); - x2apic = ecx.x2apic; - tsc_deadline = ecx.tsc_deadline; - xsave = ecx.xsave; - pcid = ecx.pcid; - fxsave = edx.fxsave; - pge = edx.pge; + cpu_features.x2apic = ecx.x2apic; + cpu_features.tsc_deadline = ecx.tsc_deadline; + cpu_features.xsave = ecx.xsave; + cpu_features.pcid = ecx.pcid; + cpu_features.fxsave = edx.fxsave; + cpu_features.pge = edx.pge; + hypervisor_flag = ecx.hypervisor; const sig = r.eax; - family = @truncate((sig >> 8) & 0xf); + var family: u8 = @truncate((sig >> 8) & 0xf); + if (family == 0xf) { family +%= @truncate((sig >> 20) & 0xff); } + + cpu_features.family = family; } if (max_basic >= 0x7) { const r = CpuidRequest.execute(.{ .ExtendedFeatures = .First }); const ecx: ExtendedFeaturesEcx = @bitCast(r.ecx); const ebx: ExtendedFeaturesEbx = @bitCast(r.ebx); - smap = ebx.smap; - smep = ebx.smep; - five_level_paging = ecx.la57; - umip = ecx.umip; + cpu_features.smap = ebx.smap; + cpu_features.smep = ebx.smep; + cpu_features.five_level_paging = ecx.la57; + cpu_features.umip = ecx.umip; } if (max_ext >= 0x80000001) { const r = CpuidRequest.execute(.ExtendedInfo); const edx: ExtendedProcessorInfoEdx = @bitCast(r.edx); - gib_pages = edx.pdpe1gb; - nx = edx.nx; - syscall = edx.syscall_sysret; + cpu_features.gib_pages = edx.pdpe1gb; + cpu_features.nx = edx.nx; + cpu_features.syscall = edx.syscall_sysret; } if (max_ext >= 0x80000004) { @@ -638,6 +627,8 @@ pub fn detect_cpu_features() void { }; var off: usize = 0; + const brand_string = &cpu_features.brand_string; + for (parts) |p| { for ([_]u32{ p.eax, p.ebx, p.ecx, p.edx }) |reg| { const dst: *[4]u8 = @ptrCast(brand_string[off..].ptr); @@ -650,32 +641,12 @@ pub fn detect_cpu_features() void { if (max_ext >= 0x80000007) { const r = CpuidRequest.execute(.PowerManagementInfo); const edx: PowerManagementInfoEdx = @bitCast(r.edx); - invtsc = edx.invtsc; + cpu_features.invariant_tsc = edx.invtsc; } if (hypervisor_flag) { hypervisor.detect(); } - - cpu_features = .{ - .x2apic = x2apic, - .five_level_paging = five_level_paging, - .gib_pages = gib_pages, - .tsc_deadline = tsc_deadline, - .fxsave = fxsave, - .xsave = xsave, - .invariant_tsc = invtsc, - .nx = nx, - .pcid = pcid, - .smap = smap, - .smep = smep, - .vendor = vendor, - .brand_string = brand_string, - .pge = pge, - .family = family, - .umip = umip, - .syscall = syscall, - }; } pub const TssDescriptor = extern struct { diff --git a/src/arch/riscv64/root.zig b/src/arch/riscv64/root.zig index a1021cf..afe3fc6 100644 --- a/src/arch/riscv64/root.zig +++ b/src/arch/riscv64/root.zig @@ -1,3 +1,5 @@ +//! Wrappers over riscv64 CPU definitions. + pub const name: []const u8 = "riscv64"; pub const BootInfo = struct {}; diff --git a/src/bootvid/bootvid.zig b/src/bootvid/bootvid.zig index f143eec..71fc8f6 100644 --- a/src/bootvid/bootvid.zig +++ b/src/bootvid/bootvid.zig @@ -39,7 +39,7 @@ const Cursor = struct { const font_file = @embedFile("sun12x22-ascii.psf"); const text_color: u32 = 0xFFFFFFFF; -const bg_color: u32 = 0xFF000000; +const bg_color: u32 = 0xFF574c8f; const crash_bg_color: u32 = 0xFF0000FF; const window_padding: usize = 10; @@ -93,8 +93,8 @@ fn load_font() void { fn plot_character(x: usize, y: usize, c: u8) void { const bytes_per_row = font.glyph_size / font.height; - const glyph_offset = font.header_size + (@as(usize, c) * font.glyph_size); - const glyph_data = font_file[glyph_offset..(glyph_offset + font.glyph_size)]; + const offset = font.header_size + (@as(usize, c) * font.glyph_size); + const glyph_data = font_file[offset .. offset + font.glyph_size]; const start_x = window_padding + x; const start_y = window_padding + y; @@ -102,14 +102,19 @@ fn plot_character(x: usize, y: usize, c: u8) void { var row_data: u32 = 0; for (0..bytes_per_row) |b| { - row_data = (row_data << 8) | @as(u32, glyph_data[row * bytes_per_row + b]); + row_data = (row_data << 8) | + @as(u32, glyph_data[row * bytes_per_row + b]); } const shift_base = bytes_per_row * 8; for (0..font.width) |col| { - if ((row_data & (@as(u32, 1) << @intCast(shift_base - 1 - col))) != 0) { - framebuffer[(start_y + row) * framebuffer_width + (start_x + col)] = text_color; + const mask = @as(u32, 1) << @intCast(shift_base - 1 - col); + + if (row_data & mask != 0) { + framebuffer[ + (start_y + row) * framebuffer_width + (start_x + col) + ] = text_color; } } } @@ -134,7 +139,9 @@ fn scroll() void { const dst_y = window_padding + (console_height - 1) * font.height + row; for (0..content_w) |col| { - framebuffer[dst_y * framebuffer_width + window_padding + col] = bg_color; + framebuffer[ + dst_y * framebuffer_width + window_padding + col + ] = bg_color; } } } @@ -187,11 +194,10 @@ pub fn init(boot_info: *r.BootInfo) void { content_h = framebuffer_height - (window_padding * 2); console_width = @intCast(content_w / font.width); - console_height = std.math.divCeil( - u16, - @intCast(content_h), + console_height = @divFloor( + @as(u16, @intCast(content_h)), font.height, - ) catch unreachable; + ); for (0..framebuffer_width * framebuffer_height) |i| { framebuffer[i] = bg_color; diff --git a/src/ex/init.zig b/src/ex/init.zig index 036ae25..b03478b 100644 --- a/src/ex/init.zig +++ b/src/ex/init.zig @@ -1,3 +1,5 @@ +//! Initialization of the executive. + const std = @import("std"); const r = @import("root"); const pl = r.pl; @@ -24,7 +26,11 @@ pub fn init() void { if (rtl.cmdline.get_string(cline, "test")) |tst| { // Run a test if specified. if (r.tests.tests.get(tst)) |func| { - const t = ps.thread.create_kernel(.Default, .{ .func = func, .arg = r.boot_info }, true) catch + const t = ps.thread.create_kernel( + .Default, + .{ .func = func, .arg = r.boot_info }, + true, + ) catch @panic("oom"); ke.sched.enqueue(&t.kern); } else { diff --git a/src/ex/private.zig b/src/ex/private.zig index 8eaf1c5..9ea7387 100644 --- a/src/ex/private.zig +++ b/src/ex/private.zig @@ -1,3 +1,4 @@ +//! Private executive definitions. pub const init_mod = @import("init.zig"); pub const work = @import("work.zig"); pub const init = init_mod.init; diff --git a/src/ex/root.zig b/src/ex/root.zig index 94a838b..cbad5cf 100644 --- a/src/ex/root.zig +++ b/src/ex/root.zig @@ -1,3 +1,5 @@ +//! Public executive definitions. + pub const private = @import("private.zig"); const p = private; diff --git a/src/ex/string.zig b/src/ex/string.zig index 4a3224a..944ad61 100644 --- a/src/ex/string.zig +++ b/src/ex/string.zig @@ -44,18 +44,16 @@ const Context = struct { pub const InternedString = packed struct(usize) { bits: usize, - const Self = @This(); - - fn from_ptr(e: *Entry) Self { + fn from_ptr(e: *Entry) InternedString { return .{ .bits = @intFromPtr(e) }; } - fn as_entry(self: *const Self) ?*Entry { + fn as_entry(self: *const InternedString) ?*Entry { if (self.bits & 1 != 0) return null; return @ptrFromInt(self.bits); } - fn from_inline(s: []const u8) Self { + fn from_inline(s: []const u8) InternedString { var buf: [@sizeOf(usize)]u8 = @splat(0); // byte 0: tag (bit 0) | len (bits 1-7) @@ -71,7 +69,7 @@ pub const InternedString = packed struct(usize) { return @sizeOf(usize) > 4 and len <= 7; } - pub fn slice(self: *const Self) []const u8 { + pub fn slice(self: *const InternedString) []const u8 { if (self.bits & 1 != 0) { const buf: *const [@sizeOf(usize)]u8 = @ptrCast(&self.bits); const len: usize = buf[0] >> 1; @@ -97,7 +95,7 @@ pub fn init() void { table.init(.{ .smr = domain, .policy = .Balanced, - }) catch unreachable; + }) catch @panic("Failed to initialize string interning"); entry_zone.init("str entry", .{ .smr = domain, @@ -114,6 +112,7 @@ pub fn retain(string: []const u8) InternedString { // Fast path where the string already exists. const ipl = ke.smr.enter(domain); defer ke.smr.exit(domain, ipl); + if (table.get(string)) |e| { _ = e.ref.fetchAdd(1, .monotonic); return .from_ptr(e); @@ -121,10 +120,17 @@ pub fn retain(string: []const u8) InternedString { } // The string (probably) doesn't exist, create a new one. - var new_entry: *Entry = entry_zone.create() catch unreachable; + var new_entry: *Entry = entry_zone.create() catch { + // This can't error since the allocation will wait for free memory. + unreachable; + }; - new_entry.string = mm.zone.gpa.dupe(u8, string) catch unreachable; - new_entry.ref = .init(1); + new_entry.string = mm.zone.gpa.dupe(u8, string) catch { + // This can't error since the allocation will wait for free memory. + unreachable; + }; + + new_entry.ref = std.atomic.Value(u32).init(1); const ipl = ke.smr.enter(domain); const entry = table.get_or_insert(string, &new_entry.link); diff --git a/src/ex/vmem.zig b/src/ex/vmem.zig index 0bee8e7..826c1ef 100644 --- a/src/ex/vmem.zig +++ b/src/ex/vmem.zig @@ -1,13 +1,17 @@ //! Generic resource allocator, based on the vmem allocator described in: //! Jeff Bonwick and Jonathan Adams' -//! "Magazines and Vmem: Extending the Slab Allocator to Many CPUs and Arbitrary Resources" -const std = @import("std"); +//! "Magazines and Vmem: Extending the Slab Allocator to Many CPUs and +//! Arbitrary Resources" + const rtl = @import("rtl"); const r = @import("root"); const mm = r.mm; const mmp = mm.private; const ke = r.ke; +const std = @import("std"); +const assert = std.debug.assert; + pub const Policy = enum { /// Use the smallest free segment that can satisfy the request. /// This tends to minimize fragmentation. @@ -37,9 +41,9 @@ const Segment = struct { }; const Linkage = union { - /// Linkage into a freelist, valid when type == .Free + /// Linkage into a freelist, valid when .Free free_link: rtl.List.Entry, - /// Linkage into the allocated segments tree, valid when type == .Allocated + /// Linkage into the allocated segments tree, valid when .Allocated tree_link: rtl.bst.Node, }; /// The type of the segment. @@ -80,6 +84,8 @@ const max_qcaches = 16; var seg_zone: mmp.zone.TypedZone(Segment) = undefined; +const SegmentTree = rtl.RBTree(Segment.cmp); + pub const Arena = struct { /// Name of the arena, for debugging purposes. name: []const u8, @@ -90,7 +96,7 @@ pub const Arena = struct { /// Power of two freelists for free segments, indexed by log2(size). freelists: [freelist_count]rtl.List, /// Tree for allocated segments, indexed by base address. - allocated_segments: rtl.RBTree(Segment.cmp), + allocated_segments: SegmentTree, /// Last segment allocated from, for NextFit policy. rotor: ?*Segment, /// Maximum size (**in bytes**) to do quantum caching on. @@ -105,14 +111,12 @@ pub const Arena = struct { lock: ke.Mutex, - const Self = @This(); - const Span = struct { addr: usize, size: usize }; const Source = struct { - arena: *Self, - import: *const fn (*Self, usize, opts: AllocOptions) mm.Error!usize, - release: *const fn (*Self, addr: usize, size: usize) void, + arena: *Arena, + import: *const fn (*Arena, usize, opts: AllocOptions) mm.Error!usize, + release: *const fn (*Arena, addr: usize, size: usize) void, }; const InitOptions = struct { @@ -124,7 +128,7 @@ pub const Arena = struct { /// Initialize the area for use. pub fn init( - self: *Self, + self: *Arena, name: []const u8, opts: InitOptions, ) !void { @@ -142,7 +146,11 @@ pub const Arena = struct { self.qshift = std.math.log2_int(usize, self.quantum); if (opts.span == null) { - rtl.assert(opts.source != null, "Arena init: null span without source", .{}); + rtl.assert( + opts.source != null, + "Arena init: null span without source", + .{}, + ); } self.list.init(); @@ -170,7 +178,7 @@ pub const Arena = struct { self.lock = ke.Mutex.init("arena"); - self.allocated_segments = .init(); + self.allocated_segments = SegmentTree.init(); self.rotor = null; // Add initial span. @@ -179,7 +187,7 @@ pub const Arena = struct { } } - pub fn deinit(self: *Self) void { + pub fn deinit(self: *Arena) void { // Free all segments in the segment list var entry = self.list.first(); while (entry != &self.list.head) { @@ -191,12 +199,12 @@ pub const Arena = struct { } /// Add a span of memory to the arena. - pub fn add(self: *Self, span: Span) !void { + pub fn add(self: *Arena, span: Span) !void { return self.add_opts(span, .Span); } /// Add a span of memory to the arena. - fn add_opts(self: *Self, span: Span, kind: Segment.Type) mm.Error!void { + fn add_opts(self: *Arena, span: Span, kind: Segment.Type) mm.Error!void { const new_free_seg = try self.alloc_segment(); new_free_seg.* = Segment{ @@ -226,12 +234,15 @@ pub const Arena = struct { /// Allocate a segment of memory from the arena. /// Options: /// - policy: allocation policy to use. Default is .InstantFit. - /// - alignment: if specified, the returned segment will be aligned to this boundary. - /// - min: if specified, the returned segment will be at least on this address. - /// - max: if specified, the returned segment will be at most on this address. + /// - alignment: if specified, the returned segment will be aligned to + /// this boundary. + /// - min: if specified, the returned segment will be at least on this + /// address. + /// - max: if specified, the returned segment will be at most on this + /// address. /// Returns the base address of the allocated segment. pub fn alloc( - self: *Self, + self: *Arena, size: usize, options: AllocOptions, ) mm.Error!usize { @@ -252,7 +263,7 @@ pub const Arena = struct { } fn alloc_impl( - self: *Self, + self: *Arena, size: usize, options: AllocOptions, ) mm.Error!usize { @@ -263,20 +274,20 @@ pub const Arena = struct { .NextFit => self.next_fit(size, options), }; - if (ret != null) break :blk ret.?; + if (ret != null) break :blk ret orelse unreachable; if (ret == null) { try self.import(size, options); } - // This is guaranteed to terminate eventually, either when the import - // brings in new free memory, or when it runs out. + // This is guaranteed to terminate eventually, either when + // the import brings in new free memory, or when it runs out. }; const start, var seg = result; - std.debug.assert(seg.type == .Free); - std.debug.assert(seg.size >= size); + assert(seg.type == .Free); + assert(seg.size >= size); // Remove the segment from the freelist. self.remove_segment_from_freelist(seg); @@ -303,11 +314,13 @@ pub const Arena = struct { self.add_segment_to_freelist(left_seg); } - // right split: seg is larger than needed, so split into allocated + free remainder. + // right split: seg is larger than needed, so split into alloc + free + // remainder. // e.g. seg=[0x100, 0x10000], size=0x1000: // before: [0x100, 0x10000] being processed // after: [0x100, 0x1100] allocated, [0x1100, 0x10000] free - // the quantum check avoids creating a remainder too small to ever be useful. + // the quantum check avoids creating a remainder too small to ever be + // useful. if (seg.size != size and (seg.size - size) > self.quantum - 1) { const new_seg = try self.alloc_segment(); new_seg.* = .{ @@ -345,7 +358,7 @@ pub const Arena = struct { } /// Try to import memory from the parent arena. - fn import(self: *Self, size: usize, options: AllocOptions) mm.Error!void { + fn import(self: *Arena, size: usize, options: AllocOptions) mm.Error!void { const src = self.source orelse return error.OutOfMemory; // Ensure no funny stuff happens wrt the lock. @@ -362,7 +375,7 @@ pub const Arena = struct { ); } - pub fn free(self: *Self, addr: usize, size: usize) void { + pub fn free(self: *Arena, addr: usize, size: usize) void { if (size <= self.qcache_max) { const i = size >> self.qshift; self.qcaches[i - 1].free(@ptrFromInt(addr)); @@ -375,7 +388,7 @@ pub const Arena = struct { return self.free_impl(addr, size); } - fn free_impl(self: *Self, addr: usize, size: usize) void { + fn free_impl(self: *Arena, addr: usize, size: usize) void { // Find the allocated segment containing addr. var search_node = Segment{ @@ -449,7 +462,7 @@ pub const Arena = struct { } /// Try to release a segment back to its source. - fn release(self: *Self, seg: *Segment) bool { + fn release(self: *Arena, seg: *Segment) bool { const prev_entry = seg.link.prev; if (prev_entry == &self.list.head) { @@ -479,7 +492,7 @@ pub const Arena = struct { // Implementation for the instant fit policy. fn instant_fit( - self: *Self, + self: *Arena, size: usize, options: AllocOptions, ) ?struct { usize, *Segment } { @@ -506,7 +519,7 @@ pub const Arena = struct { // something in the lower freelist, so check there. // This is O(n) but should happen very rarely. if (!is_pow2) { - std.debug.assert(orig_idx != 0); + assert(orig_idx != 0); var it = self.freelists[orig_idx].iterator(); while (it.next()) : (it.advance()) { @@ -522,7 +535,7 @@ pub const Arena = struct { // Implementation for the best fit policy. // Find the smallest segment that can satisfy the request. fn best_fit( - self: *Self, + self: *Arena, size: usize, options: AllocOptions, ) ?struct { usize, *Segment } { @@ -552,7 +565,7 @@ pub const Arena = struct { // Continue searching from the last allocated segment, wrapping around at // the end of the arena. fn next_fit( - self: *Self, + self: *Arena, size: usize, options: AllocOptions, ) ?struct { usize, *Segment } { @@ -588,7 +601,7 @@ pub const Arena = struct { // Try to fit the given segment to the request with the given options. // Returns the fitted address if successful. fn try_to_fit( - self: *Self, + self: *Arena, segment: *const Segment, size: usize, options: AllocOptions, @@ -610,22 +623,22 @@ pub const Arena = struct { return null; } - fn alloc_segment(self: *Self) !*Segment { + fn alloc_segment(self: *Arena) !*Segment { _ = self; return seg_zone.create(); } - fn free_segment(self: *Self, segment: *Segment) void { + fn free_segment(self: *Arena, segment: *Segment) void { seg_zone.destroy(segment); _ = self; } - fn add_segment_to_freelist(self: *Self, segment: *Segment) void { + fn add_segment_to_freelist(self: *Arena, segment: *Segment) void { const freelist = self.freelist_for_size(segment.size); freelist.insert_tail(&segment.linkage.free_link); } - fn remove_segment_from_freelist(self: *Self, segment: *Segment) void { + fn remove_segment_from_freelist(self: *Arena, segment: *Segment) void { _ = self; segment.linkage.free_link.remove(); } @@ -634,7 +647,7 @@ pub const Arena = struct { return std.math.log2_int(usize, size); } - inline fn freelist_for_size(self: *Self, size: usize) *rtl.List { + inline fn freelist_for_size(self: *Arena, size: usize) *rtl.List { return &self.freelists[freelist_index(size)]; } }; diff --git a/src/ex/work.zig b/src/ex/work.zig index 719ccbc..e6f7c9c 100644 --- a/src/ex/work.zig +++ b/src/ex/work.zig @@ -6,8 +6,8 @@ //! - High //! - Normal //! - Low -//! High and Normal worker pools are per-cpu, and there is one global worker pool -//! for low-priority work. +//! High and Normal worker pools are per-cpu, +//! and there is one global worker pool for low-priority work. //! //! Each CPU worker pool comprises of at least 2 threads, and the low pool //! has 4 threads by default. An additional thread will get spawned by the @@ -69,7 +69,7 @@ pub const WorkItem = struct { .func = func, .arg = arg, .link = undefined, - .enqueued = .init(false), + .enqueued = std.atomic.Value(bool).init(false), .priority = priority, }; } @@ -89,7 +89,7 @@ pub const DelayedWorkItem = struct { ) void { self.* = .{ .item = undefined, - .dpc = .init(work_dpc), + .dpc = ke.Dpc.init(work_dpc), .timer = undefined, }; @@ -117,14 +117,17 @@ const Pool = struct { const how_many = min_threads_for_prio(prio); - self.total_threads = .init(how_many); + self.total_threads = std.atomic.Value(usize).init(how_many); for (0..how_many) |_| { - const ctx: Context = .init(self, 0); + const ctx = Context.init(self, 0); var td = try ps.thread.create_kernel( WorkItem.Priority.to_sched_prio(self.prio), - .{ .func = work_loop, .arg = @ptrFromInt(ctx.value) }, + .{ + .func = work_loop, + .arg = @ptrFromInt(ctx.value), + }, true, ); @@ -139,7 +142,7 @@ const Pool = struct { } } - /// Return whether or not a new worker thread should be spawned on this pool. + /// Return whether or not a new worker thread should be added to this pool. fn should_grow(self: *Pool) bool { // NOTE: lockless racy check whether or not the queue is empty // and its active count. @@ -166,20 +169,23 @@ const Pool = struct { } /// Spawn a dynamic worker thread on this pool. - fn grow(self: *Pool, cpu: ?u32) !void { + fn grow(self: *Pool, cpu: ?u32) void { if (self.total_threads.load(.monotonic) >= max_threads_for_prio(self.prio)) return; _ = self.total_threads.fetchAdd(1, .monotonic); - const ctx: Context = .init(self, 1); + const ctx = Context.init(self, 1); - var td = try ps.thread.create_kernel( + var td = ps.thread.create_kernel( WorkItem.Priority.to_sched_prio(self.prio), .{ .func = work_loop, .arg = @ptrFromInt(ctx.value) }, true, - ); + ) catch { + // This will wait for free memory. + unreachable; + }; if (cpu != null) { td.kern.last_cpu = cpu; @@ -205,15 +211,15 @@ var low: Pool = undefined; fn work_loop(p: ?*anyopaque) void { const ctx: Context = .{ .value = @intFromPtr(p) }; - const dynamic = ctx.get_tag() == 1; + const dynamic = ctx.tag() == 1; while (true) { const timeout: ?r.Nanoseconds = if (dynamic) .from(r.Seconds.init(5)) else null; - const item = ctx.get_ptr().queue.remove(timeout) catch { - _ = ctx.get_ptr().total_threads.fetchSub(1, .monotonic); + const item = ctx.ptr().queue.remove(timeout) catch { + _ = ctx.ptr().total_threads.fetchSub(1, .monotonic); // We haven't had work in a while, exit. ps.thread.exit(); @@ -244,7 +250,12 @@ pub fn enqueue(item: *WorkItem) void { const ipl = ke.ipl.raise(.Dispatch); defer ke.ipl.lower(ipl); - if (item.enqueued.cmpxchgStrong(false, true, .acquire, .monotonic) != null) { + if (item.enqueued.cmpxchgStrong( + false, + true, + .acquire, + .monotonic, + ) != null) { return; } @@ -286,16 +297,16 @@ fn pool_manager(_: ?*anyopaque) void { const c = percpu.remote(cpu); if (c.high.should_grow()) { - _ = c.high.grow(cpu) catch {}; + _ = c.high.grow(cpu); } if (c.normal.should_grow()) { - _ = c.normal.grow(cpu) catch {}; + _ = c.normal.grow(cpu); } } if (low.should_grow()) { - _ = low.grow(null) catch {}; + _ = low.grow(null); } } } diff --git a/src/kern/amd64/impl.zig b/src/kern/amd64/impl.zig index 94b267c..9bde59f 100644 --- a/src/kern/amd64/impl.zig +++ b/src/kern/amd64/impl.zig @@ -1,3 +1,5 @@ +//! amd64-specific kernel code. + const r = @import("root"); const ke = r.ke; const kep = ke.private; @@ -139,7 +141,10 @@ pub fn switch_cont_to_normal( ); } -pub fn call_continuation(ctx: *Context, continuation: ke.Continuation) noreturn { +pub fn call_continuation( + ctx: *Context, + continuation: ke.Continuation, +) noreturn { const td: *ke.Thread = @alignCast(@fieldParentPtr("context", ctx)); amd64_call_continuation( diff --git a/src/kern/amd64/init.zig b/src/kern/amd64/init.zig index 5504d97..8a4d1de 100644 --- a/src/kern/amd64/init.zig +++ b/src/kern/amd64/init.zig @@ -1,3 +1,5 @@ +//! amd64-specific kernel initialization code. + const amd64 = @import("arch"); const int = @import("int.zig"); const r = @import("root"); @@ -26,7 +28,7 @@ var gdt = extern struct { .tss = undefined, }; -extern fn gdt_load(gdtr: *const amd64.Gdtr) callconv(.{ .x86_64_sysv = .{} }) void; +extern fn gdt_load(gdtr: *const amd64.Gdtr) callconv(.c) void; fn early_cpu_init() linksection(r.init) void { const gdtr: amd64.Gdtr = .{ diff --git a/src/kern/amd64/int.zig b/src/kern/amd64/int.zig index 2f82334..6c51043 100644 --- a/src/kern/amd64/int.zig +++ b/src/kern/amd64/int.zig @@ -1,3 +1,5 @@ +//! amd64 interrupt handling. + const amd64 = @import("arch"); const std = @import("std"); const r = @import("root"); @@ -45,15 +47,36 @@ const exception_msg = [32][]const u8{ "Security Exception", }; -export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sysv = .{} }) void { +fn ignore(_: std.Io.Writer.Error!void) void {} + +export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.c) void { const ipl = ke.ipl.set_hardware(.Device); if (frame.intno < 32) { - std.log.err("Unhandled exception: 0x{x} ({s}), err=0x{x}, pc=0x{x}", .{ frame.intno, exception_msg[frame.intno], frame.errcode, frame.rip }); - std.log.err("RAX=0x{x:0>16} RBX=0x{x:0>16} RCX=0x{x:0>16} RDX=0x{x:0>16}", .{ frame.rax, frame.rbx, frame.rcx, frame.rdx }); - std.log.err("RSI=0x{x:0>16} RDI=0x{x:0>16} RBP=0x{x:0>16} RSP=0x{x:0>16}", .{ frame.rsi, frame.rdi, frame.rbp, frame.rsp }); - std.log.err("R8= 0x{x:0>16} R9=0x{x:0>16} R10=0x{x:0>16} R11=0x{x:0>16}", .{ frame.r8, frame.r9, frame.r10, frame.r11 }); - std.log.err("R12=0x{x:0>16} R13=0x{x:0>16} R14=0x{x:0>16} R15=0x{x:0>16}", .{ frame.r12, frame.r13, frame.r14, frame.r15 }); + std.log.err("Unhandled exception: 0x{x} ({s}), err=0x{x}, pc=0x{x}", .{ + frame.intno, + exception_msg[frame.intno], + frame.errcode, + frame.rip, + }); + + std.log.err( + "RAX=0x{x:0>16} RBX=0x{x:0>16} RCX=0x{x:0>16} RDX=0x{x:0>16}", + .{ frame.rax, frame.rbx, frame.rcx, frame.rdx }, + ); + std.log.err( + "RSI=0x{x:0>16} RDI=0x{x:0>16} RBP=0x{x:0>16} RSP=0x{x:0>16}", + .{ frame.rsi, frame.rdi, frame.rbp, frame.rsp }, + ); + std.log.err( + "R8= 0x{x:0>16} R9=0x{x:0>16} R10=0x{x:0>16} R11=0x{x:0>16}", + .{ frame.r8, frame.r9, frame.r10, frame.r11 }, + ); + std.log.err( + "R12=0x{x:0>16} R13=0x{x:0>16} R14=0x{x:0>16} R15=0x{x:0>16}", + .{ frame.r12, frame.r13, frame.r14, frame.r15 }, + ); + const cr2 = amd64.read_cr(2); const cr3 = amd64.read_cr(3); const rflags = amd64.rflags(); @@ -61,21 +84,27 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys var buf: [128]u8 = undefined; var writer = std.Io.Writer.fixed(&buf); - writer.print("0x{x:0>8} [", .{@as(u64, @bitCast(rflags))}) catch {}; - if (rflags.carry) writer.writeAll(" CF") catch {}; - if (rflags.parity) writer.writeAll(" PF") catch {}; - if (rflags.auxiliary) writer.writeAll(" AF") catch {}; - if (rflags.zero) writer.writeAll(" ZF") catch {}; - if (rflags.sign) writer.writeAll(" SF") catch {}; - if (rflags.trap) writer.writeAll(" TF") catch {}; - if (rflags.interrupt_enable) writer.writeAll(" IF") catch {}; - if (rflags.direction) writer.writeAll(" DF") catch {}; - if (rflags.overflow) writer.writeAll(" OF") catch {}; - if (rflags.resume_) writer.writeAll(" RF") catch {}; - if (rflags.virtual_8086_mode) writer.writeAll(" VM") catch {}; - writer.print(" IOPL={d}]", .{rflags.iopl}) catch {}; + writer.print("0x{x:0>8} [", .{@as(u64, @bitCast(rflags))}) catch { + // Shouldn't fail. + }; - std.log.err("CR2=0x{x:0>16} CR3=0x{x:0>16} RFLAGS={s}", .{ cr2, cr3, buf[0..writer.end] }); + if (rflags.carry) ignore(writer.writeAll(" CF")); + if (rflags.parity) ignore(writer.writeAll(" PF")); + if (rflags.auxiliary) ignore(writer.writeAll(" AF")); + if (rflags.zero) ignore(writer.writeAll(" ZF")); + if (rflags.sign) ignore(writer.writeAll(" SF")); + if (rflags.trap) ignore(writer.writeAll(" TF")); + if (rflags.interrupt_enable) ignore(writer.writeAll(" IF")); + if (rflags.direction) ignore(writer.writeAll(" DF")); + if (rflags.overflow) ignore(writer.writeAll(" OF")); + if (rflags.resume_) ignore(writer.writeAll(" RF")); + if (rflags.virtual_8086_mode) ignore(writer.writeAll(" VM")); + ignore(writer.print(" IOPL={d}]", .{rflags.iopl})); + + std.log.err( + "CR2=0x{x:0>16} CR3=0x{x:0>16} RFLAGS={s}", + .{ cr2, cr3, buf[0..writer.end] }, + ); kep.panic.panic_with_frame("Unhandled exception", frame.rbp); } @@ -102,7 +131,7 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys } } -extern fn idt_load(idt_ptr: *const amd64.Idtr) callconv(.{ .x86_64_sysv = .{} }) void; +extern fn idt_load(idt_ptr: *const amd64.Idtr) callconv(.c) void; pub fn init() linksection(r.init) void { const idtr: amd64.Idtr = .{ @@ -111,7 +140,12 @@ pub fn init() linksection(r.init) void { }; for (&idt, 0..256) |*entry, i| { - entry.* = .init(0x28, 0, .InterruptGate, __interrupt_vectors[i]); + entry.* = amd64.IdtEntry.init( + 0x28, + 0, + .InterruptGate, + __interrupt_vectors[i], + ); } idt_load(&idtr); diff --git a/src/kern/cpu.zig b/src/kern/cpu.zig index 28fdd12..1dd2cfb 100644 --- a/src/kern/cpu.zig +++ b/src/kern/cpu.zig @@ -1,3 +1,5 @@ +//! CPU-local data. + const config = @import("config"); const rtl = @import("rtl"); const r = @import("root"); diff --git a/src/kern/dpc.zig b/src/kern/dpc.zig index 56c01a1..27235fd 100644 --- a/src/kern/dpc.zig +++ b/src/kern/dpc.zig @@ -1,10 +1,14 @@ -const std = @import("std"); +//! Deferred procedure calls (DPC). + const config = @import("config"); const rtl = @import("rtl"); const r = @import("root"); const ke = r.ke; const kep = ke.private; +const std = @import("std"); +const assert = std.debug.assert; + /// Deferred Procedure Call (DPC) structure. /// Used for scheduling work to be done when IPL is lowered below `.Dispatch`. pub const Dpc = struct { @@ -23,7 +27,7 @@ pub const Dpc = struct { .link = undefined, .func = func, .arg = undefined, - .inserted = .init(false), + .inserted = std.atomic.Value(bool).init(false), }; } }; @@ -108,7 +112,7 @@ fn dispatch_queue(cpu: u32) void { const arg = dpc.arg; dpc_cpu.lock.release(ipl); - std.debug.assert(ke.ipl.current() == .Dispatch); + assert(ke.ipl.current() == .Dispatch); dpc.func(dpc, arg); } @@ -137,7 +141,7 @@ fn dispatch_queue(cpu: u32) void { kep.shootdown.process_shootdowns(); if (sched_cpu.next_thread != null) { - const curtd = sched_cpu.current_thread.?; + const curtd = sched_cpu.current_thread orelse unreachable; if (!curtd.smr_sections.is_empty()) { kep.smr.mark_thread_stalled(curtd); diff --git a/src/kern/event.zig b/src/kern/event.zig index d8494bf..4ae69da 100644 --- a/src/kern/event.zig +++ b/src/kern/event.zig @@ -1,3 +1,5 @@ +//! Kernel events. + const ke = @import("root").ke; const kep = ke.private; @@ -5,8 +7,6 @@ const kep = ke.private; pub const Event = struct { hdr: kep.wait.DispatchHeader, - const Self = @This(); - pub const Type = enum { /// Wake only one waiter and decrease signaled. Synchronization, @@ -15,7 +15,7 @@ pub const Event = struct { }; /// Initialize an event with a specified type. - pub fn init(self: *Self, @"type": Type) void { + pub fn init(self: *Event, @"type": Type) void { self.hdr.init(switch (@"type") { .Synchronization => .Synchronization, .Notification => .Notification, @@ -23,7 +23,7 @@ pub const Event = struct { } /// Signal an event, waking its waiter(s). - pub fn signal(self: *Self) void { + pub fn signal(self: *Event) void { const ipl = self.hdr.lock.acquire(); self.hdr.signaled = 1; kep.wait.satisfy_wait(&self.hdr); @@ -31,7 +31,7 @@ pub const Event = struct { } /// Reset an event. - pub fn reset(self: *Self) void { + pub fn reset(self: *Event) void { const ipl = self.hdr.lock.acquire(); self.hdr.signaled = 0; self.hdr.lock.release(ipl); diff --git a/src/kern/ipi.zig b/src/kern/ipi.zig index f9e3f5f..475130c 100644 --- a/src/kern/ipi.zig +++ b/src/kern/ipi.zig @@ -31,7 +31,7 @@ const PerCpu = struct { const percpu = ke.CpuLocal(PerCpu, undefined); -var frozen_cpus: std.atomic.Value(u32) = .init(0); +var frozen_cpus = std.atomic.Value(u32).init(0); /// Call a function on another CPU. /// The function will be executed synchronously, i.e this will block until @@ -69,13 +69,16 @@ pub fn unicast( /// The function will be executed synchronously, i.e this will block until /// the function finishes running. The callback is called with the sender cpu /// and provided argument as arguments. -pub fn broadcast(func: *const fn (u32, ?*anyopaque) void, arg: ?*anyopaque) void { +pub fn broadcast( + func: *const fn (u32, ?*anyopaque) void, + arg: ?*anyopaque, +) void { const ipl = ke.ipl.raise(.Dispatch); const curcpu = percpu.local(); curcpu.func = func; curcpu.arg = arg; - curcpu.counter = .init(@truncate(ke.ncpus - 1)); + curcpu.counter = std.atomic.Value(u16).init(@truncate(ke.ncpus - 1)); for (0..ke.ncpus) |i| { if (i == ke.cpu.current()) continue; diff --git a/src/kern/ipl.zig b/src/kern/ipl.zig index dd94fc5..8c3d4a6 100644 --- a/src/kern/ipl.zig +++ b/src/kern/ipl.zig @@ -1,8 +1,12 @@ -const std = @import("std"); +//! Interrupt priority levels. + const r = @import("root"); const ke = r.ke; const kep = ke.private; +const std = @import("std"); +const assert = std.debug.assert; + /// Interrupt priority level (IPL) pub const Ipl = enum(u8) { /// All interrupts enabled. @@ -19,7 +23,7 @@ pub const Ipl = enum(u8) { } pub fn value(self: Ipl) u8 { - return @intFromEnum(self); + return @backingInt(self); } }; @@ -36,7 +40,7 @@ pub const cpu_ipl = ke.ExportedCpuLocal( ); pub const percpu = ke.CpuLocal(PerCpu, .{ - .pending_softints = .init(0), + .pending_softints = std.atomic.Value(u8).init(0), }); pub fn current() Ipl { @@ -109,7 +113,7 @@ pub fn set_hardware(new: Ipl) Ipl { /// Mark a software interrupt of IPL `ipl` on `cpu` as pending. pub fn set_softint_pending(cpu: u32, ipl: Ipl) void { - std.debug.assert(ipl.value() <= Ipl.Dispatch.value()); + assert(ipl.value() <= Ipl.Dispatch.value()); _ = percpu.remote(cpu).pending_softints.bitSet( @intCast(ipl.value()), .monotonic, @@ -118,7 +122,7 @@ pub fn set_softint_pending(cpu: u32, ipl: Ipl) void { /// Mark a software interrupt of IPL `ipl` on `cpu` as handled. pub fn clear_softint_pending(cpu: u32, ipl: Ipl) void { - std.debug.assert(ipl.value() <= Ipl.Dispatch.value()); + assert(ipl.value() <= Ipl.Dispatch.value()); _ = percpu.remote(cpu).pending_softints.bitReset( @intCast(ipl.value()), .monotonic, @@ -128,7 +132,7 @@ pub fn clear_softint_pending(cpu: u32, ipl: Ipl) void { /// Check whether a software interrupt of IPL `ipl` /// is pending on the current CPU. pub fn is_softint_pending(ipl: Ipl) bool { - std.debug.assert(ipl.value() <= Ipl.Dispatch.value()); + assert(ipl.value() <= Ipl.Dispatch.value()); const shift: u3 = @intCast(ipl.value()); const bit: u8 = @as(u8, 1) << shift; return percpu.local().pending_softints.load(.monotonic) & bit != 0; diff --git a/src/kern/log.zig b/src/kern/log.zig index 8c00236..ff08232 100644 --- a/src/kern/log.zig +++ b/src/kern/log.zig @@ -1,3 +1,5 @@ +//! Kernel logging. + const std = @import("std"); const config = @import("config"); const r = @import("root"); diff --git a/src/kern/log_ring.zig b/src/kern/log_ring.zig index c58f16e..e1be7a8 100644 --- a/src/kern/log_ring.zig +++ b/src/kern/log_ring.zig @@ -62,10 +62,10 @@ //! # Wrap handling //! //! Data blocks are never split across the physical boundary of the data array. -//! If an allocation would cross the boundary, the block is placed at offset 0 of -//! the next generation and a stub (containing only the descriptor ID) is left at -//! the original position so the data ring can be walked linearly during eviction. -//! +//! If an allocation would cross the boundary, the block is placed at offset 0 +//! of the next generation and a stub (containing only the descriptor ID) is +//! left at the original position so the data ring can be walked linearly +//! during eviction. const std = @import("std"); const r = @import("root"); @@ -154,10 +154,10 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { /// Beginning of the oldest data block. tail_lpos: std.atomic.Value(usize), - const Self = @This(); + const Ring = @This(); - // Return the exact Id of the descriptor that occupied this physical slot - // one generation ago. + // Return the exact Id of the descriptor that occupied this physical + // slot one generation ago. fn prev_wrap(id: Id) Id { return id -% desc_count; } @@ -178,15 +178,15 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { return sz; } - fn to_desc(self: *Self, id: Id) *Desc { + fn to_desc(self: *Ring, id: Id) *Desc { return &self.descs[id & desc_mask]; } - fn to_info(self: *Self, id: Id) *Info { + fn to_info(self: *Ring, id: Id) *Info { return &self.infos[id & desc_mask]; } - fn to_block(self: *Self, lpos: usize) *DataBlock { + fn to_block(self: *Ring, lpos: usize) *DataBlock { return @ptrCast(@alignCast(&self.data[lpos & data_mask])); } @@ -198,12 +198,12 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { // Return true if `lpos_current` has not yet reached `lpos_next`. // Uses wrapping subtraction so that: - // - if current > next (another CPU already advanced past the target), - // returns false. - // - if current == next (already at target), the -1 wraps to usize max, - // returns false. - // - if the distance exceeds data_size (should never happen), - // returns false. + // - if current > next (another CPU already advanced past the target), + // returns false. + // - if current == next (already at target), the -1 wraps to usize_max, + // returns false. + // - if the distance exceeds data_size (should never happen), + // returns false. fn need_more_space(lpos_current: usize, lpos_next: usize) bool { return lpos_next -% lpos_current -% 1 < data_size; } @@ -214,7 +214,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { desc: Desc = undefined, }; - fn read_desc(self: *Self, id: Id) ReadDescResult { + fn read_desc(self: *Ring, id: Id) ReadDescResult { const desc = self.to_desc(id); const info = self.to_info(id); var state_var = desc.state.load(.acquire); @@ -245,7 +245,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { } fn read_desc_finalized( - self: *Self, + self: *Ring, id: Id, seq: u64, desc_out: *Desc, @@ -263,7 +263,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { return error.DataLost; } - fn free_desc(self: *Self, id: Id) void { + fn free_desc(self: *Ring, id: Id) void { var desc = self.to_desc(id); const expected_state: State = .{ .state = .Published, @@ -282,7 +282,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { ); } - fn free_data(self: *Self, begin: usize, end: usize) ?usize { + fn free_data(self: *Ring, begin: usize, end: usize) ?usize { var desc: Desc = undefined; var cur_begin = begin; @@ -330,7 +330,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { return cur_begin; } - fn data_advance_tail(self: *Self, new_tail_lpos: usize) !void { + fn data_advance_tail(self: *Ring, new_tail_lpos: usize) !void { if (new_tail_lpos & 1 != 0) { // No data. return; @@ -355,8 +355,8 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { break; } } else { - // Failed to free the data, which means we either lost a race - // or the descriptor is still in use. + // Failed to free the data, which means we either lost a + // race or the descriptor is still in use. // Reload the tail position to try again. const new_tail = self.tail_lpos.load(.acquire); if (new_tail == cur_tail) { @@ -372,7 +372,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { } // Try advancing the tail. - fn advance_tail(self: *Self, tail_id: Id) !void { + fn advance_tail(self: *Ring, tail_id: Id) !void { const res = self.read_desc(tail_id); const tail_desc = res.desc; @@ -387,7 +387,8 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { return error.ReservationFailed; }, .Miss => { - if (tail_desc.state.load(.monotonic).id == prev_wrap(tail_id)) { + const tail_desc_id = tail_desc.state.load(.monotonic).id; + if (tail_desc_id == prev_wrap(tail_id)) { // This must mean that the descriptor is currently // getting reserved by another writer. return error.ReservationFailed; @@ -411,8 +412,9 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { if (next_res.state == .Published or next_res.state == .Free) { // Advance the tail to the next descriptor. - // release: push all previous state changes before the tail update, - // guaranteeing the new tail is visible after all evictions. + // release: push all previous state changes before the tail + // update, guaranteeing the new tail is visible after all + // evictions. _ = self.tail_id.cmpxchgStrong( @as(usize, tail_id), @as(usize, tail_id +% 1), @@ -433,7 +435,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { } // Reserve a new descriptor and make space for it if needed. - fn reserve_desc(self: *Self) !Id { + fn reserve_desc(self: *Ring) !Id { var head_id: Id = @truncate(self.head_id.load(.acquire)); var new_id: Id = undefined; var prev_id: Id = undefined; @@ -520,7 +522,12 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { return next_lpos; } - fn alloc_data(self: *Self, size: usize, id: Id, out_position: *BlkPos) ReserveError![]u8 { + fn alloc_data( + self: *Ring, + size: usize, + id: Id, + out_position: *BlkPos, + ) ReserveError![]u8 { const blk_size = to_block_size(size); var head_lpos = self.head_lpos.load(.monotonic); var new_head_lpos = head_lpos; @@ -534,7 +541,8 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { while (true) { new_head_lpos = get_next_lpos(head_lpos, blk_size); - // Make room if needed, try advancing the tail until we have enough space. + // Make room if needed, try advancing the tail until we have + // enough space. self.data_advance_tail(new_head_lpos -% data_size) catch |e| { out_position.begin = lpos_no_data; out_position.end = lpos_no_data; @@ -570,7 +578,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { return @as([*]u8, @ptrCast(&blk.data))[0..size]; } - fn read_internal(self: *Self, seq: u64, buf: ?[]u8) !Info { + fn read_internal(self: *Ring, seq: u64, buf: ?[]u8) !Info { const rdesc = self.to_desc(@truncate(seq)); const info = self.to_info(@truncate(seq)); var desc: Desc = undefined; @@ -601,14 +609,14 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { } /// Initialize and return a new ring buffer instance. - pub fn init() Self { + pub fn init() Ring { // Start near usize max so that the first wrap of the ID/lpos space // happens almost immediately, testing overflow handling early // rather than after billions of records. const dummy_id: Id = 0 -% @as(Id, desc_count + 1); const blk0: usize = -%@as(usize, data_size); - var self: Self = std.mem.zeroes(Self); + var self: Ring = std.mem.zeroes(Ring); // Place a dummy descriptor in the last slot. head_id and tail_id // both start here so that the first reservation (head_id + 1) lands @@ -638,7 +646,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { } /// Reserve space for a record and return a reservation. - pub fn reserve(self: *Self, size: usize) ReserveError!Reservation { + pub fn reserve(self: *Ring, size: usize) ReserveError!Reservation { if (!check_size(size)) { return error.NoSpace; } @@ -657,15 +665,15 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { if (seq == 0 and (id & desc_mask) != 0) { // First time this slot has been used. - // Slot 0 never hits this branch, its seq is set to -(desc_count) - // in init() so that the else branch below produces seq=0 on - // its first use. + // Slot 0 never hits this branch, its seq is set + // to -(desc_count) in init() so that the else branch + // below produces seq=0 on its first use. info.sequence = @as(u64, id & desc_mask); } else { // Slot is being recycled (or is slot 0 on first use). - // Adding desc_count advances the sequence by one full ring wrap, - // keeping sequence numbers monotonically increasing across - // generations. + // Adding desc_count advances the sequence by one full + // ring wrap, keeping sequence numbers monotonically increasing + // across generations. info.sequence = seq +% desc_count; } @@ -686,7 +694,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { } /// Publish a previously reserved record. - pub fn publish(self: *Self, res: Reservation) void { + pub fn publish(self: *Ring, res: Reservation) void { const desc = self.to_desc(res.id); _ = desc.state.cmpxchgStrong( @@ -707,7 +715,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { } /// Return the first readable sequence currently retained by the ring. - pub fn first_seq(self: *Self) u64 { + pub fn first_seq(self: *Ring) u64 { var seq: u64 = undefined; while (true) { @@ -723,30 +731,31 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { } /// Read a record at `seq` into `buf`. - pub fn read(self: *Self, seq: u64, buf: ?[]u8) ReadError!Info { + pub fn read(self: *Ring, seq: u64, buf: ?[]u8) ReadError!Info { var cur_seq = seq; while (true) { - return self.read_internal(cur_seq, buf) catch |err| switch (err) { - error.Invalid => { - const first = self.first_seq(); - if (cur_seq < first) { - // Behind the tail, catch up and try again. - cur_seq = first; + return self.read_internal(cur_seq, buf) catch |err| + switch (err) { + error.Invalid => { + const first = self.first_seq(); + if (cur_seq < first) { + // Behind the tail, catch up and try again. + cur_seq = first; + continue; + } + return error.NotYetAvailable; + }, + error.DataLost => { + // Record at cur_seq was overwritten, skip forward. + const first = self.first_seq(); + if (cur_seq < first) + cur_seq = first + else + cur_seq += 1; continue; - } - return error.NotYetAvailable; - }, - error.DataLost => { - // Record at cur_seq was overwritten, skip forward. - const first = self.first_seq(); - if (cur_seq < first) - cur_seq = first - else - cur_seq += 1; - continue; - }, - }; + }, + }; } } }; diff --git a/src/kern/mutex.zig b/src/kern/mutex.zig index 70f63ef..856dd2f 100644 --- a/src/kern/mutex.zig +++ b/src/kern/mutex.zig @@ -1,3 +1,5 @@ +//! Adaptive lock implementation. + const std = @import("std"); const rtl = @import("rtl"); const config = @import("config"); @@ -22,7 +24,7 @@ pub const Mutex = struct { }; } - return .{ .owner = .init(null) }; + return .{ .owner = std.atomic.Value(?*ke.Thread).init(null) }; } pub fn is_locked(self: *Mutex) bool { @@ -31,7 +33,8 @@ pub const Mutex = struct { pub fn acquire(m: *Mutex) void { const ipl = ke.ipl.raise(.Dispatch); - const curtd = kep.sched.percpu.local().current_thread.?; + const curtd = kep.sched.percpu.local().current_thread orelse + unreachable; if (config.warden) { kep.warden.check(m.wd, .Lock); @@ -98,7 +101,7 @@ pub const Mutex = struct { kep.turnstile.block( ts, m, - .{ .single = owner.? }, + .{ .single = owner orelse unreachable }, .Exclusive, ); @@ -133,6 +136,7 @@ pub const Mutex = struct { } var waiters: rtl.List = undefined; + const turn = ts orelse unreachable; // Note: wake up all waiters. // This so-called "lock barging" (name from WTF::ParkingLot) has been @@ -140,9 +144,9 @@ pub const Mutex = struct { // see this mysterious 70s paper: // https://dl.acm.org/doi/pdf/10.1145/850657.850659 kep.turnstile.signal( - ts.?, + turn, .Exclusive, - ts.?.waiters, + turn.waiters, null, &waiters, ); diff --git a/src/kern/panic.zig b/src/kern/panic.zig index acc2fc9..b4551b4 100644 --- a/src/kern/panic.zig +++ b/src/kern/panic.zig @@ -1,3 +1,5 @@ +//! Kernel panic handling code. + const std = @import("std"); const config = @import("config"); const r = @import("root"); @@ -43,7 +45,10 @@ fn get_symbol_name(addr: usize) ?Symbol { const sym = symbols[left - 1]; if (sym.addr <= addr) { - return .{ .name = sym.name_ptr[0..sym.name_len], .offset = addr - sym.addr }; + return .{ + .name = sym.name_ptr[0..sym.name_len], + .offset = addr - sym.addr, + }; } } @@ -57,7 +62,11 @@ fn is_kernel_text(addr: usize) bool { } fn walk_stack_frame(base: usize) void { - var frame: ?*StackFrame = @ptrFromInt(if (config.arch == .riscv64) base - 16 else base); + var frame: ?*StackFrame = @ptrFromInt(if (config.arch == .riscv64) + base - 16 + else + base); + var depth: usize = 0; while (frame) |f| : (depth += 1) { if (depth > 64) break; @@ -65,16 +74,27 @@ fn walk_stack_frame(base: usize) void { if (!is_kernel_text(ret_addr)) { break; } - const sym = get_symbol_name(ret_addr) orelse Symbol{ .name = "???", .offset = 0 }; - std.log.err(" #{d} {s}+0x{x} - 0x{x}", .{ depth, sym.name, sym.offset, ret_addr }); - frame = if (config.arch == .riscv64 and f.prev != null) - @ptrFromInt(@intFromPtr(f.prev.?) - 16) - else - f.prev; + const sym = get_symbol_name(ret_addr) orelse + Symbol{ .name = "???", .offset = 0 }; + + std.log.err(" #{d} {s}+0x{x} - 0x{x}", .{ + depth, + sym.name, + sym.offset, + ret_addr, + }); + + if (config.arch == .riscv64) { + if (f.prev) |fr| { + frame = @ptrFromInt(@intFromPtr(fr) - 16); + } + } else { + frame = f.prev; + } } } -var crash_count: std.atomic.Value(u8) = .init(0); +var crash_count = std.atomic.Value(u8).init(0); pub fn panic_with_frame( msg: []const u8, diff --git a/src/kern/private.zig b/src/kern/private.zig index e5a9ac1..cf65a2a 100644 --- a/src/kern/private.zig +++ b/src/kern/private.zig @@ -1,3 +1,5 @@ +//! Private kernel definitions. + const config = @import("config"); const init_mod = @import("init.zig"); const r = @import("root"); diff --git a/src/kern/queue.zig b/src/kern/queue.zig index f9e2a7a..30e9657 100644 --- a/src/kern/queue.zig +++ b/src/kern/queue.zig @@ -9,7 +9,9 @@ const r = @import("root"); const rtl = @import("rtl"); const ke = r.ke; const kep = ke.private; + const std = @import("std"); +const assert = std.debug.assert; pub const Queue = struct { hdr: kep.wait.DispatchHeader, @@ -56,11 +58,11 @@ pub const Queue = struct { /// This blocks until an item is actually popped. pub fn remove(self: *Queue, timeout: ?r.Nanoseconds) !*rtl.List.Entry { const ipl = self.hdr.lock.acquire(); - const td = kep.sched.percpu.local().current_thread.?; + const td = kep.sched.percpu.local().current_thread orelse unreachable; if (td.queue) |q| { - std.debug.assert(q == self); - std.debug.assert(self.active > 0); + assert(q == self); + assert(self.active > 0); self.active -= 1; if (self.hdr.signaled > 0 and self.active < self.max_active) { @@ -90,7 +92,7 @@ pub const Queue = struct { /// than the queue. pub fn signal_wait(queue: *Queue) void { const ipl = queue.hdr.lock.acquire(); - std.debug.assert(queue.active > 0); + assert(queue.active > 0); queue.active -= 1; diff --git a/src/kern/riscv64/impl.zig b/src/kern/riscv64/impl.zig index e390d8b..988f7a8 100644 --- a/src/kern/riscv64/impl.zig +++ b/src/kern/riscv64/impl.zig @@ -1,3 +1,5 @@ +//! Kernel riscv64-specific code. + const r = @import("root"); const ke = r.ke; const rv64 = @import("arch"); @@ -20,7 +22,12 @@ pub const ThreadContext = extern struct { return .{ .sp = stack }; } - pub fn reset(self: *@This(), stack_top: r.VAddr, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) @This() { + pub fn reset( + self: *@This(), + stack_top: r.VAddr, + entry: *const fn (?*anyopaque) void, + arg: ?*anyopaque, + ) @This() { self.sp = stack_top & ~@as(usize, 15); self.ra = @intFromPtr(&thread_start); self.s = @splat(0); @@ -35,11 +42,27 @@ pub const ThreadContext = extern struct { }; extern fn riscv_context_load(ctx: *ThreadContext) callconv(.c) noreturn; -extern fn riscv_context_switch(old: *ThreadContext, new: *ThreadContext, lock: *u8, switching: *bool) callconv(.c) void; -extern fn riscv_context_switch_cont(old: *ke.Thread.Context, new: *ThreadContext, lock: *u8, switching: *bool) callconv(.c) void; + +extern fn riscv_context_switch( + old: *ThreadContext, + new: *ThreadContext, + lock: *u8, + switching: *bool, +) callconv(.c) void; + +extern fn riscv_context_switch_cont( + old: *ke.Thread.Context, + new: *ThreadContext, + lock: *u8, + switching: *bool, +) callconv(.c) void; + extern fn thread_start() callconv(.c) noreturn; -export fn riscv_thread_entry(entry_addr: usize, arg_addr: usize) callconv(.c) noreturn { +export fn riscv_thread_entry( + entry_addr: usize, + arg_addr: usize, +) callconv(.c) noreturn { const entry: *const fn (?*anyopaque) void = @ptrFromInt(entry_addr); entry(@ptrFromInt(arg_addr)); @panic("Thread entry returned"); @@ -49,24 +72,46 @@ export fn riscv_free_old_stack(stack: usize) callconv(.c) void { ke.private.thread.stack_cache_free(stack); } -export fn riscv_run_continuation(func_addr: usize, arg: ?*anyopaque) callconv(.c) noreturn { +export fn riscv_run_continuation( + func_addr: usize, + arg: ?*anyopaque, +) callconv(.c) noreturn { ke.ipl.lower(.Passive); const func: *const fn (?*anyopaque) void = @ptrFromInt(func_addr); func(arg); @panic("Continuation returned"); } -pub fn switch_normal_to_normal(old: *ke.Thread.Context, new: *ke.Thread.Context) void { +pub fn switch_normal_to_normal( + old: *ke.Thread.Context, + new: *ke.Thread.Context, +) void { const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", old)); - riscv_context_switch(&old.impl, &new.impl, &thread.lock.inner.locked.raw, &thread.switching.raw); + riscv_context_switch( + &old.impl, + &new.impl, + &thread.lock.inner.locked.raw, + &thread.switching.raw, + ); } -pub fn switch_cont_to_normal(old: *ke.Thread.Context, new: *ke.Thread.Context) void { +pub fn switch_cont_to_normal( + old: *ke.Thread.Context, + new: *ke.Thread.Context, +) void { const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", old)); - riscv_context_switch_cont(old, &new.impl, &thread.lock.inner.locked.raw, &thread.switching.raw); + riscv_context_switch_cont( + old, + &new.impl, + &thread.lock.inner.locked.raw, + &thread.switching.raw, + ); } -pub fn call_continuation(ctx: *ke.Thread.Context, continuation: ke.Continuation) noreturn { +pub fn call_continuation( + ctx: *ke.Thread.Context, + continuation: ke.Continuation, +) noreturn { const sp = ctx.stack_top & ~@as(usize, 15); asm volatile ( \\mv sp, %[stack] diff --git a/src/kern/riscv64/int.zig b/src/kern/riscv64/int.zig index 4afd820..78ee7f6 100644 --- a/src/kern/riscv64/int.zig +++ b/src/kern/riscv64/int.zig @@ -1,3 +1,5 @@ +//! riscv64 interrupt handling. + const r = @import("root"); const ke = r.ke; const std = @import("std"); @@ -13,5 +15,9 @@ export fn riscv_trap_handler(cause: usize, pc: usize) callconv(.c) void { } return; } - std.debug.panic("trap: cause = 0x{x}, pc = 0x{x}, tval = 0x{x}", .{ cause, pc, rv64.read_csr("stval") }); + std.debug.panic("trap: cause = 0x{x}, pc = 0x{x}, tval = 0x{x}", .{ + cause, + pc, + rv64.read_csr("stval"), + }); } diff --git a/src/kern/root.zig b/src/kern/root.zig index 0aa4dde..ff0386e 100644 --- a/src/kern/root.zig +++ b/src/kern/root.zig @@ -1,3 +1,5 @@ +//! Public kernel interfaces. + // === Exported Modules === pub const private = @import("private.zig"); diff --git a/src/kern/sched.zig b/src/kern/sched.zig index 96eb1ef..f3458fc 100644 --- a/src/kern/sched.zig +++ b/src/kern/sched.zig @@ -91,9 +91,9 @@ //! balancing, the load is preferred as to prioritize latency //! (i.e. you want to run a thread ASAP). //! -//! See `sched_ule.c` in FreeBSD for the original implementation this is based on. +//! See `sched_ule.c` in FreeBSD for the original implementation this is based +//! on. -const std = @import("std"); const rtl = @import("rtl"); const config = @import("config"); const r = @import("root"); @@ -101,10 +101,13 @@ const ke = r.ke; const kep = ke.private; const pl = r.pl; +const std = @import("std"); +const assert = std.debug.assert; + const runqueues_n = 64; const interactivity_threshold = 30; const scaling_factor = 50; -const preempt_threshold = @intFromEnum(ke.Priority.LowRealtime); +const preempt_threshold = @backingInt(ke.Priority.LowRealtime); const balance_interval = @as(u64, config.sched_balance_interval); const sched_migration_cost = ke.Tunable( u64, @@ -122,8 +125,9 @@ const pct_window_max = 11 * std.time.us_per_s; // Each thread carries a decaying exponential weighted moving average (EWMA) // of the time it has spent runnable (running or in a queue). Each CPU maintains // the sum of its threads' values, which is used in load balancing decisions to -// estimate the *actual* load of a CPU (and not just its thread count). Each value -// is updated using periods of 1024us (~1ms) with a half-life of 32ms, i.e y^32 = 0.5. +// estimate the *actual* load of a CPU (and not just its thread count). +// Each value is updated using periods of 1024us (~1ms) with a half-life of +// 32ms, i.e y^32 = 0.5. // Math goes more in depth in comments below. const pelt_scale = 1024; const pelt_decay = 32; @@ -146,7 +150,7 @@ const pelt_halflife = pelt_scale * pelt_decay; pub const pelt_load_avg_max = 47742; // pelt_inv_table[k] = floor(2^32 * 2^(-k/pelt_decay)) -// NOTE: Linux uses 2^32 - 1 as base but this practically yields the same values. +// NOTE: Linux uses 2^32-1 as base but this practically yields the same values. const pelt_inv_table: [pelt_decay]u32 = .{ 0xffffffff, 0xfa83b2db, 0xf5257d15, 0xefe4b99b, 0xeac0c6e7, 0xe5b906e7, 0xe0ccdeec, 0xdbfbb797, @@ -223,7 +227,7 @@ pub const Average = struct { // Amount of the period that has been accounted for period_contrib: usize = 0, /// Timestamp (us) up to which history has been accounted. - last_update: r.Microseconds = .init(0), + last_update: r.Microseconds = r.Microseconds.init(0), }; pub const Accounting = struct { @@ -240,7 +244,7 @@ pub const Accounting = struct { pub const percpu = ke.CpuLocal(PerCpu, undefined); var balance_timer: ke.Timer = undefined; -var balance_dpc: ke.Dpc = .init(balance); +var balance_dpc = ke.Dpc.init(balance); // LCG values taken from FreeBSD. var lcg = std.Random.lcg.Wrapping(u32).init(0, 69069, 5); @@ -260,15 +264,15 @@ fn init_cpu() linksection(r.init) void { .status = 0, .queues = undefined, }, - .queues_lock = .init("runq"), + .queues_lock = ke.SpinLock.init("runq"), .idle_queue = undefined, .steal_work = false, - .load = .init(0), - .load_avg = .init(0), - .est_load_avg = .init(0), - .migratable = .init(0), - .current_thread_prio = .init(0), - .resched_dpc = .init(kep.sched.clock), + .load = std.atomic.Value(usize).init(0), + .load_avg = std.atomic.Value(usize).init(0), + .est_load_avg = std.atomic.Value(usize).init(0), + .migratable = std.atomic.Value(usize).init(0), + .current_thread_prio = std.atomic.Value(u8).init(0), + .resched_dpc = ke.Dpc.init(kep.sched.clock), .resched_timer = undefined, .start_timer = false, .preemption_reason = .None, @@ -318,7 +322,8 @@ fn pelt_do_decay(val: u64, n: u64) u64 { if (rem != 0) { // Fractional remainder, index into the table. - // The table gives us 2^32 * 2^(-k/pelt_decay), shift by 32 to remove the 2^32. + // The table gives us 2^32 * 2^(-k/pelt_decay), shift by 32 to remove + // the 2^32. v = (v * pelt_inv_table[(rem * pelt_decay) / pelt_halflife]) >> 32; } @@ -360,10 +365,12 @@ fn pelt_update(avg: *Average, runnable: bool) void { // d1 * y^p const c1 = pelt_do_decay(d1, decay_time); - // For a full period, its contribution is 1024y + 1024y^2 + ... + 1024y^(p-1) - // inf + // For a full period, its contribution is: + // 1024y + 1024y^2 + ... + 1024y^(p-1) + // // We know load_avg_max = p * Sum y^n // n=0 + // // So, load_avg_max - load_avg_max * y^p yields: // p-1 p-1 // p * Sum y^n, subtracting the first term (n=0) gives us p * Sum @@ -541,7 +548,7 @@ fn recompute_priority(td: *ke.Thread) void { // Choose a priority based on score, the lower the score, // the higher the priority it will be. // This is a simple formula I came up with that is probably good enough. - @intCast(@intFromEnum(ke.Priority.LowInteractive) + + @intCast(@backingInt(ke.Priority.LowInteractive) + ((interactivity_threshold - score) / 2)) else blk: { // Thread is not interactive, compute priority from recent CPU usage. @@ -560,14 +567,14 @@ fn recompute_priority(td: *ke.Thread) void { // positive decreases it. const nice_off: i16 = @as(i16, td.nice) + ke.Priority.nice_max; - const raw: i16 = @as(i16, @intFromEnum(ke.Priority.HighBatch)) - + const raw: i16 = @as(i16, @backingInt(ke.Priority.HighBatch)) - @as(i16, cpu_pri_off) - nice_off; break :blk @intCast(std.math.clamp( raw, - @intFromEnum(ke.Priority.LowBatch), - @intFromEnum(ke.Priority.HighBatch), + @backingInt(ke.Priority.LowBatch), + @backingInt(ke.Priority.HighBatch), )); }; @@ -589,8 +596,12 @@ fn calendar_queue_increment(cpu: *PerCpu) void { } } -fn pick_realtime_thread(cpu: *PerCpu, minimum_prio: u8, target: ?u32) ?*ke.Thread { - std.debug.assert(cpu.queues_lock.is_locked()); +fn pick_realtime_thread( + cpu: *PerCpu, + minimum_prio: u8, + target: ?u32, +) ?*ke.Thread { + assert(cpu.queues_lock.is_locked()); const runq = &cpu.realtime_queue; const minprio = to_sched_prio(minimum_prio); @@ -610,7 +621,7 @@ fn pick_realtime_thread(cpu: *PerCpu, minimum_prio: u8, target: ?u32) ?*ke.Threa const curr_queue = &runq.queues[i]; - std.debug.assert(!curr_queue.is_empty()); + assert(!curr_queue.is_empty()); // Get the first thread of the queue. var td: *ke.Thread = @fieldParentPtr( @@ -647,7 +658,7 @@ fn pick_realtime_thread(cpu: *PerCpu, minimum_prio: u8, target: ?u32) ?*ke.Threa } fn pick_batch_thread(cpu: *PerCpu, target: ?u32) ?*ke.Thread { - std.debug.assert(cpu.queues_lock.is_locked()); + assert(cpu.queues_lock.is_locked()); const runq = &cpu.calendar_queue; if (runq.status == 0) return null; @@ -669,7 +680,7 @@ fn pick_batch_thread(cpu: *PerCpu, target: ?u32) ?*ke.Thread { const curr_queue = &runq.queues[i]; - std.debug.assert(!curr_queue.is_empty()); + assert(!curr_queue.is_empty()); // Get the first thread of the queue. var td: *ke.Thread = @fieldParentPtr( @@ -710,7 +721,7 @@ fn pick_batch_thread(cpu: *PerCpu, target: ?u32) ?*ke.Thread { } fn pick_idle_thread(cpu: *PerCpu, target: ?u32) ?*ke.Thread { - std.debug.assert(cpu.queues_lock.is_locked()); + assert(cpu.queues_lock.is_locked()); if (cpu.idle_queue.is_empty()) return null; // Just get the first thread from the queue that can run. @@ -743,7 +754,7 @@ fn insert_in_list(list: *rtl.List, link: *rtl.List.Entry, head: bool) void { // Insert a thread into its respective queue. fn insert_in_queue(cpu: *PerCpu, td: *ke.Thread, preempted: bool) void { - std.debug.assert(cpu.queues_lock.is_locked()); + assert(cpu.queues_lock.is_locked()); td.cpu = null; @@ -766,7 +777,7 @@ fn insert_in_queue(cpu: *PerCpu, td: *ke.Thread, preempted: bool) void { // The insertion index is determined by insidx and the the priority of // the thread, higher priority threads will be put closer to insidx, // which ensures that they are ran more frequently in proportion. - var idx = (cpu.insidx + (@intFromEnum(ke.Priority.HighBatch) - + var idx = (cpu.insidx + (@backingInt(ke.Priority.HighBatch) - td.priority)) % runqueues_n; if (preempted) { @@ -800,8 +811,8 @@ fn insert_in_queue(cpu: *PerCpu, td: *ke.Thread, preempted: bool) void { // Remove a thread from a queue. fn remove_from_queue(cpu: *PerCpu, td: *ke.Thread) void { - std.debug.assert(td.lock.is_locked()); - std.debug.assert(cpu.queues_lock.is_locked()); + assert(td.lock.is_locked()); + assert(cpu.queues_lock.is_locked()); // Decrease load. _ = cpu.load.fetchSub(1, .monotonic); @@ -840,11 +851,11 @@ fn select_thread(cur: ?*ke.Thread, cpu: *PerCpu, target: ?u32) ?*ke.Thread { return td; } - // Nothing lower can preempt this thread. - if (cur != null and (cur.?.priority_class() == .Realtime or - cur.?.is_interactive())) - { - return null; + if (cur) |curtd| { + if (curtd.priority_class() == .Realtime or curtd.is_interactive()) { + // Nothing lower can preempt this thread. + return null; + } } // Then try picking a batch thread. @@ -856,9 +867,11 @@ fn select_thread(cur: ?*ke.Thread, cpu: *PerCpu, target: ?u32) ?*ke.Thread { return td; } - if (cur != null and cur.?.priority_class() == .Idle) { - // Idle threads don't preempt each other - return null; + if (cur) |curtd| { + if (curtd.priority_class() == .Idle) { + // Idle threads don't preempt each other. + return null; + } } // Finally try picking from the idle queue. @@ -890,7 +903,7 @@ fn should_prio_preempt(td: *ke.Thread, other_prio: u8, remote: bool) bool { } if (class == .Timeshare and td.is_interactive() and - other_prio < @intFromEnum(ke.Priority.LowInteractive) and remote) + other_prio < @backingInt(ke.Priority.LowInteractive) and remote) { // Interactive threads always preempt batch non-interactive ones // on remote CPUs. @@ -924,7 +937,9 @@ fn should_preempt_cpu(td: *ke.Thread, cpu: *PerCpu, remote: bool) bool { // Find a suitable CPU to run the thread. fn pick_cpu(td: *ke.Thread, slept: u64) u32 { // CPU selection policy in order: - // 1. Pick the last CPU the thread ran on if it would run immediately or if it's pinned. + // 1. Pick the last CPU the thread ran on if it would run immediately or + // if it's pinned. + // // 2. Pick the least loaded CPU on which the thread can run immediately. // 3. Pick the least loaded CPU overall. @@ -993,12 +1008,12 @@ fn pick_cpu(td: *ke.Thread, slept: u64) u32 { // If we found a preemptible CPU, pick that one. // Otherwise pick the least loaded one. - return least_preempt orelse least.?; + return least_preempt orelse least orelse unreachable; } // Enqueue a thread on a CPU. fn enqueue_on_cpu(c: u32, td: *ke.Thread) void { - std.debug.assert(td.lock.is_locked()); + assert(td.lock.is_locked()); const cpu = percpu.remote(c); cpu.queues_lock.acquire_no_ipl(); @@ -1097,7 +1112,7 @@ fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void { next.can_receive_stack(); if (next.continuation != null and !handoff) { - std.debug.assert(next.context.stack_top != 0); + assert(next.context.stack_top != 0); next.context.reset( next.context.stack_top, @@ -1107,7 +1122,7 @@ fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void { } if (next.context.stack_top == 0) { - std.debug.assert(handoff); + assert(handoff); } next.state.store(.Running, .monotonic); @@ -1138,7 +1153,7 @@ fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void { /// Handle preemption. /// This is called in DPC dispatch when it notices `next_thread` is set. pub fn handle_preemption(cpu: *PerCpu) void { - const cur = cpu.current_thread.?; + const cur = cpu.current_thread orelse unreachable; var next: *ke.Thread = undefined; while (true) { @@ -1157,7 +1172,7 @@ pub fn handle_preemption(cpu: *PerCpu) void { } if (next.context.stack_top == 0 and next.continuation != null) { - // This thread has no stack, try allocating one from the stack cache. + // This thread has no stack, try getting one from the stack cache. // Otherwise, the thread will be waiting for its stack to get // allocated and we pick another one in the meantime. const new_stack = allocate_stack(next); @@ -1170,14 +1185,18 @@ pub fn handle_preemption(cpu: *PerCpu) void { detach_load_avg(cpu, next); continue; } else if (next.continuation != null) { - next.context.reset(next.context.stack_top, ke.thread.call_continuation, next); + next.context.reset( + next.context.stack_top, + ke.thread.call_continuation, + next, + ); } // Found a thread with a stack. break; } - std.debug.assert(cur != next); + assert(cur != next); cpu.current_thread_prio.store(next.priority, .monotonic); @@ -1209,7 +1228,7 @@ pub fn handle_preemption(cpu: *PerCpu) void { /// Called every time slice in DPC context. pub fn clock(_: *ke.Dpc, _: ?*anyopaque) void { const cpu = percpu.local(); - std.debug.assert(kep.ipl.current() == .Dispatch); + assert(kep.ipl.current() == .Dispatch); const curtd = cpu.current_thread orelse return; @@ -1222,8 +1241,8 @@ pub fn clock(_: *ke.Dpc, _: ?*anyopaque) void { cpu.queues_lock.acquire_no_ipl(); defer cpu.queues_lock.release_no_ipl(); - // Advance the insert index every tick, while keeping a separation of 1 with runidx. - // This ensures fairness. + // Advance the insert index every tick, while keeping a separation of 1 + // with runidx. This ensures fairness. if (cpu.runidx == cpu.insidx) { cpu.insidx = (cpu.insidx + 1) % runqueues_n; @@ -1273,7 +1292,7 @@ pub fn enqueue(td: *ke.Thread) void { /// Yield on the current CPU. pub fn yield(continuation: ?ke.Continuation) void { const ipl = ke.ipl.raise(.Dispatch); - const td = percpu.local().current_thread.?; + const td = percpu.local().current_thread orelse unreachable; td.lock.acquire_no_ipl(); td.switching.store(true, .monotonic); @@ -1292,7 +1311,7 @@ pub fn yield(continuation: ?ke.Continuation) void { ke.ipl.lower(ipl); } -/// Yield the current CPU with the current thread already locked at Dispatch IPL. +/// Yield on the current CPU with the thread already locked at Dispatch IPL. pub fn yield_locked(continuation: ?ke.Continuation) void { const sched_cpu = percpu.local(); @@ -1321,38 +1340,38 @@ pub fn yield_locked(continuation: ?ke.Continuation) void { break; } - if (next == cur) { + const n = next orelse unreachable; + + if (n == cur) { // We just selected the same thread again. break; } - while (next.?.switching.load(.monotonic) == true) { + while (n.switching.load(.monotonic) == true) { std.atomic.spinLoopHint(); } - if (next) |n| { - const curstack = n.context.stack_top; + const curstack = n.context.stack_top; - const can_handoff = - cur.continuation != null and - cur.can_relinquish_stack() and - n.continuation != null and - n.context.stack_top == 0 and - n.can_receive_stack(); + const can_handoff = + cur.continuation != null and + cur.can_relinquish_stack() and + n.continuation != null and + n.context.stack_top == 0 and + n.can_receive_stack(); - if (curstack != 0 or can_handoff) { + if (curstack != 0 or can_handoff) { + break; + } + + if (curstack == 0) { + if (allocate_stack(n)) |new| { + n.context.stack_top = new; break; } - if (curstack == 0) { - if (allocate_stack(n)) |new| { - n.context.stack_top = new; - break; - } - - detach_load_avg(sched_cpu, n); - next = null; - } + detach_load_avg(sched_cpu, n); + next = null; } sched_cpu.queues_lock.acquire_no_ipl(); @@ -1385,7 +1404,7 @@ pub fn yield_locked(continuation: ?ke.Continuation) void { /// Block the currently running thread. pub fn block(cont: ?ke.Continuation) void { const ipl = ke.ipl.raise(.Dispatch); - const td = percpu.local().current_thread.?; + const td = ke.thread.current(); td.lock.acquire_no_ipl(); block_locked(td, cont); @@ -1424,8 +1443,8 @@ pub fn unblock(td: *ke.Thread) void { } pub fn unblock_locked(td: *ke.Thread) void { - std.debug.assert(td.lock.is_locked()); - std.debug.assert(td.state.load(.monotonic) == .Blocked); + assert(td.lock.is_locked()); + assert(td.state.load(.monotonic) == .Blocked); const now = ke.time.read_time(); @@ -1450,7 +1469,7 @@ pub fn unblock_locked(td: *ke.Thread) void { } pub fn update_priority_locked(td: *ke.Thread, new_prio: u8) void { - std.debug.assert(td.lock.is_locked()); + assert(td.lock.is_locked()); const old_prio = td.priority; @@ -1476,7 +1495,7 @@ pub fn update_priority_locked(td: *ke.Thread, new_prio: u8) void { } if (state == .Running) { - std.debug.assert(td.last_cpu == c); + assert(td.last_cpu == c); cpu.current_thread_prio.store(new_prio, .monotonic); @@ -1625,12 +1644,18 @@ fn find_least_loaded(exclude: ?*ke.CpuMask) ?u32 { return least; } -fn steal_thread_from_cpu(cpu: *PerCpu, curcpu: ?*PerCpu, target: ?u32) ?*ke.Thread { +fn steal_thread_from_cpu( + cpu: *PerCpu, + curcpu: ?*PerCpu, + target: ?u32, +) ?*ke.Thread { cpu.queues_lock.acquire_no_ipl(); defer cpu.queues_lock.release_no_ipl(); - if (curcpu != null and curcpu.?.load.load(.monotonic) > 0) - return null; + if (curcpu) |c| { + if (c.load.load(.monotonic) > 0) + return null; + } const td = select_thread(null, cpu, target); diff --git a/src/kern/shootdown.zig b/src/kern/shootdown.zig index 3a247e4..dd65cd8 100644 --- a/src/kern/shootdown.zig +++ b/src/kern/shootdown.zig @@ -1,4 +1,5 @@ -//! Kernel-side mechanism for asynchronous TLB shootdown and deferred memory reclamation. +//! Kernel-side mechanism for asynchronous TLB shootdowns and deferred +//! memory reclamation. const std = @import("std"); const config = @import("config"); @@ -32,8 +33,9 @@ const PerCpu = struct { /// States attached to this CPU. states: [64]ShootdownState, /// States relevant to each CPU. - /// If CPU0 sent a shootdown to CPU1, then CPU0 will have `valid_states[CPU1]` - /// pointing to that shootdown, which will sit in `states`. + /// If CPU0 sent a shootdown to CPU1, then CPU0 will have + /// `valid_states[CPU1]` pointing to that shootdown, + /// which will sit in `states`. valid_states: [config.ncpus]std.atomic.Value(u64), /// CPUs that have sent this CPU a shootdown. senders: ke.AtomicCpuMask, @@ -52,17 +54,17 @@ pub var shootdowns: rtl.HandoffList = undefined; fn pcpu_init() linksection(r.init) void { const local = percpu.local(); - local.valid_states = @splat(.init(0)); - local.senders = .init(false); + local.valid_states = @splat(std.atomic.Value(u64).init(0)); + local.senders = ke.AtomicCpuMask.init(false); local.states = @splat(.{ .base = 0, .npages = 0, - .state = .init(slot_free), + .state = std.atomic.Value(u16).init(slot_free), .payload = undefined, .link = undefined, }); - local.npages = .init(0); + local.npages = std.atomic.Value(u32).init(0); } comptime { @@ -178,10 +180,6 @@ pub fn process_shootdowns() void { } } -fn process_shootdowns_kick(_: u32, _: ?*anyopaque) void { - process_shootdowns(); -} - /// Submit a shootdown to occur asynchronously on `target_mask`. /// Returns an error if a synchronous shootdown was done instead. pub fn submit(state: ShootdownState, target_mask: ke.CpuMask) !void { diff --git a/src/kern/smr.zig b/src/kern/smr.zig index 3bf4abe..e2a68b9 100644 --- a/src/kern/smr.zig +++ b/src/kern/smr.zig @@ -1,14 +1,16 @@ //! Safe memory reclamation implementation. //! based on FreeBSD's "Global Unbounded Sequences". -const r = @import("root"); const rtl = @import("rtl"); -const std = @import("std"); const config = @import("config"); +const r = @import("root"); const ke = r.ke; const kep = ke.private; +const std = @import("std"); +const assert = std.debug.assert; + pub const Sequence = u64; pub const seq_invalid: Sequence = 0; @@ -34,15 +36,15 @@ pub const Domain = struct { } pub fn init(self: *Domain, preempt: bool) void { - self.clock.read_seq = .init(seq_init); - self.clock.write_seq = .init(seq_init); + self.clock.read_seq = std.atomic.Value(Sequence).init(seq_init); + self.clock.write_seq = std.atomic.Value(Sequence).init(seq_init); self.preempt = preempt; for (0..ke.ncpus) |i| { self.cpus[i] = .{ - .current_seq = .init(seq_invalid), + .current_seq = std.atomic.Value(Sequence).init(seq_invalid), .stall_lock = ke.SpinLock.init("stall_lock"), - .stall_seq = .init(seq_invalid), + .stall_seq = std.atomic.Value(Sequence).init(seq_invalid), .stalled = undefined, .stall_goal = seq_invalid, }; @@ -74,12 +76,17 @@ pub const Tracker = struct { }; /// Used by tests. -pub var full_scans: std.atomic.Value(usize) = .init(0); +pub var full_scans = std.atomic.Value(usize).init(0); /// Scan all CPUs and return the minimum observed value. /// If `should_wait` is true, this will spinloop (or block) /// until all CPUs have reached the given goal. -fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence { +fn scan( + dom: *Domain, + goal: Sequence, + clock: Clock, + should_wait: bool, +) Sequence { rtl.barrier.fence(.seq_cst); const clk_write = clock.write_seq.raw; @@ -92,7 +99,6 @@ fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence for (0..ke.ncpus) |i| { const cpu = &dom.cpus[i]; - var seq = cpu.current_seq.load(.monotonic); while (seq != seq_invalid) { @@ -169,8 +175,8 @@ fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence .{ .shared = &cpu.stalled }, .Exclusive, ); - ke.ipl.lower(ipl); + ke.ipl.lower(ipl); seq = cpu.stall_seq.load(.monotonic); } @@ -260,7 +266,9 @@ pub fn advance(dom: *Domain) Sequence { return dom.clock.write_seq.fetchAdd(seq_incr, .release) + seq_incr; } -/// Pretend-advance the write sequence and return the value for use as a wait goal. +/// Pretend-advance the write sequence and return the value for use as a wait +/// goal. +/// /// It is guaranteed that all previous memory writes made by /// the calling thread are visible. /// The global clock isn't advanced unlike `advance`, it is only when commit() @@ -303,7 +311,7 @@ fn enter_internal(dom: *Domain) Sequence { // on a full memory barrier, as it is guaranteed that the CPU was treated as // inactive and cannot possibly hold a reference to anything the poll // declared reclaimable. See scan(). - std.debug.assert(cpu.current_seq.load(.monotonic) == seq_invalid); + assert(cpu.current_seq.load(.monotonic) == seq_invalid); const wr_seq = dom.clock.write_seq.load(.monotonic); @@ -420,7 +428,7 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void { return; } - const cpu = tracker.cpu.?; + const cpu = tracker.cpu orelse unreachable; // We got preempted. cpu.stall_lock.acquire_no_ipl(); @@ -449,7 +457,7 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void { } if (wake and turnstile != null) { - const ts = turnstile.?; + const ts = turnstile orelse unreachable; var waiters: rtl.List = undefined; kep.turnstile.signal(ts, .Exclusive, ts.waiters, null, &waiters); kep.turnstile.exit(cpu, turnstile); diff --git a/src/kern/spinlock.zig b/src/kern/spinlock.zig index 39e5396..3ca2dfd 100644 --- a/src/kern/spinlock.zig +++ b/src/kern/spinlock.zig @@ -1,3 +1,5 @@ +//! Kernel spinlocks. + const std = @import("std"); const rtl = @import("rtl"); const config = @import("config"); @@ -8,19 +10,23 @@ const kep = ke.private; fn LockTemplate(comptime T: type) type { comptime { - if (!@hasDecl(T, "acquire_no_ipl")) @compileError("lock backend must implement acquire_no_ipl"); - if (!@hasDecl(T, "release_no_ipl")) @compileError("lock backend must implement release_no_ipl"); - if (!@hasDecl(T, "try_acquire_no_ipl")) @compileError("lock backend must implement try_acquire_no_ipl"); - if (!@hasDecl(T, "is_locked")) @compileError("lock backend must implement is_locked"); + if (!@hasDecl(T, "acquire_no_ipl")) + @compileError("lock backend must implement acquire_no_ipl"); + if (!@hasDecl(T, "release_no_ipl")) + @compileError("lock backend must implement release_no_ipl"); + if (!@hasDecl(T, "try_acquire_no_ipl")) + @compileError("lock backend must implement try_acquire_no_ipl"); + if (!@hasDecl(T, "is_locked")) + @compileError("lock backend must implement is_locked"); } return struct { inner: T, wd: kep.warden.LockData = undefined, - const Self = @This(); + const Lock = @This(); - pub fn init(comptime class: []const u8) Self { + pub fn init(comptime class: []const u8) Lock { if (config.warden) { const cl = kep.warden.find_or_create_lock_class(class); return .{ @@ -32,26 +38,28 @@ fn LockTemplate(comptime T: type) type { } /// Acquire the lock, raising IPL to `ipl`. Returns the previous IPL. - pub fn acquire_at(self: *Self, ipl: ke.Ipl) ke.Ipl { + pub fn acquire_at(self: *Lock, ipl: ke.Ipl) ke.Ipl { const old_ipl = ke.ipl.raise(ipl); self.acquire_no_ipl(); return old_ipl; } - /// Acquire the lock, raising IPL to `.Dispatch`. Returns the previous IPL. - pub fn acquire(self: *Self) ke.Ipl { + /// Acquire the lock, raising IPL to `.Dispatch`. + /// Returns the previous IPL. + pub fn acquire(self: *Lock) ke.Ipl { return self.acquire_at(.Dispatch); } /// Release the lock and restore IPL to `ipl`. - pub fn release(self: *Self, ipl: ke.Ipl) void { + pub fn release(self: *Lock, ipl: ke.Ipl) void { self.release_no_ipl(); ke.ipl.lower(ipl); } /// Try to acquire the lock at IPL `.Dispatch`. - /// Returns the previous IPL on success, null if the lock is already held. - pub fn try_acquire(self: *Self) ?ke.Ipl { + /// Returns the previous IPL on success, + /// null if the lock is already held. + pub fn try_acquire(self: *Lock) ?ke.Ipl { const old_ipl = ke.ipl.raise(.Dispatch); if (self.inner.try_acquire_no_ipl()) { if (config.warden) { @@ -66,7 +74,7 @@ fn LockTemplate(comptime T: type) type { } /// Acquire the lock without changing the IPL. - pub fn acquire_no_ipl(self: *Self) void { + pub fn acquire_no_ipl(self: *Lock) void { if (config.warden) { kep.warden.check(self.wd, .Spin); } @@ -79,7 +87,7 @@ fn LockTemplate(comptime T: type) type { } /// Release the lock without changing the IPL. - pub fn release_no_ipl(self: *Self) void { + pub fn release_no_ipl(self: *Lock) void { if (config.warden) { kep.warden.released(self, .Spin); } @@ -89,7 +97,7 @@ fn LockTemplate(comptime T: type) type { /// Try to acquire the lock without changing the IPL. /// Returns true if the lock was acquired. - pub fn try_acquire_no_ipl(self: *Self) bool { + pub fn try_acquire_no_ipl(self: *Lock) bool { const ret = self.inner.try_acquire_no_ipl(); if (ret and config.warden) { @@ -100,7 +108,7 @@ fn LockTemplate(comptime T: type) type { } /// Returns true if the lock is currently held. - pub fn is_locked(self: *Self) bool { + pub fn is_locked(self: *Lock) bool { return self.inner.is_locked(); } }; @@ -110,16 +118,16 @@ fn LockTemplate(comptime T: type) type { pub const SpinLock = LockTemplate(struct { locked: std.atomic.Value(u8), - const Self = @This(); + const Lock = @This(); - pub fn init() Self { + pub fn init() Lock { return .{ - .locked = .init(0), + .locked = std.atomic.Value(u8).init(0), }; } /// Acquire the lock without changing the IPL. - pub fn acquire_no_ipl(self: *Self) void { + pub fn acquire_no_ipl(self: *Lock) void { while (true) { if (self.locked.cmpxchgWeak(0, 1, .acquire, .monotonic) == null) return; @@ -131,16 +139,16 @@ pub const SpinLock = LockTemplate(struct { } /// Release the lock without changing the IPL. - pub fn release_no_ipl(self: *Self) void { + pub fn release_no_ipl(self: *Lock) void { self.locked.store(0, .release); } /// Try to acquire the lock. Return true if lock was acquired. - pub fn try_acquire_no_ipl(self: *Self) bool { + pub fn try_acquire_no_ipl(self: *Lock) bool { return self.locked.cmpxchgStrong(0, 1, .acquire, .monotonic) == null; } - pub fn is_locked(self: *Self) bool { + pub fn is_locked(self: *Lock) bool { return self.locked.load(.monotonic) == 1; } }); @@ -199,15 +207,15 @@ pub const QSpinLock = LockTemplate(struct { const pending_val: u32 = 0x00000100; const pending_loops = 512; - const Self = @This(); + const Lock = @This(); data: Data, - pub fn init() Self { + pub fn init() Lock { return .{ .data = .{ .val = std.atomic.Value(u32).init(0) } }; } - pub fn acquire_no_ipl(self: *Self) void { + pub fn acquire_no_ipl(self: *Lock) void { // Fast path: try to acquire the lock. var v = self.data.val.cmpxchgStrong( 0, @@ -262,11 +270,11 @@ pub const QSpinLock = LockTemplate(struct { self.data.split.locked_pending.store(1, .release); } - pub fn release_no_ipl(self: *Self) void { + pub fn release_no_ipl(self: *Lock) void { self.data.low_word.locked.store(0, .release); } - pub fn try_acquire_no_ipl(self: *Self) bool { + pub fn try_acquire_no_ipl(self: *Lock) bool { return self.data.val.cmpxchgStrong( 0, locked_val, @@ -275,11 +283,11 @@ pub const QSpinLock = LockTemplate(struct { ) == null; } - pub fn is_locked(self: *Self) bool { + pub fn is_locked(self: *Lock) bool { return self.data.low_word.locked.load(.monotonic) != 0; } - fn queue(self: *Self) void { + fn queue(self: *Lock) void { // Claim an index from our CPU. const idx = pcpu.local().curr_idx; pcpu.local().curr_idx += 1; @@ -369,6 +377,8 @@ pub const QSpinLock = LockTemplate(struct { std.atomic.spinLoopHint(); } - next.?.locked.store(1, .release); + const n = next orelse unreachable; + + n.locked.store(1, .release); } }); diff --git a/src/kern/thread.zig b/src/kern/thread.zig index 05bd620..2e974ee 100644 --- a/src/kern/thread.zig +++ b/src/kern/thread.zig @@ -1,4 +1,5 @@ -const std = @import("std"); +//! Kernel thread management. + const rtl = @import("rtl"); const config = @import("config"); @@ -6,9 +7,10 @@ const r = @import("root"); const ke = r.ke; const kep = ke.private; -pub const Priority = enum(u8) { - const Self = @This(); +const std = @import("std"); +const assert = std.debug.assert; +pub const Priority = enum(u8) { /// Reserved for the idle thread. IdleThread = 0, /// Background kernel work. @@ -42,18 +44,18 @@ pub const Priority = enum(u8) { // The low and top 20 priorities from the batch range are reserved // for nice. - pub const cpu_range = @intFromEnum(Self.HighBatch) - (nice_max * 2) - 1; + pub const cpu_range = @backingInt(Priority.HighBatch) - (nice_max * 2) - 1; pub fn class_from_prio(prio: u8) Class { - if (prio >= @intFromEnum(Self.LowRealtime) and - prio <= @intFromEnum(Self.HighRealtime)) + if (prio >= @backingInt(Priority.LowRealtime) and + prio <= @backingInt(Priority.HighRealtime)) return .Realtime; - if (prio >= @intFromEnum(Self.LowBatch) and - prio < @intFromEnum(Self.LowRealtime)) + if (prio >= @backingInt(Priority.LowBatch) and + prio < @backingInt(Priority.LowRealtime)) return .Timeshare; - if (prio <= @intFromEnum(Self.Idle)) + if (prio <= @backingInt(Priority.Idle)) return .Idle; unreachable; @@ -110,7 +112,7 @@ pub const Thread = struct { lock: ke.SpinLock, /// Niceness value. nice: i8, - /// Effective priority value of the thread: `max(base_priority, inherited_prio)`. + /// Effective priority value of the thread: `max(base, inherited)`. priority: u8, /// Base priority value of the thread. base_priority: u8, @@ -188,17 +190,17 @@ pub const Thread = struct { .context = .init_with_stack(opts.stack), .lock = ke.SpinLock.init("thread"), .nice = 0, - .priority = @intFromEnum(opts.priority), - .base_priority = @intFromEnum(opts.priority), + .priority = @backingInt(opts.priority), + .base_priority = @backingInt(opts.priority), .inherited_prio = 0, .pinned = false, - .state = .init(.Ready), + .state = std.atomic.Value(State).init(.Ready), .runq_link = .{}, .last_cpu = null, .cpu = null, .runq = null, .runq_idx = 0, - .wait_status = .init(.Satisfied), + .wait_status = std.atomic.Value(kep.wait.Status).init(.Satisfied), .inner_waitblocks = undefined, .waitblocks = &thread.inner_waitblocks, .wait_reason = null, @@ -213,11 +215,11 @@ pub const Thread = struct { .queue = null, .queue_item = null, .continuation = opts.entry, - .switching = .init(false), + .switching = std.atomic.Value(bool).init(false), .avg = .{}, .acct = .{}, .smr_sections = undefined, - .hard_affinity = .init(true), + .hard_affinity = ke.CpuMask.init(true), .warden_data = .{}, }; @@ -249,8 +251,8 @@ pub const Thread = struct { } pub fn is_interactive(self: *Thread) bool { - return self.priority >= @intFromEnum(Priority.LowInteractive) and - self.priority <= @intFromEnum(Priority.HighInteractive); + return self.priority >= @backingInt(Priority.LowInteractive) and + self.priority <= @backingInt(Priority.HighInteractive); } pub fn can_relinquish_stack(self: *Thread) bool { @@ -309,7 +311,7 @@ pub fn exit() void { /// Return the currently running thread. pub fn current() *ke.Thread { - return kep.sched.percpu.local().current_thread.?; + return kep.sched.percpu.local().current_thread orelse unreachable; } // -- Stack allocation and continuations --------------------------------------- @@ -344,7 +346,8 @@ const Cpu = struct { /// Number of excess stacks allowed in the global stack depot before trimming. /// Note that this is in addition to the global minimum of `ncpus` stacks. -/// Keep this low for minimal memory overhead, but potentially increased latency. +/// Keep this low for minimal memory overhead, but potentially +/// increased latency. const excess_stacks = ke.Tunable(u32, 1, "ke.thread.excess_stacks"); var global_depot_lock: ke.SpinLock = undefined; @@ -355,7 +358,7 @@ const percpu_count_max = 2; const wma_unit = 256; -pub var thread_stack_queue: rtl.HandoffList = .init(stack_activation); +pub var thread_stack_queue = rtl.HandoffList.init(stack_activation); inline fn wma_mix(old: u32, new: u32) u32 { // Keep 75% of old sample and 25% of new. @@ -380,14 +383,14 @@ pub fn init() void { pub fn call_continuation(ptr: ?*anyopaque) noreturn { const td: *ke.Thread = @ptrCast(@alignCast(ptr)); - std.debug.assert(td.continuation != null); + assert(td.continuation != null); // Clean things that were set up by the wait (e.g timeout). _ = kep.wait.post_wait(td) catch { // Don't care about the status. }; - const cont = td.continuation.?; + const cont = td.continuation orelse unreachable; td.continuation = null; kep.impl.call_continuation(&td.context, cont); @@ -466,7 +469,7 @@ pub fn stack_cache_update() ?*Stack { // If we have more than `excess_stacks` sitting unused, then trim. // Ensure there are still `ncpus` stacks sitting in the depot. if (excess > excess_stacks.load()) { - std.debug.assert(global_depot.count >= excess); + assert(global_depot.count >= excess); // We want every CPU to have at least one stack sitting in the depot. const floor: u32 = @truncate(ke.ncpus); diff --git a/src/kern/time.zig b/src/kern/time.zig index c4f3f29..fbc12d3 100644 --- a/src/kern/time.zig +++ b/src/kern/time.zig @@ -43,9 +43,6 @@ var state = rtl.SeqLock(State).init(.{ }); // Convert a TimeCounter's count to nanoseconds. -// For 64-bit counters (`mask == maxInt(u64)`), uses a fast multiply-shift path -// via precomputed `p` and `n` to avoid 128-bit division at runtime. -// Falls back to a division-based conversion for narrower counters. fn ticks_to_ns(tc: *TimeCounter, count: u64) u64 { if (tc.p != 0) { const val: u128 = @as(u128, @intCast(count)) * tc.n; @@ -132,8 +129,8 @@ pub fn update_overflow() void { const cur_count = tc.read_count() & tc.mask; if (s.initial_count > cur_count) { - // Counter has wrapped, use wrapping arithmetic to figure out by how much, - // then fold it into the offset and reset `initial_count`. + // Counter has wrapped, use wrapping arithmetic to figure out by how + // much, then fold it into the offset and reset `initial_count`. const overflowed_by = (cur_count -% s.initial_count) & tc.mask; state.store(.{ .initial_count = cur_count, diff --git a/src/kern/timer.zig b/src/kern/timer.zig index 9064678..9d7e753 100644 --- a/src/kern/timer.zig +++ b/src/kern/timer.zig @@ -10,9 +10,11 @@ const pl = r.pl; const std = @import("std"); const assert = std.debug.assert; +const TimerHeap = rtl.PairingHeap(.Min, kep.timer.cmp_timer); + const PerCpu = struct { /// Heap of pending timers on this CPU. - timers: rtl.PairingHeap(.min, kep.timer.cmp_timer), + timers: TimerHeap, /// Lock over the timer heap. lock: ke.SpinLock, dpc: ke.Dpc, @@ -44,8 +46,8 @@ pub const Timer = struct { pub fn init(self: *Timer) void { self.* = .{ .hdr = undefined, - .state = .init(.Stopped), - .deadline = .init(0), + .state = std.atomic.Value(State).init(.Stopped), + .deadline = r.Nanoseconds.init(0), .dpc = null, .node = .{}, .cpu = null, @@ -56,7 +58,7 @@ pub const Timer = struct { }; const percpu = ke.CpuLocal(PerCpu, .{ - .timers = .init(), + .timers = TimerHeap.init(), .lock = undefined, .dpc = .init(handle_expiry), }); @@ -127,7 +129,7 @@ pub fn cancel(timer: *Timer) void { } }, .Pending => { - const cpu = timer.cpu.?; + const cpu = timer.cpu orelse unreachable; cpu.lock.acquire_no_ipl(); // Re-check under the lock. @@ -170,7 +172,7 @@ pub fn clock() void { // Called in a DPC when a timer has expired. fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void { - std.debug.assert(kep.ipl.current() == .Dispatch); + assert(kep.ipl.current() == .Dispatch); const cpu = percpu.local(); while (true) { diff --git a/src/kern/tunable.zig b/src/kern/tunable.zig index 4695f06..6d5c1d3 100644 --- a/src/kern/tunable.zig +++ b/src/kern/tunable.zig @@ -53,12 +53,12 @@ fn store_int(comptime T: type, addr: *anyopaque, value: anytype) bool { } fn apply_num( - comptime ValueT: type, + comptime VT: type, cmdline: []const u8, entry: *const Entry, info: std.builtin.Type.Int, ) void { - const value = rtl.cmdline.get_number(ValueT, cmdline, entry.name) catch |e| { + const value = rtl.cmdline.get_number(VT, cmdline, entry.name) catch |e| { if (e == error.Format) { std.log.warn( "Invalid number value for '{s}', falling back.", @@ -68,7 +68,7 @@ fn apply_num( return; }; - const sign = @typeInfo(ValueT).int.signedness; + const sign = @typeInfo(VT).int.signedness; _ = switch (info.bits) { 8 => store_int(@Int(sign, 8), entry.address, value), diff --git a/src/kern/turnstile.zig b/src/kern/turnstile.zig index 71d1043..27fe26b 100644 --- a/src/kern/turnstile.zig +++ b/src/kern/turnstile.zig @@ -22,9 +22,10 @@ //! a lock, which might get preempted by another medium-priority thread. //! //! Priority inheritance is achieved through turnstiles by walking a -//! given turnstile's owner chain and willing the priority of the highest priority -//! waiter, i.e it will go through turnstile.owner.turnstile.owner... -//! until the end of the chain, inheriting the waiter's priority along the way. +//! given turnstile's owner chain and willing the priority of the +//! highest priority waiter, i.e it will go through +//! turnstile.owner.turnstile.owner... until the end of the chain, +//! inheriting the waiter's priority along the way. //! //! ## Locking //! ---------- @@ -41,15 +42,17 @@ //! //! ## Resources //! ------------ -//! See the original Illumos implementation, the Solaris book or FreeBSD book for -//! a more in-depth description. +//! See the original Illumos implementation, the Solaris book or FreeBSD book +//! for a more in-depth description. //! I have also written more about turnstiles here: -//! https://rdmsr.github.io/writing/turnstiles/. +//! https://gregoire.dev/writing/turnstiles/. const ke = @import("root").ke; const kep = ke.private; const rtl = @import("rtl"); + const std = @import("std"); +const assert = std.debug.assert; const num_chains = 128; const hash_mask = num_chains - 1; @@ -229,7 +232,8 @@ fn update_prio(td: *ke.Thread) void { /// `to.lock` and the owning turnstile's lock. fn donate_to(boost: *Boost, to: *ke.Thread, pri: u8) void { // A lend only ever raises. - if (boost.donated != null and pri <= boost.donated.?) return; + if (boost.donated != null and pri <= boost.donated orelse unreachable) + return; if (boost.donated == null) to.turnstiles_owned.insert_head(&boost.link); boost.donated = pri; @@ -306,7 +310,7 @@ fn propagate(curtd: *ke.Thread) void { // We hold `thread.lock` and `root.lock`. const obj = thread.waiting_on orelse break; - std.debug.assert(thread.waiting_on == obj); + assert(thread.waiting_on == obj); const ts = thread.turnstile; @@ -319,8 +323,8 @@ fn propagate(curtd: *ke.Thread) void { // on the same turnstile. if (!ts.lock.try_acquire_no_ipl()) { // The turnstile could not be acquired, drop everything and - // restart the walk from the start. Priority inheritance is idem- - // potent so there is no issue in applying it many times. + // restart the walk from the start. Priority inheritance is + // idempotent so there is no issue in applying it many times. thread.lock.release_no_ipl(); if (root) |ro| ro.lock.release_no_ipl(); @@ -443,9 +447,9 @@ pub fn block( const ipl = ke.ipl.current(); var ts = turnstile; - const queue_idx = @intFromEnum(queue); + const queue_idx = @backingInt(queue); - std.debug.assert(chain.lock.is_locked()); + assert(chain.lock.is_locked()); if (ts) |turn| { // Another thread already donated its turnstile, @@ -500,7 +504,8 @@ pub fn block( _ = ke.ipl.raise(ipl); } -/// Signal the end of a turnstile-backed wait and return a list of waiters to wake. +/// Signal the end of a turnstile-backed wait and return a list of waiters +/// to wake. /// Do hand-off to new_owner if specified. /// The locks acquired by `lookup` are still held on return. pub fn signal( @@ -510,7 +515,7 @@ pub fn signal( new_owner: ?*ke.Thread, waiters: *rtl.List, ) void { - const queue_idx = @intFromEnum(queue); + const queue_idx = @backingInt(queue); // Revoke any priority we gave to the owner(s). revoke(ts); @@ -546,8 +551,9 @@ pub fn signal( pub fn wakeup(waiters: *rtl.List) void { // Now wake all the waiters, this ensures that the turnstile and chain lock // hold times stay low. - // Also, if we did this *before* unlocking `ts`, there is no guarantee that it - // is still alive, as the last waiter could've woken up, exited, and freed it. + // Also, if we did this *before* unlocking `ts`, there is no guarantee that + // it is still alive, as the last waiter could've woken up, exited, + // and freed it. // Instead of special casing it, let's just do the wakeup in a nicer // environment here. var it = waiters.iterator(); @@ -562,9 +568,9 @@ pub fn wakeup(waiters: *rtl.List) void { fn dequeue(ts: *Turnstile, td: *ke.Thread) *Waiter { td.lock.acquire_no_ipl(); - std.debug.assert(td.turnstile == ts); - std.debug.assert(td.turnstile_waiter != null); - std.debug.assert(!ts.queues[0].is_empty() or !ts.queues[1].is_empty()); + assert(td.turnstile == ts); + assert(td.turnstile_waiter != null); + assert(!ts.queues[0].is_empty() or !ts.queues[1].is_empty()); const waiter = td.turnstile_waiter.?; waiter.link.remove(); diff --git a/src/kern/wait.zig b/src/kern/wait.zig index a94651a..74a3ec9 100644 --- a/src/kern/wait.zig +++ b/src/kern/wait.zig @@ -133,7 +133,7 @@ pub fn wait_any( const ipl = ke.ipl.raise(.Dispatch); defer ke.ipl.lower(ipl); - const curtd = kep.sched.percpu.local().current_thread.?; + const curtd = ke.thread.current(); const obj_count = objects.len; const has_timeout = opts.timeout != null; const total_count = obj_count + @intFromBool(has_timeout); @@ -264,7 +264,7 @@ pub fn wait_any( /// Satisfy a wait on an object. pub fn satisfy_wait(obj: *DispatchHeader) void { - std.debug.assert(obj.lock.is_locked()); + assert(obj.lock.is_locked()); const all = obj.type == .Notification; @@ -274,8 +274,8 @@ pub fn satisfy_wait(obj: *DispatchHeader) void { var wb: *WaitBlock = @fieldParentPtr("link", obj.waitblocks.first()); var td = wb.thread; - std.debug.assert(wb.status == .Active); - std.debug.assert(wb.object == obj); + assert(wb.status == .Active); + assert(wb.object == obj); // Remove it. wb.link.remove(); diff --git a/src/kern/warden.zig b/src/kern/warden.zig index 92521cc..a107347 100644 --- a/src/kern/warden.zig +++ b/src/kern/warden.zig @@ -124,20 +124,24 @@ pub fn find_or_create_lock_class(name: []const u8) ?*Class { const ipl = graph_lock.acquire_at(.High); defer graph_lock.release(ipl); - var class, const bucket = find_lock_class(name); + const found, const bucket = find_lock_class(name); - if (class != null) { - return class; + if (found != null) { + return found; } assert(num_active_classes < num_classes); // Get a new spot for our class. - class = &classes[num_active_classes]; - class.?.name = name; + const class = &classes[num_active_classes]; + class.name = name; // Insert it so that we can find it by name. - bucket.?.insert(&class.?.entry) catch unreachable; + bucket.?.insert(&class.entry) catch { + // This would only fail if the lock class already existed. + // Since this was accounted for earlier, the error can never happen. + unreachable; + }; num_active_classes += 1; diff --git a/src/loader/amd64/impl.zig b/src/loader/amd64/impl.zig index 10ad370..6757643 100644 --- a/src/loader/amd64/impl.zig +++ b/src/loader/amd64/impl.zig @@ -1,3 +1,5 @@ +//! amd64-specific loader code. + const amd64 = @import("arch"); const r = @import("root"); const pmap = @import("../pmap.zig"); @@ -75,7 +77,11 @@ pub inline fn make_table_pte(pa: usize) Pte { }; } -pub inline fn make_leaf_pte(pa: usize, flags: r.mem.MapFlags, level: usize) Pte { +pub inline fn make_leaf_pte( + pa: usize, + flags: r.mem.MapFlags, + level: usize, +) Pte { return Pte{ .present = true, .writable = flags.write, diff --git a/src/loader/limine/impl.zig b/src/loader/limine/impl.zig index e9f5f47..d2e52e6 100644 --- a/src/loader/limine/impl.zig +++ b/src/loader/limine/impl.zig @@ -1,56 +1,70 @@ +//! Limine-specific code for the loader. + const std = @import("std"); const limine = @import("limine.zig"); const r = @import("root"); -pub export var limine_base_revision: [3]u64 linksection(".limine_requests") = limine.base_revision(6); +pub export var base_revision: [3]u64 linksection(".limine_requests") = + limine.base_revision(6); -pub export var framebuffer_request: limine.FramebufferRequest linksection(".limine_requests") = .{ - .id = limine.framebuffer_request_id, - .revision = 0, - .response = null, -}; +pub export var framebuffer_request linksection(".limine_requests") = + limine.FramebufferRequest{ + .id = limine.framebuffer_request_id, + .revision = 0, + .response = null, + }; -pub export var limine_requests_start_marker: [4]u64 linksection(".limine_requests_start") = limine.requests_start_marker; -pub export var limine_requests_end_marker: [2]u64 linksection(".limine_requests_end") = limine.requests_end_marker; +pub export var start_marker: [4]u64 linksection(".limine_requests_start") = + limine.requests_start_marker; -pub export var memmap_request: limine.MemmapRequest linksection(".limine_requests") = .{ - .id = limine.memmap_request_id, - .revision = 0, - .response = null, -}; +pub export var end_marker: [2]u64 linksection(".limine_requests_end") = + limine.requests_end_marker; -pub export var rsdp_request: limine.RsdpRequest linksection(".limine_requests") = .{ - .id = limine.rsdp_request_id, - .revision = 0, - .response = null, -}; +pub export var memmap_request linksection(".limine_requests") = + limine.MemmapRequest{ + .id = limine.memmap_request_id, + .revision = 0, + .response = null, + }; -pub export var cmdline_request: limine.CmdlineRequest linksection(".limine_requests") = .{ - .id = limine.cmdline_request_id, - .revision = 0, - .response = null, -}; +pub export var rsdp_request linksection(".limine_requests") = + limine.RsdpRequest{ + .id = limine.rsdp_request_id, + .revision = 0, + .response = null, + }; -pub export var hhdm_request: limine.HHDMRequest linksection(".limine_requests") = .{ - .id = limine.hhdm_request_id, - .revision = 0, - .response = null, -}; +pub export var cmdline_request linksection(".limine_requests") = + limine.CmdlineRequest{ + .id = limine.cmdline_request_id, + .revision = 0, + .response = null, + }; -pub export var kernel_request: limine.ExecutableAddressRequest linksection(".limine_requests") = .{ - .id = limine.executable_address_request_id, - .revision = 0, - .response = null, -}; +pub export var hhdm_request linksection(".limine_requests") = + limine.HHDMRequest{ + .id = limine.hhdm_request_id, + .revision = 0, + .response = null, + }; -pub export var module_request: limine.ModuleRequest linksection(".limine_requests") = .{ - .id = limine.module_request_id, - .revision = 0, - .response = null, -}; +pub export var kernel_request linksection(".limine_requests") = + limine.ExecutableAddressRequest{ + .id = limine.executable_address_request_id, + .revision = 0, + .response = null, + }; +pub export var module_request linksection(".limine_requests") = + limine.ModuleRequest{ + .id = limine.module_request_id, + .revision = 0, + .response = null, + }; + +var hhdm_offset: usize = 0; pub fn p2v(pa: usize) usize { - return hhdm_request.response.?.offset + pa; + return hhdm_offset + pa; } pub fn get_image_layout() r.ImageLayout { @@ -64,36 +78,12 @@ pub fn get_image_layout() r.ImageLayout { var cmdline: [256]u8 = undefined; -export fn loader_entry() callconv(.c) void { - var kernel: ?*anyopaque = null; - var module_start: usize = 0; - var module_end: usize = 0; - - if (module_request.response) |resp| { - const modules = resp.modules.?[0..resp.module_count]; - - for (modules) |mod| { - if (std.mem.eql(u8, std.mem.span(mod.string), "kernel")) { - kernel = mod.address; - const phys = @intFromPtr(mod.address) - - hhdm_request.response.?.offset; - const size: usize = @intCast(mod.size); - - module_start = std.mem.alignForward(usize, phys, r.page_size); - module_end = std.mem.alignBackward(usize, phys + size, r.page_size); - break; - } - } - } - - if (kernel == null) { - @panic("loader: kernel not found"); - } - - const mmap = memmap_request.response.?; +fn build_mmap(module_start: usize, module_end: usize) void { + const mmap = memmap_request.response orelse unreachable; + const entries = mmap.entries orelse unreachable; for (0..mmap.entry_count) |i| { - const entry = mmap.entries.?[i]; + const entry = entries[i]; if (entry.type == .KernelAndModules) { const end = entry.base + entry.length; @@ -112,10 +102,18 @@ export fn loader_entry() callconv(.c) void { // anyway. if (reclaim_start < reclaim_end) { if (entry.base < reclaim_start) { - r.mem.add_entry(entry.base, reclaim_start - entry.base, .Reserved); + r.mem.add_entry( + entry.base, + reclaim_start - entry.base, + .Reserved, + ); } - r.mem.add_entry(reclaim_start, reclaim_end - reclaim_start, .LoaderReclaimable); + r.mem.add_entry( + reclaim_start, + reclaim_end - reclaim_start, + .LoaderReclaimable, + ); if (reclaim_end < end) { r.mem.add_entry(reclaim_end, end - reclaim_end, .Reserved); @@ -132,6 +130,46 @@ export fn loader_entry() callconv(.c) void { else => .Reserved, }); } +} + +export fn loader_entry() callconv(.c) void { + var kernel: ?*anyopaque = null; + var module_start: usize = 0; + var module_end: usize = 0; + + const hhdm_resp = hhdm_request.response orelse unreachable; + hhdm_offset = hhdm_resp.offset; + + if (module_request.response) |resp| { + const limine_mods = resp.modules orelse unreachable; + const modules = limine_mods[0..resp.module_count]; + + for (modules) |mod| { + if (std.mem.eql(u8, std.mem.span(mod.string), "kernel")) { + kernel = mod.address; + const phys = @intFromPtr(mod.address) - hhdm_offset; + const size: usize = @intCast(mod.size); + + module_start = std.mem.alignForward( + usize, + phys, + r.page_size, + ); + module_end = std.mem.alignBackward( + usize, + phys + size, + r.page_size, + ); + break; + } + } + } + + if (kernel == null) { + @panic("loader: kernel not found"); + } + + build_mmap(module_start, module_end); if (framebuffer_request.response) |resp| { const fb = resp.framebuffers[0]; diff --git a/src/loader/main.zig b/src/loader/main.zig index 2b64ff4..d8dacb8 100644 --- a/src/loader/main.zig +++ b/src/loader/main.zig @@ -4,7 +4,11 @@ const std = @import("std"); const Entry = r.BootInfo.MemMap.Entry; const stack_size = 1024 * 16; // should be enough -extern fn jump_to_kernel(loader_info: usize, entry: usize, stack_top: usize) void; +extern fn jump_to_kernel( + loader_info: usize, + entry: usize, + stack_top: usize, +) void; fn load_kernel(kernel: *anyopaque) usize { const elf: *std.elf.Elf64.Ehdr = @ptrCast(@alignCast(kernel)); @@ -22,12 +26,18 @@ fn load_kernel(kernel: *anyopaque) usize { @panic("loader: invalid ELF type (expected ET_EXEC)"); } - var phdr: *std.elf.Elf64.Phdr = @ptrFromInt(@intFromPtr(kernel) + elf.phoff); + var phdr: *std.elf.Elf64.Phdr = @ptrFromInt( + @intFromPtr(kernel) + elf.phoff, + ); for (0..elf.phnum) |_| { switch (phdr.type) { .LOAD => { - const as_pages = std.mem.alignForward(usize, phdr.memsz, r.page_size); + const as_pages = std.mem.alignForward( + usize, + phdr.memsz, + r.page_size, + ); const npages = as_pages / r.page_size; var filesz_rem = phdr.filesz; @@ -43,7 +53,9 @@ fn load_kernel(kernel: *anyopaque) usize { const file_remaining = @min(filesz_rem, r.page_size); - const off: [*]u8 = @ptrFromInt(@intFromPtr(kernel) + phdr.offset + i * r.page_size); + const off: [*]u8 = @ptrFromInt( + @intFromPtr(kernel) + phdr.offset + i * r.page_size, + ); const source = off[0..file_remaining]; const va: [*]u8 = @ptrFromInt(addr); const dest = va[0..file_remaining]; diff --git a/src/loader/mem.zig b/src/loader/mem.zig index f2240a0..111e30b 100644 --- a/src/loader/mem.zig +++ b/src/loader/mem.zig @@ -100,7 +100,11 @@ pub fn alloc(size: usize) usize { return va; } -pub fn add_entry(base: usize, size: usize, @"type": r.BootInfo.MemMap.Entry.Type) void { +pub fn add_entry( + base: usize, + size: usize, + @"type": r.BootInfo.MemMap.Entry.Type, +) void { if (memory_map.entry_count == memory_map.entries.len) { @panic("loader: memory map is full"); } @@ -190,13 +194,17 @@ fn map_self() void { }, ); - // Now go through every usable entry and map to the HHDM every part that isnt covered by [0, hhdm_minimum_max_address). - // We can't blindly map until the maximum usable physical address because on some CPUs this might cause MCEs. - // See https://github.com/torvalds/linux/commit/66520ebc2df3fe52eb4792f8101fac573b766baf + // Now go through every usable entry and map to the HHDM every part that + // isnt covered by [0, hhdm_minimum_max_address). + // We can't blindly map until the maximum usable physical address because + // on some CPUs this might cause MCEs. + // See Linux commit 66520ebc2df3fe52eb4792f8101fac573b766baf for (0..memory_map.entry_count) |i| { const entry = memory_map.entries[i]; - if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < r.page_size) { + if ((entry.type != .Free and entry.type != .LoaderReclaimable) or + entry.size < r.page_size) + { continue; } @@ -233,20 +241,33 @@ fn map_pfndb() void { for (0..memory_map.entry_count) |i| { const entry = memory_map.entries[i]; - if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < r.page_size) { + if ((entry.type != .Free and entry.type != .LoaderReclaimable) or + entry.size < r.page_size) + { continue; } - const npages = std.math.divCeil(usize, entry.size, r.page_size) catch unreachable; + const npages = std.math.divCeil(usize, entry.size, r.page_size) catch + unreachable; - // 1. Calculate the exact virtual address range needed for this region's page structs. + // 1. Calculate the exact virtual address range needed for this + // region's page structs. const start_pfn: usize = entry.base / r.page_size; const exact_start = memory_layout.pfndb + (start_pfn * size); const exact_end = exact_start + (npages * size); // 2. Ensure the addresses are aligned on page boundaries. - const map_start = std.mem.alignBackward(usize, exact_start, r.page_size); - const map_end = std.mem.alignForward(usize, exact_end, r.page_size); + const map_start = std.mem.alignBackward( + usize, + exact_start, + r.page_size, + ); + + const map_end = std.mem.alignForward( + usize, + exact_end, + r.page_size, + ); // 3. Map the virtual pages. pagemap.map_range_allocating( diff --git a/src/loader/pmap.zig b/src/loader/pmap.zig index 9078449..af15471 100644 --- a/src/loader/pmap.zig +++ b/src/loader/pmap.zig @@ -34,8 +34,9 @@ const ContiguousSource = struct { } }; -/// A mapping source that allocates physical pages on demand. Each call to `next` -/// returns a single page, so this is intended to be consumed by `map_from` in a loop until the entire range is mapped. +/// A mapping source that allocates physical pages on demand. Each call to +/// `next` returns a single page, so this is intended to be consumed by +/// `map_from` in a loop until the entire range is mapped. const AllocatingSource = struct { flags: mem.MapFlags, @@ -62,8 +63,6 @@ fn index_for_level(va: usize, level: usize) usize { } pub const PMap = struct { - const Self = @This(); - const num_levels = r.arch.levels.len; const entries_per_table = 512; const index_mask = entries_per_table - 1; @@ -78,7 +77,7 @@ pub const PMap = struct { return is_leaf_level(level) and r.arch.is_leaf_level_enabled(level); } - fn cursor(self: *Self, va: usize) Cursor { + fn cursor(self: *PMap, va: usize) Cursor { var c = Cursor{ .pmap = self, .va = va, @@ -90,11 +89,11 @@ pub const PMap = struct { return c; } - pub fn activate(self: *Self) void { + pub fn activate(self: *PMap) void { r.arch.activate(self.root_pa); } - pub fn map_from(self: *Self, va: usize, size: usize, source: anytype) void { + pub fn map_from(self: *PMap, va: usize, size: usize, source: anytype) void { var c = self.cursor(va); var remain = size; @@ -105,8 +104,15 @@ pub const PMap = struct { } } - /// Map a contiguous virtual address range to a contiguous physical address range. - pub fn map_contiguous_range(self: *Self, va: usize, pa: usize, size: usize, flags: r.mem.MapFlags) void { + /// Map a contiguous virtual address range to a contiguous physical address + /// range. + pub fn map_contiguous_range( + self: *PMap, + va: usize, + pa: usize, + size: usize, + flags: r.mem.MapFlags, + ) void { const src = ContiguousSource{ .flags = flags, .size = size, @@ -118,12 +124,22 @@ pub const PMap = struct { } /// Map a single virtual page to a physical page. - pub fn map_page(self: *Self, va: usize, pa: usize, flags: r.mem.MapFlags) void { + pub fn map_page( + self: *PMap, + va: usize, + pa: usize, + flags: r.mem.MapFlags, + ) void { self.map_contiguous_range(va, pa, r.page_size, flags); } /// Map a virtual address range to physical addresses - pub fn map_range_allocating(self: *Self, va: usize, size: usize, flags: mem.MapFlags) void { + pub fn map_range_allocating( + self: *PMap, + va: usize, + size: usize, + flags: mem.MapFlags, + ) void { const src = AllocatingSource{ .flags = flags, }; @@ -132,7 +148,7 @@ pub const PMap = struct { } pub const Cursor = struct { - pmap: *Self, + pmap: *PMap, va: usize, /// Cached pointers to each level's page table, filled as the cursor /// descends. Only indices in `[0, top_level]` are valid. @@ -142,7 +158,11 @@ pub const PMap = struct { /// Reset upward in `advance` when the VA crosses a level boundary. top_level: usize, - fn walk_down(self: *Cursor, target_level: usize, allocate: bool) error{PageNotMapped}!void { + fn walk_down( + self: *Cursor, + target_level: usize, + allocate: bool, + ) error{PageNotMapped}!void { var current_level = self.top_level; while (current_level > target_level) { @@ -156,7 +176,9 @@ pub const PMap = struct { const new_table_pa = mem.alloc_page(); - const table_ptr: [*]r.arch.Pte = @ptrFromInt(r.mem.p2v(new_table_pa)); + const table_ptr: [*]r.arch.Pte = @ptrFromInt( + r.mem.p2v(new_table_pa), + ); @memset(table_ptr[0..entries_per_table], r.arch.Pte.zero()); pte = r.arch.make_table_pte(new_table_pa); @@ -193,14 +215,23 @@ pub const PMap = struct { @panic("no leaf level can map current alignment/size"); } - pub fn map_range(self: *Cursor, pa: usize, size: usize, flags: mem.MapFlags) void { + pub fn map_range( + self: *Cursor, + pa: usize, + size: usize, + flags: mem.MapFlags, + ) void { var remain = size; var current_pa = pa; while (remain > 0) { - const target_level = self.choose_target_level(current_pa, remain); + const target_level = self.choose_target_level( + current_pa, + remain, + ); - self.walk_down(target_level, true) catch @panic("walk_down failed during map_range"); + self.walk_down(target_level, true) catch + @panic("walk_down failed during map_range"); const table = self.tables[target_level]; table[index_for_level(self.va, target_level)] = diff --git a/src/loader/riscv64/impl.zig b/src/loader/riscv64/impl.zig index c4aa6ee..1fce0ef 100644 --- a/src/loader/riscv64/impl.zig +++ b/src/loader/riscv64/impl.zig @@ -75,7 +75,11 @@ pub inline fn make_table_pte(pa: usize) Pte { }; } -pub inline fn make_leaf_pte(pa: usize, flags: r.mem.MapFlags, level: usize) Pte { +pub inline fn make_leaf_pte( + pa: usize, + flags: r.mem.MapFlags, + level: usize, +) Pte { _ = level; return Pte{ .present = true, diff --git a/src/loader/root.zig b/src/loader/root.zig index 7e09990..822d090 100644 --- a/src/loader/root.zig +++ b/src/loader/root.zig @@ -21,7 +21,11 @@ const DebugWriter = struct { }; } - fn drain(_: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { + fn drain( + _: *std.Io.Writer, + data: []const []const u8, + _: usize, + ) std.Io.Writer.Error!usize { var total_written: usize = 0; for (data) |slice| { for (slice) |byte| { @@ -47,7 +51,8 @@ pub fn log( // Calculate the length required. var buf: [256]u8 = undefined; - const written = std.fmt.bufPrint(&buf, scope_str ++ fmt ++ "\n", args) catch return; + const written = std.fmt.bufPrint(&buf, scope_str ++ fmt ++ "\n", args) catch + return; var writer = DebugWriter.init(); diff --git a/src/mm/balance.zig b/src/mm/balance.zig index 0806c6e..ec9e73d 100644 --- a/src/mm/balance.zig +++ b/src/mm/balance.zig @@ -35,10 +35,10 @@ fn reap_stacks() void { // Reap all the excess stacks. var entry: ?*ke.Stack = lst; - while (entry != null) { - const next = entry.?.next; + while (entry) |e| { + const next = e.next; mm.heap.free( - (@intFromPtr(entry) + @sizeOf(ke.Stack)) - + (@intFromPtr(e) + @sizeOf(ke.Stack)) - ps.thread.kernel_thread_stack_size, ps.thread.kernel_thread_stack_size, ); @@ -86,7 +86,6 @@ fn balance_manager(_: ?*anyopaque) void { if (stack_reap_time == 0) { stack_reap_time = stack_reap_interval; - reap_stacks(); } } diff --git a/src/mm/heap.zig b/src/mm/heap.zig index 76464ec..1fc751d 100644 --- a/src/mm/heap.zig +++ b/src/mm/heap.zig @@ -1,3 +1,5 @@ +//! Kernel heap. + const r = @import("root"); const ke = r.ke; const mm = r.mm; @@ -43,7 +45,11 @@ pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque { mmp.kernel_space.lock.acquire(); - pte.* = mmp.impl.make_leaf_pte(page, .{ .read = true, .write = true }, 0); + pte.* = mmp.impl.make_leaf_pte( + page, + .{ .read = true, .write = true }, + 0, + ); } mmp.kernel_space.lock.release(); diff --git a/src/mm/init.zig b/src/mm/init.zig index a0cd4d6..e1b3c91 100644 --- a/src/mm/init.zig +++ b/src/mm/init.zig @@ -1,3 +1,5 @@ +//! Memory manager initialization. + const r = @import("root"); const pl = r.pl; const mm = r.mm; diff --git a/src/mm/phys.zig b/src/mm/phys.zig index 02878c6..19530dd 100644 --- a/src/mm/phys.zig +++ b/src/mm/phys.zig @@ -1,17 +1,19 @@ const r = @import("root"); -const std = @import("std"); const rtl = @import("rtl"); const pl = r.pl; const mm = r.mm; const mmp = mm.private; const ke = r.ke; +const std = @import("std"); +const assert = std.debug.assert; const log = std.log.scoped(.@"mm/phys"); // How aggressive is memory reclamation. const threshold_scale_factor = 10; -/// Threshold at which most memory allocations block, only high priority allocations are allowed. +/// Threshold at which most memory allocations block, only high priority +/// allocations are allowed. pub var min_memory_threshold: usize = 0; /// Threshold at which reclamation starts. pub var low_memory_threshold: usize = 0; @@ -29,14 +31,14 @@ var free_page_event: ke.Event = undefined; var free_pages: usize = 0; -pub var usable_memory: std.atomic.Value(usize) = .init(0); +pub var usable_memory = std.atomic.Value(usize).init(0); // Wait for pages to be available. // List lock is held. fn wait_for_pages(old_ipl: ke.Ipl) void { var ipl = old_ipl; - std.debug.assert(@intFromEnum(ipl) < @intFromEnum(ke.Ipl.Dispatch)); + assert(@backingInt(ipl) < @backingInt(ke.Ipl.Dispatch)); if (free_pages >= min_memory_threshold) { return; @@ -65,7 +67,7 @@ pub fn alloc() r.PAddr { wait_for_pages(ipl); - std.debug.assert(!free_list.is_empty()); + assert(!free_list.is_empty()); const head = free_list.first(); head.remove(); @@ -94,8 +96,8 @@ pub fn alloc_opts(opts: struct { policy: mm.WaitPolicy }) ?r.PAddr { wait_for_pages(ipl); } - std.debug.assert(free_pages > 0); - std.debug.assert(!free_list.is_empty()); + assert(free_pages > 0); + assert(!free_list.is_empty()); const head = free_list.first(); head.remove(); diff --git a/src/mm/pmap.zig b/src/mm/pmap.zig index 41936b6..4092aca 100644 --- a/src/mm/pmap.zig +++ b/src/mm/pmap.zig @@ -2,10 +2,12 @@ const r = @import("root"); const rtl = @import("rtl"); -const std = @import("std"); const mm = r.mm; const mmp = mm.private; +const std = @import("std"); +const assert = std.debug.assert; + /// Describes a contiguous region of physical memory to be mapped /// at a given virtual address. const MapItem = struct { @@ -45,8 +47,6 @@ fn index_for_level(va: usize, level: usize) usize { } pub const PMap = struct { - const Self = @This(); - const num_levels = mmp.impl.levels.len; const entries_per_table = 512; const index_mask = entries_per_table - 1; @@ -61,7 +61,7 @@ pub const PMap = struct { return is_leaf_level(level) and mmp.impl.is_leaf_level_enabled(level); } - fn cursor(self: *Self, va: r.VAddr) Cursor { + fn cursor(self: *PMap, va: r.VAddr) Cursor { var c = Cursor{ .pmap = self, .va = va, @@ -73,11 +73,16 @@ pub const PMap = struct { return c; } - pub fn activate(self: *Self) void { + pub fn activate(self: *PMap) void { mmp.impl.activate(self.root_pa); } - pub fn map_from(self: *Self, va: r.VAddr, size: usize, source: anytype) void { + pub fn map_from( + self: *PMap, + va: r.VAddr, + size: usize, + source: anytype, + ) void { var c = self.cursor(va); var remain = size; @@ -91,7 +96,7 @@ pub const PMap = struct { /// Map a contiguous virtual address range to a /// contiguous physical address range. pub fn map_contiguous_range( - self: *Self, + self: *PMap, va: r.VAddr, pa: r.PAddr, size: usize, @@ -107,7 +112,7 @@ pub const PMap = struct { self.map_from(va, size, src); } - pub fn query(self: *Self, va: r.VAddr) ?r.PAddr { + pub fn query(self: *PMap, va: r.VAddr) ?r.PAddr { var c = self.cursor(va); var level: usize = num_levels - 1; @@ -129,8 +134,8 @@ pub const PMap = struct { /// Unmap a contiguous range of virtual pages. /// Only small pages are supported. /// Return a pfn that points to the physical pages that were unmapped. - pub fn unmap(self: *Self, va: r.VAddr, size: usize) ?mmp.PfnList { - std.debug.assert(std.mem.isAligned(va, mm.page_size)); + pub fn unmap(self: *PMap, va: r.VAddr, size: usize) ?mmp.PfnList { + assert(std.mem.isAligned(va, mm.page_size)); var c = self.cursor(va); @@ -178,7 +183,7 @@ pub const PMap = struct { } pub const Cursor = struct { - pmap: *Self, + pmap: *PMap, va: r.VAddr, /// Cached pointers to each level's page table, filled as the cursor /// descends. Only indices in `[0, top_level]` are valid. @@ -283,9 +288,12 @@ pub fn wire_pte( va: r.VAddr, policy: mm.WaitPolicy, ) mm.Error!*mmp.impl.Pte { - std.debug.assert(space.lock.is_locked()); + assert(space.lock.is_locked()); + + var table: [*]mmp.impl.Pte = @ptrFromInt( + mm.p2v(mmp.kernel_space.pmap.root_pa), + ); - var table: [*]mmp.impl.Pte = @ptrFromInt(mm.p2v(mmp.kernel_space.pmap.root_pa)); var level = mmp.impl.levels.len - 1; while (true) { @@ -293,7 +301,7 @@ pub fn wire_pte( var entry = table[idx]; if (level == 0) { - std.debug.assert(!entry.present); + assert(!entry.present); return &table[idx]; } @@ -302,10 +310,13 @@ pub fn wire_pte( // since that may block. space.lock.release(); - const new_table_pa = mmp.phys.alloc_opts(.{ .policy = policy }) orelse { - space.lock.acquire(); - return mm.Error.OutOfMemory; - }; + const new_table_pa = mmp.phys.alloc_opts( + .{ .policy = policy }, + ) orelse + { + space.lock.acquire(); + return mm.Error.OutOfMemory; + }; // Re-acquire the lock and re-validate the entry, // since another thread may have raced to allocate it. diff --git a/src/mm/riscv64/impl.zig b/src/mm/riscv64/impl.zig index 1c27b68..8410538 100644 --- a/src/mm/riscv64/impl.zig +++ b/src/mm/riscv64/impl.zig @@ -88,7 +88,11 @@ pub inline fn make_table_pte(pa: r.PAddr) Pte { }; } -pub inline fn make_leaf_pte(pa: r.PAddr, flags: mm.MapFlags, level: usize) Pte { +pub inline fn make_leaf_pte( + pa: r.PAddr, + flags: mm.MapFlags, + level: usize, +) Pte { _ = level; return Pte{ .present = true, @@ -118,5 +122,6 @@ pub fn is_leaf_level_enabled(level: usize) bool { } pub fn init_kernel() void { - mmp.kernel_space.pmap.root_pa = (rv64.read_csr("satp") & ((@as(u64, 1) << 44) - 1)) << 12; + mmp.kernel_space.pmap.root_pa = (rv64.read_csr("satp") & + ((@as(u64, 1) << 44) - 1)) << 12; } diff --git a/src/mm/root.zig b/src/mm/root.zig index 31147ce..f215e04 100644 --- a/src/mm/root.zig +++ b/src/mm/root.zig @@ -35,7 +35,7 @@ comptime { if (@sizeOf(Page) != r.BootInfo.page_struct_size) { @compileError( std.fmt.comptimePrint( - "'Page' size mismatch between loader and kernel: kernel={} loader={}", + "'Page' size mismatch between loader and kernel: k={} l={}", .{ @sizeOf(Page), r.BootInfo.page_struct_size }, ), ); @@ -68,11 +68,13 @@ pub inline fn pfn_to_page(pfn: Pfn) usize { } pub inline fn struct_page_to_pfn(page: *Page) Pfn { - return @intCast((@intFromPtr(page) - private.impl.pfndb_base) / @sizeOf(Page)); + return @intCast((@intFromPtr(page) - private.impl.pfndb_base) / + @sizeOf(Page)); } pub inline fn pfn_to_struct_page(pfn: Pfn) *Page { - return @ptrFromInt(private.impl.pfndb_base + (@as(usize, @intCast(pfn)) * @sizeOf(Page))); + return @ptrFromInt(private.impl.pfndb_base + + (@as(usize, pfn) * @sizeOf(Page))); } pub var pfndb: [*]Page = @ptrFromInt(private.impl.pfndb_base); diff --git a/src/mm/tlb.zig b/src/mm/tlb.zig index 7379b10..a77ba2e 100644 --- a/src/mm/tlb.zig +++ b/src/mm/tlb.zig @@ -1,7 +1,6 @@ //! MM-level TLB shootdown support const r = @import("root"); -const std = @import("std"); const rtl = @import("rtl"); const ke = r.ke; @@ -10,9 +9,12 @@ const mmp = mm.private; const ps = r.ps; const ex = r.ex; -pub var sync_shootdowns: std.atomic.Value(usize) = .init(0); -pub var async_shootdowns: std.atomic.Value(usize) = .init(0); -pub var sent_shootdowns: std.atomic.Value(usize) = .init(0); +const std = @import("std"); +const assert = std.debug.assert; + +pub var sync_shootdowns = std.atomic.Value(usize).init(0); +pub var async_shootdowns = std.atomic.Value(usize).init(0); +pub var sent_shootdowns = std.atomic.Value(usize).init(0); var work_item: ex.WorkItem = undefined; @@ -54,7 +56,7 @@ fn reap_pages(_: ?*anyopaque) void { /// Unmap and flush a virtual range on the given space. /// Space lock is held on entry and IPL is raised to Dispatch. pub fn reclaim_range(space: *mm.Space, va: r.VAddr, size: usize) void { - std.debug.assert(std.mem.isAligned(va, mm.page_size)); + assert(std.mem.isAligned(va, mm.page_size)); // Unmap the virtual addresses and get the backing physical pages. const list = space.pmap.unmap(va, size) orelse { @@ -67,7 +69,7 @@ pub fn reclaim_range(space: *mm.Space, va: r.VAddr, size: usize) void { .base = va, .npages = @truncate(size / mm.page_size), .link = undefined, - .state = .init(0), + .state = std.atomic.Value(u16).init(0), .payload = .{ @intFromPtr(space), @as(*const usize, @ptrCast(@alignCast(&list))).*, @@ -77,7 +79,7 @@ pub fn reclaim_range(space: *mm.Space, va: r.VAddr, size: usize) void { space.lock.release(); const ipl = ke.ipl.raise(.Dispatch); - var mask: ke.CpuMask = .init(false); + var mask = ke.CpuMask.init(false); for (0..ke.ncpus) |i| { if (i != ke.cpu.current()) { @@ -110,6 +112,6 @@ fn activation(_: *rtl.HandoffList) void { } pub fn init() void { - ke.shootdown.shootdowns.* = .init(activation); + ke.shootdown.shootdowns.* = rtl.HandoffList.init(activation); work_item.init(.High, reap_pages, null); } diff --git a/src/mm/zone.zig b/src/mm/zone.zig index c3a2a06..ba60bfb 100644 --- a/src/mm/zone.zig +++ b/src/mm/zone.zig @@ -114,16 +114,18 @@ //! an object leaves the slab layer and destructors run when it returns to it, //! slabs hold raw objects while magazines hold constructed ones. -const rtl = @import("rtl"); -const r = @import("root"); -const std = @import("std"); const config = @import("config"); +const rtl = @import("rtl"); +const r = @import("root"); const ke = r.ke; const mm = r.mm; const ex = r.ex; const mmp = mm.private; +const std = @import("std"); +const assert = std.debug.assert; + /// Interval at which we do housekeeping (working set update, reaping, etc.) pub const update_interval_s = 15; @@ -276,7 +278,7 @@ const Slab = struct { const bm = self.bitmap(); const bit = @as(u64, 1) << @intCast(idx % 64); - std.debug.assert(bm[idx / 64] & (bit) == 0); + assert(bm[idx / 64] & (bit) == 0); bm[idx / 64] |= bit; } @@ -558,8 +560,6 @@ pub const Zone = struct { arg: ?*anyopaque, - const Self = @This(); - const InitOptions = struct { alignment: u16 = slab_align, ctor: ?*const fn (*anyopaque) void = null, @@ -586,7 +586,7 @@ pub const Zone = struct { /// Initialize a zone. pub fn init( - self: *Self, + self: *Zone, name: []const u8, size: usize, options: InitOptions, @@ -634,9 +634,9 @@ pub const Zone = struct { self.smr_reclaim = options.smr_reclaim; if (options.import) |i| { - std.debug.assert(options.release != null); + assert(options.release != null); self.import = i; - self.release = options.release.?; + self.release = options.release orelse unreachable; self.arg = options.arg; } else { self.import = zone_import; @@ -669,7 +669,7 @@ pub const Zone = struct { } /// Allocate an object from the zone. - pub fn alloc(self: *Self, opts: AllocOpts) mm.Error!*anyopaque { + pub fn alloc(self: *Zone, opts: AllocOpts) mm.Error!*anyopaque { // Try grabbing an object from the magazine layer. if (magazines_initialized and self.use_magazines) { @branchHint(.likely); @@ -682,12 +682,13 @@ pub const Zone = struct { while (true) { if (cpu.alloc_rounds > 0) { // Fast path: just pop from the alloc magazine. + const alloc_mag = cpu.alloc orelse unreachable; cpu.alloc_rounds -= 1; - buf = cpu.alloc.?.rounds_ptr()[cpu.alloc_rounds]; + buf = alloc_mag.rounds_ptr()[cpu.alloc_rounds]; if (is_poison_enabled and !self.dont_touch) { if (!check_poison(buf, self.obj_size, free_poison)) { - @panic("slab poison mismatch: object corrupted after free"); + @panic("poison mismatch: possible use-after-free"); } fill_with_poison(buf, self.obj_size, alloc_poison); @@ -702,7 +703,7 @@ pub const Zone = struct { } if (cpu.free_rounds > 0 and self.smr == null) { - std.debug.assert(cpu.free != null); + assert(cpu.free != null); // Alloc magazine is empty. If the free magazine has items, // swap them. The free magazine now becomes our alloc @@ -715,7 +716,8 @@ pub const Zone = struct { // Both magazines are empty. // Try to get a full magazine from the CPU-local depot. if (self.full_mags_ready(&cpu.depot.full_mags)) { - const full_mag = Depot.alloc(&cpu.depot.full_mags).?; + const full_mag = Depot.alloc(&cpu.depot.full_mags) orelse + unreachable; // Discard our empty alloc magazine to the depot. if (cpu.alloc) |empty_mag| { @@ -741,7 +743,8 @@ pub const Zone = struct { // Try getting a full magazine from the CPU-local depot again. if (self.full_mags_ready(&cpu.depot.full_mags)) { - const full_mag = Depot.alloc(&cpu.depot.full_mags).?; + const full_mag = Depot.alloc(&cpu.depot.full_mags) orelse + unreachable; // Discard our empty alloc magazine to the depot. if (cpu.alloc) |empty_mag| { @@ -758,14 +761,15 @@ pub const Zone = struct { continue; } - // Nothing was found. Allocate a new magazine and fill it from the - // import layer, then replace the CPU's alloc magazine with it. - // It is preferable (maybe even required?) to drop the CPU lock - // while we do this work; when we do re-acquire the lock we then - // might notice that someone has already done the work for us and - // put a magazine in the CPU's alloc storage. In that case, it would - // be better to take our fresh magazine and insert it into the depot - // instead, since there is a higher chance the impostor magazine + // Nothing was found. Allocate a new magazine and fill it from + // the import layer, then replace the CPU's alloc magazine with + // it. It is preferable (maybe even required?) to drop the CPU + // lock while we do this work; when we do re-acquire the lock + // we then might notice that someone has already done the work + // for us and put a magazine in the CPU's alloc storage. + // In that case, it would be better to take our fresh magazine + // and insert it into the depot instead, + // since there is a higher chance the impostor magazine // contains objects that are hotter in cache. cpu.lock.release(ipl); @@ -783,7 +787,8 @@ pub const Zone = struct { opts.policy, ); - // We dropped the lock and IPL, so we might've been migrated. + // We dropped the lock and IPL, so we might've been + // migrated. ipl = ke.ipl.raise(.Dispatch); cpu = &self.cpus[ke.cpu.current()].value; cpu.lock.acquire_no_ipl(); @@ -793,14 +798,19 @@ pub const Zone = struct { cpu.alloc_rounds = cnt; continue; } else { - // We lost the race. If the magazine we got was fully filled, - // then put it in the depot. Otherwise, dispose of it. + // We lost the race. If the magazine we got was fully + // filled, then put it in the depot. + // Otherwise, dispose of it. if (cnt == magazine_size.load()) { - // Ensure this magazine isn't counted as a freed one, - // it is instantly reclaimable. + // Ensure this magazine isn't counted as a freed + // one, it is instantly reclaimable. nm.seq = ke.smr.seq_invalid; - Depot.free(nm, &cpu.depot.full_mags, self.reuse_policy); + Depot.free( + nm, + &cpu.depot.full_mags, + self.reuse_policy, + ); continue; } @@ -828,8 +838,8 @@ pub const Zone = struct { } } - // Fallback in case this is a zone with magazines disabled or we got into trouble above, - // just fetch an object from the import layer. + // Fallback in case this is a zone with magazines disabled or we got + // into trouble above, just fetch an object from the import layer. var elems: [1]*anyopaque = undefined; _ = try self.import( @@ -843,7 +853,7 @@ pub const Zone = struct { } /// Free an object back to the zone. - pub fn free(self: *Self, obj: *anyopaque) void { + pub fn free(self: *Zone, obj: *anyopaque) void { if (magazines_initialized and self.use_magazines) { @branchHint(.likely); @@ -857,8 +867,12 @@ pub const Zone = struct { { // Fast path: push directly to free magazine. // For SMR zones the dtor and poisoning are deferred until - // the magazine is reused, as readers may still hold the object. - if (is_poison_enabled and self.smr == null and !self.dont_touch) { + // the magazine is reused, as readers may still hold + // the object. + const free_mag = cpu.free orelse unreachable; + if (is_poison_enabled and self.smr == null and + !self.dont_touch) + { if (self.dtor) |dtor| { dtor(obj); } @@ -866,12 +880,16 @@ pub const Zone = struct { fill_with_poison(obj, self.obj_size, free_poison); } - cpu.free.?.rounds_ptr()[cpu.free_rounds] = obj; + free_mag.rounds_ptr()[cpu.free_rounds] = obj; cpu.free_rounds += 1; - if (self.smr != null and cpu.free_rounds == magazine_size.load()) { + if (self.smr != null and + cpu.free_rounds == magazine_size.load()) + { + const smr = self.smr orelse unreachable; + // Stamp the now-full magazine. - cpu.free.?.seq = ke.smr.deferred_advance(self.smr.?); + free_mag.seq = ke.smr.deferred_advance(smr); } break true; @@ -929,8 +947,8 @@ pub const Zone = struct { } // No empty magazine in the depot, try to allocate a new one. - // We need to drop IPL and the CPU lock here because we need to acquire - // the magazine's allocator's blocking lock. + // We need to drop IPL and the CPU lock here because we need to + // acquire the magazine's allocator's blocking lock. cpu.lock.release(ipl); const raw_mag = magazine_zone.alloc( @@ -970,7 +988,7 @@ pub const Zone = struct { self.release(self.arg, &slice, self.chunk_size); } - fn slab_free(self: *Self, obj: *anyopaque, poison: bool) void { + fn slab_free(self: *Zone, obj: *anyopaque, poison: bool) void { if (!is_poison_enabled or poison and !self.dont_touch) { if (self.dtor) |dtor| { dtor(obj); @@ -988,7 +1006,8 @@ pub const Zone = struct { @intFromPtr(obj), mm.page_size, ); - const phys = mmp.kernel_space.pmap.query(page_va).?; + const phys = mmp.kernel_space.pmap.query(page_va) orelse + unreachable; const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys)); break :blk page.alloced.slab_data.slab; } else blk: { @@ -1023,7 +1042,7 @@ pub const Zone = struct { elem_size: usize, policy: mm.WaitPolicy, ) mm.Error!usize { - const self: *Self = @ptrCast(@alignCast(arg)); + const self: *Zone = @ptrCast(@alignCast(arg)); _ = elem_size; @@ -1046,18 +1065,21 @@ pub const Zone = struct { self.lock.acquire(); - // Note: this is racy and could lead to two threads creating a slab - // at the same time. Worst case scenario, we have one slab too many, - // so we don't care; we already did the work required for slab - // construction and this ideally should not happen very often. + // Note: this is racy and could lead to two threads creating a + // slab at the same time. Worst case scenario, we have one slab + // too many, so we don't care; we already did the work + // required for slab construction and this ideally should not + // happen very often. self.partial_slabs.insert_tail(&slab.link); } else { slab = @fieldParentPtr("link", self.partial_slabs.first()); } while (slab.refcount != slab.capacity and i < requested) { - const idx = slab.bitmap_alloc().?; - const buf: *anyopaque = @ptrFromInt(slab.base + idx * self.chunk_size); + const idx = slab.bitmap_alloc() orelse unreachable; + const buf: *anyopaque = @ptrFromInt( + slab.base + idx * self.chunk_size, + ); slab.refcount += 1; @@ -1068,7 +1090,7 @@ pub const Zone = struct { if (is_poison_enabled) { if (!check_poison(buf, self.obj_size, free_poison)) { - @panic("slab poison mismatch: object corrupted after free"); + @panic("poison mismatch: possible use-after-free"); } fill_with_poison(buf, self.obj_size, alloc_poison); @@ -1087,8 +1109,12 @@ pub const Zone = struct { return i; } - fn zone_release(arg: ?*anyopaque, elems: []*anyopaque, elem_size: usize) void { - const self: *Self = @ptrCast(@alignCast(arg)); + fn zone_release( + arg: ?*anyopaque, + elems: []*anyopaque, + elem_size: usize, + ) void { + const self: *Zone = @ptrCast(@alignCast(arg)); _ = elem_size; @@ -1112,7 +1138,7 @@ pub const Zone = struct { /// Periodic updates on the zone. /// Updates the magazine working set and trims or grows them if needed. - fn update(self: *Self) void { + fn update(self: *Zone) void { const ipl = self.depot_lock.acquire(); // Update the working set size. @@ -1187,7 +1213,7 @@ pub const Zone = struct { } /// Returns whether or not we should trim excess magazines. - fn trim_needed(self: *Self) bool { + fn trim_needed(self: *Zone) bool { if (!self.use_magazines) return false; if (self.trim_depot) return true; @@ -1221,8 +1247,9 @@ pub const Zone = struct { /// Called periodically to trim our magazines. /// This frees the excess magazines. - fn trim(self: *Self, trim_depot: bool) void { - // Detach magazines under the locks, then release their objects unlocked. + fn trim(self: *Zone, trim_depot: bool) void { + // Detach magazines under the locks, then release their objects + // unlocked. self.lock.acquire(); const empty_mags = self.trim_maglist(&self.depot.empty_mags); @@ -1262,7 +1289,7 @@ pub const Zone = struct { /// Called when memory is low and we need to make memory ASAP. /// Drains all magazines from the depot. - fn drain(self: *Self) void { + fn drain(self: *Zone) void { if (!self.use_magazines) return; // Detach magazines before invoking release callbacks. @@ -1304,7 +1331,7 @@ pub const Zone = struct { } /// Detach excess magazines for destruction. - fn trim_maglist(self: *Self, maglist: *MagazineList) ?*Magazine { + fn trim_maglist(self: *Zone, maglist: *MagazineList) ?*Magazine { const ipl = self.depot_lock.acquire(); // Grab the magazines and WSS under the depot lock. @@ -1345,7 +1372,7 @@ pub const Zone = struct { } /// Trim a CPU's depot to `target`. - fn trim_cpu(self: *Self, cpu: *Cpu, depot: *Depot, target: usize) void { + fn trim_cpu(self: *Zone, cpu: *Cpu, depot: *Depot, target: usize) void { const ipl = cpu.lock.acquire(); // Split the target (cpu_depot_size) in 2 for each magazine type. @@ -1361,7 +1388,7 @@ pub const Zone = struct { // Move them to our depot. for (0..n) |_| { Depot.free( - Depot.alloc(&cpu.depot.full_mags).?, + Depot.alloc(&cpu.depot.full_mags) orelse unreachable, &depot.full_mags, self.reuse_policy, ); @@ -1375,7 +1402,7 @@ pub const Zone = struct { // Move them to our depot. for (0..n) |_| { Depot.free( - Depot.alloc(&cpu.depot.empty_mags).?, + Depot.alloc(&cpu.depot.empty_mags) orelse unreachable, &depot.empty_mags, self.reuse_policy, ); @@ -1385,7 +1412,7 @@ pub const Zone = struct { cpu.lock.release(ipl); } - fn maglist_destroy(self: *Self, head: ?*Magazine, n: usize) void { + fn maglist_destroy(self: *Zone, head: ?*Magazine, n: usize) void { var elems = head; while (elems) |mag| { elems = mag.next; @@ -1394,7 +1421,7 @@ pub const Zone = struct { } /// Destroy a magazine. - fn magazine_destroy(self: *Self, magazine: *Magazine, rounds: usize) void { + fn magazine_destroy(self: *Zone, magazine: *Magazine, rounds: usize) void { if (rounds != 0) { if (self.smr) |smr| { // The stamp may still be uncommitted if the magazine never @@ -1439,7 +1466,7 @@ pub const Zone = struct { mmp.phys.free(phys); } - fn slab_create_small(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab { + fn slab_create_small(self: *Zone, policy: mm.WaitPolicy) mm.Error!*Slab { const buf = try alloc_page(policy); const capacity = Slab.calc_capacity(self.chunk_size, self.alignment); @@ -1463,7 +1490,9 @@ pub const Zone = struct { } for (0..capacity) |i| { - const obj: *anyopaque = @ptrFromInt(slab.base + i * self.chunk_size); + const obj: *anyopaque = @ptrFromInt( + slab.base + i * self.chunk_size, + ); if (is_poison_enabled) { fill_with_poison(obj, self.obj_size, free_poison); } @@ -1472,7 +1501,7 @@ pub const Zone = struct { return slab; } - fn slab_create_large(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab { + fn slab_create_large(self: *Zone, policy: mm.WaitPolicy) mm.Error!*Slab { const capacity = (self.slab_size) / self.chunk_size; const buf = try mmp.heap.alloc(self.slab_size, policy); @@ -1515,7 +1544,7 @@ pub const Zone = struct { return ret; } - fn slab_create(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab { + fn slab_create(self: *Zone, policy: mm.WaitPolicy) mm.Error!*Slab { if (self.obj_size <= small_slab_size) { return self.slab_create_small(policy); } else { @@ -1523,7 +1552,7 @@ pub const Zone = struct { } } - fn slab_destroy(self: *Self, slab: *Slab) void { + fn slab_destroy(self: *Zone, slab: *Slab) void { if (self.obj_size > small_slab_size) { const base = slab.base; const alloc_size = @sizeOf(Slab) + Slab.bitmap_bytes(slab.capacity); @@ -1538,7 +1567,7 @@ pub const Zone = struct { } /// Check that a full magazine can be taken from the list head. - fn full_mags_ready(self: *Self, list: *MagazineList) bool { + fn full_mags_ready(self: *Zone, list: *MagazineList) bool { const head = list.head orelse return false; const smr = self.smr orelse return true; @@ -1547,7 +1576,7 @@ pub const Zone = struct { /// Deferred free-time work for SMR zones, ran when a full magazine /// is reused after its grace period expired. - fn magazine_reuse(self: *Self, mag: *Magazine) void { + fn magazine_reuse(self: *Zone, mag: *Magazine) void { if (self.smr == null) return; for (0..magazine_size.load()) |i| { @@ -1564,7 +1593,7 @@ pub const Zone = struct { /// Move all local full magazines to the zone depot so their sequences /// can age, committing their deferred advances. Depot lock is held. - fn smr_rotate_full(self: *Self, smr: *ke.smr.Domain, cpu: *Cpu) void { + fn smr_rotate_full(self: *Zone, smr: *ke.smr.Domain, cpu: *Cpu) void { const src = &cpu.depot.full_mags; const dst = &self.depot.full_mags; const tail = src.tail orelse return; @@ -1584,7 +1613,8 @@ pub const Zone = struct { src.num = 0; src.min = 0; - // The last magazine has the newest stamp, committing it covers them all. + // The last magazine has the newest stamp, committing it covers them + // all. ke.smr.deferred_advance_commit(smr, tail.seq); } @@ -1592,7 +1622,7 @@ pub const Zone = struct { /// after an allocation. The CPU depot moves its extra full magazines to the /// zone depot, and tries to get full magazines from the zone depot up to /// `target`. CPU lock is held. - fn alloc_depot_rebalance(self: *Self, target: usize, cpu: *Cpu) void { + fn alloc_depot_rebalance(self: *Zone, target: usize, cpu: *Cpu) void { if (!self.depot_lock.try_acquire_no_ipl()) { // Register the contention. self.depot_lock.acquire_no_ipl(); @@ -1605,7 +1635,7 @@ pub const Zone = struct { for (0..n) |_| { Depot.free( - Depot.alloc(&cpu.depot.empty_mags).?, + Depot.alloc(&cpu.depot.empty_mags) orelse unreachable, &self.depot.empty_mags, self.reuse_policy, ); @@ -1631,7 +1661,7 @@ pub const Zone = struct { if (n != 0 and self.full_mags_ready(&self.depot.full_mags)) { for (0..n) |_| { Depot.free( - Depot.alloc(&self.depot.full_mags).?, + Depot.alloc(&self.depot.full_mags) orelse unreachable, &cpu.depot.full_mags, self.reuse_policy, ); @@ -1645,7 +1675,7 @@ pub const Zone = struct { /// after a free.The CPU depot moves its extra full magazines to the /// zone depot, and tries to get empty magazines from the zone depot up to /// `target`. CPU lock is held. - fn free_depot_rebalance(self: *Self, target: usize, cpu: *Cpu) void { + fn free_depot_rebalance(self: *Zone, target: usize, cpu: *Cpu) void { if (!self.depot_lock.try_acquire_no_ipl()) { // Register the contention. self.depot_lock.acquire_no_ipl(); @@ -1663,7 +1693,8 @@ pub const Zone = struct { if (n != 0 and self.full_mags_ready(&self.depot.full_mags)) { for (0..n) |_| { Depot.free( - Depot.alloc(&self.depot.full_mags).?, + Depot.alloc(&self.depot.full_mags) orelse + unreachable, &cpu.depot.full_mags, self.reuse_policy, ); @@ -1675,7 +1706,7 @@ pub const Zone = struct { for (0..n) |_| { Depot.free( - Depot.alloc(&cpu.depot.full_mags).?, + Depot.alloc(&cpu.depot.full_mags) orelse unreachable, &self.depot.full_mags, self.reuse_policy, ); @@ -1691,7 +1722,7 @@ pub const Zone = struct { ); for (0..n) |_| { Depot.free( - Depot.alloc(&self.depot.empty_mags).?, + Depot.alloc(&self.depot.empty_mags) orelse unreachable, &cpu.depot.empty_mags, self.reuse_policy, ); @@ -1719,7 +1750,7 @@ pub fn TypedZone(comptime T: type) type { return struct { zone: Zone, - const Self = @This(); + const TZone = @This(); pub const InitOptions = struct { ctor: ?*const fn (*T) void = null, @@ -1728,7 +1759,11 @@ pub fn TypedZone(comptime T: type) type { smr_reclaim: ?*const fn (*T) void = null, }; - pub fn init(self: *@This(), name: []const u8, options: InitOptions) void { + pub fn init( + self: *TZone, + name: []const u8, + options: InitOptions, + ) void { self.zone.init(name, @sizeOf(T), .{ .alignment = @alignOf(T), .ctor = @ptrCast(options.ctor), @@ -1738,17 +1773,17 @@ pub fn TypedZone(comptime T: type) type { }); } - pub fn create(self: *Self) mm.Error!*T { + pub fn create(self: *TZone) mm.Error!*T { const ret = try self.zone.alloc(.{}); return @ptrCast(@alignCast(ret)); } - pub fn create_opts(self: *Self, opts: AllocOpts) mm.Error!*T { + pub fn create_opts(self: *TZone, opts: AllocOpts) mm.Error!*T { const ret = try self.zone.alloc(opts); return @ptrCast(@alignCast(ret)); } - pub fn destroy(self: *Self, obj: *T) void { + pub fn destroy(self: *TZone, obj: *T) void { self.zone.free(obj); } }; @@ -1787,7 +1822,7 @@ pub fn late_init() linksection(r.init) void { z.cpus = cpus_alloc(); for (0..ke.ncpus) |i| { - z.cpus[i] = .init(.{ + z.cpus[i] = rtl.CachePadded(Cpu).init(.{ .lock = ke.SpinLock.init("Magazine"), .alloc = null, .free = null, @@ -1889,7 +1924,8 @@ fn gpa_remap( new_len: usize, ret_addr: usize, ) ?[*]u8 { - if (gpa_resize(undefined, buf, alignment, new_len, ret_addr)) return buf.ptr; + if (gpa_resize(undefined, buf, alignment, new_len, ret_addr)) + return buf.ptr; const new_ptr = gpa_alloc(undefined, new_len, alignment, ret_addr) orelse return null; diff --git a/src/platform/acpi/root.zig b/src/platform/acpi/root.zig index 8fb822e..4a57615 100644 --- a/src/platform/acpi/root.zig +++ b/src/platform/acpi/root.zig @@ -276,7 +276,8 @@ pub var rsdt: ?*Rsdt = null; pub fn find_table(signature: []const u8) ?*SdtHeader { if (xsdt) |x| { - const entry_count = (x.header.length - @sizeOf(SdtHeader)) / @sizeOf(u64); + const entry_count = (x.header.length - @sizeOf(SdtHeader)) / + @sizeOf(u64); const entries_ptr: [*]align(1) u64 = @ptrCast(&x.entries); for (0..entry_count) |i| { @@ -286,7 +287,8 @@ pub fn find_table(signature: []const u8) ?*SdtHeader { } } } else if (rsdt) |rsdt_table| { - const entry_count = (rsdt_table.header.length - @sizeOf(SdtHeader)) / @sizeOf(u32); + const entry_count = (rsdt_table.header.length - @sizeOf(SdtHeader)) / + @sizeOf(u32); const entries_ptr: [*]align(1) u32 = @ptrCast(&rsdt_table.entries); for (0..entry_count) |i| { @@ -316,7 +318,8 @@ fn format_table(hdr: *SdtHeader, phys_addr: u64) void { fn enumerate_tables() void { if (xsdt) |x| { - const entry_count = (x.header.length - @sizeOf(SdtHeader)) / @sizeOf(u64); + const entry_count = (x.header.length - @sizeOf(SdtHeader)) / + @sizeOf(u64); const entries_ptr: [*]align(1) u64 = @ptrCast(&x.entries); for (0..entry_count) |i| { @@ -325,7 +328,8 @@ fn enumerate_tables() void { format_table(hdr, phys); } } else if (rsdt) |rsdt_table| { - const entry_count = (rsdt_table.header.length - @sizeOf(SdtHeader)) / @sizeOf(u32); + const entry_count = (rsdt_table.header.length - @sizeOf(SdtHeader)) / + @sizeOf(u32); const entries_ptr: [*]align(1) u32 = @ptrCast(&rsdt_table.entries); for (0..entry_count) |i| { diff --git a/src/platform/pc/smp.zig b/src/platform/pc/smp.zig index e9847db..6434ecb 100644 --- a/src/platform/pc/smp.zig +++ b/src/platform/pc/smp.zig @@ -23,7 +23,7 @@ const start_stack = ke.ExportedCpuLocal(usize, 0, "ap_start_stack"); pub const start_thread = ke.CpuLocal(*ke.Thread, undefined); -var aps_booted: std.atomic.Value(usize) = .init(0); +var aps_booted = std.atomic.Value(usize).init(0); const ApData = extern struct { entry: usize align(1), @@ -77,15 +77,18 @@ pub fn init() linksection(r.init) void { .Unknown => false, }; - const init_delay: r.Nanoseconds = .init( + const init_delay = r.Nanoseconds.init( if (skip_delay) 0 else std.time.ns_per_ms * 10, ); - const sipi_delay: r.Nanoseconds = .init( + const sipi_delay = r.Nanoseconds.init( if (skip_delay) std.time.ns_per_us * 10 else std.time.ns_per_us * 300, ); const page: [*]u8 = @ptrFromInt(mm.p2v(0x8000)); - @memcpy(page[0..trampoline_size], @as([*]u8, @ptrFromInt(trampoline_start))); + @memcpy( + page[0..trampoline_size], + @as([*]u8, @ptrFromInt(trampoline_start)), + ); const data_offset = @intFromPtr(&AP_TRAMPOLINE_DATA) - trampoline_start; const data_phys: *ApData = @ptrFromInt(mm.p2v(0x8000 + data_offset)); @@ -97,7 +100,8 @@ pub fn init() linksection(r.init) void { // Allocate per-cpu offsets for CPU-local data. kep.impl.cpu_offsets = @ptrCast(offsets); - const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start); + const percpu_size = @intFromPtr(&__percpu_end) - + @intFromPtr(&__percpu_start); log.info("per-CPU data size: {} bytes", .{percpu_size}); diff --git a/src/platform/root.zig b/src/platform/root.zig index c97270d..42bd26c 100644 --- a/src/platform/root.zig +++ b/src/platform/root.zig @@ -1,5 +1,6 @@ //! Platform interface: each port must implement these functions and variables. -//! To add a new platform, create a new file and implement all required functions. +//! To add a new platform, create a new file and implement all required +//! functions. const config = @import("config"); const r = @import("root"); diff --git a/src/proc/private.zig b/src/proc/private.zig index 01c5858..7449adc 100644 --- a/src/proc/private.zig +++ b/src/proc/private.zig @@ -15,7 +15,7 @@ fn turnstile_ctor(ts: *kep.turnstile.Turnstile) void { .queues = undefined, .waiters = 0, .owners = .none, - .lock = .init("turnstile"), + .lock = ke.SpinLock.init("turnstile"), }; ts.reset(); } diff --git a/src/proc/thread.zig b/src/proc/thread.zig index 151de83..474d576 100644 --- a/src/proc/thread.zig +++ b/src/proc/thread.zig @@ -51,7 +51,7 @@ pub fn stack_activation() void { pub fn init() void { thread_zone.init("thread", .{}); - ke.thread.reaper_list.* = .init(reaper_activation); + ke.thread.reaper_list.* = rtl.HandoffList.init(reaper_activation); reaper_item.init(.High, reaper_fn, null); stack_item.init(.High, stack_fn, null); } @@ -68,7 +68,9 @@ pub fn create_kernel( var td = try thread_zone.create(); const stack = if (start_with_stack) - @intFromPtr(try mm.heap.alloc(kernel_thread_stack_size, .WaitForMemory)) + + @intFromPtr( + try mm.heap.alloc(kernel_thread_stack_size, .WaitForMemory), + ) + kernel_thread_stack_size else 0; diff --git a/src/rcu/hashtable.zig b/src/rcu/hashtable.zig index 19c5517..548e979 100644 --- a/src/rcu/hashtable.zig +++ b/src/rcu/hashtable.zig @@ -3,14 +3,17 @@ //! the buckets for write operations. //! The general design is inspired by XNU's smr_shash. -const std = @import("std"); const rtl = @import("rtl"); + const r = @import("root"); const mm = r.mm; const ex = r.ex; const ke = r.ke; const rcu = r.rcu; +const std = @import("std"); +const assert = std.debug.assert; + /// Atomic state for the table const State = packed struct(u32) { /// The current index. @@ -30,8 +33,6 @@ const State = packed struct(u32) { const Bucket = struct { word: std.atomic.Value(usize), - const Self = @This(); - const lock_tag = 0b01; const stop_tag = 0b10; const mask = lock_tag | stop_tag; @@ -46,12 +47,12 @@ const Bucket = struct { return @intFromPtr(head) | stop_tag; } - fn load(self: *const Self) usize { + fn load(self: *const Bucket) usize { return self.word.load(.acquire) & ~@as(usize, lock_tag); } /// Lock the bucket and return the address. - fn lock(self: *Self) usize { + fn lock(self: *Bucket) usize { while (true) { const v = self.word.fetchOr(lock_tag, .acquire); if (v & lock_tag == 0) return v; @@ -62,17 +63,17 @@ const Bucket = struct { } /// Unlock the bucket and store `val`. - fn unlock(self: *Self, val: usize) void { - std.debug.assert(val & lock_tag == 0); + fn unlock(self: *Bucket, val: usize) void { + assert(val & lock_tag == 0); self.word.store(val, .release); } - fn store_locked(self: *Self, val: usize) void { - std.debug.assert(val & lock_tag == 0); + fn store_locked(self: *Bucket, val: usize) void { + assert(val & lock_tag == 0); self.word.store(val | lock_tag, .release); } - fn unlock_no_changes(self: *Self) void { + fn unlock_no_changes(self: *Bucket) void { _ = self.word.fetchAnd(~@as(usize, lock_tag), .release); } }; @@ -145,10 +146,8 @@ const Core = struct { smr: *ke.smr.Domain, hash_link: *const fn (link: *rcu.SList.Entry, seed: u32) u32, - const Self = @This(); - fn init( - self: *Self, + self: *Core, smr: *ke.smr.Domain, policy: ResizePolicy, min_size: u32, @@ -168,29 +167,29 @@ const Core = struct { const arr = try mm.zone.gpa.alloc(Bucket, size); for (arr) |*elem| { - elem.word = .init(Bucket.stop_for(elem)); + elem.word = std.atomic.Value(usize).init(Bucket.stop_for(elem)); } self.min_shift = @intCast(std.math.log2_int_ceil(usize, min)); - self.state = .init(.{ + self.state = std.atomic.Value(State).init(.{ .current_shift = shift, .new_shift = shift, .current_index = 0, .new_index = 0, }); - self.rehashing_state = .init(rehashing_none); - self.elems = .init(0); + self.rehashing_state = std.atomic.Value(u8).init(rehashing_none); + self.elems = std.atomic.Value(u32).init(0); self.smr = smr; - self.buckets[0] = .init(@ptrCast(arr)); - self.seeds[0] = .init(0); + self.buckets[0] = std.atomic.Value([*]Bucket).init(@ptrCast(arr)); + self.seeds[0] = std.atomic.Value(u32).init(0); self.rehash_work.init(.Normal, rehash, self); } fn migrate_bucket( - self: *Self, + self: *Core, old: *Bucket, new_array: [*]Bucket, new_seed: u32, @@ -244,7 +243,7 @@ const Core = struct { old.unlock(Bucket.moved); } - fn enqueue_rehash(self: *Self, reason: u8) void { + fn enqueue_rehash(self: *Core, reason: u8) void { const prev = self.rehashing_state.fetchOr(reason, .monotonic); if (prev == rehashing_none) { @@ -254,10 +253,14 @@ const Core = struct { /// Called in worker thread context when the table needs rehashing. fn rehash(ptr: ?*anyopaque) void { - const self: *Self = @ptrCast(@alignCast(ptr.?)); + const ctx = ptr orelse unreachable; + const self: *Core = @ptrCast(@alignCast(ctx)); while (true) { - const reason = self.rehashing_state.swap(rehashing_running, .monotonic); + const reason = self.rehashing_state.swap( + rehashing_running, + .monotonic, + ); const elems = self.elems.load(.monotonic); const state = self.state.load(.monotonic); @@ -288,7 +291,7 @@ const Core = struct { } /// Resize (and re-seed) the hash table to a given target shift. - fn resize(self: *Self, new_shift: u5) void { + fn resize(self: *Core, new_shift: u5) void { var state = self.state.load(.monotonic); state.new_shift = new_shift; @@ -307,7 +310,7 @@ const Core = struct { // 1. Prepare the new array and publish it. for (new_array) |*elem| { - elem.word = .init(Bucket.stop_for(elem)); + elem.word = std.atomic.Value(usize).init(Bucket.stop_for(elem)); } self.buckets[state.new_index].store(new_array.ptr, .monotonic); @@ -338,7 +341,7 @@ const Core = struct { } /// Remove a given element from the table, SMR must be entered. - fn remove(self: *Self, link: *rcu.SList.Entry) void { + fn remove(self: *Core, link: *rcu.SList.Entry) void { var state = self.state.load(.acquire); var h = self.hash_link( link, @@ -394,7 +397,7 @@ const Core = struct { } } - fn bucket(self: *Self, state: State, which: Which, h: u32) *Bucket { + fn bucket(self: *Core, state: State, which: Which, h: u32) *Bucket { const idx: usize, const shift: u5 = switch (which) { .Cur => .{ state.current_index, @intCast(state.current_shift) }, .New => .{ state.new_index, @intCast(state.new_shift) }, @@ -408,12 +411,12 @@ const Core = struct { return @as(u32, 1) << shift; } - fn should_grow(self: *Self, elems: u32, size: u32) bool { + fn should_grow(self: *Core, elems: u32, size: u32) bool { const target_depth = ResizePolicy.depth_for(self.policy); return elems > target_depth * size; } - fn should_shrink(self: *Self, elems: u32, size: u32) bool { + fn should_shrink(self: *Core, elems: u32, size: u32) bool { const min_size = size_for_shift(self.min_shift); return switch (self.policy) { @@ -429,9 +432,9 @@ const Core = struct { /// This avoids going through multiple steps when resizing a busy table, /// i.e instead of doubling the size and doing 16 -> 32 -> 64, /// we can just do 16 -> 64 in a single resize. - fn target_shift_for(self: *Self, elems: u32) u5 { + fn target_shift_for(self: *Core, elems: u32) u5 { const depth = ResizePolicy.depth_for(self.policy); - const needed = std.math.divCeil(usize, elems, depth) catch unreachable; + const needed = @divCeil(elems, depth); var shift = self.min_shift; while (size_for_shift(shift) < needed) : (shift += 1) {} @@ -470,7 +473,8 @@ pub fn Table( return struct { core: Core, - const Self = @This(); + + const Map = @This(); const Options = struct { smr: *ke.smr.Domain, @@ -481,13 +485,13 @@ pub fn Table( /// Initialize a new table, `min_size` provides the minimum count of /// buckets. If `min_size` is 0, it will get clamped to sane defaults /// depending on the chosen policy. - pub fn init(self: *Self, opts: Options) !void { + pub fn init(self: *Map, opts: Options) !void { try self.core.init(opts.smr, opts.policy, opts.min_size); self.core.hash_link = hash_link; } /// Get an element given a key, SMR must be entered. - pub fn get(self: *Self, key: Context.Key) ?*T { + pub fn get(self: *Map, key: Context.Key) ?*T { const state = self.core.state.load(.acquire); const h = Context.hash( key, @@ -521,7 +525,7 @@ pub fn Table( /// Find an element or insert it given a key, SMR must be entered. pub fn get_or_insert( - self: *Self, + self: *Map, key: Context.Key, value: *rcu.SList.Entry, ) ?*T { @@ -590,11 +594,11 @@ pub fn Table( } /// Remove a given element from the table, SMR must be entered. - pub fn remove(self: *Self, link: *rcu.SList.Entry) void { + pub fn remove(self: *Map, link: *rcu.SList.Entry) void { self.core.remove(link); } - fn get_slow(self: *Self, key: Context.Key, bucket: *Bucket) ?*T { + fn get_slow(self: *Map, key: Context.Key, bucket: *Bucket) ?*T { // Wait until the bucket finishes moving. while (bucket.word.load(.monotonic) != Bucket.moved) { std.atomic.spinLoopHint(); diff --git a/src/rtl/avl.zig b/src/rtl/avl.zig index f0f640c..7c84125 100644 --- a/src/rtl/avl.zig +++ b/src/rtl/avl.zig @@ -1,4 +1,4 @@ -//! AVL tree implementation +//! AVL tree impLementation //! Invariant: For every node, the difference in height between its left //! and right subtrees is strictly limited to -1, 0 or 1. const bst = @import("bst.zig"); @@ -12,26 +12,28 @@ const zero: u2 = 0; const one: u2 = 2; fn get_bf(node: *bst.Node) u2 { - return node.parent.get_tag(); + return node.parent.tag(); } fn set_bf(node: *bst.Node, bf: u2) void { node.parent.set_tag(bf); } -pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order) type { +pub fn AVLTree( + comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order, +) type { return struct { tree: bst.BST(cmp), - const Self = @This(); + const Tree = @This(); - pub fn init() Self { - return Self{ + pub fn init() Tree { + return Tree{ .tree = bst.BST(cmp).init(), }; } - fn rotate_left(self: *Self, x: *bst.Node, z: *bst.Node) *bst.Node { + fn rotate_left(self: *Tree, x: *bst.Node, z: *bst.Node) *bst.Node { x.right = z.left; if (!self.tree.is_nil(z.left)) { @@ -53,7 +55,7 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde return z; } - fn rotate_right(self: *Self, x: *bst.Node, z: *bst.Node) *bst.Node { + fn rotate_right(self: *Tree, x: *bst.Node, z: *bst.Node) *bst.Node { x.left = z.right; if (!self.tree.is_nil(z.right)) { @@ -75,7 +77,11 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde return z; } - fn rotate_right_left(self: *Self, x: *bst.Node, z: *bst.Node) *bst.Node { + fn rotate_right_left( + self: *Tree, + x: *bst.Node, + z: *bst.Node, + ) *bst.Node { var y = z.left; var t2 = y.right; @@ -123,7 +129,11 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde return y; } - fn rotate_left_right(self: *Self, x: *bst.Node, z: *bst.Node) *bst.Node { + fn rotate_left_right( + self: *Tree, + x: *bst.Node, + z: *bst.Node, + ) *bst.Node { var y = z.right; var t2 = y.left; @@ -172,9 +182,14 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde } /// Insert the node `elem` into the tree. - pub fn insert(self: *Self, elem: *bst.Node) !void { + pub fn insert(self: *Tree, elem: *bst.Node) !void { if (self.tree.is_empty()) { - self.tree.insert(elem) catch unreachable; + self.tree.insert(elem) catch { + // This can't ever error since the tree is empty, so the + // element can't already exist. + unreachable; + }; + set_bf(elem, zero); return; } @@ -188,17 +203,18 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde // Now the tree may be unbalanced, rebalance it. var node = elem; - var parent = elem.parent.get_ptr(); + var parent = elem.parent.ptr(); var n = &bst.nil; - while (!self.tree.is_nil(parent)) : (parent = parent.parent.get_ptr()) { - const orig_parent = parent.parent.get_ptr(); + while (!self.tree.is_nil(parent)) : (parent = parent.parent.ptr()) { + const orig_parent = parent.parent.ptr(); if (node == parent.right) { if (get_bf(parent) == one) { - // This node has been inserted on the right of the parent, - // and the parent already has a balance factor of 1 (right-heavy), - // the parent's BF would then become +2, which breaks the invariant. + // This node has been inserted on the right of the + // parent, and the parent already has a balance factor + // of 1 (right-heavy), the parent's BF would then + // become +2, which breaks the invariant. // We need to rebalance. if (get_bf(node) == minus_one) { @@ -214,13 +230,16 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde // 2 // / \ // 1 3 - // In the code, `parent` would be 1 and `node` would be 2 + // In the code, `parent` would be 1 and `node` would + // be 2 n = self.rotate_left(parent, node); } } else { // The balance factor is either 0 or -1, increase it. - // If it was -1, now it is 0 so the subtree is perfectly balanced, we can stop. - // If it was 0, now it is +1, so we may need to check higher up for imbalances. + // If it was -1, now it is 0 so the subtree is + // perfectly balanced, we can stop. + // If it was 0, now it is +1, so we may need to check + // higher up for imbalances. if (get_bf(parent) == minus_one) { set_bf(parent, zero); @@ -236,9 +255,10 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde // The logic is the same as the other case but inverted. else { if (get_bf(parent) == minus_one) { - // This node has been inserted on the left of the parent, - // and the parent already has a balance factor of -1 (left-heavy), - // the parent's balance factor would then become -2, which breaks the invariant. + // This node has been inserted on the left of the + // parent, and the parent already has a balance factor + // of -1 (left-heavy), the parent's balance factor would + // then become -2, which breaks the invariant. // We need to rebalance. if (get_bf(node) == one) { @@ -276,10 +296,10 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde } /// Delete the node `elem` from the tree. - pub fn delete(self: *Self, elem: *bst.Node) void { + pub fn delete(self: *Tree, elem: *bst.Node) void { var node = elem; var original_parent = &bst.nil; - var parent = node.parent.get_ptr(); + var parent = node.parent.ptr(); var was_on_left = !self.tree.is_nil(parent) and parent.left == node; var b = zero; @@ -289,7 +309,7 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde self.tree.transplant(elem, elem.left); } else { var succ = self.tree.successor(node); - parent = succ.parent.get_ptr(); + parent = succ.parent.ptr(); was_on_left = !self.tree.is_nil(parent) and succ == parent.left; if (parent != elem) { @@ -297,7 +317,8 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde succ.right = elem.right; succ.right.parent.set_ptr(succ); } else { - // The successor is the direct child, so start rebalancing from it. + // The successor is the direct child, so start rebalancing + // from it. parent = succ; } @@ -311,7 +332,7 @@ pub fn AVLTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Orde // Rebalance the tree. while (!self.tree.is_nil(parent)) { - original_parent = parent.parent.get_ptr(); + original_parent = parent.parent.ptr(); if (node == parent.left or was_on_left) { was_on_left = false; diff --git a/src/rtl/barrier.zig b/src/rtl/barrier.zig index f0f64a6..38b0ab0 100644 --- a/src/rtl/barrier.zig +++ b/src/rtl/barrier.zig @@ -1,9 +1,11 @@ //! Atomic barrier implementations on various architectures. //! This is needed since Zig removed @fence. -const std = @import("std"); const builtin = @import("builtin"); +const std = @import("std"); +const assert = std.debug.assert; + pub inline fn fence(comptime ordering: std.builtin.AtomicOrder) void { switch (ordering) { .seq_cst, .acq_rel => mb(), @@ -67,7 +69,11 @@ inline fn rel() void { /// Copy to dest from src with atomic loads. /// Both addresses need to be aligned. -pub fn atomic_load_memcpy(dest: anytype, src: anytype, comptime ordering: std.builtin.AtomicOrder) void { +pub fn atomic_load_memcpy( + dest: anytype, + src: anytype, + comptime ordering: std.builtin.AtomicOrder, +) void { const T = @TypeOf(dest.*); const Word = switch (@alignOf(T)) { 1 => u8, @@ -78,7 +84,7 @@ pub fn atomic_load_memcpy(dest: anytype, src: anytype, comptime ordering: std.bu const word_size = @sizeOf(Word); const n = @sizeOf(T); - comptime std.debug.assert(n % word_size == 0); + comptime assert(n % word_size == 0); const src_words: [*]const volatile Word = @ptrCast(src); const dest_words: [*]Word = @ptrCast(dest); @@ -91,7 +97,11 @@ pub fn atomic_load_memcpy(dest: anytype, src: anytype, comptime ordering: std.bu /// Copy to dest from src with atomic stores. /// Both addresses need to be aligned. -pub fn atomic_store_memcpy(dest: anytype, src: anytype, comptime ordering: std.builtin.AtomicOrder) void { +pub fn atomic_store_memcpy( + dest: anytype, + src: anytype, + comptime ordering: std.builtin.AtomicOrder, +) void { const T = @TypeOf(dest.*); const Word = switch (@alignOf(T)) { 1 => u8, @@ -102,7 +112,7 @@ pub fn atomic_store_memcpy(dest: anytype, src: anytype, comptime ordering: std.b const word_size = @sizeOf(Word); const n = @sizeOf(T); - comptime std.debug.assert(n % word_size == 0); + comptime assert(n % word_size == 0); const src_words: [*]const volatile Word = @ptrCast(src); const dest_words: [*]Word = @ptrCast(dest); diff --git a/src/rtl/bitmap.zig b/src/rtl/bitmap.zig index b763e17..8199817 100644 --- a/src/rtl/bitmap.zig +++ b/src/rtl/bitmap.zig @@ -1,7 +1,13 @@ const std = @import("std"); +const AtomicOrder = std.builtin.AtomicOrder; +const assert = std.debug.assert; + +fn ClosestType(comptime N: usize) type { + const bits = std.math.ceilPowerOfTwo(usize, N) catch { + // Shouldn't happen. + unreachable; + }; -fn get_closest_type(comptime N: usize) type { - const bits = std.math.ceilPowerOfTwo(usize, N) catch unreachable; return @Int(.unsigned, bits); } @@ -12,15 +18,15 @@ fn bit_mask(comptime Word: type, bit_index: usize) Word { /// Static bitmap that uses the most efficient underlying word size for N < 64 pub fn BitMap(comptime N: usize) type { return struct { - pub const Word = if (N > 32) u64 else get_closest_type(N); - const Self = @This(); + pub const Word = if (N > 32) u64 else ClosestType(N); + const Bitmap = @This(); const bits_per_word = @bitSizeOf(Word); const arr_len = (N + bits_per_word - 1) / bits_per_word; storage: [arr_len]Word, fn valid_word_mask(word_index: usize) Word { - std.debug.assert(word_index < arr_len); + assert(word_index < arr_len); if (word_index != arr_len - 1) return std.math.maxInt(Word); @@ -30,39 +36,39 @@ pub fn BitMap(comptime N: usize) type { } /// Initialize a bitmap, setting all bits to `val`. - pub fn init(val: bool) Self { + pub fn init(val: bool) Bitmap { return .{ - .storage = @splat(if (val) std.math.maxInt(Self.Word) else 0), + .storage = @splat(if (val) std.math.maxInt(Bitmap.Word) else 0), }; } /// Set a given bit. - pub fn set(self: *Self, bit: usize) void { - std.debug.assert(bit < N); - const word_index = bit / Self.bits_per_word; - const bit_index = bit % Self.bits_per_word; + pub fn set(self: *Bitmap, bit: usize) void { + assert(bit < N); + const word_index = bit / Bitmap.bits_per_word; + const bit_index = bit % Bitmap.bits_per_word; self.storage[word_index] |= bit_mask(Word, bit_index); } /// Clear a given bit. - pub fn clear(self: *Self, bit: usize) void { - std.debug.assert(bit < N); - const word_index = bit / Self.bits_per_word; - const bit_index = bit % Self.bits_per_word; + pub fn clear(self: *Bitmap, bit: usize) void { + assert(bit < N); + const word_index = bit / Bitmap.bits_per_word; + const bit_index = bit % Bitmap.bits_per_word; self.storage[word_index] &= ~bit_mask(Word, bit_index); } /// Get a given bit. - pub fn get(self: *const Self, bit: usize) bool { - std.debug.assert(bit < N); - const word_index = bit / Self.bits_per_word; - const bit_index = bit % Self.bits_per_word; + pub fn get(self: *const Bitmap, bit: usize) bool { + assert(bit < N); + const word_index = bit / Bitmap.bits_per_word; + const bit_index = bit % Bitmap.bits_per_word; return self.storage[word_index] & bit_mask(Word, bit_index) != 0; } /// Count the number of set bits in the bitmap. - pub fn count(self: *const Self) usize { + pub fn count(self: *const Bitmap) usize { var total: usize = 0; for (self.storage, 0..) |word, i| { total += @intCast(@popCount(word & valid_word_mask(i))); @@ -71,7 +77,7 @@ pub fn BitMap(comptime N: usize) type { } pub const Iterator = struct { - bitmap: *const Self, + bitmap: *const Bitmap, word_index: usize = 0, word: Word = 0, @@ -81,25 +87,29 @@ pub fn BitMap(comptime N: usize) type { if (self.word != 0) { const bit_index: usize = @intCast(@ctz(self.word)); self.word &= self.word - 1; - return ((self.word_index - 1) * bits_per_word) + bit_index; + return ((self.word_index - 1) * bits_per_word) + + bit_index; } if (self.word_index >= arr_len) return null; - self.word = self.bitmap.storage[self.word_index] & valid_word_mask(self.word_index); + self.word = self.bitmap.storage[self.word_index] & + valid_word_mask(self.word_index); self.word_index += 1; } } }; /// Iterate over the indexes of set bits in ascending order. - pub fn iter(self: *const Self) Iterator { + pub fn iter(self: *const Bitmap) Iterator { return .{ .bitmap = self }; } /// Check whether all bits in the bitmap are equal to a given value. - pub fn is_all(self: *const Self, value: bool) bool { - const v: Self.Word = if (value) std.math.maxInt(Self.Word) else 0; + pub fn is_all(self: *const Bitmap, value: bool) bool { + const v: Bitmap.Word = if (value) std.math.maxInt(Bitmap.Word) + else + 0; for (self.storage) |word| { if (word != v) return false; } @@ -108,18 +118,19 @@ pub fn BitMap(comptime N: usize) type { }; } -/// Atomic static bitmap with the same API as `BitMap`, but each operation accepts a memory ordering. +/// Atomic static bitmap with the same API as `BitMap`, but each operation +/// accepts a memory ordering. pub fn AtomicBitMap(comptime N: usize) type { return struct { - pub const Word = if (N > 32) u64 else get_closest_type(N); - const Self = @This(); + pub const Word = if (N > 32) u64 else ClosestType(N); + const Bitmap = @This(); const bits_per_word = @bitSizeOf(Word); const arr_len = (N + bits_per_word - 1) / bits_per_word; storage: [arr_len]std.atomic.Value(Word), fn valid_word_mask(word_index: usize) Word { - std.debug.assert(word_index < arr_len); + assert(word_index < arr_len); if (word_index != arr_len - 1) return std.math.maxInt(Word); @@ -129,9 +140,9 @@ pub fn AtomicBitMap(comptime N: usize) type { } /// Initialize a bitmap, setting all bits to `val`. - pub fn init(val: bool) Self { + pub fn init(val: bool) Bitmap { var storage: [arr_len]std.atomic.Value(Word) = undefined; - const word: Word = if (val) std.math.maxInt(Self.Word) else 0; + const word: Word = if (val) std.math.maxInt(Bitmap.Word) else 0; inline for (&storage) |*elem| { elem.* = std.atomic.Value(Word).init(word); } @@ -139,58 +150,86 @@ pub fn AtomicBitMap(comptime N: usize) type { } /// Set a given bit. - pub fn set(self: *Self, bit: usize, comptime ordering: std.builtin.AtomicOrder) void { - std.debug.assert(bit < N); - const word_index = bit / Self.bits_per_word; - const bit_index = bit % Self.bits_per_word; + pub fn set( + self: *Bitmap, + bit: usize, + comptime order: AtomicOrder, + ) void { + assert(bit < N); + const word_index = bit / Bitmap.bits_per_word; + const bit_index = bit % Bitmap.bits_per_word; - _ = self.storage[word_index].fetchOr(bit_mask(Word, bit_index), ordering); + _ = self.storage[word_index].fetchOr( + bit_mask(Word, bit_index), + order, + ); } /// Clear a given bit. - pub fn clear(self: *Self, bit: usize, comptime ordering: std.builtin.AtomicOrder) void { - std.debug.assert(bit < N); - const word_index = bit / Self.bits_per_word; - const bit_index = bit % Self.bits_per_word; + pub fn clear( + self: *Bitmap, + bit: usize, + comptime order: AtomicOrder, + ) void { + assert(bit < N); + const word_index = bit / Bitmap.bits_per_word; + const bit_index = bit % Bitmap.bits_per_word; - _ = self.storage[word_index].fetchAnd(~bit_mask(Word, bit_index), ordering); + _ = self.storage[word_index].fetchAnd( + ~bit_mask(Word, bit_index), + order, + ); } /// Get a given bit. - pub fn get(self: *const Self, bit: usize, comptime ordering: std.builtin.AtomicOrder) bool { - std.debug.assert(bit < N); - const word_index = bit / Self.bits_per_word; - const bit_index = bit % Self.bits_per_word; - return self.storage[word_index].load(ordering) & bit_mask(Word, bit_index) != 0; + pub fn get( + self: *const Bitmap, + bit: usize, + comptime order: AtomicOrder, + ) bool { + assert(bit < N); + const word_index = bit / Bitmap.bits_per_word; + const bit_index = bit % Bitmap.bits_per_word; + return self.storage[word_index].load(order) & + bit_mask(Word, bit_index) != 0; } /// Count the number of set bits in the bitmap. - pub fn count(self: *const Self, comptime ordering: std.builtin.AtomicOrder) usize { + pub fn count( + self: *const Bitmap, + comptime order: AtomicOrder, + ) usize { var total: usize = 0; for (self.storage, 0..) |word, i| { - total += @intCast(@popCount(word.load(ordering) & valid_word_mask(i))); + total += @intCast(@popCount(word.load(order) & + valid_word_mask(i))); } return total; } - pub fn Iterator(comptime ordering: std.builtin.AtomicOrder) type { + pub fn Iterator(comptime order: AtomicOrder) type { return struct { - bitmap: *const Self, + bitmap: *const Bitmap, word_index: usize = 0, word: Word = 0, - /// Return the index of the next set bit, or null when exhausted. + /// Return the index of the next set bit, or null when + /// exhausted. pub fn next(self: *@This()) ?usize { while (true) { if (self.word != 0) { const bit_index: usize = @intCast(@ctz(self.word)); self.word &= self.word - 1; - return ((self.word_index - 1) * bits_per_word) + bit_index; + return ((self.word_index - 1) * bits_per_word) + + bit_index; } if (self.word_index >= arr_len) return null; - self.word = self.bitmap.storage[self.word_index].load(ordering) & valid_word_mask(self.word_index); + const word = &self.bitmap.storage[self.word_index]; + + self.word = word.load(order) & + valid_word_mask(self.word_index); self.word_index += 1; } } @@ -198,15 +237,23 @@ pub fn AtomicBitMap(comptime N: usize) type { } /// Iterate over the indexes of set bits in ascending order. - pub fn iter(self: *const Self, comptime ordering: std.builtin.AtomicOrder) Iterator(ordering) { + pub fn iter( + self: *const Bitmap, + comptime order: AtomicOrder, + ) Iterator(order) { return .{ .bitmap = self }; } /// Check whether all bits in the bitmap are equal to a given value. - pub fn is_all(self: *const Self, value: bool, comptime ordering: std.builtin.AtomicOrder) bool { - const v: Self.Word = if (value) std.math.maxInt(Self.Word) else 0; + pub fn is_all( + self: *const Bitmap, + value: bool, + comptime order: AtomicOrder, + ) bool { + const v: Bitmap.Word = if (value) std.math.maxInt(Bitmap.Word) else + 0; for (self.storage) |word| { - if (word.load(ordering) != v) return false; + if (word.load(order) != v) return false; } return true; } @@ -214,37 +261,37 @@ pub fn AtomicBitMap(comptime N: usize) type { } test "BitMap has proper underlying type" { - std.debug.assert(BitMap(8).Word == u8); - std.debug.assert(BitMap(10).Word == u16); - std.debug.assert(BitMap(16).Word == u16); - std.debug.assert(BitMap(21).Word == u32); - std.debug.assert(BitMap(32).Word == u32); - std.debug.assert(BitMap(64).Word == u64); - std.debug.assert(BitMap(128).Word == u64); - std.debug.assert(BitMap(1024).Word == u64); + assert(BitMap(8).Word == u8); + assert(BitMap(10).Word == u16); + assert(BitMap(16).Word == u16); + assert(BitMap(21).Word == u32); + assert(BitMap(32).Word == u32); + assert(BitMap(64).Word == u64); + assert(BitMap(128).Word == u64); + assert(BitMap(1024).Word == u64); } test BitMap { var bitmap = BitMap(10).init(false); - std.debug.assert(@TypeOf(bitmap).Word == u16); - std.debug.assert(bitmap.get(0) == false); + assert(@TypeOf(bitmap).Word == u16); + assert(bitmap.get(0) == false); bitmap.set(0); - std.debug.assert(bitmap.get(0) == true); + assert(bitmap.get(0) == true); bitmap.clear(0); - std.debug.assert(bitmap.get(0) == false); + assert(bitmap.get(0) == false); } test "BitMap count and iter" { var bitmap = BitMap(70).init(false); - std.debug.assert(bitmap.count() == 0); + assert(bitmap.count() == 0); { var it = bitmap.iter(); - std.debug.assert(it.next() == null); + assert(it.next() == null); } bitmap.set(0); @@ -253,54 +300,54 @@ test "BitMap count and iter" { bitmap.set(64); bitmap.set(69); - std.debug.assert(bitmap.count() == 5); + assert(bitmap.count() == 5); var it = bitmap.iter(); - std.debug.assert(it.next().? == 0); - std.debug.assert(it.next().? == 3); - std.debug.assert(it.next().? == 63); - std.debug.assert(it.next().? == 64); - std.debug.assert(it.next().? == 69); - std.debug.assert(it.next() == null); + assert(it.next().? == 0); + assert(it.next().? == 3); + assert(it.next().? == 63); + assert(it.next().? == 64); + assert(it.next().? == 69); + assert(it.next() == null); bitmap.clear(63); - std.debug.assert(bitmap.count() == 4); + assert(bitmap.count() == 4); var all = BitMap(70).init(true); - std.debug.assert(all.count() == 70); + assert(all.count() == 70); var all_it = all.iter(); var expected: usize = 0; while (all_it.next()) |bit| : (expected += 1) { - std.debug.assert(bit == expected); + assert(bit == expected); } - std.debug.assert(expected == 70); + assert(expected == 70); } test AtomicBitMap { var bitmap = AtomicBitMap(10).init(false); - std.debug.assert(@TypeOf(bitmap).Word == u16); - std.debug.assert(bitmap.get(0, .seq_cst) == false); + assert(@TypeOf(bitmap).Word == u16); + assert(bitmap.get(0, .seq_cst) == false); bitmap.set(0, .seq_cst); - std.debug.assert(bitmap.get(0, .seq_cst) == true); - std.debug.assert(bitmap.is_all(false, .seq_cst) == false); + assert(bitmap.get(0, .seq_cst) == true); + assert(bitmap.is_all(false, .seq_cst) == false); bitmap.clear(0, .seq_cst); - std.debug.assert(bitmap.get(0, .seq_cst) == false); - std.debug.assert(bitmap.is_all(false, .seq_cst) == true); + assert(bitmap.get(0, .seq_cst) == false); + assert(bitmap.is_all(false, .seq_cst) == true); } test "AtomicBitMap count and iter" { var bitmap = AtomicBitMap(70).init(false); - std.debug.assert(bitmap.count(.seq_cst) == 0); + assert(bitmap.count(.seq_cst) == 0); { var it = bitmap.iter(.seq_cst); - std.debug.assert(it.next() == null); + assert(it.next() == null); } bitmap.set(0, .seq_cst); @@ -309,27 +356,27 @@ test "AtomicBitMap count and iter" { bitmap.set(64, .seq_cst); bitmap.set(69, .seq_cst); - std.debug.assert(bitmap.count(.seq_cst) == 5); + assert(bitmap.count(.seq_cst) == 5); var it = bitmap.iter(.seq_cst); - std.debug.assert(it.next().? == 0); - std.debug.assert(it.next().? == 3); - std.debug.assert(it.next().? == 63); - std.debug.assert(it.next().? == 64); - std.debug.assert(it.next().? == 69); - std.debug.assert(it.next() == null); + assert(it.next().? == 0); + assert(it.next().? == 3); + assert(it.next().? == 63); + assert(it.next().? == 64); + assert(it.next().? == 69); + assert(it.next() == null); bitmap.clear(63, .seq_cst); - std.debug.assert(bitmap.count(.seq_cst) == 4); + assert(bitmap.count(.seq_cst) == 4); var all = AtomicBitMap(70).init(true); - std.debug.assert(all.count(.seq_cst) == 70); + assert(all.count(.seq_cst) == 70); var all_it = all.iter(.seq_cst); var expected: usize = 0; while (all_it.next()) |bit| : (expected += 1) { - std.debug.assert(bit == expected); + assert(bit == expected); } - std.debug.assert(expected == 70); + assert(expected == 70); } diff --git a/src/rtl/bst.zig b/src/rtl/bst.zig index 1a6a378..d43ce5e 100644 --- a/src/rtl/bst.zig +++ b/src/rtl/bst.zig @@ -14,27 +14,29 @@ pub var nil: Node = .{ }; /// Base struct for binary search trees. -/// This should be used as a field in other types that are based on binary search trees, such as red-black trees or AVL trees. +/// This should be used as a field in other types that are based on +/// binary search trees, such as red-black trees or AVL trees. pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type { return struct { - const Self = @This(); + const Tree = @This(); root: *Node = &nil, - pub fn init() Self { - return Self{ + pub fn init() Tree { + return Tree{ .root = &nil, }; } /// Return whether or not the given node is the sentinel nil node. - pub fn is_nil(self: *Self, node: *Node) bool { + pub fn is_nil(self: *Tree, node: *Node) bool { _ = self; return node == &nil; } - /// Search for a node in the BST that matches `elem`, and return a pointer to it if found. - pub fn search(self: *Self, elem: *Node) ?*Node { + /// Search for a node in the BST that matches `elem`. + /// Return a pointer to it if found. + pub fn search(self: *Tree, elem: *Node) ?*Node { var node = self.root; while (!self.is_nil(node)) { const order = cmp(elem, node); @@ -49,8 +51,9 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type { return null; } - /// Return a pointer to the minimum node in the subtree rooted at `node`. - pub fn minimum(self: *Self, node: *Node) *Node { + /// Return a pointer to the minimum node in the subtree + /// rooted at `node`. + pub fn minimum(self: *Tree, node: *Node) *Node { var current = node; while (!self.is_nil(current.left)) { current = current.left; @@ -58,8 +61,9 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type { return current; } - /// Return a pointer to the maximum node in the subtree rooted at `node`. - pub fn maximum(self: *Self, node: *Node) *Node { + /// Return a pointer to the maximum node in the subtree + /// rooted at `node`. + pub fn maximum(self: *Tree, node: *Node) *Node { var current = node; while (!self.is_nil(current.right)) { current = current.right; @@ -67,8 +71,9 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type { return current; } - /// Return a pointer to the successor of `node`, or nil if `node` is the maximum element in the tree. - pub fn successor(self: *Self, node: *Node) *Node { + /// Return a pointer to the successor of `node`, or nil if `node` + /// is the maximum element in the tree. + pub fn successor(self: *Tree, node: *Node) *Node { if (!self.is_nil(node.right)) { // Minimum of right subtree return self.minimum(node.right); @@ -76,10 +81,10 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type { // Go up and find it. var current = node; - var parent = current.parent.get_ptr(); + var parent = current.parent.ptr(); while (!self.is_nil(parent) and current == parent.right) { current = parent; - parent = parent.parent.get_ptr(); + parent = parent.parent.ptr(); } if (parent == self.root) { @@ -89,9 +94,9 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type { return parent; } - /// Transplant the subtree rooted at `u` with the subtree rooted at `v`. This is used as a helper function. - pub fn transplant(self: *Self, u: *Node, v: *Node) void { - const u_parent = u.parent.get_ptr(); + /// Transplant the subtree rooted at `u` with the subtree rooted at `v`. + pub fn transplant(self: *Tree, u: *Node, v: *Node) void { + const u_parent = u.parent.ptr(); if (self.is_nil(u_parent)) { self.root = v; } else if (u == u_parent.left) { @@ -104,7 +109,7 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type { // Insert `elem` into the BST, if already inserted, do nothing. // This does not perform any balancing. - pub fn insert(self: *Self, elem: *Node) !void { + pub fn insert(self: *Tree, elem: *Node) !void { var current = self.root; if (self.is_nil(current)) { @@ -142,7 +147,7 @@ pub fn BST(comptime cmp: fn (*const Node, *const Node) std.math.Order) type { } /// Return whether or not the tree is empty. - pub fn is_empty(self: *Self) bool { + pub fn is_empty(self: *Tree) bool { return self.is_nil(self.root); } }; @@ -162,7 +167,7 @@ fn node_cmp(a: *const Node, b: *const Node) std.math.Order { } test BST { - var tree: BST(node_cmp) = .init(); + var tree = BST(node_cmp).init(); var nodes = [_]MyNode{ .{ .value = 5, .node = undefined }, @@ -181,6 +186,11 @@ test BST { for (&nodes) |*node| { const found = tree.search(&node.node); try std.testing.expect(found != null); - try std.testing.expectEqual(node.value, @as(*MyNode, @fieldParentPtr("node", found.?)).value); + const f = found orelse unreachable; + + try std.testing.expectEqual( + node.value, + @as(*MyNode, @fieldParentPtr("node", f)).value, + ); } } diff --git a/src/rtl/cmdline.zig b/src/rtl/cmdline.zig index 5a573ee..d673c42 100644 --- a/src/rtl/cmdline.zig +++ b/src/rtl/cmdline.zig @@ -51,7 +51,7 @@ test "get_number() should return the integer value of a key" { try std.testing.expectEqual(100, get_number(cmdline, "baz").?); } -test "get_number() should return null if the key is not found or value is not an integer" { +test "get_number() returns null if the key not found or value not an integer" { const cmdline = "foo=42 baz=qux"; try std.testing.expect(get_number(cmdline, "quux") == null); try std.testing.expect(get_number(cmdline, "baz") == null); diff --git a/src/rtl/handoff.zig b/src/rtl/handoff.zig index 1a3ba86..1db15c2 100644 --- a/src/rtl/handoff.zig +++ b/src/rtl/handoff.zig @@ -1,15 +1,15 @@ -//! Lock-free singly linked list with an activation callback when items are first inserted. -//! This is meant for use where there is a set of objects being worked upon in an -//! asynchronous context. +//! Lock-free singly linked list with an activation callback invoked when items +//! are first inserted. +//! This is meant for use where there is a set of objects being worked upon in +//! an asynchronous context. //! Data structure generalized from the NT kernel reaper lists, and inspired by -//! MINTIA's custody list (https://github.com/xrarch/mintia2/blob/main/OS/Executive/Ke/KeCustodyList.jkl) +//! MINTIA's custody list (https://github.com/xrarch/mintia2). const std = @import("std"); pub const HandoffList = struct { - const ActivationFn = *const fn (*Self) void; - - const Self = @This(); + const ActivationFn = *const fn (*HandoffList) void; + const Callback = *const fn (obj: *anyopaque, ctx: ?*anyopaque) void; /// Non-null value indicating processing is in progress. const processing_tag: *anyopaque = @ptrFromInt(1); @@ -17,21 +17,26 @@ pub const HandoffList = struct { activation: ActivationFn, head: std.atomic.Value(?*anyopaque), - pub fn init(activation: ActivationFn) Self { + pub fn init(activation: ActivationFn) HandoffList { return .{ .activation = activation, - .head = .init(null), + .head = std.atomic.Value(?*anyopaque).init(null), }; } /// Insert an element at the head of the list. /// If the list was previously empty, the activation routine will be called. - pub fn insert(self: *Self, link: *?*anyopaque) void { + pub fn insert(self: *HandoffList, link: *?*anyopaque) void { var head = self.head.load(.monotonic); while (true) { link.* = head; - head = self.head.cmpxchgWeak(head, @ptrCast(link), .release, .monotonic) orelse break; + head = self.head.cmpxchgWeak( + head, + @ptrCast(link), + .release, + .monotonic, + ) orelse break; } if (head == null) { @@ -43,17 +48,27 @@ pub const HandoffList = struct { /// Pop the entire list and process it. /// The callback takes the object pointer as its first parameter /// and ctx as its second. - pub fn process(self: *Self, callback: *const fn (*anyopaque, ?*anyopaque) void, ctx: ?*anyopaque) void { + pub fn process( + self: *HandoffList, + callback: Callback, + ctx: ?*anyopaque, + ) void { while (true) { const head = self.head.swap(processing_tag, .acquire); - // Someone else is already processing it (or we set it previously and got nothing). + // Someone else is already processing it, or we set it previously + // and got nothing. if (head == processing_tag) return; - // List is empty; this shouldn't happen, but it'll just return on the next - // iteration if it is truly empty. + // List is empty; this shouldn't happen, but it'll just return + // on the next iteration if it is truly empty. if (head == null) { - _ = self.head.cmpxchgStrong(processing_tag, null, .release, .monotonic) orelse return; + _ = self.head.cmpxchgStrong( + processing_tag, + null, + .release, + .monotonic, + ) orelse return; continue; } @@ -68,7 +83,12 @@ pub const HandoffList = struct { } // If the cmpxchg fails, a new item was added; also process it. - _ = self.head.cmpxchgStrong(processing_tag, null, .release, .monotonic) orelse return; + _ = self.head.cmpxchgStrong( + processing_tag, + null, + .release, + .monotonic, + ) orelse return; } } }; @@ -99,7 +119,7 @@ fn test_activation(list: *HandoffList) void { fn record_node(node_ptr: *anyopaque, ctx_ptr: ?*anyopaque) void { const node: *TestNode = @ptrCast(@alignCast(node_ptr)); - const ctx: *TestContext = @ptrCast(@alignCast(ctx_ptr.?)); + const ctx: *TestContext = @ptrCast(@alignCast(ctx_ptr orelse unreachable)); ctx.processed[ctx.processed_len] = node.value; ctx.processed_len += 1; @@ -107,7 +127,7 @@ fn record_node(node_ptr: *anyopaque, ctx_ptr: ?*anyopaque) void { fn record_node_and_insert(node_ptr: *anyopaque, ctx_ptr: ?*anyopaque) void { const node: *TestNode = @ptrCast(@alignCast(node_ptr)); - const ctx: *TestContext = @ptrCast(@alignCast(ctx_ptr.?)); + const ctx: *TestContext = @ptrCast(@alignCast(ctx_ptr orelse unreachable)); ctx.processed[ctx.processed_len] = node.value; ctx.processed_len += 1; diff --git a/src/rtl/linker_set.zig b/src/rtl/linker_set.zig index 92935c5..389605d 100644 --- a/src/rtl/linker_set.zig +++ b/src/rtl/linker_set.zig @@ -1,3 +1,5 @@ +//! Linker sets, inspired by FreeBSD. + const std = @import("std"); /// Used for declaring global objects that get collected by the linker @@ -13,8 +15,6 @@ pub fn LinkerSet(name: []const u8, comptime T: type) type { const start = @extern(*u8, .{ .name = "__start_set_" ++ name }); const end = @extern(*u8, .{ .name = "__stop_set_" ++ name }); - const Self = @This(); - pub fn count() usize { return (@intFromPtr(end) - @intFromPtr(start)) / @sizeOf(T); } @@ -41,7 +41,8 @@ pub fn LinkerSet(name: []const u8, comptime T: type) type { @export( @as(*const usize, @ptrCast(&storage)), .{ - .name = "_set_elem_" ++ name ++ "_" ++ @typeName(Tag(elem)), + .name = "_set_elem_" ++ name ++ "_" ++ + @typeName(Tag(elem)), .section = "set_" ++ name, }, ); diff --git a/src/rtl/pairing_heap.zig b/src/rtl/pairing_heap.zig index f416556..1053d98 100644 --- a/src/rtl/pairing_heap.zig +++ b/src/rtl/pairing_heap.zig @@ -1,5 +1,6 @@ //! Intrusive Pairing Heap implementation. const std = @import("std"); +const assert = std.debug.assert; pub const Node = struct { child: ?*Node = null, @@ -7,9 +8,12 @@ pub const Node = struct { prev: ?*Node = null, }; -pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, *Node) std.math.Order) type { +pub fn PairingHeap( + comptime order: enum { Min, Max }, + comptime cmp: fn (*Node, *Node) std.math.Order, +) type { return struct { - const Self = @This(); + const Heap = @This(); /// Number of elements in the heap size: usize = 0, @@ -19,8 +23,8 @@ pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, * fn meld(a: *Node, b: *Node) *Node { const a_wins = switch (order) { - .min => cmp(a, b) != .gt, // a <= b - .max => cmp(a, b) == .gt, // a > b + .Min => cmp(a, b) != .gt, // a <= b + .Max => cmp(a, b) == .gt, // a > b }; if (a_wins) { @@ -70,7 +74,7 @@ pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, * } // Second pass: fold pairs right to left - var result = pairs.?; + var result = pairs orelse unreachable; var rest = result.next; result.next = null; while (rest) |p| { @@ -81,7 +85,7 @@ pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, * return result; } - pub fn init() Self { + pub fn init() Heap { return .{ .size = 0, .root = null, @@ -89,7 +93,7 @@ pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, * } /// Insert `node` into the heap. - pub fn insert(self: *Self, node: *Node) void { + pub fn insert(self: *Heap, node: *Node) void { node.* = .{ .child = null, .next = null, @@ -107,7 +111,7 @@ pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, * } /// Return and remove the topmost element. - pub fn pop(self: *Self) ?*Node { + pub fn pop(self: *Heap) ?*Node { const root = self.root orelse return null; const child = root.child; @@ -125,13 +129,13 @@ pub fn PairingHeap(comptime order: enum { min, max }, comptime cmp: fn (*Node, * } /// Remove `node` from the heap. - pub fn remove(self: *Self, node: *Node) void { + pub fn remove(self: *Heap, node: *Node) void { if (self.root == node) { _ = self.pop(); return; } - std.debug.assert(self.root != null); + assert(self.root != null); self.size -= 1; @@ -174,8 +178,8 @@ fn my_cmp(a: *Node, b: *Node) std.math.Order { return std.math.order(ta.value, tb.value); } -const MinHeap = PairingHeap(.min, my_cmp); -const MaxHeap = PairingHeap(.max, my_cmp); +const MinHeap = PairingHeap(.Min, my_cmp); +const MaxHeap = PairingHeap(.Max, my_cmp); test "empty heap" { var heap = MinHeap.init(); diff --git a/src/rtl/rbtree.zig b/src/rtl/rbtree.zig index b2ec8e0..6e96a90 100644 --- a/src/rtl/rbtree.zig +++ b/src/rtl/rbtree.zig @@ -1,5 +1,6 @@ //! Red-Black Tree implementation mostly based on pseudocode from CLRS. -//! Refer to the book for details on the algorithms and properties of red-black trees. +//! Refer to the book for details on the algorithms and properties of red-black +//! trees. const bst = @import("bst.zig"); const std = @import("std"); @@ -7,22 +8,24 @@ const black: u2 = 0; const red: u2 = 1; fn get_color(node: *bst.Node) u2 { - return node.parent.get_tag(); + return node.parent.tag(); } fn set_color(node: *bst.Node, color: u2) void { node.parent.set_tag(color); } -pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order) type { +pub fn RBTree( + comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order, +) type { return struct { tree: bst.BST(cmp), - const Self = @This(); + const Tree = @This(); - fn rotate_right(self: *Self, node: *bst.Node) void { + fn rotate_right(self: *Tree, node: *bst.Node) void { var y = node.left; - const parent = node.parent.get_ptr(); + const parent = node.parent.ptr(); node.left = y.right; @@ -44,9 +47,9 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order node.parent.set_ptr(y); } - fn rotate_left(self: *Self, node: *bst.Node) void { + fn rotate_left(self: *Tree, node: *bst.Node) void { var y = node.right; - const parent = node.parent.get_ptr(); + const parent = node.parent.ptr(); node.right = y.left; @@ -68,16 +71,19 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order node.parent.set_ptr(y); } - pub fn init() Self { - return Self{ + pub fn init() Tree { + return Tree{ .tree = bst.BST(cmp).init(), }; } /// Insert the node `elem` into the tree. - pub fn insert(self: *Self, elem: *bst.Node) !void { + pub fn insert(self: *Tree, elem: *bst.Node) !void { if (self.tree.is_empty()) { - self.tree.insert(elem) catch unreachable; + self.tree.insert(elem) catch { + // This can't happen as the tree is empty. + unreachable; + }; set_color(elem, black); return; } @@ -90,11 +96,11 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order set_color(elem, red); var node = elem; - var parent = node.parent.get_ptr(); + var parent = node.parent.ptr(); // The tree might be unbalanced, rebalance it. while (get_color(parent) == red) { - var grandparent = parent.parent.get_ptr(); + var grandparent = parent.parent.ptr(); // The parent of the node is a left child if (parent == grandparent.left) { @@ -108,19 +114,19 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order set_color(uncle, black); set_color(grandparent, red); node = grandparent; - parent = node.parent.get_ptr(); + parent = node.parent.ptr(); } else { // Otherwise, if the node is a right child, rotate left // to make it a left child. if (node == parent.right) { node = parent; self.rotate_left(node); - parent = node.parent.get_ptr(); - grandparent = parent.parent.get_ptr(); + parent = node.parent.ptr(); + grandparent = parent.parent.ptr(); } - // Then, recolor the parent and grandparent and rotate right - // to maintain the properties of the red-black tree. + // Then, recolor the parent and grandparent and rotate + // right to maintain the invariants. set_color(parent, black); set_color(grandparent, red); self.rotate_right(grandparent); @@ -134,13 +140,13 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order set_color(uncle, black); set_color(grandparent, red); node = grandparent; - parent = node.parent.get_ptr(); + parent = node.parent.ptr(); } else { if (node == parent.left) { node = parent; self.rotate_right(node); - parent = node.parent.get_ptr(); - grandparent = parent.parent.get_ptr(); + parent = node.parent.ptr(); + grandparent = parent.parent.ptr(); } set_color(parent, black); @@ -148,7 +154,7 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order self.rotate_left(grandparent); } - parent = node.parent.get_ptr(); + parent = node.parent.ptr(); } } @@ -156,7 +162,7 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order } /// Delete the node `elem` from the tree. - pub fn delete(self: *Self, elem: *bst.Node) void { + pub fn delete(self: *Tree, elem: *bst.Node) void { var node = elem; var child = &bst.nil; var orig_color = get_color(node); @@ -175,7 +181,7 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order orig_color = get_color(node); child = node.right; - if (node.parent.get_ptr() == elem) { + if (node.parent.ptr() == elem) { child.parent.set_ptr(node); } else { self.tree.transplant(node, node.right); @@ -198,7 +204,7 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order // Fix the tree while (node != self.tree.root and get_color(node) == black) { - var parent = node.parent.get_ptr(); + var parent = node.parent.ptr(); if (node == parent.left) { var sibling = parent.right; @@ -208,11 +214,13 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order set_color(parent, red); self.rotate_left(parent); - parent = node.parent.get_ptr(); + parent = node.parent.ptr(); sibling = parent.right; } - if (get_color(sibling.left) == black and get_color(sibling.right) == black) { + if (get_color(sibling.left) == black and + get_color(sibling.right) == black) + { set_color(sibling, red); node = parent; } else { @@ -220,7 +228,7 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order set_color(sibling.left, black); set_color(sibling, red); self.rotate_right(sibling); - parent = node.parent.get_ptr(); + parent = node.parent.ptr(); sibling = parent.right; } @@ -238,11 +246,13 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order set_color(parent, red); self.rotate_right(parent); - parent = node.parent.get_ptr(); + parent = node.parent.ptr(); sibling = parent.left; } - if (get_color(sibling.left) == black and get_color(sibling.right) == black) { + if (get_color(sibling.left) == black and + get_color(sibling.right) == black) + { set_color(sibling, red); node = parent; } else { @@ -250,7 +260,7 @@ pub fn RBTree(comptime cmp: fn (*const bst.Node, *const bst.Node) std.math.Order set_color(sibling.right, black); set_color(sibling, red); self.rotate_left(sibling); - parent = node.parent.get_ptr(); + parent = node.parent.ptr(); sibling = parent.left; } diff --git a/src/rtl/root.zig b/src/rtl/root.zig index fb9f848..6430265 100644 --- a/src/rtl/root.zig +++ b/src/rtl/root.zig @@ -22,6 +22,45 @@ pub fn comptime_error(comptime msg: []const u8, args: anytype) void { @compileError(std.fmt.comptimePrint(msg, args)); } +const InterfaceError = enum { + MissingField, + FieldType, + MissingMethod, + ParameterCount, + ReturnType, + ParameterType, + MissingType, + TypeDeclaration, + MissingVariable, + VariableType, +}; + +fn interface_error(comptime err: InterfaceError, args: anytype) void { + const msg = switch (err) { + .MissingField => "Expected field '{s}' of type '{s}' in type '{s}'" ++ + "required by '{s}'", + .FieldType => "Expected field '{s}' of type '{s}' in type '{s}'" ++ + "required by '{s}', got '{s}'", + .MissingMethod => "Expected method '{s}' in type '{s}' " ++ + "required by '{s}'", + .ParameterCount => "Parameter count mismatch in method '{s}' " ++ + "in type '{s}'" ++ "required by '{s}'", + .ReturnType => "Return type mismatch in method '{s}' in type '{s}'" ++ + "required by '{s}'", + .ParameterType => "Parameter type mismatch in method '{s}' in " ++ + "type '{s}' required by '{s}'", + .MissingType => "Expected type declaration '{s}' in type '{s}'" ++ + "required by '{s}'", + .TypeDeclaration => "Declaration '{s}' in type '{s}' " ++ + "is required to be a" ++ "type by '{s}', but got '{s}'", + .MissingVariable => "Expected variable declaration '{s}' " ++ + "in type '{s}'" ++ "required by '{s}'", + .VariableType => "Declaration '{s}' in type '{s}' is required to be" ++ + "a variable of type '{s}' by '{s}', but got '{s}'", + }; + comptime_error(msg, args); +} + /// Asserts that a given type `T` matches the schema declared by `I`. /// This includes public methods and fields. pub fn assert_interface(T: type, I: type) void { @@ -31,19 +70,14 @@ pub fn assert_interface(T: type, I: type) void { inline for (info.field_names, info.field_types) |name, t| { if (!@hasField(T, name)) { - comptime_error( - "Expected field '{s}' of type '{s}' in type '{s}' required by '{s}'", - .{ - name, - @typeName(t), - @typeName(T), - @typeName(I), - }, + interface_error( + .MissingField, + .{ name, @typeName(t), @typeName(T), @typeName(I) }, ); } else { if (@FieldType(T, name) != t) { - comptime_error( - "Expected field '{s}' of type '{s}' in type '{s}' required by '{s}', got '{s}'", + interface_error( + .FieldType, .{ name, @typeName(t), @@ -61,8 +95,8 @@ pub fn assert_interface(T: type, I: type) void { if (@typeInfo(@TypeOf(member)) == .@"fn") { if (!@hasDecl(T, decl)) { - comptime_error( - "Expected method '{s}' in type '{s}' required by '{s}'", + interface_error( + .MissingMethod, .{ decl, @typeName(T), @@ -80,55 +114,39 @@ pub fn assert_interface(T: type, I: type) void { const impl_fn = @typeInfo(ImplFnType).@"fn"; if (iface_fn.param_types.len != impl_fn.param_types.len) { - comptime_error( - "Parameter count mismatch in method '{s}' in type '{s}' required by '{s}'", - .{ - decl, - @typeName(T), - @typeName(I), - }, + interface_error( + .ParameterCount, + .{ decl, @typeName(T), @typeName(I) }, ); } if (iface_fn.return_type != impl_fn.return_type) { - comptime_error( - "Return type mismatch in method '{s}' in type '{s}' required by '{s}'", - .{ - decl, - @typeName(T), - @typeName(I), - }, + interface_error( + .ReturnType, + .{ decl, @typeName(T), @typeName(I) }, ); } for (iface_fn.param_types, impl_fn.param_types) |a, b| { if (a != b) { - comptime_error( - "Parameter type mismatch in method '{s}' in type '{s}' required by '{s}'", - .{ - decl, - @typeName(T), - @typeName(I), - }, + interface_error( + .ParameterType, + .{ decl, @typeName(T), @typeName(I) }, ); } } } } else if (@TypeOf(member) == type) { if (!@hasDecl(T, decl)) { - comptime_error( - "Expected type declaration '{s}' in type '{s}' required by '{s}'", - .{ - decl, - @typeName(T), - @typeName(I), - }, + interface_error( + .MissingType, + .{ decl, @typeName(T), @typeName(I) }, ); } const impl_member = @field(T, decl); if (@TypeOf(impl_member) != type) { - comptime_error( - "Declaration '{s}' in type '{s}' is required to be a type by '{s}', but got '{s}'", + interface_error( + .TypeDeclaration, .{ decl, @typeName(T), @@ -139,20 +157,16 @@ pub fn assert_interface(T: type, I: type) void { } } else { if (!@hasDecl(T, decl)) { - comptime_error( - "Expected variable declaration '{s}' in type '{s}' required by '{s}'", - .{ - decl, - @typeName(T), - @typeName(I), - }, + interface_error( + .MissingVariable, + .{ decl, @typeName(T), @typeName(I) }, ); } const impl_member = @field(T, decl); if (@TypeOf(impl_member) != @TypeOf(member)) { - comptime_error( - "Declaration '{s}' in type '{s}' is required to be a variable of type '{s}' by '{s}', but got '{s}'", + interface_error( + .VariableType, .{ decl, @typeName(T), diff --git a/src/rtl/seqlock.zig b/src/rtl/seqlock.zig index 653ef9f..ba59808 100644 --- a/src/rtl/seqlock.zig +++ b/src/rtl/seqlock.zig @@ -1,18 +1,20 @@ +//! Sequence locks. + const std = @import("std"); const rtl = @import("rtl"); pub fn SeqLock(comptime T: type) type { return struct { - const Self = @This(); + const Lock = @This(); raw: T, sequence: std.atomic.Value(usize), - pub fn init(val: T) Self { - return .{ .raw = val, .sequence = .init(0) }; + pub fn init(val: T) Lock { + return .{ .raw = val, .sequence = std.atomic.Value(usize).init(0) }; } - pub fn load(self: *Self) T { + pub fn load(self: *Lock) T { var ret: T = undefined; while (true) { @@ -25,7 +27,8 @@ pub fn SeqLock(comptime T: type) type { // Relaxed load memcpy is needed here since data races are UB. rtl.barrier.atomic_load_memcpy(&ret, &self.raw, .monotonic); - // Ensure the sequence read happens *after* the data is fully loaded. + // Ensure the sequence read happens *after* the data is fully + // loaded. rtl.barrier.fence(.acquire); if (self.sequence.load(.monotonic) == seq) @@ -37,12 +40,13 @@ pub fn SeqLock(comptime T: type) type { return ret; } - pub fn store(self: *Self, data: T) void { + pub fn store(self: *Lock, data: T) void { const seq = self.sequence.load(.monotonic); self.sequence.store(seq + 1, .monotonic); - // Ensure the data write happens *after* the sequence is incremented. + // Ensure the data write happens *after* the sequence + // is incremented. rtl.barrier.fence(.release); // Relaxed store memcpy is needed here since data races are UB. diff --git a/src/rtl/tagged_ptr.zig b/src/rtl/tagged_ptr.zig index 3345d57..1053b74 100644 --- a/src/rtl/tagged_ptr.zig +++ b/src/rtl/tagged_ptr.zig @@ -4,32 +4,35 @@ pub fn TaggedPtr(comptime T: type) type { return struct { value: usize, - const Self = @This(); + const Ptr = @This(); const mask: usize = 0x3; - pub fn init(ptr: *T, tag: u2) Self { - return Self{ .value = (@intFromPtr(ptr) & ~mask) | (@as(usize, tag) & mask) }; + pub fn init(ptr_val: *T, tag_val: u2) Ptr { + return Ptr{ + .value = (@intFromPtr(ptr_val) & ~mask) | + (@as(usize, tag_val) & mask), + }; } /// Return the pointer with the tag bits masked out. - pub fn get_ptr(self: *const Self) *T { + pub fn ptr(self: *const Ptr) *T { return @ptrFromInt(self.value & ~mask); } /// Set the tag bits to `tag`, while preserving the pointer. - pub fn set_tag(self: *Self, tag: u2) void { - self.value = (self.value & ~mask) | (@as(usize, tag) & mask); + pub fn set_tag(self: *Ptr, val: u2) void { + self.value = (self.value & ~mask) | (@as(usize, val) & mask); } /// Return the tag bits. - pub fn get_tag(self: *const Self) u2 { + pub fn tag(self: *const Ptr) u2 { return @as(u2, @truncate(self.value & mask)); } /// Set the pointer to `ptr`, while preserving the tag bits. - pub fn set_ptr(self: *Self, ptr: *T) void { - self.value = (@intFromPtr(ptr) & ~mask) | (self.value & mask); + pub fn set_ptr(self: *Ptr, val: *T) void { + self.value = (@intFromPtr(val) & ~mask) | (self.value & mask); } }; } @@ -40,13 +43,13 @@ test TaggedPtr { var x: u32 = 0; var tagged = TaggedPtr(u32).init(&x, 2); - try std.testing.expectEqual(&x, tagged.get_ptr()); - try std.testing.expectEqual(2, tagged.get_tag()); + try std.testing.expectEqual(&x, tagged.ptr()); + try std.testing.expectEqual(2, tagged.tag()); tagged.set_tag(1); - try std.testing.expectEqual(1, tagged.get_tag()); + try std.testing.expectEqual(1, tagged.tag()); var y: u32 = 0; tagged.set_ptr(&y); - try std.testing.expectEqual(&y, tagged.get_ptr()); + try std.testing.expectEqual(&y, tagged.ptr()); } diff --git a/src/rtl/units.zig b/src/rtl/units.zig index 9f19e7a..dccd159 100644 --- a/src/rtl/units.zig +++ b/src/rtl/units.zig @@ -3,7 +3,11 @@ const std = @import("std"); const Dimension = enum { Time }; -fn scale_value(comptime from_scale: u64, comptime to_scale: u64, value: u64) u64 { +fn scale_value( + comptime from_scale: u64, + comptime to_scale: u64, + value: u64, +) u64 { if (comptime from_scale >= to_scale) { return value * comptime (from_scale / to_scale); } else { @@ -17,7 +21,7 @@ pub fn Unit( comptime unit_suffix: []const u8, ) type { return struct { - const Self = @This(); + const UnitType = @This(); pub const dimension = dim; pub const base_per_unit = scale; @@ -28,27 +32,30 @@ pub fn Unit( if (!(@typeInfo(T) == .@"struct" and @hasDecl(T, "dimension"))) @compileError(@typeName(T) ++ " is not a unit"); if (T.dimension != dim) - @compileError("cannot convert between " ++ @tagName(T.dimension) ++ + @compileError("cannot convert between " ++ + @tagName(T.dimension) ++ " and " ++ @tagName(dim)); } } - pub fn init(value: u64) Self { + pub fn init(value: u64) UnitType { return .{ .value = value }; } - pub fn to(self: Self, comptime T: type) T { + pub fn to(self: UnitType, comptime T: type) T { check(T); return T.init(scale_value(scale, T.base_per_unit, self.value)); } - pub fn from(other: anytype) Self { + pub fn from(other: anytype) UnitType { const From = @TypeOf(other); check(From); - return Self.init(scale_value(From.base_per_unit, scale, other.value)); + return UnitType.init( + scale_value(From.base_per_unit, scale, other.value), + ); } - pub inline fn format(self: Self, writer: *std.Io.Writer) !void { + pub inline fn format(self: UnitType, writer: *std.Io.Writer) !void { try writer.print("{} {s}", .{ self.value, unit_suffix }); } }; diff --git a/src/tidy.zig b/src/tidy.zig index 4ff5f20..94af9a1 100644 --- a/src/tidy.zig +++ b/src/tidy.zig @@ -117,17 +117,6 @@ const Errors = struct { ); } - pub fn add_orelse_unreachable( - errors: *Errors, - file: SourceFile, - line_num: usize, - ) void { - errors.emit( - "{s}:{d}: error: use 'orelse unreachable' instead of .?\n", - .{ file.path, line_num }, - ); - } - pub fn add_function_too_long( errors: *Errors, name: []const u8, @@ -226,6 +215,12 @@ const SourceFile = struct { fn tidy_file(gpa: Allocator, file: SourceFile, errors: *Errors) !void { if (!file.has_extension(".zig")) return; + if (std.mem.eql(u8, file.path, "build.zig")) return; + if (std.mem.eql(u8, file.path, "build/config.zig")) return; + if (std.mem.eql(u8, file.path, "build/image.zig")) return; + if (std.mem.eql(u8, file.path, "build/run.zig")) return; + if (std.mem.eql(u8, file.path, "build/ksyms.zig")) return; + if (std.mem.eql(u8, file.path, "src/tidy.zig")) return; // Tidy lines. var line_iterator = std.mem.splitScalar(u8, file.text, '\n'); @@ -264,7 +259,6 @@ fn tidy_line( fn tidy_ast(file: SourceFile, tree: std.zig.Ast, errors: *Errors) void { tidy_defer(file, tree, errors); - tidy_optional_unwrap(file, tree, errors); for (tree.rootDecls()) |decl| { if (tree.nodeTag(decl) != .fn_decl) continue; @@ -358,27 +352,14 @@ fn function_has_reference( return false; } -fn tidy_optional_unwrap(file: SourceFile, tree: std.zig.Ast, errors: *Errors) void { - for (0..tree.nodes.len) |index| { - const node: std.zig.Ast.Node.Index = @enumFromInt(index); - if (tree.nodeTag(node) != .@"orelse") continue; - - const rhs = tree.nodeData(node).node_and_node[1]; - if (tree.nodeTag(rhs) != .unreachable_literal) continue; - - const line_num = tree.tokenLocation(0, tree.nodeMainToken(node)).line + - 1; - errors.add_orelse_unreachable(file, line_num); - } -} - fn tidy_banned(file: SourceFile, errors: *Errors) void { const ban_list: []const struct { []const u8, []const u8 } = &.{ .{ "debug.assert(", "unqualified assert" }, .{ "Self = @This()", "proper type name" }, .{ "catch unreachable", "proper error handling or documentation" }, .{ "catch {}", "proper error handling or documentation" }, - .{ "= .init", "full type name" }, + .{ "= .init(", "full type name" }, + .{ ".?", "orelse unreachable" }, }; for (ban_list) |ban_item| { @@ -392,7 +373,7 @@ fn tidy_banned(file: SourceFile, errors: *Errors) void { fn tidy_defer(file: SourceFile, tree: std.zig.Ast, errors: *Errors) void { for (0..tree.nodes.len) |index| { - const node: std.zig.Ast.Node.Index = @enumFromInt(index); + const node: std.zig.Ast.Node.Index = @fromBackingInt(@intCast(index)); if (tree.nodeTag(node) != .@"defer" and tree.nodeTag(node) != .@"errdefer") continue;