From bec872bc262e08dbba76cdd754022b2c45e26842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Gr=C3=A9goire?= Date: Wed, 5 Aug 2026 17:43:46 -0400 Subject: [PATCH] all: refactor coding style --- src/arch/amd64/root.zig | 6 +- src/ex/console.zig | 15 +- src/ex/work.zig | 37 ++-- src/kern/amd64/impl.zig | 26 ++- src/kern/cpu.zig | 6 +- src/kern/dpc.zig | 14 +- src/kern/init.zig | 6 +- src/kern/ipi.zig | 10 +- src/kern/log.zig | 19 ++- src/kern/shootdown.zig | 20 ++- src/kern/smr.zig | 49 ++++-- src/kern/spinlock.zig | 33 +++- src/kern/thread.zig | 11 +- src/kern/time.zig | 12 +- src/kern/timer.zig | 35 +++- src/kern/tunable.zig | 115 +++++++------ src/kern/turnstile.zig | 21 ++- src/kern/wait.zig | 67 +++++--- src/mm/heap.zig | 13 +- src/mm/phys.zig | 37 +++- src/mm/pmap.zig | 59 +++++-- src/mm/tlb.zig | 12 +- src/mm/vmem.zig | 74 ++++++-- src/mm/zone.zig | 340 +++++++++++++++++++++++++++---------- src/platform/acpi/hpet.zig | 3 +- src/platform/acpi/madt.zig | 4 +- src/platform/acpi/root.zig | 24 ++- src/platform/pc/apic.zig | 34 ++-- src/platform/pc/smp.zig | 58 +++++-- src/proc/thread.zig | 6 +- src/rcu/hashtable.zig | 51 ++++-- 31 files changed, 887 insertions(+), 330 deletions(-) diff --git a/src/arch/amd64/root.zig b/src/arch/amd64/root.zig index b2b2819..510b6eb 100644 --- a/src/arch/amd64/root.zig +++ b/src/arch/amd64/root.zig @@ -476,7 +476,11 @@ const PowerManagementInfoEdx = packed struct(u32) { reserved1: u23, }; -fn assert_bit(comptime T: type, comptime field_name: []const u8, comptime expected_bit: u5) void { +fn assert_bit( + comptime T: type, + comptime field_name: []const u8, + comptime expected_bit: u5, +) void { var v: T = @bitCast(@as(u32, 0)); @field(v, field_name) = true; const raw: u32 = @bitCast(v); diff --git a/src/ex/console.zig b/src/ex/console.zig index 8adf4b0..67dd700 100644 --- a/src/ex/console.zig +++ b/src/ex/console.zig @@ -27,7 +27,11 @@ const ConWriter = struct { }; } - fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { + fn drain( + w: *std.Io.Writer, + data: []const []const u8, + _: usize, + ) std.Io.Writer.Error!usize { const self: *ConWriter = @fieldParentPtr("interface", w); var total_written: usize = 0; for (data) |slice| { @@ -87,7 +91,8 @@ fn drain_from_all(seq: usize) usize { fn worker(_: ?*anyopaque) void { while (true) { - _ = ke.wait.wait_one(&ke.log.event.hdr, "console_worker", .{}) catch unreachable; + _ = ke.wait.wait_one(&ke.log.event.hdr, "console_worker", .{}) catch + unreachable; ke.log.event.reset(); last_seq = drain_from_all(last_seq); @@ -119,6 +124,10 @@ pub fn register(cons: *Console) void { pub fn init() void { console_list.init(); - const td = ps.thread.create_kernel(ke.Thread.Priority.default, worker, null) catch @panic("handle me"); + const td = ps.thread.create_kernel( + ke.Thread.Priority.default, + worker, + null, + ) catch @panic("handle me"); ke.sched.enqueue(&td.kern); } diff --git a/src/ex/work.zig b/src/ex/work.zig index d13fcc4..d65684a 100644 --- a/src/ex/work.zig +++ b/src/ex/work.zig @@ -1,6 +1,7 @@ //! Worker pools and work items. //! -//! Work items are used to enqueue work to be done in a thread context asynchronously. +//! Work items are used to enqueue work to be done in a thread context +//! asynchronously. //! There are three worker priorities: //! - High //! - Normal @@ -10,8 +11,8 @@ //! //! Each CPU worker pool comprises of at least 2 threads, and the low pool //! has 4 threads by default. An additional thread will get spawned by the -//! pool manager thread if a queue has a backlog, up to 4 additional dynamic threads -//! (tunable) per pool. +//! pool manager thread if a queue has a backlog, up to 4 additional dynamic +//! threads (tunable) per pool. const std = @import("std"); const rtl = @import("rtl"); @@ -58,7 +59,12 @@ pub const WorkItem = struct { } }; - pub fn init(self: *WorkItem, priority: Priority, func: *const fn (arg: ?*anyopaque) void, arg: ?*anyopaque) void { + pub fn init( + self: *WorkItem, + priority: Priority, + func: *const fn (arg: ?*anyopaque) void, + arg: ?*anyopaque, + ) void { self.* = .{ .func = func, .arg = arg, @@ -145,7 +151,9 @@ const Pool = struct { const active = @atomicLoad(usize, &self.queue.active, .monotonic); const threads = self.total_threads.load(.monotonic); - if (threads < max_threads_for_prio(self.prio) and !is_empty and active < self.queue.max_active) { + if (threads < max_threads_for_prio(self.prio) and !is_empty and + active < self.queue.max_active) + { // We have room to grow, the list still has items and // not enough runnable threads are present, create an // additional thread to ease the backlog. @@ -157,7 +165,8 @@ const Pool = struct { /// Spawn a dynamic worker thread on this pool. fn grow(self: *Pool, cpu: ?u32) !void { - if (self.total_threads.load(.monotonic) >= max_threads_for_prio(self.prio)) + if (self.total_threads.load(.monotonic) >= + max_threads_for_prio(self.prio)) return; _ = self.total_threads.fetchAdd(1, .monotonic); @@ -197,7 +206,10 @@ fn work_loop(p: ?*anyopaque) void { const dynamic = ctx.get_tag() == 1; while (true) { - const timeout: ?r.Nanoseconds = if (dynamic) .from(r.Seconds.init(5)) else null; + const timeout: ?r.Nanoseconds = if (dynamic) + .from(r.Seconds.init(5)) + else + null; const item = ctx.get_ptr().queue.remove(timeout) catch { _ = ctx.get_ptr().total_threads.fetchSub(1, .monotonic); @@ -257,11 +269,16 @@ pub fn enqueue(item: *WorkItem) void { fn pool_manager(_: ?*anyopaque) void { while (true) { - _ = ke.wait.wait_one(&pool_manager_event.hdr, "work_pool_manager", .{}) catch unreachable; + _ = ke.wait.wait_one( + &pool_manager_event.hdr, + "work_pool_manager", + .{}, + ) catch + unreachable; // We have been signaled, go through every pool and grow it if needed. - // Failure to grow is ignored, it is only a latency problem and should be fixed - // eventually as memory is freed. + // Failure to grow is ignored, it is only a latency problem and should + // be fixed eventually as memory is freed. for (0..ke.ncpus) |i| { const cpu: u32 = @intCast(i); const c = percpu.remote(cpu); diff --git a/src/kern/amd64/impl.zig b/src/kern/amd64/impl.zig index ea61cbe..315e56c 100644 --- a/src/kern/amd64/impl.zig +++ b/src/kern/amd64/impl.zig @@ -25,7 +25,12 @@ const ThreadFrame = extern struct { extern fn asm_thread_entry() void; -extern fn do_context_switch(old: *ThreadContext, new: *ThreadContext, lock: *u8, switching: *bool) callconv(.c) void; +extern fn do_context_switch( + old: *ThreadContext, + new: *ThreadContext, + lock: *u8, + switching: *bool, +) callconv(.c) void; extern fn do_context_load(td: *ThreadContext) callconv(.c) void; pub const ThreadContext = extern struct { @@ -38,7 +43,12 @@ pub const ThreadContext = extern struct { entry_fn(@ptrFromInt(arg)); } - pub fn init(stack: r.VAddr, stack_size: usize, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) @This() { + pub fn init( + stack: r.VAddr, + stack_size: usize, + entry: *const fn (?*anyopaque) void, + arg: ?*anyopaque, + ) @This() { var ctx: @This() = undefined; var sp: usize = stack + stack_size; @@ -58,9 +68,17 @@ pub const ThreadContext = extern struct { return ctx; } - pub fn switch_to(self: *ThreadContext, new: *ThreadContext) callconv(.c) void { + pub fn switch_to( + self: *ThreadContext, + new: *ThreadContext, + ) callconv(.c) void { const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", self)); - do_context_switch(self, new, &thread.lock.inner.locked.raw, &thread.switching.raw); + do_context_switch( + self, + new, + &thread.lock.inner.locked.raw, + &thread.switching.raw, + ); } pub fn load(self: *ThreadContext) callconv(.c) void { diff --git a/src/kern/cpu.zig b/src/kern/cpu.zig index fbf9042..40d8e05 100644 --- a/src/kern/cpu.zig +++ b/src/kern/cpu.zig @@ -40,7 +40,11 @@ pub fn CpuLocal(comptime T: type, comptime init: T) type { } /// Wraps around CPU-local data with a designated symbol name. -pub fn ExportedCpuLocal(comptime T: type, comptime init: T, comptime name: []const u8) type { +pub fn ExportedCpuLocal( + comptime T: type, + comptime init: T, + comptime name: []const u8, +) type { const S = struct { var storage: T linksection(".data.percpu") = init; diff --git a/src/kern/dpc.zig b/src/kern/dpc.zig index c9a07fe..d4575ec 100644 --- a/src/kern/dpc.zig +++ b/src/kern/dpc.zig @@ -6,7 +6,7 @@ const ke = r.ke; const ki = ke.private; /// Deferred Procedure Call (DPC) structure. -/// Used for scheduling work to be done when IPL is lowered below `ke.Ipl.Dispatch`. +/// Used for scheduling work to be done when IPL is lowered below `.Dispatch`. pub const Dpc = struct { /// Entry into per-CPU DPC queue link: rtl.List.Entry, @@ -78,7 +78,8 @@ fn dispatch_queue(cpu: u32) void { ipl_cpu.ipl = .Dispatch; - // Mark as handled, this must be done before enabling interrupts or we will race. + // Mark as handled, this must be done before enabling interrupts or we will + // race. ki.ipl.clear_softint_pending(cpu, .Dispatch); _ = ki.impl.enable_interrupts(); @@ -100,8 +101,9 @@ fn dispatch_queue(cpu: u32) void { dpc = @fieldParentPtr("link", first_elem); dpc.inserted.store(false, .release); - // An interrupt which would enqueue this DPC could occur between loading the argument and calling the routine, - // which is why we capture it here to ensure that we get the intended context. + // An interrupt which would enqueue this DPC could occur between + // loading the argument and calling the routine, which is why we capture + // it here to ensure that we get the intended context. const arg = dpc.arg; dpc_cpu.lock.release(ipl); @@ -148,8 +150,8 @@ fn dispatch_queue(cpu: u32) void { /// Dispatch the DPC queue on `cpu`. pub fn dispatch(cpu: u32) void { - // DPC processing is done at Ipl.Dispatch - // We can't call lower/raise here because they might call us again recursively. + // DPC processing is done at IPL Dispatch. + // Don't call lower/raise here because they might call us again recursively. var mycpu = cpu; const old_ipl = ki.ipl.percpu.remote(mycpu).ipl; const int_state = ki.impl.disable_interrupts(); diff --git a/src/kern/init.zig b/src/kern/init.zig index 000f26e..e43ef53 100644 --- a/src/kern/init.zig +++ b/src/kern/init.zig @@ -37,7 +37,11 @@ pub fn init(_: ?*anyopaque) linksection(r.init) void { ki.sched.percpu.local().current_thread = &thread0; ki.sched.percpu.local().idle_thread = &thread0; - std.log.info("Zag for {s} ({s}), cmdline is \"{?s}\"", .{ pl.name, arch.name, r.boot_info.cmdline }); + std.log.info("Zag for {s} ({s}), cmdline is \"{?s}\"", .{ + pl.name, + arch.name, + r.boot_info.cmdline, + }); ex.init(); ki.sched.idle(null); diff --git a/src/kern/ipi.zig b/src/kern/ipi.zig index f7d524c..9e45170 100644 --- a/src/kern/ipi.zig +++ b/src/kern/ipi.zig @@ -35,7 +35,11 @@ const percpu = ke.CpuLocal(PerCpu, undefined); /// The function will be executed synchronously, i.e this will block until /// the function finishes running. The callback is called with the sender cpu /// and provided argument as arguments. -pub fn unicast(cpu: u32, func: *const fn (u32, ?*anyopaque) void, arg: ?*anyopaque) void { +pub fn unicast( + cpu: u32, + func: *const fn (u32, ?*anyopaque) void, + arg: ?*anyopaque, +) void { const ipl = ke.ipl.raise(.Dispatch); const curcpu = percpu.local(); const remote = percpu.remote(cpu); @@ -93,8 +97,8 @@ pub fn ipi_handler() void { while (true) { // Mark the current CPU as processing IPIs, if the CAS fails here, - // then it's probably some kind of IPI that was sent to force us to check - // software interrupts (like a rescheduling IPI). + // then it's probably some kind of IPI that was sent to force us to + // check software interrupts (like a rescheduling IPI). const val = curcpu.state.cmpxchgStrong( .Pending, .Running, diff --git a/src/kern/log.zig b/src/kern/log.zig index 4e5e395..e719879 100644 --- a/src/kern/log.zig +++ b/src/kern/log.zig @@ -21,7 +21,11 @@ const DebugWriter = struct { }; } - fn drain(_: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { + fn drain( + _: *std.Io.Writer, + data: []const []const u8, + _: usize, + ) std.Io.Writer.Error!usize { var total_written: usize = 0; for (data) |slice| { for (slice) |byte| { @@ -37,7 +41,10 @@ const DebugWriter = struct { // Messages are on average 2^5 = 32 bytes. const avg_msg_size_bits = 5; -pub var ringbuffer = ki.log_ring.RingBuffer(config.log_buffer_shift, avg_msg_size_bits).init(); +pub var ringbuffer = ki.log_ring.RingBuffer( + config.log_buffer_shift, + avg_msg_size_bits, +).init(); // Initialize manually because we need this ASAP. pub var event: ke.Event = .{ @@ -46,7 +53,10 @@ pub var event: ke.Event = .{ .signaled = 0, .type = .Notification, .waitblocks = .{ - .head = .{ .next = &event.hdr.waitblocks.head, .prev = &event.hdr.waitblocks.head }, + .head = .{ + .next = &event.hdr.waitblocks.head, + .prev = &event.hdr.waitblocks.head, + }, }, }, }; @@ -75,7 +85,8 @@ pub fn log( res.info.length = @truncate(required_len); // Format the log message into the reserved buffer. - res.buf = std.fmt.bufPrint(res.buf, scope_str ++ fmt ++ "\n", args) catch return; + res.buf = std.fmt.bufPrint(res.buf, scope_str ++ fmt ++ "\n", args) catch + return; ringbuffer.publish(res); diff --git a/src/kern/shootdown.zig b/src/kern/shootdown.zig index 6317d99..dea329c 100644 --- a/src/kern/shootdown.zig +++ b/src/kern/shootdown.zig @@ -53,7 +53,13 @@ fn pcpu_init() linksection(r.init) void { local.valid_states = @splat(.init(0)); local.senders = .init(false); - local.states = @splat(.{ .base = 0, .npages = 0, .state = .init(slot_free), .payload = undefined, .link = undefined }); + local.states = @splat(.{ + .base = 0, + .npages = 0, + .state = .init(slot_free), + .payload = undefined, + .link = undefined, + }); } comptime { @@ -66,7 +72,12 @@ fn allocate_slot(cpu: *PerCpu) ?usize { for (0.., &cpu.states) |slot, *state| { if (state.state.load(.monotonic) != slot_free) continue; - if (state.state.cmpxchgStrong(slot_free, slot_reserved, .acquire, .monotonic) == null) { + if (state.state.cmpxchgStrong( + slot_free, + slot_reserved, + .acquire, + .monotonic, + ) == null) { return slot; } } @@ -167,7 +178,10 @@ pub fn submit(state: ShootdownState, target_mask: ke.CpuMask) !void { while (iter.next()) |bit| { if (bit == curcpu) continue; - _ = cpu.valid_states[bit].fetchOr(@as(u64, 1) << @truncate(slot), .release); + _ = cpu.valid_states[bit].fetchOr( + @as(u64, 1) << @truncate(slot), + .release, + ); ki.ipl.set_softint_pending(@truncate(bit), .Dispatch); diff --git a/src/kern/smr.zig b/src/kern/smr.zig index 80c51a0..0943beb 100644 --- a/src/kern/smr.zig +++ b/src/kern/smr.zig @@ -77,7 +77,8 @@ pub const Tracker = struct { pub var full_scans: std.atomic.Value(usize) = .init(0); /// Scan all CPUs and return the minimum observed value. -/// If `should_wait` is true, this will spinloop (or block) until all CPUs have reached the given goal. +/// If `should_wait` is true, this will spinloop (or block) +/// until all CPUs have reached the given goal. fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence { rtl.barrier.fence(.seq_cst); @@ -162,7 +163,12 @@ fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence break; } - ki.turnstile.block(ts, cpu, .{ .shared = &cpu.stalled }, .Exclusive); + ki.turnstile.block( + ts, + cpu, + .{ .shared = &cpu.stalled }, + .Exclusive, + ); ke.ipl.lower(ipl); seq = cpu.stall_seq.load(.monotonic); @@ -256,8 +262,9 @@ pub fn advance(dom: *Domain) Sequence { /// Pretend-advance the write sequence and return the value for use as a wait goal. /// It is guaranteed that all previous memory writes made by -/// the calling thread are visible. The global clock isn't advanced unlike `advance`, -/// it is only when commit() is called. +/// the calling thread are visible. +/// The global clock isn't advanced unlike `advance`, it is only when commit() +/// is called. pub fn deferred_advance(dom: *Domain) Sequence { rtl.barrier.fence(.seq_cst); @@ -267,7 +274,12 @@ pub fn deferred_advance(dom: *Domain) Sequence { /// Actually advance the global clock to a value previously returned by /// deferred_advance(). pub fn deferred_advance_commit(dom: *Domain, seq: Sequence) void { - _ = dom.clock.write_seq.cmpxchgStrong(seq - seq_incr, seq, .monotonic, .monotonic); + _ = dom.clock.write_seq.cmpxchgStrong( + seq - seq_incr, + seq, + .monotonic, + .monotonic, + ); } /// Enter a read section. @@ -276,18 +288,20 @@ fn enter_internal(dom: *Domain) Sequence { // Store the currently observed write sequence into our CPU state. // Subsequent loads must not be re-ordered w.r.t to the store here. - // On AMD64, we can simply use XADD as this is faster and provides the same guarantees, - // on other architectures we have to rely on a full (seq_cst) fence. - // The add operation works because cpu.current_seq is always only 0 (seq_invalid) - // when we write to it. + // On AMD64, we can simply use XADD as this is faster and provides the + // same guarantees, on other architectures we have to rely on a full fence. + // The add operation works because cpu.current_seq is always only 0 + // (seq_invalid) when we write to it. // - // There can be a large pause between the load from the domain write sequence - // and the store to the per-cpu state because of e.g an interrupt. This could lead - // to a state where another CPU advances the global write sequence and current_seq - // becomes lower than the global write sequence. This is not an issue with the ordering - // in scan(), which also relies on a full memory barrier, as it is guaranteed that the - // CPU was treated as inactive and cannot possibly hold a reference to anything the - // poll declared reclaimable. See scan(). + // There can be a large pause between the load from the domain write + // sequence and the store to the per-cpu state because of e.g an interrupt. + // This could lead to a state where another CPU advances + // the global write sequence and current_seq ends up lower + // than the global write sequence. + // This is not an issue with the ordering in scan(), which also relies + // on a full memory barrier, as it is guaranteed that the CPU was treated as + // inactive and cannot possibly hold a reference to anything the poll + // declared reclaimable. See scan(). std.debug.assert(cpu.current_seq.load(.monotonic) == seq_invalid); const wr_seq = dom.clock.write_seq.load(.monotonic); @@ -434,7 +448,8 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void { } if (wake) { - if (turnstile) |ts| ki.turnstile.wakeup(ts, .Exclusive, ts.waiters, null); + if (turnstile) |ts| + ki.turnstile.wakeup(ts, .Exclusive, ts.waiters, null); } ki.turnstile.exit(cpu); diff --git a/src/kern/spinlock.zig b/src/kern/spinlock.zig index a91337e..861b95d 100644 --- a/src/kern/spinlock.zig +++ b/src/kern/spinlock.zig @@ -176,7 +176,12 @@ pub const QSpinLock = LockTemplate(struct { pub fn acquire_no_ipl(self: *Self) void { // Fast path: try to acquire the lock. - var v = self.data.val.cmpxchgStrong(0, locked_val, .acquire, .monotonic) orelse return; + var v = self.data.val.cmpxchgStrong( + 0, + locked_val, + .acquire, + .monotonic, + ) orelse return; // Medium path: if only pending is set, the pending waiter is // in the process of clearing pending and setting `locked` to take @@ -229,7 +234,12 @@ pub const QSpinLock = LockTemplate(struct { } pub fn try_acquire_no_ipl(self: *Self) bool { - return self.data.val.cmpxchgStrong(0, locked_val, .acquire, .monotonic) == null; + return self.data.val.cmpxchgStrong( + 0, + locked_val, + .acquire, + .monotonic, + ) == null; } pub fn is_locked(self: *Self) bool { @@ -247,7 +257,12 @@ pub const QSpinLock = LockTemplate(struct { if (idx >= 4) { // Exhausted all indices, spin. while (true) { - _ = self.data.val.cmpxchgWeak(0, locked_val, .acquire, .monotonic) orelse break; + _ = self.data.val.cmpxchgWeak( + 0, + locked_val, + .acquire, + .monotonic, + ) orelse break; std.atomic.spinLoopHint(); } @@ -266,7 +281,10 @@ pub const QSpinLock = LockTemplate(struct { // Store cpu + 1 so that we can distinguish from an empty tail field. rtl.barrier.fence(.release); - const tail: u16 = @bitCast(Tail{ .cpu = @intCast(ke.cpu.current() + 1), .idx = @intCast(idx) }); + const tail: u16 = @bitCast(Tail{ + .cpu = @intCast(ke.cpu.current() + 1), + .idx = @intCast(idx), + }); const old: Tail = @bitCast(self.data.split.tail.swap(tail, .acq_rel)); if (old.cpu != 0) { @@ -293,7 +311,12 @@ pub const QSpinLock = LockTemplate(struct { if ((v & tail_mask) == @as(u32, tail) << 16) { // We are the only waiter in the queue. // Try to claim the lock and clear the tail atomically. - if (self.data.val.cmpxchgStrong(v, locked_val, .acquire, .monotonic) == null) { + if (self.data.val.cmpxchgStrong( + v, + locked_val, + .acquire, + .monotonic, + ) == null) { return; } } diff --git a/src/kern/thread.zig b/src/kern/thread.zig index a931457..5344423 100644 --- a/src/kern/thread.zig +++ b/src/kern/thread.zig @@ -123,12 +123,19 @@ pub const Thread = struct { smr_sections: rtl.List, /// Initialize a thread. - /// - `stack`: Address of the **base** of the stack on which the initial context for the thread is built + /// - `stack`: Address of the base of the stack used by the thread. /// - `stack_size`: Size of the stack /// - `prio`: Base priority of the thread /// - `entry`: Entry point of the thread /// - `arg`: Extraneous argument to be passed to `entry` - pub fn init(thread: *Thread, stack: r.VAddr, stack_size: usize, prio: u8, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) void { + pub fn init( + thread: *Thread, + stack: r.VAddr, + stack_size: usize, + prio: u8, + entry: *const fn (?*anyopaque) void, + arg: ?*anyopaque, + ) void { thread.* = .{ .context = .init(stack, stack_size, entry, arg), .lock = .init(), diff --git a/src/kern/time.zig b/src/kern/time.zig index 55b7f84..f35d142 100644 --- a/src/kern/time.zig +++ b/src/kern/time.zig @@ -18,11 +18,9 @@ pub const TimeCounter = struct { quality: i16, /// Mask to apply to the counter value. mask: u64, - /// Binary scaling exponent for fast ns conversion (0 if unused). - /// See `ticks_to_ns`. + /// Binary scaling exponent for fast ns conversion. See `ticks_to_ns`. p: u64, - /// Scaled nanoseconds-per-tick multiplier for fast ns conversion (0 if unused). - /// See `ticks_to_ns`. + /// Scaled for fast ns conversion (0 if unused). See `ticks_to_ns`. n: u64, }; @@ -59,10 +57,12 @@ fn ticks_to_ns(tc: *TimeCounter, count: u64) u64 { /// Register a time-keeping source. /// The TimeCounter will be used if it is the highest quality one available. -/// This function cannot be safely called when the hardware timer has been started, as data races on `best_tc` are not avoided. +/// This function cannot be safely called when the hardware timer +/// has been started, as data races on `best_tc` are not avoided. pub fn register_source(tc: *TimeCounter) void { // Precompute scaling factors for the fast multiply-shift path. - // This is meant to avoid 128-bit division and only use multiplication for 64-bit counters. + // This is meant to avoid 128-bit division and only use multiplication for + // 64-bit counters. if (tc.mask == std.math.maxInt(u64)) { tc.p = std.math.log2_int_ceil(u64, tc.frequency); tc.n = (@as(u64, std.time.ns_per_s) << @intCast(tc.p)) / tc.frequency; diff --git a/src/kern/timer.zig b/src/kern/timer.zig index bda5392..0a1ed3b 100644 --- a/src/kern/timer.zig +++ b/src/kern/timer.zig @@ -1,5 +1,6 @@ //! Timer object implementation. -//! Timer objects are useful when one wants to wait a fixed amount of time for an event to occur. +//! Timer objects are useful when one wants to wait a given amount of time for +//! an event to occur. const std = @import("std"); const rtl = @import("rtl"); const r = @import("root"); @@ -70,7 +71,12 @@ pub fn set(timer: *Timer, time: r.Nanoseconds, opts: Options) void { const cpu = percpu.local(); - if (timer.state.cmpxchgStrong(.Stopped, .Pending, .acquire, .monotonic) != null) { + if (timer.state.cmpxchgStrong( + .Stopped, + .Pending, + .acquire, + .monotonic, + ) != null) { // Could not set the timer. return; } @@ -107,7 +113,12 @@ pub fn cancel(timer: *Timer) void { const ipl = timer.hdr.lock.acquire(); defer timer.hdr.lock.release(ipl); - if (timer.state.cmpxchgStrong(.Pending, .Stopped, .acquire, .monotonic) != null) { + if (timer.state.cmpxchgStrong( + .Pending, + .Stopped, + .acquire, + .monotonic, + ) != null) { // Something changed, it probably expired. return; } @@ -137,7 +148,8 @@ pub fn cmp_timer(a: *rtl.pairing_heap.Node, b_: *rtl.pairing_heap.Node) std.math pub fn clock() void { ke.dpc.enqueue(&percpu.local().dpc, null); // Check for overflows. - // This is fine to call a lot, as the function only gets expensive (i.e seqlock store) when an overflow actually happens. + // This is fine to call a lot, as the function only gets expensive + // (i.e seqlock store) when an overflow actually happens. ki.time.update_overflow(); } @@ -161,13 +173,17 @@ fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void { // If the timer expires more than 1ms in the future, consider it not yet due. // Sub-millisecond differences are close enough to expire immediately. - if (timer.deadline.value > curtime.value and timer.deadline.value - curtime.value > std.time.ns_per_ms) { + if (timer.deadline.value > curtime.value and + timer.deadline.value - curtime.value > std.time.ns_per_ms) + { const next = timer; cpu.lock.release_no_ipl(); // Re-check in case the timer expired while we were here. const now = ke.time.read_time(); - if (next.deadline.value <= now.value or next.deadline.value - now.value <= std.time.ns_per_ms) { + if (next.deadline.value <= now.value or + next.deadline.value - now.value <= std.time.ns_per_ms) + { continue; // expired in the meantime, loop again } @@ -176,7 +192,12 @@ fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void { return; } - if (timer.state.cmpxchgStrong(.Pending, .Running, .acquire, .monotonic) != null) { + if (timer.state.cmpxchgStrong( + .Pending, + .Running, + .acquire, + .monotonic, + ) != null) { // Timer must've been canceled. cpu.lock.release_no_ipl(); continue; diff --git a/src/kern/tunable.zig b/src/kern/tunable.zig index 7bd4a36..a65a903 100644 --- a/src/kern/tunable.zig +++ b/src/kern/tunable.zig @@ -16,7 +16,11 @@ const Entry = struct { }, }; -pub fn Tunable(comptime T: type, comptime default: T, comptime name: []const u8) type { +pub fn Tunable( + comptime T: type, + comptime default: T, + comptime name: []const u8, +) type { return struct { var storage: T = default; @@ -42,66 +46,65 @@ pub fn Tunable(comptime T: type, comptime default: T, comptime name: []const u8) }; } +fn store_int(comptime T: type, addr: *anyopaque, value: anytype) bool { + const ptr: *T = @ptrCast(@alignCast(addr)); + ptr.* = std.math.cast(T, value) orelse return false; + return true; +} + +fn apply_num( + comptime ValueT: type, + cmdline: []const u8, + entry: *const Entry, + info: std.builtin.Type.Int, +) void { + const value = rtl.cmdline.get_number(ValueT, cmdline, entry.name) catch |e| { + if (e == error.Format) { + std.log.warn( + "Invalid number value for '{s}', falling back.", + .{entry.name}, + ); + } + return; + }; + + const sign = @typeInfo(ValueT).int.signedness; + + _ = switch (info.bits) { + 8 => store_int(std.meta.Int(sign, 8), entry.address, value), + 16 => store_int(std.meta.Int(sign, 16), entry.address, value), + 32 => store_int(std.meta.Int(sign, 32), entry.address, value), + 64 => store_int(std.meta.Int(sign, 64), entry.address, value), + else => unreachable, + }; +} + +fn apply_bool(cmdline: []const u8, entry: *const Entry) void { + const value = rtl.cmdline.get_string(cmdline, entry.name) orelse return; + const ptr: *bool = @ptrCast(@alignCast(entry.address)); + + if (std.mem.eql(u8, value, "true")) { + ptr.* = true; + } else if (std.mem.eql(u8, value, "false")) { + ptr.* = false; + } else { + std.log.warn( + "Invalid cmdline boolean value for tunable '{s}'", + .{entry.name}, + ); + } +} + pub fn init() void { const cmdline = r.boot_info.cmdline orelse return; - const elems = set.elems(); - - for (elems) |entry| { + for (set.elems()) |entry| { switch (entry.type) { - .num => |info| { - const a = entry.address; - - switch (info.signedness) { - .unsigned => { - const value = rtl.cmdline.get_number(u64, cmdline, entry.name) catch |e| { - if (e == error.Format) { - std.log.warn("Invalid number value for '{s}', falling back.", .{entry.name}); - } - continue; - }; - - switch (info.bits) { - 8 => (@as(*u8, @ptrCast(@alignCast(a)))).* = std.math.cast(u8, value) orelse continue, - 16 => (@as(*u16, @ptrCast(@alignCast(a)))).* = std.math.cast(u16, value) orelse continue, - 32 => (@as(*u32, @ptrCast(@alignCast(a)))).* = std.math.cast(u32, value) orelse continue, - 64 => (@as(*u64, @ptrCast(@alignCast(a)))).* = value, - else => unreachable, - } - }, - - .signed => { - const value = rtl.cmdline.get_number(i64, cmdline, entry.name) catch |e| { - if (e == error.Format) { - std.log.warn("Invalid number value for '{s}', falling back.", .{entry.name}); - } - - continue; - }; - - switch (info.bits) { - 8 => (@as(*i8, @ptrCast(@alignCast(a)))).* = std.math.cast(i8, value) orelse continue, - 16 => (@as(*i16, @ptrCast(@alignCast(a)))).* = std.math.cast(i16, value) orelse continue, - 32 => (@as(*i32, @ptrCast(@alignCast(a)))).* = std.math.cast(i32, value) orelse continue, - 64 => (@as(*i64, @ptrCast(@alignCast(a)))).* = value, - else => unreachable, - } - }, - } - }, - - .bool => { - const value = rtl.cmdline.get_string(cmdline, entry.name) orelse continue; - const ptr: *bool = @ptrCast(@alignCast(entry.address)); - - if (std.mem.eql(u8, value, "true")) { - ptr.* = true; - } else if (std.mem.eql(u8, value, "false")) { - ptr.* = false; - } else { - std.log.warn("Invalid cmdline boolean value for tunable '{s}'", .{entry.name}); - } + .num => |info| switch (info.signedness) { + .unsigned => apply_num(u64, cmdline, entry, info), + .signed => apply_num(i64, cmdline, entry, info), }, + .bool => apply_bool(cmdline, entry), } } } diff --git a/src/kern/turnstile.zig b/src/kern/turnstile.zig index 6fc8700..814e2c7 100644 --- a/src/kern/turnstile.zig +++ b/src/kern/turnstile.zig @@ -314,7 +314,8 @@ fn propagate(curtd: *ke.Thread, root: *Chain) void { // and they are forbidden to explicitly block. var it = targets(ts); while (it.next()) |t| { - if (t.thread == curtd) @panic("turnstile: cycle in blocking chain"); + if (t.thread == curtd) + @panic("turnstile: cycle in blocking chain"); t.thread.lock.acquire_no_ipl(); _ = donate_to(t.boost, t.thread, donate); t.thread.lock.release_no_ipl(); @@ -384,7 +385,12 @@ pub fn exit(obj: *const anyopaque) void { /// This must be called with the appropriate turnstile chain lock held, and /// returns with the chain lock released. /// IPL is kept as before on return. -pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, own: Ownership, queue: Queue) void { +pub fn block( + turnstile: ?*Turnstile, + obj: *anyopaque, + own: Ownership, + queue: Queue, +) void { const chain = chain_for(obj); const curtd = ki.sched.percpu.local().current_thread.?; const ipl = ke.ipl.current(); @@ -441,7 +447,8 @@ pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, own: Ownership, queue: Que // Now block on the event. _ = ke.ipl.lower(.Passive); - _ = ke.wait.wait_one(&waiter.event.hdr, "turnstile:waiter", .{}) catch unreachable; + _ = ke.wait.wait_one(&waiter.event.hdr, "turnstile:waiter", .{}) catch + unreachable; _ = ke.ipl.raise(ipl); } @@ -464,7 +471,10 @@ pub fn wakeup(ts: *Turnstile, queue: Queue, count: usize, new_owner: ?*ke.Thread } else { for (0..count) |_| { if (ts.queues[queue_idx].is_empty()) break; - const waiter: *Waiter = @fieldParentPtr("link", ts.queues[queue_idx].first()); + const waiter: *Waiter = @fieldParentPtr( + "link", + ts.queues[queue_idx].first(), + ); dequeue(ts, waiter.thread); } if (ts.waiters > 0) ts.detach(); @@ -487,7 +497,8 @@ fn dequeue(ts: *Turnstile, td: *ke.Thread) void { ts.next_free = free.next_free; free.next_free = null; } else { - // Last waiter, pull the turnstile off the chain and keep it for ourselves. + // Last waiter, pull the turnstile off the chain + // and keep it for ourselves. ts.link.remove(); ts.reset(); } diff --git a/src/kern/wait.zig b/src/kern/wait.zig index 4819930..6c38c30 100644 --- a/src/kern/wait.zig +++ b/src/kern/wait.zig @@ -13,7 +13,8 @@ pub const DispatchHeader = struct { pub const Type = enum { /// When signaled, `signaled` is kept high and all waiters are woken up. Notification, - /// When signaled, `signaled` is decreased until 0, and a single waiter is woken up. + /// When signaled, `signaled` is decreased until 0, and a single waiter + /// is woken up. Synchronization, /// Special type for queue objects. Queue, @@ -70,11 +71,11 @@ pub const DispatchHeader = struct { pub const WaitBlock = struct { pub const Status = enum(u8) { - /// WaitBlock is linked to an object as part of a thread that is waiting. + /// Block is linked to an object as part of a thread that is waiting. Active, - /// The wait associated with this WaitBlock has been satisfied (or timed out). + /// The wait associated has been satisfied (or timed out). Inactive, - /// A signal was delivered to the wait (this could be before it was committed). + /// A signal was delivered to the wait. Signaled, }; /// List linkage. @@ -114,8 +115,9 @@ pub fn wait_one(object: *DispatchHeader, reason: []const u8, opts: Options) !usi /// Returns the index of the object that was signaled, or an error /// in the case of timeout. /// If `timeout` is not provided, it will wait indefinitely. -/// If `waitblocks` is specified, then the wait will use those waitblocks for the operation. -/// Note that if `timeout` is provided, then one additional waitblock must be allocated. +/// If `waitblocks` is specified, then the wait will use those waitblocks +/// for the operation. Note that if `timeout` is provided, then one additional +/// waitblock must be allocated. pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !usize { const ipl = ke.ipl.raise(.Dispatch); defer ke.ipl.lower(ipl); @@ -123,7 +125,9 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) ! const obj_count = objects.len; const has_timeout = opts.timeout != null; const blocks = opts.waitblocks orelse blk: { - std.debug.assert(obj_count <= curtd.waitblocks.len - @intFromBool(has_timeout)); + std.debug.assert( + obj_count <= curtd.waitblocks.len - @intFromBool(has_timeout), + ); break :blk &curtd.waitblocks; }; @@ -146,7 +150,12 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) ! if (obj.can_satisfy()) { // Object was already signaled. Try consuming it. - if (curtd.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acq_rel, .monotonic) == null) { + if (curtd.wait_status.cmpxchgStrong( + .InProgress, + .Satisfied, + .acq_rel, + .monotonic, + ) == null) { obj.consume(curtd); satisfier = i; } @@ -161,7 +170,7 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) ! wb.object = obj; wb.thread = curtd; wb.status = .Active; - // We are not satisfied yet, so add a waitblock to the object's waitblocks + // We are not satisfied yet, so add a waitblock to the object. obj.waitblocks.insert_tail(&wb.link); } @@ -197,9 +206,15 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) ! curtd.lock.acquire_no_ipl(); // Now try committing the wait. - // While we're trying to commit the wait, the object locks have been released, and the state could therefore change. + // While we're trying to commit the wait, the object locks + // have been released, and the state could therefore change. // We need to re-check the state of the wait before actually blocking. - if (curtd.wait_status.cmpxchgStrong(.InProgress, .Committed, .acq_rel, .monotonic) == null) { + if (curtd.wait_status.cmpxchgStrong( + .InProgress, + .Committed, + .acq_rel, + .monotonic, + ) == null) { if (queue == null) { if (curtd.queue) |q| { ki.queue.signal_wait(q); @@ -236,7 +251,8 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) ! wb.link.remove(); } if (wb.status == .Signaled) { - // This waitblock was signaled, it must be the one that satisfied the wait. + // This waitblock was signaled, it must be the one that satisfied + // the wait. std.debug.assert(satisfier == null); satisfier = i; } @@ -245,7 +261,10 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) ! } const final_sat = satisfier orelse return error.Timeout; - return if (has_timeout and final_sat == timer_i) error.Timeout else final_sat; + return if (has_timeout and final_sat == timer_i) + error.Timeout + else + final_sat; } /// Satisfy a wait on an object. @@ -267,19 +286,29 @@ pub fn satisfy_wait(obj: *DispatchHeader) void { wb.link.remove(); // Three cases may occur here: - // 1. The wait was still preparing (status == .InProgress) and we interrupted it. - // 2. The wait was already committed (status == .Committed) and we satisfied it. - // 3. The wait was already satisfied by another object (status == .Satisfied). - + // 1. The wait was still preparing (.InProgress) and we interrupted it. + // 2. The wait was already committed (.Committed) and we satisfied it. + // 3. The wait was already satisfied by another object (.Satisfied). + // // 1. - if (td.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acq_rel, .monotonic) == null) { + if (td.wait_status.cmpxchgStrong( + .InProgress, + .Satisfied, + .acq_rel, + .monotonic, + ) == null) { // We interrupted the wait while it was being prepared. wb.status = .Signaled; obj.consume(td); } // 2. - else if (td.wait_status.cmpxchgStrong(.Committed, .Satisfied, .acq_rel, .monotonic) == null) { + else if (td.wait_status.cmpxchgStrong( + .Committed, + .Satisfied, + .acq_rel, + .monotonic, + ) == null) { // We interrupted the wait while it was committed. // Wake the thread. wb.status = .Signaled; diff --git a/src/mm/heap.zig b/src/mm/heap.zig index e1a6fa6..4e3d4f5 100644 --- a/src/mm/heap.zig +++ b/src/mm/heap.zig @@ -4,7 +4,12 @@ const mm = r.mm; const mi = mm.private; pub fn init() void { - mi.kernel_space.arena.init("kernel heap", mi.impl.kernel_heap_base, r.tib(16), mm.page_size) catch @panic("failed to initialize kernel heap arena"); + mi.kernel_space.arena.init( + "kernel heap", + mi.impl.kernel_heap_base, + r.tib(16), + mm.page_size, + ) catch @panic("failed to initialize kernel heap arena"); } pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque { @@ -18,7 +23,11 @@ pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque { const npages = size / mm.page_size; for (0..npages) |i| { - const pte = mi.pmap.wire_pte(&mi.kernel_space, addr + (i * mm.page_size), policy) catch { + const pte = mi.pmap.wire_pte( + &mi.kernel_space, + addr + (i * mm.page_size), + policy, + ) catch { mi.kernel_space.lock.release(); return mm.Error.OutOfMemory; }; diff --git a/src/mm/phys.zig b/src/mm/phys.zig index 6e9580e..7742fbe 100644 --- a/src/mm/phys.zig +++ b/src/mm/phys.zig @@ -47,7 +47,8 @@ fn wait_for_pages(old_ipl: ke.Ipl) void { list_lock.release(ipl); - _ = ke.wait.wait_one(&free_page_event.hdr, "wait_for_pages", .{}) catch unreachable; + _ = ke.wait.wait_one(&free_page_event.hdr, "wait_for_pages", .{}) catch + unreachable; ipl = list_lock.acquire(); cnt = free_pages; @@ -150,7 +151,11 @@ pub fn init() linksection(r.init) void { for (0..memory_map.entry_count) |i| { const entry = memory_map.entries[i]; - log.info("[{x:0>16}-{x:0>16}] {s}", .{ entry.base, entry.base + entry.size, @tagName(entry.type) }); + log.info("[{x:0>16}-{x:0>16}] {s}", .{ + entry.base, + entry.base + entry.size, + @tagName(entry.type), + }); if (entry.type == .Free or entry.type == .LoaderReclaimable) { total_usable_memory += entry.size; @@ -158,17 +163,22 @@ pub fn init() linksection(r.init) void { } const pfndb_size_required = @sizeOf(mm.Page) * - (std.math.divCeil(usize, total_usable_memory, mm.page_size) catch unreachable); + (std.math.divCeil(usize, total_usable_memory, mm.page_size) catch + unreachable); - log.info("using {} KiB for pfndb ({} bytes per page)", .{ pfndb_size_required / 1024, @sizeOf(mm.Page) }); + log.info("using {} KiB for pfndb ({} bytes per page)", .{ + pfndb_size_required / 1024, + @sizeOf(mm.Page), + }); // Get the kernel pagemap from the loader, // we are already in it so no need to activate it. mi.impl.init_kernel(); // Calculate the minimum amount of free memory we accept, - // this formula is from Linux and does not grow linearly; it would be wasteful - // to take a fixed percentage of memory from a large server. + // this formula is from Linux and does not grow linearly; + // it would be wasteful to take a fixed percentage of memory from a large + // server. // We clamp this between 128 KiB and 256 MiB. const min_free_kb = std.math.clamp( std.math.sqrt((usable_memory.load(.monotonic) / 1024) * 16), @@ -180,7 +190,10 @@ pub fn init() linksection(r.init) void { const total_usable_pages = total_usable_memory / mm.page_size; - const gap = @max(min_memory_threshold / 4, total_usable_pages * threshold_scale_factor / 10000); + const gap = @max( + min_memory_threshold / 4, + total_usable_pages * threshold_scale_factor / 10000, + ); low_memory_threshold = min_memory_threshold + gap; high_memory_threshold = low_memory_threshold + gap; @@ -189,14 +202,20 @@ pub fn init() linksection(r.init) void { for (0..memory_map.entry_count) |i| { const entry = memory_map.entries[i]; - if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < mm.page_size) { + if (entry.type != .Free and entry.type != .LoaderReclaimable) { + continue; + } + + if (entry.size < mm.page_size) { continue; } const npages = entry.size / mm.page_size; for (0..npages) |j| { - const page: *mm.Page = mm.pfn_to_struct_page(mm.page_to_pfn(entry.base + (j * mm.page_size))); + const page: *mm.Page = mm.pfn_to_struct_page( + mm.page_to_pfn(entry.base + (j * mm.page_size)), + ); free_list.insert_head(&page.free.link); free_pages += 1; diff --git a/src/mm/pmap.zig b/src/mm/pmap.zig index 6222070..db46ffa 100644 --- a/src/mm/pmap.zig +++ b/src/mm/pmap.zig @@ -6,7 +6,8 @@ const std = @import("std"); const mm = r.mm; const mi = mm.private; -/// Describes a contiguous region of physical memory to be mapped at a given virtual address. +/// Describes a contiguous region of physical memory to be mapped +/// at a given virtual address. const MapItem = struct { pa: r.PAddr, len: usize, @@ -87,8 +88,15 @@ pub const PMap = struct { } } - /// Map a contiguous virtual address range to a contiguous physical address range. - pub fn map_contiguous_range(self: *Self, va: r.VAddr, pa: r.PAddr, size: usize, flags: mm.MapFlags) void { + /// Map a contiguous virtual address range to a + /// contiguous physical address range. + pub fn map_contiguous_range( + self: *Self, + va: r.VAddr, + pa: r.PAddr, + size: usize, + flags: mm.MapFlags, + ) void { const src = ContiguousSource{ .flags = flags, .size = size, @@ -163,7 +171,10 @@ pub const PMap = struct { const head: *mm.Page = @ptrCast(head_free); const tail: *mm.Page = @ptrCast(tail_free); - return .{ .head = mm.struct_page_to_pfn(head), .tail = mm.struct_page_to_pfn(tail) }; + return .{ + .head = mm.struct_page_to_pfn(head), + .tail = mm.struct_page_to_pfn(tail), + }; } pub const Cursor = struct { @@ -177,7 +188,11 @@ pub const PMap = struct { /// Reset upward in `advance` when the VA crosses a level boundary. top_level: usize, - fn walk_down(self: *Cursor, target_level: usize, allocate: bool) error{PageNotMapped}!void { + fn walk_down( + self: *Cursor, + target_level: usize, + allocate: bool, + ) error{PageNotMapped}!void { var current_level = self.top_level; while (current_level > target_level) { @@ -191,8 +206,13 @@ pub const PMap = struct { const new_table_pa = mi.phys.alloc(); - const table_ptr: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(new_table_pa)); - @memset(table_ptr[0..entries_per_table], std.mem.zeroes(mi.impl.Pte)); + const table_ptr: [*]mi.impl.Pte = @ptrFromInt( + mm.p2v(new_table_pa), + ); + @memset( + table_ptr[0..entries_per_table], + std.mem.zeroes(mi.impl.Pte), + ); pte = mi.impl.make_table_pte(new_table_pa); self.tables[current_level][idx] = pte; @@ -210,14 +230,20 @@ pub const PMap = struct { return true; } - pub fn map_range(self: *Cursor, pa: r.PAddr, size: usize, flags: mm.MapFlags) void { + pub fn map_range( + self: *Cursor, + pa: r.PAddr, + size: usize, + flags: mm.MapFlags, + ) void { var remain = size; var current_pa = pa; while (remain > 0) { const target_level = 0; - self.walk_down(target_level, true) catch @panic("walk_down failed during map_range"); + self.walk_down(target_level, true) catch + @panic("walk_down failed during map_range"); const table = self.tables[target_level]; table[index_for_level(self.va, target_level)] = @@ -252,7 +278,11 @@ pub const PMap = struct { }; }; -pub fn wire_pte(space: *mi.Space, va: r.VAddr, policy: mm.WaitPolicy) mm.Error!*mi.impl.Pte { +pub fn wire_pte( + space: *mi.Space, + va: r.VAddr, + policy: mm.WaitPolicy, +) mm.Error!*mi.impl.Pte { std.debug.assert(space.lock.is_locked()); var table: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(mi.kernel_space.pmap.root_pa)); @@ -268,7 +298,8 @@ pub fn wire_pte(space: *mi.Space, va: r.VAddr, policy: mm.WaitPolicy) mm.Error!* } if (!entry.present) { - // Release the lock while allocating a new page table, since that may block. + // Release the lock while allocating a new page table, + // since that may block. space.lock.release(); const new_table_pa = mi.phys.alloc_opts(.{ .policy = policy }) orelse { @@ -276,13 +307,15 @@ pub fn wire_pte(space: *mi.Space, va: r.VAddr, policy: mm.WaitPolicy) mm.Error!* return mm.Error.OutOfMemory; }; - // Re-acquire the lock and re-validate the entry, since another thread may have raced to allocate it. + // Re-acquire the lock and re-validate the entry, + // since another thread may have raced to allocate it. space.lock.acquire(); entry = table[idx]; if (entry.present) { - // Another thread beat us to it, free the page we just allocated and continue. + // Another thread beat us to it, + // free the page we just allocated and continue. mi.phys.free(new_table_pa); } else { const new_table_ptr: [*]u64 = @ptrFromInt(mm.p2v(new_table_pa)); diff --git a/src/mm/tlb.zig b/src/mm/tlb.zig index 6c39115..849c6c5 100644 --- a/src/mm/tlb.zig +++ b/src/mm/tlb.zig @@ -35,7 +35,11 @@ fn reclaim_state(obj: *anyopaque, _: ?*anyopaque) void { space.lock.release(); // Free the physical pages - mm.phys.free_batch(mm.pfn_to_struct_page(pfn_list.head), mm.pfn_to_struct_page(pfn_list.tail), npages); + mm.phys.free_batch( + mm.pfn_to_struct_page(pfn_list.head), + mm.pfn_to_struct_page(pfn_list.tail), + npages, + ); _ = async_shootdowns.fetchAdd(1, .monotonic); @@ -83,7 +87,11 @@ pub fn reclaim_range(space: *mm.Space, va: r.VAddr, size: usize) void { space.arena.free(va, size) catch unreachable; // Free the physical pages - mm.phys.free_batch(mm.pfn_to_struct_page(list.head), mm.pfn_to_struct_page(list.tail), size / mm.page_size); + mm.phys.free_batch( + mm.pfn_to_struct_page(list.head), + mm.pfn_to_struct_page(list.tail), + size / mm.page_size, + ); return; }; } diff --git a/src/mm/vmem.zig b/src/mm/vmem.zig index 7eb840a..4ccf230 100644 --- a/src/mm/vmem.zig +++ b/src/mm/vmem.zig @@ -113,7 +113,10 @@ const Segment = struct { return @fieldParentPtr("linkage", linkage); } - fn cmp(a_node: *const rtl.bst.Node, b_node: *const rtl.bst.Node) std.math.Order { + fn cmp( + a_node: *const rtl.bst.Node, + b_node: *const rtl.bst.Node, + ) std.math.Order { const a = Segment.from_tree_link(@constCast(a_node)); const b = Segment.from_tree_link(@constCast(b_node)); return std.math.order(a.base, b.base); @@ -147,7 +150,13 @@ pub const Arena = struct { const Self = @This(); /// Initialize the area for use. - pub fn init(self: *Self, name: []const u8, base: usize, size: usize, quantum: usize) !void { + pub fn init( + self: *Self, + name: []const u8, + base: usize, + size: usize, + quantum: usize, + ) !void { self.name = name; self.base = base; self.size = size; @@ -260,7 +269,13 @@ pub const Arena = struct { // the quantum check avoids creating a remainder too small to ever be useful. if (seg.size != size and (seg.size - size) > self.quantum - 1) { const new_seg = try self.alloc_segment(); - new_seg.* = .{ .type = .Allocated, .base = seg.base, .size = size, .link = undefined, .linkage = .{ .tree_link = undefined } }; + new_seg.* = .{ + .type = .Allocated, + .base = seg.base, + .size = size, + .link = undefined, + .linkage = .{ .tree_link = undefined }, + }; seg.base += size; seg.size -= size; @@ -270,15 +285,18 @@ pub const Arena = struct { new_seg.link.insert_before(&seg.link); self.add_segment_to_freelist(seg); - self.allocated_segments.insert(&new_seg.linkage.tree_link) catch return error.InvalidAddress; + self.allocated_segments.insert(&new_seg.linkage.tree_link) catch + return error.InvalidAddress; if (options.policy == .NextFit) self.rotor = new_seg; return new_seg.base; } else { - // seg is exactly the right size (or remainder is too small), use it directly. + // seg is exactly the right size (or remainder is too small), + // use it directly. seg.type = .Allocated; seg.linkage = .{ .tree_link = undefined }; - self.allocated_segments.insert(&seg.linkage.tree_link) catch return error.InvalidAddress; + self.allocated_segments.insert(&seg.linkage.tree_link) catch + return error.InvalidAddress; if (options.policy == .NextFit) self.rotor = seg; return seg.base; @@ -295,7 +313,9 @@ pub const Arena = struct { .linkage = .{ .tree_link = undefined }, }; - const node = self.allocated_segments.tree.search(&search_node.linkage.tree_link) orelse return error.InvalidAddress; + const node = self.allocated_segments.tree.search( + &search_node.linkage.tree_link, + ) orelse return error.InvalidAddress; const seg = Segment.from_tree_link(node); @@ -310,7 +330,8 @@ pub const Arena = struct { const next_seg: *Segment = @fieldParentPtr("link", next_entry); if (next_seg.type == .Free) { - // Remove next segment from freelist and segment list, then merge into seg. + // Remove next segment from freelist and segment list, + // then merge into seg. self.remove_segment_from_freelist(next_seg); next_seg.link.remove(); seg.size += next_seg.size; @@ -326,7 +347,8 @@ pub const Arena = struct { const prev_seg: *Segment = @fieldParentPtr("link", prev_entry); if (prev_seg.type == .Free) { - // Remove previous segment from freelist and segment list, then merge into seg. + // Remove previous segment from freelist and segment list, + // then merge into seg. self.remove_segment_from_freelist(prev_seg); prev_seg.link.remove(); seg.base = prev_seg.base; @@ -344,7 +366,11 @@ pub const Arena = struct { } // Implementation for the instant fit policy. - fn instant_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } { + fn instant_fit( + self: *Self, + size: usize, + options: AllocOptions, + ) ?struct { usize, *Segment } { var idx = freelist_index(size); if (!std.math.isPowerOfTwo(size)) idx += 1; @@ -356,7 +382,8 @@ pub const Arena = struct { var it = self.freelists[idx].iterator(); while (it.next()) : (it.advance()) { const seg = Segment.from_free_link(it.get()); - if (self.try_to_fit(seg, size, options)) |addr| return .{ addr, seg }; + if (self.try_to_fit(seg, size, options)) |addr| + return .{ addr, seg }; } } return null; @@ -364,7 +391,11 @@ pub const Arena = struct { // Implementation for the best fit policy. // Find the smallest segment that can satisfy the request. - fn best_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } { + fn best_fit( + self: *Self, + size: usize, + options: AllocOptions, + ) ?struct { usize, *Segment } { var idx = freelist_index(size); var best_seg: ?*Segment = null; var best_start: usize = 0; @@ -388,8 +419,13 @@ pub const Arena = struct { } // Implementation for the next fit policy. - // Continue searching from the last allocated segment, wrapping around at the end of the arena. - fn next_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } { + // Continue searching from the last allocated segment, wrapping around at + // the end of the arena. + fn next_fit( + self: *Self, + size: usize, + options: AllocOptions, + ) ?struct { usize, *Segment } { // Start from rotor if we have one, otherwise from the beginning. const start_entry = if (self.rotor) |r| r.link.next else self.list.first(); @@ -418,9 +454,15 @@ pub const Arena = struct { // Try to fit the given segment to the request with the given options. // Returns the fitted address if successful. - fn try_to_fit(self: *Self, segment: *const Segment, size: usize, options: AllocOptions) ?usize { + fn try_to_fit( + self: *Self, + segment: *const Segment, + size: usize, + options: AllocOptions, + ) ?usize { var start = @max(segment.base, options.min orelse 0); - const end = @min(segment.base + segment.size, options.max orelse std.math.maxInt(usize)); + const end = @min(segment.base + segment.size, options.max orelse + std.math.maxInt(usize)); const alignment = options.alignment orelse self.quantum; if (start > end) return null; diff --git a/src/mm/zone.zig b/src/mm/zone.zig index dacd4d5..4f3e8ce 100644 --- a/src/mm/zone.zig +++ b/src/mm/zone.zig @@ -18,9 +18,10 @@ //! ## Per-CPU layer //! ---------------- //! Each CPU has an `alloc` and `free` magazine, where objects are allocated -//! from and freed to, respectively. This separation avoids hysteresis and is useful -//! for safe memory reclamation (see below). -//! When one side runs dry/full, the magazines may be swapped, if SMR is disabled. +//! from and freed to, respectively. This separation avoids hysteresis and is +//! useful for safe memory reclamation (see below). +//! When one side runs dry/full, the magazines may be swapped, +//! if SMR is disabled. //! Each CPU also has its own depot, from which it allocates and frees its //! magazines to first. If there is no magazine in the local depot, the magazine //! will be provided from the zone depot layer. We keep track of the size of the @@ -42,11 +43,11 @@ //! ## Slab layer //! ------------- //! The slab layer is the fallback when all previous layers failed. A slab -//! represents a contiguous chunk of memory holding a fixed number of constructed -//! objects, determined when the zone is created, and is allocated via the -//! kernel's virtual memory allocator. Allocation in the slab is managed by a -//! bitmap and slab metadata is retrieved through the `mm.Page` structure -//! representing the page of allocated memory. +//! represents a contiguous chunk of memory holding a fixed number of +//! constructed objects, and is allocated via the kernel's virtual memory +//! allocator. Allocation in the slab is managed by a bitmap and slab metadata +//! is retrieved through the `mm.Page` structure representing the page +//! of allocated memory. //! //! ## Hardening features //! --------------------- @@ -90,10 +91,10 @@ //! commits. When alloc needs a full magazine, it first polls the sequence of //! the head of the depot magazine list (since SMR zones are forced //! FIFO and stamps are taken in monotonic order, the head always carries the -//! minimum sequence). If the local head has not expired, the local full magazines -//! are moved and appended to the zone depot to age, and their sequences -//! are committed, while expired magazines from the zone depot are pulled back in -//! to satisfy the request. +//! minimum sequence). If the local head has not expired, the local full +//! magazines are moved and appended to the zone depot to age, +//! and their sequences are committed, while expired magazines from the zone +//! depot are pulled back in to satisfy the request. //! //! All this machinery is to avoid expensive scans during SMR polling, //! growing the per-CPU depots widens the window between a stamp and its poll @@ -134,19 +135,35 @@ const zones_num = 32; const magazine_size = ke.Tunable(u8, 8, "mm.zone.mag_size"); /// How much extra memory CPUs are allowed to keep around. -const max_local_memory = ke.Tunable(u32, @intCast(r.kib(128)), "mm.zone.max_local_mem"); +const max_local_memory = ke.Tunable( + u32, + @intCast(r.kib(128)), + "mm.zone.max_local_mem", +); /// Number of contentions allowed per second before the depot grows. -const depot_grow_level = ke.Tunable(u32, 5 * wma_unit, "mm.zone.depot_grow_level"); +const depot_grow_level = ke.Tunable( + u32, + 5 * wma_unit, + "mm.zone.depot_grow_level", +); /// Number of contentions allowed per second before the depot shrinks. -const depot_shrink_level = ke.Tunable(u32, wma_unit / 2, "mm.zone.depot_shrink_level"); +const depot_shrink_level = ke.Tunable( + u32, + wma_unit / 2, + "mm.zone.depot_shrink_level", +); /// Number of excess magazines in a zone before they are trimmed. const excess_magazines = ke.Tunable(u32, 8, "mm.zone.excess_mags"); /// Excess memory in a zone before magazines are trimmed. -const excess_memory = ke.Tunable(u32, @intCast(r.kib(16)), "mm.zone.excess_mem"); +const excess_memory = ke.Tunable( + u32, + @intCast(r.kib(16)), + "mm.zone.excess_mem", +); /// Every power-of-two-size from 8 to 2048 (inclusively). const generic_zones_num = 9; @@ -189,7 +206,8 @@ const should_check_poison = config.slab_check_poison; const AllocOpts = struct { policy: mm.WaitPolicy = .WaitForMemory }; -/// A slab is a contiguous memory region that holds multiple objects of the same type. +/// A slab is a contiguous memory region that holds multiple objects of the same +/// type. const Slab = struct { /// Linkage into a zone's list of slabs. link: rtl.List.Entry, @@ -300,7 +318,11 @@ const Slab = struct { // since a smaller capacity means a smaller bitmap, which may then fit. while (capacity > 0) : (capacity -= 1) { const objects_off = - std.mem.alignForward(usize, @sizeOf(Slab) + Slab.bitmap_bytes(capacity), alignment); + std.mem.alignForward( + usize, + @sizeOf(Slab) + Slab.bitmap_bytes(capacity), + alignment, + ); if (objects_off + capacity * chunk_size <= mm.page_size) { break; @@ -352,7 +374,11 @@ const Depot = struct { return ret; } - fn maglist_free(list: *MagazineList, mag: *Magazine, policy: ReusePolicy) void { + fn maglist_free( + list: *MagazineList, + mag: *Magazine, + policy: ReusePolicy, + ) void { mag.next = null; if (policy == .FIFO) { @@ -430,7 +456,8 @@ inline fn check_poison(ptr: *anyopaque, len: usize, pattern: u32) bool { var i: usize = 0; while (i + 4 <= len) : (i += 4) { - if (std.mem.readInt(u32, bytes[i..][0..4], .little) != pattern) return false; + if (std.mem.readInt(u32, bytes[i..][0..4], .little) != pattern) + return false; } if (i < len) { @@ -454,7 +481,8 @@ inline fn wma_mix(old: usize, new: usize) usize { } fn cpus_alloc() []rtl.CachePadded(Cpu) { - const ptr = cpu_zone.alloc(.{}) catch @panic("failed to allocate zone CPU state"); + const ptr = cpu_zone.alloc(.{}) catch + @panic("failed to allocate zone CPU state"); const raw: [*]rtl.CachePadded(Cpu) = @ptrCast(@alignCast(ptr)); return raw[0..ke.ncpus]; } @@ -505,7 +533,7 @@ pub const Zone = struct { const Self = @This(); const InitOptions = struct { - alignment: u16 = 0, + alignment: u16 = slab_align, ctor: ?*const fn (*anyopaque) void = null, dtor: ?*const fn (*anyopaque) void = null, magazines: bool = true, @@ -515,11 +543,19 @@ pub const Zone = struct { }; /// Initialize a zone. - pub fn init(self: *Self, name: []const u8, size: usize, options: InitOptions) void { - const obj_align = if (options.alignment == 0) slab_align else options.alignment; + pub fn init( + self: *Self, + name: []const u8, + size: usize, + options: InitOptions, + ) void { + const obj_align = options.alignment; const chunk_size = std.mem.alignForward(usize, size, obj_align); - const slab_size = if (size <= small_slab_size) mm.page_size else Slab.calc_slab_size(chunk_size); + const slab_size = if (size <= small_slab_size) + mm.page_size + else + Slab.calc_slab_size(chunk_size); self.full_slabs.init(); self.partial_slabs.init(); @@ -538,12 +574,17 @@ pub const Zone = struct { self.depot_contention_cur = 0; self.depot_contention_wma = 0; self.cpu_depot_size = 0; - self.cpu_depot_limit = max_local_memory.load() / (self.chunk_size * magazine_size.load()); + self.cpu_depot_limit = max_local_memory.load() / + (self.chunk_size * magazine_size.load()); self.trim_depot = false; // SMR zones have to be FIFO so that the minimum sequence can be // obtained by checking the list head. - self.reuse_policy = if (options.smr != null) .FIFO else options.reuse_policy; + self.reuse_policy = if (options.smr != null) + .FIFO + else + options.reuse_policy; + self.smr = options.smr; self.smr_reclaim = options.smr_reclaim; @@ -608,8 +649,9 @@ pub const Zone = struct { if (cpu.free_rounds > 0 and self.smr == null) { std.debug.assert(cpu.free != null); - // Alloc magazine is empty. If the free magazine has items, swap them. - // The free magazine now becomes our alloc magazine. + // Alloc magazine is empty. If the free magazine has items, + // swap them. The free magazine now becomes our alloc + // magazine. std.mem.swap(?*Magazine, &cpu.alloc, &cpu.free); std.mem.swap(usize, &cpu.alloc_rounds, &cpu.free_rounds); continue; @@ -622,7 +664,11 @@ pub const Zone = struct { // Discard our empty alloc magazine to the depot. if (cpu.alloc) |empty_mag| { - Depot.free(empty_mag, &cpu.depot.empty_mags, self.reuse_policy); + Depot.free( + empty_mag, + &cpu.depot.empty_mags, + self.reuse_policy, + ); } self.magazine_reuse(full_mag); @@ -632,8 +678,8 @@ pub const Zone = struct { } // Local depot is empty, rebalance with the zone depot. - // Purposefully race against cpu_depot_size because locking is tricky here, - // we may get a stale value but it is harmless. + // Purposefully race against cpu_depot_size because locking is + // tricky here, we may get a stale value but it is harmless. const n = @atomicLoad(usize, &self.cpu_depot_size, .monotonic); self.alloc_depot_rebalance(n, cpu); @@ -644,7 +690,11 @@ pub const Zone = struct { // Discard our empty alloc magazine to the depot. if (cpu.alloc) |empty_mag| { - Depot.free(empty_mag, &cpu.depot.empty_mags, self.reuse_policy); + Depot.free( + empty_mag, + &cpu.depot.empty_mags, + self.reuse_policy, + ); } self.magazine_reuse(full_mag); @@ -670,9 +720,10 @@ pub const Zone = struct { self.lock.acquire(); - // Note: this is racy and could lead to two threads creating a slab at the same time. - // Worst case scenario, we have one slab too many, so we don't care; we already did the work required - // for slab construction and this ideally should not happen very often. + // Note: this is racy and could lead to two threads creating a slab + // at the same time. Worst case scenario, we have one slab too many, + // so we don't care; we already did the work required for slab + // construction and this ideally should not happen very often. self.partial_slabs.insert_tail(&slab.link); } else { slab = @fieldParentPtr("link", self.partial_slabs.first()); @@ -718,7 +769,9 @@ pub const Zone = struct { cpu.lock.acquire_no_ipl(); const freed = while (true) { - if (cpu.free != null and cpu.free_rounds < magazine_size.load()) { + if (cpu.free != null and + cpu.free_rounds < magazine_size.load()) + { // Fast path: push directly to free magazine. // For SMR zones the dtor and poisoning are deferred until // the magazine is reused, readers may still hold the object. @@ -741,8 +794,12 @@ pub const Zone = struct { break true; } - if (cpu.alloc_rounds == 0 and cpu.alloc != null and self.smr == null) { - // Free magazine is full. If the alloc magazine is completely empty, swap them. + if (cpu.alloc_rounds == 0 and + cpu.alloc != null and + self.smr == null) + { + // Free magazine is full. + // If the alloc magazine is completely empty, swap them. // The empty alloc magazine becomes our new free magazine. std.mem.swap(?*Magazine, &cpu.alloc, &cpu.free); std.mem.swap(usize, &cpu.alloc_rounds, &cpu.free_rounds); @@ -754,7 +811,11 @@ pub const Zone = struct { if (Depot.alloc(&cpu.depot.empty_mags)) |empty_mag| { // Put the free magazine on the full list. if (cpu.free) |full_mag| { - Depot.free(full_mag, &cpu.depot.full_mags, self.reuse_policy); + Depot.free( + full_mag, + &cpu.depot.full_mags, + self.reuse_policy, + ); } cpu.free = empty_mag; @@ -772,7 +833,11 @@ pub const Zone = struct { if (Depot.alloc(&cpu.depot.empty_mags)) |empty_mag| { // Put the free magazine on the full list. if (cpu.free) |full_mag| { - Depot.free(full_mag, &cpu.depot.full_mags, self.reuse_policy); + Depot.free( + full_mag, + &cpu.depot.full_mags, + self.reuse_policy, + ); } cpu.free = empty_mag; @@ -784,7 +849,12 @@ pub const Zone = struct { // We need to drop IPL and the CPU lock here because we need to acquire // the magazine's allocator's blocking lock. cpu.lock.release(ipl); - const new_mag: ?*Magazine = @ptrCast(@alignCast(magazine_zone.alloc(.{ .policy = .DontWaitForMemory }) catch null)); + + const raw_mag = magazine_zone.alloc( + .{ .policy = .DontWaitForMemory }, + ) catch null; + + const new_mag: ?*Magazine = @ptrCast(@alignCast(raw_mag)); // Re-grab our current context. ipl = ke.ipl.raise(.Dispatch); @@ -831,12 +901,18 @@ pub const Zone = struct { // Find the slab for the buffer. const slab: *Slab = if (self.obj_size > small_slab_size) blk: { - const page_va = std.mem.alignBackward(usize, @intFromPtr(obj), mm.page_size); + const page_va = std.mem.alignBackward( + usize, + @intFromPtr(obj), + mm.page_size, + ); const phys = mi.kernel_space.pmap.query(page_va).?; const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys)); break :blk page.alloced.slab_data.slab; } else blk: { - break :blk @ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(obj), mm.page_size)); + break :blk @ptrFromInt( + std.mem.alignBackward(usize, @intFromPtr(obj), mm.page_size), + ); }; const was_full = slab.refcount == slab.capacity; @@ -860,34 +936,46 @@ pub const Zone = struct { } /// Periodic updates on the zone. - /// Updates the working set and trims it if needed (for now, will do magazine resizing later...). + /// Updates the magazine working set and trims or grows them if needed. fn update(self: *Self) void { const ipl = self.depot_lock.acquire(); // Update the working set size. - // This tracks the minimum number of magazines required, we assume it is safe - // to reclaim `min` magazines, since the number of magazines never went below that, and - // those magazines sat unused. e.g if the number of magazines ranged between 47 and 37 in an update - // interval, then the working set size is 10 and we can reclaim 37 magazines. We keep track of - // this in a weighted moving average (WMA) to bias recent utilization when trimming. - self.depot.empty_mags.wma = wma_mix(self.depot.empty_mags.wma, self.depot.empty_mags.min); + // This tracks the minimum number of magazines required, + // we assume it is safe to reclaim `min` magazines, since the number of + // magazines never went below that, and those magazines sat unused. e.g + // if the number of magazines ranged between 47 and 37 in an update + // interval, then the working set size is 10 and we can + // reclaim 37 magazines. We keep track of this in a weighted moving + // average (WMA) to bias recent utilization when trimming. + self.depot.empty_mags.wma = wma_mix( + self.depot.empty_mags.wma, + self.depot.empty_mags.min, + ); self.depot.empty_mags.min = self.depot.empty_mags.num; - self.depot.full_mags.wma = wma_mix(self.depot.full_mags.wma, self.depot.full_mags.min); + self.depot.full_mags.wma = wma_mix( + self.depot.full_mags.wma, + self.depot.full_mags.min, + ); self.depot.full_mags.min = self.depot.full_mags.num; // Calculate the number of contentions/second in fixed-point. const old = self.depot_contention_wma; - var cur = self.depot_contention_cur * wma_unit / (ke.ncpus * update_interval_s); + var cur = self.depot_contention_cur * wma_unit / + (ke.ncpus * update_interval_s); // WMA formula. cur = (3 * old + cur) / 4; if (self.use_magazines) { - if (self.cpu_depot_size < self.cpu_depot_limit and cur > depot_grow_level.load()) { + if (self.cpu_depot_size < self.cpu_depot_limit and + cur > depot_grow_level.load()) + { // We have room to grow the depot and we should. - // Put the new WMA at around midpoint between shrink and growth, so that - // we have time to check whether what we just did is good or not. + // Put the new WMA at around midpoint between shrink and growth, + // so that we have time to check whether what we just did is + // good or not. cur = (depot_grow_level.load() + depot_shrink_level.load()) / 2; const size = if (self.cpu_depot_size == 0) @@ -898,7 +986,9 @@ pub const Zone = struct { // Clamp it. self.cpu_depot_size = @min(size, self.cpu_depot_limit); - } else if (self.cpu_depot_size > 0 and cur <= depot_shrink_level.load()) { + } else if (self.cpu_depot_size > 0 and + cur <= depot_shrink_level.load()) + { // We should shrink the depot. cur = (depot_grow_level.load() + depot_shrink_level.load()) / 2; self.cpu_depot_size -= 1; @@ -926,19 +1016,28 @@ pub const Zone = struct { if (!self.use_magazines) return false; if (self.trim_depot) return true; - const empty = @min(self.depot.empty_mags.wma, self.depot.empty_mags.min * wma_unit); + const empty = @min( + self.depot.empty_mags.wma, + self.depot.empty_mags.min * wma_unit, + ); if (empty > excess_magazines.load() * wma_unit) { // Too many excess magazines. return true; } - const full = @min(self.depot.full_mags.wma, self.depot.full_mags.min * wma_unit); + const full = @min( + self.depot.full_mags.wma, + self.depot.full_mags.min * wma_unit, + ); const full_bytes = full * magazine_size.load() * self.chunk_size; - if (full >= 2 * wma_unit and full_bytes >= excess_memory.load() * wma_unit) { - // We have at least 2 excess full magazines and they take up at least 16 KiB. + if (full >= 2 * wma_unit and + full_bytes >= excess_memory.load() * wma_unit) + { + // We have at least 2 excess full magazines + // and they take up at least 16 KiB. return true; } @@ -948,8 +1047,8 @@ pub const Zone = struct { /// Called periodically to trim our magazines. /// This frees the excess magazines. fn trim(self: *Self, trim_depot: bool) void { - // Note: we do all of this under the zone lock, - // but it *should* be a fairly quick operation, even though it is O(ncpus). + // Note: we do all of this under the zone lock, but it *should* be a + // a fairly quick operation, even though it is O(ncpus). self.lock.acquire(); self.trim_maglist(&self.depot.empty_mags, 0); @@ -975,8 +1074,7 @@ pub const Zone = struct { fn drain(self: *Self) void { if (!self.use_magazines) return; - // Note: we do all of this under the zone lock, - // but it *should* be a fairly quick operation, even though it is O(ncpus). + // Note: we do all of this under the zone lock, same as above. self.lock.acquire(); const ipl = self.depot_lock.acquire(); @@ -1055,7 +1153,11 @@ pub const Zone = struct { // Move them to our depot. for (0..n) |_| { - Depot.free(Depot.alloc(&cpu.depot.full_mags).?, &depot.full_mags, self.reuse_policy); + Depot.free( + Depot.alloc(&cpu.depot.full_mags).?, + &depot.full_mags, + self.reuse_policy, + ); } } @@ -1065,7 +1167,11 @@ pub const Zone = struct { // Move them to our depot. for (0..n) |_| { - Depot.free(Depot.alloc(&cpu.depot.empty_mags).?, &depot.empty_mags, self.reuse_policy); + Depot.free( + Depot.alloc(&cpu.depot.empty_mags).?, + &depot.empty_mags, + self.reuse_policy, + ); } } @@ -1108,7 +1214,8 @@ pub const Zone = struct { } fn alloc_page(policy: mm.WaitPolicy) !*anyopaque { - const alloc_ret = mi.phys.alloc_opts(.{ .policy = policy }) orelse return error.OutOfMemory; + const alloc_ret = mi.phys.alloc_opts(.{ .policy = policy }) orelse + return error.OutOfMemory; return @ptrFromInt(mm.p2v(alloc_ret)); } @@ -1129,7 +1236,8 @@ pub const Zone = struct { slab.alloc_rr = 0; slab.refcount = 0; slab.capacity = @intCast(capacity); - const meta_end = @intFromPtr(slab) + @sizeOf(Slab) + Slab.bitmap_bytes(capacity); + const meta_end = @intFromPtr(slab) + @sizeOf(Slab) + + Slab.bitmap_bytes(capacity); slab.base = std.mem.alignForward(usize, meta_end, self.alignment); const bm = slab.bitmap(); @@ -1154,7 +1262,11 @@ pub const Zone = struct { const capacity = (self.slab_size) / self.chunk_size; const buf = try mi.heap.alloc(self.slab_size, policy); - var ret: *Slab = @ptrCast(@alignCast(try gpa.alloc(u8, @sizeOf(Slab) + Slab.bitmap_bytes(capacity)))); + var ret: *Slab = @ptrCast( + @alignCast( + try gpa.alloc(u8, @sizeOf(Slab) + Slab.bitmap_bytes(capacity)), + ), + ); // Mark the bitmap as all free. const bm = ret.bitmap(); @@ -1171,7 +1283,9 @@ pub const Zone = struct { ret.alloc_rr = 0; for (0..self.slab_size / mm.page_size) |i| { - const phys_page = mi.kernel_space.pmap.query(@intFromPtr(buf) + i * mm.page_size) orelse @panic("Could not query page"); + const phys_page = mi.kernel_space.pmap.query( + @intFromPtr(buf) + i * mm.page_size, + ) orelse @panic("Could not query page"); const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys_page)); page.alloced.slab_data.slab = ret; } @@ -1202,7 +1316,9 @@ pub const Zone = struct { mm.heap.free(slab.base, self.slab_size); } else { - free_page(@ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(slab), mm.page_size))); + free_page(@ptrFromInt( + std.mem.alignBackward(usize, @intFromPtr(slab), mm.page_size), + )); } } @@ -1257,9 +1373,10 @@ pub const Zone = struct { ke.smr.deferred_advance_commit(smr, tail.seq); } - /// Try to rebalance the zone and cpu depots w.r.t to each other after an allocation. - /// The CPU depot moves its extra empty magazines to the zone depot, and tries - /// to get full magazines from the zone depot up to `target`. CPU lock is held. + /// Try to rebalance the zone and cpu depots w.r.t to each other + /// after an allocation. The CPU depot moves its extra full magazines to the + /// zone depot, and tries to get full magazines from the zone depot up to + /// `target`. CPU lock is held. fn alloc_depot_rebalance(self: *Self, target: usize, cpu: *Cpu) void { if (!self.depot_lock.try_acquire_no_ipl()) { // Register the contention. @@ -1272,7 +1389,11 @@ pub const Zone = struct { const n = cpu.depot.empty_mags.num - target / 2; for (0..n) |_| { - Depot.free(Depot.alloc(&cpu.depot.empty_mags).?, &self.depot.empty_mags, self.reuse_policy); + Depot.free( + Depot.alloc(&cpu.depot.empty_mags).?, + &self.depot.empty_mags, + self.reuse_policy, + ); } } @@ -1280,27 +1401,35 @@ pub const Zone = struct { // We got here because the local head hasn't expired or there is // no local magazine at all, rotate the local magazines through the // zone depot to let their sequences age. - // We then hope that the head of the zone depot has expired, otherwise we just return - // and fall back to the slab layer. + // We then hope that the head of the zone depot has expired, + // otherwise we just return and fall back to the slab layer. self.smr_rotate_full(smr, cpu); } // Grab a batch of full magazines from the zone depot. // Exchange at least one magazine even when the depot size is 0. - const n = @min(@max(target, 1) - cpu.depot.empty_mags.num, self.depot.full_mags.num); + const n = @min( + @max(target, 1) - cpu.depot.empty_mags.num, + self.depot.full_mags.num, + ); if (n != 0 and self.full_mags_ready(&self.depot.full_mags)) { for (0..n) |_| { - Depot.free(Depot.alloc(&self.depot.full_mags).?, &cpu.depot.full_mags, self.reuse_policy); + Depot.free( + Depot.alloc(&self.depot.full_mags).?, + &cpu.depot.full_mags, + self.reuse_policy, + ); } } self.depot_lock.release_no_ipl(); } - /// Try to rebalance the zone and cpu depots w.r.t to each other after a free. - /// The CPU depot moves its extra full magazines to the zone depot, and tries - /// to get empty magazines from the zone depot up to `target`. CPU lock is held. + /// Try to rebalance the zone and cpu depots w.r.t to each other + /// after a free.The CPU depot moves its extra full magazines to the + /// zone depot, and tries to get empty magazines from the zone depot up to + /// `target`. CPU lock is held. fn free_depot_rebalance(self: *Self, target: usize, cpu: *Cpu) void { if (!self.depot_lock.try_acquire_no_ipl()) { // Register the contention. @@ -1318,7 +1447,11 @@ pub const Zone = struct { if (n != 0 and self.full_mags_ready(&self.depot.full_mags)) { for (0..n) |_| { - Depot.free(Depot.alloc(&self.depot.full_mags).?, &cpu.depot.full_mags, self.reuse_policy); + Depot.free( + Depot.alloc(&self.depot.full_mags).?, + &cpu.depot.full_mags, + self.reuse_policy, + ); } } } else { @@ -1326,16 +1459,27 @@ pub const Zone = struct { const n = cpu.depot.full_mags.num - target / 2; for (0..n) |_| { - Depot.free(Depot.alloc(&cpu.depot.full_mags).?, &self.depot.full_mags, self.reuse_policy); + Depot.free( + Depot.alloc(&cpu.depot.full_mags).?, + &self.depot.full_mags, + self.reuse_policy, + ); } } } // Grab a batch of empty magazines from the zone depot. // Exchange at least one magazine even when the depot size is 0. - const n = @min(@max(target, 1) - cpu.depot.full_mags.num, self.depot.empty_mags.num); + const n = @min( + @max(target, 1) - cpu.depot.full_mags.num, + self.depot.empty_mags.num, + ); for (0..n) |_| { - Depot.free(Depot.alloc(&self.depot.empty_mags).?, &cpu.depot.empty_mags, self.reuse_policy); + Depot.free( + Depot.alloc(&self.depot.empty_mags).?, + &cpu.depot.empty_mags, + self.reuse_policy, + ); } self.depot_lock.release_no_ipl(); @@ -1403,9 +1547,13 @@ pub fn early_init() linksection(r.init) void { zone.init(generic_zone_names[i], @as(usize, 1) << @intCast(i + 3), .{}); } - magazine_zone.init("magazine", @sizeOf(Magazine) + magazine_size.load() * @sizeOf(*anyopaque), .{ - .magazines = false, - }); + magazine_zone.init( + "magazine", + @sizeOf(Magazine) + magazine_size.load() * @sizeOf(*anyopaque), + .{ + .magazines = false, + }, + ); } /// Post-SMP initialization. @@ -1415,7 +1563,8 @@ pub fn late_init() linksection(r.init) void { .alignment = std.atomic.cache_line, }); - // Go through all caches with magazines enabled and initialize their per-CPU magazines. + // Go through all caches with magazines enabled and initialize + // their per-CPU magazines. zone_list_lock.acquire(); var zone = all_zones; @@ -1446,8 +1595,10 @@ pub fn late_init() linksection(r.init) void { smr_zone.init("SMR", .{}); smr_cpu_zone.init("SMR CPU", @sizeOf(ke.smr.Cpu) * ke.ncpus, .{}); - smr_sys_domain = smr_domain_create(false) catch @panic("Failed to create smr domain"); - smr_sys_preempt_domain = smr_domain_create(true) catch @panic("Failed to create smr domain"); + smr_sys_domain = smr_domain_create(false) catch + @panic("Failed to create smr domain"); + smr_sys_preempt_domain = smr_domain_create(true) catch + @panic("Failed to create smr domain"); } pub fn smr_domain_create(preempt: bool) !*ke.smr.Domain { @@ -1528,7 +1679,8 @@ fn gpa_remap( ) ?[*]u8 { if (gpa_resize(undefined, buf, alignment, new_len, ret_addr)) return buf.ptr; - const new_ptr = gpa_alloc(undefined, new_len, alignment, ret_addr) orelse return null; + const new_ptr = gpa_alloc(undefined, new_len, alignment, ret_addr) orelse + return null; const copy_len = @min(buf.len, new_len); @memcpy(new_ptr[0..copy_len], buf[0..copy_len]); gpa_free(undefined, buf, alignment, ret_addr); diff --git a/src/platform/acpi/hpet.zig b/src/platform/acpi/hpet.zig index 77d1789..ade2d7d 100644 --- a/src/platform/acpi/hpet.zig +++ b/src/platform/acpi/hpet.zig @@ -21,7 +21,8 @@ const femtos_per_s = 1_000_000_000_000_000; pub fn init(hpet: *acpi.HpetTable) linksection(r.init) void { hpet_regs = @ptrFromInt(mm.p2v(hpet.base_address.address)); - hpet_timer.frequency = femtos_per_s / (r.mmio_read(u64, @intFromPtr(&hpet_regs.general_capabilities)) >> 32); + hpet_timer.frequency = femtos_per_s / + (r.mmio_read(u64, @intFromPtr(&hpet_regs.general_capabilities)) >> 32); // Disable timer. r.mmio_write(u64, @intFromPtr(&hpet_regs.general_configuration), 0); diff --git a/src/platform/acpi/madt.zig b/src/platform/acpi/madt.zig index 2dec0e0..bfd64c8 100644 --- a/src/platform/acpi/madt.zig +++ b/src/platform/acpi/madt.zig @@ -11,7 +11,9 @@ pub const MadtIterator = struct { pub fn next(self: *MadtIterator) ?*const acpi.MadtEntryHeader { const entries_size = self.madt.header.length - @sizeOf(acpi.Madt); if (self.i >= entries_size) return null; - const entry: *const acpi.MadtEntryHeader = @ptrFromInt(@intFromPtr(&self.madt.entries) + self.i); + const entry: *const acpi.MadtEntryHeader = @ptrFromInt( + @intFromPtr(&self.madt.entries) + self.i, + ); self.i += @max(2, entry.length); return entry; } diff --git a/src/platform/acpi/root.zig b/src/platform/acpi/root.zig index 7807fbd..8fb822e 100644 --- a/src/platform/acpi/root.zig +++ b/src/platform/acpi/root.zig @@ -97,7 +97,8 @@ pub const Gas = extern struct { register_bit_width: u8 align(1), /// The bit offset of the given register at the given address. register_bit_offset: u8 align(1), - /// The access size of the given register. (1 = byte, 2 = word, 3 = dword, 4 = qword). + /// The access size of the given register. + /// (1 = byte, 2 = word, 3 = dword, 4 = qword). access_size: u8 align(1), address: u64 align(1), pub fn read(self: Gas) u64 { @@ -142,9 +143,21 @@ pub const Gas = extern struct { else => unreachable, }, .SystemIo => switch (self.access_size) { - 1 => arch.pio_write(u8, @truncate(self.address), @truncate(shifted_value)), - 2 => arch.pio_write(u16, @truncate(self.address), @truncate(shifted_value)), - 3 => arch.pio_write(u32, @truncate(self.address), @truncate(shifted_value)), + 1 => arch.pio_write( + u8, + @truncate(self.address), + @truncate(shifted_value), + ), + 2 => arch.pio_write( + u16, + @truncate(self.address), + @truncate(shifted_value), + ), + 3 => arch.pio_write( + u32, + @truncate(self.address), + @truncate(shifted_value), + ), else => unreachable, }, else => @panic("unsupported GAS address space"), @@ -213,7 +226,8 @@ pub const Fadt = extern struct { const version_sizes = [_]usize{ 116, 132, 244, 244, 268, 276 }; pub fn revision(self: Fadt) u8 { - // We can't rely on firmware to set the revision field correctly, so we infer it from the length field. + // We can't rely on firmware to set the revision field correctly, + // so we infer it from the length field. const declared: usize = self.header.revision; const len: u32 = self.header.length; diff --git a/src/platform/pc/apic.zig b/src/platform/pc/apic.zig index ae02c9a..70dd335 100644 --- a/src/platform/pc/apic.zig +++ b/src/platform/pc/apic.zig @@ -90,11 +90,16 @@ fn xapic_read(register: ApicRegisters) u32 { } fn x2apic_write(register: ApicRegisters, value: u64) void { - amd64.wrmsr(@intFromEnum(amd64.Msr.X2ApicBase) + (@as(u32, @intFromEnum(register)) >> 4), value); + amd64.wrmsr( + @intFromEnum(amd64.Msr.X2ApicBase) + (@intFromEnum(register) >> 4), + value, + ); } fn x2apic_read(register: ApicRegisters) u64 { - return amd64.rdmsr(@intFromEnum(amd64.Msr.X2ApicBase) + (@as(u32, @intFromEnum(register)) >> 4)); + return amd64.rdmsr( + @intFromEnum(amd64.Msr.X2ApicBase) + (@intFromEnum(register) >> 4), + ); } pub fn write(register: ApicRegisters, value: u32) void { @@ -133,7 +138,10 @@ pub fn send_init(apic_id: u32) void { pub fn send_sipi(apic_id: u32, startup_page: u8) void { if (in_x2apic_mode.local().*) { - x2apic_write(.Icr, (@as(u64, apic_id) << 32) | 0x00004600 | startup_page); + x2apic_write( + .Icr, + (@as(u64, apic_id) << 32) | 0x00004600 | startup_page, + ); } else { xapic_write(.Icr1, @as(u32, apic_id) << 24); xapic_write(.Icr, @as(u32, 0x00004600) | startup_page); @@ -143,7 +151,8 @@ pub fn send_sipi(apic_id: u32, startup_page: u8) void { pub fn send_ipi(apic_id: u32, vector: u8, delivery_mode: u8) void { if (in_x2apic_mode.local().*) { - const icr_value = (@as(u64, apic_id) << 32) | (@as(u64, delivery_mode) << 8) | vector; + const icr_value = (@as(u64, apic_id) << 32) | + (@as(u64, delivery_mode) << 8) | vector; x2apic_write(.Icr, icr_value); } else { const icr_value = (@as(u32, delivery_mode) << 8) | vector; @@ -200,10 +209,11 @@ pub fn arm_timer(ns: r.Nanoseconds) void { if (amd64.cpu_features.tsc_deadline) { const now = amd64.rdtsc(); - const delta: u64 = @intCast((@as(u128, ns.value) * tsc.tsc_timer.frequency) / std.time.ns_per_s); + const delta: u64 = @intCast( + (@as(u128, ns.value) * tsc.tsc_timer.frequency) / std.time.ns_per_s, + ); const deadline = now +% delta; - amd64.write_msr(.TscDeadline, deadline); return; } @@ -242,7 +252,8 @@ pub fn init() linksection(r.init) void { const cur_cpu_id = get_id(); - // First, we iterate to check if we find *any* APICs. If we do, then we must use them over x2APIC entries for IDs under 255. + // First, we iterate to check if we find *any* APICs. + // If we do, then we must use them over x2APIC entries for IDs under 255. var has_lapics = false; while (iter.next()) |entry| { switch (entry.type) { @@ -264,13 +275,16 @@ pub fn init() linksection(r.init) void { .LocalApic => { const lapic: *const MadtLapic = @ptrCast(entry); if (lapic.flags & 1 == 0 or lapic.id == cur_cpu_id) continue; - apics.append(mm.zone.gpa, lapic.id) catch @panic("Could not append APIC ID to list"); + apics.append(mm.zone.gpa, lapic.id) catch + @panic("Could not append APIC ID to list"); }, .X2LocalApic => { const x2lapic: *const MadtX2Lapic = @ptrCast(entry); if (has_lapics and x2lapic.id <= 0xFF) continue; - if (x2lapic.flags & 1 == 0 or x2lapic.id == cur_cpu_id) continue; - apics.append(mm.zone.gpa, x2lapic.id) catch @panic("Could not append APIC ID to list"); + if (x2lapic.flags & 1 == 0 or x2lapic.id == cur_cpu_id) + continue; + apics.append(mm.zone.gpa, x2lapic.id) catch + @panic("Could not append APIC ID to list"); }, else => {}, } diff --git a/src/platform/pc/smp.zig b/src/platform/pc/smp.zig index 37ee63f..09b87ce 100644 --- a/src/platform/pc/smp.zig +++ b/src/platform/pc/smp.zig @@ -37,10 +37,15 @@ fn ap_entry(cpu_id: u32) callconv(.c) noreturn { ki.impl.init.ap_entry(cpu_id, &aps_booted); } -fn make_thread(entrypoint: *const fn (?*anyopaque) void, stack: usize) *ke.Thread { - const td = mm.zone.gpa.create(ke.Thread) catch @panic("Failed to allocate thread for AP"); +fn make_thread( + entrypoint: *const fn (?*anyopaque) void, + stack: usize, + size: usize, +) *ke.Thread { + const td = mm.zone.gpa.create(ke.Thread) catch + @panic("Failed to allocate thread for AP"); - td.init(stack, r.kib(32), ke.Thread.Priority.idle_low, entrypoint, null); + td.init(stack, size, ke.Thread.Priority.idle_low, entrypoint, null); td.pinned = true; return td; @@ -65,8 +70,12 @@ pub fn init() linksection(r.init) void { .Unknown => false, }; - const init_delay: r.Nanoseconds = .init(if (skip_delay) 0 else std.time.ns_per_ms * 10); - const sipi_delay: r.Nanoseconds = .init(if (skip_delay) std.time.ns_per_us * 10 else std.time.ns_per_us * 300); + const init_delay: r.Nanoseconds = .init( + if (skip_delay) 0 else std.time.ns_per_ms * 10, + ); + const sipi_delay: r.Nanoseconds = .init( + if (skip_delay) std.time.ns_per_us * 10 else std.time.ns_per_us * 300, + ); const page: [*]u8 = @ptrFromInt(mm.p2v(0x8000)); @memcpy(page[0..trampoline_size], @as([*]u8, @ptrFromInt(trampoline_start))); @@ -76,8 +85,11 @@ pub fn init() linksection(r.init) void { const idtr = amd64.sidtr(); + const offsets = mm.zone.gpa.alloc(usize, apic.apics.items.len + 1) catch + @panic("Failed to allocate AP local data offsets"); + // Allocate per-cpu offsets for CPU-local data. - ki.impl.cpu_offsets = @ptrCast(mm.zone.gpa.alloc(usize, apic.apics.items.len + 1) catch @panic("Failed to allocate AP local data offsets")); + ki.impl.cpu_offsets = @ptrCast(offsets); const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start); log.info("per-CPU data size: {} bytes", .{percpu_size}); @@ -98,19 +110,39 @@ pub fn init() linksection(r.init) void { cpu_id_to_apic_id[cpu_id] = apic_id; // Allocate per-cpu data. - const cpu_data = mm.zone.gpa.alloc(u8, percpu_size) catch @panic("Failed to allocate per-cpu data"); - @memcpy(cpu_data, @as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size]); + const cpu_data = mm.zone.gpa.alloc(u8, percpu_size) catch + @panic("Failed to allocate per-cpu data"); - const self_offset_offset = @intFromPtr(&ki.impl.cpu_self_offset) - @intFromPtr(&__percpu_start); - ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -% @intFromPtr(&__percpu_start); + @memcpy( + cpu_data, + @as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size], + ); + + const self_offset_offset = @intFromPtr(&ki.impl.cpu_self_offset) - + @intFromPtr(&__percpu_start); + + ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -% + @intFromPtr(&__percpu_start); + + const off: *usize = @ptrCast(@alignCast(&cpu_data[self_offset_offset])); // copy the offset into the AP's self_offset variable - @as(*usize, @ptrCast(@alignCast(&cpu_data[self_offset_offset]))).* = ki.impl.cpu_offsets[cpu_id]; + off.* = ki.impl.cpu_offsets[cpu_id]; - const stack_top = @intFromPtr(mm.heap.alloc(r.kib(32), .DontWaitForMemory) catch @panic("Failed to allocate AP stack")) + r.kib(32); + const stack_size = r.kib(16); + + const stack_top = @intFromPtr(mm.heap.alloc( + stack_size, + .DontWaitForMemory, + ) catch + @panic("Failed to allocate AP stack")) + stack_size; start_stack.remote(@intCast(cpu_id)).* = stack_top & ~@as(usize, 15); - start_thread.remote(@intCast(cpu_id)).* = make_thread(@ptrCast(&ap_entry), stack_top - r.kib(32)); + start_thread.remote(@intCast(cpu_id)).* = make_thread( + @ptrCast(&ap_entry), + stack_top - stack_size, + stack_size, + ); rtl.barrier.fence(.release); diff --git a/src/proc/thread.zig b/src/proc/thread.zig index 3dc9349..742d807 100644 --- a/src/proc/thread.zig +++ b/src/proc/thread.zig @@ -55,7 +55,11 @@ pub fn init() void { /// - `prio`: base priority of the thread. /// - `entry`: entry point. /// - `arg`: argument passed to `entry`. -pub fn create_kernel(prio: u8, entry: *const fn (arg: ?*anyopaque) void, arg: ?*anyopaque) !*Thread { +pub fn create_kernel( + prio: u8, + entry: *const fn (arg: ?*anyopaque) void, + arg: ?*anyopaque, +) !*Thread { var td = try thread_zone.create(); const stack = try mm.heap.alloc(kernel_thread_stack_size, .WaitForMemory); diff --git a/src/rcu/hashtable.zig b/src/rcu/hashtable.zig index 6683dff..19c5517 100644 --- a/src/rcu/hashtable.zig +++ b/src/rcu/hashtable.zig @@ -133,7 +133,7 @@ const rehashing_reseed: u8 = 1 << 2; const Core = struct { /// The current state of the hash table. state: std.atomic.Value(State), - /// One pointer for current buckets, another for new buckets (during resizing). + /// One pointer for current buckets, another for new buckets. buckets: [2]std.atomic.Value([*]Bucket), /// One seed for current table, another for the new table. seeds: [2]std.atomic.Value(u32), @@ -153,7 +153,10 @@ const Core = struct { policy: ResizePolicy, min_size: u32, ) !void { - const min: u32 = if (policy == .Compact) @max(2, min_size) else @max(16, min_size); + const min: u32 = if (policy == .Compact) + @max(2, min_size) + else + @max(16, min_size); const start_size = min; @@ -186,7 +189,13 @@ const Core = struct { self.rehash_work.init(.Normal, rehash, self); } - fn migrate_bucket(self: *Self, old: *Bucket, new_array: [*]Bucket, new_seed: u32, new_shift: u5) void { + fn migrate_bucket( + self: *Self, + old: *Bucket, + new_array: [*]Bucket, + new_seed: u32, + new_shift: u5, + ) void { // This is called in worker thread context, so we have ample stack size // to work with. const flat_size = 256; @@ -331,7 +340,10 @@ const Core = struct { /// Remove a given element from the table, SMR must be entered. fn remove(self: *Self, link: *rcu.SList.Entry) void { var state = self.state.load(.acquire); - var h = self.hash_link(link, self.seeds[state.current_index].load(.monotonic)); + var h = self.hash_link( + link, + self.seeds[state.current_index].load(.monotonic), + ); var b = self.bucket(state, .Cur, h); @@ -343,7 +355,10 @@ const Core = struct { // Get the new state instead. state = self.state.load(.acquire); - h = self.hash_link(link, self.seeds[state.new_index].load(.monotonic)); + h = self.hash_link( + link, + self.seeds[state.new_index].load(.monotonic), + ); b = self.bucket(state, .New, h); first = b.lock(); } @@ -425,7 +440,11 @@ const Core = struct { }; /// Scalable, lock-free hashtable backed by SMR. -pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context: type) type { +pub fn Table( + comptime T: type, + comptime link_field: []const u8, + comptime Context: type, +) type { comptime rtl.assert_interface(Context, struct { pub const Key = Context.Key; pub fn hash(key: Key, seed: usize) u32 { @@ -470,7 +489,10 @@ pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context /// Get an element given a key, SMR must be entered. pub fn get(self: *Self, key: Context.Key) ?*T { const state = self.core.state.load(.acquire); - const h = Context.hash(key, self.core.seeds[state.current_index].load(.monotonic)); + const h = Context.hash( + key, + self.core.seeds[state.current_index].load(.monotonic), + ); const b = self.core.bucket(state, .Cur, h); var word = b.load(); @@ -504,7 +526,10 @@ pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context value: *rcu.SList.Entry, ) ?*T { var state = self.core.state.load(.acquire); - var h = Context.hash(key, self.core.seeds[state.current_index].load(.monotonic)); + var h = Context.hash( + key, + self.core.seeds[state.current_index].load(.monotonic), + ); var b = self.core.bucket(state, .Cur, h); var first = b.lock(); @@ -515,7 +540,10 @@ pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context // Get the new state instead. state = self.core.state.load(.acquire); - h = Context.hash(key, self.core.seeds[state.new_index].load(.monotonic)); + h = Context.hash( + key, + self.core.seeds[state.new_index].load(.monotonic), + ); b = self.core.bucket(state, .New, h); first = b.lock(); } @@ -574,7 +602,10 @@ pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context // Now do the same as in get(), but with the new state. const state = self.core.state.load(.acquire); - const h = Context.hash(key, self.core.seeds[state.new_index].load(.monotonic)); + const h = Context.hash( + key, + self.core.seeds[state.new_index].load(.monotonic), + ); const b = self.core.bucket(state, .New, h); var word: usize = b.load();