From 497e08bd7b85f49a11dc64df325378ebf4f5f61d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Gr=C3=A9goire?= Date: Wed, 2 Sep 2026 22:20:25 -0400 Subject: [PATCH] all: standardize suffixes ki -> kep mi -> mmp --- src/kern/amd64/impl.zig | 4 ++-- src/kern/amd64/init.zig | 10 ++++----- src/kern/amd64/int.zig | 16 +++++++------- src/kern/cpu.zig | 12 +++++------ src/kern/dpc.zig | 30 +++++++++++++------------- src/kern/event.zig | 6 +++--- src/kern/init.zig | 18 ++++++++-------- src/kern/ipi.zig | 4 ++-- src/kern/ipl.zig | 18 ++++++++-------- src/kern/log.zig | 4 ++-- src/kern/log_ring.zig | 6 +++--- src/kern/mutex.zig | 18 ++++++++-------- src/kern/panic.zig | 12 +++++------ src/kern/queue.zig | 12 +++++------ src/kern/sched.zig | 28 ++++++++++++------------ src/kern/shootdown.zig | 14 ++++++------ src/kern/smr.zig | 28 ++++++++++++------------ src/kern/thread.zig | 28 ++++++++++++------------ src/kern/time.zig | 2 +- src/kern/timer.zig | 12 +++++------ src/kern/turnstile.zig | 6 +++--- src/kern/wait.zig | 12 +++++------ src/mm/amd64/impl.zig | 6 +++--- src/mm/balance.zig | 2 +- src/mm/heap.zig | 34 ++++++++++++++--------------- src/mm/init.zig | 16 +++++++------- src/mm/phys.zig | 4 ++-- src/mm/pmap.zig | 46 ++++++++++++++++++++-------------------- src/mm/tlb.zig | 4 ++-- src/mm/vmem.zig | 4 ++-- src/mm/zone.zig | 14 ++++++------ src/platform/pc/impl.zig | 2 +- src/platform/pc/smp.zig | 14 ++++++------ src/proc/private.zig | 6 +++--- src/proc/thread.zig | 2 +- src/tests/smr.zig | 11 +++++----- src/tests/smr_pi.zig | 24 ++++++++++----------- 37 files changed, 244 insertions(+), 245 deletions(-) diff --git a/src/kern/amd64/impl.zig b/src/kern/amd64/impl.zig index 159a333..94b267c 100644 --- a/src/kern/amd64/impl.zig +++ b/src/kern/amd64/impl.zig @@ -1,6 +1,6 @@ const r = @import("root"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; const pl = r.pl; const amd64 = r.arch; @@ -52,7 +52,7 @@ extern fn amd64_call_continuation( ) callconv(.c) noreturn; export fn ki_free_old_stack(stack_top: usize) callconv(.c) void { - ki.thread.stack_cache_free(stack_top); + kep.thread.stack_cache_free(stack_top); } /// Trampoline to match the zig ABI. diff --git a/src/kern/amd64/init.zig b/src/kern/amd64/init.zig index 7d6d9ca..f96a723 100644 --- a/src/kern/amd64/init.zig +++ b/src/kern/amd64/init.zig @@ -4,7 +4,7 @@ const r = @import("root"); const std = @import("std"); const impl = @import("impl.zig"); const ke = r.ke; -const ki = r.ke.private; +const kep = r.ke.private; const pl = r.pl; var gdt = extern struct { @@ -85,13 +85,13 @@ fn early_cpu_init() linksection(r.init) void { } pub fn ap_entry(cpu_id: u32, booted: *std.atomic.Value(usize)) noreturn { - amd64.write_msr(.GsBase, ki.impl.cpu_offsets[cpu_id]); + amd64.write_msr(.GsBase, kep.impl.cpu_offsets[cpu_id]); early_cpu_init(); - ki.cpu.init_cpu(cpu_id); + kep.cpu.init_cpu(cpu_id); pl.impl.init_ap(); - const sched = ki.sched.percpu.local(); + const sched = kep.sched.percpu.local(); sched.current_thread = pl.impl.smp.start_thread.local().*; sched.idle_thread = sched.current_thread; @@ -101,7 +101,7 @@ pub fn ap_entry(cpu_id: u32, booted: *std.atomic.Value(usize)) noreturn { _ = booted.fetchAdd(1, .monotonic); ke.ipl.lower(.Passive); - ki.sched.idle(null); + kep.sched.idle(null); } var initial_offsets: [1]usize = .{0}; diff --git a/src/kern/amd64/int.zig b/src/kern/amd64/int.zig index a23100a..7dba84d 100644 --- a/src/kern/amd64/int.zig +++ b/src/kern/amd64/int.zig @@ -2,9 +2,9 @@ const amd64 = @import("arch"); const std = @import("std"); const r = @import("root"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; const pl = r.pl; -const mi = r.mm.private; +const mmp = r.mm.private; extern const __interrupt_vectors: [256]usize; @@ -49,7 +49,7 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys const ipl = ke.ipl.set_hardware(.Device); if (frame.intno < 32) { - ki.panic.panic_lock.acquire_no_ipl(); + kep.panic.panic_lock.acquire_no_ipl(); std.log.err("Unhandled exception: 0x{x} ({s}), err=0x{x}, pc=0x{x}", .{ frame.intno, exception_msg[frame.intno], frame.errcode, frame.rip }); std.log.err("RAX=0x{x:0>16} RBX=0x{x:0>16} RCX=0x{x:0>16} RDX=0x{x:0>16}", .{ frame.rax, frame.rbx, frame.rcx, frame.rdx }); std.log.err("RSI=0x{x:0>16} RDI=0x{x:0>16} RBP=0x{x:0>16} RSP=0x{x:0>16}", .{ frame.rsi, frame.rdi, frame.rbp, frame.rsp }); @@ -78,16 +78,16 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys std.log.err("CR2=0x{x:0>16} CR3=0x{x:0>16} RFLAGS={s}", .{ cr2, cr3, buf[0..writer.end] }); - ki.panic.panic_with_frame("Unhandled exception", frame.rbp); + kep.panic.panic_with_frame("Unhandled exception", frame.rbp); } if (frame.intno == 32) { - ki.timer.clock(); + kep.timer.clock(); } // Platform IPI number. if (frame.intno == 240) { - ki.ipi.ipi_handler(); + kep.ipi.ipi_handler(); } if (frame.intno >= 32) { @@ -97,9 +97,9 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys _ = ke.ipl.set_hardware(ipl); if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and - ki.ipl.is_softint_pending(.Dispatch)) + kep.ipl.is_softint_pending(.Dispatch)) { - ki.dpc.dispatch(); + kep.dpc.dispatch(); } } diff --git a/src/kern/cpu.zig b/src/kern/cpu.zig index 40d8e05..28fdd12 100644 --- a/src/kern/cpu.zig +++ b/src/kern/cpu.zig @@ -3,7 +3,7 @@ const rtl = @import("rtl"); const r = @import("root"); const ke = r.ke; -const ki = r.ke.private; +const kep = r.ke.private; const id = CpuLocal(u32, 0); @@ -29,12 +29,12 @@ pub fn CpuLocal(comptime T: type, comptime init: T) type { /// Return a pointer to local CPU data. pub fn local() *T { - return ki.impl.percpu_ptr(&storage); + return kep.impl.percpu_ptr(&storage); } /// Return a pointer to remote CPU data. pub fn remote(cpu: u32) *T { - return ki.impl.percpu_ptr_other(&storage, cpu); + return kep.impl.percpu_ptr_other(&storage, cpu); } }; } @@ -48,11 +48,11 @@ pub fn ExportedCpuLocal( const S = struct { var storage: T linksection(".data.percpu") = init; - pub fn local() @TypeOf(ki.impl.percpu_ptr(&storage)) { - return ki.impl.percpu_ptr(&storage); + pub fn local() @TypeOf(kep.impl.percpu_ptr(&storage)) { + return kep.impl.percpu_ptr(&storage); } pub fn remote(cpu: u32) *T { - return ki.impl.percpu_ptr_other(&storage, cpu); + return kep.impl.percpu_ptr_other(&storage, cpu); } }; @export(&S.storage, .{ .name = name, .linkage = .strong }); diff --git a/src/kern/dpc.zig b/src/kern/dpc.zig index cf2cee4..928583a 100644 --- a/src/kern/dpc.zig +++ b/src/kern/dpc.zig @@ -3,7 +3,7 @@ const config = @import("config"); const rtl = @import("rtl"); const r = @import("root"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; /// Deferred Procedure Call (DPC) structure. /// Used for scheduling work to be done when IPL is lowered below `.Dispatch`. @@ -63,7 +63,7 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void { dpc_cpu.queue.insert_tail(&dpc.link); // Mark the DPC as pending on this CPU - ki.ipl.set_softint_pending(mycpu, .Dispatch); + kep.ipl.set_softint_pending(mycpu, .Dispatch); dpc_cpu.lock.release_no_ipl(); } @@ -72,17 +72,17 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void { } fn dispatch_queue(cpu: u32) void { - const ipl_cpu = ki.ipl.cpu_ipl.local(); + const ipl_cpu = kep.ipl.cpu_ipl.local(); const dpc_cpu = pcpu.local(); - const sched_cpu = ki.sched.percpu.local(); + const sched_cpu = kep.sched.percpu.local(); ipl_cpu.* = .Dispatch; // Mark as handled, this must be done before enabling interrupts or we will // race. - ki.ipl.clear_softint_pending(cpu, .Dispatch); + kep.ipl.clear_softint_pending(cpu, .Dispatch); - _ = ki.impl.enable_interrupts(); + _ = kep.impl.enable_interrupts(); while (true) { const ipl = dpc_cpu.lock.acquire_at(.High); @@ -133,35 +133,35 @@ fn dispatch_queue(cpu: u32) void { sched_cpu.start_timer = false; sched_cpu.preemption_reason = .None; - ki.shootdown.process_shootdowns(); + kep.shootdown.process_shootdowns(); if (sched_cpu.next_thread != null) { const curtd = sched_cpu.current_thread.?; if (!curtd.smr_sections.is_empty()) { - ki.smr.mark_thread_stalled(curtd); + kep.smr.mark_thread_stalled(curtd); } - ki.sched.handle_preemption(sched_cpu); + kep.sched.handle_preemption(sched_cpu); } - _ = ki.impl.disable_interrupts(); + _ = kep.impl.disable_interrupts(); } /// Dispatch the DPC queue on the current CPU. pub fn dispatch() void { // DPC processing is done at IPL Dispatch. // Don't call lower/raise here because they might call us again recursively. - const int_state = ki.impl.disable_interrupts(); - const old_ipl = ki.ipl.cpu_ipl.local().*; + const int_state = kep.impl.disable_interrupts(); + const old_ipl = kep.ipl.cpu_ipl.local().*; var mycpu = ke.cpu.current(); - while (ki.ipl.is_softint_pending(.Dispatch)) { + while (kep.ipl.is_softint_pending(.Dispatch)) { dispatch_queue(mycpu); // Our CPU might have changed. mycpu = ke.cpu.current(); } - ki.ipl.cpu_ipl.local().* = old_ipl; - ki.impl.restore_interrupts(int_state); + kep.ipl.cpu_ipl.local().* = old_ipl; + kep.impl.restore_interrupts(int_state); } diff --git a/src/kern/event.zig b/src/kern/event.zig index eeef3b8..d8494bf 100644 --- a/src/kern/event.zig +++ b/src/kern/event.zig @@ -1,9 +1,9 @@ const ke = @import("root").ke; -const ki = ke.private; +const kep = ke.private; /// Generic waitable Event, exported for use everywhere in the kernel. pub const Event = struct { - hdr: ki.wait.DispatchHeader, + hdr: kep.wait.DispatchHeader, const Self = @This(); @@ -26,7 +26,7 @@ pub const Event = struct { pub fn signal(self: *Self) void { const ipl = self.hdr.lock.acquire(); self.hdr.signaled = 1; - ki.wait.satisfy_wait(&self.hdr); + kep.wait.satisfy_wait(&self.hdr); self.hdr.lock.release(ipl); } diff --git a/src/kern/init.zig b/src/kern/init.zig index aed14c0..e6275fd 100644 --- a/src/kern/init.zig +++ b/src/kern/init.zig @@ -3,7 +3,7 @@ const r = @import("root"); const pl = r.pl; const ke = r.ke; const arch = r.arch; -const ki = ke.private; +const kep = ke.private; const ex = r.ex; var thread0: ke.Thread = undefined; @@ -29,16 +29,16 @@ export fn kmain(boot_info: *r.BootInfo) callconv(.c) void { fn init() linksection(r.init) void { ke.ncpus = 1; - ki.impl.early_init(); - ki.tunable.init(); + kep.impl.early_init(); + kep.tunable.init(); std.log.info("Welcome to the machine", .{}); pl.early_init(); - ki.log.init(); - ki.cpu.init_cpu(0); - ki.turnstile.init_turnstiles(); + kep.log.init(); + kep.cpu.init_cpu(0); + kep.turnstile.init_turnstiles(); - ki.sched.percpu.local().current_thread = &thread0; - ki.sched.percpu.local().idle_thread = &thread0; + kep.sched.percpu.local().current_thread = &thread0; + kep.sched.percpu.local().idle_thread = &thread0; std.log.info("Zag for {s} ({s}), cmdline is \"{?s}\"", .{ pl.name, @@ -47,5 +47,5 @@ fn init() linksection(r.init) void { }); ex.init(); - ki.sched.idle(null); + kep.sched.idle(null); } diff --git a/src/kern/ipi.zig b/src/kern/ipi.zig index 285b947..f9e3f5f 100644 --- a/src/kern/ipi.zig +++ b/src/kern/ipi.zig @@ -4,7 +4,7 @@ const r = @import("root"); const std = @import("std"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; const pl = r.pl; const State = enum(u8) { @@ -138,7 +138,7 @@ pub fn ipi_handler() void { _ = frozen_cpus.fetchAdd(1, .monotonic); while (true) { - ki.impl.halt(); + kep.impl.halt(); } } diff --git a/src/kern/ipl.zig b/src/kern/ipl.zig index ccd4863..72e1e10 100644 --- a/src/kern/ipl.zig +++ b/src/kern/ipl.zig @@ -1,7 +1,7 @@ const std = @import("std"); const r = @import("root"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; /// Interrupt priority level (IPL) pub const Ipl = enum(u8) { @@ -44,7 +44,7 @@ pub fn current() Ipl { pub fn raise(new: Ipl) Ipl { var old: ke.Ipl = undefined; - if (ki.impl.raise_software_ipl(new, &old)) { + if (kep.impl.raise_software_ipl(new, &old)) { @branchHint(.unlikely); std.debug.panic( "ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})", @@ -55,7 +55,7 @@ pub fn raise(new: Ipl) Ipl { if (@intFromEnum(new) > @intFromEnum(Ipl.get_max_software())) { // Moving between hardware levels is rare. @branchHint(.unlikely); - ki.impl.set_hardware_ipl(new); + kep.impl.set_hardware_ipl(new); } return old; @@ -66,7 +66,7 @@ pub fn raise(new: Ipl) Ipl { pub fn lower(new: Ipl) void { var old: ke.Ipl = undefined; - if (ki.impl.lower_software_ipl(new, &old)) { + if (kep.impl.lower_software_ipl(new, &old)) { @branchHint(.unlikely); std.debug.panic( "ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})", @@ -78,27 +78,27 @@ pub fn lower(new: Ipl) void { @intFromEnum(old) > @intFromEnum(Ipl.get_max_software())) { @branchHint(.unlikely); - ki.impl.set_hardware_ipl(.Passive); + kep.impl.set_hardware_ipl(.Passive); } if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and is_softint_pending(.Dispatch)) { // Dispatch DPCs if necessary. - ki.dpc.dispatch(); + kep.dpc.dispatch(); } } /// Set the hardware IPL to `new`. pub fn set_hardware(new: Ipl) Ipl { - const ints = ki.impl.disable_interrupts(); + const ints = kep.impl.disable_interrupts(); const cpu = cpu_ipl.local(); const old = cpu.*; cpu.* = new; - ki.impl.set_hardware_ipl(new); - ki.impl.restore_interrupts(ints); + kep.impl.set_hardware_ipl(new); + kep.impl.restore_interrupts(ints); return old; } diff --git a/src/kern/log.zig b/src/kern/log.zig index e719879..f8b3cb9 100644 --- a/src/kern/log.zig +++ b/src/kern/log.zig @@ -4,7 +4,7 @@ const r = @import("root"); const pl = r.pl; const ke = r.ke; -const ki = ke.private; +const kep = ke.private; var out_lock: ke.SpinLock = .init(); @@ -41,7 +41,7 @@ const DebugWriter = struct { // Messages are on average 2^5 = 32 bytes. const avg_msg_size_bits = 5; -pub var ringbuffer = ki.log_ring.RingBuffer( +pub var ringbuffer = kep.log_ring.RingBuffer( config.log_buffer_shift, avg_msg_size_bits, ).init(); diff --git a/src/kern/log_ring.zig b/src/kern/log_ring.zig index 8f7756d..c58f16e 100644 --- a/src/kern/log_ring.zig +++ b/src/kern/log_ring.zig @@ -71,7 +71,7 @@ const std = @import("std"); const r = @import("root"); const rtl = @import("rtl"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; const BlkPos = struct { begin: usize, @@ -645,7 +645,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { var reservation: Reservation = undefined; - reservation.irq_state = ki.impl.disable_interrupts(); + reservation.irq_state = kep.impl.disable_interrupts(); const id = try self.reserve_desc(); @@ -703,7 +703,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type { ); // Restore IRQ state after publishing the new entry. - ki.impl.restore_interrupts(res.irq_state); + kep.impl.restore_interrupts(res.irq_state); } /// Return the first readable sequence currently retained by the ring. diff --git a/src/kern/mutex.zig b/src/kern/mutex.zig index 08a0b8b..1584b26 100644 --- a/src/kern/mutex.zig +++ b/src/kern/mutex.zig @@ -1,6 +1,6 @@ const std = @import("std"); const ke = @import("root").ke; -const ki = ke.private; +const kep = ke.private; /// Number of spins before blocking. const optimistic_spins = 100; @@ -19,7 +19,7 @@ pub const Mutex = struct { pub fn acquire(m: *Mutex) void { const ipl = ke.ipl.raise(.Dispatch); - const curtd = ki.sched.percpu.local().current_thread.?; + const curtd = kep.sched.percpu.local().current_thread.?; // Very fast path: the lock is uncontended and we can acquire it //immediately. @@ -57,12 +57,12 @@ pub const Mutex = struct { // Slow path: contended, block on a turnstile. while (true) { - const ts = ki.turnstile.lookup(m); + const ts = kep.turnstile.lookup(m); const owner = m.owner.load(.monotonic); if (owner == null) { // Lock was released between lookup and here. - ki.turnstile.exit(m); + kep.turnstile.exit(m); if (m.owner.cmpxchgStrong( null, curtd, @@ -75,7 +75,7 @@ pub const Mutex = struct { } // Block until woken. - ki.turnstile.block( + kep.turnstile.block( ts, m, .{ .single = owner.? }, @@ -99,12 +99,12 @@ pub const Mutex = struct { pub fn release(m: *Mutex) void { const ipl = ke.ipl.raise(.Dispatch); - const ts = ki.turnstile.lookup(m); + const ts = kep.turnstile.lookup(m); m.owner.store(null, .release); if (ts == null) { - ki.turnstile.exit(m); + kep.turnstile.exit(m); ke.ipl.lower(ipl); return; } @@ -114,13 +114,13 @@ pub const Mutex = struct { // to be better because this avoids lock convoys, // see this mysterious 70s paper: // https://dl.acm.org/doi/pdf/10.1145/850657.850659 - ki.turnstile.wakeup( + kep.turnstile.wakeup( ts.?, .Exclusive, ts.?.waiters, null, ); - ki.turnstile.exit(m); + kep.turnstile.exit(m); ke.ipl.lower(ipl); } diff --git a/src/kern/panic.zig b/src/kern/panic.zig index 2c3c6db..6cdc393 100644 --- a/src/kern/panic.zig +++ b/src/kern/panic.zig @@ -3,7 +3,7 @@ const config = @import("config"); const r = @import("root"); const ksyms = @import("ksyms"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; extern var text_start_addr: u8; extern var text_end_addr: u8; @@ -77,7 +77,7 @@ pub fn panic_with_frame( msg: []const u8, frame: usize, ) noreturn { - _ = ki.impl.disable_interrupts(); + _ = kep.impl.disable_interrupts(); std.log.info("Crash count is {} on cpu {}", .{ crash_count.load(.monotonic), ke.cpu.current() }); @@ -86,21 +86,21 @@ pub fn panic_with_frame( // Another CPU has already crashed, just freeze manually right now. while (true) { - ki.impl.halt(); + kep.impl.halt(); } } // We're the first one who actually crashed, that means we get to handle // things like dumping the system info and freezing other CPUs. Start by // doing the freeze. - ki.ipi.freeze_cpus(); + kep.ipi.freeze_cpus(); _ = ke.ipl.raise(.High); std.log.err("KERNEL PANIC: {s} on CPU {}, curthread is {*}", .{ msg, ke.cpu.current(), - ki.sched.percpu.local().current_thread.?, + kep.sched.percpu.local().current_thread.?, }); std.log.err("Stack trace:", .{}); @@ -108,7 +108,7 @@ pub fn panic_with_frame( walk_stack_frame(frame); while (true) { - ki.impl.halt(); + kep.impl.halt(); } } diff --git a/src/kern/queue.zig b/src/kern/queue.zig index a185dcc..f9e2a7a 100644 --- a/src/kern/queue.zig +++ b/src/kern/queue.zig @@ -8,11 +8,11 @@ const r = @import("root"); const rtl = @import("rtl"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; const std = @import("std"); pub const Queue = struct { - hdr: ki.wait.DispatchHeader, + hdr: kep.wait.DispatchHeader, items: rtl.List, active: usize, max_active: usize, @@ -49,14 +49,14 @@ pub const Queue = struct { }, } - ki.wait.satisfy_wait(&self.hdr); + kep.wait.satisfy_wait(&self.hdr); } /// Remove the item at the head. /// This blocks until an item is actually popped. pub fn remove(self: *Queue, timeout: ?r.Nanoseconds) !*rtl.List.Entry { const ipl = self.hdr.lock.acquire(); - const td = ki.sched.percpu.local().current_thread.?; + const td = kep.sched.percpu.local().current_thread.?; if (td.queue) |q| { std.debug.assert(q == self); @@ -64,7 +64,7 @@ pub const Queue = struct { self.active -= 1; if (self.hdr.signaled > 0 and self.active < self.max_active) { - ki.wait.satisfy_wait(&self.hdr); + kep.wait.satisfy_wait(&self.hdr); } } else { td.queue = self; @@ -95,7 +95,7 @@ pub fn signal_wait(queue: *Queue) void { queue.active -= 1; if (queue.hdr.signaled > 0 and queue.active < queue.max_active) { - ki.wait.satisfy_wait(&queue.hdr); + kep.wait.satisfy_wait(&queue.hdr); } queue.hdr.lock.release(ipl); diff --git a/src/kern/sched.zig b/src/kern/sched.zig index 53ba5df..b851778 100644 --- a/src/kern/sched.zig +++ b/src/kern/sched.zig @@ -98,7 +98,7 @@ const rtl = @import("rtl"); const config = @import("config"); const r = @import("root"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; const pl = r.pl; const runqueues_n = 64; @@ -268,7 +268,7 @@ fn init_cpu() linksection(r.init) void { .est_load_avg = .init(0), .migratable = .init(0), .current_thread_prio = .init(0), - .resched_dpc = .init(ki.sched.clock), + .resched_dpc = .init(kep.sched.clock), .resched_timer = undefined, .start_timer = false, .preemption_reason = .None, @@ -1034,7 +1034,7 @@ fn enqueue_on_cpu(c: u32, td: *ke.Thread) void { cpu.preemption_reason = .HigherPriority; td.cpu = c; td.state.store(.Selected, .monotonic); - ki.ipl.set_softint_pending(c, .Dispatch); + kep.ipl.set_softint_pending(c, .Dispatch); cpu.current_thread_prio.store(td.priority, .monotonic); @@ -1056,19 +1056,19 @@ fn enqueue_on_cpu(c: u32, td: *ke.Thread) void { if (cpu.resched_timer.state.load(.monotonic) == .Stopped) { cpu.start_timer = true; - ki.ipl.set_softint_pending(c, .Dispatch); + kep.ipl.set_softint_pending(c, .Dispatch); } // queues_lock dropped } fn allocate_stack(td: *ke.Thread) ?usize { - if (ki.thread.stack_cache_pop()) |stack| { + if (kep.thread.stack_cache_pop()) |stack| { return stack; } // Enqueue the thread for allocation - ki.thread.thread_stack_queue.insert(@ptrCast(&td.runq_link.next)); + kep.thread.thread_stack_queue.insert(@ptrCast(&td.runq_link.next)); return null; } @@ -1122,13 +1122,13 @@ fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void { cur.switching.store(false, .release); cur.lock.release_no_ipl(); - ki.impl.call_continuation(&next.context, continuation); + kep.impl.call_continuation(&next.context, continuation); } if (relinquish) { - ki.impl.switch_cont_to_normal(&cur.context, &next.context); + kep.impl.switch_cont_to_normal(&cur.context, &next.context); } else { - ki.impl.switch_normal_to_normal(&cur.context, &next.context); + kep.impl.switch_normal_to_normal(&cur.context, &next.context); } } @@ -1206,7 +1206,7 @@ pub fn handle_preemption(cpu: *PerCpu) void { /// Called every time slice in DPC context. pub fn clock(_: *ke.Dpc, _: ?*anyopaque) void { const cpu = percpu.local(); - std.debug.assert(ki.ipl.current() == .Dispatch); + std.debug.assert(kep.ipl.current() == .Dispatch); const curtd = cpu.current_thread orelse return; @@ -1369,7 +1369,7 @@ pub fn yield_locked(continuation: ?ke.Continuation) void { cur.switching.store(false, .monotonic); cur.lock.release_no_ipl(); - if (cont) |c| ki.impl.call_continuation(&cur.context, c); + if (cont) |c| kep.impl.call_continuation(&cur.context, c); } // cur lock dropped @@ -1508,7 +1508,7 @@ pub fn update_priority_locked(td: *ke.Thread, new_prio: u8) void { cpu.preemption_reason = .HigherPriority; cpu.current_thread_prio.store(next.priority, .monotonic); - ki.ipl.set_softint_pending(c, .Dispatch); + kep.ipl.set_softint_pending(c, .Dispatch); if (c != ke.cpu.current()) { pl.send_ipi(c); @@ -1770,8 +1770,8 @@ pub fn idle(_: ?*anyopaque) noreturn { } // Dispatch any pending DPCs. - if (ki.ipl.is_softint_pending(.Dispatch)) { - ki.dpc.dispatch(); + if (kep.ipl.is_softint_pending(.Dispatch)) { + kep.dpc.dispatch(); } std.atomic.spinLoopHint(); diff --git a/src/kern/shootdown.zig b/src/kern/shootdown.zig index ac7e7b4..758d90c 100644 --- a/src/kern/shootdown.zig +++ b/src/kern/shootdown.zig @@ -5,7 +5,7 @@ const config = @import("config"); const rtl = @import("rtl"); const r = @import("root"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; const mm = r.mm; const pl = r.pl; @@ -87,14 +87,14 @@ fn allocate_slot(cpu: *PerCpu) ?usize { } fn flush_range(va: r.VAddr, npages: usize) void { - if (npages > ki.impl.tlb_max_pages) { + if (npages > kep.impl.tlb_max_pages) { // Just flush the entire thing. - ki.impl.flush_full_tlb(); + kep.impl.flush_full_tlb(); return; } for (0..npages) |i| { - ki.impl.flush_tlb(va + i * mm.page_size); + kep.impl.flush_tlb(va + i * mm.page_size); } } @@ -148,12 +148,12 @@ pub fn process_shootdowns() void { if (nreq == 0) return; // npages is just a hint, we don't need it to be accurate. - const full = cpu.npages.load(.monotonic) >= ki.impl.tlb_max_pages; + const full = cpu.npages.load(.monotonic) >= kep.impl.tlb_max_pages; cpu.npages.store(0, .monotonic); if (full) { // Just flush the entire thing. - ki.impl.flush_full_tlb(); + kep.impl.flush_full_tlb(); } // Go through every sender and release their states. @@ -220,6 +220,6 @@ pub fn submit(state: ShootdownState, target_mask: ke.CpuMask) !void { remote.senders.set(curcpu, .release); rtl.barrier.fence(.release); - ki.ipl.set_softint_pending(@truncate(bit), .Dispatch); + kep.ipl.set_softint_pending(@truncate(bit), .Dispatch); } } diff --git a/src/kern/smr.zig b/src/kern/smr.zig index 0943beb..ce76949 100644 --- a/src/kern/smr.zig +++ b/src/kern/smr.zig @@ -7,7 +7,7 @@ const std = @import("std"); const config = @import("config"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; pub const Sequence = u64; @@ -70,7 +70,7 @@ pub const Tracker = struct { dom: *Domain, seq: Sequence, cpu: ?*Cpu, - owner: ki.turnstile.Owner, + owner: kep.turnstile.Owner, }; /// Used by tests. @@ -152,18 +152,18 @@ fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence cpu.stall_goal = goal; cpu.stall_lock.release_no_ipl(); - const ts = ki.turnstile.lookup(cpu); + const ts = kep.turnstile.lookup(cpu); seq = cpu.stall_seq.load(.monotonic); // Re-check under the chain lock. if (seq == seq_invalid or goal <= seq) { - ki.turnstile.exit(cpu); + kep.turnstile.exit(cpu); ke.ipl.lower(ipl); break; } - ki.turnstile.block( + kep.turnstile.block( ts, cpu, .{ .shared = &cpu.stalled }, @@ -354,7 +354,7 @@ pub fn mark_thread_stalled(td: *ke.Thread) void { const entry = it.get(); it.advance(); - const owner: *ki.turnstile.Owner = @fieldParentPtr("link", entry); + const owner: *kep.turnstile.Owner = @fieldParentPtr("link", entry); const tracker: *Tracker = @fieldParentPtr("owner", owner); if (tracker.cpu == null) { @@ -370,16 +370,16 @@ pub fn mark_thread_stalled(td: *ke.Thread) void { cpu.current_seq.store(seq_invalid, .release); - const turnstile = ki.turnstile.lookup(cpu); + const turnstile = kep.turnstile.lookup(cpu); entry.remove(); cpu.stalled.insert_tail(&tracker.owner.link); if (turnstile) |ts| { - ki.turnstile.owner_enter(ts, &tracker.owner); + kep.turnstile.owner_enter(ts, &tracker.owner); } - ki.turnstile.exit(cpu); + kep.turnstile.exit(cpu); cpu.stall_lock.release_no_ipl(); } @@ -424,12 +424,12 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void { // We got preempted. cpu.stall_lock.acquire_no_ipl(); - const turnstile = ki.turnstile.lookup(cpu); + const turnstile = kep.turnstile.lookup(cpu); // Remove ourselves from the stalled list. tracker.owner.link.remove(); - ki.turnstile.owner_leave(&tracker.owner); + kep.turnstile.owner_leave(&tracker.owner); var wake = false; @@ -437,7 +437,7 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void { cpu.stall_seq.store(seq_invalid, .release); wake = true; } else { - const first_owner: *ki.turnstile.Owner = @fieldParentPtr( + const first_owner: *kep.turnstile.Owner = @fieldParentPtr( "link", cpu.stalled.first(), ); @@ -449,10 +449,10 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void { if (wake) { if (turnstile) |ts| - ki.turnstile.wakeup(ts, .Exclusive, ts.waiters, null); + kep.turnstile.wakeup(ts, .Exclusive, ts.waiters, null); } - ki.turnstile.exit(cpu); + kep.turnstile.exit(cpu); cpu.stall_lock.release_no_ipl(); ke.ipl.lower(ipl); diff --git a/src/kern/thread.zig b/src/kern/thread.zig index ecbea24..8091712 100644 --- a/src/kern/thread.zig +++ b/src/kern/thread.zig @@ -3,7 +3,7 @@ const r = @import("root"); const rtl = @import("rtl"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; pub const Priority = enum(u8) { const Self = @This(); @@ -79,7 +79,7 @@ pub const Thread = struct { pub const Context = extern struct { /// Implementation-dependent context. - impl: ki.impl.ThreadContext, + impl: kep.impl.ThreadContext, stack_top: usize, @@ -127,20 +127,20 @@ pub const Thread = struct { /// CPU this thread is enqueued on. cpu: ?u32, /// Run queue this thread is in - runq: ?*ki.sched.RunQueue, + runq: ?*kep.sched.RunQueue, /// Index into the run queue this thread is currently in. /// Only valid when `runq` is non-null. runq_idx: u8, /// Current wait status. - wait_status: std.atomic.Value(ki.wait.Status), - waitblocks: [4]ki.wait.WaitBlock, + wait_status: std.atomic.Value(kep.wait.Status), + waitblocks: [4]kep.wait.WaitBlock, /// Reason for the wait, if any. wait_reason: ?[]const u8, /// Timer used for timeouts. timer: ke.Timer, /// Turnstile. - turnstile: *ki.turnstile.Turnstile, - turnstile_waiter: ?*ki.turnstile.Waiter, + turnstile: *kep.turnstile.Turnstile, + turnstile_waiter: ?*kep.turnstile.Waiter, turnstiles_owned: rtl.List, /// Object this thread is currently blocked on, or null. waiting_on: ?*anyopaque, @@ -148,9 +148,9 @@ pub const Thread = struct { queue: ?*ke.Queue, queue_item: ?*rtl.List.Entry, /// PELT load average, - avg: ki.sched.Average, + avg: kep.sched.Average, /// Accounting statistics. - acct: ki.sched.Accounting, + acct: kep.sched.Accounting, /// Set whenever the thread is switching off its stack. /// This is used to avoid taking thread next lock to wait for switch off /// to complete. @@ -165,7 +165,7 @@ pub const Thread = struct { /// Top of the stack for the thread. stack: r.VAddr, /// Turnstile associated with the thread. - turnstile: *ki.turnstile.Turnstile, + turnstile: *kep.turnstile.Turnstile, /// Thread's priority. priority: Priority, }; @@ -272,13 +272,13 @@ pub fn exit() void { // Reuse the runq linkage to put on reaper list. reaper_list.insert(@ptrCast(&curtd.runq_link.next)); - ki.sched.detach_load_avg(ki.sched.percpu.local(), curtd); - ki.sched.yield_locked(Continuation.dummy); + kep.sched.detach_load_avg(kep.sched.percpu.local(), curtd); + kep.sched.yield_locked(Continuation.dummy); } /// Return the currently running thread. pub fn current() *ke.Thread { - return ki.sched.percpu.local().current_thread.?; + return kep.sched.percpu.local().current_thread.?; } // -- Stack allocation and continuations --------------------------------------- @@ -350,7 +350,7 @@ pub fn call_continuation(ptr: ?*anyopaque) noreturn { const cont = td.continuation.?; td.continuation = null; - ki.impl.call_continuation(&td.context, cont); + kep.impl.call_continuation(&td.context, cont); } /// Try to get a stack from the stack cache. diff --git a/src/kern/time.zig b/src/kern/time.zig index 8dd8c52..c4f3f29 100644 --- a/src/kern/time.zig +++ b/src/kern/time.zig @@ -2,7 +2,7 @@ const std = @import("std"); const rtl = @import("rtl"); const r = @import("root"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; /// TimeCounter structure. /// Represents a time-keeping source upon which the kernel can rely. diff --git a/src/kern/timer.zig b/src/kern/timer.zig index 0a1ed3b..128c106 100644 --- a/src/kern/timer.zig +++ b/src/kern/timer.zig @@ -5,12 +5,12 @@ const std = @import("std"); const rtl = @import("rtl"); const r = @import("root"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; const pl = r.pl; const PerCpu = struct { /// Heap of pending timers on this CPU. - timers: rtl.PairingHeap(.min, ki.timer.cmp_timer), + timers: rtl.PairingHeap(.min, kep.timer.cmp_timer), /// Lock over the timer heap. lock: ke.SpinLock, dpc: ke.Dpc, @@ -26,7 +26,7 @@ pub const Timer = struct { Stopped, }; - hdr: ki.wait.DispatchHeader, + hdr: kep.wait.DispatchHeader, /// Timer state. state: std.atomic.Value(State), @@ -150,12 +150,12 @@ pub fn clock() void { // Check for overflows. // This is fine to call a lot, as the function only gets expensive // (i.e seqlock store) when an overflow actually happens. - ki.time.update_overflow(); + kep.time.update_overflow(); } // Called in a DPC when a timer has expired. fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void { - std.debug.assert(ki.ipl.current() == .Dispatch); + std.debug.assert(kep.ipl.current() == .Dispatch); const cpu = percpu.local(); while (true) { @@ -217,7 +217,7 @@ fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void { timer.state.store(.Stopped, .release); // Wake whomever was waiting on the timer. - ki.wait.satisfy_wait(&timer.hdr); + kep.wait.satisfy_wait(&timer.hdr); timer.hdr.lock.release_no_ipl(); if (maybe_dpc) |dpc| ke.dpc.enqueue(dpc, null); diff --git a/src/kern/turnstile.zig b/src/kern/turnstile.zig index 814e2c7..48c9e0b 100644 --- a/src/kern/turnstile.zig +++ b/src/kern/turnstile.zig @@ -1,5 +1,5 @@ const ke = @import("root").ke; -const ki = ke.private; +const kep = ke.private; const rtl = @import("rtl"); const std = @import("std"); @@ -170,7 +170,7 @@ fn recompute_inherited(td: *ke.Thread) void { fn update_prio(td: *ke.Thread) void { const pri = td.effective_priority(); - if (pri != td.priority) ki.sched.update_priority_locked(td, pri); + if (pri != td.priority) kep.sched.update_priority_locked(td, pri); } /// Donate priority `pri` to `to` along the edge `boost`. The caller holds @@ -392,7 +392,7 @@ pub fn block( queue: Queue, ) void { const chain = chain_for(obj); - const curtd = ki.sched.percpu.local().current_thread.?; + const curtd = kep.sched.percpu.local().current_thread.?; const ipl = ke.ipl.current(); var ts = turnstile; diff --git a/src/kern/wait.zig b/src/kern/wait.zig index dde6134..394574e 100644 --- a/src/kern/wait.zig +++ b/src/kern/wait.zig @@ -5,7 +5,7 @@ const rtl = @import("rtl"); const std = @import("std"); const r = @import("root"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; /// Header for waitable objects. /// This must be added to any structured which is considered waitable. @@ -122,7 +122,7 @@ pub fn wait_one(object: *DispatchHeader, reason: []const u8, opts: Options) !usi pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !usize { const ipl = ke.ipl.raise(.Dispatch); defer ke.ipl.lower(ipl); - const curtd = ki.sched.percpu.local().current_thread.?; + const curtd = kep.sched.percpu.local().current_thread.?; const obj_count = objects.len; const has_timeout = opts.timeout != null; const blocks = opts.waitblocks orelse blk: { @@ -222,13 +222,13 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) ! ) == null) { if (queue == null) { if (curtd.queue) |q| { - ki.queue.signal_wait(q); + kep.queue.signal_wait(q); } } curtd.wait_reason = reason; // We're good, now actually block. - ki.sched.block_locked(curtd, opts.continuation); + kep.sched.block_locked(curtd, opts.continuation); } else { queue = null; @@ -323,11 +323,11 @@ pub fn satisfy_wait(obj: *DispatchHeader) void { if (obj.type != .Queue) { if (td.queue) |q| { - ki.queue.signal_wake(q); + kep.queue.signal_wake(q); } } - ki.sched.unblock_locked(td); + kep.sched.unblock_locked(td); td.lock.release(ipl); } diff --git a/src/mm/amd64/impl.zig b/src/mm/amd64/impl.zig index e6edfb7..bc4b611 100644 --- a/src/mm/amd64/impl.zig +++ b/src/mm/amd64/impl.zig @@ -1,7 +1,7 @@ const amd64 = @import("arch"); const r = @import("root"); const mm = r.mm; -const mi = mm.private; +const mmp = mm.private; pub const hhdm_minimum_max_address = r.gib(4); @@ -17,7 +17,7 @@ pub const hhdm_base = 0xffff800000000000; pub const kernel_heap_base = 0xffffc00000000000; pub const pfndb_base = 0xffffd00000000000; -pub const levels = [_]mi.PMapLevel{ +pub const levels = [_]mmp.PMapLevel{ .{ .shift = 12, .mask = 0x1ff, .leaf = true }, // 4K .{ .shift = 21, .mask = 0x1ff, .leaf = true }, // 2M .{ .shift = 30, .mask = 0x1ff, .leaf = true }, // 1G @@ -117,5 +117,5 @@ pub fn is_leaf_level_enabled(level: usize) bool { } pub fn init_kernel() void { - mi.kernel_space.pmap.root_pa = amd64.read_cr(3); + mmp.kernel_space.pmap.root_pa = amd64.read_cr(3); } diff --git a/src/mm/balance.zig b/src/mm/balance.zig index 30ee25e..4e89b73 100644 --- a/src/mm/balance.zig +++ b/src/mm/balance.zig @@ -10,7 +10,7 @@ const std = @import("std"); const ps = r.ps; const ke = r.ke; const mm = r.mm; -const mi = mm.private; +const mmp = mm.private; const balance_interval = ke.Tunable(u32, 1000, "mm.balance.interval_ms"); diff --git a/src/mm/heap.zig b/src/mm/heap.zig index 4e3d4f5..fdf2c14 100644 --- a/src/mm/heap.zig +++ b/src/mm/heap.zig @@ -1,54 +1,54 @@ const r = @import("root"); const ke = r.ke; const mm = r.mm; -const mi = mm.private; +const mmp = mm.private; pub fn init() void { - mi.kernel_space.arena.init( + mmp.kernel_space.arena.init( "kernel heap", - mi.impl.kernel_heap_base, + mmp.impl.kernel_heap_base, r.tib(16), mm.page_size, ) catch @panic("failed to initialize kernel heap arena"); } pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque { - mi.kernel_space.lock.acquire(); + mmp.kernel_space.lock.acquire(); - const addr = mi.kernel_space.arena.alloc(size, .{}) catch { - mi.kernel_space.lock.release(); + const addr = mmp.kernel_space.arena.alloc(size, .{}) catch { + mmp.kernel_space.lock.release(); return mm.Error.OutOfMemory; }; const npages = size / mm.page_size; for (0..npages) |i| { - const pte = mi.pmap.wire_pte( - &mi.kernel_space, + const pte = mmp.pmap.wire_pte( + &mmp.kernel_space, addr + (i * mm.page_size), policy, ) catch { - mi.kernel_space.lock.release(); + mmp.kernel_space.lock.release(); return mm.Error.OutOfMemory; }; - mi.kernel_space.lock.release(); + mmp.kernel_space.lock.release(); - const page = mi.phys.alloc_opts(.{ .policy = policy }) orelse { + const page = mmp.phys.alloc_opts(.{ .policy = policy }) orelse { return mm.Error.OutOfMemory; }; - mi.kernel_space.lock.acquire(); + mmp.kernel_space.lock.acquire(); - pte.* = mi.impl.make_leaf_pte(page, .{ .read = true, .write = true }, 0); + pte.* = mmp.impl.make_leaf_pte(page, .{ .read = true, .write = true }, 0); } - mi.kernel_space.lock.release(); + mmp.kernel_space.lock.release(); return @ptrFromInt(addr); } pub fn free(va: r.VAddr, size: usize) void { - mi.kernel_space.lock.acquire(); - mi.tlb.reclaim_range(&mi.kernel_space, va, size); - mi.kernel_space.lock.release(); + mmp.kernel_space.lock.acquire(); + mmp.tlb.reclaim_range(&mmp.kernel_space, va, size); + mmp.kernel_space.lock.release(); } diff --git a/src/mm/init.zig b/src/mm/init.zig index b9a6695..3796891 100644 --- a/src/mm/init.zig +++ b/src/mm/init.zig @@ -1,18 +1,18 @@ const r = @import("root"); const pl = r.pl; const mm = r.mm; -const mi = mm.private; +const mmp = mm.private; const config = @import("config"); pub fn init() linksection(r.init) void { - mi.phys.init(); - mi.zone.early_init(); - mi.vmem.init(); - mi.heap.init(); + mmp.phys.init(); + mmp.zone.early_init(); + mmp.vmem.init(); + mmp.heap.init(); } pub fn late_init() linksection(r.init) void { - mi.zone.late_init(); - mi.tlb.init(); - mi.balance.init(); + mmp.zone.late_init(); + mmp.tlb.init(); + mmp.balance.init(); } diff --git a/src/mm/phys.zig b/src/mm/phys.zig index 7742fbe..f3d28b0 100644 --- a/src/mm/phys.zig +++ b/src/mm/phys.zig @@ -3,7 +3,7 @@ const std = @import("std"); const rtl = @import("rtl"); const pl = r.pl; const mm = r.mm; -const mi = mm.private; +const mmp = mm.private; const ke = r.ke; const log = std.log.scoped(.@"mm/phys"); @@ -173,7 +173,7 @@ pub fn init() linksection(r.init) void { // Get the kernel pagemap from the loader, // we are already in it so no need to activate it. - mi.impl.init_kernel(); + mmp.impl.init_kernel(); // Calculate the minimum amount of free memory we accept, // this formula is from Linux and does not grow linearly; diff --git a/src/mm/pmap.zig b/src/mm/pmap.zig index db46ffa..41936b6 100644 --- a/src/mm/pmap.zig +++ b/src/mm/pmap.zig @@ -4,7 +4,7 @@ const r = @import("root"); const rtl = @import("rtl"); const std = @import("std"); const mm = r.mm; -const mi = mm.private; +const mmp = mm.private; /// Describes a contiguous region of physical memory to be mapped /// at a given virtual address. @@ -33,7 +33,7 @@ const ContiguousSource = struct { }; fn level_shift(level: usize) u6 { - return mi.impl.levels[level].shift; + return mmp.impl.levels[level].shift; } fn page_size(level: usize) usize { @@ -41,24 +41,24 @@ fn page_size(level: usize) usize { } fn index_for_level(va: usize, level: usize) usize { - return (va >> level_shift(level)) & mi.impl.levels[level].mask; + return (va >> level_shift(level)) & mmp.impl.levels[level].mask; } pub const PMap = struct { const Self = @This(); - const num_levels = mi.impl.levels.len; + const num_levels = mmp.impl.levels.len; const entries_per_table = 512; const index_mask = entries_per_table - 1; root_pa: r.PAddr, fn is_leaf_level(level: usize) bool { - return mi.impl.levels[level].leaf; + return mmp.impl.levels[level].leaf; } fn leaf_level_enabled(level: usize) bool { - return is_leaf_level(level) and mi.impl.is_leaf_level_enabled(level); + return is_leaf_level(level) and mmp.impl.is_leaf_level_enabled(level); } fn cursor(self: *Self, va: r.VAddr) Cursor { @@ -74,7 +74,7 @@ pub const PMap = struct { } pub fn activate(self: *Self) void { - mi.impl.activate(self.root_pa); + mmp.impl.activate(self.root_pa); } pub fn map_from(self: *Self, va: r.VAddr, size: usize, source: anytype) void { @@ -129,7 +129,7 @@ pub const PMap = struct { /// Unmap a contiguous range of virtual pages. /// Only small pages are supported. /// Return a pfn that points to the physical pages that were unmapped. - pub fn unmap(self: *Self, va: r.VAddr, size: usize) ?mi.PfnList { + pub fn unmap(self: *Self, va: r.VAddr, size: usize) ?mmp.PfnList { std.debug.assert(std.mem.isAligned(va, mm.page_size)); var c = self.cursor(va); @@ -153,7 +153,7 @@ pub const PMap = struct { const page = mm.pfn_to_struct_page(pfn); list.insert_tail(&page.free.link); - c.tables[level][idx] = std.mem.zeroes(mi.impl.Pte); + c.tables[level][idx] = std.mem.zeroes(mmp.impl.Pte); break; } @@ -182,7 +182,7 @@ pub const PMap = struct { va: r.VAddr, /// Cached pointers to each level's page table, filled as the cursor /// descends. Only indices in `[0, top_level]` are valid. - tables: [num_levels][*]mi.impl.Pte, + tables: [num_levels][*]mmp.impl.Pte, /// Index of the highest level for which `tables` currently holds a /// valid pointer. Descends toward 0 (the leaf) as `walk_down` runs. /// Reset upward in `advance` when the VA crosses a level boundary. @@ -197,24 +197,24 @@ pub const PMap = struct { while (current_level > target_level) { const idx = index_for_level(self.va, current_level); - var pte = mi.impl.Pte.load(&self.tables[current_level][idx]); + var pte = mmp.impl.Pte.load(&self.tables[current_level][idx]); if (!pte.present) { if (!allocate) { return error.PageNotMapped; } - const new_table_pa = mi.phys.alloc(); + const new_table_pa = mmp.phys.alloc(); - const table_ptr: [*]mi.impl.Pte = @ptrFromInt( + const table_ptr: [*]mmp.impl.Pte = @ptrFromInt( mm.p2v(new_table_pa), ); @memset( table_ptr[0..entries_per_table], - std.mem.zeroes(mi.impl.Pte), + std.mem.zeroes(mmp.impl.Pte), ); - pte = mi.impl.make_table_pte(new_table_pa); + pte = mmp.impl.make_table_pte(new_table_pa); self.tables[current_level][idx] = pte; } @@ -247,7 +247,7 @@ pub const PMap = struct { const table = self.tables[target_level]; table[index_for_level(self.va, target_level)] = - mi.impl.make_leaf_pte(current_pa, flags, target_level); + mmp.impl.make_leaf_pte(current_pa, flags, target_level); const step = page_size(target_level); current_pa += step; @@ -279,14 +279,14 @@ pub const PMap = struct { }; pub fn wire_pte( - space: *mi.Space, + space: *mmp.Space, va: r.VAddr, policy: mm.WaitPolicy, -) mm.Error!*mi.impl.Pte { +) mm.Error!*mmp.impl.Pte { std.debug.assert(space.lock.is_locked()); - var table: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(mi.kernel_space.pmap.root_pa)); - var level = mi.impl.levels.len - 1; + var table: [*]mmp.impl.Pte = @ptrFromInt(mm.p2v(mmp.kernel_space.pmap.root_pa)); + var level = mmp.impl.levels.len - 1; while (true) { const idx = index_for_level(va, level); @@ -302,7 +302,7 @@ pub fn wire_pte( // since that may block. space.lock.release(); - const new_table_pa = mi.phys.alloc_opts(.{ .policy = policy }) orelse { + const new_table_pa = mmp.phys.alloc_opts(.{ .policy = policy }) orelse { space.lock.acquire(); return mm.Error.OutOfMemory; }; @@ -316,12 +316,12 @@ pub fn wire_pte( if (entry.present) { // Another thread beat us to it, // free the page we just allocated and continue. - mi.phys.free(new_table_pa); + mmp.phys.free(new_table_pa); } else { const new_table_ptr: [*]u64 = @ptrFromInt(mm.p2v(new_table_pa)); @memset(new_table_ptr[0..512], 0); - table[idx] = mi.impl.make_table_pte(new_table_pa); + table[idx] = mmp.impl.make_table_pte(new_table_pa); entry = table[idx]; } } diff --git a/src/mm/tlb.zig b/src/mm/tlb.zig index 9a9102e..21ea2c3 100644 --- a/src/mm/tlb.zig +++ b/src/mm/tlb.zig @@ -6,7 +6,7 @@ const rtl = @import("rtl"); const ke = r.ke; const mm = r.mm; -const mi = mm.private; +const mmp = mm.private; const ps = r.ps; const ex = r.ex; @@ -23,7 +23,7 @@ fn reclaim_state(obj: *anyopaque, _: ?*anyopaque) void { const state: *ke.ShootdownState = @fieldParentPtr("link", link); const space: *mm.Space = @ptrFromInt(state.payload[0]); - const pfn_list: mi.PfnList = @bitCast(state.payload[1]); + const pfn_list: mmp.PfnList = @bitCast(state.payload[1]); const base = state.base; const npages = state.npages; diff --git a/src/mm/vmem.zig b/src/mm/vmem.zig index 4ccf230..c27a9b4 100644 --- a/src/mm/vmem.zig +++ b/src/mm/vmem.zig @@ -58,7 +58,7 @@ const std = @import("std"); const rtl = @import("rtl"); const mm = @import("root").mm; -const mi = mm.private; +const mmp = mm.private; pub const Policy = enum { /// Use the smallest free segment that can satisfy the request. @@ -127,7 +127,7 @@ const Segment = struct { /// address space. const freelist_count = @bitSizeOf(usize); -var seg_zone: mi.zone.TypedZone(Segment) = undefined; +var seg_zone: mmp.zone.TypedZone(Segment) = undefined; pub const Arena = struct { /// Name of the arena, for debugging purposes. diff --git a/src/mm/zone.zig b/src/mm/zone.zig index 966439a..dc7d51b 100644 --- a/src/mm/zone.zig +++ b/src/mm/zone.zig @@ -115,7 +115,7 @@ const config = @import("config"); const ke = r.ke; const mm = r.mm; const ex = r.ex; -const mi = mm.private; +const mmp = mm.private; /// Interval at which we do housekeeping (working set update, reaping, etc.) pub const update_interval_s = 15; @@ -907,7 +907,7 @@ pub const Zone = struct { @intFromPtr(obj), mm.page_size, ); - const phys = mi.kernel_space.pmap.query(page_va).?; + const phys = mmp.kernel_space.pmap.query(page_va).?; const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys)); break :blk page.alloced.slab_data.slab; } else blk: { @@ -1227,7 +1227,7 @@ pub const Zone = struct { } fn alloc_page(policy: mm.WaitPolicy) !*anyopaque { - const alloc_ret = mi.phys.alloc_opts(.{ .policy = policy }) orelse + const alloc_ret = mmp.phys.alloc_opts(.{ .policy = policy }) orelse return error.OutOfMemory; return @ptrFromInt(mm.p2v(alloc_ret)); } @@ -1235,7 +1235,7 @@ pub const Zone = struct { fn free_page(p: *anyopaque) void { const phys = mm.v2p(@intFromPtr(p)); - mi.phys.free(phys); + mmp.phys.free(phys); } fn slab_create_small(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab { @@ -1273,7 +1273,7 @@ pub const Zone = struct { fn slab_create_large(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab { const capacity = (self.slab_size) / self.chunk_size; - const buf = try mi.heap.alloc(self.slab_size, policy); + const buf = try mmp.heap.alloc(self.slab_size, policy); var ret: *Slab = @ptrCast( @alignCast( @@ -1296,7 +1296,7 @@ pub const Zone = struct { ret.alloc_rr = 0; for (0..self.slab_size / mm.page_size) |i| { - const phys_page = mi.kernel_space.pmap.query( + const phys_page = mmp.kernel_space.pmap.query( @intFromPtr(buf) + i * mm.page_size, ) orelse @panic("Could not query page"); const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys_page)); @@ -1650,7 +1650,7 @@ fn gpa_alloc( if (len > 2048) { if (ptr_align.toByteUnits() > mm.page_size) return null; const pages = std.mem.alignForward(usize, len, mm.page_size); - const ptr = mi.heap.alloc(pages, .WaitForMemory) catch return null; + const ptr = mmp.heap.alloc(pages, .WaitForMemory) catch return null; return @ptrCast(ptr); } diff --git a/src/platform/pc/impl.zig b/src/platform/pc/impl.zig index 256000b..9985fe6 100644 --- a/src/platform/pc/impl.zig +++ b/src/platform/pc/impl.zig @@ -7,7 +7,7 @@ const pvclock = @import("pvclock.zig"); pub const apic = @import("apic.zig"); pub const smp = @import("smp.zig"); -const ki = r.ke.private; +const kep = r.ke.private; pub const name: []const u8 = "PC"; diff --git a/src/platform/pc/smp.zig b/src/platform/pc/smp.zig index 93ff072..e9847db 100644 --- a/src/platform/pc/smp.zig +++ b/src/platform/pc/smp.zig @@ -5,7 +5,7 @@ const rtl = @import("rtl"); const config = @import("config"); const mm = r.mm; const ke = r.ke; -const ki = ke.private; +const kep = ke.private; const amd64 = @import("arch"); const log = std.log.scoped(.smp); @@ -34,7 +34,7 @@ const ApData = extern struct { }; fn ap_entry(cpu_id: u32) callconv(.c) noreturn { - ki.impl.init.ap_entry(cpu_id, &aps_booted); + kep.impl.init.ap_entry(cpu_id, &aps_booted); } fn make_thread( @@ -96,12 +96,12 @@ pub fn init() linksection(r.init) void { @panic("Failed to allocate AP local data offsets"); // Allocate per-cpu offsets for CPU-local data. - ki.impl.cpu_offsets = @ptrCast(offsets); + kep.impl.cpu_offsets = @ptrCast(offsets); const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start); log.info("per-CPU data size: {} bytes", .{percpu_size}); - ki.impl.cpu_offsets[0] = 0; + kep.impl.cpu_offsets[0] = 0; cpu_id_to_apic_id[0] = apic.get_id(); @@ -125,16 +125,16 @@ pub fn init() linksection(r.init) void { @as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size], ); - const self_offset_offset = @intFromPtr(&ki.impl.cpu_self_offset) - + const self_offset_offset = @intFromPtr(&kep.impl.cpu_self_offset) - @intFromPtr(&__percpu_start); - ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -% + kep.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -% @intFromPtr(&__percpu_start); const off: *usize = @ptrCast(@alignCast(&cpu_data[self_offset_offset])); // copy the offset into the AP's self_offset variable - off.* = ki.impl.cpu_offsets[cpu_id]; + off.* = kep.impl.cpu_offsets[cpu_id]; const stack_size = r.kib(16); diff --git a/src/proc/private.zig b/src/proc/private.zig index 40c42cd..22ec443 100644 --- a/src/proc/private.zig +++ b/src/proc/private.zig @@ -1,13 +1,13 @@ const r = @import("root"); const ke = r.ke; -const ki = ke.private; +const kep = ke.private; const mm = r.mm; pub const thread = @import("thread.zig"); -pub var turnstile_zone: mm.zone.TypedZone(ki.turnstile.Turnstile) = undefined; +pub var turnstile_zone: mm.zone.TypedZone(kep.turnstile.Turnstile) = undefined; -fn turnstile_ctor(ts: *ki.turnstile.Turnstile) void { +fn turnstile_ctor(ts: *kep.turnstile.Turnstile) void { ts.* = .{ .link = undefined, .next_free = null, diff --git a/src/proc/thread.zig b/src/proc/thread.zig index f24a586..151de83 100644 --- a/src/proc/thread.zig +++ b/src/proc/thread.zig @@ -5,7 +5,7 @@ const r = @import("root"); const ex = r.ex; const psp = r.ps.private; const ke = r.ke; -const ki = ke.private; +const kep = ke.private; const mm = r.mm; pub const kernel_thread_stack_size = r.kib(16); diff --git a/src/tests/smr.zig b/src/tests/smr.zig index c220b27..3b00eac 100644 --- a/src/tests/smr.zig +++ b/src/tests/smr.zig @@ -5,7 +5,7 @@ const ke = r.ke; const mm = r.mm; const ps = r.ps; -const ki = ke.private; +const kep = ke.private; const nslots = 128; const obj_words = 8; @@ -50,7 +50,7 @@ fn reader(param: ?*anyopaque) void { var iter: u64 = 0; while (true) : (iter += 1) { - const ipl = ki.smr.enter(dom); + const ipl = kep.smr.enter(dom); var done: u64 = 0; for (0..reads_per_section) |_| { @@ -68,7 +68,7 @@ fn reader(param: ?*anyopaque) void { done += 1; } - ki.smr.exit(dom, ipl); + kep.smr.exit(dom, ipl); _ = reads.fetchAdd(done, .monotonic); @@ -135,9 +135,8 @@ fn poller(_: ?*anyopaque) void { } fn spawn(entry: *const fn (?*anyopaque) void, id: usize) void { - const t = ps.thread.create_kernel(.Default, .{.func = entry, .arg = @ptrFromInt(id + 1)}, true) - catch -@panic("oom"); + const t = ps.thread.create_kernel(.Default, .{ .func = entry, .arg = @ptrFromInt(id + 1) }, true) catch + @panic("oom"); ke.sched.enqueue(&t.kern); } diff --git a/src/tests/smr_pi.zig b/src/tests/smr_pi.zig index 1564d42..3be5ba0 100644 --- a/src/tests/smr_pi.zig +++ b/src/tests/smr_pi.zig @@ -9,7 +9,7 @@ const mm = r.mm; const ps = r.ps; const rtl = @import("rtl"); -const ki = ke.private; +const kep = ke.private; const prio_reader: u8 = 4; const prio_writer: u8 = 6; @@ -90,8 +90,8 @@ fn reader(param: ?*anyopaque) void { while (true) { park_point(); - var tracker: ki.smr.Tracker = undefined; - ki.smr.enter_preempt(dom, &tracker); + var tracker: kep.smr.Tracker = undefined; + kep.smr.enter_preempt(dom, &tracker); var done: u64 = 0; for (0..reads_per_section) |_| { @@ -113,7 +113,7 @@ fn reader(param: ?*anyopaque) void { // land on the stalled list. A blocked poller must boost us out of here. spin_us(section_us); - ki.smr.exit_preempt(dom, &tracker); + kep.smr.exit_preempt(dom, &tracker); _ = reads.fetchAdd(done, .monotonic); } @@ -175,13 +175,13 @@ fn cur_inherited(td: *ke.Thread) u8 { } /// Highest priority among the threads blocked on `ts`, or null if none. -fn blocked_prio(ts: *ki.turnstile.Turnstile) ?u8 { +fn blocked_prio(ts: *kep.turnstile.Turnstile) ?u8 { var best: ?u8 = null; for (&ts.queues) |*q| { var it = q.iterator(); while (it.next()) : (it.advance()) { - const w: *ki.turnstile.Waiter = @fieldParentPtr("link", it.get()); + const w: *kep.turnstile.Waiter = @fieldParentPtr("link", it.get()); best = @max(best orelse 0, cur_prio(w.thread)); } } @@ -195,7 +195,7 @@ fn check_cpu(cpu: *ke.smr.Cpu) void { const ipl = ke.ipl.raise(.Dispatch); cpu.stall_lock.acquire_no_ipl(); - const turnstile = ki.turnstile.lookup(cpu); + const turnstile = kep.turnstile.lookup(cpu); if (turnstile) |ts| { if (blocked_prio(ts)) |want| { @@ -203,8 +203,8 @@ fn check_cpu(cpu: *ke.smr.Cpu) void { var it = cpu.stalled.iterator(); while (it.next()) : (it.advance()) { - const owner: *ki.turnstile.Owner = @fieldParentPtr("link", it.get()); - const tracker: *ki.smr.Tracker = @fieldParentPtr("owner", owner); + const owner: *kep.turnstile.Owner = @fieldParentPtr("link", it.get()); + const tracker: *kep.smr.Tracker = @fieldParentPtr("owner", owner); const td = tracker.owner.thread; stalled += 1; @@ -226,7 +226,7 @@ fn check_cpu(cpu: *ke.smr.Cpu) void { } } - ki.turnstile.exit(cpu); + kep.turnstile.exit(cpu); cpu.stall_lock.release_no_ipl(); ke.ipl.lower(ipl); } @@ -279,13 +279,13 @@ fn quiesce_check() void { } fn spawn(prio: u8, entry: *const fn (?*anyopaque) void, id: usize) *ps.Thread { - const t = ps.thread.create_kernel(@enumFromInt(prio), .{.func = entry, .arg= @ptrFromInt(id + 1)}, true) catch @panic("oom"); + const t = ps.thread.create_kernel(@enumFromInt(prio), .{ .func = entry, .arg = @ptrFromInt(id + 1) }, true) catch @panic("oom"); ke.sched.enqueue(&t.kern); return t; } pub fn start(_: ?*anyopaque) void { - std.log.info("{}", .{@sizeOf(ki.smr.Tracker)}); + std.log.info("{}", .{@sizeOf(kep.smr.Tracker)}); dom = mm.zone.smr_domain_create(true) catch @panic("failed to create SMR domain"); zone.init("smr-pi", @sizeOf(Object), .{