mirrors/Zag
1
0
Fork 0
mirror of https://github.com/rdmsr/zag.git synced 2026-10-27 13:58:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

all: standardize suffixes

Browse Source 
ki -> kep
mi -> mmp
This commit is contained in:
Loïc Grégoire
2026-09-02 22:20:25 -04:00
parent cf58ad369e
commit 497e08bd7b
37 changed files with 244 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/kern/amd64/impl.zig
+2 -2
View File
@@ -1,6 +1,6 @@
const r = @import("root");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
const pl = r.pl;
const amd64 = r.arch;
@@ -52,7 +52,7 @@ extern fn amd64_call_continuation(
) callconv(.c) noreturn;
export fn ki_free_old_stack(stack_top: usize) callconv(.c) void {
ki.thread.stack_cache_free(stack_top);
kep.thread.stack_cache_free(stack_top);
}
/// Trampoline to match the zig ABI.
src/kern/amd64/init.zig
+5 -5
View File
@@ -4,7 +4,7 @@ const r = @import("root");
const std = @import("std");
const impl = @import("impl.zig");
const ke = r.ke;
const ki = r.ke.private;
const kep = r.ke.private;
const pl = r.pl;
var gdt = extern struct {
@@ -85,13 +85,13 @@ fn early_cpu_init() linksection(r.init) void {
}
pub fn ap_entry(cpu_id: u32, booted: *std.atomic.Value(usize)) noreturn {
amd64.write_msr(.GsBase, ki.impl.cpu_offsets[cpu_id]);
amd64.write_msr(.GsBase, kep.impl.cpu_offsets[cpu_id]);
early_cpu_init();
ki.cpu.init_cpu(cpu_id);
kep.cpu.init_cpu(cpu_id);
pl.impl.init_ap();
const sched = ki.sched.percpu.local();
const sched = kep.sched.percpu.local();
sched.current_thread = pl.impl.smp.start_thread.local().*;
sched.idle_thread = sched.current_thread;
@@ -101,7 +101,7 @@ pub fn ap_entry(cpu_id: u32, booted: *std.atomic.Value(usize)) noreturn {
_ = booted.fetchAdd(1, .monotonic);
ke.ipl.lower(.Passive);
ki.sched.idle(null);
kep.sched.idle(null);
}
var initial_offsets: [1]usize = .{0};
src/kern/amd64/int.zig
+8 -8
View File
@@ -2,9 +2,9 @@ const amd64 = @import("arch");
const std = @import("std");
const r = @import("root");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
const pl = r.pl;
const mi = r.mm.private;
const mmp = r.mm.private;
extern const __interrupt_vectors: [256]usize;
@@ -49,7 +49,7 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
const ipl = ke.ipl.set_hardware(.Device);
if (frame.intno < 32) {
ki.panic.panic_lock.acquire_no_ipl();
kep.panic.panic_lock.acquire_no_ipl();
std.log.err("Unhandled exception: 0x{x} ({s}), err=0x{x}, pc=0x{x}", .{ frame.intno, exception_msg[frame.intno], frame.errcode, frame.rip });
std.log.err("RAX=0x{x:0>16} RBX=0x{x:0>16} RCX=0x{x:0>16} RDX=0x{x:0>16}", .{ frame.rax, frame.rbx, frame.rcx, frame.rdx });
std.log.err("RSI=0x{x:0>16} RDI=0x{x:0>16} RBP=0x{x:0>16} RSP=0x{x:0>16}", .{ frame.rsi, frame.rdi, frame.rbp, frame.rsp });
@@ -78,16 +78,16 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
std.log.err("CR2=0x{x:0>16} CR3=0x{x:0>16} RFLAGS={s}", .{ cr2, cr3, buf[0..writer.end] });
ki.panic.panic_with_frame("Unhandled exception", frame.rbp);
kep.panic.panic_with_frame("Unhandled exception", frame.rbp);
}
if (frame.intno == 32) {
ki.timer.clock();
kep.timer.clock();
}
// Platform IPI number.
if (frame.intno == 240) {
ki.ipi.ipi_handler();
kep.ipi.ipi_handler();
}
if (frame.intno >= 32) {
@@ -97,9 +97,9 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
_ = ke.ipl.set_hardware(ipl);
if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and
ki.ipl.is_softint_pending(.Dispatch))
kep.ipl.is_softint_pending(.Dispatch))
{
ki.dpc.dispatch();
kep.dpc.dispatch();
}
}
src/kern/cpu.zig
+6 -6
View File
@@ -3,7 +3,7 @@ const rtl = @import("rtl");
const r = @import("root");
const ke = r.ke;
const ki = r.ke.private;
const kep = r.ke.private;
const id = CpuLocal(u32, 0);
@@ -29,12 +29,12 @@ pub fn CpuLocal(comptime T: type, comptime init: T) type {
/// Return a pointer to local CPU data.
pub fn local() *T {
return ki.impl.percpu_ptr(&storage);
return kep.impl.percpu_ptr(&storage);
}
/// Return a pointer to remote CPU data.
pub fn remote(cpu: u32) *T {
return ki.impl.percpu_ptr_other(&storage, cpu);
return kep.impl.percpu_ptr_other(&storage, cpu);
}
};
}
@@ -48,11 +48,11 @@ pub fn ExportedCpuLocal(
const S = struct {
var storage: T linksection(".data.percpu") = init;
pub fn local() @TypeOf(ki.impl.percpu_ptr(&storage)) {
return ki.impl.percpu_ptr(&storage);
pub fn local() @TypeOf(kep.impl.percpu_ptr(&storage)) {
return kep.impl.percpu_ptr(&storage);
}
pub fn remote(cpu: u32) *T {
return ki.impl.percpu_ptr_other(&storage, cpu);
return kep.impl.percpu_ptr_other(&storage, cpu);
}
};
@export(&S.storage, .{ .name = name, .linkage = .strong });
src/kern/dpc.zig
+15 -15
View File
@@ -3,7 +3,7 @@ const config = @import("config");
const rtl = @import("rtl");
const r = @import("root");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
/// Deferred Procedure Call (DPC) structure.
/// Used for scheduling work to be done when IPL is lowered below `.Dispatch`.
@@ -63,7 +63,7 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void {
dpc_cpu.queue.insert_tail(&dpc.link);
// Mark the DPC as pending on this CPU
ki.ipl.set_softint_pending(mycpu, .Dispatch);
kep.ipl.set_softint_pending(mycpu, .Dispatch);
dpc_cpu.lock.release_no_ipl();
}
@@ -72,17 +72,17 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void {
}
fn dispatch_queue(cpu: u32) void {
const ipl_cpu = ki.ipl.cpu_ipl.local();
const ipl_cpu = kep.ipl.cpu_ipl.local();
const dpc_cpu = pcpu.local();
const sched_cpu = ki.sched.percpu.local();
const sched_cpu = kep.sched.percpu.local();
ipl_cpu.* = .Dispatch;
// Mark as handled, this must be done before enabling interrupts or we will
// race.
ki.ipl.clear_softint_pending(cpu, .Dispatch);
kep.ipl.clear_softint_pending(cpu, .Dispatch);
_ = ki.impl.enable_interrupts();
_ = kep.impl.enable_interrupts();
while (true) {
const ipl = dpc_cpu.lock.acquire_at(.High);
@@ -133,35 +133,35 @@ fn dispatch_queue(cpu: u32) void {
sched_cpu.start_timer = false;
sched_cpu.preemption_reason = .None;
ki.shootdown.process_shootdowns();
kep.shootdown.process_shootdowns();
if (sched_cpu.next_thread != null) {
const curtd = sched_cpu.current_thread.?;
if (!curtd.smr_sections.is_empty()) {
ki.smr.mark_thread_stalled(curtd);
kep.smr.mark_thread_stalled(curtd);
}
ki.sched.handle_preemption(sched_cpu);
kep.sched.handle_preemption(sched_cpu);
}
_ = ki.impl.disable_interrupts();
_ = kep.impl.disable_interrupts();
}
/// Dispatch the DPC queue on the current CPU.
pub fn dispatch() void {
// DPC processing is done at IPL Dispatch.
// Don't call lower/raise here because they might call us again recursively.
const int_state = ki.impl.disable_interrupts();
const old_ipl = ki.ipl.cpu_ipl.local().*;
const int_state = kep.impl.disable_interrupts();
const old_ipl = kep.ipl.cpu_ipl.local().*;
var mycpu = ke.cpu.current();
while (ki.ipl.is_softint_pending(.Dispatch)) {
while (kep.ipl.is_softint_pending(.Dispatch)) {
dispatch_queue(mycpu);
// Our CPU might have changed.
mycpu = ke.cpu.current();
}
ki.ipl.cpu_ipl.local().* = old_ipl;
ki.impl.restore_interrupts(int_state);
kep.ipl.cpu_ipl.local().* = old_ipl;
kep.impl.restore_interrupts(int_state);
}
src/kern/event.zig
+3 -3
View File
@@ -1,9 +1,9 @@
const ke = @import("root").ke;
const ki = ke.private;
const kep = ke.private;
/// Generic waitable Event, exported for use everywhere in the kernel.
pub const Event = struct {
hdr: ki.wait.DispatchHeader,
hdr: kep.wait.DispatchHeader,
const Self = @This();
@@ -26,7 +26,7 @@ pub const Event = struct {
pub fn signal(self: *Self) void {
const ipl = self.hdr.lock.acquire();
self.hdr.signaled = 1;
ki.wait.satisfy_wait(&self.hdr);
kep.wait.satisfy_wait(&self.hdr);
self.hdr.lock.release(ipl);
}
src/kern/init.zig
+9 -9
View File
@@ -3,7 +3,7 @@ const r = @import("root");
const pl = r.pl;
const ke = r.ke;
const arch = r.arch;
const ki = ke.private;
const kep = ke.private;
const ex = r.ex;
var thread0: ke.Thread = undefined;
@@ -29,16 +29,16 @@ export fn kmain(boot_info: *r.BootInfo) callconv(.c) void {
fn init() linksection(r.init) void {
ke.ncpus = 1;
ki.impl.early_init();
ki.tunable.init();
kep.impl.early_init();
kep.tunable.init();
std.log.info("Welcome to the machine", .{});
pl.early_init();
ki.log.init();
ki.cpu.init_cpu(0);
ki.turnstile.init_turnstiles();
kep.log.init();
kep.cpu.init_cpu(0);
kep.turnstile.init_turnstiles();
ki.sched.percpu.local().current_thread = &thread0;
ki.sched.percpu.local().idle_thread = &thread0;
kep.sched.percpu.local().current_thread = &thread0;
kep.sched.percpu.local().idle_thread = &thread0;
std.log.info("Zag for {s} ({s}), cmdline is \"{?s}\"", .{
pl.name,
@@ -47,5 +47,5 @@ fn init() linksection(r.init) void {
});
ex.init();
ki.sched.idle(null);
kep.sched.idle(null);
}
src/kern/ipi.zig
+2 -2
View File
@@ -4,7 +4,7 @@ const r = @import("root");
const std = @import("std");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
const pl = r.pl;
const State = enum(u8) {
@@ -138,7 +138,7 @@ pub fn ipi_handler() void {
_ = frozen_cpus.fetchAdd(1, .monotonic);
while (true) {
ki.impl.halt();
kep.impl.halt();
}
}
src/kern/ipl.zig
+9 -9
View File
@@ -1,7 +1,7 @@
const std = @import("std");
const r = @import("root");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
/// Interrupt priority level (IPL)
pub const Ipl = enum(u8) {
@@ -44,7 +44,7 @@ pub fn current() Ipl {
pub fn raise(new: Ipl) Ipl {
var old: ke.Ipl = undefined;
if (ki.impl.raise_software_ipl(new, &old)) {
if (kep.impl.raise_software_ipl(new, &old)) {
@branchHint(.unlikely);
std.debug.panic(
"ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})",
@@ -55,7 +55,7 @@ pub fn raise(new: Ipl) Ipl {
if (@intFromEnum(new) > @intFromEnum(Ipl.get_max_software())) {
// Moving between hardware levels is rare.
@branchHint(.unlikely);
ki.impl.set_hardware_ipl(new);
kep.impl.set_hardware_ipl(new);
}
return old;
@@ -66,7 +66,7 @@ pub fn raise(new: Ipl) Ipl {
pub fn lower(new: Ipl) void {
var old: ke.Ipl = undefined;
if (ki.impl.lower_software_ipl(new, &old)) {
if (kep.impl.lower_software_ipl(new, &old)) {
@branchHint(.unlikely);
std.debug.panic(
"ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})",
@@ -78,27 +78,27 @@ pub fn lower(new: Ipl) void {
@intFromEnum(old) > @intFromEnum(Ipl.get_max_software()))
{
@branchHint(.unlikely);
ki.impl.set_hardware_ipl(.Passive);
kep.impl.set_hardware_ipl(.Passive);
}
if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and
is_softint_pending(.Dispatch))
{
// Dispatch DPCs if necessary.
ki.dpc.dispatch();
kep.dpc.dispatch();
}
}
/// Set the hardware IPL to `new`.
pub fn set_hardware(new: Ipl) Ipl {
const ints = ki.impl.disable_interrupts();
const ints = kep.impl.disable_interrupts();
const cpu = cpu_ipl.local();
const old = cpu.*;
cpu.* = new;
ki.impl.set_hardware_ipl(new);
ki.impl.restore_interrupts(ints);
kep.impl.set_hardware_ipl(new);
kep.impl.restore_interrupts(ints);
return old;
}
src/kern/log.zig
+2 -2
View File
@@ -4,7 +4,7 @@ const r = @import("root");
const pl = r.pl;
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
var out_lock: ke.SpinLock = .init();
@@ -41,7 +41,7 @@ const DebugWriter = struct {
// Messages are on average 2^5 = 32 bytes.
const avg_msg_size_bits = 5;
pub var ringbuffer = ki.log_ring.RingBuffer(
pub var ringbuffer = kep.log_ring.RingBuffer(
config.log_buffer_shift,
avg_msg_size_bits,
).init();
src/kern/log_ring.zig
+3 -3
View File
@@ -71,7 +71,7 @@ const std = @import("std");
const r = @import("root");
const rtl = @import("rtl");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
const BlkPos = struct {
begin: usize,
@@ -645,7 +645,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
var reservation: Reservation = undefined;
reservation.irq_state = ki.impl.disable_interrupts();
reservation.irq_state = kep.impl.disable_interrupts();
const id = try self.reserve_desc();
@@ -703,7 +703,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
);
// Restore IRQ state after publishing the new entry.
ki.impl.restore_interrupts(res.irq_state);
kep.impl.restore_interrupts(res.irq_state);
}
/// Return the first readable sequence currently retained by the ring.
src/kern/mutex.zig
+9 -9
View File
@@ -1,6 +1,6 @@
const std = @import("std");
const ke = @import("root").ke;
const ki = ke.private;
const kep = ke.private;
/// Number of spins before blocking.
const optimistic_spins = 100;
@@ -19,7 +19,7 @@ pub const Mutex = struct {
pub fn acquire(m: *Mutex) void {
const ipl = ke.ipl.raise(.Dispatch);
const curtd = ki.sched.percpu.local().current_thread.?;
const curtd = kep.sched.percpu.local().current_thread.?;
// Very fast path: the lock is uncontended and we can acquire it
//immediately.
@@ -57,12 +57,12 @@ pub const Mutex = struct {
// Slow path: contended, block on a turnstile.
while (true) {
const ts = ki.turnstile.lookup(m);
const ts = kep.turnstile.lookup(m);
const owner = m.owner.load(.monotonic);
if (owner == null) {
// Lock was released between lookup and here.
ki.turnstile.exit(m);
kep.turnstile.exit(m);
if (m.owner.cmpxchgStrong(
null,
curtd,
@@ -75,7 +75,7 @@ pub const Mutex = struct {
}
// Block until woken.
ki.turnstile.block(
kep.turnstile.block(
ts,
m,
.{ .single = owner.? },
@@ -99,12 +99,12 @@ pub const Mutex = struct {
pub fn release(m: *Mutex) void {
const ipl = ke.ipl.raise(.Dispatch);
const ts = ki.turnstile.lookup(m);
const ts = kep.turnstile.lookup(m);
m.owner.store(null, .release);
if (ts == null) {
ki.turnstile.exit(m);
kep.turnstile.exit(m);
ke.ipl.lower(ipl);
return;
}
@@ -114,13 +114,13 @@ pub const Mutex = struct {
// to be better because this avoids lock convoys,
// see this mysterious 70s paper:
// https://dl.acm.org/doi/pdf/10.1145/850657.850659
ki.turnstile.wakeup(
kep.turnstile.wakeup(
ts.?,
.Exclusive,
ts.?.waiters,
null,
);
ki.turnstile.exit(m);
kep.turnstile.exit(m);
ke.ipl.lower(ipl);
}
src/kern/panic.zig
+6 -6
View File
@@ -3,7 +3,7 @@ const config = @import("config");
const r = @import("root");
const ksyms = @import("ksyms");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
extern var text_start_addr: u8;
extern var text_end_addr: u8;
@@ -77,7 +77,7 @@ pub fn panic_with_frame(
msg: []const u8,
frame: usize,
) noreturn {
_ = ki.impl.disable_interrupts();
_ = kep.impl.disable_interrupts();
std.log.info("Crash count is {} on cpu {}", .{ crash_count.load(.monotonic), ke.cpu.current() });
@@ -86,21 +86,21 @@ pub fn panic_with_frame(
// Another CPU has already crashed, just freeze manually right now.
while (true) {
ki.impl.halt();
kep.impl.halt();
}
}
// We're the first one who actually crashed, that means we get to handle
// things like dumping the system info and freezing other CPUs. Start by
// doing the freeze.
ki.ipi.freeze_cpus();
kep.ipi.freeze_cpus();
_ = ke.ipl.raise(.High);
std.log.err("KERNEL PANIC: {s} on CPU {}, curthread is {*}", .{
msg,
ke.cpu.current(),
ki.sched.percpu.local().current_thread.?,
kep.sched.percpu.local().current_thread.?,
});
std.log.err("Stack trace:", .{});
@@ -108,7 +108,7 @@ pub fn panic_with_frame(
walk_stack_frame(frame);
while (true) {
ki.impl.halt();
kep.impl.halt();
}
}
src/kern/queue.zig
+6 -6
View File
@@ -8,11 +8,11 @@
const r = @import("root");
const rtl = @import("rtl");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
const std = @import("std");
pub const Queue = struct {
hdr: ki.wait.DispatchHeader,
hdr: kep.wait.DispatchHeader,
items: rtl.List,
active: usize,
max_active: usize,
@@ -49,14 +49,14 @@ pub const Queue = struct {
},
}
ki.wait.satisfy_wait(&self.hdr);
kep.wait.satisfy_wait(&self.hdr);
}
/// Remove the item at the head.
/// This blocks until an item is actually popped.
pub fn remove(self: *Queue, timeout: ?r.Nanoseconds) !*rtl.List.Entry {
const ipl = self.hdr.lock.acquire();
const td = ki.sched.percpu.local().current_thread.?;
const td = kep.sched.percpu.local().current_thread.?;
if (td.queue) |q| {
std.debug.assert(q == self);
@@ -64,7 +64,7 @@ pub const Queue = struct {
self.active -= 1;
if (self.hdr.signaled > 0 and self.active < self.max_active) {
ki.wait.satisfy_wait(&self.hdr);
kep.wait.satisfy_wait(&self.hdr);
}
} else {
td.queue = self;
@@ -95,7 +95,7 @@ pub fn signal_wait(queue: *Queue) void {
queue.active -= 1;
if (queue.hdr.signaled > 0 and queue.active < queue.max_active) {
ki.wait.satisfy_wait(&queue.hdr);
kep.wait.satisfy_wait(&queue.hdr);
}
queue.hdr.lock.release(ipl);
src/kern/sched.zig
+14 -14
View File
@@ -98,7 +98,7 @@ const rtl = @import("rtl");
const config = @import("config");
const r = @import("root");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
const pl = r.pl;
const runqueues_n = 64;
@@ -268,7 +268,7 @@ fn init_cpu() linksection(r.init) void {
.est_load_avg = .init(0),
.migratable = .init(0),
.current_thread_prio = .init(0),
.resched_dpc = .init(ki.sched.clock),
.resched_dpc = .init(kep.sched.clock),
.resched_timer = undefined,
.start_timer = false,
.preemption_reason = .None,
@@ -1034,7 +1034,7 @@ fn enqueue_on_cpu(c: u32, td: *ke.Thread) void {
cpu.preemption_reason = .HigherPriority;
td.cpu = c;
td.state.store(.Selected, .monotonic);
ki.ipl.set_softint_pending(c, .Dispatch);
kep.ipl.set_softint_pending(c, .Dispatch);
cpu.current_thread_prio.store(td.priority, .monotonic);
@@ -1056,19 +1056,19 @@ fn enqueue_on_cpu(c: u32, td: *ke.Thread) void {
if (cpu.resched_timer.state.load(.monotonic) == .Stopped) {
cpu.start_timer = true;
ki.ipl.set_softint_pending(c, .Dispatch);
kep.ipl.set_softint_pending(c, .Dispatch);
}
// queues_lock dropped
}
fn allocate_stack(td: *ke.Thread) ?usize {
if (ki.thread.stack_cache_pop()) |stack| {
if (kep.thread.stack_cache_pop()) |stack| {
return stack;
}
// Enqueue the thread for allocation
ki.thread.thread_stack_queue.insert(@ptrCast(&td.runq_link.next));
kep.thread.thread_stack_queue.insert(@ptrCast(&td.runq_link.next));
return null;
}
@@ -1122,13 +1122,13 @@ fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void {
cur.switching.store(false, .release);
cur.lock.release_no_ipl();
ki.impl.call_continuation(&next.context, continuation);
kep.impl.call_continuation(&next.context, continuation);
}
if (relinquish) {
ki.impl.switch_cont_to_normal(&cur.context, &next.context);
kep.impl.switch_cont_to_normal(&cur.context, &next.context);
} else {
ki.impl.switch_normal_to_normal(&cur.context, &next.context);
kep.impl.switch_normal_to_normal(&cur.context, &next.context);
}
}
@@ -1206,7 +1206,7 @@ pub fn handle_preemption(cpu: *PerCpu) void {
/// Called every time slice in DPC context.
pub fn clock(_: *ke.Dpc, _: ?*anyopaque) void {
const cpu = percpu.local();
std.debug.assert(ki.ipl.current() == .Dispatch);
std.debug.assert(kep.ipl.current() == .Dispatch);
const curtd = cpu.current_thread orelse return;
@@ -1369,7 +1369,7 @@ pub fn yield_locked(continuation: ?ke.Continuation) void {
cur.switching.store(false, .monotonic);
cur.lock.release_no_ipl();
if (cont) |c| ki.impl.call_continuation(&cur.context, c);
if (cont) |c| kep.impl.call_continuation(&cur.context, c);
}
// cur lock dropped
@@ -1508,7 +1508,7 @@ pub fn update_priority_locked(td: *ke.Thread, new_prio: u8) void {
cpu.preemption_reason = .HigherPriority;
cpu.current_thread_prio.store(next.priority, .monotonic);
ki.ipl.set_softint_pending(c, .Dispatch);
kep.ipl.set_softint_pending(c, .Dispatch);
if (c != ke.cpu.current()) {
pl.send_ipi(c);
@@ -1770,8 +1770,8 @@ pub fn idle(_: ?*anyopaque) noreturn {
}
// Dispatch any pending DPCs.
if (ki.ipl.is_softint_pending(.Dispatch)) {
ki.dpc.dispatch();
if (kep.ipl.is_softint_pending(.Dispatch)) {
kep.dpc.dispatch();
}
std.atomic.spinLoopHint();
src/kern/shootdown.zig
+7 -7
View File
@@ -5,7 +5,7 @@ const config = @import("config");
const rtl = @import("rtl");
const r = @import("root");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
const mm = r.mm;
const pl = r.pl;
@@ -87,14 +87,14 @@ fn allocate_slot(cpu: *PerCpu) ?usize {
}
fn flush_range(va: r.VAddr, npages: usize) void {
if (npages > ki.impl.tlb_max_pages) {
if (npages > kep.impl.tlb_max_pages) {
// Just flush the entire thing.
ki.impl.flush_full_tlb();
kep.impl.flush_full_tlb();
return;
}
for (0..npages) |i| {
ki.impl.flush_tlb(va + i * mm.page_size);
kep.impl.flush_tlb(va + i * mm.page_size);
}
}
@@ -148,12 +148,12 @@ pub fn process_shootdowns() void {
if (nreq == 0) return;
// npages is just a hint, we don't need it to be accurate.
const full = cpu.npages.load(.monotonic) >= ki.impl.tlb_max_pages;
const full = cpu.npages.load(.monotonic) >= kep.impl.tlb_max_pages;
cpu.npages.store(0, .monotonic);
if (full) {
// Just flush the entire thing.
ki.impl.flush_full_tlb();
kep.impl.flush_full_tlb();
}
// Go through every sender and release their states.
@@ -220,6 +220,6 @@ pub fn submit(state: ShootdownState, target_mask: ke.CpuMask) !void {
remote.senders.set(curcpu, .release);
rtl.barrier.fence(.release);
ki.ipl.set_softint_pending(@truncate(bit), .Dispatch);
kep.ipl.set_softint_pending(@truncate(bit), .Dispatch);
}
}
src/kern/smr.zig
+14 -14
View File
@@ -7,7 +7,7 @@ const std = @import("std");
const config = @import("config");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
pub const Sequence = u64;
@@ -70,7 +70,7 @@ pub const Tracker = struct {
dom: *Domain,
seq: Sequence,
cpu: ?*Cpu,
owner: ki.turnstile.Owner,
owner: kep.turnstile.Owner,
};
/// Used by tests.
@@ -152,18 +152,18 @@ fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence
cpu.stall_goal = goal;
cpu.stall_lock.release_no_ipl();
const ts = ki.turnstile.lookup(cpu);
const ts = kep.turnstile.lookup(cpu);
seq = cpu.stall_seq.load(.monotonic);
// Re-check under the chain lock.
if (seq == seq_invalid or goal <= seq) {
ki.turnstile.exit(cpu);
kep.turnstile.exit(cpu);
ke.ipl.lower(ipl);
break;
}
ki.turnstile.block(
kep.turnstile.block(
ts,
cpu,
.{ .shared = &cpu.stalled },
@@ -354,7 +354,7 @@ pub fn mark_thread_stalled(td: *ke.Thread) void {
const entry = it.get();
it.advance();
const owner: *ki.turnstile.Owner = @fieldParentPtr("link", entry);
const owner: *kep.turnstile.Owner = @fieldParentPtr("link", entry);
const tracker: *Tracker = @fieldParentPtr("owner", owner);
if (tracker.cpu == null) {
@@ -370,16 +370,16 @@ pub fn mark_thread_stalled(td: *ke.Thread) void {
cpu.current_seq.store(seq_invalid, .release);
const turnstile = ki.turnstile.lookup(cpu);
const turnstile = kep.turnstile.lookup(cpu);
entry.remove();
cpu.stalled.insert_tail(&tracker.owner.link);
if (turnstile) |ts| {
ki.turnstile.owner_enter(ts, &tracker.owner);
kep.turnstile.owner_enter(ts, &tracker.owner);
}
ki.turnstile.exit(cpu);
kep.turnstile.exit(cpu);
cpu.stall_lock.release_no_ipl();
}
@@ -424,12 +424,12 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void {
// We got preempted.
cpu.stall_lock.acquire_no_ipl();
const turnstile = ki.turnstile.lookup(cpu);
const turnstile = kep.turnstile.lookup(cpu);
// Remove ourselves from the stalled list.
tracker.owner.link.remove();
ki.turnstile.owner_leave(&tracker.owner);
kep.turnstile.owner_leave(&tracker.owner);
var wake = false;
@@ -437,7 +437,7 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void {
cpu.stall_seq.store(seq_invalid, .release);
wake = true;
} else {
const first_owner: *ki.turnstile.Owner = @fieldParentPtr(
const first_owner: *kep.turnstile.Owner = @fieldParentPtr(
"link",
cpu.stalled.first(),
);
@@ -449,10 +449,10 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void {
if (wake) {
if (turnstile) |ts|
ki.turnstile.wakeup(ts, .Exclusive, ts.waiters, null);
kep.turnstile.wakeup(ts, .Exclusive, ts.waiters, null);
}
ki.turnstile.exit(cpu);
kep.turnstile.exit(cpu);
cpu.stall_lock.release_no_ipl();
ke.ipl.lower(ipl);
src/kern/thread.zig
+14 -14
View File
@@ -3,7 +3,7 @@ const r = @import("root");
const rtl = @import("rtl");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
pub const Priority = enum(u8) {
const Self = @This();
@@ -79,7 +79,7 @@ pub const Thread = struct {
pub const Context = extern struct {
/// Implementation-dependent context.
impl: ki.impl.ThreadContext,
impl: kep.impl.ThreadContext,
stack_top: usize,
@@ -127,20 +127,20 @@ pub const Thread = struct {
/// CPU this thread is enqueued on.
cpu: ?u32,
/// Run queue this thread is in
runq: ?*ki.sched.RunQueue,
runq: ?*kep.sched.RunQueue,
/// Index into the run queue this thread is currently in.
/// Only valid when `runq` is non-null.
runq_idx: u8,
/// Current wait status.
wait_status: std.atomic.Value(ki.wait.Status),
waitblocks: [4]ki.wait.WaitBlock,
wait_status: std.atomic.Value(kep.wait.Status),
waitblocks: [4]kep.wait.WaitBlock,
/// Reason for the wait, if any.
wait_reason: ?[]const u8,
/// Timer used for timeouts.
timer: ke.Timer,
/// Turnstile.
turnstile: *ki.turnstile.Turnstile,
turnstile_waiter: ?*ki.turnstile.Waiter,
turnstile: *kep.turnstile.Turnstile,
turnstile_waiter: ?*kep.turnstile.Waiter,
turnstiles_owned: rtl.List,
/// Object this thread is currently blocked on, or null.
waiting_on: ?*anyopaque,
@@ -148,9 +148,9 @@ pub const Thread = struct {
queue: ?*ke.Queue,
queue_item: ?*rtl.List.Entry,
/// PELT load average,
avg: ki.sched.Average,
avg: kep.sched.Average,
/// Accounting statistics.
acct: ki.sched.Accounting,
acct: kep.sched.Accounting,
/// Set whenever the thread is switching off its stack.
/// This is used to avoid taking thread next lock to wait for switch off
/// to complete.
@@ -165,7 +165,7 @@ pub const Thread = struct {
/// Top of the stack for the thread.
stack: r.VAddr,
/// Turnstile associated with the thread.
turnstile: *ki.turnstile.Turnstile,
turnstile: *kep.turnstile.Turnstile,
/// Thread's priority.
priority: Priority,
};
@@ -272,13 +272,13 @@ pub fn exit() void {
// Reuse the runq linkage to put on reaper list.
reaper_list.insert(@ptrCast(&curtd.runq_link.next));
ki.sched.detach_load_avg(ki.sched.percpu.local(), curtd);
ki.sched.yield_locked(Continuation.dummy);
kep.sched.detach_load_avg(kep.sched.percpu.local(), curtd);
kep.sched.yield_locked(Continuation.dummy);
}
/// Return the currently running thread.
pub fn current() *ke.Thread {
return ki.sched.percpu.local().current_thread.?;
return kep.sched.percpu.local().current_thread.?;
}
// -- Stack allocation and continuations ---------------------------------------
@@ -350,7 +350,7 @@ pub fn call_continuation(ptr: ?*anyopaque) noreturn {
const cont = td.continuation.?;
td.continuation = null;
ki.impl.call_continuation(&td.context, cont);
kep.impl.call_continuation(&td.context, cont);
}
/// Try to get a stack from the stack cache.
src/kern/time.zig
+1 -1
View File
@@ -2,7 +2,7 @@ const std = @import("std");
const rtl = @import("rtl");
const r = @import("root");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
/// TimeCounter structure.
/// Represents a time-keeping source upon which the kernel can rely.
src/kern/timer.zig
+6 -6
View File
@@ -5,12 +5,12 @@ const std = @import("std");
const rtl = @import("rtl");
const r = @import("root");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
const pl = r.pl;
const PerCpu = struct {
/// Heap of pending timers on this CPU.
timers: rtl.PairingHeap(.min, ki.timer.cmp_timer),
timers: rtl.PairingHeap(.min, kep.timer.cmp_timer),
/// Lock over the timer heap.
lock: ke.SpinLock,
dpc: ke.Dpc,
@@ -26,7 +26,7 @@ pub const Timer = struct {
Stopped,
};
hdr: ki.wait.DispatchHeader,
hdr: kep.wait.DispatchHeader,
/// Timer state.
state: std.atomic.Value(State),
@@ -150,12 +150,12 @@ pub fn clock() void {
// Check for overflows.
// This is fine to call a lot, as the function only gets expensive
// (i.e seqlock store) when an overflow actually happens.
ki.time.update_overflow();
kep.time.update_overflow();
}
// Called in a DPC when a timer has expired.
fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void {
std.debug.assert(ki.ipl.current() == .Dispatch);
std.debug.assert(kep.ipl.current() == .Dispatch);
const cpu = percpu.local();
while (true) {
@@ -217,7 +217,7 @@ fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void {
timer.state.store(.Stopped, .release);
// Wake whomever was waiting on the timer.
ki.wait.satisfy_wait(&timer.hdr);
kep.wait.satisfy_wait(&timer.hdr);
timer.hdr.lock.release_no_ipl();
if (maybe_dpc) |dpc| ke.dpc.enqueue(dpc, null);
src/kern/turnstile.zig
+3 -3
View File
@@ -1,5 +1,5 @@
const ke = @import("root").ke;
const ki = ke.private;
const kep = ke.private;
const rtl = @import("rtl");
const std = @import("std");
@@ -170,7 +170,7 @@ fn recompute_inherited(td: *ke.Thread) void {
fn update_prio(td: *ke.Thread) void {
const pri = td.effective_priority();
if (pri != td.priority) ki.sched.update_priority_locked(td, pri);
if (pri != td.priority) kep.sched.update_priority_locked(td, pri);
}
/// Donate priority `pri` to `to` along the edge `boost`. The caller holds
@@ -392,7 +392,7 @@ pub fn block(
queue: Queue,
) void {
const chain = chain_for(obj);
const curtd = ki.sched.percpu.local().current_thread.?;
const curtd = kep.sched.percpu.local().current_thread.?;
const ipl = ke.ipl.current();
var ts = turnstile;
src/kern/wait.zig
+6 -6
View File
@@ -5,7 +5,7 @@ const rtl = @import("rtl");
const std = @import("std");
const r = @import("root");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
/// Header for waitable objects.
/// This must be added to any structured which is considered waitable.
@@ -122,7 +122,7 @@ pub fn wait_one(object: *DispatchHeader, reason: []const u8, opts: Options) !usi
pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !usize {
const ipl = ke.ipl.raise(.Dispatch);
defer ke.ipl.lower(ipl);
const curtd = ki.sched.percpu.local().current_thread.?;
const curtd = kep.sched.percpu.local().current_thread.?;
const obj_count = objects.len;
const has_timeout = opts.timeout != null;
const blocks = opts.waitblocks orelse blk: {
@@ -222,13 +222,13 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
) == null) {
if (queue == null) {
if (curtd.queue) |q| {
ki.queue.signal_wait(q);
kep.queue.signal_wait(q);
}
}
curtd.wait_reason = reason;
// We're good, now actually block.
ki.sched.block_locked(curtd, opts.continuation);
kep.sched.block_locked(curtd, opts.continuation);
} else {
queue = null;
@@ -323,11 +323,11 @@ pub fn satisfy_wait(obj: *DispatchHeader) void {
if (obj.type != .Queue) {
if (td.queue) |q| {
ki.queue.signal_wake(q);
kep.queue.signal_wake(q);
}
}
ki.sched.unblock_locked(td);
kep.sched.unblock_locked(td);
td.lock.release(ipl);
}
src/mm/amd64/impl.zig
+3 -3
View File
@@ -1,7 +1,7 @@
const amd64 = @import("arch");
const r = @import("root");
const mm = r.mm;
const mi = mm.private;
const mmp = mm.private;
pub const hhdm_minimum_max_address = r.gib(4);
@@ -17,7 +17,7 @@ pub const hhdm_base = 0xffff800000000000;
pub const kernel_heap_base = 0xffffc00000000000;
pub const pfndb_base = 0xffffd00000000000;
pub const levels = [_]mi.PMapLevel{
pub const levels = [_]mmp.PMapLevel{
.{ .shift = 12, .mask = 0x1ff, .leaf = true }, // 4K
.{ .shift = 21, .mask = 0x1ff, .leaf = true }, // 2M
.{ .shift = 30, .mask = 0x1ff, .leaf = true }, // 1G
@@ -117,5 +117,5 @@ pub fn is_leaf_level_enabled(level: usize) bool {
}
pub fn init_kernel() void {
mi.kernel_space.pmap.root_pa = amd64.read_cr(3);
mmp.kernel_space.pmap.root_pa = amd64.read_cr(3);
}
src/mm/balance.zig
+1 -1
View File
@@ -10,7 +10,7 @@ const std = @import("std");
const ps = r.ps;
const ke = r.ke;
const mm = r.mm;
const mi = mm.private;
const mmp = mm.private;
const balance_interval = ke.Tunable(u32, 1000, "mm.balance.interval_ms");
src/mm/heap.zig
+17 -17
View File
@@ -1,54 +1,54 @@
const r = @import("root");
const ke = r.ke;
const mm = r.mm;
const mi = mm.private;
const mmp = mm.private;
pub fn init() void {
mi.kernel_space.arena.init(
mmp.kernel_space.arena.init(
"kernel heap",
mi.impl.kernel_heap_base,
mmp.impl.kernel_heap_base,
r.tib(16),
mm.page_size,
) catch @panic("failed to initialize kernel heap arena");
}
pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque {
mi.kernel_space.lock.acquire();
mmp.kernel_space.lock.acquire();
const addr = mi.kernel_space.arena.alloc(size, .{}) catch {
mi.kernel_space.lock.release();
const addr = mmp.kernel_space.arena.alloc(size, .{}) catch {
mmp.kernel_space.lock.release();
return mm.Error.OutOfMemory;
};
const npages = size / mm.page_size;
for (0..npages) |i| {
const pte = mi.pmap.wire_pte(
&mi.kernel_space,
const pte = mmp.pmap.wire_pte(
&mmp.kernel_space,
addr + (i * mm.page_size),
policy,
) catch {
mi.kernel_space.lock.release();
mmp.kernel_space.lock.release();
return mm.Error.OutOfMemory;
};
mi.kernel_space.lock.release();
mmp.kernel_space.lock.release();
const page = mi.phys.alloc_opts(.{ .policy = policy }) orelse {
const page = mmp.phys.alloc_opts(.{ .policy = policy }) orelse {
return mm.Error.OutOfMemory;
};
mi.kernel_space.lock.acquire();
mmp.kernel_space.lock.acquire();
pte.* = mi.impl.make_leaf_pte(page, .{ .read = true, .write = true }, 0);
pte.* = mmp.impl.make_leaf_pte(page, .{ .read = true, .write = true }, 0);
}
mi.kernel_space.lock.release();
mmp.kernel_space.lock.release();
return @ptrFromInt(addr);
}
pub fn free(va: r.VAddr, size: usize) void {
mi.kernel_space.lock.acquire();
mi.tlb.reclaim_range(&mi.kernel_space, va, size);
mi.kernel_space.lock.release();
mmp.kernel_space.lock.acquire();
mmp.tlb.reclaim_range(&mmp.kernel_space, va, size);
mmp.kernel_space.lock.release();
}
src/mm/init.zig
+8 -8
View File
@@ -1,18 +1,18 @@
const r = @import("root");
const pl = r.pl;
const mm = r.mm;
const mi = mm.private;
const mmp = mm.private;
const config = @import("config");
pub fn init() linksection(r.init) void {
mi.phys.init();
mi.zone.early_init();
mi.vmem.init();
mi.heap.init();
mmp.phys.init();
mmp.zone.early_init();
mmp.vmem.init();
mmp.heap.init();
}
pub fn late_init() linksection(r.init) void {
mi.zone.late_init();
mi.tlb.init();
mi.balance.init();
mmp.zone.late_init();
mmp.tlb.init();
mmp.balance.init();
}
src/mm/phys.zig
+2 -2
View File
@@ -3,7 +3,7 @@ const std = @import("std");
const rtl = @import("rtl");
const pl = r.pl;
const mm = r.mm;
const mi = mm.private;
const mmp = mm.private;
const ke = r.ke;
const log = std.log.scoped(.@"mm/phys");
@@ -173,7 +173,7 @@ pub fn init() linksection(r.init) void {
// Get the kernel pagemap from the loader,
// we are already in it so no need to activate it.
mi.impl.init_kernel();
mmp.impl.init_kernel();
// Calculate the minimum amount of free memory we accept,
// this formula is from Linux and does not grow linearly;
src/mm/pmap.zig
+23 -23
View File
@@ -4,7 +4,7 @@ const r = @import("root");
const rtl = @import("rtl");
const std = @import("std");
const mm = r.mm;
const mi = mm.private;
const mmp = mm.private;
/// Describes a contiguous region of physical memory to be mapped
/// at a given virtual address.
@@ -33,7 +33,7 @@ const ContiguousSource = struct {
};
fn level_shift(level: usize) u6 {
return mi.impl.levels[level].shift;
return mmp.impl.levels[level].shift;
}
fn page_size(level: usize) usize {
@@ -41,24 +41,24 @@ fn page_size(level: usize) usize {
}
fn index_for_level(va: usize, level: usize) usize {
return (va >> level_shift(level)) & mi.impl.levels[level].mask;
return (va >> level_shift(level)) & mmp.impl.levels[level].mask;
}
pub const PMap = struct {
const Self = @This();
const num_levels = mi.impl.levels.len;
const num_levels = mmp.impl.levels.len;
const entries_per_table = 512;
const index_mask = entries_per_table - 1;
root_pa: r.PAddr,
fn is_leaf_level(level: usize) bool {
return mi.impl.levels[level].leaf;
return mmp.impl.levels[level].leaf;
}
fn leaf_level_enabled(level: usize) bool {
return is_leaf_level(level) and mi.impl.is_leaf_level_enabled(level);
return is_leaf_level(level) and mmp.impl.is_leaf_level_enabled(level);
}
fn cursor(self: *Self, va: r.VAddr) Cursor {
@@ -74,7 +74,7 @@ pub const PMap = struct {
}
pub fn activate(self: *Self) void {
mi.impl.activate(self.root_pa);
mmp.impl.activate(self.root_pa);
}
pub fn map_from(self: *Self, va: r.VAddr, size: usize, source: anytype) void {
@@ -129,7 +129,7 @@ pub const PMap = struct {
/// Unmap a contiguous range of virtual pages.
/// Only small pages are supported.
/// Return a pfn that points to the physical pages that were unmapped.
pub fn unmap(self: *Self, va: r.VAddr, size: usize) ?mi.PfnList {
pub fn unmap(self: *Self, va: r.VAddr, size: usize) ?mmp.PfnList {
std.debug.assert(std.mem.isAligned(va, mm.page_size));
var c = self.cursor(va);
@@ -153,7 +153,7 @@ pub const PMap = struct {
const page = mm.pfn_to_struct_page(pfn);
list.insert_tail(&page.free.link);
c.tables[level][idx] = std.mem.zeroes(mi.impl.Pte);
c.tables[level][idx] = std.mem.zeroes(mmp.impl.Pte);
break;
}
@@ -182,7 +182,7 @@ pub const PMap = struct {
va: r.VAddr,
/// Cached pointers to each level's page table, filled as the cursor
/// descends. Only indices in `[0, top_level]` are valid.
tables: [num_levels][*]mi.impl.Pte,
tables: [num_levels][*]mmp.impl.Pte,
/// Index of the highest level for which `tables` currently holds a
/// valid pointer. Descends toward 0 (the leaf) as `walk_down` runs.
/// Reset upward in `advance` when the VA crosses a level boundary.
@@ -197,24 +197,24 @@ pub const PMap = struct {
while (current_level > target_level) {
const idx = index_for_level(self.va, current_level);
var pte = mi.impl.Pte.load(&self.tables[current_level][idx]);
var pte = mmp.impl.Pte.load(&self.tables[current_level][idx]);
if (!pte.present) {
if (!allocate) {
return error.PageNotMapped;
}
const new_table_pa = mi.phys.alloc();
const new_table_pa = mmp.phys.alloc();
const table_ptr: [*]mi.impl.Pte = @ptrFromInt(
const table_ptr: [*]mmp.impl.Pte = @ptrFromInt(
mm.p2v(new_table_pa),
);
@memset(
table_ptr[0..entries_per_table],
std.mem.zeroes(mi.impl.Pte),
std.mem.zeroes(mmp.impl.Pte),
);
pte = mi.impl.make_table_pte(new_table_pa);
pte = mmp.impl.make_table_pte(new_table_pa);
self.tables[current_level][idx] = pte;
}
@@ -247,7 +247,7 @@ pub const PMap = struct {
const table = self.tables[target_level];
table[index_for_level(self.va, target_level)] =
mi.impl.make_leaf_pte(current_pa, flags, target_level);
mmp.impl.make_leaf_pte(current_pa, flags, target_level);
const step = page_size(target_level);
current_pa += step;
@@ -279,14 +279,14 @@ pub const PMap = struct {
};
pub fn wire_pte(
space: *mi.Space,
space: *mmp.Space,
va: r.VAddr,
policy: mm.WaitPolicy,
) mm.Error!*mi.impl.Pte {
) mm.Error!*mmp.impl.Pte {
std.debug.assert(space.lock.is_locked());
var table: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(mi.kernel_space.pmap.root_pa));
var level = mi.impl.levels.len - 1;
var table: [*]mmp.impl.Pte = @ptrFromInt(mm.p2v(mmp.kernel_space.pmap.root_pa));
var level = mmp.impl.levels.len - 1;
while (true) {
const idx = index_for_level(va, level);
@@ -302,7 +302,7 @@ pub fn wire_pte(
// since that may block.
space.lock.release();
const new_table_pa = mi.phys.alloc_opts(.{ .policy = policy }) orelse {
const new_table_pa = mmp.phys.alloc_opts(.{ .policy = policy }) orelse {
space.lock.acquire();
return mm.Error.OutOfMemory;
};
@@ -316,12 +316,12 @@ pub fn wire_pte(
if (entry.present) {
// Another thread beat us to it,
// free the page we just allocated and continue.
mi.phys.free(new_table_pa);
mmp.phys.free(new_table_pa);
} else {
const new_table_ptr: [*]u64 = @ptrFromInt(mm.p2v(new_table_pa));
@memset(new_table_ptr[0..512], 0);
table[idx] = mi.impl.make_table_pte(new_table_pa);
table[idx] = mmp.impl.make_table_pte(new_table_pa);
entry = table[idx];
}
}
src/mm/tlb.zig
+2 -2
View File
@@ -6,7 +6,7 @@ const rtl = @import("rtl");
const ke = r.ke;
const mm = r.mm;
const mi = mm.private;
const mmp = mm.private;
const ps = r.ps;
const ex = r.ex;
@@ -23,7 +23,7 @@ fn reclaim_state(obj: *anyopaque, _: ?*anyopaque) void {
const state: *ke.ShootdownState = @fieldParentPtr("link", link);
const space: *mm.Space = @ptrFromInt(state.payload[0]);
const pfn_list: mi.PfnList = @bitCast(state.payload[1]);
const pfn_list: mmp.PfnList = @bitCast(state.payload[1]);
const base = state.base;
const npages = state.npages;
src/mm/vmem.zig
+2 -2
View File
@@ -58,7 +58,7 @@
const std = @import("std");
const rtl = @import("rtl");
const mm = @import("root").mm;
const mi = mm.private;
const mmp = mm.private;
pub const Policy = enum {
/// Use the smallest free segment that can satisfy the request.
@@ -127,7 +127,7 @@ const Segment = struct {
/// address space.
const freelist_count = @bitSizeOf(usize);
var seg_zone: mi.zone.TypedZone(Segment) = undefined;
var seg_zone: mmp.zone.TypedZone(Segment) = undefined;
pub const Arena = struct {
/// Name of the arena, for debugging purposes.
src/mm/zone.zig
+7 -7
View File
@@ -115,7 +115,7 @@ const config = @import("config");
const ke = r.ke;
const mm = r.mm;
const ex = r.ex;
const mi = mm.private;
const mmp = mm.private;
/// Interval at which we do housekeeping (working set update, reaping, etc.)
pub const update_interval_s = 15;
@@ -907,7 +907,7 @@ pub const Zone = struct {
@intFromPtr(obj),
mm.page_size,
);
const phys = mi.kernel_space.pmap.query(page_va).?;
const phys = mmp.kernel_space.pmap.query(page_va).?;
const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys));
break :blk page.alloced.slab_data.slab;
} else blk: {
@@ -1227,7 +1227,7 @@ pub const Zone = struct {
}
fn alloc_page(policy: mm.WaitPolicy) !*anyopaque {
const alloc_ret = mi.phys.alloc_opts(.{ .policy = policy }) orelse
const alloc_ret = mmp.phys.alloc_opts(.{ .policy = policy }) orelse
return error.OutOfMemory;
return @ptrFromInt(mm.p2v(alloc_ret));
}
@@ -1235,7 +1235,7 @@ pub const Zone = struct {
fn free_page(p: *anyopaque) void {
const phys = mm.v2p(@intFromPtr(p));
mi.phys.free(phys);
mmp.phys.free(phys);
}
fn slab_create_small(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab {
@@ -1273,7 +1273,7 @@ pub const Zone = struct {
fn slab_create_large(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab {
const capacity = (self.slab_size) / self.chunk_size;
const buf = try mi.heap.alloc(self.slab_size, policy);
const buf = try mmp.heap.alloc(self.slab_size, policy);
var ret: *Slab = @ptrCast(
@alignCast(
@@ -1296,7 +1296,7 @@ pub const Zone = struct {
ret.alloc_rr = 0;
for (0..self.slab_size / mm.page_size) |i| {
const phys_page = mi.kernel_space.pmap.query(
const phys_page = mmp.kernel_space.pmap.query(
@intFromPtr(buf) + i * mm.page_size,
) orelse @panic("Could not query page");
const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys_page));
@@ -1650,7 +1650,7 @@ fn gpa_alloc(
if (len > 2048) {
if (ptr_align.toByteUnits() > mm.page_size) return null;
const pages = std.mem.alignForward(usize, len, mm.page_size);
const ptr = mi.heap.alloc(pages, .WaitForMemory) catch return null;
const ptr = mmp.heap.alloc(pages, .WaitForMemory) catch return null;
return @ptrCast(ptr);
}
src/platform/pc/impl.zig
+1 -1
View File
@@ -7,7 +7,7 @@ const pvclock = @import("pvclock.zig");
pub const apic = @import("apic.zig");
pub const smp = @import("smp.zig");
const ki = r.ke.private;
const kep = r.ke.private;
pub const name: []const u8 = "PC";
src/platform/pc/smp.zig
+7 -7
View File
@@ -5,7 +5,7 @@ const rtl = @import("rtl");
const config = @import("config");
const mm = r.mm;
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
const amd64 = @import("arch");
const log = std.log.scoped(.smp);
@@ -34,7 +34,7 @@ const ApData = extern struct {
};
fn ap_entry(cpu_id: u32) callconv(.c) noreturn {
ki.impl.init.ap_entry(cpu_id, &aps_booted);
kep.impl.init.ap_entry(cpu_id, &aps_booted);
}
fn make_thread(
@@ -96,12 +96,12 @@ pub fn init() linksection(r.init) void {
@panic("Failed to allocate AP local data offsets");
// Allocate per-cpu offsets for CPU-local data.
ki.impl.cpu_offsets = @ptrCast(offsets);
kep.impl.cpu_offsets = @ptrCast(offsets);
const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start);
log.info("per-CPU data size: {} bytes", .{percpu_size});
ki.impl.cpu_offsets[0] = 0;
kep.impl.cpu_offsets[0] = 0;
cpu_id_to_apic_id[0] = apic.get_id();
@@ -125,16 +125,16 @@ pub fn init() linksection(r.init) void {
@as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size],
);
const self_offset_offset = @intFromPtr(&ki.impl.cpu_self_offset) -
const self_offset_offset = @intFromPtr(&kep.impl.cpu_self_offset) -
@intFromPtr(&__percpu_start);
ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -%
kep.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -%
@intFromPtr(&__percpu_start);
const off: *usize = @ptrCast(@alignCast(&cpu_data[self_offset_offset]));
// copy the offset into the AP's self_offset variable
off.* = ki.impl.cpu_offsets[cpu_id];
off.* = kep.impl.cpu_offsets[cpu_id];
const stack_size = r.kib(16);
src/proc/private.zig
+3 -3
View File
@@ -1,13 +1,13 @@
const r = @import("root");
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
const mm = r.mm;
pub const thread = @import("thread.zig");
pub var turnstile_zone: mm.zone.TypedZone(ki.turnstile.Turnstile) = undefined;
pub var turnstile_zone: mm.zone.TypedZone(kep.turnstile.Turnstile) = undefined;
fn turnstile_ctor(ts: *ki.turnstile.Turnstile) void {
fn turnstile_ctor(ts: *kep.turnstile.Turnstile) void {
ts.* = .{
.link = undefined,
.next_free = null,
src/proc/thread.zig
+1 -1
View File
@@ -5,7 +5,7 @@ const r = @import("root");
const ex = r.ex;
const psp = r.ps.private;
const ke = r.ke;
const ki = ke.private;
const kep = ke.private;
const mm = r.mm;
pub const kernel_thread_stack_size = r.kib(16);
src/tests/smr.zig
+5 -6
View File
@@ -5,7 +5,7 @@ const ke = r.ke;
const mm = r.mm;
const ps = r.ps;
const ki = ke.private;
const kep = ke.private;
const nslots = 128;
const obj_words = 8;
@@ -50,7 +50,7 @@ fn reader(param: ?*anyopaque) void {
var iter: u64 = 0;
while (true) : (iter += 1) {
const ipl = ki.smr.enter(dom);
const ipl = kep.smr.enter(dom);
var done: u64 = 0;
for (0..reads_per_section) |_| {
@@ -68,7 +68,7 @@ fn reader(param: ?*anyopaque) void {
done += 1;
}
ki.smr.exit(dom, ipl);
kep.smr.exit(dom, ipl);
_ = reads.fetchAdd(done, .monotonic);
@@ -135,9 +135,8 @@ fn poller(_: ?*anyopaque) void {
}
fn spawn(entry: *const fn (?*anyopaque) void, id: usize) void {
const t = ps.thread.create_kernel(.Default, .{.func = entry, .arg = @ptrFromInt(id + 1)}, true)
catch
@panic("oom");
const t = ps.thread.create_kernel(.Default, .{ .func = entry, .arg = @ptrFromInt(id + 1) }, true) catch
@panic("oom");
ke.sched.enqueue(&t.kern);
}
src/tests/smr_pi.zig
+12 -12
View File
@@ -9,7 +9,7 @@ const mm = r.mm;
const ps = r.ps;
const rtl = @import("rtl");
const ki = ke.private;
const kep = ke.private;
const prio_reader: u8 = 4;
const prio_writer: u8 = 6;
@@ -90,8 +90,8 @@ fn reader(param: ?*anyopaque) void {
while (true) {
park_point();
var tracker: ki.smr.Tracker = undefined;
ki.smr.enter_preempt(dom, &tracker);
var tracker: kep.smr.Tracker = undefined;
kep.smr.enter_preempt(dom, &tracker);
var done: u64 = 0;
for (0..reads_per_section) |_| {
@@ -113,7 +113,7 @@ fn reader(param: ?*anyopaque) void {
// land on the stalled list. A blocked poller must boost us out of here.
spin_us(section_us);
ki.smr.exit_preempt(dom, &tracker);
kep.smr.exit_preempt(dom, &tracker);
_ = reads.fetchAdd(done, .monotonic);
}
@@ -175,13 +175,13 @@ fn cur_inherited(td: *ke.Thread) u8 {
}
/// Highest priority among the threads blocked on `ts`, or null if none.
fn blocked_prio(ts: *ki.turnstile.Turnstile) ?u8 {
fn blocked_prio(ts: *kep.turnstile.Turnstile) ?u8 {
var best: ?u8 = null;
for (&ts.queues) |*q| {
var it = q.iterator();
while (it.next()) : (it.advance()) {
const w: *ki.turnstile.Waiter = @fieldParentPtr("link", it.get());
const w: *kep.turnstile.Waiter = @fieldParentPtr("link", it.get());
best = @max(best orelse 0, cur_prio(w.thread));
}
}
@@ -195,7 +195,7 @@ fn check_cpu(cpu: *ke.smr.Cpu) void {
const ipl = ke.ipl.raise(.Dispatch);
cpu.stall_lock.acquire_no_ipl();
const turnstile = ki.turnstile.lookup(cpu);
const turnstile = kep.turnstile.lookup(cpu);
if (turnstile) |ts| {
if (blocked_prio(ts)) |want| {
@@ -203,8 +203,8 @@ fn check_cpu(cpu: *ke.smr.Cpu) void {
var it = cpu.stalled.iterator();
while (it.next()) : (it.advance()) {
const owner: *ki.turnstile.Owner = @fieldParentPtr("link", it.get());
const tracker: *ki.smr.Tracker = @fieldParentPtr("owner", owner);
const owner: *kep.turnstile.Owner = @fieldParentPtr("link", it.get());
const tracker: *kep.smr.Tracker = @fieldParentPtr("owner", owner);
const td = tracker.owner.thread;
stalled += 1;
@@ -226,7 +226,7 @@ fn check_cpu(cpu: *ke.smr.Cpu) void {
}
}
ki.turnstile.exit(cpu);
kep.turnstile.exit(cpu);
cpu.stall_lock.release_no_ipl();
ke.ipl.lower(ipl);
}
@@ -279,13 +279,13 @@ fn quiesce_check() void {
}
fn spawn(prio: u8, entry: *const fn (?*anyopaque) void, id: usize) *ps.Thread {
const t = ps.thread.create_kernel(@enumFromInt(prio), .{.func = entry, .arg= @ptrFromInt(id + 1)}, true) catch @panic("oom");
const t = ps.thread.create_kernel(@enumFromInt(prio), .{ .func = entry, .arg = @ptrFromInt(id + 1) }, true) catch @panic("oom");
ke.sched.enqueue(&t.kern);
return t;
}
pub fn start(_: ?*anyopaque) void {
std.log.info("{}", .{@sizeOf(ki.smr.Tracker)});
std.log.info("{}", .{@sizeOf(kep.smr.Tracker)});
dom = mm.zone.smr_domain_create(true) catch @panic("failed to create SMR domain");
zone.init("smr-pi", @sizeOf(Object), .{