mirrors/Zag
1
0
Fork 0
mirror of https://github.com/rdmsr/zag.git synced 2026-11-01 23:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

all: standardize suffixes

Browse Source 
ki -> kep
mi -> mmp
This commit is contained in:
Loïc Grégoire
2026-09-02 22:20:25 -04:00
parent cf58ad369e
commit 497e08bd7b
37 changed files with 244 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/kern/amd64/impl.zig
+2 -2
View File
@@ -1,6 +1,6 @@
const r = @import("root"); const r = @import("root");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
const pl = r.pl; const pl = r.pl;
const amd64 = r.arch; const amd64 = r.arch;
@@ -52,7 +52,7 @@ extern fn amd64_call_continuation(
) callconv(.c) noreturn; ) callconv(.c) noreturn;
export fn ki_free_old_stack(stack_top: usize) callconv(.c) void { export fn ki_free_old_stack(stack_top: usize) callconv(.c) void {
ki.thread.stack_cache_free(stack_top); kep.thread.stack_cache_free(stack_top);
} }
/// Trampoline to match the zig ABI. /// Trampoline to match the zig ABI.
src/kern/amd64/init.zig
+5 -5
View File
@@ -4,7 +4,7 @@ const r = @import("root");
const std = @import("std"); const std = @import("std");
const impl = @import("impl.zig"); const impl = @import("impl.zig");
const ke = r.ke; const ke = r.ke;
const ki = r.ke.private; const kep = r.ke.private;
const pl = r.pl; const pl = r.pl;
var gdt = extern struct { var gdt = extern struct {
@@ -85,13 +85,13 @@ fn early_cpu_init() linksection(r.init) void {
} }
pub fn ap_entry(cpu_id: u32, booted: *std.atomic.Value(usize)) noreturn { pub fn ap_entry(cpu_id: u32, booted: *std.atomic.Value(usize)) noreturn {
amd64.write_msr(.GsBase, ki.impl.cpu_offsets[cpu_id]); amd64.write_msr(.GsBase, kep.impl.cpu_offsets[cpu_id]);
early_cpu_init(); early_cpu_init();
ki.cpu.init_cpu(cpu_id); kep.cpu.init_cpu(cpu_id);
pl.impl.init_ap(); pl.impl.init_ap();
const sched = ki.sched.percpu.local(); const sched = kep.sched.percpu.local();
sched.current_thread = pl.impl.smp.start_thread.local().*; sched.current_thread = pl.impl.smp.start_thread.local().*;
sched.idle_thread = sched.current_thread; sched.idle_thread = sched.current_thread;
@@ -101,7 +101,7 @@ pub fn ap_entry(cpu_id: u32, booted: *std.atomic.Value(usize)) noreturn {
_ = booted.fetchAdd(1, .monotonic); _ = booted.fetchAdd(1, .monotonic);
ke.ipl.lower(.Passive); ke.ipl.lower(.Passive);
ki.sched.idle(null); kep.sched.idle(null);
} }
var initial_offsets: [1]usize = .{0}; var initial_offsets: [1]usize = .{0};
src/kern/amd64/int.zig
+8 -8
View File
@@ -2,9 +2,9 @@ const amd64 = @import("arch");
const std = @import("std"); const std = @import("std");
const r = @import("root"); const r = @import("root");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
const pl = r.pl; const pl = r.pl;
const mi = r.mm.private; const mmp = r.mm.private;
extern const __interrupt_vectors: [256]usize; extern const __interrupt_vectors: [256]usize;
@@ -49,7 +49,7 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
const ipl = ke.ipl.set_hardware(.Device); const ipl = ke.ipl.set_hardware(.Device);
if (frame.intno < 32) { if (frame.intno < 32) {
ki.panic.panic_lock.acquire_no_ipl(); kep.panic.panic_lock.acquire_no_ipl();
std.log.err("Unhandled exception: 0x{x} ({s}), err=0x{x}, pc=0x{x}", .{ frame.intno, exception_msg[frame.intno], frame.errcode, frame.rip }); std.log.err("Unhandled exception: 0x{x} ({s}), err=0x{x}, pc=0x{x}", .{ frame.intno, exception_msg[frame.intno], frame.errcode, frame.rip });
std.log.err("RAX=0x{x:0>16} RBX=0x{x:0>16} RCX=0x{x:0>16} RDX=0x{x:0>16}", .{ frame.rax, frame.rbx, frame.rcx, frame.rdx }); std.log.err("RAX=0x{x:0>16} RBX=0x{x:0>16} RCX=0x{x:0>16} RDX=0x{x:0>16}", .{ frame.rax, frame.rbx, frame.rcx, frame.rdx });
std.log.err("RSI=0x{x:0>16} RDI=0x{x:0>16} RBP=0x{x:0>16} RSP=0x{x:0>16}", .{ frame.rsi, frame.rdi, frame.rbp, frame.rsp }); std.log.err("RSI=0x{x:0>16} RDI=0x{x:0>16} RBP=0x{x:0>16} RSP=0x{x:0>16}", .{ frame.rsi, frame.rdi, frame.rbp, frame.rsp });
@@ -78,16 +78,16 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
std.log.err("CR2=0x{x:0>16} CR3=0x{x:0>16} RFLAGS={s}", .{ cr2, cr3, buf[0..writer.end] }); std.log.err("CR2=0x{x:0>16} CR3=0x{x:0>16} RFLAGS={s}", .{ cr2, cr3, buf[0..writer.end] });
ki.panic.panic_with_frame("Unhandled exception", frame.rbp); kep.panic.panic_with_frame("Unhandled exception", frame.rbp);
} }
if (frame.intno == 32) { if (frame.intno == 32) {
ki.timer.clock(); kep.timer.clock();
} }
// Platform IPI number. // Platform IPI number.
if (frame.intno == 240) { if (frame.intno == 240) {
ki.ipi.ipi_handler(); kep.ipi.ipi_handler();
} }
if (frame.intno >= 32) { if (frame.intno >= 32) {
@@ -97,9 +97,9 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
_ = ke.ipl.set_hardware(ipl); _ = ke.ipl.set_hardware(ipl);
if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and
ki.ipl.is_softint_pending(.Dispatch)) kep.ipl.is_softint_pending(.Dispatch))
{ {
ki.dpc.dispatch(); kep.dpc.dispatch();
} }
} }
src/kern/cpu.zig
+6 -6
View File
@@ -3,7 +3,7 @@ const rtl = @import("rtl");
const r = @import("root"); const r = @import("root");
const ke = r.ke; const ke = r.ke;
const ki = r.ke.private; const kep = r.ke.private;
const id = CpuLocal(u32, 0); const id = CpuLocal(u32, 0);
@@ -29,12 +29,12 @@ pub fn CpuLocal(comptime T: type, comptime init: T) type {
/// Return a pointer to local CPU data. /// Return a pointer to local CPU data.
pub fn local() *T { pub fn local() *T {
return ki.impl.percpu_ptr(&storage); return kep.impl.percpu_ptr(&storage);
} }
/// Return a pointer to remote CPU data. /// Return a pointer to remote CPU data.
pub fn remote(cpu: u32) *T { pub fn remote(cpu: u32) *T {
return ki.impl.percpu_ptr_other(&storage, cpu); return kep.impl.percpu_ptr_other(&storage, cpu);
} }
}; };
} }
@@ -48,11 +48,11 @@ pub fn ExportedCpuLocal(
const S = struct { const S = struct {
var storage: T linksection(".data.percpu") = init; var storage: T linksection(".data.percpu") = init;
pub fn local() @TypeOf(ki.impl.percpu_ptr(&storage)) { pub fn local() @TypeOf(kep.impl.percpu_ptr(&storage)) {
return ki.impl.percpu_ptr(&storage); return kep.impl.percpu_ptr(&storage);
} }
pub fn remote(cpu: u32) *T { pub fn remote(cpu: u32) *T {
return ki.impl.percpu_ptr_other(&storage, cpu); return kep.impl.percpu_ptr_other(&storage, cpu);
} }
}; };
@export(&S.storage, .{ .name = name, .linkage = .strong }); @export(&S.storage, .{ .name = name, .linkage = .strong });
src/kern/dpc.zig
+15 -15
View File
@@ -3,7 +3,7 @@ const config = @import("config");
const rtl = @import("rtl"); const rtl = @import("rtl");
const r = @import("root"); const r = @import("root");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
/// Deferred Procedure Call (DPC) structure. /// Deferred Procedure Call (DPC) structure.
/// Used for scheduling work to be done when IPL is lowered below `.Dispatch`. /// Used for scheduling work to be done when IPL is lowered below `.Dispatch`.
@@ -63,7 +63,7 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void {
dpc_cpu.queue.insert_tail(&dpc.link); dpc_cpu.queue.insert_tail(&dpc.link);
// Mark the DPC as pending on this CPU // Mark the DPC as pending on this CPU
ki.ipl.set_softint_pending(mycpu, .Dispatch); kep.ipl.set_softint_pending(mycpu, .Dispatch);
dpc_cpu.lock.release_no_ipl(); dpc_cpu.lock.release_no_ipl();
} }
@@ -72,17 +72,17 @@ pub fn enqueue(dpc: *Dpc, arg: ?*anyopaque) void {
} }
fn dispatch_queue(cpu: u32) void { fn dispatch_queue(cpu: u32) void {
const ipl_cpu = ki.ipl.cpu_ipl.local(); const ipl_cpu = kep.ipl.cpu_ipl.local();
const dpc_cpu = pcpu.local(); const dpc_cpu = pcpu.local();
const sched_cpu = ki.sched.percpu.local(); const sched_cpu = kep.sched.percpu.local();
ipl_cpu.* = .Dispatch; ipl_cpu.* = .Dispatch;
// Mark as handled, this must be done before enabling interrupts or we will // Mark as handled, this must be done before enabling interrupts or we will
// race. // race.
ki.ipl.clear_softint_pending(cpu, .Dispatch); kep.ipl.clear_softint_pending(cpu, .Dispatch);
_ = ki.impl.enable_interrupts(); _ = kep.impl.enable_interrupts();
while (true) { while (true) {
const ipl = dpc_cpu.lock.acquire_at(.High); const ipl = dpc_cpu.lock.acquire_at(.High);
@@ -133,35 +133,35 @@ fn dispatch_queue(cpu: u32) void {
sched_cpu.start_timer = false; sched_cpu.start_timer = false;
sched_cpu.preemption_reason = .None; sched_cpu.preemption_reason = .None;
ki.shootdown.process_shootdowns(); kep.shootdown.process_shootdowns();
if (sched_cpu.next_thread != null) { if (sched_cpu.next_thread != null) {
const curtd = sched_cpu.current_thread.?; const curtd = sched_cpu.current_thread.?;
if (!curtd.smr_sections.is_empty()) { if (!curtd.smr_sections.is_empty()) {
ki.smr.mark_thread_stalled(curtd); kep.smr.mark_thread_stalled(curtd);
} }
ki.sched.handle_preemption(sched_cpu); kep.sched.handle_preemption(sched_cpu);
} }
_ = ki.impl.disable_interrupts(); _ = kep.impl.disable_interrupts();
} }
/// Dispatch the DPC queue on the current CPU. /// Dispatch the DPC queue on the current CPU.
pub fn dispatch() void { pub fn dispatch() void {
// DPC processing is done at IPL Dispatch. // DPC processing is done at IPL Dispatch.
// Don't call lower/raise here because they might call us again recursively. // Don't call lower/raise here because they might call us again recursively.
const int_state = ki.impl.disable_interrupts(); const int_state = kep.impl.disable_interrupts();
const old_ipl = ki.ipl.cpu_ipl.local().*; const old_ipl = kep.ipl.cpu_ipl.local().*;
var mycpu = ke.cpu.current(); var mycpu = ke.cpu.current();
while (ki.ipl.is_softint_pending(.Dispatch)) { while (kep.ipl.is_softint_pending(.Dispatch)) {
dispatch_queue(mycpu); dispatch_queue(mycpu);
// Our CPU might have changed. // Our CPU might have changed.
mycpu = ke.cpu.current(); mycpu = ke.cpu.current();
} }
ki.ipl.cpu_ipl.local().* = old_ipl; kep.ipl.cpu_ipl.local().* = old_ipl;
ki.impl.restore_interrupts(int_state); kep.impl.restore_interrupts(int_state);
} }
src/kern/event.zig
+3 -3
View File
@@ -1,9 +1,9 @@
const ke = @import("root").ke; const ke = @import("root").ke;
const ki = ke.private; const kep = ke.private;
/// Generic waitable Event, exported for use everywhere in the kernel. /// Generic waitable Event, exported for use everywhere in the kernel.
pub const Event = struct { pub const Event = struct {
hdr: ki.wait.DispatchHeader, hdr: kep.wait.DispatchHeader,
const Self = @This(); const Self = @This();
@@ -26,7 +26,7 @@ pub const Event = struct {
pub fn signal(self: *Self) void { pub fn signal(self: *Self) void {
const ipl = self.hdr.lock.acquire(); const ipl = self.hdr.lock.acquire();
self.hdr.signaled = 1; self.hdr.signaled = 1;
ki.wait.satisfy_wait(&self.hdr); kep.wait.satisfy_wait(&self.hdr);
self.hdr.lock.release(ipl); self.hdr.lock.release(ipl);
} }
src/kern/init.zig
+9 -9
View File
@@ -3,7 +3,7 @@ const r = @import("root");
const pl = r.pl; const pl = r.pl;
const ke = r.ke; const ke = r.ke;
const arch = r.arch; const arch = r.arch;
const ki = ke.private; const kep = ke.private;
const ex = r.ex; const ex = r.ex;
var thread0: ke.Thread = undefined; var thread0: ke.Thread = undefined;
@@ -29,16 +29,16 @@ export fn kmain(boot_info: *r.BootInfo) callconv(.c) void {
fn init() linksection(r.init) void { fn init() linksection(r.init) void {
ke.ncpus = 1; ke.ncpus = 1;
ki.impl.early_init(); kep.impl.early_init();
ki.tunable.init(); kep.tunable.init();
std.log.info("Welcome to the machine", .{}); std.log.info("Welcome to the machine", .{});
pl.early_init(); pl.early_init();
ki.log.init(); kep.log.init();
ki.cpu.init_cpu(0); kep.cpu.init_cpu(0);
ki.turnstile.init_turnstiles(); kep.turnstile.init_turnstiles();
ki.sched.percpu.local().current_thread = &thread0; kep.sched.percpu.local().current_thread = &thread0;
ki.sched.percpu.local().idle_thread = &thread0; kep.sched.percpu.local().idle_thread = &thread0;
std.log.info("Zag for {s} ({s}), cmdline is \"{?s}\"", .{ std.log.info("Zag for {s} ({s}), cmdline is \"{?s}\"", .{
pl.name, pl.name,
@@ -47,5 +47,5 @@ fn init() linksection(r.init) void {
}); });
ex.init(); ex.init();
ki.sched.idle(null); kep.sched.idle(null);
} }
src/kern/ipi.zig
+2 -2
View File
@@ -4,7 +4,7 @@ const r = @import("root");
const std = @import("std"); const std = @import("std");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
const pl = r.pl; const pl = r.pl;
const State = enum(u8) { const State = enum(u8) {
@@ -138,7 +138,7 @@ pub fn ipi_handler() void {
_ = frozen_cpus.fetchAdd(1, .monotonic); _ = frozen_cpus.fetchAdd(1, .monotonic);
while (true) { while (true) {
ki.impl.halt(); kep.impl.halt();
} }
} }
src/kern/ipl.zig
+9 -9
View File
@@ -1,7 +1,7 @@
const std = @import("std"); const std = @import("std");
const r = @import("root"); const r = @import("root");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
/// Interrupt priority level (IPL) /// Interrupt priority level (IPL)
pub const Ipl = enum(u8) { pub const Ipl = enum(u8) {
@@ -44,7 +44,7 @@ pub fn current() Ipl {
pub fn raise(new: Ipl) Ipl { pub fn raise(new: Ipl) Ipl {
var old: ke.Ipl = undefined; var old: ke.Ipl = undefined;
if (ki.impl.raise_software_ipl(new, &old)) { if (kep.impl.raise_software_ipl(new, &old)) {
@branchHint(.unlikely); @branchHint(.unlikely);
std.debug.panic( std.debug.panic(
"ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})", "ke.ipl.raise(): Target IPL ({}) is lower than current IPL ({})",
@@ -55,7 +55,7 @@ pub fn raise(new: Ipl) Ipl {
if (@intFromEnum(new) > @intFromEnum(Ipl.get_max_software())) { if (@intFromEnum(new) > @intFromEnum(Ipl.get_max_software())) {
// Moving between hardware levels is rare. // Moving between hardware levels is rare.
@branchHint(.unlikely); @branchHint(.unlikely);
ki.impl.set_hardware_ipl(new); kep.impl.set_hardware_ipl(new);
} }
return old; return old;
@@ -66,7 +66,7 @@ pub fn raise(new: Ipl) Ipl {
pub fn lower(new: Ipl) void { pub fn lower(new: Ipl) void {
var old: ke.Ipl = undefined; var old: ke.Ipl = undefined;
if (ki.impl.lower_software_ipl(new, &old)) { if (kep.impl.lower_software_ipl(new, &old)) {
@branchHint(.unlikely); @branchHint(.unlikely);
std.debug.panic( std.debug.panic(
"ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})", "ke.ipl.lower(): Target IPL ({}) is higher than current IPL ({})",
@@ -78,27 +78,27 @@ pub fn lower(new: Ipl) void {
@intFromEnum(old) > @intFromEnum(Ipl.get_max_software())) @intFromEnum(old) > @intFromEnum(Ipl.get_max_software()))
{ {
@branchHint(.unlikely); @branchHint(.unlikely);
ki.impl.set_hardware_ipl(.Passive); kep.impl.set_hardware_ipl(.Passive);
} }
if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and if (@intFromEnum(new) < @intFromEnum(Ipl.Dispatch) and
is_softint_pending(.Dispatch)) is_softint_pending(.Dispatch))
{ {
// Dispatch DPCs if necessary. // Dispatch DPCs if necessary.
ki.dpc.dispatch(); kep.dpc.dispatch();
} }
} }
/// Set the hardware IPL to `new`. /// Set the hardware IPL to `new`.
pub fn set_hardware(new: Ipl) Ipl { pub fn set_hardware(new: Ipl) Ipl {
const ints = ki.impl.disable_interrupts(); const ints = kep.impl.disable_interrupts();
const cpu = cpu_ipl.local(); const cpu = cpu_ipl.local();
const old = cpu.*; const old = cpu.*;
cpu.* = new; cpu.* = new;
ki.impl.set_hardware_ipl(new); kep.impl.set_hardware_ipl(new);
ki.impl.restore_interrupts(ints); kep.impl.restore_interrupts(ints);
return old; return old;
} }
src/kern/log.zig
+2 -2
View File
@@ -4,7 +4,7 @@ const r = @import("root");
const pl = r.pl; const pl = r.pl;
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
var out_lock: ke.SpinLock = .init(); var out_lock: ke.SpinLock = .init();
@@ -41,7 +41,7 @@ const DebugWriter = struct {
// Messages are on average 2^5 = 32 bytes. // Messages are on average 2^5 = 32 bytes.
const avg_msg_size_bits = 5; const avg_msg_size_bits = 5;
pub var ringbuffer = ki.log_ring.RingBuffer( pub var ringbuffer = kep.log_ring.RingBuffer(
config.log_buffer_shift, config.log_buffer_shift,
avg_msg_size_bits, avg_msg_size_bits,
).init(); ).init();
src/kern/log_ring.zig
+3 -3
View File
@@ -71,7 +71,7 @@ const std = @import("std");
const r = @import("root"); const r = @import("root");
const rtl = @import("rtl"); const rtl = @import("rtl");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
const BlkPos = struct { const BlkPos = struct {
begin: usize, begin: usize,
@@ -645,7 +645,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
var reservation: Reservation = undefined; var reservation: Reservation = undefined;
reservation.irq_state = ki.impl.disable_interrupts(); reservation.irq_state = kep.impl.disable_interrupts();
const id = try self.reserve_desc(); const id = try self.reserve_desc();
@@ -703,7 +703,7 @@ pub fn RingBuffer(data_size_bits: usize, avg_msg_bits: usize) type {
); );
// Restore IRQ state after publishing the new entry. // Restore IRQ state after publishing the new entry.
ki.impl.restore_interrupts(res.irq_state); kep.impl.restore_interrupts(res.irq_state);
} }
/// Return the first readable sequence currently retained by the ring. /// Return the first readable sequence currently retained by the ring.
src/kern/mutex.zig
+9 -9
View File
@@ -1,6 +1,6 @@
const std = @import("std"); const std = @import("std");
const ke = @import("root").ke; const ke = @import("root").ke;
const ki = ke.private; const kep = ke.private;
/// Number of spins before blocking. /// Number of spins before blocking.
const optimistic_spins = 100; const optimistic_spins = 100;
@@ -19,7 +19,7 @@ pub const Mutex = struct {
pub fn acquire(m: *Mutex) void { pub fn acquire(m: *Mutex) void {
const ipl = ke.ipl.raise(.Dispatch); const ipl = ke.ipl.raise(.Dispatch);
const curtd = ki.sched.percpu.local().current_thread.?; const curtd = kep.sched.percpu.local().current_thread.?;
// Very fast path: the lock is uncontended and we can acquire it // Very fast path: the lock is uncontended and we can acquire it
//immediately. //immediately.
@@ -57,12 +57,12 @@ pub const Mutex = struct {
// Slow path: contended, block on a turnstile. // Slow path: contended, block on a turnstile.
while (true) { while (true) {
const ts = ki.turnstile.lookup(m); const ts = kep.turnstile.lookup(m);
const owner = m.owner.load(.monotonic); const owner = m.owner.load(.monotonic);
if (owner == null) { if (owner == null) {
// Lock was released between lookup and here. // Lock was released between lookup and here.
ki.turnstile.exit(m); kep.turnstile.exit(m);
if (m.owner.cmpxchgStrong( if (m.owner.cmpxchgStrong(
null, null,
curtd, curtd,
@@ -75,7 +75,7 @@ pub const Mutex = struct {
} }
// Block until woken. // Block until woken.
ki.turnstile.block( kep.turnstile.block(
ts, ts,
m, m,
.{ .single = owner.? }, .{ .single = owner.? },
@@ -99,12 +99,12 @@ pub const Mutex = struct {
pub fn release(m: *Mutex) void { pub fn release(m: *Mutex) void {
const ipl = ke.ipl.raise(.Dispatch); const ipl = ke.ipl.raise(.Dispatch);
const ts = ki.turnstile.lookup(m); const ts = kep.turnstile.lookup(m);
m.owner.store(null, .release); m.owner.store(null, .release);
if (ts == null) { if (ts == null) {
ki.turnstile.exit(m); kep.turnstile.exit(m);
ke.ipl.lower(ipl); ke.ipl.lower(ipl);
return; return;
} }
@@ -114,13 +114,13 @@ pub const Mutex = struct {
// to be better because this avoids lock convoys, // to be better because this avoids lock convoys,
// see this mysterious 70s paper: // see this mysterious 70s paper:
// https://dl.acm.org/doi/pdf/10.1145/850657.850659 // https://dl.acm.org/doi/pdf/10.1145/850657.850659
ki.turnstile.wakeup( kep.turnstile.wakeup(
ts.?, ts.?,
.Exclusive, .Exclusive,
ts.?.waiters, ts.?.waiters,
null, null,
); );
ki.turnstile.exit(m); kep.turnstile.exit(m);
ke.ipl.lower(ipl); ke.ipl.lower(ipl);
} }
src/kern/panic.zig
+6 -6
View File
@@ -3,7 +3,7 @@ const config = @import("config");
const r = @import("root"); const r = @import("root");
const ksyms = @import("ksyms"); const ksyms = @import("ksyms");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
extern var text_start_addr: u8; extern var text_start_addr: u8;
extern var text_end_addr: u8; extern var text_end_addr: u8;
@@ -77,7 +77,7 @@ pub fn panic_with_frame(
msg: []const u8, msg: []const u8,
frame: usize, frame: usize,
) noreturn { ) noreturn {
_ = ki.impl.disable_interrupts(); _ = kep.impl.disable_interrupts();
std.log.info("Crash count is {} on cpu {}", .{ crash_count.load(.monotonic), ke.cpu.current() }); std.log.info("Crash count is {} on cpu {}", .{ crash_count.load(.monotonic), ke.cpu.current() });
@@ -86,21 +86,21 @@ pub fn panic_with_frame(
// Another CPU has already crashed, just freeze manually right now. // Another CPU has already crashed, just freeze manually right now.
while (true) { while (true) {
ki.impl.halt(); kep.impl.halt();
} }
} }
// We're the first one who actually crashed, that means we get to handle // We're the first one who actually crashed, that means we get to handle
// things like dumping the system info and freezing other CPUs. Start by // things like dumping the system info and freezing other CPUs. Start by
// doing the freeze. // doing the freeze.
ki.ipi.freeze_cpus(); kep.ipi.freeze_cpus();
_ = ke.ipl.raise(.High); _ = ke.ipl.raise(.High);
std.log.err("KERNEL PANIC: {s} on CPU {}, curthread is {*}", .{ std.log.err("KERNEL PANIC: {s} on CPU {}, curthread is {*}", .{
msg, msg,
ke.cpu.current(), ke.cpu.current(),
ki.sched.percpu.local().current_thread.?, kep.sched.percpu.local().current_thread.?,
}); });
std.log.err("Stack trace:", .{}); std.log.err("Stack trace:", .{});
@@ -108,7 +108,7 @@ pub fn panic_with_frame(
walk_stack_frame(frame); walk_stack_frame(frame);
while (true) { while (true) {
ki.impl.halt(); kep.impl.halt();
} }
} }
src/kern/queue.zig
+6 -6
View File
@@ -8,11 +8,11 @@
const r = @import("root"); const r = @import("root");
const rtl = @import("rtl"); const rtl = @import("rtl");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
const std = @import("std"); const std = @import("std");
pub const Queue = struct { pub const Queue = struct {
hdr: ki.wait.DispatchHeader, hdr: kep.wait.DispatchHeader,
items: rtl.List, items: rtl.List,
active: usize, active: usize,
max_active: usize, max_active: usize,
@@ -49,14 +49,14 @@ pub const Queue = struct {
}, },
} }
ki.wait.satisfy_wait(&self.hdr); kep.wait.satisfy_wait(&self.hdr);
} }
/// Remove the item at the head. /// Remove the item at the head.
/// This blocks until an item is actually popped. /// This blocks until an item is actually popped.
pub fn remove(self: *Queue, timeout: ?r.Nanoseconds) !*rtl.List.Entry { pub fn remove(self: *Queue, timeout: ?r.Nanoseconds) !*rtl.List.Entry {
const ipl = self.hdr.lock.acquire(); const ipl = self.hdr.lock.acquire();
const td = ki.sched.percpu.local().current_thread.?; const td = kep.sched.percpu.local().current_thread.?;
if (td.queue) |q| { if (td.queue) |q| {
std.debug.assert(q == self); std.debug.assert(q == self);
@@ -64,7 +64,7 @@ pub const Queue = struct {
self.active -= 1; self.active -= 1;
if (self.hdr.signaled > 0 and self.active < self.max_active) { if (self.hdr.signaled > 0 and self.active < self.max_active) {
ki.wait.satisfy_wait(&self.hdr); kep.wait.satisfy_wait(&self.hdr);
} }
} else { } else {
td.queue = self; td.queue = self;
@@ -95,7 +95,7 @@ pub fn signal_wait(queue: *Queue) void {
queue.active -= 1; queue.active -= 1;
if (queue.hdr.signaled > 0 and queue.active < queue.max_active) { if (queue.hdr.signaled > 0 and queue.active < queue.max_active) {
ki.wait.satisfy_wait(&queue.hdr); kep.wait.satisfy_wait(&queue.hdr);
} }
queue.hdr.lock.release(ipl); queue.hdr.lock.release(ipl);
src/kern/sched.zig
+14 -14
View File
@@ -98,7 +98,7 @@ const rtl = @import("rtl");
const config = @import("config"); const config = @import("config");
const r = @import("root"); const r = @import("root");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
const pl = r.pl; const pl = r.pl;
const runqueues_n = 64; const runqueues_n = 64;
@@ -268,7 +268,7 @@ fn init_cpu() linksection(r.init) void {
.est_load_avg = .init(0), .est_load_avg = .init(0),
.migratable = .init(0), .migratable = .init(0),
.current_thread_prio = .init(0), .current_thread_prio = .init(0),
.resched_dpc = .init(ki.sched.clock), .resched_dpc = .init(kep.sched.clock),
.resched_timer = undefined, .resched_timer = undefined,
.start_timer = false, .start_timer = false,
.preemption_reason = .None, .preemption_reason = .None,
@@ -1034,7 +1034,7 @@ fn enqueue_on_cpu(c: u32, td: *ke.Thread) void {
cpu.preemption_reason = .HigherPriority; cpu.preemption_reason = .HigherPriority;
td.cpu = c; td.cpu = c;
td.state.store(.Selected, .monotonic); td.state.store(.Selected, .monotonic);
ki.ipl.set_softint_pending(c, .Dispatch); kep.ipl.set_softint_pending(c, .Dispatch);
cpu.current_thread_prio.store(td.priority, .monotonic); cpu.current_thread_prio.store(td.priority, .monotonic);
@@ -1056,19 +1056,19 @@ fn enqueue_on_cpu(c: u32, td: *ke.Thread) void {
if (cpu.resched_timer.state.load(.monotonic) == .Stopped) { if (cpu.resched_timer.state.load(.monotonic) == .Stopped) {
cpu.start_timer = true; cpu.start_timer = true;
ki.ipl.set_softint_pending(c, .Dispatch); kep.ipl.set_softint_pending(c, .Dispatch);
} }
// queues_lock dropped // queues_lock dropped
} }
fn allocate_stack(td: *ke.Thread) ?usize { fn allocate_stack(td: *ke.Thread) ?usize {
if (ki.thread.stack_cache_pop()) |stack| { if (kep.thread.stack_cache_pop()) |stack| {
return stack; return stack;
} }
// Enqueue the thread for allocation // Enqueue the thread for allocation
ki.thread.thread_stack_queue.insert(@ptrCast(&td.runq_link.next)); kep.thread.thread_stack_queue.insert(@ptrCast(&td.runq_link.next));
return null; return null;
} }
@@ -1122,13 +1122,13 @@ fn do_switch(cpu: *PerCpu, cur: *ke.Thread, next: *ke.Thread) void {
cur.switching.store(false, .release); cur.switching.store(false, .release);
cur.lock.release_no_ipl(); cur.lock.release_no_ipl();
ki.impl.call_continuation(&next.context, continuation); kep.impl.call_continuation(&next.context, continuation);
} }
if (relinquish) { if (relinquish) {
ki.impl.switch_cont_to_normal(&cur.context, &next.context); kep.impl.switch_cont_to_normal(&cur.context, &next.context);
} else { } else {
ki.impl.switch_normal_to_normal(&cur.context, &next.context); kep.impl.switch_normal_to_normal(&cur.context, &next.context);
} }
} }
@@ -1206,7 +1206,7 @@ pub fn handle_preemption(cpu: *PerCpu) void {
/// Called every time slice in DPC context. /// Called every time slice in DPC context.
pub fn clock(_: *ke.Dpc, _: ?*anyopaque) void { pub fn clock(_: *ke.Dpc, _: ?*anyopaque) void {
const cpu = percpu.local(); const cpu = percpu.local();
std.debug.assert(ki.ipl.current() == .Dispatch); std.debug.assert(kep.ipl.current() == .Dispatch);
const curtd = cpu.current_thread orelse return; const curtd = cpu.current_thread orelse return;
@@ -1369,7 +1369,7 @@ pub fn yield_locked(continuation: ?ke.Continuation) void {
cur.switching.store(false, .monotonic); cur.switching.store(false, .monotonic);
cur.lock.release_no_ipl(); cur.lock.release_no_ipl();
if (cont) |c| ki.impl.call_continuation(&cur.context, c); if (cont) |c| kep.impl.call_continuation(&cur.context, c);
} }
// cur lock dropped // cur lock dropped
@@ -1508,7 +1508,7 @@ pub fn update_priority_locked(td: *ke.Thread, new_prio: u8) void {
cpu.preemption_reason = .HigherPriority; cpu.preemption_reason = .HigherPriority;
cpu.current_thread_prio.store(next.priority, .monotonic); cpu.current_thread_prio.store(next.priority, .monotonic);
ki.ipl.set_softint_pending(c, .Dispatch); kep.ipl.set_softint_pending(c, .Dispatch);
if (c != ke.cpu.current()) { if (c != ke.cpu.current()) {
pl.send_ipi(c); pl.send_ipi(c);
@@ -1770,8 +1770,8 @@ pub fn idle(_: ?*anyopaque) noreturn {
} }
// Dispatch any pending DPCs. // Dispatch any pending DPCs.
if (ki.ipl.is_softint_pending(.Dispatch)) { if (kep.ipl.is_softint_pending(.Dispatch)) {
ki.dpc.dispatch(); kep.dpc.dispatch();
} }
std.atomic.spinLoopHint(); std.atomic.spinLoopHint();
src/kern/shootdown.zig
+7 -7
View File
@@ -5,7 +5,7 @@ const config = @import("config");
const rtl = @import("rtl"); const rtl = @import("rtl");
const r = @import("root"); const r = @import("root");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
const mm = r.mm; const mm = r.mm;
const pl = r.pl; const pl = r.pl;
@@ -87,14 +87,14 @@ fn allocate_slot(cpu: *PerCpu) ?usize {
} }
fn flush_range(va: r.VAddr, npages: usize) void { fn flush_range(va: r.VAddr, npages: usize) void {
if (npages > ki.impl.tlb_max_pages) { if (npages > kep.impl.tlb_max_pages) {
// Just flush the entire thing. // Just flush the entire thing.
ki.impl.flush_full_tlb(); kep.impl.flush_full_tlb();
return; return;
} }
for (0..npages) |i| { for (0..npages) |i| {
ki.impl.flush_tlb(va + i * mm.page_size); kep.impl.flush_tlb(va + i * mm.page_size);
} }
} }
@@ -148,12 +148,12 @@ pub fn process_shootdowns() void {
if (nreq == 0) return; if (nreq == 0) return;
// npages is just a hint, we don't need it to be accurate. // npages is just a hint, we don't need it to be accurate.
const full = cpu.npages.load(.monotonic) >= ki.impl.tlb_max_pages; const full = cpu.npages.load(.monotonic) >= kep.impl.tlb_max_pages;
cpu.npages.store(0, .monotonic); cpu.npages.store(0, .monotonic);
if (full) { if (full) {
// Just flush the entire thing. // Just flush the entire thing.
ki.impl.flush_full_tlb(); kep.impl.flush_full_tlb();
} }
// Go through every sender and release their states. // Go through every sender and release their states.
@@ -220,6 +220,6 @@ pub fn submit(state: ShootdownState, target_mask: ke.CpuMask) !void {
remote.senders.set(curcpu, .release); remote.senders.set(curcpu, .release);
rtl.barrier.fence(.release); rtl.barrier.fence(.release);
ki.ipl.set_softint_pending(@truncate(bit), .Dispatch); kep.ipl.set_softint_pending(@truncate(bit), .Dispatch);
} }
} }
src/kern/smr.zig
+14 -14
View File
@@ -7,7 +7,7 @@ const std = @import("std");
const config = @import("config"); const config = @import("config");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
pub const Sequence = u64; pub const Sequence = u64;
@@ -70,7 +70,7 @@ pub const Tracker = struct {
dom: *Domain, dom: *Domain,
seq: Sequence, seq: Sequence,
cpu: ?*Cpu, cpu: ?*Cpu,
owner: ki.turnstile.Owner, owner: kep.turnstile.Owner,
}; };
/// Used by tests. /// Used by tests.
@@ -152,18 +152,18 @@ fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence
cpu.stall_goal = goal; cpu.stall_goal = goal;
cpu.stall_lock.release_no_ipl(); cpu.stall_lock.release_no_ipl();
const ts = ki.turnstile.lookup(cpu); const ts = kep.turnstile.lookup(cpu);
seq = cpu.stall_seq.load(.monotonic); seq = cpu.stall_seq.load(.monotonic);
// Re-check under the chain lock. // Re-check under the chain lock.
if (seq == seq_invalid or goal <= seq) { if (seq == seq_invalid or goal <= seq) {
ki.turnstile.exit(cpu); kep.turnstile.exit(cpu);
ke.ipl.lower(ipl); ke.ipl.lower(ipl);
break; break;
} }
ki.turnstile.block( kep.turnstile.block(
ts, ts,
cpu, cpu,
.{ .shared = &cpu.stalled }, .{ .shared = &cpu.stalled },
@@ -354,7 +354,7 @@ pub fn mark_thread_stalled(td: *ke.Thread) void {
const entry = it.get(); const entry = it.get();
it.advance(); it.advance();
const owner: *ki.turnstile.Owner = @fieldParentPtr("link", entry); const owner: *kep.turnstile.Owner = @fieldParentPtr("link", entry);
const tracker: *Tracker = @fieldParentPtr("owner", owner); const tracker: *Tracker = @fieldParentPtr("owner", owner);
if (tracker.cpu == null) { if (tracker.cpu == null) {
@@ -370,16 +370,16 @@ pub fn mark_thread_stalled(td: *ke.Thread) void {
cpu.current_seq.store(seq_invalid, .release); cpu.current_seq.store(seq_invalid, .release);
const turnstile = ki.turnstile.lookup(cpu); const turnstile = kep.turnstile.lookup(cpu);
entry.remove(); entry.remove();
cpu.stalled.insert_tail(&tracker.owner.link); cpu.stalled.insert_tail(&tracker.owner.link);
if (turnstile) |ts| { if (turnstile) |ts| {
ki.turnstile.owner_enter(ts, &tracker.owner); kep.turnstile.owner_enter(ts, &tracker.owner);
} }
ki.turnstile.exit(cpu); kep.turnstile.exit(cpu);
cpu.stall_lock.release_no_ipl(); cpu.stall_lock.release_no_ipl();
} }
@@ -424,12 +424,12 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void {
// We got preempted. // We got preempted.
cpu.stall_lock.acquire_no_ipl(); cpu.stall_lock.acquire_no_ipl();
const turnstile = ki.turnstile.lookup(cpu); const turnstile = kep.turnstile.lookup(cpu);
// Remove ourselves from the stalled list. // Remove ourselves from the stalled list.
tracker.owner.link.remove(); tracker.owner.link.remove();
ki.turnstile.owner_leave(&tracker.owner); kep.turnstile.owner_leave(&tracker.owner);
var wake = false; var wake = false;
@@ -437,7 +437,7 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void {
cpu.stall_seq.store(seq_invalid, .release); cpu.stall_seq.store(seq_invalid, .release);
wake = true; wake = true;
} else { } else {
const first_owner: *ki.turnstile.Owner = @fieldParentPtr( const first_owner: *kep.turnstile.Owner = @fieldParentPtr(
"link", "link",
cpu.stalled.first(), cpu.stalled.first(),
); );
@@ -449,10 +449,10 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void {
if (wake) { if (wake) {
if (turnstile) |ts| if (turnstile) |ts|
ki.turnstile.wakeup(ts, .Exclusive, ts.waiters, null); kep.turnstile.wakeup(ts, .Exclusive, ts.waiters, null);
} }
ki.turnstile.exit(cpu); kep.turnstile.exit(cpu);
cpu.stall_lock.release_no_ipl(); cpu.stall_lock.release_no_ipl();
ke.ipl.lower(ipl); ke.ipl.lower(ipl);
src/kern/thread.zig
+14 -14
View File
@@ -3,7 +3,7 @@ const r = @import("root");
const rtl = @import("rtl"); const rtl = @import("rtl");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
pub const Priority = enum(u8) { pub const Priority = enum(u8) {
const Self = @This(); const Self = @This();
@@ -79,7 +79,7 @@ pub const Thread = struct {
pub const Context = extern struct { pub const Context = extern struct {
/// Implementation-dependent context. /// Implementation-dependent context.
impl: ki.impl.ThreadContext, impl: kep.impl.ThreadContext,
stack_top: usize, stack_top: usize,
@@ -127,20 +127,20 @@ pub const Thread = struct {
/// CPU this thread is enqueued on. /// CPU this thread is enqueued on.
cpu: ?u32, cpu: ?u32,
/// Run queue this thread is in /// Run queue this thread is in
runq: ?*ki.sched.RunQueue, runq: ?*kep.sched.RunQueue,
/// Index into the run queue this thread is currently in. /// Index into the run queue this thread is currently in.
/// Only valid when `runq` is non-null. /// Only valid when `runq` is non-null.
runq_idx: u8, runq_idx: u8,
/// Current wait status. /// Current wait status.
wait_status: std.atomic.Value(ki.wait.Status), wait_status: std.atomic.Value(kep.wait.Status),
waitblocks: [4]ki.wait.WaitBlock, waitblocks: [4]kep.wait.WaitBlock,
/// Reason for the wait, if any. /// Reason for the wait, if any.
wait_reason: ?[]const u8, wait_reason: ?[]const u8,
/// Timer used for timeouts. /// Timer used for timeouts.
timer: ke.Timer, timer: ke.Timer,
/// Turnstile. /// Turnstile.
turnstile: *ki.turnstile.Turnstile, turnstile: *kep.turnstile.Turnstile,
turnstile_waiter: ?*ki.turnstile.Waiter, turnstile_waiter: ?*kep.turnstile.Waiter,
turnstiles_owned: rtl.List, turnstiles_owned: rtl.List,
/// Object this thread is currently blocked on, or null. /// Object this thread is currently blocked on, or null.
waiting_on: ?*anyopaque, waiting_on: ?*anyopaque,
@@ -148,9 +148,9 @@ pub const Thread = struct {
queue: ?*ke.Queue, queue: ?*ke.Queue,
queue_item: ?*rtl.List.Entry, queue_item: ?*rtl.List.Entry,
/// PELT load average, /// PELT load average,
avg: ki.sched.Average, avg: kep.sched.Average,
/// Accounting statistics. /// Accounting statistics.
acct: ki.sched.Accounting, acct: kep.sched.Accounting,
/// Set whenever the thread is switching off its stack. /// Set whenever the thread is switching off its stack.
/// This is used to avoid taking thread next lock to wait for switch off /// This is used to avoid taking thread next lock to wait for switch off
/// to complete. /// to complete.
@@ -165,7 +165,7 @@ pub const Thread = struct {
/// Top of the stack for the thread. /// Top of the stack for the thread.
stack: r.VAddr, stack: r.VAddr,
/// Turnstile associated with the thread. /// Turnstile associated with the thread.
turnstile: *ki.turnstile.Turnstile, turnstile: *kep.turnstile.Turnstile,
/// Thread's priority. /// Thread's priority.
priority: Priority, priority: Priority,
}; };
@@ -272,13 +272,13 @@ pub fn exit() void {
// Reuse the runq linkage to put on reaper list. // Reuse the runq linkage to put on reaper list.
reaper_list.insert(@ptrCast(&curtd.runq_link.next)); reaper_list.insert(@ptrCast(&curtd.runq_link.next));
ki.sched.detach_load_avg(ki.sched.percpu.local(), curtd); kep.sched.detach_load_avg(kep.sched.percpu.local(), curtd);
ki.sched.yield_locked(Continuation.dummy); kep.sched.yield_locked(Continuation.dummy);
} }
/// Return the currently running thread. /// Return the currently running thread.
pub fn current() *ke.Thread { pub fn current() *ke.Thread {
return ki.sched.percpu.local().current_thread.?; return kep.sched.percpu.local().current_thread.?;
} }
// -- Stack allocation and continuations --------------------------------------- // -- Stack allocation and continuations ---------------------------------------
@@ -350,7 +350,7 @@ pub fn call_continuation(ptr: ?*anyopaque) noreturn {
const cont = td.continuation.?; const cont = td.continuation.?;
td.continuation = null; td.continuation = null;
ki.impl.call_continuation(&td.context, cont); kep.impl.call_continuation(&td.context, cont);
} }
/// Try to get a stack from the stack cache. /// Try to get a stack from the stack cache.
src/kern/time.zig
+1 -1
View File
@@ -2,7 +2,7 @@ const std = @import("std");
const rtl = @import("rtl"); const rtl = @import("rtl");
const r = @import("root"); const r = @import("root");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
/// TimeCounter structure. /// TimeCounter structure.
/// Represents a time-keeping source upon which the kernel can rely. /// Represents a time-keeping source upon which the kernel can rely.
src/kern/timer.zig
+6 -6
View File
@@ -5,12 +5,12 @@ const std = @import("std");
const rtl = @import("rtl"); const rtl = @import("rtl");
const r = @import("root"); const r = @import("root");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
const pl = r.pl; const pl = r.pl;
const PerCpu = struct { const PerCpu = struct {
/// Heap of pending timers on this CPU. /// Heap of pending timers on this CPU.
timers: rtl.PairingHeap(.min, ki.timer.cmp_timer), timers: rtl.PairingHeap(.min, kep.timer.cmp_timer),
/// Lock over the timer heap. /// Lock over the timer heap.
lock: ke.SpinLock, lock: ke.SpinLock,
dpc: ke.Dpc, dpc: ke.Dpc,
@@ -26,7 +26,7 @@ pub const Timer = struct {
Stopped, Stopped,
}; };
hdr: ki.wait.DispatchHeader, hdr: kep.wait.DispatchHeader,
/// Timer state. /// Timer state.
state: std.atomic.Value(State), state: std.atomic.Value(State),
@@ -150,12 +150,12 @@ pub fn clock() void {
// Check for overflows. // Check for overflows.
// This is fine to call a lot, as the function only gets expensive // This is fine to call a lot, as the function only gets expensive
// (i.e seqlock store) when an overflow actually happens. // (i.e seqlock store) when an overflow actually happens.
ki.time.update_overflow(); kep.time.update_overflow();
} }
// Called in a DPC when a timer has expired. // Called in a DPC when a timer has expired.
fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void { fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void {
std.debug.assert(ki.ipl.current() == .Dispatch); std.debug.assert(kep.ipl.current() == .Dispatch);
const cpu = percpu.local(); const cpu = percpu.local();
while (true) { while (true) {
@@ -217,7 +217,7 @@ fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void {
timer.state.store(.Stopped, .release); timer.state.store(.Stopped, .release);
// Wake whomever was waiting on the timer. // Wake whomever was waiting on the timer.
ki.wait.satisfy_wait(&timer.hdr); kep.wait.satisfy_wait(&timer.hdr);
timer.hdr.lock.release_no_ipl(); timer.hdr.lock.release_no_ipl();
if (maybe_dpc) |dpc| ke.dpc.enqueue(dpc, null); if (maybe_dpc) |dpc| ke.dpc.enqueue(dpc, null);
src/kern/turnstile.zig
+3 -3
View File
@@ -1,5 +1,5 @@
const ke = @import("root").ke; const ke = @import("root").ke;
const ki = ke.private; const kep = ke.private;
const rtl = @import("rtl"); const rtl = @import("rtl");
const std = @import("std"); const std = @import("std");
@@ -170,7 +170,7 @@ fn recompute_inherited(td: *ke.Thread) void {
fn update_prio(td: *ke.Thread) void { fn update_prio(td: *ke.Thread) void {
const pri = td.effective_priority(); const pri = td.effective_priority();
if (pri != td.priority) ki.sched.update_priority_locked(td, pri); if (pri != td.priority) kep.sched.update_priority_locked(td, pri);
} }
/// Donate priority `pri` to `to` along the edge `boost`. The caller holds /// Donate priority `pri` to `to` along the edge `boost`. The caller holds
@@ -392,7 +392,7 @@ pub fn block(
queue: Queue, queue: Queue,
) void { ) void {
const chain = chain_for(obj); const chain = chain_for(obj);
const curtd = ki.sched.percpu.local().current_thread.?; const curtd = kep.sched.percpu.local().current_thread.?;
const ipl = ke.ipl.current(); const ipl = ke.ipl.current();
var ts = turnstile; var ts = turnstile;
src/kern/wait.zig
+6 -6
View File
@@ -5,7 +5,7 @@ const rtl = @import("rtl");
const std = @import("std"); const std = @import("std");
const r = @import("root"); const r = @import("root");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
/// Header for waitable objects. /// Header for waitable objects.
/// This must be added to any structured which is considered waitable. /// This must be added to any structured which is considered waitable.
@@ -122,7 +122,7 @@ pub fn wait_one(object: *DispatchHeader, reason: []const u8, opts: Options) !usi
pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !usize { pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !usize {
const ipl = ke.ipl.raise(.Dispatch); const ipl = ke.ipl.raise(.Dispatch);
defer ke.ipl.lower(ipl); defer ke.ipl.lower(ipl);
const curtd = ki.sched.percpu.local().current_thread.?; const curtd = kep.sched.percpu.local().current_thread.?;
const obj_count = objects.len; const obj_count = objects.len;
const has_timeout = opts.timeout != null; const has_timeout = opts.timeout != null;
const blocks = opts.waitblocks orelse blk: { const blocks = opts.waitblocks orelse blk: {
@@ -222,13 +222,13 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
) == null) { ) == null) {
if (queue == null) { if (queue == null) {
if (curtd.queue) |q| { if (curtd.queue) |q| {
ki.queue.signal_wait(q); kep.queue.signal_wait(q);
} }
} }
curtd.wait_reason = reason; curtd.wait_reason = reason;
// We're good, now actually block. // We're good, now actually block.
ki.sched.block_locked(curtd, opts.continuation); kep.sched.block_locked(curtd, opts.continuation);
} else { } else {
queue = null; queue = null;
@@ -323,11 +323,11 @@ pub fn satisfy_wait(obj: *DispatchHeader) void {
if (obj.type != .Queue) { if (obj.type != .Queue) {
if (td.queue) |q| { if (td.queue) |q| {
ki.queue.signal_wake(q); kep.queue.signal_wake(q);
} }
} }
ki.sched.unblock_locked(td); kep.sched.unblock_locked(td);
td.lock.release(ipl); td.lock.release(ipl);
} }
src/mm/amd64/impl.zig
+3 -3
View File
@@ -1,7 +1,7 @@
const amd64 = @import("arch"); const amd64 = @import("arch");
const r = @import("root"); const r = @import("root");
const mm = r.mm; const mm = r.mm;
const mi = mm.private; const mmp = mm.private;
pub const hhdm_minimum_max_address = r.gib(4); pub const hhdm_minimum_max_address = r.gib(4);
@@ -17,7 +17,7 @@ pub const hhdm_base = 0xffff800000000000;
pub const kernel_heap_base = 0xffffc00000000000; pub const kernel_heap_base = 0xffffc00000000000;
pub const pfndb_base = 0xffffd00000000000; pub const pfndb_base = 0xffffd00000000000;
pub const levels = [_]mi.PMapLevel{ pub const levels = [_]mmp.PMapLevel{
.{ .shift = 12, .mask = 0x1ff, .leaf = true }, // 4K .{ .shift = 12, .mask = 0x1ff, .leaf = true }, // 4K
.{ .shift = 21, .mask = 0x1ff, .leaf = true }, // 2M .{ .shift = 21, .mask = 0x1ff, .leaf = true }, // 2M
.{ .shift = 30, .mask = 0x1ff, .leaf = true }, // 1G .{ .shift = 30, .mask = 0x1ff, .leaf = true }, // 1G
@@ -117,5 +117,5 @@ pub fn is_leaf_level_enabled(level: usize) bool {
} }
pub fn init_kernel() void { pub fn init_kernel() void {
mi.kernel_space.pmap.root_pa = amd64.read_cr(3); mmp.kernel_space.pmap.root_pa = amd64.read_cr(3);
} }
src/mm/balance.zig
+1 -1
View File
@@ -10,7 +10,7 @@ const std = @import("std");
const ps = r.ps; const ps = r.ps;
const ke = r.ke; const ke = r.ke;
const mm = r.mm; const mm = r.mm;
const mi = mm.private; const mmp = mm.private;
const balance_interval = ke.Tunable(u32, 1000, "mm.balance.interval_ms"); const balance_interval = ke.Tunable(u32, 1000, "mm.balance.interval_ms");
src/mm/heap.zig
+17 -17
View File
@@ -1,54 +1,54 @@
const r = @import("root"); const r = @import("root");
const ke = r.ke; const ke = r.ke;
const mm = r.mm; const mm = r.mm;
const mi = mm.private; const mmp = mm.private;
pub fn init() void { pub fn init() void {
mi.kernel_space.arena.init( mmp.kernel_space.arena.init(
"kernel heap", "kernel heap",
mi.impl.kernel_heap_base, mmp.impl.kernel_heap_base,
r.tib(16), r.tib(16),
mm.page_size, mm.page_size,
) catch @panic("failed to initialize kernel heap arena"); ) catch @panic("failed to initialize kernel heap arena");
} }
pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque { pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque {
mi.kernel_space.lock.acquire(); mmp.kernel_space.lock.acquire();
const addr = mi.kernel_space.arena.alloc(size, .{}) catch { const addr = mmp.kernel_space.arena.alloc(size, .{}) catch {
mi.kernel_space.lock.release(); mmp.kernel_space.lock.release();
return mm.Error.OutOfMemory; return mm.Error.OutOfMemory;
}; };
const npages = size / mm.page_size; const npages = size / mm.page_size;
for (0..npages) |i| { for (0..npages) |i| {
const pte = mi.pmap.wire_pte( const pte = mmp.pmap.wire_pte(
&mi.kernel_space, &mmp.kernel_space,
addr + (i * mm.page_size), addr + (i * mm.page_size),
policy, policy,
) catch { ) catch {
mi.kernel_space.lock.release(); mmp.kernel_space.lock.release();
return mm.Error.OutOfMemory; return mm.Error.OutOfMemory;
}; };
mi.kernel_space.lock.release(); mmp.kernel_space.lock.release();
const page = mi.phys.alloc_opts(.{ .policy = policy }) orelse { const page = mmp.phys.alloc_opts(.{ .policy = policy }) orelse {
return mm.Error.OutOfMemory; return mm.Error.OutOfMemory;
}; };
mi.kernel_space.lock.acquire(); mmp.kernel_space.lock.acquire();
pte.* = mi.impl.make_leaf_pte(page, .{ .read = true, .write = true }, 0); pte.* = mmp.impl.make_leaf_pte(page, .{ .read = true, .write = true }, 0);
} }
mi.kernel_space.lock.release(); mmp.kernel_space.lock.release();
return @ptrFromInt(addr); return @ptrFromInt(addr);
} }
pub fn free(va: r.VAddr, size: usize) void { pub fn free(va: r.VAddr, size: usize) void {
mi.kernel_space.lock.acquire(); mmp.kernel_space.lock.acquire();
mi.tlb.reclaim_range(&mi.kernel_space, va, size); mmp.tlb.reclaim_range(&mmp.kernel_space, va, size);
mi.kernel_space.lock.release(); mmp.kernel_space.lock.release();
} }
src/mm/init.zig
+8 -8
View File
@@ -1,18 +1,18 @@
const r = @import("root"); const r = @import("root");
const pl = r.pl; const pl = r.pl;
const mm = r.mm; const mm = r.mm;
const mi = mm.private; const mmp = mm.private;
const config = @import("config"); const config = @import("config");
pub fn init() linksection(r.init) void { pub fn init() linksection(r.init) void {
mi.phys.init(); mmp.phys.init();
mi.zone.early_init(); mmp.zone.early_init();
mi.vmem.init(); mmp.vmem.init();
mi.heap.init(); mmp.heap.init();
} }
pub fn late_init() linksection(r.init) void { pub fn late_init() linksection(r.init) void {
mi.zone.late_init(); mmp.zone.late_init();
mi.tlb.init(); mmp.tlb.init();
mi.balance.init(); mmp.balance.init();
} }
src/mm/phys.zig
+2 -2
View File
@@ -3,7 +3,7 @@ const std = @import("std");
const rtl = @import("rtl"); const rtl = @import("rtl");
const pl = r.pl; const pl = r.pl;
const mm = r.mm; const mm = r.mm;
const mi = mm.private; const mmp = mm.private;
const ke = r.ke; const ke = r.ke;
const log = std.log.scoped(.@"mm/phys"); const log = std.log.scoped(.@"mm/phys");
@@ -173,7 +173,7 @@ pub fn init() linksection(r.init) void {
// Get the kernel pagemap from the loader, // Get the kernel pagemap from the loader,
// we are already in it so no need to activate it. // we are already in it so no need to activate it.
mi.impl.init_kernel(); mmp.impl.init_kernel();
// Calculate the minimum amount of free memory we accept, // Calculate the minimum amount of free memory we accept,
// this formula is from Linux and does not grow linearly; // this formula is from Linux and does not grow linearly;
src/mm/pmap.zig
+23 -23
View File
@@ -4,7 +4,7 @@ const r = @import("root");
const rtl = @import("rtl"); const rtl = @import("rtl");
const std = @import("std"); const std = @import("std");
const mm = r.mm; const mm = r.mm;
const mi = mm.private; const mmp = mm.private;
/// Describes a contiguous region of physical memory to be mapped /// Describes a contiguous region of physical memory to be mapped
/// at a given virtual address. /// at a given virtual address.
@@ -33,7 +33,7 @@ const ContiguousSource = struct {
}; };
fn level_shift(level: usize) u6 { fn level_shift(level: usize) u6 {
return mi.impl.levels[level].shift; return mmp.impl.levels[level].shift;
} }
fn page_size(level: usize) usize { fn page_size(level: usize) usize {
@@ -41,24 +41,24 @@ fn page_size(level: usize) usize {
} }
fn index_for_level(va: usize, level: usize) usize { fn index_for_level(va: usize, level: usize) usize {
return (va >> level_shift(level)) & mi.impl.levels[level].mask; return (va >> level_shift(level)) & mmp.impl.levels[level].mask;
} }
pub const PMap = struct { pub const PMap = struct {
const Self = @This(); const Self = @This();
const num_levels = mi.impl.levels.len; const num_levels = mmp.impl.levels.len;
const entries_per_table = 512; const entries_per_table = 512;
const index_mask = entries_per_table - 1; const index_mask = entries_per_table - 1;
root_pa: r.PAddr, root_pa: r.PAddr,
fn is_leaf_level(level: usize) bool { fn is_leaf_level(level: usize) bool {
return mi.impl.levels[level].leaf; return mmp.impl.levels[level].leaf;
} }
fn leaf_level_enabled(level: usize) bool { fn leaf_level_enabled(level: usize) bool {
return is_leaf_level(level) and mi.impl.is_leaf_level_enabled(level); return is_leaf_level(level) and mmp.impl.is_leaf_level_enabled(level);
} }
fn cursor(self: *Self, va: r.VAddr) Cursor { fn cursor(self: *Self, va: r.VAddr) Cursor {
@@ -74,7 +74,7 @@ pub const PMap = struct {
} }
pub fn activate(self: *Self) void { pub fn activate(self: *Self) void {
mi.impl.activate(self.root_pa); mmp.impl.activate(self.root_pa);
} }
pub fn map_from(self: *Self, va: r.VAddr, size: usize, source: anytype) void { pub fn map_from(self: *Self, va: r.VAddr, size: usize, source: anytype) void {
@@ -129,7 +129,7 @@ pub const PMap = struct {
/// Unmap a contiguous range of virtual pages. /// Unmap a contiguous range of virtual pages.
/// Only small pages are supported. /// Only small pages are supported.
/// Return a pfn that points to the physical pages that were unmapped. /// Return a pfn that points to the physical pages that were unmapped.
pub fn unmap(self: *Self, va: r.VAddr, size: usize) ?mi.PfnList { pub fn unmap(self: *Self, va: r.VAddr, size: usize) ?mmp.PfnList {
std.debug.assert(std.mem.isAligned(va, mm.page_size)); std.debug.assert(std.mem.isAligned(va, mm.page_size));
var c = self.cursor(va); var c = self.cursor(va);
@@ -153,7 +153,7 @@ pub const PMap = struct {
const page = mm.pfn_to_struct_page(pfn); const page = mm.pfn_to_struct_page(pfn);
list.insert_tail(&page.free.link); list.insert_tail(&page.free.link);
c.tables[level][idx] = std.mem.zeroes(mi.impl.Pte); c.tables[level][idx] = std.mem.zeroes(mmp.impl.Pte);
break; break;
} }
@@ -182,7 +182,7 @@ pub const PMap = struct {
va: r.VAddr, va: r.VAddr,
/// Cached pointers to each level's page table, filled as the cursor /// Cached pointers to each level's page table, filled as the cursor
/// descends. Only indices in `[0, top_level]` are valid. /// descends. Only indices in `[0, top_level]` are valid.
tables: [num_levels][*]mi.impl.Pte, tables: [num_levels][*]mmp.impl.Pte,
/// Index of the highest level for which `tables` currently holds a /// Index of the highest level for which `tables` currently holds a
/// valid pointer. Descends toward 0 (the leaf) as `walk_down` runs. /// valid pointer. Descends toward 0 (the leaf) as `walk_down` runs.
/// Reset upward in `advance` when the VA crosses a level boundary. /// Reset upward in `advance` when the VA crosses a level boundary.
@@ -197,24 +197,24 @@ pub const PMap = struct {
while (current_level > target_level) { while (current_level > target_level) {
const idx = index_for_level(self.va, current_level); const idx = index_for_level(self.va, current_level);
var pte = mi.impl.Pte.load(&self.tables[current_level][idx]); var pte = mmp.impl.Pte.load(&self.tables[current_level][idx]);
if (!pte.present) { if (!pte.present) {
if (!allocate) { if (!allocate) {
return error.PageNotMapped; return error.PageNotMapped;
} }
const new_table_pa = mi.phys.alloc(); const new_table_pa = mmp.phys.alloc();
const table_ptr: [*]mi.impl.Pte = @ptrFromInt( const table_ptr: [*]mmp.impl.Pte = @ptrFromInt(
mm.p2v(new_table_pa), mm.p2v(new_table_pa),
); );
@memset( @memset(
table_ptr[0..entries_per_table], table_ptr[0..entries_per_table],
std.mem.zeroes(mi.impl.Pte), std.mem.zeroes(mmp.impl.Pte),
); );
pte = mi.impl.make_table_pte(new_table_pa); pte = mmp.impl.make_table_pte(new_table_pa);
self.tables[current_level][idx] = pte; self.tables[current_level][idx] = pte;
} }
@@ -247,7 +247,7 @@ pub const PMap = struct {
const table = self.tables[target_level]; const table = self.tables[target_level];
table[index_for_level(self.va, target_level)] = table[index_for_level(self.va, target_level)] =
mi.impl.make_leaf_pte(current_pa, flags, target_level); mmp.impl.make_leaf_pte(current_pa, flags, target_level);
const step = page_size(target_level); const step = page_size(target_level);
current_pa += step; current_pa += step;
@@ -279,14 +279,14 @@ pub const PMap = struct {
}; };
pub fn wire_pte( pub fn wire_pte(
space: *mi.Space, space: *mmp.Space,
va: r.VAddr, va: r.VAddr,
policy: mm.WaitPolicy, policy: mm.WaitPolicy,
) mm.Error!*mi.impl.Pte { ) mm.Error!*mmp.impl.Pte {
std.debug.assert(space.lock.is_locked()); std.debug.assert(space.lock.is_locked());
var table: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(mi.kernel_space.pmap.root_pa)); var table: [*]mmp.impl.Pte = @ptrFromInt(mm.p2v(mmp.kernel_space.pmap.root_pa));
var level = mi.impl.levels.len - 1; var level = mmp.impl.levels.len - 1;
while (true) { while (true) {
const idx = index_for_level(va, level); const idx = index_for_level(va, level);
@@ -302,7 +302,7 @@ pub fn wire_pte(
// since that may block. // since that may block.
space.lock.release(); space.lock.release();
const new_table_pa = mi.phys.alloc_opts(.{ .policy = policy }) orelse { const new_table_pa = mmp.phys.alloc_opts(.{ .policy = policy }) orelse {
space.lock.acquire(); space.lock.acquire();
return mm.Error.OutOfMemory; return mm.Error.OutOfMemory;
}; };
@@ -316,12 +316,12 @@ pub fn wire_pte(
if (entry.present) { if (entry.present) {
// Another thread beat us to it, // Another thread beat us to it,
// free the page we just allocated and continue. // free the page we just allocated and continue.
mi.phys.free(new_table_pa); mmp.phys.free(new_table_pa);
} else { } else {
const new_table_ptr: [*]u64 = @ptrFromInt(mm.p2v(new_table_pa)); const new_table_ptr: [*]u64 = @ptrFromInt(mm.p2v(new_table_pa));
@memset(new_table_ptr[0..512], 0); @memset(new_table_ptr[0..512], 0);
table[idx] = mi.impl.make_table_pte(new_table_pa); table[idx] = mmp.impl.make_table_pte(new_table_pa);
entry = table[idx]; entry = table[idx];
} }
} }
src/mm/tlb.zig
+2 -2
View File
@@ -6,7 +6,7 @@ const rtl = @import("rtl");
const ke = r.ke; const ke = r.ke;
const mm = r.mm; const mm = r.mm;
const mi = mm.private; const mmp = mm.private;
const ps = r.ps; const ps = r.ps;
const ex = r.ex; const ex = r.ex;
@@ -23,7 +23,7 @@ fn reclaim_state(obj: *anyopaque, _: ?*anyopaque) void {
const state: *ke.ShootdownState = @fieldParentPtr("link", link); const state: *ke.ShootdownState = @fieldParentPtr("link", link);
const space: *mm.Space = @ptrFromInt(state.payload[0]); const space: *mm.Space = @ptrFromInt(state.payload[0]);
const pfn_list: mi.PfnList = @bitCast(state.payload[1]); const pfn_list: mmp.PfnList = @bitCast(state.payload[1]);
const base = state.base; const base = state.base;
const npages = state.npages; const npages = state.npages;
src/mm/vmem.zig
+2 -2
View File
@@ -58,7 +58,7 @@
const std = @import("std"); const std = @import("std");
const rtl = @import("rtl"); const rtl = @import("rtl");
const mm = @import("root").mm; const mm = @import("root").mm;
const mi = mm.private; const mmp = mm.private;
pub const Policy = enum { pub const Policy = enum {
/// Use the smallest free segment that can satisfy the request. /// Use the smallest free segment that can satisfy the request.
@@ -127,7 +127,7 @@ const Segment = struct {
/// address space. /// address space.
const freelist_count = @bitSizeOf(usize); const freelist_count = @bitSizeOf(usize);
var seg_zone: mi.zone.TypedZone(Segment) = undefined; var seg_zone: mmp.zone.TypedZone(Segment) = undefined;
pub const Arena = struct { pub const Arena = struct {
/// Name of the arena, for debugging purposes. /// Name of the arena, for debugging purposes.
src/mm/zone.zig
+7 -7
View File
@@ -115,7 +115,7 @@ const config = @import("config");
const ke = r.ke; const ke = r.ke;
const mm = r.mm; const mm = r.mm;
const ex = r.ex; const ex = r.ex;
const mi = mm.private; const mmp = mm.private;
/// Interval at which we do housekeeping (working set update, reaping, etc.) /// Interval at which we do housekeeping (working set update, reaping, etc.)
pub const update_interval_s = 15; pub const update_interval_s = 15;
@@ -907,7 +907,7 @@ pub const Zone = struct {
@intFromPtr(obj), @intFromPtr(obj),
mm.page_size, mm.page_size,
); );
const phys = mi.kernel_space.pmap.query(page_va).?; const phys = mmp.kernel_space.pmap.query(page_va).?;
const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys)); const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys));
break :blk page.alloced.slab_data.slab; break :blk page.alloced.slab_data.slab;
} else blk: { } else blk: {
@@ -1227,7 +1227,7 @@ pub const Zone = struct {
} }
fn alloc_page(policy: mm.WaitPolicy) !*anyopaque { fn alloc_page(policy: mm.WaitPolicy) !*anyopaque {
const alloc_ret = mi.phys.alloc_opts(.{ .policy = policy }) orelse const alloc_ret = mmp.phys.alloc_opts(.{ .policy = policy }) orelse
return error.OutOfMemory; return error.OutOfMemory;
return @ptrFromInt(mm.p2v(alloc_ret)); return @ptrFromInt(mm.p2v(alloc_ret));
} }
@@ -1235,7 +1235,7 @@ pub const Zone = struct {
fn free_page(p: *anyopaque) void { fn free_page(p: *anyopaque) void {
const phys = mm.v2p(@intFromPtr(p)); const phys = mm.v2p(@intFromPtr(p));
mi.phys.free(phys); mmp.phys.free(phys);
} }
fn slab_create_small(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab { fn slab_create_small(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab {
@@ -1273,7 +1273,7 @@ pub const Zone = struct {
fn slab_create_large(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab { fn slab_create_large(self: *Self, policy: mm.WaitPolicy) mm.Error!*Slab {
const capacity = (self.slab_size) / self.chunk_size; const capacity = (self.slab_size) / self.chunk_size;
const buf = try mi.heap.alloc(self.slab_size, policy); const buf = try mmp.heap.alloc(self.slab_size, policy);
var ret: *Slab = @ptrCast( var ret: *Slab = @ptrCast(
@alignCast( @alignCast(
@@ -1296,7 +1296,7 @@ pub const Zone = struct {
ret.alloc_rr = 0; ret.alloc_rr = 0;
for (0..self.slab_size / mm.page_size) |i| { for (0..self.slab_size / mm.page_size) |i| {
const phys_page = mi.kernel_space.pmap.query( const phys_page = mmp.kernel_space.pmap.query(
@intFromPtr(buf) + i * mm.page_size, @intFromPtr(buf) + i * mm.page_size,
) orelse @panic("Could not query page"); ) orelse @panic("Could not query page");
const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys_page)); const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys_page));
@@ -1650,7 +1650,7 @@ fn gpa_alloc(
if (len > 2048) { if (len > 2048) {
if (ptr_align.toByteUnits() > mm.page_size) return null; if (ptr_align.toByteUnits() > mm.page_size) return null;
const pages = std.mem.alignForward(usize, len, mm.page_size); const pages = std.mem.alignForward(usize, len, mm.page_size);
const ptr = mi.heap.alloc(pages, .WaitForMemory) catch return null; const ptr = mmp.heap.alloc(pages, .WaitForMemory) catch return null;
return @ptrCast(ptr); return @ptrCast(ptr);
} }
src/platform/pc/impl.zig
+1 -1
View File
@@ -7,7 +7,7 @@ const pvclock = @import("pvclock.zig");
pub const apic = @import("apic.zig"); pub const apic = @import("apic.zig");
pub const smp = @import("smp.zig"); pub const smp = @import("smp.zig");
const ki = r.ke.private; const kep = r.ke.private;
pub const name: []const u8 = "PC"; pub const name: []const u8 = "PC";
src/platform/pc/smp.zig
+7 -7
View File
@@ -5,7 +5,7 @@ const rtl = @import("rtl");
const config = @import("config"); const config = @import("config");
const mm = r.mm; const mm = r.mm;
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
const amd64 = @import("arch"); const amd64 = @import("arch");
const log = std.log.scoped(.smp); const log = std.log.scoped(.smp);
@@ -34,7 +34,7 @@ const ApData = extern struct {
}; };
fn ap_entry(cpu_id: u32) callconv(.c) noreturn { fn ap_entry(cpu_id: u32) callconv(.c) noreturn {
ki.impl.init.ap_entry(cpu_id, &aps_booted); kep.impl.init.ap_entry(cpu_id, &aps_booted);
} }
fn make_thread( fn make_thread(
@@ -96,12 +96,12 @@ pub fn init() linksection(r.init) void {
@panic("Failed to allocate AP local data offsets"); @panic("Failed to allocate AP local data offsets");
// Allocate per-cpu offsets for CPU-local data. // Allocate per-cpu offsets for CPU-local data.
ki.impl.cpu_offsets = @ptrCast(offsets); kep.impl.cpu_offsets = @ptrCast(offsets);
const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start); const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start);
log.info("per-CPU data size: {} bytes", .{percpu_size}); log.info("per-CPU data size: {} bytes", .{percpu_size});
ki.impl.cpu_offsets[0] = 0; kep.impl.cpu_offsets[0] = 0;
cpu_id_to_apic_id[0] = apic.get_id(); cpu_id_to_apic_id[0] = apic.get_id();
@@ -125,16 +125,16 @@ pub fn init() linksection(r.init) void {
@as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size], @as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size],
); );
const self_offset_offset = @intFromPtr(&ki.impl.cpu_self_offset) - const self_offset_offset = @intFromPtr(&kep.impl.cpu_self_offset) -
@intFromPtr(&__percpu_start); @intFromPtr(&__percpu_start);
ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -% kep.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -%
@intFromPtr(&__percpu_start); @intFromPtr(&__percpu_start);
const off: *usize = @ptrCast(@alignCast(&cpu_data[self_offset_offset])); const off: *usize = @ptrCast(@alignCast(&cpu_data[self_offset_offset]));
// copy the offset into the AP's self_offset variable // copy the offset into the AP's self_offset variable
off.* = ki.impl.cpu_offsets[cpu_id]; off.* = kep.impl.cpu_offsets[cpu_id];
const stack_size = r.kib(16); const stack_size = r.kib(16);
src/proc/private.zig
+3 -3
View File
@@ -1,13 +1,13 @@
const r = @import("root"); const r = @import("root");
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
const mm = r.mm; const mm = r.mm;
pub const thread = @import("thread.zig"); pub const thread = @import("thread.zig");
pub var turnstile_zone: mm.zone.TypedZone(ki.turnstile.Turnstile) = undefined; pub var turnstile_zone: mm.zone.TypedZone(kep.turnstile.Turnstile) = undefined;
fn turnstile_ctor(ts: *ki.turnstile.Turnstile) void { fn turnstile_ctor(ts: *kep.turnstile.Turnstile) void {
ts.* = .{ ts.* = .{
.link = undefined, .link = undefined,
.next_free = null, .next_free = null,
src/proc/thread.zig
+1 -1
View File
@@ -5,7 +5,7 @@ const r = @import("root");
const ex = r.ex; const ex = r.ex;
const psp = r.ps.private; const psp = r.ps.private;
const ke = r.ke; const ke = r.ke;
const ki = ke.private; const kep = ke.private;
const mm = r.mm; const mm = r.mm;
pub const kernel_thread_stack_size = r.kib(16); pub const kernel_thread_stack_size = r.kib(16);
src/tests/smr.zig
+5 -6
View File
@@ -5,7 +5,7 @@ const ke = r.ke;
const mm = r.mm; const mm = r.mm;
const ps = r.ps; const ps = r.ps;
const ki = ke.private; const kep = ke.private;
const nslots = 128; const nslots = 128;
const obj_words = 8; const obj_words = 8;
@@ -50,7 +50,7 @@ fn reader(param: ?*anyopaque) void {
var iter: u64 = 0; var iter: u64 = 0;
while (true) : (iter += 1) { while (true) : (iter += 1) {
const ipl = ki.smr.enter(dom); const ipl = kep.smr.enter(dom);
var done: u64 = 0; var done: u64 = 0;
for (0..reads_per_section) |_| { for (0..reads_per_section) |_| {
@@ -68,7 +68,7 @@ fn reader(param: ?*anyopaque) void {
done += 1; done += 1;
} }
ki.smr.exit(dom, ipl); kep.smr.exit(dom, ipl);
_ = reads.fetchAdd(done, .monotonic); _ = reads.fetchAdd(done, .monotonic);
@@ -135,9 +135,8 @@ fn poller(_: ?*anyopaque) void {
} }
fn spawn(entry: *const fn (?*anyopaque) void, id: usize) void { fn spawn(entry: *const fn (?*anyopaque) void, id: usize) void {
const t = ps.thread.create_kernel(.Default, .{.func = entry, .arg = @ptrFromInt(id + 1)}, true) const t = ps.thread.create_kernel(.Default, .{ .func = entry, .arg = @ptrFromInt(id + 1) }, true) catch
catch @panic("oom");
@panic("oom");
ke.sched.enqueue(&t.kern); ke.sched.enqueue(&t.kern);
} }
src/tests/smr_pi.zig
+12 -12
View File
@@ -9,7 +9,7 @@ const mm = r.mm;
const ps = r.ps; const ps = r.ps;
const rtl = @import("rtl"); const rtl = @import("rtl");
const ki = ke.private; const kep = ke.private;
const prio_reader: u8 = 4; const prio_reader: u8 = 4;
const prio_writer: u8 = 6; const prio_writer: u8 = 6;
@@ -90,8 +90,8 @@ fn reader(param: ?*anyopaque) void {
while (true) { while (true) {
park_point(); park_point();
var tracker: ki.smr.Tracker = undefined; var tracker: kep.smr.Tracker = undefined;
ki.smr.enter_preempt(dom, &tracker); kep.smr.enter_preempt(dom, &tracker);
var done: u64 = 0; var done: u64 = 0;
for (0..reads_per_section) |_| { for (0..reads_per_section) |_| {
@@ -113,7 +113,7 @@ fn reader(param: ?*anyopaque) void {
// land on the stalled list. A blocked poller must boost us out of here. // land on the stalled list. A blocked poller must boost us out of here.
spin_us(section_us); spin_us(section_us);
ki.smr.exit_preempt(dom, &tracker); kep.smr.exit_preempt(dom, &tracker);
_ = reads.fetchAdd(done, .monotonic); _ = reads.fetchAdd(done, .monotonic);
} }
@@ -175,13 +175,13 @@ fn cur_inherited(td: *ke.Thread) u8 {
} }
/// Highest priority among the threads blocked on `ts`, or null if none. /// Highest priority among the threads blocked on `ts`, or null if none.
fn blocked_prio(ts: *ki.turnstile.Turnstile) ?u8 { fn blocked_prio(ts: *kep.turnstile.Turnstile) ?u8 {
var best: ?u8 = null; var best: ?u8 = null;
for (&ts.queues) |*q| { for (&ts.queues) |*q| {
var it = q.iterator(); var it = q.iterator();
while (it.next()) : (it.advance()) { while (it.next()) : (it.advance()) {
const w: *ki.turnstile.Waiter = @fieldParentPtr("link", it.get()); const w: *kep.turnstile.Waiter = @fieldParentPtr("link", it.get());
best = @max(best orelse 0, cur_prio(w.thread)); best = @max(best orelse 0, cur_prio(w.thread));
} }
} }
@@ -195,7 +195,7 @@ fn check_cpu(cpu: *ke.smr.Cpu) void {
const ipl = ke.ipl.raise(.Dispatch); const ipl = ke.ipl.raise(.Dispatch);
cpu.stall_lock.acquire_no_ipl(); cpu.stall_lock.acquire_no_ipl();
const turnstile = ki.turnstile.lookup(cpu); const turnstile = kep.turnstile.lookup(cpu);
if (turnstile) |ts| { if (turnstile) |ts| {
if (blocked_prio(ts)) |want| { if (blocked_prio(ts)) |want| {
@@ -203,8 +203,8 @@ fn check_cpu(cpu: *ke.smr.Cpu) void {
var it = cpu.stalled.iterator(); var it = cpu.stalled.iterator();
while (it.next()) : (it.advance()) { while (it.next()) : (it.advance()) {
const owner: *ki.turnstile.Owner = @fieldParentPtr("link", it.get()); const owner: *kep.turnstile.Owner = @fieldParentPtr("link", it.get());
const tracker: *ki.smr.Tracker = @fieldParentPtr("owner", owner); const tracker: *kep.smr.Tracker = @fieldParentPtr("owner", owner);
const td = tracker.owner.thread; const td = tracker.owner.thread;
stalled += 1; stalled += 1;
@@ -226,7 +226,7 @@ fn check_cpu(cpu: *ke.smr.Cpu) void {
} }
} }
ki.turnstile.exit(cpu); kep.turnstile.exit(cpu);
cpu.stall_lock.release_no_ipl(); cpu.stall_lock.release_no_ipl();
ke.ipl.lower(ipl); ke.ipl.lower(ipl);
} }
@@ -279,13 +279,13 @@ fn quiesce_check() void {
} }
fn spawn(prio: u8, entry: *const fn (?*anyopaque) void, id: usize) *ps.Thread { fn spawn(prio: u8, entry: *const fn (?*anyopaque) void, id: usize) *ps.Thread {
const t = ps.thread.create_kernel(@enumFromInt(prio), .{.func = entry, .arg= @ptrFromInt(id + 1)}, true) catch @panic("oom"); const t = ps.thread.create_kernel(@enumFromInt(prio), .{ .func = entry, .arg = @ptrFromInt(id + 1) }, true) catch @panic("oom");
ke.sched.enqueue(&t.kern); ke.sched.enqueue(&t.kern);
return t; return t;
} }
pub fn start(_: ?*anyopaque) void { pub fn start(_: ?*anyopaque) void {
std.log.info("{}", .{@sizeOf(ki.smr.Tracker)}); std.log.info("{}", .{@sizeOf(kep.smr.Tracker)});
dom = mm.zone.smr_domain_create(true) catch @panic("failed to create SMR domain"); dom = mm.zone.smr_domain_create(true) catch @panic("failed to create SMR domain");
zone.init("smr-pi", @sizeOf(Object), .{ zone.init("smr-pi", @sizeOf(Object), .{