mirrors/Zag
1
0
Fork 0
mirror of https://github.com/rdmsr/zag.git synced 2026-11-01 15:48:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

all: refactor coding style

Browse Source
This commit is contained in:
Loïc Grégoire
2026-08-05 17:43:46 -04:00
parent 2a09642f4d
commit bec872bc26
31 changed files with 887 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/arch/amd64/root.zig
+5 -1
View File
@@ -476,7 +476,11 @@ const PowerManagementInfoEdx = packed struct(u32) {
reserved1: u23, reserved1: u23,
}; };
fn assert_bit(comptime T: type, comptime field_name: []const u8, comptime expected_bit: u5) void { fn assert_bit(
comptime T: type,
comptime field_name: []const u8,
comptime expected_bit: u5,
) void {
var v: T = @bitCast(@as(u32, 0)); var v: T = @bitCast(@as(u32, 0));
@field(v, field_name) = true; @field(v, field_name) = true;
const raw: u32 = @bitCast(v); const raw: u32 = @bitCast(v);
src/ex/console.zig
+12 -3
View File
@@ -27,7 +27,11 @@ const ConWriter = struct {
}; };
} }
fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { fn drain(
w: *std.Io.Writer,
data: []const []const u8,
_: usize,
) std.Io.Writer.Error!usize {
const self: *ConWriter = @fieldParentPtr("interface", w); const self: *ConWriter = @fieldParentPtr("interface", w);
var total_written: usize = 0; var total_written: usize = 0;
for (data) |slice| { for (data) |slice| {
@@ -87,7 +91,8 @@ fn drain_from_all(seq: usize) usize {
fn worker(_: ?*anyopaque) void { fn worker(_: ?*anyopaque) void {
while (true) { while (true) {
_ = ke.wait.wait_one(&ke.log.event.hdr, "console_worker", .{}) catch unreachable; _ = ke.wait.wait_one(&ke.log.event.hdr, "console_worker", .{}) catch
unreachable;
ke.log.event.reset(); ke.log.event.reset();
last_seq = drain_from_all(last_seq); last_seq = drain_from_all(last_seq);
@@ -119,6 +124,10 @@ pub fn register(cons: *Console) void {
pub fn init() void { pub fn init() void {
console_list.init(); console_list.init();
const td = ps.thread.create_kernel(ke.Thread.Priority.default, worker, null) catch @panic("handle me"); const td = ps.thread.create_kernel(
ke.Thread.Priority.default,
worker,
null,
) catch @panic("handle me");
ke.sched.enqueue(&td.kern); ke.sched.enqueue(&td.kern);
} }
src/ex/work.zig
+27 -10
View File
@@ -1,6 +1,7 @@
//! Worker pools and work items. //! Worker pools and work items.
//! //!
//! Work items are used to enqueue work to be done in a thread context asynchronously. //! Work items are used to enqueue work to be done in a thread context
//! asynchronously.
//! There are three worker priorities: //! There are three worker priorities:
//! - High //! - High
//! - Normal //! - Normal
@@ -10,8 +11,8 @@
//! //!
//! Each CPU worker pool comprises of at least 2 threads, and the low pool //! Each CPU worker pool comprises of at least 2 threads, and the low pool
//! has 4 threads by default. An additional thread will get spawned by the //! has 4 threads by default. An additional thread will get spawned by the
//! pool manager thread if a queue has a backlog, up to 4 additional dynamic threads //! pool manager thread if a queue has a backlog, up to 4 additional dynamic
//! (tunable) per pool. //! threads (tunable) per pool.
const std = @import("std"); const std = @import("std");
const rtl = @import("rtl"); const rtl = @import("rtl");
@@ -58,7 +59,12 @@ pub const WorkItem = struct {
} }
}; };
pub fn init(self: *WorkItem, priority: Priority, func: *const fn (arg: ?*anyopaque) void, arg: ?*anyopaque) void { pub fn init(
self: *WorkItem,
priority: Priority,
func: *const fn (arg: ?*anyopaque) void,
arg: ?*anyopaque,
) void {
self.* = .{ self.* = .{
.func = func, .func = func,
.arg = arg, .arg = arg,
@@ -145,7 +151,9 @@ const Pool = struct {
const active = @atomicLoad(usize, &self.queue.active, .monotonic); const active = @atomicLoad(usize, &self.queue.active, .monotonic);
const threads = self.total_threads.load(.monotonic); const threads = self.total_threads.load(.monotonic);
if (threads < max_threads_for_prio(self.prio) and !is_empty and active < self.queue.max_active) { if (threads < max_threads_for_prio(self.prio) and !is_empty and
active < self.queue.max_active)
{
// We have room to grow, the list still has items and // We have room to grow, the list still has items and
// not enough runnable threads are present, create an // not enough runnable threads are present, create an
// additional thread to ease the backlog. // additional thread to ease the backlog.
@@ -157,7 +165,8 @@ const Pool = struct {
/// Spawn a dynamic worker thread on this pool. /// Spawn a dynamic worker thread on this pool.
fn grow(self: *Pool, cpu: ?u32) !void { fn grow(self: *Pool, cpu: ?u32) !void {
if (self.total_threads.load(.monotonic) >= max_threads_for_prio(self.prio)) if (self.total_threads.load(.monotonic) >=
max_threads_for_prio(self.prio))
return; return;
_ = self.total_threads.fetchAdd(1, .monotonic); _ = self.total_threads.fetchAdd(1, .monotonic);
@@ -197,7 +206,10 @@ fn work_loop(p: ?*anyopaque) void {
const dynamic = ctx.get_tag() == 1; const dynamic = ctx.get_tag() == 1;
while (true) { while (true) {
const timeout: ?r.Nanoseconds = if (dynamic) .from(r.Seconds.init(5)) else null; const timeout: ?r.Nanoseconds = if (dynamic)
.from(r.Seconds.init(5))
else
null;
const item = ctx.get_ptr().queue.remove(timeout) catch { const item = ctx.get_ptr().queue.remove(timeout) catch {
_ = ctx.get_ptr().total_threads.fetchSub(1, .monotonic); _ = ctx.get_ptr().total_threads.fetchSub(1, .monotonic);
@@ -257,11 +269,16 @@ pub fn enqueue(item: *WorkItem) void {
fn pool_manager(_: ?*anyopaque) void { fn pool_manager(_: ?*anyopaque) void {
while (true) { while (true) {
_ = ke.wait.wait_one(&pool_manager_event.hdr, "work_pool_manager", .{}) catch unreachable; _ = ke.wait.wait_one(
&pool_manager_event.hdr,
"work_pool_manager",
.{},
) catch
unreachable;
// We have been signaled, go through every pool and grow it if needed. // We have been signaled, go through every pool and grow it if needed.
// Failure to grow is ignored, it is only a latency problem and should be fixed // Failure to grow is ignored, it is only a latency problem and should
// eventually as memory is freed. // be fixed eventually as memory is freed.
for (0..ke.ncpus) |i| { for (0..ke.ncpus) |i| {
const cpu: u32 = @intCast(i); const cpu: u32 = @intCast(i);
const c = percpu.remote(cpu); const c = percpu.remote(cpu);
src/kern/amd64/impl.zig
+22 -4
View File
@@ -25,7 +25,12 @@ const ThreadFrame = extern struct {
extern fn asm_thread_entry() void; extern fn asm_thread_entry() void;
extern fn do_context_switch(old: *ThreadContext, new: *ThreadContext, lock: *u8, switching: *bool) callconv(.c) void; extern fn do_context_switch(
old: *ThreadContext,
new: *ThreadContext,
lock: *u8,
switching: *bool,
) callconv(.c) void;
extern fn do_context_load(td: *ThreadContext) callconv(.c) void; extern fn do_context_load(td: *ThreadContext) callconv(.c) void;
pub const ThreadContext = extern struct { pub const ThreadContext = extern struct {
@@ -38,7 +43,12 @@ pub const ThreadContext = extern struct {
entry_fn(@ptrFromInt(arg)); entry_fn(@ptrFromInt(arg));
} }
pub fn init(stack: r.VAddr, stack_size: usize, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) @This() { pub fn init(
stack: r.VAddr,
stack_size: usize,
entry: *const fn (?*anyopaque) void,
arg: ?*anyopaque,
) @This() {
var ctx: @This() = undefined; var ctx: @This() = undefined;
var sp: usize = stack + stack_size; var sp: usize = stack + stack_size;
@@ -58,9 +68,17 @@ pub const ThreadContext = extern struct {
return ctx; return ctx;
} }
pub fn switch_to(self: *ThreadContext, new: *ThreadContext) callconv(.c) void { pub fn switch_to(
self: *ThreadContext,
new: *ThreadContext,
) callconv(.c) void {
const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", self)); const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", self));
do_context_switch(self, new, &thread.lock.inner.locked.raw, &thread.switching.raw); do_context_switch(
self,
new,
&thread.lock.inner.locked.raw,
&thread.switching.raw,
);
} }
pub fn load(self: *ThreadContext) callconv(.c) void { pub fn load(self: *ThreadContext) callconv(.c) void {
src/kern/cpu.zig
+5 -1
View File
@@ -40,7 +40,11 @@ pub fn CpuLocal(comptime T: type, comptime init: T) type {
} }
/// Wraps around CPU-local data with a designated symbol name. /// Wraps around CPU-local data with a designated symbol name.
pub fn ExportedCpuLocal(comptime T: type, comptime init: T, comptime name: []const u8) type { pub fn ExportedCpuLocal(
comptime T: type,
comptime init: T,
comptime name: []const u8,
) type {
const S = struct { const S = struct {
var storage: T linksection(".data.percpu") = init; var storage: T linksection(".data.percpu") = init;
src/kern/dpc.zig
+8 -6
View File
@@ -6,7 +6,7 @@ const ke = r.ke;
const ki = ke.private; const ki = ke.private;
/// Deferred Procedure Call (DPC) structure. /// Deferred Procedure Call (DPC) structure.
/// Used for scheduling work to be done when IPL is lowered below `ke.Ipl.Dispatch`. /// Used for scheduling work to be done when IPL is lowered below `.Dispatch`.
pub const Dpc = struct { pub const Dpc = struct {
/// Entry into per-CPU DPC queue /// Entry into per-CPU DPC queue
link: rtl.List.Entry, link: rtl.List.Entry,
@@ -78,7 +78,8 @@ fn dispatch_queue(cpu: u32) void {
ipl_cpu.ipl = .Dispatch; ipl_cpu.ipl = .Dispatch;
// Mark as handled, this must be done before enabling interrupts or we will race. // Mark as handled, this must be done before enabling interrupts or we will
// race.
ki.ipl.clear_softint_pending(cpu, .Dispatch); ki.ipl.clear_softint_pending(cpu, .Dispatch);
_ = ki.impl.enable_interrupts(); _ = ki.impl.enable_interrupts();
@@ -100,8 +101,9 @@ fn dispatch_queue(cpu: u32) void {
dpc = @fieldParentPtr("link", first_elem); dpc = @fieldParentPtr("link", first_elem);
dpc.inserted.store(false, .release); dpc.inserted.store(false, .release);
// An interrupt which would enqueue this DPC could occur between loading the argument and calling the routine, // An interrupt which would enqueue this DPC could occur between
// which is why we capture it here to ensure that we get the intended context. // loading the argument and calling the routine, which is why we capture
// it here to ensure that we get the intended context.
const arg = dpc.arg; const arg = dpc.arg;
dpc_cpu.lock.release(ipl); dpc_cpu.lock.release(ipl);
@@ -148,8 +150,8 @@ fn dispatch_queue(cpu: u32) void {
/// Dispatch the DPC queue on `cpu`. /// Dispatch the DPC queue on `cpu`.
pub fn dispatch(cpu: u32) void { pub fn dispatch(cpu: u32) void {
// DPC processing is done at Ipl.Dispatch // DPC processing is done at IPL Dispatch.
// We can't call lower/raise here because they might call us again recursively. // Don't call lower/raise here because they might call us again recursively.
var mycpu = cpu; var mycpu = cpu;
const old_ipl = ki.ipl.percpu.remote(mycpu).ipl; const old_ipl = ki.ipl.percpu.remote(mycpu).ipl;
const int_state = ki.impl.disable_interrupts(); const int_state = ki.impl.disable_interrupts();
src/kern/init.zig
+5 -1
View File
@@ -37,7 +37,11 @@ pub fn init(_: ?*anyopaque) linksection(r.init) void {
ki.sched.percpu.local().current_thread = &thread0; ki.sched.percpu.local().current_thread = &thread0;
ki.sched.percpu.local().idle_thread = &thread0; ki.sched.percpu.local().idle_thread = &thread0;
std.log.info("Zag for {s} ({s}), cmdline is \"{?s}\"", .{ pl.name, arch.name, r.boot_info.cmdline }); std.log.info("Zag for {s} ({s}), cmdline is \"{?s}\"", .{
pl.name,
arch.name,
r.boot_info.cmdline,
});
ex.init(); ex.init();
ki.sched.idle(null); ki.sched.idle(null);
src/kern/ipi.zig
+7 -3
View File
@@ -35,7 +35,11 @@ const percpu = ke.CpuLocal(PerCpu, undefined);
/// The function will be executed synchronously, i.e this will block until /// The function will be executed synchronously, i.e this will block until
/// the function finishes running. The callback is called with the sender cpu /// the function finishes running. The callback is called with the sender cpu
/// and provided argument as arguments. /// and provided argument as arguments.
pub fn unicast(cpu: u32, func: *const fn (u32, ?*anyopaque) void, arg: ?*anyopaque) void { pub fn unicast(
cpu: u32,
func: *const fn (u32, ?*anyopaque) void,
arg: ?*anyopaque,
) void {
const ipl = ke.ipl.raise(.Dispatch); const ipl = ke.ipl.raise(.Dispatch);
const curcpu = percpu.local(); const curcpu = percpu.local();
const remote = percpu.remote(cpu); const remote = percpu.remote(cpu);
@@ -93,8 +97,8 @@ pub fn ipi_handler() void {
while (true) { while (true) {
// Mark the current CPU as processing IPIs, if the CAS fails here, // Mark the current CPU as processing IPIs, if the CAS fails here,
// then it's probably some kind of IPI that was sent to force us to check // then it's probably some kind of IPI that was sent to force us to
// software interrupts (like a rescheduling IPI). // check software interrupts (like a rescheduling IPI).
const val = curcpu.state.cmpxchgStrong( const val = curcpu.state.cmpxchgStrong(
.Pending, .Pending,
.Running, .Running,
src/kern/log.zig
+15 -4
View File
@@ -21,7 +21,11 @@ const DebugWriter = struct {
}; };
} }
fn drain(_: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { fn drain(
_: *std.Io.Writer,
data: []const []const u8,
_: usize,
) std.Io.Writer.Error!usize {
var total_written: usize = 0; var total_written: usize = 0;
for (data) |slice| { for (data) |slice| {
for (slice) |byte| { for (slice) |byte| {
@@ -37,7 +41,10 @@ const DebugWriter = struct {
// Messages are on average 2^5 = 32 bytes. // Messages are on average 2^5 = 32 bytes.
const avg_msg_size_bits = 5; const avg_msg_size_bits = 5;
pub var ringbuffer = ki.log_ring.RingBuffer(config.log_buffer_shift, avg_msg_size_bits).init(); pub var ringbuffer = ki.log_ring.RingBuffer(
config.log_buffer_shift,
avg_msg_size_bits,
).init();
// Initialize manually because we need this ASAP. // Initialize manually because we need this ASAP.
pub var event: ke.Event = .{ pub var event: ke.Event = .{
@@ -46,7 +53,10 @@ pub var event: ke.Event = .{
.signaled = 0, .signaled = 0,
.type = .Notification, .type = .Notification,
.waitblocks = .{ .waitblocks = .{
.head = .{ .next = &event.hdr.waitblocks.head, .prev = &event.hdr.waitblocks.head }, .head = .{
.next = &event.hdr.waitblocks.head,
.prev = &event.hdr.waitblocks.head,
},
}, },
}, },
}; };
@@ -75,7 +85,8 @@ pub fn log(
res.info.length = @truncate(required_len); res.info.length = @truncate(required_len);
// Format the log message into the reserved buffer. // Format the log message into the reserved buffer.
res.buf = std.fmt.bufPrint(res.buf, scope_str ++ fmt ++ "\n", args) catch return; res.buf = std.fmt.bufPrint(res.buf, scope_str ++ fmt ++ "\n", args) catch
return;
ringbuffer.publish(res); ringbuffer.publish(res);
src/kern/shootdown.zig
+17 -3
View File
@@ -53,7 +53,13 @@ fn pcpu_init() linksection(r.init) void {
local.valid_states = @splat(.init(0)); local.valid_states = @splat(.init(0));
local.senders = .init(false); local.senders = .init(false);
local.states = @splat(.{ .base = 0, .npages = 0, .state = .init(slot_free), .payload = undefined, .link = undefined }); local.states = @splat(.{
.base = 0,
.npages = 0,
.state = .init(slot_free),
.payload = undefined,
.link = undefined,
});
} }
comptime { comptime {
@@ -66,7 +72,12 @@ fn allocate_slot(cpu: *PerCpu) ?usize {
for (0.., &cpu.states) |slot, *state| { for (0.., &cpu.states) |slot, *state| {
if (state.state.load(.monotonic) != slot_free) continue; if (state.state.load(.monotonic) != slot_free) continue;
if (state.state.cmpxchgStrong(slot_free, slot_reserved, .acquire, .monotonic) == null) { if (state.state.cmpxchgStrong(
slot_free,
slot_reserved,
.acquire,
.monotonic,
) == null) {
return slot; return slot;
} }
} }
@@ -167,7 +178,10 @@ pub fn submit(state: ShootdownState, target_mask: ke.CpuMask) !void {
while (iter.next()) |bit| { while (iter.next()) |bit| {
if (bit == curcpu) continue; if (bit == curcpu) continue;
_ = cpu.valid_states[bit].fetchOr(@as(u64, 1) << @truncate(slot), .release); _ = cpu.valid_states[bit].fetchOr(
@as(u64, 1) << @truncate(slot),
.release,
);
ki.ipl.set_softint_pending(@truncate(bit), .Dispatch); ki.ipl.set_softint_pending(@truncate(bit), .Dispatch);
src/kern/smr.zig
+32 -17
View File
@@ -77,7 +77,8 @@ pub const Tracker = struct {
pub var full_scans: std.atomic.Value(usize) = .init(0); pub var full_scans: std.atomic.Value(usize) = .init(0);
/// Scan all CPUs and return the minimum observed value. /// Scan all CPUs and return the minimum observed value.
/// If `should_wait` is true, this will spinloop (or block) until all CPUs have reached the given goal. /// If `should_wait` is true, this will spinloop (or block)
/// until all CPUs have reached the given goal.
fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence { fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence {
rtl.barrier.fence(.seq_cst); rtl.barrier.fence(.seq_cst);
@@ -162,7 +163,12 @@ fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence
break; break;
} }
ki.turnstile.block(ts, cpu, .{ .shared = &cpu.stalled }, .Exclusive); ki.turnstile.block(
ts,
cpu,
.{ .shared = &cpu.stalled },
.Exclusive,
);
ke.ipl.lower(ipl); ke.ipl.lower(ipl);
seq = cpu.stall_seq.load(.monotonic); seq = cpu.stall_seq.load(.monotonic);
@@ -256,8 +262,9 @@ pub fn advance(dom: *Domain) Sequence {
/// Pretend-advance the write sequence and return the value for use as a wait goal. /// Pretend-advance the write sequence and return the value for use as a wait goal.
/// It is guaranteed that all previous memory writes made by /// It is guaranteed that all previous memory writes made by
/// the calling thread are visible. The global clock isn't advanced unlike `advance`, /// the calling thread are visible.
/// it is only when commit() is called. /// The global clock isn't advanced unlike `advance`, it is only when commit()
/// is called.
pub fn deferred_advance(dom: *Domain) Sequence { pub fn deferred_advance(dom: *Domain) Sequence {
rtl.barrier.fence(.seq_cst); rtl.barrier.fence(.seq_cst);
@@ -267,7 +274,12 @@ pub fn deferred_advance(dom: *Domain) Sequence {
/// Actually advance the global clock to a value previously returned by /// Actually advance the global clock to a value previously returned by
/// deferred_advance(). /// deferred_advance().
pub fn deferred_advance_commit(dom: *Domain, seq: Sequence) void { pub fn deferred_advance_commit(dom: *Domain, seq: Sequence) void {
_ = dom.clock.write_seq.cmpxchgStrong(seq - seq_incr, seq, .monotonic, .monotonic); _ = dom.clock.write_seq.cmpxchgStrong(
seq - seq_incr,
seq,
.monotonic,
.monotonic,
);
} }
/// Enter a read section. /// Enter a read section.
@@ -276,18 +288,20 @@ fn enter_internal(dom: *Domain) Sequence {
// Store the currently observed write sequence into our CPU state. // Store the currently observed write sequence into our CPU state.
// Subsequent loads must not be re-ordered w.r.t to the store here. // Subsequent loads must not be re-ordered w.r.t to the store here.
// On AMD64, we can simply use XADD as this is faster and provides the same guarantees, // On AMD64, we can simply use XADD as this is faster and provides the
// on other architectures we have to rely on a full (seq_cst) fence. // same guarantees, on other architectures we have to rely on a full fence.
// The add operation works because cpu.current_seq is always only 0 (seq_invalid) // The add operation works because cpu.current_seq is always only 0
// when we write to it. // (seq_invalid) when we write to it.
// //
// There can be a large pause between the load from the domain write sequence // There can be a large pause between the load from the domain write
// and the store to the per-cpu state because of e.g an interrupt. This could lead // sequence and the store to the per-cpu state because of e.g an interrupt.
// to a state where another CPU advances the global write sequence and current_seq // This could lead to a state where another CPU advances
// becomes lower than the global write sequence. This is not an issue with the ordering // the global write sequence and current_seq ends up lower
// in scan(), which also relies on a full memory barrier, as it is guaranteed that the // than the global write sequence.
// CPU was treated as inactive and cannot possibly hold a reference to anything the // This is not an issue with the ordering in scan(), which also relies
// poll declared reclaimable. See scan(). // on a full memory barrier, as it is guaranteed that the CPU was treated as
// inactive and cannot possibly hold a reference to anything the poll
// declared reclaimable. See scan().
std.debug.assert(cpu.current_seq.load(.monotonic) == seq_invalid); std.debug.assert(cpu.current_seq.load(.monotonic) == seq_invalid);
const wr_seq = dom.clock.write_seq.load(.monotonic); const wr_seq = dom.clock.write_seq.load(.monotonic);
@@ -434,7 +448,8 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void {
} }
if (wake) { if (wake) {
if (turnstile) |ts| ki.turnstile.wakeup(ts, .Exclusive, ts.waiters, null); if (turnstile) |ts|
ki.turnstile.wakeup(ts, .Exclusive, ts.waiters, null);
} }
ki.turnstile.exit(cpu); ki.turnstile.exit(cpu);
src/kern/spinlock.zig
+28 -5
View File
@@ -176,7 +176,12 @@ pub const QSpinLock = LockTemplate(struct {
pub fn acquire_no_ipl(self: *Self) void { pub fn acquire_no_ipl(self: *Self) void {
// Fast path: try to acquire the lock. // Fast path: try to acquire the lock.
var v = self.data.val.cmpxchgStrong(0, locked_val, .acquire, .monotonic) orelse return; var v = self.data.val.cmpxchgStrong(
0,
locked_val,
.acquire,
.monotonic,
) orelse return;
// Medium path: if only pending is set, the pending waiter is // Medium path: if only pending is set, the pending waiter is
// in the process of clearing pending and setting `locked` to take // in the process of clearing pending and setting `locked` to take
@@ -229,7 +234,12 @@ pub const QSpinLock = LockTemplate(struct {
} }
pub fn try_acquire_no_ipl(self: *Self) bool { pub fn try_acquire_no_ipl(self: *Self) bool {
return self.data.val.cmpxchgStrong(0, locked_val, .acquire, .monotonic) == null; return self.data.val.cmpxchgStrong(
0,
locked_val,
.acquire,
.monotonic,
) == null;
} }
pub fn is_locked(self: *Self) bool { pub fn is_locked(self: *Self) bool {
@@ -247,7 +257,12 @@ pub const QSpinLock = LockTemplate(struct {
if (idx >= 4) { if (idx >= 4) {
// Exhausted all indices, spin. // Exhausted all indices, spin.
while (true) { while (true) {
_ = self.data.val.cmpxchgWeak(0, locked_val, .acquire, .monotonic) orelse break; _ = self.data.val.cmpxchgWeak(
0,
locked_val,
.acquire,
.monotonic,
) orelse break;
std.atomic.spinLoopHint(); std.atomic.spinLoopHint();
} }
@@ -266,7 +281,10 @@ pub const QSpinLock = LockTemplate(struct {
// Store cpu + 1 so that we can distinguish from an empty tail field. // Store cpu + 1 so that we can distinguish from an empty tail field.
rtl.barrier.fence(.release); rtl.barrier.fence(.release);
const tail: u16 = @bitCast(Tail{ .cpu = @intCast(ke.cpu.current() + 1), .idx = @intCast(idx) }); const tail: u16 = @bitCast(Tail{
.cpu = @intCast(ke.cpu.current() + 1),
.idx = @intCast(idx),
});
const old: Tail = @bitCast(self.data.split.tail.swap(tail, .acq_rel)); const old: Tail = @bitCast(self.data.split.tail.swap(tail, .acq_rel));
if (old.cpu != 0) { if (old.cpu != 0) {
@@ -293,7 +311,12 @@ pub const QSpinLock = LockTemplate(struct {
if ((v & tail_mask) == @as(u32, tail) << 16) { if ((v & tail_mask) == @as(u32, tail) << 16) {
// We are the only waiter in the queue. // We are the only waiter in the queue.
// Try to claim the lock and clear the tail atomically. // Try to claim the lock and clear the tail atomically.
if (self.data.val.cmpxchgStrong(v, locked_val, .acquire, .monotonic) == null) { if (self.data.val.cmpxchgStrong(
v,
locked_val,
.acquire,
.monotonic,
) == null) {
return; return;
} }
} }
src/kern/thread.zig
+9 -2
View File
@@ -123,12 +123,19 @@ pub const Thread = struct {
smr_sections: rtl.List, smr_sections: rtl.List,
/// Initialize a thread. /// Initialize a thread.
/// - `stack`: Address of the **base** of the stack on which the initial context for the thread is built /// - `stack`: Address of the base of the stack used by the thread.
/// - `stack_size`: Size of the stack /// - `stack_size`: Size of the stack
/// - `prio`: Base priority of the thread /// - `prio`: Base priority of the thread
/// - `entry`: Entry point of the thread /// - `entry`: Entry point of the thread
/// - `arg`: Extraneous argument to be passed to `entry` /// - `arg`: Extraneous argument to be passed to `entry`
pub fn init(thread: *Thread, stack: r.VAddr, stack_size: usize, prio: u8, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) void { pub fn init(
thread: *Thread,
stack: r.VAddr,
stack_size: usize,
prio: u8,
entry: *const fn (?*anyopaque) void,
arg: ?*anyopaque,
) void {
thread.* = .{ thread.* = .{
.context = .init(stack, stack_size, entry, arg), .context = .init(stack, stack_size, entry, arg),
.lock = .init(), .lock = .init(),
src/kern/time.zig
+6 -6
View File
@@ -18,11 +18,9 @@ pub const TimeCounter = struct {
quality: i16, quality: i16,
/// Mask to apply to the counter value. /// Mask to apply to the counter value.
mask: u64, mask: u64,
/// Binary scaling exponent for fast ns conversion (0 if unused). /// Binary scaling exponent for fast ns conversion. See `ticks_to_ns`.
/// See `ticks_to_ns`.
p: u64, p: u64,
/// Scaled nanoseconds-per-tick multiplier for fast ns conversion (0 if unused). /// Scaled for fast ns conversion (0 if unused). See `ticks_to_ns`.
/// See `ticks_to_ns`.
n: u64, n: u64,
}; };
@@ -59,10 +57,12 @@ fn ticks_to_ns(tc: *TimeCounter, count: u64) u64 {
/// Register a time-keeping source. /// Register a time-keeping source.
/// The TimeCounter will be used if it is the highest quality one available. /// The TimeCounter will be used if it is the highest quality one available.
/// This function cannot be safely called when the hardware timer has been started, as data races on `best_tc` are not avoided. /// This function cannot be safely called when the hardware timer
/// has been started, as data races on `best_tc` are not avoided.
pub fn register_source(tc: *TimeCounter) void { pub fn register_source(tc: *TimeCounter) void {
// Precompute scaling factors for the fast multiply-shift path. // Precompute scaling factors for the fast multiply-shift path.
// This is meant to avoid 128-bit division and only use multiplication for 64-bit counters. // This is meant to avoid 128-bit division and only use multiplication for
// 64-bit counters.
if (tc.mask == std.math.maxInt(u64)) { if (tc.mask == std.math.maxInt(u64)) {
tc.p = std.math.log2_int_ceil(u64, tc.frequency); tc.p = std.math.log2_int_ceil(u64, tc.frequency);
tc.n = (@as(u64, std.time.ns_per_s) << @intCast(tc.p)) / tc.frequency; tc.n = (@as(u64, std.time.ns_per_s) << @intCast(tc.p)) / tc.frequency;
src/kern/timer.zig
+28 -7
View File
@@ -1,5 +1,6 @@
//! Timer object implementation. //! Timer object implementation.
//! Timer objects are useful when one wants to wait a fixed amount of time for an event to occur. //! Timer objects are useful when one wants to wait a given amount of time for
//! an event to occur.
const std = @import("std"); const std = @import("std");
const rtl = @import("rtl"); const rtl = @import("rtl");
const r = @import("root"); const r = @import("root");
@@ -70,7 +71,12 @@ pub fn set(timer: *Timer, time: r.Nanoseconds, opts: Options) void {
const cpu = percpu.local(); const cpu = percpu.local();
if (timer.state.cmpxchgStrong(.Stopped, .Pending, .acquire, .monotonic) != null) { if (timer.state.cmpxchgStrong(
.Stopped,
.Pending,
.acquire,
.monotonic,
) != null) {
// Could not set the timer. // Could not set the timer.
return; return;
} }
@@ -107,7 +113,12 @@ pub fn cancel(timer: *Timer) void {
const ipl = timer.hdr.lock.acquire(); const ipl = timer.hdr.lock.acquire();
defer timer.hdr.lock.release(ipl); defer timer.hdr.lock.release(ipl);
if (timer.state.cmpxchgStrong(.Pending, .Stopped, .acquire, .monotonic) != null) { if (timer.state.cmpxchgStrong(
.Pending,
.Stopped,
.acquire,
.monotonic,
) != null) {
// Something changed, it probably expired. // Something changed, it probably expired.
return; return;
} }
@@ -137,7 +148,8 @@ pub fn cmp_timer(a: *rtl.pairing_heap.Node, b_: *rtl.pairing_heap.Node) std.math
pub fn clock() void { pub fn clock() void {
ke.dpc.enqueue(&percpu.local().dpc, null); ke.dpc.enqueue(&percpu.local().dpc, null);
// Check for overflows. // Check for overflows.
// This is fine to call a lot, as the function only gets expensive (i.e seqlock store) when an overflow actually happens. // This is fine to call a lot, as the function only gets expensive
// (i.e seqlock store) when an overflow actually happens.
ki.time.update_overflow(); ki.time.update_overflow();
} }
@@ -161,13 +173,17 @@ fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void {
// If the timer expires more than 1ms in the future, consider it not yet due. // If the timer expires more than 1ms in the future, consider it not yet due.
// Sub-millisecond differences are close enough to expire immediately. // Sub-millisecond differences are close enough to expire immediately.
if (timer.deadline.value > curtime.value and timer.deadline.value - curtime.value > std.time.ns_per_ms) { if (timer.deadline.value > curtime.value and
timer.deadline.value - curtime.value > std.time.ns_per_ms)
{
const next = timer; const next = timer;
cpu.lock.release_no_ipl(); cpu.lock.release_no_ipl();
// Re-check in case the timer expired while we were here. // Re-check in case the timer expired while we were here.
const now = ke.time.read_time(); const now = ke.time.read_time();
if (next.deadline.value <= now.value or next.deadline.value - now.value <= std.time.ns_per_ms) { if (next.deadline.value <= now.value or
next.deadline.value - now.value <= std.time.ns_per_ms)
{
continue; // expired in the meantime, loop again continue; // expired in the meantime, loop again
} }
@@ -176,7 +192,12 @@ fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void {
return; return;
} }
if (timer.state.cmpxchgStrong(.Pending, .Running, .acquire, .monotonic) != null) { if (timer.state.cmpxchgStrong(
.Pending,
.Running,
.acquire,
.monotonic,
) != null) {
// Timer must've been canceled. // Timer must've been canceled.
cpu.lock.release_no_ipl(); cpu.lock.release_no_ipl();
continue; continue;
src/kern/tunable.zig
+59 -56
View File
@@ -16,7 +16,11 @@ const Entry = struct {
}, },
}; };
pub fn Tunable(comptime T: type, comptime default: T, comptime name: []const u8) type { pub fn Tunable(
comptime T: type,
comptime default: T,
comptime name: []const u8,
) type {
return struct { return struct {
var storage: T = default; var storage: T = default;
@@ -42,66 +46,65 @@ pub fn Tunable(comptime T: type, comptime default: T, comptime name: []const u8)
}; };
} }
fn store_int(comptime T: type, addr: *anyopaque, value: anytype) bool {
const ptr: *T = @ptrCast(@alignCast(addr));
ptr.* = std.math.cast(T, value) orelse return false;
return true;
}
fn apply_num(
comptime ValueT: type,
cmdline: []const u8,
entry: *const Entry,
info: std.builtin.Type.Int,
) void {
const value = rtl.cmdline.get_number(ValueT, cmdline, entry.name) catch |e| {
if (e == error.Format) {
std.log.warn(
"Invalid number value for '{s}', falling back.",
.{entry.name},
);
}
return;
};
const sign = @typeInfo(ValueT).int.signedness;
_ = switch (info.bits) {
8 => store_int(std.meta.Int(sign, 8), entry.address, value),
16 => store_int(std.meta.Int(sign, 16), entry.address, value),
32 => store_int(std.meta.Int(sign, 32), entry.address, value),
64 => store_int(std.meta.Int(sign, 64), entry.address, value),
else => unreachable,
};
}
fn apply_bool(cmdline: []const u8, entry: *const Entry) void {
const value = rtl.cmdline.get_string(cmdline, entry.name) orelse return;
const ptr: *bool = @ptrCast(@alignCast(entry.address));
if (std.mem.eql(u8, value, "true")) {
ptr.* = true;
} else if (std.mem.eql(u8, value, "false")) {
ptr.* = false;
} else {
std.log.warn(
"Invalid cmdline boolean value for tunable '{s}'",
.{entry.name},
);
}
}
pub fn init() void { pub fn init() void {
const cmdline = r.boot_info.cmdline orelse return; const cmdline = r.boot_info.cmdline orelse return;
const elems = set.elems(); for (set.elems()) |entry| {
for (elems) |entry| {
switch (entry.type) { switch (entry.type) {
.num => |info| { .num => |info| switch (info.signedness) {
const a = entry.address; .unsigned => apply_num(u64, cmdline, entry, info),
.signed => apply_num(i64, cmdline, entry, info),
switch (info.signedness) {
.unsigned => {
const value = rtl.cmdline.get_number(u64, cmdline, entry.name) catch |e| {
if (e == error.Format) {
std.log.warn("Invalid number value for '{s}', falling back.", .{entry.name});
}
continue;
};
switch (info.bits) {
8 => (@as(*u8, @ptrCast(@alignCast(a)))).* = std.math.cast(u8, value) orelse continue,
16 => (@as(*u16, @ptrCast(@alignCast(a)))).* = std.math.cast(u16, value) orelse continue,
32 => (@as(*u32, @ptrCast(@alignCast(a)))).* = std.math.cast(u32, value) orelse continue,
64 => (@as(*u64, @ptrCast(@alignCast(a)))).* = value,
else => unreachable,
}
},
.signed => {
const value = rtl.cmdline.get_number(i64, cmdline, entry.name) catch |e| {
if (e == error.Format) {
std.log.warn("Invalid number value for '{s}', falling back.", .{entry.name});
}
continue;
};
switch (info.bits) {
8 => (@as(*i8, @ptrCast(@alignCast(a)))).* = std.math.cast(i8, value) orelse continue,
16 => (@as(*i16, @ptrCast(@alignCast(a)))).* = std.math.cast(i16, value) orelse continue,
32 => (@as(*i32, @ptrCast(@alignCast(a)))).* = std.math.cast(i32, value) orelse continue,
64 => (@as(*i64, @ptrCast(@alignCast(a)))).* = value,
else => unreachable,
}
},
}
},
.bool => {
const value = rtl.cmdline.get_string(cmdline, entry.name) orelse continue;
const ptr: *bool = @ptrCast(@alignCast(entry.address));
if (std.mem.eql(u8, value, "true")) {
ptr.* = true;
} else if (std.mem.eql(u8, value, "false")) {
ptr.* = false;
} else {
std.log.warn("Invalid cmdline boolean value for tunable '{s}'", .{entry.name});
}
}, },
.bool => apply_bool(cmdline, entry),
} }
} }
} }
src/kern/turnstile.zig
+16 -5
View File
@@ -314,7 +314,8 @@ fn propagate(curtd: *ke.Thread, root: *Chain) void {
// and they are forbidden to explicitly block. // and they are forbidden to explicitly block.
var it = targets(ts); var it = targets(ts);
while (it.next()) |t| { while (it.next()) |t| {
if (t.thread == curtd) @panic("turnstile: cycle in blocking chain"); if (t.thread == curtd)
@panic("turnstile: cycle in blocking chain");
t.thread.lock.acquire_no_ipl(); t.thread.lock.acquire_no_ipl();
_ = donate_to(t.boost, t.thread, donate); _ = donate_to(t.boost, t.thread, donate);
t.thread.lock.release_no_ipl(); t.thread.lock.release_no_ipl();
@@ -384,7 +385,12 @@ pub fn exit(obj: *const anyopaque) void {
/// This must be called with the appropriate turnstile chain lock held, and /// This must be called with the appropriate turnstile chain lock held, and
/// returns with the chain lock released. /// returns with the chain lock released.
/// IPL is kept as before on return. /// IPL is kept as before on return.
pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, own: Ownership, queue: Queue) void { pub fn block(
turnstile: ?*Turnstile,
obj: *anyopaque,
own: Ownership,
queue: Queue,
) void {
const chain = chain_for(obj); const chain = chain_for(obj);
const curtd = ki.sched.percpu.local().current_thread.?; const curtd = ki.sched.percpu.local().current_thread.?;
const ipl = ke.ipl.current(); const ipl = ke.ipl.current();
@@ -441,7 +447,8 @@ pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, own: Ownership, queue: Que
// Now block on the event. // Now block on the event.
_ = ke.ipl.lower(.Passive); _ = ke.ipl.lower(.Passive);
_ = ke.wait.wait_one(&waiter.event.hdr, "turnstile:waiter", .{}) catch unreachable; _ = ke.wait.wait_one(&waiter.event.hdr, "turnstile:waiter", .{}) catch
unreachable;
_ = ke.ipl.raise(ipl); _ = ke.ipl.raise(ipl);
} }
@@ -464,7 +471,10 @@ pub fn wakeup(ts: *Turnstile, queue: Queue, count: usize, new_owner: ?*ke.Thread
} else { } else {
for (0..count) |_| { for (0..count) |_| {
if (ts.queues[queue_idx].is_empty()) break; if (ts.queues[queue_idx].is_empty()) break;
const waiter: *Waiter = @fieldParentPtr("link", ts.queues[queue_idx].first()); const waiter: *Waiter = @fieldParentPtr(
"link",
ts.queues[queue_idx].first(),
);
dequeue(ts, waiter.thread); dequeue(ts, waiter.thread);
} }
if (ts.waiters > 0) ts.detach(); if (ts.waiters > 0) ts.detach();
@@ -487,7 +497,8 @@ fn dequeue(ts: *Turnstile, td: *ke.Thread) void {
ts.next_free = free.next_free; ts.next_free = free.next_free;
free.next_free = null; free.next_free = null;
} else { } else {
// Last waiter, pull the turnstile off the chain and keep it for ourselves. // Last waiter, pull the turnstile off the chain
// and keep it for ourselves.
ts.link.remove(); ts.link.remove();
ts.reset(); ts.reset();
} }
src/kern/wait.zig
+48 -19
View File
@@ -13,7 +13,8 @@ pub const DispatchHeader = struct {
pub const Type = enum { pub const Type = enum {
/// When signaled, `signaled` is kept high and all waiters are woken up. /// When signaled, `signaled` is kept high and all waiters are woken up.
Notification, Notification,
/// When signaled, `signaled` is decreased until 0, and a single waiter is woken up. /// When signaled, `signaled` is decreased until 0, and a single waiter
/// is woken up.
Synchronization, Synchronization,
/// Special type for queue objects. /// Special type for queue objects.
Queue, Queue,
@@ -70,11 +71,11 @@ pub const DispatchHeader = struct {
pub const WaitBlock = struct { pub const WaitBlock = struct {
pub const Status = enum(u8) { pub const Status = enum(u8) {
/// WaitBlock is linked to an object as part of a thread that is waiting. /// Block is linked to an object as part of a thread that is waiting.
Active, Active,
/// The wait associated with this WaitBlock has been satisfied (or timed out). /// The wait associated has been satisfied (or timed out).
Inactive, Inactive,
/// A signal was delivered to the wait (this could be before it was committed). /// A signal was delivered to the wait.
Signaled, Signaled,
}; };
/// List linkage. /// List linkage.
@@ -114,8 +115,9 @@ pub fn wait_one(object: *DispatchHeader, reason: []const u8, opts: Options) !usi
/// Returns the index of the object that was signaled, or an error /// Returns the index of the object that was signaled, or an error
/// in the case of timeout. /// in the case of timeout.
/// If `timeout` is not provided, it will wait indefinitely. /// If `timeout` is not provided, it will wait indefinitely.
/// If `waitblocks` is specified, then the wait will use those waitblocks for the operation. /// If `waitblocks` is specified, then the wait will use those waitblocks
/// Note that if `timeout` is provided, then one additional waitblock must be allocated. /// for the operation. Note that if `timeout` is provided, then one additional
/// waitblock must be allocated.
pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !usize { pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !usize {
const ipl = ke.ipl.raise(.Dispatch); const ipl = ke.ipl.raise(.Dispatch);
defer ke.ipl.lower(ipl); defer ke.ipl.lower(ipl);
@@ -123,7 +125,9 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
const obj_count = objects.len; const obj_count = objects.len;
const has_timeout = opts.timeout != null; const has_timeout = opts.timeout != null;
const blocks = opts.waitblocks orelse blk: { const blocks = opts.waitblocks orelse blk: {
std.debug.assert(obj_count <= curtd.waitblocks.len - @intFromBool(has_timeout)); std.debug.assert(
obj_count <= curtd.waitblocks.len - @intFromBool(has_timeout),
);
break :blk &curtd.waitblocks; break :blk &curtd.waitblocks;
}; };
@@ -146,7 +150,12 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
if (obj.can_satisfy()) { if (obj.can_satisfy()) {
// Object was already signaled. Try consuming it. // Object was already signaled. Try consuming it.
if (curtd.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acq_rel, .monotonic) == null) { if (curtd.wait_status.cmpxchgStrong(
.InProgress,
.Satisfied,
.acq_rel,
.monotonic,
) == null) {
obj.consume(curtd); obj.consume(curtd);
satisfier = i; satisfier = i;
} }
@@ -161,7 +170,7 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
wb.object = obj; wb.object = obj;
wb.thread = curtd; wb.thread = curtd;
wb.status = .Active; wb.status = .Active;
// We are not satisfied yet, so add a waitblock to the object's waitblocks // We are not satisfied yet, so add a waitblock to the object.
obj.waitblocks.insert_tail(&wb.link); obj.waitblocks.insert_tail(&wb.link);
} }
@@ -197,9 +206,15 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
curtd.lock.acquire_no_ipl(); curtd.lock.acquire_no_ipl();
// Now try committing the wait. // Now try committing the wait.
// While we're trying to commit the wait, the object locks have been released, and the state could therefore change. // While we're trying to commit the wait, the object locks
// have been released, and the state could therefore change.
// We need to re-check the state of the wait before actually blocking. // We need to re-check the state of the wait before actually blocking.
if (curtd.wait_status.cmpxchgStrong(.InProgress, .Committed, .acq_rel, .monotonic) == null) { if (curtd.wait_status.cmpxchgStrong(
.InProgress,
.Committed,
.acq_rel,
.monotonic,
) == null) {
if (queue == null) { if (queue == null) {
if (curtd.queue) |q| { if (curtd.queue) |q| {
ki.queue.signal_wait(q); ki.queue.signal_wait(q);
@@ -236,7 +251,8 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
wb.link.remove(); wb.link.remove();
} }
if (wb.status == .Signaled) { if (wb.status == .Signaled) {
// This waitblock was signaled, it must be the one that satisfied the wait. // This waitblock was signaled, it must be the one that satisfied
// the wait.
std.debug.assert(satisfier == null); std.debug.assert(satisfier == null);
satisfier = i; satisfier = i;
} }
@@ -245,7 +261,10 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
} }
const final_sat = satisfier orelse return error.Timeout; const final_sat = satisfier orelse return error.Timeout;
return if (has_timeout and final_sat == timer_i) error.Timeout else final_sat; return if (has_timeout and final_sat == timer_i)
error.Timeout
else
final_sat;
} }
/// Satisfy a wait on an object. /// Satisfy a wait on an object.
@@ -267,19 +286,29 @@ pub fn satisfy_wait(obj: *DispatchHeader) void {
wb.link.remove(); wb.link.remove();
// Three cases may occur here: // Three cases may occur here:
// 1. The wait was still preparing (status == .InProgress) and we interrupted it. // 1. The wait was still preparing (.InProgress) and we interrupted it.
// 2. The wait was already committed (status == .Committed) and we satisfied it. // 2. The wait was already committed (.Committed) and we satisfied it.
// 3. The wait was already satisfied by another object (status == .Satisfied). // 3. The wait was already satisfied by another object (.Satisfied).
//
// 1. // 1.
if (td.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acq_rel, .monotonic) == null) { if (td.wait_status.cmpxchgStrong(
.InProgress,
.Satisfied,
.acq_rel,
.monotonic,
) == null) {
// We interrupted the wait while it was being prepared. // We interrupted the wait while it was being prepared.
wb.status = .Signaled; wb.status = .Signaled;
obj.consume(td); obj.consume(td);
} }
// 2. // 2.
else if (td.wait_status.cmpxchgStrong(.Committed, .Satisfied, .acq_rel, .monotonic) == null) { else if (td.wait_status.cmpxchgStrong(
.Committed,
.Satisfied,
.acq_rel,
.monotonic,
) == null) {
// We interrupted the wait while it was committed. // We interrupted the wait while it was committed.
// Wake the thread. // Wake the thread.
wb.status = .Signaled; wb.status = .Signaled;
src/mm/heap.zig
+11 -2
View File
@@ -4,7 +4,12 @@ const mm = r.mm;
const mi = mm.private; const mi = mm.private;
pub fn init() void { pub fn init() void {
mi.kernel_space.arena.init("kernel heap", mi.impl.kernel_heap_base, r.tib(16), mm.page_size) catch @panic("failed to initialize kernel heap arena"); mi.kernel_space.arena.init(
"kernel heap",
mi.impl.kernel_heap_base,
r.tib(16),
mm.page_size,
) catch @panic("failed to initialize kernel heap arena");
} }
pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque { pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque {
@@ -18,7 +23,11 @@ pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque {
const npages = size / mm.page_size; const npages = size / mm.page_size;
for (0..npages) |i| { for (0..npages) |i| {
const pte = mi.pmap.wire_pte(&mi.kernel_space, addr + (i * mm.page_size), policy) catch { const pte = mi.pmap.wire_pte(
&mi.kernel_space,
addr + (i * mm.page_size),
policy,
) catch {
mi.kernel_space.lock.release(); mi.kernel_space.lock.release();
return mm.Error.OutOfMemory; return mm.Error.OutOfMemory;
}; };
src/mm/phys.zig
+28 -9
View File
@@ -47,7 +47,8 @@ fn wait_for_pages(old_ipl: ke.Ipl) void {
list_lock.release(ipl); list_lock.release(ipl);
_ = ke.wait.wait_one(&free_page_event.hdr, "wait_for_pages", .{}) catch unreachable; _ = ke.wait.wait_one(&free_page_event.hdr, "wait_for_pages", .{}) catch
unreachable;
ipl = list_lock.acquire(); ipl = list_lock.acquire();
cnt = free_pages; cnt = free_pages;
@@ -150,7 +151,11 @@ pub fn init() linksection(r.init) void {
for (0..memory_map.entry_count) |i| { for (0..memory_map.entry_count) |i| {
const entry = memory_map.entries[i]; const entry = memory_map.entries[i];
log.info("[{x:0>16}-{x:0>16}] {s}", .{ entry.base, entry.base + entry.size, @tagName(entry.type) }); log.info("[{x:0>16}-{x:0>16}] {s}", .{
entry.base,
entry.base + entry.size,
@tagName(entry.type),
});
if (entry.type == .Free or entry.type == .LoaderReclaimable) { if (entry.type == .Free or entry.type == .LoaderReclaimable) {
total_usable_memory += entry.size; total_usable_memory += entry.size;
@@ -158,17 +163,22 @@ pub fn init() linksection(r.init) void {
} }
const pfndb_size_required = @sizeOf(mm.Page) * const pfndb_size_required = @sizeOf(mm.Page) *
(std.math.divCeil(usize, total_usable_memory, mm.page_size) catch unreachable); (std.math.divCeil(usize, total_usable_memory, mm.page_size) catch
unreachable);
log.info("using {} KiB for pfndb ({} bytes per page)", .{ pfndb_size_required / 1024, @sizeOf(mm.Page) }); log.info("using {} KiB for pfndb ({} bytes per page)", .{
pfndb_size_required / 1024,
@sizeOf(mm.Page),
});
// Get the kernel pagemap from the loader, // Get the kernel pagemap from the loader,
// we are already in it so no need to activate it. // we are already in it so no need to activate it.
mi.impl.init_kernel(); mi.impl.init_kernel();
// Calculate the minimum amount of free memory we accept, // Calculate the minimum amount of free memory we accept,
// this formula is from Linux and does not grow linearly; it would be wasteful // this formula is from Linux and does not grow linearly;
// to take a fixed percentage of memory from a large server. // it would be wasteful to take a fixed percentage of memory from a large
// server.
// We clamp this between 128 KiB and 256 MiB. // We clamp this between 128 KiB and 256 MiB.
const min_free_kb = std.math.clamp( const min_free_kb = std.math.clamp(
std.math.sqrt((usable_memory.load(.monotonic) / 1024) * 16), std.math.sqrt((usable_memory.load(.monotonic) / 1024) * 16),
@@ -180,7 +190,10 @@ pub fn init() linksection(r.init) void {
const total_usable_pages = total_usable_memory / mm.page_size; const total_usable_pages = total_usable_memory / mm.page_size;
const gap = @max(min_memory_threshold / 4, total_usable_pages * threshold_scale_factor / 10000); const gap = @max(
min_memory_threshold / 4,
total_usable_pages * threshold_scale_factor / 10000,
);
low_memory_threshold = min_memory_threshold + gap; low_memory_threshold = min_memory_threshold + gap;
high_memory_threshold = low_memory_threshold + gap; high_memory_threshold = low_memory_threshold + gap;
@@ -189,14 +202,20 @@ pub fn init() linksection(r.init) void {
for (0..memory_map.entry_count) |i| { for (0..memory_map.entry_count) |i| {
const entry = memory_map.entries[i]; const entry = memory_map.entries[i];
if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < mm.page_size) { if (entry.type != .Free and entry.type != .LoaderReclaimable) {
continue;
}
if (entry.size < mm.page_size) {
continue; continue;
} }
const npages = entry.size / mm.page_size; const npages = entry.size / mm.page_size;
for (0..npages) |j| { for (0..npages) |j| {
const page: *mm.Page = mm.pfn_to_struct_page(mm.page_to_pfn(entry.base + (j * mm.page_size))); const page: *mm.Page = mm.pfn_to_struct_page(
mm.page_to_pfn(entry.base + (j * mm.page_size)),
);
free_list.insert_head(&page.free.link); free_list.insert_head(&page.free.link);
free_pages += 1; free_pages += 1;
src/mm/pmap.zig
+46 -13
View File
@@ -6,7 +6,8 @@ const std = @import("std");
const mm = r.mm; const mm = r.mm;
const mi = mm.private; const mi = mm.private;
/// Describes a contiguous region of physical memory to be mapped at a given virtual address. /// Describes a contiguous region of physical memory to be mapped
/// at a given virtual address.
const MapItem = struct { const MapItem = struct {
pa: r.PAddr, pa: r.PAddr,
len: usize, len: usize,
@@ -87,8 +88,15 @@ pub const PMap = struct {
} }
} }
/// Map a contiguous virtual address range to a contiguous physical address range. /// Map a contiguous virtual address range to a
pub fn map_contiguous_range(self: *Self, va: r.VAddr, pa: r.PAddr, size: usize, flags: mm.MapFlags) void { /// contiguous physical address range.
pub fn map_contiguous_range(
self: *Self,
va: r.VAddr,
pa: r.PAddr,
size: usize,
flags: mm.MapFlags,
) void {
const src = ContiguousSource{ const src = ContiguousSource{
.flags = flags, .flags = flags,
.size = size, .size = size,
@@ -163,7 +171,10 @@ pub const PMap = struct {
const head: *mm.Page = @ptrCast(head_free); const head: *mm.Page = @ptrCast(head_free);
const tail: *mm.Page = @ptrCast(tail_free); const tail: *mm.Page = @ptrCast(tail_free);
return .{ .head = mm.struct_page_to_pfn(head), .tail = mm.struct_page_to_pfn(tail) }; return .{
.head = mm.struct_page_to_pfn(head),
.tail = mm.struct_page_to_pfn(tail),
};
} }
pub const Cursor = struct { pub const Cursor = struct {
@@ -177,7 +188,11 @@ pub const PMap = struct {
/// Reset upward in `advance` when the VA crosses a level boundary. /// Reset upward in `advance` when the VA crosses a level boundary.
top_level: usize, top_level: usize,
fn walk_down(self: *Cursor, target_level: usize, allocate: bool) error{PageNotMapped}!void { fn walk_down(
self: *Cursor,
target_level: usize,
allocate: bool,
) error{PageNotMapped}!void {
var current_level = self.top_level; var current_level = self.top_level;
while (current_level > target_level) { while (current_level > target_level) {
@@ -191,8 +206,13 @@ pub const PMap = struct {
const new_table_pa = mi.phys.alloc(); const new_table_pa = mi.phys.alloc();
const table_ptr: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(new_table_pa)); const table_ptr: [*]mi.impl.Pte = @ptrFromInt(
@memset(table_ptr[0..entries_per_table], std.mem.zeroes(mi.impl.Pte)); mm.p2v(new_table_pa),
);
@memset(
table_ptr[0..entries_per_table],
std.mem.zeroes(mi.impl.Pte),
);
pte = mi.impl.make_table_pte(new_table_pa); pte = mi.impl.make_table_pte(new_table_pa);
self.tables[current_level][idx] = pte; self.tables[current_level][idx] = pte;
@@ -210,14 +230,20 @@ pub const PMap = struct {
return true; return true;
} }
pub fn map_range(self: *Cursor, pa: r.PAddr, size: usize, flags: mm.MapFlags) void { pub fn map_range(
self: *Cursor,
pa: r.PAddr,
size: usize,
flags: mm.MapFlags,
) void {
var remain = size; var remain = size;
var current_pa = pa; var current_pa = pa;
while (remain > 0) { while (remain > 0) {
const target_level = 0; const target_level = 0;
self.walk_down(target_level, true) catch @panic("walk_down failed during map_range"); self.walk_down(target_level, true) catch
@panic("walk_down failed during map_range");
const table = self.tables[target_level]; const table = self.tables[target_level];
table[index_for_level(self.va, target_level)] = table[index_for_level(self.va, target_level)] =
@@ -252,7 +278,11 @@ pub const PMap = struct {
}; };
}; };
pub fn wire_pte(space: *mi.Space, va: r.VAddr, policy: mm.WaitPolicy) mm.Error!*mi.impl.Pte { pub fn wire_pte(
space: *mi.Space,
va: r.VAddr,
policy: mm.WaitPolicy,
) mm.Error!*mi.impl.Pte {
std.debug.assert(space.lock.is_locked()); std.debug.assert(space.lock.is_locked());
var table: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(mi.kernel_space.pmap.root_pa)); var table: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(mi.kernel_space.pmap.root_pa));
@@ -268,7 +298,8 @@ pub fn wire_pte(space: *mi.Space, va: r.VAddr, policy: mm.WaitPolicy) mm.Error!*
} }
if (!entry.present) { if (!entry.present) {
// Release the lock while allocating a new page table, since that may block. // Release the lock while allocating a new page table,
// since that may block.
space.lock.release(); space.lock.release();
const new_table_pa = mi.phys.alloc_opts(.{ .policy = policy }) orelse { const new_table_pa = mi.phys.alloc_opts(.{ .policy = policy }) orelse {
@@ -276,13 +307,15 @@ pub fn wire_pte(space: *mi.Space, va: r.VAddr, policy: mm.WaitPolicy) mm.Error!*
return mm.Error.OutOfMemory; return mm.Error.OutOfMemory;
}; };
// Re-acquire the lock and re-validate the entry, since another thread may have raced to allocate it. // Re-acquire the lock and re-validate the entry,
// since another thread may have raced to allocate it.
space.lock.acquire(); space.lock.acquire();
entry = table[idx]; entry = table[idx];
if (entry.present) { if (entry.present) {
// Another thread beat us to it, free the page we just allocated and continue. // Another thread beat us to it,
// free the page we just allocated and continue.
mi.phys.free(new_table_pa); mi.phys.free(new_table_pa);
} else { } else {
const new_table_ptr: [*]u64 = @ptrFromInt(mm.p2v(new_table_pa)); const new_table_ptr: [*]u64 = @ptrFromInt(mm.p2v(new_table_pa));
src/mm/tlb.zig
+10 -2
View File
@@ -35,7 +35,11 @@ fn reclaim_state(obj: *anyopaque, _: ?*anyopaque) void {
space.lock.release(); space.lock.release();
// Free the physical pages // Free the physical pages
mm.phys.free_batch(mm.pfn_to_struct_page(pfn_list.head), mm.pfn_to_struct_page(pfn_list.tail), npages); mm.phys.free_batch(
mm.pfn_to_struct_page(pfn_list.head),
mm.pfn_to_struct_page(pfn_list.tail),
npages,
);
_ = async_shootdowns.fetchAdd(1, .monotonic); _ = async_shootdowns.fetchAdd(1, .monotonic);
@@ -83,7 +87,11 @@ pub fn reclaim_range(space: *mm.Space, va: r.VAddr, size: usize) void {
space.arena.free(va, size) catch unreachable; space.arena.free(va, size) catch unreachable;
// Free the physical pages // Free the physical pages
mm.phys.free_batch(mm.pfn_to_struct_page(list.head), mm.pfn_to_struct_page(list.tail), size / mm.page_size); mm.phys.free_batch(
mm.pfn_to_struct_page(list.head),
mm.pfn_to_struct_page(list.tail),
size / mm.page_size,
);
return; return;
}; };
} }
src/mm/vmem.zig
+58 -16
View File
@@ -113,7 +113,10 @@ const Segment = struct {
return @fieldParentPtr("linkage", linkage); return @fieldParentPtr("linkage", linkage);
} }
fn cmp(a_node: *const rtl.bst.Node, b_node: *const rtl.bst.Node) std.math.Order { fn cmp(
a_node: *const rtl.bst.Node,
b_node: *const rtl.bst.Node,
) std.math.Order {
const a = Segment.from_tree_link(@constCast(a_node)); const a = Segment.from_tree_link(@constCast(a_node));
const b = Segment.from_tree_link(@constCast(b_node)); const b = Segment.from_tree_link(@constCast(b_node));
return std.math.order(a.base, b.base); return std.math.order(a.base, b.base);
@@ -147,7 +150,13 @@ pub const Arena = struct {
const Self = @This(); const Self = @This();
/// Initialize the area for use. /// Initialize the area for use.
pub fn init(self: *Self, name: []const u8, base: usize, size: usize, quantum: usize) !void { pub fn init(
self: *Self,
name: []const u8,
base: usize,
size: usize,
quantum: usize,
) !void {
self.name = name; self.name = name;
self.base = base; self.base = base;
self.size = size; self.size = size;
@@ -260,7 +269,13 @@ pub const Arena = struct {
// the quantum check avoids creating a remainder too small to ever be useful. // the quantum check avoids creating a remainder too small to ever be useful.
if (seg.size != size and (seg.size - size) > self.quantum - 1) { if (seg.size != size and (seg.size - size) > self.quantum - 1) {
const new_seg = try self.alloc_segment(); const new_seg = try self.alloc_segment();
new_seg.* = .{ .type = .Allocated, .base = seg.base, .size = size, .link = undefined, .linkage = .{ .tree_link = undefined } }; new_seg.* = .{
.type = .Allocated,
.base = seg.base,
.size = size,
.link = undefined,
.linkage = .{ .tree_link = undefined },
};
seg.base += size; seg.base += size;
seg.size -= size; seg.size -= size;
@@ -270,15 +285,18 @@ pub const Arena = struct {
new_seg.link.insert_before(&seg.link); new_seg.link.insert_before(&seg.link);
self.add_segment_to_freelist(seg); self.add_segment_to_freelist(seg);
self.allocated_segments.insert(&new_seg.linkage.tree_link) catch return error.InvalidAddress; self.allocated_segments.insert(&new_seg.linkage.tree_link) catch
return error.InvalidAddress;
if (options.policy == .NextFit) self.rotor = new_seg; if (options.policy == .NextFit) self.rotor = new_seg;
return new_seg.base; return new_seg.base;
} else { } else {
// seg is exactly the right size (or remainder is too small), use it directly. // seg is exactly the right size (or remainder is too small),
// use it directly.
seg.type = .Allocated; seg.type = .Allocated;
seg.linkage = .{ .tree_link = undefined }; seg.linkage = .{ .tree_link = undefined };
self.allocated_segments.insert(&seg.linkage.tree_link) catch return error.InvalidAddress; self.allocated_segments.insert(&seg.linkage.tree_link) catch
return error.InvalidAddress;
if (options.policy == .NextFit) self.rotor = seg; if (options.policy == .NextFit) self.rotor = seg;
return seg.base; return seg.base;
@@ -295,7 +313,9 @@ pub const Arena = struct {
.linkage = .{ .tree_link = undefined }, .linkage = .{ .tree_link = undefined },
}; };
const node = self.allocated_segments.tree.search(&search_node.linkage.tree_link) orelse return error.InvalidAddress; const node = self.allocated_segments.tree.search(
&search_node.linkage.tree_link,
) orelse return error.InvalidAddress;
const seg = Segment.from_tree_link(node); const seg = Segment.from_tree_link(node);
@@ -310,7 +330,8 @@ pub const Arena = struct {
const next_seg: *Segment = @fieldParentPtr("link", next_entry); const next_seg: *Segment = @fieldParentPtr("link", next_entry);
if (next_seg.type == .Free) { if (next_seg.type == .Free) {
// Remove next segment from freelist and segment list, then merge into seg. // Remove next segment from freelist and segment list,
// then merge into seg.
self.remove_segment_from_freelist(next_seg); self.remove_segment_from_freelist(next_seg);
next_seg.link.remove(); next_seg.link.remove();
seg.size += next_seg.size; seg.size += next_seg.size;
@@ -326,7 +347,8 @@ pub const Arena = struct {
const prev_seg: *Segment = @fieldParentPtr("link", prev_entry); const prev_seg: *Segment = @fieldParentPtr("link", prev_entry);
if (prev_seg.type == .Free) { if (prev_seg.type == .Free) {
// Remove previous segment from freelist and segment list, then merge into seg. // Remove previous segment from freelist and segment list,
// then merge into seg.
self.remove_segment_from_freelist(prev_seg); self.remove_segment_from_freelist(prev_seg);
prev_seg.link.remove(); prev_seg.link.remove();
seg.base = prev_seg.base; seg.base = prev_seg.base;
@@ -344,7 +366,11 @@ pub const Arena = struct {
} }
// Implementation for the instant fit policy. // Implementation for the instant fit policy.
fn instant_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } { fn instant_fit(
self: *Self,
size: usize,
options: AllocOptions,
) ?struct { usize, *Segment } {
var idx = freelist_index(size); var idx = freelist_index(size);
if (!std.math.isPowerOfTwo(size)) idx += 1; if (!std.math.isPowerOfTwo(size)) idx += 1;
@@ -356,7 +382,8 @@ pub const Arena = struct {
var it = self.freelists[idx].iterator(); var it = self.freelists[idx].iterator();
while (it.next()) : (it.advance()) { while (it.next()) : (it.advance()) {
const seg = Segment.from_free_link(it.get()); const seg = Segment.from_free_link(it.get());
if (self.try_to_fit(seg, size, options)) |addr| return .{ addr, seg }; if (self.try_to_fit(seg, size, options)) |addr|
return .{ addr, seg };
} }
} }
return null; return null;
@@ -364,7 +391,11 @@ pub const Arena = struct {
// Implementation for the best fit policy. // Implementation for the best fit policy.
// Find the smallest segment that can satisfy the request. // Find the smallest segment that can satisfy the request.
fn best_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } { fn best_fit(
self: *Self,
size: usize,
options: AllocOptions,
) ?struct { usize, *Segment } {
var idx = freelist_index(size); var idx = freelist_index(size);
var best_seg: ?*Segment = null; var best_seg: ?*Segment = null;
var best_start: usize = 0; var best_start: usize = 0;
@@ -388,8 +419,13 @@ pub const Arena = struct {
} }
// Implementation for the next fit policy. // Implementation for the next fit policy.
// Continue searching from the last allocated segment, wrapping around at the end of the arena. // Continue searching from the last allocated segment, wrapping around at
fn next_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } { // the end of the arena.
fn next_fit(
self: *Self,
size: usize,
options: AllocOptions,
) ?struct { usize, *Segment } {
// Start from rotor if we have one, otherwise from the beginning. // Start from rotor if we have one, otherwise from the beginning.
const start_entry = if (self.rotor) |r| r.link.next else self.list.first(); const start_entry = if (self.rotor) |r| r.link.next else self.list.first();
@@ -418,9 +454,15 @@ pub const Arena = struct {
// Try to fit the given segment to the request with the given options. // Try to fit the given segment to the request with the given options.
// Returns the fitted address if successful. // Returns the fitted address if successful.
fn try_to_fit(self: *Self, segment: *const Segment, size: usize, options: AllocOptions) ?usize { fn try_to_fit(
self: *Self,
segment: *const Segment,
size: usize,
options: AllocOptions,
) ?usize {
var start = @max(segment.base, options.min orelse 0); var start = @max(segment.base, options.min orelse 0);
const end = @min(segment.base + segment.size, options.max orelse std.math.maxInt(usize)); const end = @min(segment.base + segment.size, options.max orelse
std.math.maxInt(usize));
const alignment = options.alignment orelse self.quantum; const alignment = options.alignment orelse self.quantum;
if (start > end) return null; if (start > end) return null;
src/mm/zone.zig
+246 -94
View File
@@ -18,9 +18,10 @@
//! ## Per-CPU layer //! ## Per-CPU layer
//! ---------------- //! ----------------
//! Each CPU has an `alloc` and `free` magazine, where objects are allocated //! Each CPU has an `alloc` and `free` magazine, where objects are allocated
//! from and freed to, respectively. This separation avoids hysteresis and is useful //! from and freed to, respectively. This separation avoids hysteresis and is
//! for safe memory reclamation (see below). //! useful for safe memory reclamation (see below).
//! When one side runs dry/full, the magazines may be swapped, if SMR is disabled. //! When one side runs dry/full, the magazines may be swapped,
//! if SMR is disabled.
//! Each CPU also has its own depot, from which it allocates and frees its //! Each CPU also has its own depot, from which it allocates and frees its
//! magazines to first. If there is no magazine in the local depot, the magazine //! magazines to first. If there is no magazine in the local depot, the magazine
//! will be provided from the zone depot layer. We keep track of the size of the //! will be provided from the zone depot layer. We keep track of the size of the
@@ -42,11 +43,11 @@
//! ## Slab layer //! ## Slab layer
//! ------------- //! -------------
//! The slab layer is the fallback when all previous layers failed. A slab //! The slab layer is the fallback when all previous layers failed. A slab
//! represents a contiguous chunk of memory holding a fixed number of constructed //! represents a contiguous chunk of memory holding a fixed number of
//! objects, determined when the zone is created, and is allocated via the //! constructed objects, and is allocated via the kernel's virtual memory
//! kernel's virtual memory allocator. Allocation in the slab is managed by a //! allocator. Allocation in the slab is managed by a bitmap and slab metadata
//! bitmap and slab metadata is retrieved through the `mm.Page` structure //! is retrieved through the `mm.Page` structure representing the page
//! representing the page of allocated memory. //! of allocated memory.
//! //!
//! ## Hardening features //! ## Hardening features
//! --------------------- //! ---------------------
@@ -90,10 +91,10 @@
//! commits. When alloc needs a full magazine, it first polls the sequence of //! commits. When alloc needs a full magazine, it first polls the sequence of
//! the head of the depot magazine list (since SMR zones are forced //! the head of the depot magazine list (since SMR zones are forced
//! FIFO and stamps are taken in monotonic order, the head always carries the //! FIFO and stamps are taken in monotonic order, the head always carries the
//! minimum sequence). If the local head has not expired, the local full magazines //! minimum sequence). If the local head has not expired, the local full
//! are moved and appended to the zone depot to age, and their sequences //! magazines are moved and appended to the zone depot to age,
//! are committed, while expired magazines from the zone depot are pulled back in //! and their sequences are committed, while expired magazines from the zone
//! to satisfy the request. //! depot are pulled back in to satisfy the request.
//! //!
//! All this machinery is to avoid expensive scans during SMR polling, //! All this machinery is to avoid expensive scans during SMR polling,
//! growing the per-CPU depots widens the window between a stamp and its poll //! growing the per-CPU depots widens the window between a stamp and its poll
@@ -134,19 +135,35 @@ const zones_num = 32;
const magazine_size = ke.Tunable(u8, 8, "mm.zone.mag_size"); const magazine_size = ke.Tunable(u8, 8, "mm.zone.mag_size");
/// How much extra memory CPUs are allowed to keep around. /// How much extra memory CPUs are allowed to keep around.
const max_local_memory = ke.Tunable(u32, @intCast(r.kib(128)), "mm.zone.max_local_mem"); const max_local_memory = ke.Tunable(
u32,
@intCast(r.kib(128)),
"mm.zone.max_local_mem",
);
/// Number of contentions allowed per second before the depot grows. /// Number of contentions allowed per second before the depot grows.
const depot_grow_level = ke.Tunable(u32, 5 * wma_unit, "mm.zone.depot_grow_level"); const depot_grow_level = ke.Tunable(
u32,
5 * wma_unit,
"mm.zone.depot_grow_level",
);
/// Number of contentions allowed per second before the depot shrinks. /// Number of contentions allowed per second before the depot shrinks.
const depot_shrink_level = ke.Tunable(u32, wma_unit / 2, "mm.zone.depot_shrink_level"); const depot_shrink_level = ke.Tunable(
u32,
wma_unit / 2,
"mm.zone.depot_shrink_level",
);
/// Number of excess magazines in a zone before they are trimmed. /// Number of excess magazines in a zone before they are trimmed.
const excess_magazines = ke.Tunable(u32, 8, "mm.zone.excess_mags"); const excess_magazines = ke.Tunable(u32, 8, "mm.zone.excess_mags");
/// Excess memory in a zone before magazines are trimmed. /// Excess memory in a zone before magazines are trimmed.
const excess_memory = ke.Tunable(u32, @intCast(r.kib(16)), "mm.zone.excess_mem"); const excess_memory = ke.Tunable(
u32,
@intCast(r.kib(16)),
"mm.zone.excess_mem",
);
/// Every power-of-two-size from 8 to 2048 (inclusively). /// Every power-of-two-size from 8 to 2048 (inclusively).
const generic_zones_num = 9; const generic_zones_num = 9;
@@ -189,7 +206,8 @@ const should_check_poison = config.slab_check_poison;
const AllocOpts = struct { policy: mm.WaitPolicy = .WaitForMemory }; const AllocOpts = struct { policy: mm.WaitPolicy = .WaitForMemory };
/// A slab is a contiguous memory region that holds multiple objects of the same type. /// A slab is a contiguous memory region that holds multiple objects of the same
/// type.
const Slab = struct { const Slab = struct {
/// Linkage into a zone's list of slabs. /// Linkage into a zone's list of slabs.
link: rtl.List.Entry, link: rtl.List.Entry,
@@ -300,7 +318,11 @@ const Slab = struct {
// since a smaller capacity means a smaller bitmap, which may then fit. // since a smaller capacity means a smaller bitmap, which may then fit.
while (capacity > 0) : (capacity -= 1) { while (capacity > 0) : (capacity -= 1) {
const objects_off = const objects_off =
std.mem.alignForward(usize, @sizeOf(Slab) + Slab.bitmap_bytes(capacity), alignment); std.mem.alignForward(
usize,
@sizeOf(Slab) + Slab.bitmap_bytes(capacity),
alignment,
);
if (objects_off + capacity * chunk_size <= mm.page_size) { if (objects_off + capacity * chunk_size <= mm.page_size) {
break; break;
@@ -352,7 +374,11 @@ const Depot = struct {
return ret; return ret;
} }
fn maglist_free(list: *MagazineList, mag: *Magazine, policy: ReusePolicy) void { fn maglist_free(
list: *MagazineList,
mag: *Magazine,
policy: ReusePolicy,
) void {
mag.next = null; mag.next = null;
if (policy == .FIFO) { if (policy == .FIFO) {
@@ -430,7 +456,8 @@ inline fn check_poison(ptr: *anyopaque, len: usize, pattern: u32) bool {
var i: usize = 0; var i: usize = 0;
while (i + 4 <= len) : (i += 4) { while (i + 4 <= len) : (i += 4) {
if (std.mem.readInt(u32, bytes[i..][0..4], .little) != pattern) return false; if (std.mem.readInt(u32, bytes[i..][0..4], .little) != pattern)
return false;
} }
if (i < len) { if (i < len) {
@@ -454,7 +481,8 @@ inline fn wma_mix(old: usize, new: usize) usize {
} }
fn cpus_alloc() []rtl.CachePadded(Cpu) { fn cpus_alloc() []rtl.CachePadded(Cpu) {
const ptr = cpu_zone.alloc(.{}) catch @panic("failed to allocate zone CPU state"); const ptr = cpu_zone.alloc(.{}) catch
@panic("failed to allocate zone CPU state");
const raw: [*]rtl.CachePadded(Cpu) = @ptrCast(@alignCast(ptr)); const raw: [*]rtl.CachePadded(Cpu) = @ptrCast(@alignCast(ptr));
return raw[0..ke.ncpus]; return raw[0..ke.ncpus];
} }
@@ -505,7 +533,7 @@ pub const Zone = struct {
const Self = @This(); const Self = @This();
const InitOptions = struct { const InitOptions = struct {
alignment: u16 = 0, alignment: u16 = slab_align,
ctor: ?*const fn (*anyopaque) void = null, ctor: ?*const fn (*anyopaque) void = null,
dtor: ?*const fn (*anyopaque) void = null, dtor: ?*const fn (*anyopaque) void = null,
magazines: bool = true, magazines: bool = true,
@@ -515,11 +543,19 @@ pub const Zone = struct {
}; };
/// Initialize a zone. /// Initialize a zone.
pub fn init(self: *Self, name: []const u8, size: usize, options: InitOptions) void { pub fn init(
const obj_align = if (options.alignment == 0) slab_align else options.alignment; self: *Self,
name: []const u8,
size: usize,
options: InitOptions,
) void {
const obj_align = options.alignment;
const chunk_size = std.mem.alignForward(usize, size, obj_align); const chunk_size = std.mem.alignForward(usize, size, obj_align);
const slab_size = if (size <= small_slab_size) mm.page_size else Slab.calc_slab_size(chunk_size); const slab_size = if (size <= small_slab_size)
mm.page_size
else
Slab.calc_slab_size(chunk_size);
self.full_slabs.init(); self.full_slabs.init();
self.partial_slabs.init(); self.partial_slabs.init();
@@ -538,12 +574,17 @@ pub const Zone = struct {
self.depot_contention_cur = 0; self.depot_contention_cur = 0;
self.depot_contention_wma = 0; self.depot_contention_wma = 0;
self.cpu_depot_size = 0; self.cpu_depot_size = 0;
self.cpu_depot_limit = max_local_memory.load() / (self.chunk_size * magazine_size.load()); self.cpu_depot_limit = max_local_memory.load() /
(self.chunk_size * magazine_size.load());
self.trim_depot = false; self.trim_depot = false;
// SMR zones have to be FIFO so that the minimum sequence can be // SMR zones have to be FIFO so that the minimum sequence can be
// obtained by checking the list head. // obtained by checking the list head.
self.reuse_policy = if (options.smr != null) .FIFO else options.reuse_policy; self.reuse_policy = if (options.smr != null)
.FIFO
else
options.reuse_policy;
self.smr = options.smr; self.smr = options.smr;
self.smr_reclaim = options.smr_reclaim; self.smr_reclaim = options.smr_reclaim;
@@ -608,8 +649,9 @@ pub const Zone = struct {
if (cpu.free_rounds > 0 and self.smr == null) { if (cpu.free_rounds > 0 and self.smr == null) {
std.debug.assert(cpu.free != null); std.debug.assert(cpu.free != null);
// Alloc magazine is empty. If the free magazine has items, swap them. // Alloc magazine is empty. If the free magazine has items,
// The free magazine now becomes our alloc magazine. // swap them. The free magazine now becomes our alloc
// magazine.
std.mem.swap(?*Magazine, &cpu.alloc, &cpu.free); std.mem.swap(?*Magazine, &cpu.alloc, &cpu.free);
std.mem.swap(usize, &cpu.alloc_rounds, &cpu.free_rounds); std.mem.swap(usize, &cpu.alloc_rounds, &cpu.free_rounds);
continue; continue;
@@ -622,7 +664,11 @@ pub const Zone = struct {
// Discard our empty alloc magazine to the depot. // Discard our empty alloc magazine to the depot.
if (cpu.alloc) |empty_mag| { if (cpu.alloc) |empty_mag| {
Depot.free(empty_mag, &cpu.depot.empty_mags, self.reuse_policy); Depot.free(
empty_mag,
&cpu.depot.empty_mags,
self.reuse_policy,
);
} }
self.magazine_reuse(full_mag); self.magazine_reuse(full_mag);
@@ -632,8 +678,8 @@ pub const Zone = struct {
} }
// Local depot is empty, rebalance with the zone depot. // Local depot is empty, rebalance with the zone depot.
// Purposefully race against cpu_depot_size because locking is tricky here, // Purposefully race against cpu_depot_size because locking is
// we may get a stale value but it is harmless. // tricky here, we may get a stale value but it is harmless.
const n = @atomicLoad(usize, &self.cpu_depot_size, .monotonic); const n = @atomicLoad(usize, &self.cpu_depot_size, .monotonic);
self.alloc_depot_rebalance(n, cpu); self.alloc_depot_rebalance(n, cpu);
@@ -644,7 +690,11 @@ pub const Zone = struct {
// Discard our empty alloc magazine to the depot. // Discard our empty alloc magazine to the depot.
if (cpu.alloc) |empty_mag| { if (cpu.alloc) |empty_mag| {
Depot.free(empty_mag, &cpu.depot.empty_mags, self.reuse_policy); Depot.free(
empty_mag,
&cpu.depot.empty_mags,
self.reuse_policy,
);
} }
self.magazine_reuse(full_mag); self.magazine_reuse(full_mag);
@@ -670,9 +720,10 @@ pub const Zone = struct {
self.lock.acquire(); self.lock.acquire();
// Note: this is racy and could lead to two threads creating a slab at the same time. // Note: this is racy and could lead to two threads creating a slab
// Worst case scenario, we have one slab too many, so we don't care; we already did the work required // at the same time. Worst case scenario, we have one slab too many,
// for slab construction and this ideally should not happen very often. // so we don't care; we already did the work required for slab
// construction and this ideally should not happen very often.
self.partial_slabs.insert_tail(&slab.link); self.partial_slabs.insert_tail(&slab.link);
} else { } else {
slab = @fieldParentPtr("link", self.partial_slabs.first()); slab = @fieldParentPtr("link", self.partial_slabs.first());
@@ -718,7 +769,9 @@ pub const Zone = struct {
cpu.lock.acquire_no_ipl(); cpu.lock.acquire_no_ipl();
const freed = while (true) { const freed = while (true) {
if (cpu.free != null and cpu.free_rounds < magazine_size.load()) { if (cpu.free != null and
cpu.free_rounds < magazine_size.load())
{
// Fast path: push directly to free magazine. // Fast path: push directly to free magazine.
// For SMR zones the dtor and poisoning are deferred until // For SMR zones the dtor and poisoning are deferred until
// the magazine is reused, readers may still hold the object. // the magazine is reused, readers may still hold the object.
@@ -741,8 +794,12 @@ pub const Zone = struct {
break true; break true;
} }
if (cpu.alloc_rounds == 0 and cpu.alloc != null and self.smr == null) { if (cpu.alloc_rounds == 0 and
// Free magazine is full. If the alloc magazine is completely empty, swap them. cpu.alloc != null and
self.smr == null)
{
// Free magazine is full.
// If the alloc magazine is completely empty, swap them.
// The empty alloc magazine becomes our new free magazine. // The empty alloc magazine becomes our new free magazine.
std.mem.swap(?*Magazine, &cpu.alloc, &cpu.free); std.mem.swap(?*Magazine, &cpu.alloc, &cpu.free);
std.mem.swap(usize, &cpu.alloc_rounds, &cpu.free_rounds); std.mem.swap(usize, &cpu.alloc_rounds, &cpu.free_rounds);
@@ -754,7 +811,11 @@ pub const Zone = struct {
if (Depot.alloc(&cpu.depot.empty_mags)) |empty_mag| { if (Depot.alloc(&cpu.depot.empty_mags)) |empty_mag| {
// Put the free magazine on the full list. // Put the free magazine on the full list.
if (cpu.free) |full_mag| { if (cpu.free) |full_mag| {
Depot.free(full_mag, &cpu.depot.full_mags, self.reuse_policy); Depot.free(
full_mag,
&cpu.depot.full_mags,
self.reuse_policy,
);
} }
cpu.free = empty_mag; cpu.free = empty_mag;
@@ -772,7 +833,11 @@ pub const Zone = struct {
if (Depot.alloc(&cpu.depot.empty_mags)) |empty_mag| { if (Depot.alloc(&cpu.depot.empty_mags)) |empty_mag| {
// Put the free magazine on the full list. // Put the free magazine on the full list.
if (cpu.free) |full_mag| { if (cpu.free) |full_mag| {
Depot.free(full_mag, &cpu.depot.full_mags, self.reuse_policy); Depot.free(
full_mag,
&cpu.depot.full_mags,
self.reuse_policy,
);
} }
cpu.free = empty_mag; cpu.free = empty_mag;
@@ -784,7 +849,12 @@ pub const Zone = struct {
// We need to drop IPL and the CPU lock here because we need to acquire // We need to drop IPL and the CPU lock here because we need to acquire
// the magazine's allocator's blocking lock. // the magazine's allocator's blocking lock.
cpu.lock.release(ipl); cpu.lock.release(ipl);
const new_mag: ?*Magazine = @ptrCast(@alignCast(magazine_zone.alloc(.{ .policy = .DontWaitForMemory }) catch null));
const raw_mag = magazine_zone.alloc(
.{ .policy = .DontWaitForMemory },
) catch null;
const new_mag: ?*Magazine = @ptrCast(@alignCast(raw_mag));
// Re-grab our current context. // Re-grab our current context.
ipl = ke.ipl.raise(.Dispatch); ipl = ke.ipl.raise(.Dispatch);
@@ -831,12 +901,18 @@ pub const Zone = struct {
// Find the slab for the buffer. // Find the slab for the buffer.
const slab: *Slab = if (self.obj_size > small_slab_size) blk: { const slab: *Slab = if (self.obj_size > small_slab_size) blk: {
const page_va = std.mem.alignBackward(usize, @intFromPtr(obj), mm.page_size); const page_va = std.mem.alignBackward(
usize,
@intFromPtr(obj),
mm.page_size,
);
const phys = mi.kernel_space.pmap.query(page_va).?; const phys = mi.kernel_space.pmap.query(page_va).?;
const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys)); const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys));
break :blk page.alloced.slab_data.slab; break :blk page.alloced.slab_data.slab;
} else blk: { } else blk: {
break :blk @ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(obj), mm.page_size)); break :blk @ptrFromInt(
std.mem.alignBackward(usize, @intFromPtr(obj), mm.page_size),
);
}; };
const was_full = slab.refcount == slab.capacity; const was_full = slab.refcount == slab.capacity;
@@ -860,34 +936,46 @@ pub const Zone = struct {
} }
/// Periodic updates on the zone. /// Periodic updates on the zone.
/// Updates the working set and trims it if needed (for now, will do magazine resizing later...). /// Updates the magazine working set and trims or grows them if needed.
fn update(self: *Self) void { fn update(self: *Self) void {
const ipl = self.depot_lock.acquire(); const ipl = self.depot_lock.acquire();
// Update the working set size. // Update the working set size.
// This tracks the minimum number of magazines required, we assume it is safe // This tracks the minimum number of magazines required,
// to reclaim `min` magazines, since the number of magazines never went below that, and // we assume it is safe to reclaim `min` magazines, since the number of
// those magazines sat unused. e.g if the number of magazines ranged between 47 and 37 in an update // magazines never went below that, and those magazines sat unused. e.g
// interval, then the working set size is 10 and we can reclaim 37 magazines. We keep track of // if the number of magazines ranged between 47 and 37 in an update
// this in a weighted moving average (WMA) to bias recent utilization when trimming. // interval, then the working set size is 10 and we can
self.depot.empty_mags.wma = wma_mix(self.depot.empty_mags.wma, self.depot.empty_mags.min); // reclaim 37 magazines. We keep track of this in a weighted moving
// average (WMA) to bias recent utilization when trimming.
self.depot.empty_mags.wma = wma_mix(
self.depot.empty_mags.wma,
self.depot.empty_mags.min,
);
self.depot.empty_mags.min = self.depot.empty_mags.num; self.depot.empty_mags.min = self.depot.empty_mags.num;
self.depot.full_mags.wma = wma_mix(self.depot.full_mags.wma, self.depot.full_mags.min); self.depot.full_mags.wma = wma_mix(
self.depot.full_mags.wma,
self.depot.full_mags.min,
);
self.depot.full_mags.min = self.depot.full_mags.num; self.depot.full_mags.min = self.depot.full_mags.num;
// Calculate the number of contentions/second in fixed-point. // Calculate the number of contentions/second in fixed-point.
const old = self.depot_contention_wma; const old = self.depot_contention_wma;
var cur = self.depot_contention_cur * wma_unit / (ke.ncpus * update_interval_s); var cur = self.depot_contention_cur * wma_unit /
(ke.ncpus * update_interval_s);
// WMA formula. // WMA formula.
cur = (3 * old + cur) / 4; cur = (3 * old + cur) / 4;
if (self.use_magazines) { if (self.use_magazines) {
if (self.cpu_depot_size < self.cpu_depot_limit and cur > depot_grow_level.load()) { if (self.cpu_depot_size < self.cpu_depot_limit and
cur > depot_grow_level.load())
{
// We have room to grow the depot and we should. // We have room to grow the depot and we should.
// Put the new WMA at around midpoint between shrink and growth, so that // Put the new WMA at around midpoint between shrink and growth,
// we have time to check whether what we just did is good or not. // so that we have time to check whether what we just did is
// good or not.
cur = (depot_grow_level.load() + depot_shrink_level.load()) / 2; cur = (depot_grow_level.load() + depot_shrink_level.load()) / 2;
const size = if (self.cpu_depot_size == 0) const size = if (self.cpu_depot_size == 0)
@@ -898,7 +986,9 @@ pub const Zone = struct {
// Clamp it. // Clamp it.
self.cpu_depot_size = @min(size, self.cpu_depot_limit); self.cpu_depot_size = @min(size, self.cpu_depot_limit);
} else if (self.cpu_depot_size > 0 and cur <= depot_shrink_level.load()) { } else if (self.cpu_depot_size > 0 and
cur <= depot_shrink_level.load())
{
// We should shrink the depot. // We should shrink the depot.
cur = (depot_grow_level.load() + depot_shrink_level.load()) / 2; cur = (depot_grow_level.load() + depot_shrink_level.load()) / 2;
self.cpu_depot_size -= 1; self.cpu_depot_size -= 1;
@@ -926,19 +1016,28 @@ pub const Zone = struct {
if (!self.use_magazines) return false; if (!self.use_magazines) return false;
if (self.trim_depot) return true; if (self.trim_depot) return true;
const empty = @min(self.depot.empty_mags.wma, self.depot.empty_mags.min * wma_unit); const empty = @min(
self.depot.empty_mags.wma,
self.depot.empty_mags.min * wma_unit,
);
if (empty > excess_magazines.load() * wma_unit) { if (empty > excess_magazines.load() * wma_unit) {
// Too many excess magazines. // Too many excess magazines.
return true; return true;
} }
const full = @min(self.depot.full_mags.wma, self.depot.full_mags.min * wma_unit); const full = @min(
self.depot.full_mags.wma,
self.depot.full_mags.min * wma_unit,
);
const full_bytes = full * magazine_size.load() * self.chunk_size; const full_bytes = full * magazine_size.load() * self.chunk_size;
if (full >= 2 * wma_unit and full_bytes >= excess_memory.load() * wma_unit) { if (full >= 2 * wma_unit and
// We have at least 2 excess full magazines and they take up at least 16 KiB. full_bytes >= excess_memory.load() * wma_unit)
{
// We have at least 2 excess full magazines
// and they take up at least 16 KiB.
return true; return true;
} }
@@ -948,8 +1047,8 @@ pub const Zone = struct {
/// Called periodically to trim our magazines. /// Called periodically to trim our magazines.
/// This frees the excess magazines. /// This frees the excess magazines.
fn trim(self: *Self, trim_depot: bool) void { fn trim(self: *Self, trim_depot: bool) void {
// Note: we do all of this under the zone lock, // Note: we do all of this under the zone lock, but it *should* be a
// but it *should* be a fairly quick operation, even though it is O(ncpus). // a fairly quick operation, even though it is O(ncpus).
self.lock.acquire(); self.lock.acquire();
self.trim_maglist(&self.depot.empty_mags, 0); self.trim_maglist(&self.depot.empty_mags, 0);
@@ -975,8 +1074,7 @@ pub const Zone = struct {
fn drain(self: *Self) void { fn drain(self: *Self) void {
if (!self.use_magazines) return; if (!self.use_magazines) return;
// Note: we do all of this under the zone lock, // Note: we do all of this under the zone lock, same as above.
// but it *should* be a fairly quick operation, even though it is O(ncpus).
self.lock.acquire(); self.lock.acquire();
const ipl = self.depot_lock.acquire(); const ipl = self.depot_lock.acquire();
@@ -1055,7 +1153,11 @@ pub const Zone = struct {
// Move them to our depot. // Move them to our depot.
for (0..n) |_| { for (0..n) |_| {
Depot.free(Depot.alloc(&cpu.depot.full_mags).?, &depot.full_mags, self.reuse_policy); Depot.free(
Depot.alloc(&cpu.depot.full_mags).?,
&depot.full_mags,
self.reuse_policy,
);
} }
} }
@@ -1065,7 +1167,11 @@ pub const Zone = struct {
// Move them to our depot. // Move them to our depot.
for (0..n) |_| { for (0..n) |_| {
Depot.free(Depot.alloc(&cpu.depot.empty_mags).?, &depot.empty_mags, self.reuse_policy); Depot.free(
Depot.alloc(&cpu.depot.empty_mags).?,
&depot.empty_mags,
self.reuse_policy,
);
} }
} }
@@ -1108,7 +1214,8 @@ pub const Zone = struct {
} }
fn alloc_page(policy: mm.WaitPolicy) !*anyopaque { fn alloc_page(policy: mm.WaitPolicy) !*anyopaque {
const alloc_ret = mi.phys.alloc_opts(.{ .policy = policy }) orelse return error.OutOfMemory; const alloc_ret = mi.phys.alloc_opts(.{ .policy = policy }) orelse
return error.OutOfMemory;
return @ptrFromInt(mm.p2v(alloc_ret)); return @ptrFromInt(mm.p2v(alloc_ret));
} }
@@ -1129,7 +1236,8 @@ pub const Zone = struct {
slab.alloc_rr = 0; slab.alloc_rr = 0;
slab.refcount = 0; slab.refcount = 0;
slab.capacity = @intCast(capacity); slab.capacity = @intCast(capacity);
const meta_end = @intFromPtr(slab) + @sizeOf(Slab) + Slab.bitmap_bytes(capacity); const meta_end = @intFromPtr(slab) + @sizeOf(Slab) +
Slab.bitmap_bytes(capacity);
slab.base = std.mem.alignForward(usize, meta_end, self.alignment); slab.base = std.mem.alignForward(usize, meta_end, self.alignment);
const bm = slab.bitmap(); const bm = slab.bitmap();
@@ -1154,7 +1262,11 @@ pub const Zone = struct {
const capacity = (self.slab_size) / self.chunk_size; const capacity = (self.slab_size) / self.chunk_size;
const buf = try mi.heap.alloc(self.slab_size, policy); const buf = try mi.heap.alloc(self.slab_size, policy);
var ret: *Slab = @ptrCast(@alignCast(try gpa.alloc(u8, @sizeOf(Slab) + Slab.bitmap_bytes(capacity)))); var ret: *Slab = @ptrCast(
@alignCast(
try gpa.alloc(u8, @sizeOf(Slab) + Slab.bitmap_bytes(capacity)),
),
);
// Mark the bitmap as all free. // Mark the bitmap as all free.
const bm = ret.bitmap(); const bm = ret.bitmap();
@@ -1171,7 +1283,9 @@ pub const Zone = struct {
ret.alloc_rr = 0; ret.alloc_rr = 0;
for (0..self.slab_size / mm.page_size) |i| { for (0..self.slab_size / mm.page_size) |i| {
const phys_page = mi.kernel_space.pmap.query(@intFromPtr(buf) + i * mm.page_size) orelse @panic("Could not query page"); const phys_page = mi.kernel_space.pmap.query(
@intFromPtr(buf) + i * mm.page_size,
) orelse @panic("Could not query page");
const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys_page)); const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys_page));
page.alloced.slab_data.slab = ret; page.alloced.slab_data.slab = ret;
} }
@@ -1202,7 +1316,9 @@ pub const Zone = struct {
mm.heap.free(slab.base, self.slab_size); mm.heap.free(slab.base, self.slab_size);
} else { } else {
free_page(@ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(slab), mm.page_size))); free_page(@ptrFromInt(
std.mem.alignBackward(usize, @intFromPtr(slab), mm.page_size),
));
} }
} }
@@ -1257,9 +1373,10 @@ pub const Zone = struct {
ke.smr.deferred_advance_commit(smr, tail.seq); ke.smr.deferred_advance_commit(smr, tail.seq);
} }
/// Try to rebalance the zone and cpu depots w.r.t to each other after an allocation. /// Try to rebalance the zone and cpu depots w.r.t to each other
/// The CPU depot moves its extra empty magazines to the zone depot, and tries /// after an allocation. The CPU depot moves its extra full magazines to the
/// to get full magazines from the zone depot up to `target`. CPU lock is held. /// zone depot, and tries to get full magazines from the zone depot up to
/// `target`. CPU lock is held.
fn alloc_depot_rebalance(self: *Self, target: usize, cpu: *Cpu) void { fn alloc_depot_rebalance(self: *Self, target: usize, cpu: *Cpu) void {
if (!self.depot_lock.try_acquire_no_ipl()) { if (!self.depot_lock.try_acquire_no_ipl()) {
// Register the contention. // Register the contention.
@@ -1272,7 +1389,11 @@ pub const Zone = struct {
const n = cpu.depot.empty_mags.num - target / 2; const n = cpu.depot.empty_mags.num - target / 2;
for (0..n) |_| { for (0..n) |_| {
Depot.free(Depot.alloc(&cpu.depot.empty_mags).?, &self.depot.empty_mags, self.reuse_policy); Depot.free(
Depot.alloc(&cpu.depot.empty_mags).?,
&self.depot.empty_mags,
self.reuse_policy,
);
} }
} }
@@ -1280,27 +1401,35 @@ pub const Zone = struct {
// We got here because the local head hasn't expired or there is // We got here because the local head hasn't expired or there is
// no local magazine at all, rotate the local magazines through the // no local magazine at all, rotate the local magazines through the
// zone depot to let their sequences age. // zone depot to let their sequences age.
// We then hope that the head of the zone depot has expired, otherwise we just return // We then hope that the head of the zone depot has expired,
// and fall back to the slab layer. // otherwise we just return and fall back to the slab layer.
self.smr_rotate_full(smr, cpu); self.smr_rotate_full(smr, cpu);
} }
// Grab a batch of full magazines from the zone depot. // Grab a batch of full magazines from the zone depot.
// Exchange at least one magazine even when the depot size is 0. // Exchange at least one magazine even when the depot size is 0.
const n = @min(@max(target, 1) - cpu.depot.empty_mags.num, self.depot.full_mags.num); const n = @min(
@max(target, 1) - cpu.depot.empty_mags.num,
self.depot.full_mags.num,
);
if (n != 0 and self.full_mags_ready(&self.depot.full_mags)) { if (n != 0 and self.full_mags_ready(&self.depot.full_mags)) {
for (0..n) |_| { for (0..n) |_| {
Depot.free(Depot.alloc(&self.depot.full_mags).?, &cpu.depot.full_mags, self.reuse_policy); Depot.free(
Depot.alloc(&self.depot.full_mags).?,
&cpu.depot.full_mags,
self.reuse_policy,
);
} }
} }
self.depot_lock.release_no_ipl(); self.depot_lock.release_no_ipl();
} }
/// Try to rebalance the zone and cpu depots w.r.t to each other after a free. /// Try to rebalance the zone and cpu depots w.r.t to each other
/// The CPU depot moves its extra full magazines to the zone depot, and tries /// after a free.The CPU depot moves its extra full magazines to the
/// to get empty magazines from the zone depot up to `target`. CPU lock is held. /// zone depot, and tries to get empty magazines from the zone depot up to
/// `target`. CPU lock is held.
fn free_depot_rebalance(self: *Self, target: usize, cpu: *Cpu) void { fn free_depot_rebalance(self: *Self, target: usize, cpu: *Cpu) void {
if (!self.depot_lock.try_acquire_no_ipl()) { if (!self.depot_lock.try_acquire_no_ipl()) {
// Register the contention. // Register the contention.
@@ -1318,7 +1447,11 @@ pub const Zone = struct {
if (n != 0 and self.full_mags_ready(&self.depot.full_mags)) { if (n != 0 and self.full_mags_ready(&self.depot.full_mags)) {
for (0..n) |_| { for (0..n) |_| {
Depot.free(Depot.alloc(&self.depot.full_mags).?, &cpu.depot.full_mags, self.reuse_policy); Depot.free(
Depot.alloc(&self.depot.full_mags).?,
&cpu.depot.full_mags,
self.reuse_policy,
);
} }
} }
} else { } else {
@@ -1326,16 +1459,27 @@ pub const Zone = struct {
const n = cpu.depot.full_mags.num - target / 2; const n = cpu.depot.full_mags.num - target / 2;
for (0..n) |_| { for (0..n) |_| {
Depot.free(Depot.alloc(&cpu.depot.full_mags).?, &self.depot.full_mags, self.reuse_policy); Depot.free(
Depot.alloc(&cpu.depot.full_mags).?,
&self.depot.full_mags,
self.reuse_policy,
);
} }
} }
} }
// Grab a batch of empty magazines from the zone depot. // Grab a batch of empty magazines from the zone depot.
// Exchange at least one magazine even when the depot size is 0. // Exchange at least one magazine even when the depot size is 0.
const n = @min(@max(target, 1) - cpu.depot.full_mags.num, self.depot.empty_mags.num); const n = @min(
@max(target, 1) - cpu.depot.full_mags.num,
self.depot.empty_mags.num,
);
for (0..n) |_| { for (0..n) |_| {
Depot.free(Depot.alloc(&self.depot.empty_mags).?, &cpu.depot.empty_mags, self.reuse_policy); Depot.free(
Depot.alloc(&self.depot.empty_mags).?,
&cpu.depot.empty_mags,
self.reuse_policy,
);
} }
self.depot_lock.release_no_ipl(); self.depot_lock.release_no_ipl();
@@ -1403,9 +1547,13 @@ pub fn early_init() linksection(r.init) void {
zone.init(generic_zone_names[i], @as(usize, 1) << @intCast(i + 3), .{}); zone.init(generic_zone_names[i], @as(usize, 1) << @intCast(i + 3), .{});
} }
magazine_zone.init("magazine", @sizeOf(Magazine) + magazine_size.load() * @sizeOf(*anyopaque), .{ magazine_zone.init(
.magazines = false, "magazine",
}); @sizeOf(Magazine) + magazine_size.load() * @sizeOf(*anyopaque),
.{
.magazines = false,
},
);
} }
/// Post-SMP initialization. /// Post-SMP initialization.
@@ -1415,7 +1563,8 @@ pub fn late_init() linksection(r.init) void {
.alignment = std.atomic.cache_line, .alignment = std.atomic.cache_line,
}); });
// Go through all caches with magazines enabled and initialize their per-CPU magazines. // Go through all caches with magazines enabled and initialize
// their per-CPU magazines.
zone_list_lock.acquire(); zone_list_lock.acquire();
var zone = all_zones; var zone = all_zones;
@@ -1446,8 +1595,10 @@ pub fn late_init() linksection(r.init) void {
smr_zone.init("SMR", .{}); smr_zone.init("SMR", .{});
smr_cpu_zone.init("SMR CPU", @sizeOf(ke.smr.Cpu) * ke.ncpus, .{}); smr_cpu_zone.init("SMR CPU", @sizeOf(ke.smr.Cpu) * ke.ncpus, .{});
smr_sys_domain = smr_domain_create(false) catch @panic("Failed to create smr domain"); smr_sys_domain = smr_domain_create(false) catch
smr_sys_preempt_domain = smr_domain_create(true) catch @panic("Failed to create smr domain"); @panic("Failed to create smr domain");
smr_sys_preempt_domain = smr_domain_create(true) catch
@panic("Failed to create smr domain");
} }
pub fn smr_domain_create(preempt: bool) !*ke.smr.Domain { pub fn smr_domain_create(preempt: bool) !*ke.smr.Domain {
@@ -1528,7 +1679,8 @@ fn gpa_remap(
) ?[*]u8 { ) ?[*]u8 {
if (gpa_resize(undefined, buf, alignment, new_len, ret_addr)) return buf.ptr; if (gpa_resize(undefined, buf, alignment, new_len, ret_addr)) return buf.ptr;
const new_ptr = gpa_alloc(undefined, new_len, alignment, ret_addr) orelse return null; const new_ptr = gpa_alloc(undefined, new_len, alignment, ret_addr) orelse
return null;
const copy_len = @min(buf.len, new_len); const copy_len = @min(buf.len, new_len);
@memcpy(new_ptr[0..copy_len], buf[0..copy_len]); @memcpy(new_ptr[0..copy_len], buf[0..copy_len]);
gpa_free(undefined, buf, alignment, ret_addr); gpa_free(undefined, buf, alignment, ret_addr);
src/platform/acpi/hpet.zig
+2 -1
View File
@@ -21,7 +21,8 @@ const femtos_per_s = 1_000_000_000_000_000;
pub fn init(hpet: *acpi.HpetTable) linksection(r.init) void { pub fn init(hpet: *acpi.HpetTable) linksection(r.init) void {
hpet_regs = @ptrFromInt(mm.p2v(hpet.base_address.address)); hpet_regs = @ptrFromInt(mm.p2v(hpet.base_address.address));
hpet_timer.frequency = femtos_per_s / (r.mmio_read(u64, @intFromPtr(&hpet_regs.general_capabilities)) >> 32); hpet_timer.frequency = femtos_per_s /
(r.mmio_read(u64, @intFromPtr(&hpet_regs.general_capabilities)) >> 32);
// Disable timer. // Disable timer.
r.mmio_write(u64, @intFromPtr(&hpet_regs.general_configuration), 0); r.mmio_write(u64, @intFromPtr(&hpet_regs.general_configuration), 0);
src/platform/acpi/madt.zig
+3 -1
View File
@@ -11,7 +11,9 @@ pub const MadtIterator = struct {
pub fn next(self: *MadtIterator) ?*const acpi.MadtEntryHeader { pub fn next(self: *MadtIterator) ?*const acpi.MadtEntryHeader {
const entries_size = self.madt.header.length - @sizeOf(acpi.Madt); const entries_size = self.madt.header.length - @sizeOf(acpi.Madt);
if (self.i >= entries_size) return null; if (self.i >= entries_size) return null;
const entry: *const acpi.MadtEntryHeader = @ptrFromInt(@intFromPtr(&self.madt.entries) + self.i); const entry: *const acpi.MadtEntryHeader = @ptrFromInt(
@intFromPtr(&self.madt.entries) + self.i,
);
self.i += @max(2, entry.length); self.i += @max(2, entry.length);
return entry; return entry;
} }
src/platform/acpi/root.zig
+19 -5
View File
@@ -97,7 +97,8 @@ pub const Gas = extern struct {
register_bit_width: u8 align(1), register_bit_width: u8 align(1),
/// The bit offset of the given register at the given address. /// The bit offset of the given register at the given address.
register_bit_offset: u8 align(1), register_bit_offset: u8 align(1),
/// The access size of the given register. (1 = byte, 2 = word, 3 = dword, 4 = qword). /// The access size of the given register.
/// (1 = byte, 2 = word, 3 = dword, 4 = qword).
access_size: u8 align(1), access_size: u8 align(1),
address: u64 align(1), address: u64 align(1),
pub fn read(self: Gas) u64 { pub fn read(self: Gas) u64 {
@@ -142,9 +143,21 @@ pub const Gas = extern struct {
else => unreachable, else => unreachable,
}, },
.SystemIo => switch (self.access_size) { .SystemIo => switch (self.access_size) {
1 => arch.pio_write(u8, @truncate(self.address), @truncate(shifted_value)), 1 => arch.pio_write(
2 => arch.pio_write(u16, @truncate(self.address), @truncate(shifted_value)), u8,
3 => arch.pio_write(u32, @truncate(self.address), @truncate(shifted_value)), @truncate(self.address),
@truncate(shifted_value),
),
2 => arch.pio_write(
u16,
@truncate(self.address),
@truncate(shifted_value),
),
3 => arch.pio_write(
u32,
@truncate(self.address),
@truncate(shifted_value),
),
else => unreachable, else => unreachable,
}, },
else => @panic("unsupported GAS address space"), else => @panic("unsupported GAS address space"),
@@ -213,7 +226,8 @@ pub const Fadt = extern struct {
const version_sizes = [_]usize{ 116, 132, 244, 244, 268, 276 }; const version_sizes = [_]usize{ 116, 132, 244, 244, 268, 276 };
pub fn revision(self: Fadt) u8 { pub fn revision(self: Fadt) u8 {
// We can't rely on firmware to set the revision field correctly, so we infer it from the length field. // We can't rely on firmware to set the revision field correctly,
// so we infer it from the length field.
const declared: usize = self.header.revision; const declared: usize = self.header.revision;
const len: u32 = self.header.length; const len: u32 = self.header.length;
src/platform/pc/apic.zig
+24 -10
View File
@@ -90,11 +90,16 @@ fn xapic_read(register: ApicRegisters) u32 {
} }
fn x2apic_write(register: ApicRegisters, value: u64) void { fn x2apic_write(register: ApicRegisters, value: u64) void {
amd64.wrmsr(@intFromEnum(amd64.Msr.X2ApicBase) + (@as(u32, @intFromEnum(register)) >> 4), value); amd64.wrmsr(
@intFromEnum(amd64.Msr.X2ApicBase) + (@intFromEnum(register) >> 4),
value,
);
} }
fn x2apic_read(register: ApicRegisters) u64 { fn x2apic_read(register: ApicRegisters) u64 {
return amd64.rdmsr(@intFromEnum(amd64.Msr.X2ApicBase) + (@as(u32, @intFromEnum(register)) >> 4)); return amd64.rdmsr(
@intFromEnum(amd64.Msr.X2ApicBase) + (@intFromEnum(register) >> 4),
);
} }
pub fn write(register: ApicRegisters, value: u32) void { pub fn write(register: ApicRegisters, value: u32) void {
@@ -133,7 +138,10 @@ pub fn send_init(apic_id: u32) void {
pub fn send_sipi(apic_id: u32, startup_page: u8) void { pub fn send_sipi(apic_id: u32, startup_page: u8) void {
if (in_x2apic_mode.local().*) { if (in_x2apic_mode.local().*) {
x2apic_write(.Icr, (@as(u64, apic_id) << 32) | 0x00004600 | startup_page); x2apic_write(
.Icr,
(@as(u64, apic_id) << 32) | 0x00004600 | startup_page,
);
} else { } else {
xapic_write(.Icr1, @as(u32, apic_id) << 24); xapic_write(.Icr1, @as(u32, apic_id) << 24);
xapic_write(.Icr, @as(u32, 0x00004600) | startup_page); xapic_write(.Icr, @as(u32, 0x00004600) | startup_page);
@@ -143,7 +151,8 @@ pub fn send_sipi(apic_id: u32, startup_page: u8) void {
pub fn send_ipi(apic_id: u32, vector: u8, delivery_mode: u8) void { pub fn send_ipi(apic_id: u32, vector: u8, delivery_mode: u8) void {
if (in_x2apic_mode.local().*) { if (in_x2apic_mode.local().*) {
const icr_value = (@as(u64, apic_id) << 32) | (@as(u64, delivery_mode) << 8) | vector; const icr_value = (@as(u64, apic_id) << 32) |
(@as(u64, delivery_mode) << 8) | vector;
x2apic_write(.Icr, icr_value); x2apic_write(.Icr, icr_value);
} else { } else {
const icr_value = (@as(u32, delivery_mode) << 8) | vector; const icr_value = (@as(u32, delivery_mode) << 8) | vector;
@@ -200,10 +209,11 @@ pub fn arm_timer(ns: r.Nanoseconds) void {
if (amd64.cpu_features.tsc_deadline) { if (amd64.cpu_features.tsc_deadline) {
const now = amd64.rdtsc(); const now = amd64.rdtsc();
const delta: u64 = @intCast((@as(u128, ns.value) * tsc.tsc_timer.frequency) / std.time.ns_per_s); const delta: u64 = @intCast(
(@as(u128, ns.value) * tsc.tsc_timer.frequency) / std.time.ns_per_s,
);
const deadline = now +% delta; const deadline = now +% delta;
amd64.write_msr(.TscDeadline, deadline); amd64.write_msr(.TscDeadline, deadline);
return; return;
} }
@@ -242,7 +252,8 @@ pub fn init() linksection(r.init) void {
const cur_cpu_id = get_id(); const cur_cpu_id = get_id();
// First, we iterate to check if we find *any* APICs. If we do, then we must use them over x2APIC entries for IDs under 255. // First, we iterate to check if we find *any* APICs.
// If we do, then we must use them over x2APIC entries for IDs under 255.
var has_lapics = false; var has_lapics = false;
while (iter.next()) |entry| { while (iter.next()) |entry| {
switch (entry.type) { switch (entry.type) {
@@ -264,13 +275,16 @@ pub fn init() linksection(r.init) void {
.LocalApic => { .LocalApic => {
const lapic: *const MadtLapic = @ptrCast(entry); const lapic: *const MadtLapic = @ptrCast(entry);
if (lapic.flags & 1 == 0 or lapic.id == cur_cpu_id) continue; if (lapic.flags & 1 == 0 or lapic.id == cur_cpu_id) continue;
apics.append(mm.zone.gpa, lapic.id) catch @panic("Could not append APIC ID to list"); apics.append(mm.zone.gpa, lapic.id) catch
@panic("Could not append APIC ID to list");
}, },
.X2LocalApic => { .X2LocalApic => {
const x2lapic: *const MadtX2Lapic = @ptrCast(entry); const x2lapic: *const MadtX2Lapic = @ptrCast(entry);
if (has_lapics and x2lapic.id <= 0xFF) continue; if (has_lapics and x2lapic.id <= 0xFF) continue;
if (x2lapic.flags & 1 == 0 or x2lapic.id == cur_cpu_id) continue; if (x2lapic.flags & 1 == 0 or x2lapic.id == cur_cpu_id)
apics.append(mm.zone.gpa, x2lapic.id) catch @panic("Could not append APIC ID to list"); continue;
apics.append(mm.zone.gpa, x2lapic.id) catch
@panic("Could not append APIC ID to list");
}, },
else => {}, else => {},
} }
src/platform/pc/smp.zig
+45 -13
View File
@@ -37,10 +37,15 @@ fn ap_entry(cpu_id: u32) callconv(.c) noreturn {
ki.impl.init.ap_entry(cpu_id, &aps_booted); ki.impl.init.ap_entry(cpu_id, &aps_booted);
} }
fn make_thread(entrypoint: *const fn (?*anyopaque) void, stack: usize) *ke.Thread { fn make_thread(
const td = mm.zone.gpa.create(ke.Thread) catch @panic("Failed to allocate thread for AP"); entrypoint: *const fn (?*anyopaque) void,
stack: usize,
size: usize,
) *ke.Thread {
const td = mm.zone.gpa.create(ke.Thread) catch
@panic("Failed to allocate thread for AP");
td.init(stack, r.kib(32), ke.Thread.Priority.idle_low, entrypoint, null); td.init(stack, size, ke.Thread.Priority.idle_low, entrypoint, null);
td.pinned = true; td.pinned = true;
return td; return td;
@@ -65,8 +70,12 @@ pub fn init() linksection(r.init) void {
.Unknown => false, .Unknown => false,
}; };
const init_delay: r.Nanoseconds = .init(if (skip_delay) 0 else std.time.ns_per_ms * 10); const init_delay: r.Nanoseconds = .init(
const sipi_delay: r.Nanoseconds = .init(if (skip_delay) std.time.ns_per_us * 10 else std.time.ns_per_us * 300); if (skip_delay) 0 else std.time.ns_per_ms * 10,
);
const sipi_delay: r.Nanoseconds = .init(
if (skip_delay) std.time.ns_per_us * 10 else std.time.ns_per_us * 300,
);
const page: [*]u8 = @ptrFromInt(mm.p2v(0x8000)); const page: [*]u8 = @ptrFromInt(mm.p2v(0x8000));
@memcpy(page[0..trampoline_size], @as([*]u8, @ptrFromInt(trampoline_start))); @memcpy(page[0..trampoline_size], @as([*]u8, @ptrFromInt(trampoline_start)));
@@ -76,8 +85,11 @@ pub fn init() linksection(r.init) void {
const idtr = amd64.sidtr(); const idtr = amd64.sidtr();
const offsets = mm.zone.gpa.alloc(usize, apic.apics.items.len + 1) catch
@panic("Failed to allocate AP local data offsets");
// Allocate per-cpu offsets for CPU-local data. // Allocate per-cpu offsets for CPU-local data.
ki.impl.cpu_offsets = @ptrCast(mm.zone.gpa.alloc(usize, apic.apics.items.len + 1) catch @panic("Failed to allocate AP local data offsets")); ki.impl.cpu_offsets = @ptrCast(offsets);
const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start); const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start);
log.info("per-CPU data size: {} bytes", .{percpu_size}); log.info("per-CPU data size: {} bytes", .{percpu_size});
@@ -98,19 +110,39 @@ pub fn init() linksection(r.init) void {
cpu_id_to_apic_id[cpu_id] = apic_id; cpu_id_to_apic_id[cpu_id] = apic_id;
// Allocate per-cpu data. // Allocate per-cpu data.
const cpu_data = mm.zone.gpa.alloc(u8, percpu_size) catch @panic("Failed to allocate per-cpu data"); const cpu_data = mm.zone.gpa.alloc(u8, percpu_size) catch
@memcpy(cpu_data, @as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size]); @panic("Failed to allocate per-cpu data");
const self_offset_offset = @intFromPtr(&ki.impl.cpu_self_offset) - @intFromPtr(&__percpu_start); @memcpy(
ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -% @intFromPtr(&__percpu_start); cpu_data,
@as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size],
);
const self_offset_offset = @intFromPtr(&ki.impl.cpu_self_offset) -
@intFromPtr(&__percpu_start);
ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -%
@intFromPtr(&__percpu_start);
const off: *usize = @ptrCast(@alignCast(&cpu_data[self_offset_offset]));
// copy the offset into the AP's self_offset variable // copy the offset into the AP's self_offset variable
@as(*usize, @ptrCast(@alignCast(&cpu_data[self_offset_offset]))).* = ki.impl.cpu_offsets[cpu_id]; off.* = ki.impl.cpu_offsets[cpu_id];
const stack_top = @intFromPtr(mm.heap.alloc(r.kib(32), .DontWaitForMemory) catch @panic("Failed to allocate AP stack")) + r.kib(32); const stack_size = r.kib(16);
const stack_top = @intFromPtr(mm.heap.alloc(
stack_size,
.DontWaitForMemory,
) catch
@panic("Failed to allocate AP stack")) + stack_size;
start_stack.remote(@intCast(cpu_id)).* = stack_top & ~@as(usize, 15); start_stack.remote(@intCast(cpu_id)).* = stack_top & ~@as(usize, 15);
start_thread.remote(@intCast(cpu_id)).* = make_thread(@ptrCast(&ap_entry), stack_top - r.kib(32)); start_thread.remote(@intCast(cpu_id)).* = make_thread(
@ptrCast(&ap_entry),
stack_top - stack_size,
stack_size,
);
rtl.barrier.fence(.release); rtl.barrier.fence(.release);
src/proc/thread.zig
+5 -1
View File
@@ -55,7 +55,11 @@ pub fn init() void {
/// - `prio`: base priority of the thread. /// - `prio`: base priority of the thread.
/// - `entry`: entry point. /// - `entry`: entry point.
/// - `arg`: argument passed to `entry`. /// - `arg`: argument passed to `entry`.
pub fn create_kernel(prio: u8, entry: *const fn (arg: ?*anyopaque) void, arg: ?*anyopaque) !*Thread { pub fn create_kernel(
prio: u8,
entry: *const fn (arg: ?*anyopaque) void,
arg: ?*anyopaque,
) !*Thread {
var td = try thread_zone.create(); var td = try thread_zone.create();
const stack = try mm.heap.alloc(kernel_thread_stack_size, .WaitForMemory); const stack = try mm.heap.alloc(kernel_thread_stack_size, .WaitForMemory);
src/rcu/hashtable.zig
+41 -10
View File
@@ -133,7 +133,7 @@ const rehashing_reseed: u8 = 1 << 2;
const Core = struct { const Core = struct {
/// The current state of the hash table. /// The current state of the hash table.
state: std.atomic.Value(State), state: std.atomic.Value(State),
/// One pointer for current buckets, another for new buckets (during resizing). /// One pointer for current buckets, another for new buckets.
buckets: [2]std.atomic.Value([*]Bucket), buckets: [2]std.atomic.Value([*]Bucket),
/// One seed for current table, another for the new table. /// One seed for current table, another for the new table.
seeds: [2]std.atomic.Value(u32), seeds: [2]std.atomic.Value(u32),
@@ -153,7 +153,10 @@ const Core = struct {
policy: ResizePolicy, policy: ResizePolicy,
min_size: u32, min_size: u32,
) !void { ) !void {
const min: u32 = if (policy == .Compact) @max(2, min_size) else @max(16, min_size); const min: u32 = if (policy == .Compact)
@max(2, min_size)
else
@max(16, min_size);
const start_size = min; const start_size = min;
@@ -186,7 +189,13 @@ const Core = struct {
self.rehash_work.init(.Normal, rehash, self); self.rehash_work.init(.Normal, rehash, self);
} }
fn migrate_bucket(self: *Self, old: *Bucket, new_array: [*]Bucket, new_seed: u32, new_shift: u5) void { fn migrate_bucket(
self: *Self,
old: *Bucket,
new_array: [*]Bucket,
new_seed: u32,
new_shift: u5,
) void {
// This is called in worker thread context, so we have ample stack size // This is called in worker thread context, so we have ample stack size
// to work with. // to work with.
const flat_size = 256; const flat_size = 256;
@@ -331,7 +340,10 @@ const Core = struct {
/// Remove a given element from the table, SMR must be entered. /// Remove a given element from the table, SMR must be entered.
fn remove(self: *Self, link: *rcu.SList.Entry) void { fn remove(self: *Self, link: *rcu.SList.Entry) void {
var state = self.state.load(.acquire); var state = self.state.load(.acquire);
var h = self.hash_link(link, self.seeds[state.current_index].load(.monotonic)); var h = self.hash_link(
link,
self.seeds[state.current_index].load(.monotonic),
);
var b = self.bucket(state, .Cur, h); var b = self.bucket(state, .Cur, h);
@@ -343,7 +355,10 @@ const Core = struct {
// Get the new state instead. // Get the new state instead.
state = self.state.load(.acquire); state = self.state.load(.acquire);
h = self.hash_link(link, self.seeds[state.new_index].load(.monotonic)); h = self.hash_link(
link,
self.seeds[state.new_index].load(.monotonic),
);
b = self.bucket(state, .New, h); b = self.bucket(state, .New, h);
first = b.lock(); first = b.lock();
} }
@@ -425,7 +440,11 @@ const Core = struct {
}; };
/// Scalable, lock-free hashtable backed by SMR. /// Scalable, lock-free hashtable backed by SMR.
pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context: type) type { pub fn Table(
comptime T: type,
comptime link_field: []const u8,
comptime Context: type,
) type {
comptime rtl.assert_interface(Context, struct { comptime rtl.assert_interface(Context, struct {
pub const Key = Context.Key; pub const Key = Context.Key;
pub fn hash(key: Key, seed: usize) u32 { pub fn hash(key: Key, seed: usize) u32 {
@@ -470,7 +489,10 @@ pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context
/// Get an element given a key, SMR must be entered. /// Get an element given a key, SMR must be entered.
pub fn get(self: *Self, key: Context.Key) ?*T { pub fn get(self: *Self, key: Context.Key) ?*T {
const state = self.core.state.load(.acquire); const state = self.core.state.load(.acquire);
const h = Context.hash(key, self.core.seeds[state.current_index].load(.monotonic)); const h = Context.hash(
key,
self.core.seeds[state.current_index].load(.monotonic),
);
const b = self.core.bucket(state, .Cur, h); const b = self.core.bucket(state, .Cur, h);
var word = b.load(); var word = b.load();
@@ -504,7 +526,10 @@ pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context
value: *rcu.SList.Entry, value: *rcu.SList.Entry,
) ?*T { ) ?*T {
var state = self.core.state.load(.acquire); var state = self.core.state.load(.acquire);
var h = Context.hash(key, self.core.seeds[state.current_index].load(.monotonic)); var h = Context.hash(
key,
self.core.seeds[state.current_index].load(.monotonic),
);
var b = self.core.bucket(state, .Cur, h); var b = self.core.bucket(state, .Cur, h);
var first = b.lock(); var first = b.lock();
@@ -515,7 +540,10 @@ pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context
// Get the new state instead. // Get the new state instead.
state = self.core.state.load(.acquire); state = self.core.state.load(.acquire);
h = Context.hash(key, self.core.seeds[state.new_index].load(.monotonic)); h = Context.hash(
key,
self.core.seeds[state.new_index].load(.monotonic),
);
b = self.core.bucket(state, .New, h); b = self.core.bucket(state, .New, h);
first = b.lock(); first = b.lock();
} }
@@ -574,7 +602,10 @@ pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context
// Now do the same as in get(), but with the new state. // Now do the same as in get(), but with the new state.
const state = self.core.state.load(.acquire); const state = self.core.state.load(.acquire);
const h = Context.hash(key, self.core.seeds[state.new_index].load(.monotonic)); const h = Context.hash(
key,
self.core.seeds[state.new_index].load(.monotonic),
);
const b = self.core.bucket(state, .New, h); const b = self.core.bucket(state, .New, h);
var word: usize = b.load(); var word: usize = b.load();