mirrors/Zag
1
0
Fork 0
mirror of https://github.com/rdmsr/zag.git synced 2026-10-27 13:58:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

all: refactor coding style

Browse Source
This commit is contained in:
Loïc Grégoire
2026-08-05 17:43:46 -04:00
parent 2a09642f4d
commit bec872bc26
31 changed files with 887 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/arch/amd64/root.zig
+5 -1
View File
@@ -476,7 +476,11 @@ const PowerManagementInfoEdx = packed struct(u32) {
reserved1: u23,
};
fn assert_bit(comptime T: type, comptime field_name: []const u8, comptime expected_bit: u5) void {
fn assert_bit(
comptime T: type,
comptime field_name: []const u8,
comptime expected_bit: u5,
) void {
var v: T = @bitCast(@as(u32, 0));
@field(v, field_name) = true;
const raw: u32 = @bitCast(v);
src/ex/console.zig
+12 -3
View File
@@ -27,7 +27,11 @@ const ConWriter = struct {
};
}
fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize {
fn drain(
w: *std.Io.Writer,
data: []const []const u8,
_: usize,
) std.Io.Writer.Error!usize {
const self: *ConWriter = @fieldParentPtr("interface", w);
var total_written: usize = 0;
for (data) |slice| {
@@ -87,7 +91,8 @@ fn drain_from_all(seq: usize) usize {
fn worker(_: ?*anyopaque) void {
while (true) {
_ = ke.wait.wait_one(&ke.log.event.hdr, "console_worker", .{}) catch unreachable;
_ = ke.wait.wait_one(&ke.log.event.hdr, "console_worker", .{}) catch
unreachable;
ke.log.event.reset();
last_seq = drain_from_all(last_seq);
@@ -119,6 +124,10 @@ pub fn register(cons: *Console) void {
pub fn init() void {
console_list.init();
const td = ps.thread.create_kernel(ke.Thread.Priority.default, worker, null) catch @panic("handle me");
const td = ps.thread.create_kernel(
ke.Thread.Priority.default,
worker,
null,
) catch @panic("handle me");
ke.sched.enqueue(&td.kern);
}
src/ex/work.zig
+27 -10
View File
@@ -1,6 +1,7 @@
//! Worker pools and work items.
//!
//! Work items are used to enqueue work to be done in a thread context asynchronously.
//! Work items are used to enqueue work to be done in a thread context
//! asynchronously.
//! There are three worker priorities:
//! - High
//! - Normal
@@ -10,8 +11,8 @@
//!
//! Each CPU worker pool comprises of at least 2 threads, and the low pool
//! has 4 threads by default. An additional thread will get spawned by the
//! pool manager thread if a queue has a backlog, up to 4 additional dynamic threads
//! (tunable) per pool.
//! pool manager thread if a queue has a backlog, up to 4 additional dynamic
//! threads (tunable) per pool.
const std = @import("std");
const rtl = @import("rtl");
@@ -58,7 +59,12 @@ pub const WorkItem = struct {
}
};
pub fn init(self: *WorkItem, priority: Priority, func: *const fn (arg: ?*anyopaque) void, arg: ?*anyopaque) void {
pub fn init(
self: *WorkItem,
priority: Priority,
func: *const fn (arg: ?*anyopaque) void,
arg: ?*anyopaque,
) void {
self.* = .{
.func = func,
.arg = arg,
@@ -145,7 +151,9 @@ const Pool = struct {
const active = @atomicLoad(usize, &self.queue.active, .monotonic);
const threads = self.total_threads.load(.monotonic);
if (threads < max_threads_for_prio(self.prio) and !is_empty and active < self.queue.max_active) {
if (threads < max_threads_for_prio(self.prio) and !is_empty and
active < self.queue.max_active)
{
// We have room to grow, the list still has items and
// not enough runnable threads are present, create an
// additional thread to ease the backlog.
@@ -157,7 +165,8 @@ const Pool = struct {
/// Spawn a dynamic worker thread on this pool.
fn grow(self: *Pool, cpu: ?u32) !void {
if (self.total_threads.load(.monotonic) >= max_threads_for_prio(self.prio))
if (self.total_threads.load(.monotonic) >=
max_threads_for_prio(self.prio))
return;
_ = self.total_threads.fetchAdd(1, .monotonic);
@@ -197,7 +206,10 @@ fn work_loop(p: ?*anyopaque) void {
const dynamic = ctx.get_tag() == 1;
while (true) {
const timeout: ?r.Nanoseconds = if (dynamic) .from(r.Seconds.init(5)) else null;
const timeout: ?r.Nanoseconds = if (dynamic)
.from(r.Seconds.init(5))
else
null;
const item = ctx.get_ptr().queue.remove(timeout) catch {
_ = ctx.get_ptr().total_threads.fetchSub(1, .monotonic);
@@ -257,11 +269,16 @@ pub fn enqueue(item: *WorkItem) void {
fn pool_manager(_: ?*anyopaque) void {
while (true) {
_ = ke.wait.wait_one(&pool_manager_event.hdr, "work_pool_manager", .{}) catch unreachable;
_ = ke.wait.wait_one(
&pool_manager_event.hdr,
"work_pool_manager",
.{},
) catch
unreachable;
// We have been signaled, go through every pool and grow it if needed.
// Failure to grow is ignored, it is only a latency problem and should be fixed
// eventually as memory is freed.
// Failure to grow is ignored, it is only a latency problem and should
// be fixed eventually as memory is freed.
for (0..ke.ncpus) |i| {
const cpu: u32 = @intCast(i);
const c = percpu.remote(cpu);
src/kern/amd64/impl.zig
+22 -4
View File
@@ -25,7 +25,12 @@ const ThreadFrame = extern struct {
extern fn asm_thread_entry() void;
extern fn do_context_switch(old: *ThreadContext, new: *ThreadContext, lock: *u8, switching: *bool) callconv(.c) void;
extern fn do_context_switch(
old: *ThreadContext,
new: *ThreadContext,
lock: *u8,
switching: *bool,
) callconv(.c) void;
extern fn do_context_load(td: *ThreadContext) callconv(.c) void;
pub const ThreadContext = extern struct {
@@ -38,7 +43,12 @@ pub const ThreadContext = extern struct {
entry_fn(@ptrFromInt(arg));
}
pub fn init(stack: r.VAddr, stack_size: usize, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) @This() {
pub fn init(
stack: r.VAddr,
stack_size: usize,
entry: *const fn (?*anyopaque) void,
arg: ?*anyopaque,
) @This() {
var ctx: @This() = undefined;
var sp: usize = stack + stack_size;
@@ -58,9 +68,17 @@ pub const ThreadContext = extern struct {
return ctx;
}
pub fn switch_to(self: *ThreadContext, new: *ThreadContext) callconv(.c) void {
pub fn switch_to(
self: *ThreadContext,
new: *ThreadContext,
) callconv(.c) void {
const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", self));
do_context_switch(self, new, &thread.lock.inner.locked.raw, &thread.switching.raw);
do_context_switch(
self,
new,
&thread.lock.inner.locked.raw,
&thread.switching.raw,
);
}
pub fn load(self: *ThreadContext) callconv(.c) void {
src/kern/cpu.zig
+5 -1
View File
@@ -40,7 +40,11 @@ pub fn CpuLocal(comptime T: type, comptime init: T) type {
}
/// Wraps around CPU-local data with a designated symbol name.
pub fn ExportedCpuLocal(comptime T: type, comptime init: T, comptime name: []const u8) type {
pub fn ExportedCpuLocal(
comptime T: type,
comptime init: T,
comptime name: []const u8,
) type {
const S = struct {
var storage: T linksection(".data.percpu") = init;
src/kern/dpc.zig
+8 -6
View File
@@ -6,7 +6,7 @@ const ke = r.ke;
const ki = ke.private;
/// Deferred Procedure Call (DPC) structure.
/// Used for scheduling work to be done when IPL is lowered below `ke.Ipl.Dispatch`.
/// Used for scheduling work to be done when IPL is lowered below `.Dispatch`.
pub const Dpc = struct {
/// Entry into per-CPU DPC queue
link: rtl.List.Entry,
@@ -78,7 +78,8 @@ fn dispatch_queue(cpu: u32) void {
ipl_cpu.ipl = .Dispatch;
// Mark as handled, this must be done before enabling interrupts or we will race.
// Mark as handled, this must be done before enabling interrupts or we will
// race.
ki.ipl.clear_softint_pending(cpu, .Dispatch);
_ = ki.impl.enable_interrupts();
@@ -100,8 +101,9 @@ fn dispatch_queue(cpu: u32) void {
dpc = @fieldParentPtr("link", first_elem);
dpc.inserted.store(false, .release);
// An interrupt which would enqueue this DPC could occur between loading the argument and calling the routine,
// which is why we capture it here to ensure that we get the intended context.
// An interrupt which would enqueue this DPC could occur between
// loading the argument and calling the routine, which is why we capture
// it here to ensure that we get the intended context.
const arg = dpc.arg;
dpc_cpu.lock.release(ipl);
@@ -148,8 +150,8 @@ fn dispatch_queue(cpu: u32) void {
/// Dispatch the DPC queue on `cpu`.
pub fn dispatch(cpu: u32) void {
// DPC processing is done at Ipl.Dispatch
// We can't call lower/raise here because they might call us again recursively.
// DPC processing is done at IPL Dispatch.
// Don't call lower/raise here because they might call us again recursively.
var mycpu = cpu;
const old_ipl = ki.ipl.percpu.remote(mycpu).ipl;
const int_state = ki.impl.disable_interrupts();
src/kern/init.zig
+5 -1
View File
@@ -37,7 +37,11 @@ pub fn init(_: ?*anyopaque) linksection(r.init) void {
ki.sched.percpu.local().current_thread = &thread0;
ki.sched.percpu.local().idle_thread = &thread0;
std.log.info("Zag for {s} ({s}), cmdline is \"{?s}\"", .{ pl.name, arch.name, r.boot_info.cmdline });
std.log.info("Zag for {s} ({s}), cmdline is \"{?s}\"", .{
pl.name,
arch.name,
r.boot_info.cmdline,
});
ex.init();
ki.sched.idle(null);
src/kern/ipi.zig
+7 -3
View File
@@ -35,7 +35,11 @@ const percpu = ke.CpuLocal(PerCpu, undefined);
/// The function will be executed synchronously, i.e this will block until
/// the function finishes running. The callback is called with the sender cpu
/// and provided argument as arguments.
pub fn unicast(cpu: u32, func: *const fn (u32, ?*anyopaque) void, arg: ?*anyopaque) void {
pub fn unicast(
cpu: u32,
func: *const fn (u32, ?*anyopaque) void,
arg: ?*anyopaque,
) void {
const ipl = ke.ipl.raise(.Dispatch);
const curcpu = percpu.local();
const remote = percpu.remote(cpu);
@@ -93,8 +97,8 @@ pub fn ipi_handler() void {
while (true) {
// Mark the current CPU as processing IPIs, if the CAS fails here,
// then it's probably some kind of IPI that was sent to force us to check
// software interrupts (like a rescheduling IPI).
// then it's probably some kind of IPI that was sent to force us to
// check software interrupts (like a rescheduling IPI).
const val = curcpu.state.cmpxchgStrong(
.Pending,
.Running,
src/kern/log.zig
+15 -4
View File
@@ -21,7 +21,11 @@ const DebugWriter = struct {
};
}
fn drain(_: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize {
fn drain(
_: *std.Io.Writer,
data: []const []const u8,
_: usize,
) std.Io.Writer.Error!usize {
var total_written: usize = 0;
for (data) |slice| {
for (slice) |byte| {
@@ -37,7 +41,10 @@ const DebugWriter = struct {
// Messages are on average 2^5 = 32 bytes.
const avg_msg_size_bits = 5;
pub var ringbuffer = ki.log_ring.RingBuffer(config.log_buffer_shift, avg_msg_size_bits).init();
pub var ringbuffer = ki.log_ring.RingBuffer(
config.log_buffer_shift,
avg_msg_size_bits,
).init();
// Initialize manually because we need this ASAP.
pub var event: ke.Event = .{
@@ -46,7 +53,10 @@ pub var event: ke.Event = .{
.signaled = 0,
.type = .Notification,
.waitblocks = .{
.head = .{ .next = &event.hdr.waitblocks.head, .prev = &event.hdr.waitblocks.head },
.head = .{
.next = &event.hdr.waitblocks.head,
.prev = &event.hdr.waitblocks.head,
},
},
},
};
@@ -75,7 +85,8 @@ pub fn log(
res.info.length = @truncate(required_len);
// Format the log message into the reserved buffer.
res.buf = std.fmt.bufPrint(res.buf, scope_str ++ fmt ++ "\n", args) catch return;
res.buf = std.fmt.bufPrint(res.buf, scope_str ++ fmt ++ "\n", args) catch
return;
ringbuffer.publish(res);
src/kern/shootdown.zig
+17 -3
View File
@@ -53,7 +53,13 @@ fn pcpu_init() linksection(r.init) void {
local.valid_states = @splat(.init(0));
local.senders = .init(false);
local.states = @splat(.{ .base = 0, .npages = 0, .state = .init(slot_free), .payload = undefined, .link = undefined });
local.states = @splat(.{
.base = 0,
.npages = 0,
.state = .init(slot_free),
.payload = undefined,
.link = undefined,
});
}
comptime {
@@ -66,7 +72,12 @@ fn allocate_slot(cpu: *PerCpu) ?usize {
for (0.., &cpu.states) |slot, *state| {
if (state.state.load(.monotonic) != slot_free) continue;
if (state.state.cmpxchgStrong(slot_free, slot_reserved, .acquire, .monotonic) == null) {
if (state.state.cmpxchgStrong(
slot_free,
slot_reserved,
.acquire,
.monotonic,
) == null) {
return slot;
}
}
@@ -167,7 +178,10 @@ pub fn submit(state: ShootdownState, target_mask: ke.CpuMask) !void {
while (iter.next()) |bit| {
if (bit == curcpu) continue;
_ = cpu.valid_states[bit].fetchOr(@as(u64, 1) << @truncate(slot), .release);
_ = cpu.valid_states[bit].fetchOr(
@as(u64, 1) << @truncate(slot),
.release,
);
ki.ipl.set_softint_pending(@truncate(bit), .Dispatch);
src/kern/smr.zig
+32 -17
View File
@@ -77,7 +77,8 @@ pub const Tracker = struct {
pub var full_scans: std.atomic.Value(usize) = .init(0);
/// Scan all CPUs and return the minimum observed value.
/// If `should_wait` is true, this will spinloop (or block) until all CPUs have reached the given goal.
/// If `should_wait` is true, this will spinloop (or block)
/// until all CPUs have reached the given goal.
fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence {
rtl.barrier.fence(.seq_cst);
@@ -162,7 +163,12 @@ fn scan(dom: *Domain, goal: Sequence, clock: Clock, should_wait: bool) Sequence
break;
}
ki.turnstile.block(ts, cpu, .{ .shared = &cpu.stalled }, .Exclusive);
ki.turnstile.block(
ts,
cpu,
.{ .shared = &cpu.stalled },
.Exclusive,
);
ke.ipl.lower(ipl);
seq = cpu.stall_seq.load(.monotonic);
@@ -256,8 +262,9 @@ pub fn advance(dom: *Domain) Sequence {
/// Pretend-advance the write sequence and return the value for use as a wait goal.
/// It is guaranteed that all previous memory writes made by
/// the calling thread are visible. The global clock isn't advanced unlike `advance`,
/// it is only when commit() is called.
/// the calling thread are visible.
/// The global clock isn't advanced unlike `advance`, it is only when commit()
/// is called.
pub fn deferred_advance(dom: *Domain) Sequence {
rtl.barrier.fence(.seq_cst);
@@ -267,7 +274,12 @@ pub fn deferred_advance(dom: *Domain) Sequence {
/// Actually advance the global clock to a value previously returned by
/// deferred_advance().
pub fn deferred_advance_commit(dom: *Domain, seq: Sequence) void {
_ = dom.clock.write_seq.cmpxchgStrong(seq - seq_incr, seq, .monotonic, .monotonic);
_ = dom.clock.write_seq.cmpxchgStrong(
seq - seq_incr,
seq,
.monotonic,
.monotonic,
);
}
/// Enter a read section.
@@ -276,18 +288,20 @@ fn enter_internal(dom: *Domain) Sequence {
// Store the currently observed write sequence into our CPU state.
// Subsequent loads must not be re-ordered w.r.t to the store here.
// On AMD64, we can simply use XADD as this is faster and provides the same guarantees,
// on other architectures we have to rely on a full (seq_cst) fence.
// The add operation works because cpu.current_seq is always only 0 (seq_invalid)
// when we write to it.
// On AMD64, we can simply use XADD as this is faster and provides the
// same guarantees, on other architectures we have to rely on a full fence.
// The add operation works because cpu.current_seq is always only 0
// (seq_invalid) when we write to it.
//
// There can be a large pause between the load from the domain write sequence
// and the store to the per-cpu state because of e.g an interrupt. This could lead
// to a state where another CPU advances the global write sequence and current_seq
// becomes lower than the global write sequence. This is not an issue with the ordering
// in scan(), which also relies on a full memory barrier, as it is guaranteed that the
// CPU was treated as inactive and cannot possibly hold a reference to anything the
// poll declared reclaimable. See scan().
// There can be a large pause between the load from the domain write
// sequence and the store to the per-cpu state because of e.g an interrupt.
// This could lead to a state where another CPU advances
// the global write sequence and current_seq ends up lower
// than the global write sequence.
// This is not an issue with the ordering in scan(), which also relies
// on a full memory barrier, as it is guaranteed that the CPU was treated as
// inactive and cannot possibly hold a reference to anything the poll
// declared reclaimable. See scan().
std.debug.assert(cpu.current_seq.load(.monotonic) == seq_invalid);
const wr_seq = dom.clock.write_seq.load(.monotonic);
@@ -434,7 +448,8 @@ pub fn exit_preempt(dom: *Domain, tracker: *Tracker) void {
}
if (wake) {
if (turnstile) |ts| ki.turnstile.wakeup(ts, .Exclusive, ts.waiters, null);
if (turnstile) |ts|
ki.turnstile.wakeup(ts, .Exclusive, ts.waiters, null);
}
ki.turnstile.exit(cpu);
src/kern/spinlock.zig
+28 -5
View File
@@ -176,7 +176,12 @@ pub const QSpinLock = LockTemplate(struct {
pub fn acquire_no_ipl(self: *Self) void {
// Fast path: try to acquire the lock.
var v = self.data.val.cmpxchgStrong(0, locked_val, .acquire, .monotonic) orelse return;
var v = self.data.val.cmpxchgStrong(
0,
locked_val,
.acquire,
.monotonic,
) orelse return;
// Medium path: if only pending is set, the pending waiter is
// in the process of clearing pending and setting `locked` to take
@@ -229,7 +234,12 @@ pub const QSpinLock = LockTemplate(struct {
}
pub fn try_acquire_no_ipl(self: *Self) bool {
return self.data.val.cmpxchgStrong(0, locked_val, .acquire, .monotonic) == null;
return self.data.val.cmpxchgStrong(
0,
locked_val,
.acquire,
.monotonic,
) == null;
}
pub fn is_locked(self: *Self) bool {
@@ -247,7 +257,12 @@ pub const QSpinLock = LockTemplate(struct {
if (idx >= 4) {
// Exhausted all indices, spin.
while (true) {
_ = self.data.val.cmpxchgWeak(0, locked_val, .acquire, .monotonic) orelse break;
_ = self.data.val.cmpxchgWeak(
0,
locked_val,
.acquire,
.monotonic,
) orelse break;
std.atomic.spinLoopHint();
}
@@ -266,7 +281,10 @@ pub const QSpinLock = LockTemplate(struct {
// Store cpu + 1 so that we can distinguish from an empty tail field.
rtl.barrier.fence(.release);
const tail: u16 = @bitCast(Tail{ .cpu = @intCast(ke.cpu.current() + 1), .idx = @intCast(idx) });
const tail: u16 = @bitCast(Tail{
.cpu = @intCast(ke.cpu.current() + 1),
.idx = @intCast(idx),
});
const old: Tail = @bitCast(self.data.split.tail.swap(tail, .acq_rel));
if (old.cpu != 0) {
@@ -293,7 +311,12 @@ pub const QSpinLock = LockTemplate(struct {
if ((v & tail_mask) == @as(u32, tail) << 16) {
// We are the only waiter in the queue.
// Try to claim the lock and clear the tail atomically.
if (self.data.val.cmpxchgStrong(v, locked_val, .acquire, .monotonic) == null) {
if (self.data.val.cmpxchgStrong(
v,
locked_val,
.acquire,
.monotonic,
) == null) {
return;
}
}
src/kern/thread.zig
+9 -2
View File
@@ -123,12 +123,19 @@ pub const Thread = struct {
smr_sections: rtl.List,
/// Initialize a thread.
/// - `stack`: Address of the **base** of the stack on which the initial context for the thread is built
/// - `stack`: Address of the base of the stack used by the thread.
/// - `stack_size`: Size of the stack
/// - `prio`: Base priority of the thread
/// - `entry`: Entry point of the thread
/// - `arg`: Extraneous argument to be passed to `entry`
pub fn init(thread: *Thread, stack: r.VAddr, stack_size: usize, prio: u8, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) void {
pub fn init(
thread: *Thread,
stack: r.VAddr,
stack_size: usize,
prio: u8,
entry: *const fn (?*anyopaque) void,
arg: ?*anyopaque,
) void {
thread.* = .{
.context = .init(stack, stack_size, entry, arg),
.lock = .init(),
src/kern/time.zig
+6 -6
View File
@@ -18,11 +18,9 @@ pub const TimeCounter = struct {
quality: i16,
/// Mask to apply to the counter value.
mask: u64,
/// Binary scaling exponent for fast ns conversion (0 if unused).
/// See `ticks_to_ns`.
/// Binary scaling exponent for fast ns conversion. See `ticks_to_ns`.
p: u64,
/// Scaled nanoseconds-per-tick multiplier for fast ns conversion (0 if unused).
/// See `ticks_to_ns`.
/// Scaled for fast ns conversion (0 if unused). See `ticks_to_ns`.
n: u64,
};
@@ -59,10 +57,12 @@ fn ticks_to_ns(tc: *TimeCounter, count: u64) u64 {
/// Register a time-keeping source.
/// The TimeCounter will be used if it is the highest quality one available.
/// This function cannot be safely called when the hardware timer has been started, as data races on `best_tc` are not avoided.
/// This function cannot be safely called when the hardware timer
/// has been started, as data races on `best_tc` are not avoided.
pub fn register_source(tc: *TimeCounter) void {
// Precompute scaling factors for the fast multiply-shift path.
// This is meant to avoid 128-bit division and only use multiplication for 64-bit counters.
// This is meant to avoid 128-bit division and only use multiplication for
// 64-bit counters.
if (tc.mask == std.math.maxInt(u64)) {
tc.p = std.math.log2_int_ceil(u64, tc.frequency);
tc.n = (@as(u64, std.time.ns_per_s) << @intCast(tc.p)) / tc.frequency;
src/kern/timer.zig
+28 -7
View File
@@ -1,5 +1,6 @@
//! Timer object implementation.
//! Timer objects are useful when one wants to wait a fixed amount of time for an event to occur.
//! Timer objects are useful when one wants to wait a given amount of time for
//! an event to occur.
const std = @import("std");
const rtl = @import("rtl");
const r = @import("root");
@@ -70,7 +71,12 @@ pub fn set(timer: *Timer, time: r.Nanoseconds, opts: Options) void {
const cpu = percpu.local();
if (timer.state.cmpxchgStrong(.Stopped, .Pending, .acquire, .monotonic) != null) {
if (timer.state.cmpxchgStrong(
.Stopped,
.Pending,
.acquire,
.monotonic,
) != null) {
// Could not set the timer.
return;
}
@@ -107,7 +113,12 @@ pub fn cancel(timer: *Timer) void {
const ipl = timer.hdr.lock.acquire();
defer timer.hdr.lock.release(ipl);
if (timer.state.cmpxchgStrong(.Pending, .Stopped, .acquire, .monotonic) != null) {
if (timer.state.cmpxchgStrong(
.Pending,
.Stopped,
.acquire,
.monotonic,
) != null) {
// Something changed, it probably expired.
return;
}
@@ -137,7 +148,8 @@ pub fn cmp_timer(a: *rtl.pairing_heap.Node, b_: *rtl.pairing_heap.Node) std.math
pub fn clock() void {
ke.dpc.enqueue(&percpu.local().dpc, null);
// Check for overflows.
// This is fine to call a lot, as the function only gets expensive (i.e seqlock store) when an overflow actually happens.
// This is fine to call a lot, as the function only gets expensive
// (i.e seqlock store) when an overflow actually happens.
ki.time.update_overflow();
}
@@ -161,13 +173,17 @@ fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void {
// If the timer expires more than 1ms in the future, consider it not yet due.
// Sub-millisecond differences are close enough to expire immediately.
if (timer.deadline.value > curtime.value and timer.deadline.value - curtime.value > std.time.ns_per_ms) {
if (timer.deadline.value > curtime.value and
timer.deadline.value - curtime.value > std.time.ns_per_ms)
{
const next = timer;
cpu.lock.release_no_ipl();
// Re-check in case the timer expired while we were here.
const now = ke.time.read_time();
if (next.deadline.value <= now.value or next.deadline.value - now.value <= std.time.ns_per_ms) {
if (next.deadline.value <= now.value or
next.deadline.value - now.value <= std.time.ns_per_ms)
{
continue; // expired in the meantime, loop again
}
@@ -176,7 +192,12 @@ fn handle_expiry(_: *ke.Dpc, _: ?*anyopaque) void {
return;
}
if (timer.state.cmpxchgStrong(.Pending, .Running, .acquire, .monotonic) != null) {
if (timer.state.cmpxchgStrong(
.Pending,
.Running,
.acquire,
.monotonic,
) != null) {
// Timer must've been canceled.
cpu.lock.release_no_ipl();
continue;
src/kern/tunable.zig
+48 -45
View File
@@ -16,7 +16,11 @@ const Entry = struct {
},
};
pub fn Tunable(comptime T: type, comptime default: T, comptime name: []const u8) type {
pub fn Tunable(
comptime T: type,
comptime default: T,
comptime name: []const u8,
) type {
return struct {
var storage: T = default;
@@ -42,56 +46,41 @@ pub fn Tunable(comptime T: type, comptime default: T, comptime name: []const u8)
};
}
pub fn init() void {
const cmdline = r.boot_info.cmdline orelse return;
const elems = set.elems();
for (elems) |entry| {
switch (entry.type) {
.num => |info| {
const a = entry.address;
switch (info.signedness) {
.unsigned => {
const value = rtl.cmdline.get_number(u64, cmdline, entry.name) catch |e| {
if (e == error.Format) {
std.log.warn("Invalid number value for '{s}', falling back.", .{entry.name});
fn store_int(comptime T: type, addr: *anyopaque, value: anytype) bool {
const ptr: *T = @ptrCast(@alignCast(addr));
ptr.* = std.math.cast(T, value) orelse return false;
return true;
}
continue;
fn apply_num(
comptime ValueT: type,
cmdline: []const u8,
entry: *const Entry,
info: std.builtin.Type.Int,
) void {
const value = rtl.cmdline.get_number(ValueT, cmdline, entry.name) catch |e| {
if (e == error.Format) {
std.log.warn(
"Invalid number value for '{s}', falling back.",
.{entry.name},
);
}
return;
};
switch (info.bits) {
8 => (@as(*u8, @ptrCast(@alignCast(a)))).* = std.math.cast(u8, value) orelse continue,
16 => (@as(*u16, @ptrCast(@alignCast(a)))).* = std.math.cast(u16, value) orelse continue,
32 => (@as(*u32, @ptrCast(@alignCast(a)))).* = std.math.cast(u32, value) orelse continue,
64 => (@as(*u64, @ptrCast(@alignCast(a)))).* = value,
const sign = @typeInfo(ValueT).int.signedness;
_ = switch (info.bits) {
8 => store_int(std.meta.Int(sign, 8), entry.address, value),
16 => store_int(std.meta.Int(sign, 16), entry.address, value),
32 => store_int(std.meta.Int(sign, 32), entry.address, value),
64 => store_int(std.meta.Int(sign, 64), entry.address, value),
else => unreachable,
}
},
.signed => {
const value = rtl.cmdline.get_number(i64, cmdline, entry.name) catch |e| {
if (e == error.Format) {
std.log.warn("Invalid number value for '{s}', falling back.", .{entry.name});
}
continue;
};
switch (info.bits) {
8 => (@as(*i8, @ptrCast(@alignCast(a)))).* = std.math.cast(i8, value) orelse continue,
16 => (@as(*i16, @ptrCast(@alignCast(a)))).* = std.math.cast(i16, value) orelse continue,
32 => (@as(*i32, @ptrCast(@alignCast(a)))).* = std.math.cast(i32, value) orelse continue,
64 => (@as(*i64, @ptrCast(@alignCast(a)))).* = value,
else => unreachable,
}
},
}
},
.bool => {
const value = rtl.cmdline.get_string(cmdline, entry.name) orelse continue;
fn apply_bool(cmdline: []const u8, entry: *const Entry) void {
const value = rtl.cmdline.get_string(cmdline, entry.name) orelse return;
const ptr: *bool = @ptrCast(@alignCast(entry.address));
if (std.mem.eql(u8, value, "true")) {
@@ -99,9 +88,23 @@ pub fn init() void {
} else if (std.mem.eql(u8, value, "false")) {
ptr.* = false;
} else {
std.log.warn("Invalid cmdline boolean value for tunable '{s}'", .{entry.name});
std.log.warn(
"Invalid cmdline boolean value for tunable '{s}'",
.{entry.name},
);
}
}
pub fn init() void {
const cmdline = r.boot_info.cmdline orelse return;
for (set.elems()) |entry| {
switch (entry.type) {
.num => |info| switch (info.signedness) {
.unsigned => apply_num(u64, cmdline, entry, info),
.signed => apply_num(i64, cmdline, entry, info),
},
.bool => apply_bool(cmdline, entry),
}
}
}
src/kern/turnstile.zig
+16 -5
View File
@@ -314,7 +314,8 @@ fn propagate(curtd: *ke.Thread, root: *Chain) void {
// and they are forbidden to explicitly block.
var it = targets(ts);
while (it.next()) |t| {
if (t.thread == curtd) @panic("turnstile: cycle in blocking chain");
if (t.thread == curtd)
@panic("turnstile: cycle in blocking chain");
t.thread.lock.acquire_no_ipl();
_ = donate_to(t.boost, t.thread, donate);
t.thread.lock.release_no_ipl();
@@ -384,7 +385,12 @@ pub fn exit(obj: *const anyopaque) void {
/// This must be called with the appropriate turnstile chain lock held, and
/// returns with the chain lock released.
/// IPL is kept as before on return.
pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, own: Ownership, queue: Queue) void {
pub fn block(
turnstile: ?*Turnstile,
obj: *anyopaque,
own: Ownership,
queue: Queue,
) void {
const chain = chain_for(obj);
const curtd = ki.sched.percpu.local().current_thread.?;
const ipl = ke.ipl.current();
@@ -441,7 +447,8 @@ pub fn block(turnstile: ?*Turnstile, obj: *anyopaque, own: Ownership, queue: Que
// Now block on the event.
_ = ke.ipl.lower(.Passive);
_ = ke.wait.wait_one(&waiter.event.hdr, "turnstile:waiter", .{}) catch unreachable;
_ = ke.wait.wait_one(&waiter.event.hdr, "turnstile:waiter", .{}) catch
unreachable;
_ = ke.ipl.raise(ipl);
}
@@ -464,7 +471,10 @@ pub fn wakeup(ts: *Turnstile, queue: Queue, count: usize, new_owner: ?*ke.Thread
} else {
for (0..count) |_| {
if (ts.queues[queue_idx].is_empty()) break;
const waiter: *Waiter = @fieldParentPtr("link", ts.queues[queue_idx].first());
const waiter: *Waiter = @fieldParentPtr(
"link",
ts.queues[queue_idx].first(),
);
dequeue(ts, waiter.thread);
}
if (ts.waiters > 0) ts.detach();
@@ -487,7 +497,8 @@ fn dequeue(ts: *Turnstile, td: *ke.Thread) void {
ts.next_free = free.next_free;
free.next_free = null;
} else {
// Last waiter, pull the turnstile off the chain and keep it for ourselves.
// Last waiter, pull the turnstile off the chain
// and keep it for ourselves.
ts.link.remove();
ts.reset();
}
src/kern/wait.zig
+48 -19
View File
@@ -13,7 +13,8 @@ pub const DispatchHeader = struct {
pub const Type = enum {
/// When signaled, `signaled` is kept high and all waiters are woken up.
Notification,
/// When signaled, `signaled` is decreased until 0, and a single waiter is woken up.
/// When signaled, `signaled` is decreased until 0, and a single waiter
/// is woken up.
Synchronization,
/// Special type for queue objects.
Queue,
@@ -70,11 +71,11 @@ pub const DispatchHeader = struct {
pub const WaitBlock = struct {
pub const Status = enum(u8) {
/// WaitBlock is linked to an object as part of a thread that is waiting.
/// Block is linked to an object as part of a thread that is waiting.
Active,
/// The wait associated with this WaitBlock has been satisfied (or timed out).
/// The wait associated has been satisfied (or timed out).
Inactive,
/// A signal was delivered to the wait (this could be before it was committed).
/// A signal was delivered to the wait.
Signaled,
};
/// List linkage.
@@ -114,8 +115,9 @@ pub fn wait_one(object: *DispatchHeader, reason: []const u8, opts: Options) !usi
/// Returns the index of the object that was signaled, or an error
/// in the case of timeout.
/// If `timeout` is not provided, it will wait indefinitely.
/// If `waitblocks` is specified, then the wait will use those waitblocks for the operation.
/// Note that if `timeout` is provided, then one additional waitblock must be allocated.
/// If `waitblocks` is specified, then the wait will use those waitblocks
/// for the operation. Note that if `timeout` is provided, then one additional
/// waitblock must be allocated.
pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !usize {
const ipl = ke.ipl.raise(.Dispatch);
defer ke.ipl.lower(ipl);
@@ -123,7 +125,9 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
const obj_count = objects.len;
const has_timeout = opts.timeout != null;
const blocks = opts.waitblocks orelse blk: {
std.debug.assert(obj_count <= curtd.waitblocks.len - @intFromBool(has_timeout));
std.debug.assert(
obj_count <= curtd.waitblocks.len - @intFromBool(has_timeout),
);
break :blk &curtd.waitblocks;
};
@@ -146,7 +150,12 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
if (obj.can_satisfy()) {
// Object was already signaled. Try consuming it.
if (curtd.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acq_rel, .monotonic) == null) {
if (curtd.wait_status.cmpxchgStrong(
.InProgress,
.Satisfied,
.acq_rel,
.monotonic,
) == null) {
obj.consume(curtd);
satisfier = i;
}
@@ -161,7 +170,7 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
wb.object = obj;
wb.thread = curtd;
wb.status = .Active;
// We are not satisfied yet, so add a waitblock to the object's waitblocks
// We are not satisfied yet, so add a waitblock to the object.
obj.waitblocks.insert_tail(&wb.link);
}
@@ -197,9 +206,15 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
curtd.lock.acquire_no_ipl();
// Now try committing the wait.
// While we're trying to commit the wait, the object locks have been released, and the state could therefore change.
// While we're trying to commit the wait, the object locks
// have been released, and the state could therefore change.
// We need to re-check the state of the wait before actually blocking.
if (curtd.wait_status.cmpxchgStrong(.InProgress, .Committed, .acq_rel, .monotonic) == null) {
if (curtd.wait_status.cmpxchgStrong(
.InProgress,
.Committed,
.acq_rel,
.monotonic,
) == null) {
if (queue == null) {
if (curtd.queue) |q| {
ki.queue.signal_wait(q);
@@ -236,7 +251,8 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
wb.link.remove();
}
if (wb.status == .Signaled) {
// This waitblock was signaled, it must be the one that satisfied the wait.
// This waitblock was signaled, it must be the one that satisfied
// the wait.
std.debug.assert(satisfier == null);
satisfier = i;
}
@@ -245,7 +261,10 @@ pub fn wait_any(objects: []*DispatchHeader, reason: []const u8, opts: Options) !
}
const final_sat = satisfier orelse return error.Timeout;
return if (has_timeout and final_sat == timer_i) error.Timeout else final_sat;
return if (has_timeout and final_sat == timer_i)
error.Timeout
else
final_sat;
}
/// Satisfy a wait on an object.
@@ -267,19 +286,29 @@ pub fn satisfy_wait(obj: *DispatchHeader) void {
wb.link.remove();
// Three cases may occur here:
// 1. The wait was still preparing (status == .InProgress) and we interrupted it.
// 2. The wait was already committed (status == .Committed) and we satisfied it.
// 3. The wait was already satisfied by another object (status == .Satisfied).
// 1. The wait was still preparing (.InProgress) and we interrupted it.
// 2. The wait was already committed (.Committed) and we satisfied it.
// 3. The wait was already satisfied by another object (.Satisfied).
//
// 1.
if (td.wait_status.cmpxchgStrong(.InProgress, .Satisfied, .acq_rel, .monotonic) == null) {
if (td.wait_status.cmpxchgStrong(
.InProgress,
.Satisfied,
.acq_rel,
.monotonic,
) == null) {
// We interrupted the wait while it was being prepared.
wb.status = .Signaled;
obj.consume(td);
}
// 2.
else if (td.wait_status.cmpxchgStrong(.Committed, .Satisfied, .acq_rel, .monotonic) == null) {
else if (td.wait_status.cmpxchgStrong(
.Committed,
.Satisfied,
.acq_rel,
.monotonic,
) == null) {
// We interrupted the wait while it was committed.
// Wake the thread.
wb.status = .Signaled;
src/mm/heap.zig
+11 -2
View File
@@ -4,7 +4,12 @@ const mm = r.mm;
const mi = mm.private;
pub fn init() void {
mi.kernel_space.arena.init("kernel heap", mi.impl.kernel_heap_base, r.tib(16), mm.page_size) catch @panic("failed to initialize kernel heap arena");
mi.kernel_space.arena.init(
"kernel heap",
mi.impl.kernel_heap_base,
r.tib(16),
mm.page_size,
) catch @panic("failed to initialize kernel heap arena");
}
pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque {
@@ -18,7 +23,11 @@ pub fn alloc(size: usize, policy: mm.WaitPolicy) mm.Error!*anyopaque {
const npages = size / mm.page_size;
for (0..npages) |i| {
const pte = mi.pmap.wire_pte(&mi.kernel_space, addr + (i * mm.page_size), policy) catch {
const pte = mi.pmap.wire_pte(
&mi.kernel_space,
addr + (i * mm.page_size),
policy,
) catch {
mi.kernel_space.lock.release();
return mm.Error.OutOfMemory;
};
src/mm/phys.zig
+28 -9
View File
@@ -47,7 +47,8 @@ fn wait_for_pages(old_ipl: ke.Ipl) void {
list_lock.release(ipl);
_ = ke.wait.wait_one(&free_page_event.hdr, "wait_for_pages", .{}) catch unreachable;
_ = ke.wait.wait_one(&free_page_event.hdr, "wait_for_pages", .{}) catch
unreachable;
ipl = list_lock.acquire();
cnt = free_pages;
@@ -150,7 +151,11 @@ pub fn init() linksection(r.init) void {
for (0..memory_map.entry_count) |i| {
const entry = memory_map.entries[i];
log.info("[{x:0>16}-{x:0>16}] {s}", .{ entry.base, entry.base + entry.size, @tagName(entry.type) });
log.info("[{x:0>16}-{x:0>16}] {s}", .{
entry.base,
entry.base + entry.size,
@tagName(entry.type),
});
if (entry.type == .Free or entry.type == .LoaderReclaimable) {
total_usable_memory += entry.size;
@@ -158,17 +163,22 @@ pub fn init() linksection(r.init) void {
}
const pfndb_size_required = @sizeOf(mm.Page) *
(std.math.divCeil(usize, total_usable_memory, mm.page_size) catch unreachable);
(std.math.divCeil(usize, total_usable_memory, mm.page_size) catch
unreachable);
log.info("using {} KiB for pfndb ({} bytes per page)", .{ pfndb_size_required / 1024, @sizeOf(mm.Page) });
log.info("using {} KiB for pfndb ({} bytes per page)", .{
pfndb_size_required / 1024,
@sizeOf(mm.Page),
});
// Get the kernel pagemap from the loader,
// we are already in it so no need to activate it.
mi.impl.init_kernel();
// Calculate the minimum amount of free memory we accept,
// this formula is from Linux and does not grow linearly; it would be wasteful
// to take a fixed percentage of memory from a large server.
// this formula is from Linux and does not grow linearly;
// it would be wasteful to take a fixed percentage of memory from a large
// server.
// We clamp this between 128 KiB and 256 MiB.
const min_free_kb = std.math.clamp(
std.math.sqrt((usable_memory.load(.monotonic) / 1024) * 16),
@@ -180,7 +190,10 @@ pub fn init() linksection(r.init) void {
const total_usable_pages = total_usable_memory / mm.page_size;
const gap = @max(min_memory_threshold / 4, total_usable_pages * threshold_scale_factor / 10000);
const gap = @max(
min_memory_threshold / 4,
total_usable_pages * threshold_scale_factor / 10000,
);
low_memory_threshold = min_memory_threshold + gap;
high_memory_threshold = low_memory_threshold + gap;
@@ -189,14 +202,20 @@ pub fn init() linksection(r.init) void {
for (0..memory_map.entry_count) |i| {
const entry = memory_map.entries[i];
if ((entry.type != .Free and entry.type != .LoaderReclaimable) or entry.size < mm.page_size) {
if (entry.type != .Free and entry.type != .LoaderReclaimable) {
continue;
}
if (entry.size < mm.page_size) {
continue;
}
const npages = entry.size / mm.page_size;
for (0..npages) |j| {
const page: *mm.Page = mm.pfn_to_struct_page(mm.page_to_pfn(entry.base + (j * mm.page_size)));
const page: *mm.Page = mm.pfn_to_struct_page(
mm.page_to_pfn(entry.base + (j * mm.page_size)),
);
free_list.insert_head(&page.free.link);
free_pages += 1;
src/mm/pmap.zig
+46 -13
View File
@@ -6,7 +6,8 @@ const std = @import("std");
const mm = r.mm;
const mi = mm.private;
/// Describes a contiguous region of physical memory to be mapped at a given virtual address.
/// Describes a contiguous region of physical memory to be mapped
/// at a given virtual address.
const MapItem = struct {
pa: r.PAddr,
len: usize,
@@ -87,8 +88,15 @@ pub const PMap = struct {
}
}
/// Map a contiguous virtual address range to a contiguous physical address range.
pub fn map_contiguous_range(self: *Self, va: r.VAddr, pa: r.PAddr, size: usize, flags: mm.MapFlags) void {
/// Map a contiguous virtual address range to a
/// contiguous physical address range.
pub fn map_contiguous_range(
self: *Self,
va: r.VAddr,
pa: r.PAddr,
size: usize,
flags: mm.MapFlags,
) void {
const src = ContiguousSource{
.flags = flags,
.size = size,
@@ -163,7 +171,10 @@ pub const PMap = struct {
const head: *mm.Page = @ptrCast(head_free);
const tail: *mm.Page = @ptrCast(tail_free);
return .{ .head = mm.struct_page_to_pfn(head), .tail = mm.struct_page_to_pfn(tail) };
return .{
.head = mm.struct_page_to_pfn(head),
.tail = mm.struct_page_to_pfn(tail),
};
}
pub const Cursor = struct {
@@ -177,7 +188,11 @@ pub const PMap = struct {
/// Reset upward in `advance` when the VA crosses a level boundary.
top_level: usize,
fn walk_down(self: *Cursor, target_level: usize, allocate: bool) error{PageNotMapped}!void {
fn walk_down(
self: *Cursor,
target_level: usize,
allocate: bool,
) error{PageNotMapped}!void {
var current_level = self.top_level;
while (current_level > target_level) {
@@ -191,8 +206,13 @@ pub const PMap = struct {
const new_table_pa = mi.phys.alloc();
const table_ptr: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(new_table_pa));
@memset(table_ptr[0..entries_per_table], std.mem.zeroes(mi.impl.Pte));
const table_ptr: [*]mi.impl.Pte = @ptrFromInt(
mm.p2v(new_table_pa),
);
@memset(
table_ptr[0..entries_per_table],
std.mem.zeroes(mi.impl.Pte),
);
pte = mi.impl.make_table_pte(new_table_pa);
self.tables[current_level][idx] = pte;
@@ -210,14 +230,20 @@ pub const PMap = struct {
return true;
}
pub fn map_range(self: *Cursor, pa: r.PAddr, size: usize, flags: mm.MapFlags) void {
pub fn map_range(
self: *Cursor,
pa: r.PAddr,
size: usize,
flags: mm.MapFlags,
) void {
var remain = size;
var current_pa = pa;
while (remain > 0) {
const target_level = 0;
self.walk_down(target_level, true) catch @panic("walk_down failed during map_range");
self.walk_down(target_level, true) catch
@panic("walk_down failed during map_range");
const table = self.tables[target_level];
table[index_for_level(self.va, target_level)] =
@@ -252,7 +278,11 @@ pub const PMap = struct {
};
};
pub fn wire_pte(space: *mi.Space, va: r.VAddr, policy: mm.WaitPolicy) mm.Error!*mi.impl.Pte {
pub fn wire_pte(
space: *mi.Space,
va: r.VAddr,
policy: mm.WaitPolicy,
) mm.Error!*mi.impl.Pte {
std.debug.assert(space.lock.is_locked());
var table: [*]mi.impl.Pte = @ptrFromInt(mm.p2v(mi.kernel_space.pmap.root_pa));
@@ -268,7 +298,8 @@ pub fn wire_pte(space: *mi.Space, va: r.VAddr, policy: mm.WaitPolicy) mm.Error!*
}
if (!entry.present) {
// Release the lock while allocating a new page table, since that may block.
// Release the lock while allocating a new page table,
// since that may block.
space.lock.release();
const new_table_pa = mi.phys.alloc_opts(.{ .policy = policy }) orelse {
@@ -276,13 +307,15 @@ pub fn wire_pte(space: *mi.Space, va: r.VAddr, policy: mm.WaitPolicy) mm.Error!*
return mm.Error.OutOfMemory;
};
// Re-acquire the lock and re-validate the entry, since another thread may have raced to allocate it.
// Re-acquire the lock and re-validate the entry,
// since another thread may have raced to allocate it.
space.lock.acquire();
entry = table[idx];
if (entry.present) {
// Another thread beat us to it, free the page we just allocated and continue.
// Another thread beat us to it,
// free the page we just allocated and continue.
mi.phys.free(new_table_pa);
} else {
const new_table_ptr: [*]u64 = @ptrFromInt(mm.p2v(new_table_pa));
src/mm/tlb.zig
+10 -2
View File
@@ -35,7 +35,11 @@ fn reclaim_state(obj: *anyopaque, _: ?*anyopaque) void {
space.lock.release();
// Free the physical pages
mm.phys.free_batch(mm.pfn_to_struct_page(pfn_list.head), mm.pfn_to_struct_page(pfn_list.tail), npages);
mm.phys.free_batch(
mm.pfn_to_struct_page(pfn_list.head),
mm.pfn_to_struct_page(pfn_list.tail),
npages,
);
_ = async_shootdowns.fetchAdd(1, .monotonic);
@@ -83,7 +87,11 @@ pub fn reclaim_range(space: *mm.Space, va: r.VAddr, size: usize) void {
space.arena.free(va, size) catch unreachable;
// Free the physical pages
mm.phys.free_batch(mm.pfn_to_struct_page(list.head), mm.pfn_to_struct_page(list.tail), size / mm.page_size);
mm.phys.free_batch(
mm.pfn_to_struct_page(list.head),
mm.pfn_to_struct_page(list.tail),
size / mm.page_size,
);
return;
};
}
src/mm/vmem.zig
+58 -16
View File
@@ -113,7 +113,10 @@ const Segment = struct {
return @fieldParentPtr("linkage", linkage);
}
fn cmp(a_node: *const rtl.bst.Node, b_node: *const rtl.bst.Node) std.math.Order {
fn cmp(
a_node: *const rtl.bst.Node,
b_node: *const rtl.bst.Node,
) std.math.Order {
const a = Segment.from_tree_link(@constCast(a_node));
const b = Segment.from_tree_link(@constCast(b_node));
return std.math.order(a.base, b.base);
@@ -147,7 +150,13 @@ pub const Arena = struct {
const Self = @This();
/// Initialize the area for use.
pub fn init(self: *Self, name: []const u8, base: usize, size: usize, quantum: usize) !void {
pub fn init(
self: *Self,
name: []const u8,
base: usize,
size: usize,
quantum: usize,
) !void {
self.name = name;
self.base = base;
self.size = size;
@@ -260,7 +269,13 @@ pub const Arena = struct {
// the quantum check avoids creating a remainder too small to ever be useful.
if (seg.size != size and (seg.size - size) > self.quantum - 1) {
const new_seg = try self.alloc_segment();
new_seg.* = .{ .type = .Allocated, .base = seg.base, .size = size, .link = undefined, .linkage = .{ .tree_link = undefined } };
new_seg.* = .{
.type = .Allocated,
.base = seg.base,
.size = size,
.link = undefined,
.linkage = .{ .tree_link = undefined },
};
seg.base += size;
seg.size -= size;
@@ -270,15 +285,18 @@ pub const Arena = struct {
new_seg.link.insert_before(&seg.link);
self.add_segment_to_freelist(seg);
self.allocated_segments.insert(&new_seg.linkage.tree_link) catch return error.InvalidAddress;
self.allocated_segments.insert(&new_seg.linkage.tree_link) catch
return error.InvalidAddress;
if (options.policy == .NextFit) self.rotor = new_seg;
return new_seg.base;
} else {
// seg is exactly the right size (or remainder is too small), use it directly.
// seg is exactly the right size (or remainder is too small),
// use it directly.
seg.type = .Allocated;
seg.linkage = .{ .tree_link = undefined };
self.allocated_segments.insert(&seg.linkage.tree_link) catch return error.InvalidAddress;
self.allocated_segments.insert(&seg.linkage.tree_link) catch
return error.InvalidAddress;
if (options.policy == .NextFit) self.rotor = seg;
return seg.base;
@@ -295,7 +313,9 @@ pub const Arena = struct {
.linkage = .{ .tree_link = undefined },
};
const node = self.allocated_segments.tree.search(&search_node.linkage.tree_link) orelse return error.InvalidAddress;
const node = self.allocated_segments.tree.search(
&search_node.linkage.tree_link,
) orelse return error.InvalidAddress;
const seg = Segment.from_tree_link(node);
@@ -310,7 +330,8 @@ pub const Arena = struct {
const next_seg: *Segment = @fieldParentPtr("link", next_entry);
if (next_seg.type == .Free) {
// Remove next segment from freelist and segment list, then merge into seg.
// Remove next segment from freelist and segment list,
// then merge into seg.
self.remove_segment_from_freelist(next_seg);
next_seg.link.remove();
seg.size += next_seg.size;
@@ -326,7 +347,8 @@ pub const Arena = struct {
const prev_seg: *Segment = @fieldParentPtr("link", prev_entry);
if (prev_seg.type == .Free) {
// Remove previous segment from freelist and segment list, then merge into seg.
// Remove previous segment from freelist and segment list,
// then merge into seg.
self.remove_segment_from_freelist(prev_seg);
prev_seg.link.remove();
seg.base = prev_seg.base;
@@ -344,7 +366,11 @@ pub const Arena = struct {
}
// Implementation for the instant fit policy.
fn instant_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } {
fn instant_fit(
self: *Self,
size: usize,
options: AllocOptions,
) ?struct { usize, *Segment } {
var idx = freelist_index(size);
if (!std.math.isPowerOfTwo(size)) idx += 1;
@@ -356,7 +382,8 @@ pub const Arena = struct {
var it = self.freelists[idx].iterator();
while (it.next()) : (it.advance()) {
const seg = Segment.from_free_link(it.get());
if (self.try_to_fit(seg, size, options)) |addr| return .{ addr, seg };
if (self.try_to_fit(seg, size, options)) |addr|
return .{ addr, seg };
}
}
return null;
@@ -364,7 +391,11 @@ pub const Arena = struct {
// Implementation for the best fit policy.
// Find the smallest segment that can satisfy the request.
fn best_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } {
fn best_fit(
self: *Self,
size: usize,
options: AllocOptions,
) ?struct { usize, *Segment } {
var idx = freelist_index(size);
var best_seg: ?*Segment = null;
var best_start: usize = 0;
@@ -388,8 +419,13 @@ pub const Arena = struct {
}
// Implementation for the next fit policy.
// Continue searching from the last allocated segment, wrapping around at the end of the arena.
fn next_fit(self: *Self, size: usize, options: AllocOptions) ?struct { usize, *Segment } {
// Continue searching from the last allocated segment, wrapping around at
// the end of the arena.
fn next_fit(
self: *Self,
size: usize,
options: AllocOptions,
) ?struct { usize, *Segment } {
// Start from rotor if we have one, otherwise from the beginning.
const start_entry = if (self.rotor) |r| r.link.next else self.list.first();
@@ -418,9 +454,15 @@ pub const Arena = struct {
// Try to fit the given segment to the request with the given options.
// Returns the fitted address if successful.
fn try_to_fit(self: *Self, segment: *const Segment, size: usize, options: AllocOptions) ?usize {
fn try_to_fit(
self: *Self,
segment: *const Segment,
size: usize,
options: AllocOptions,
) ?usize {
var start = @max(segment.base, options.min orelse 0);
const end = @min(segment.base + segment.size, options.max orelse std.math.maxInt(usize));
const end = @min(segment.base + segment.size, options.max orelse
std.math.maxInt(usize));
const alignment = options.alignment orelse self.quantum;
if (start > end) return null;
src/mm/zone.zig
+245 -93
View File
@@ -18,9 +18,10 @@
//! ## Per-CPU layer
//! ----------------
//! Each CPU has an `alloc` and `free` magazine, where objects are allocated
//! from and freed to, respectively. This separation avoids hysteresis and is useful
//! for safe memory reclamation (see below).
//! When one side runs dry/full, the magazines may be swapped, if SMR is disabled.
//! from and freed to, respectively. This separation avoids hysteresis and is
//! useful for safe memory reclamation (see below).
//! When one side runs dry/full, the magazines may be swapped,
//! if SMR is disabled.
//! Each CPU also has its own depot, from which it allocates and frees its
//! magazines to first. If there is no magazine in the local depot, the magazine
//! will be provided from the zone depot layer. We keep track of the size of the
@@ -42,11 +43,11 @@
//! ## Slab layer
//! -------------
//! The slab layer is the fallback when all previous layers failed. A slab
//! represents a contiguous chunk of memory holding a fixed number of constructed
//! objects, determined when the zone is created, and is allocated via the
//! kernel's virtual memory allocator. Allocation in the slab is managed by a
//! bitmap and slab metadata is retrieved through the `mm.Page` structure
//! representing the page of allocated memory.
//! represents a contiguous chunk of memory holding a fixed number of
//! constructed objects, and is allocated via the kernel's virtual memory
//! allocator. Allocation in the slab is managed by a bitmap and slab metadata
//! is retrieved through the `mm.Page` structure representing the page
//! of allocated memory.
//!
//! ## Hardening features
//! ---------------------
@@ -90,10 +91,10 @@
//! commits. When alloc needs a full magazine, it first polls the sequence of
//! the head of the depot magazine list (since SMR zones are forced
//! FIFO and stamps are taken in monotonic order, the head always carries the
//! minimum sequence). If the local head has not expired, the local full magazines
//! are moved and appended to the zone depot to age, and their sequences
//! are committed, while expired magazines from the zone depot are pulled back in
//! to satisfy the request.
//! minimum sequence). If the local head has not expired, the local full
//! magazines are moved and appended to the zone depot to age,
//! and their sequences are committed, while expired magazines from the zone
//! depot are pulled back in to satisfy the request.
//!
//! All this machinery is to avoid expensive scans during SMR polling,
//! growing the per-CPU depots widens the window between a stamp and its poll
@@ -134,19 +135,35 @@ const zones_num = 32;
const magazine_size = ke.Tunable(u8, 8, "mm.zone.mag_size");
/// How much extra memory CPUs are allowed to keep around.
const max_local_memory = ke.Tunable(u32, @intCast(r.kib(128)), "mm.zone.max_local_mem");
const max_local_memory = ke.Tunable(
u32,
@intCast(r.kib(128)),
"mm.zone.max_local_mem",
);
/// Number of contentions allowed per second before the depot grows.
const depot_grow_level = ke.Tunable(u32, 5 * wma_unit, "mm.zone.depot_grow_level");
const depot_grow_level = ke.Tunable(
u32,
5 * wma_unit,
"mm.zone.depot_grow_level",
);
/// Number of contentions allowed per second before the depot shrinks.
const depot_shrink_level = ke.Tunable(u32, wma_unit / 2, "mm.zone.depot_shrink_level");
const depot_shrink_level = ke.Tunable(
u32,
wma_unit / 2,
"mm.zone.depot_shrink_level",
);
/// Number of excess magazines in a zone before they are trimmed.
const excess_magazines = ke.Tunable(u32, 8, "mm.zone.excess_mags");
/// Excess memory in a zone before magazines are trimmed.
const excess_memory = ke.Tunable(u32, @intCast(r.kib(16)), "mm.zone.excess_mem");
const excess_memory = ke.Tunable(
u32,
@intCast(r.kib(16)),
"mm.zone.excess_mem",
);
/// Every power-of-two-size from 8 to 2048 (inclusively).
const generic_zones_num = 9;
@@ -189,7 +206,8 @@ const should_check_poison = config.slab_check_poison;
const AllocOpts = struct { policy: mm.WaitPolicy = .WaitForMemory };
/// A slab is a contiguous memory region that holds multiple objects of the same type.
/// A slab is a contiguous memory region that holds multiple objects of the same
/// type.
const Slab = struct {
/// Linkage into a zone's list of slabs.
link: rtl.List.Entry,
@@ -300,7 +318,11 @@ const Slab = struct {
// since a smaller capacity means a smaller bitmap, which may then fit.
while (capacity > 0) : (capacity -= 1) {
const objects_off =
std.mem.alignForward(usize, @sizeOf(Slab) + Slab.bitmap_bytes(capacity), alignment);
std.mem.alignForward(
usize,
@sizeOf(Slab) + Slab.bitmap_bytes(capacity),
alignment,
);
if (objects_off + capacity * chunk_size <= mm.page_size) {
break;
@@ -352,7 +374,11 @@ const Depot = struct {
return ret;
}
fn maglist_free(list: *MagazineList, mag: *Magazine, policy: ReusePolicy) void {
fn maglist_free(
list: *MagazineList,
mag: *Magazine,
policy: ReusePolicy,
) void {
mag.next = null;
if (policy == .FIFO) {
@@ -430,7 +456,8 @@ inline fn check_poison(ptr: *anyopaque, len: usize, pattern: u32) bool {
var i: usize = 0;
while (i + 4 <= len) : (i += 4) {
if (std.mem.readInt(u32, bytes[i..][0..4], .little) != pattern) return false;
if (std.mem.readInt(u32, bytes[i..][0..4], .little) != pattern)
return false;
}
if (i < len) {
@@ -454,7 +481,8 @@ inline fn wma_mix(old: usize, new: usize) usize {
}
fn cpus_alloc() []rtl.CachePadded(Cpu) {
const ptr = cpu_zone.alloc(.{}) catch @panic("failed to allocate zone CPU state");
const ptr = cpu_zone.alloc(.{}) catch
@panic("failed to allocate zone CPU state");
const raw: [*]rtl.CachePadded(Cpu) = @ptrCast(@alignCast(ptr));
return raw[0..ke.ncpus];
}
@@ -505,7 +533,7 @@ pub const Zone = struct {
const Self = @This();
const InitOptions = struct {
alignment: u16 = 0,
alignment: u16 = slab_align,
ctor: ?*const fn (*anyopaque) void = null,
dtor: ?*const fn (*anyopaque) void = null,
magazines: bool = true,
@@ -515,11 +543,19 @@ pub const Zone = struct {
};
/// Initialize a zone.
pub fn init(self: *Self, name: []const u8, size: usize, options: InitOptions) void {
const obj_align = if (options.alignment == 0) slab_align else options.alignment;
pub fn init(
self: *Self,
name: []const u8,
size: usize,
options: InitOptions,
) void {
const obj_align = options.alignment;
const chunk_size = std.mem.alignForward(usize, size, obj_align);
const slab_size = if (size <= small_slab_size) mm.page_size else Slab.calc_slab_size(chunk_size);
const slab_size = if (size <= small_slab_size)
mm.page_size
else
Slab.calc_slab_size(chunk_size);
self.full_slabs.init();
self.partial_slabs.init();
@@ -538,12 +574,17 @@ pub const Zone = struct {
self.depot_contention_cur = 0;
self.depot_contention_wma = 0;
self.cpu_depot_size = 0;
self.cpu_depot_limit = max_local_memory.load() / (self.chunk_size * magazine_size.load());
self.cpu_depot_limit = max_local_memory.load() /
(self.chunk_size * magazine_size.load());
self.trim_depot = false;
// SMR zones have to be FIFO so that the minimum sequence can be
// obtained by checking the list head.
self.reuse_policy = if (options.smr != null) .FIFO else options.reuse_policy;
self.reuse_policy = if (options.smr != null)
.FIFO
else
options.reuse_policy;
self.smr = options.smr;
self.smr_reclaim = options.smr_reclaim;
@@ -608,8 +649,9 @@ pub const Zone = struct {
if (cpu.free_rounds > 0 and self.smr == null) {
std.debug.assert(cpu.free != null);
// Alloc magazine is empty. If the free magazine has items, swap them.
// The free magazine now becomes our alloc magazine.
// Alloc magazine is empty. If the free magazine has items,
// swap them. The free magazine now becomes our alloc
// magazine.
std.mem.swap(?*Magazine, &cpu.alloc, &cpu.free);
std.mem.swap(usize, &cpu.alloc_rounds, &cpu.free_rounds);
continue;
@@ -622,7 +664,11 @@ pub const Zone = struct {
// Discard our empty alloc magazine to the depot.
if (cpu.alloc) |empty_mag| {
Depot.free(empty_mag, &cpu.depot.empty_mags, self.reuse_policy);
Depot.free(
empty_mag,
&cpu.depot.empty_mags,
self.reuse_policy,
);
}
self.magazine_reuse(full_mag);
@@ -632,8 +678,8 @@ pub const Zone = struct {
}
// Local depot is empty, rebalance with the zone depot.
// Purposefully race against cpu_depot_size because locking is tricky here,
// we may get a stale value but it is harmless.
// Purposefully race against cpu_depot_size because locking is
// tricky here, we may get a stale value but it is harmless.
const n = @atomicLoad(usize, &self.cpu_depot_size, .monotonic);
self.alloc_depot_rebalance(n, cpu);
@@ -644,7 +690,11 @@ pub const Zone = struct {
// Discard our empty alloc magazine to the depot.
if (cpu.alloc) |empty_mag| {
Depot.free(empty_mag, &cpu.depot.empty_mags, self.reuse_policy);
Depot.free(
empty_mag,
&cpu.depot.empty_mags,
self.reuse_policy,
);
}
self.magazine_reuse(full_mag);
@@ -670,9 +720,10 @@ pub const Zone = struct {
self.lock.acquire();
// Note: this is racy and could lead to two threads creating a slab at the same time.
// Worst case scenario, we have one slab too many, so we don't care; we already did the work required
// for slab construction and this ideally should not happen very often.
// Note: this is racy and could lead to two threads creating a slab
// at the same time. Worst case scenario, we have one slab too many,
// so we don't care; we already did the work required for slab
// construction and this ideally should not happen very often.
self.partial_slabs.insert_tail(&slab.link);
} else {
slab = @fieldParentPtr("link", self.partial_slabs.first());
@@ -718,7 +769,9 @@ pub const Zone = struct {
cpu.lock.acquire_no_ipl();
const freed = while (true) {
if (cpu.free != null and cpu.free_rounds < magazine_size.load()) {
if (cpu.free != null and
cpu.free_rounds < magazine_size.load())
{
// Fast path: push directly to free magazine.
// For SMR zones the dtor and poisoning are deferred until
// the magazine is reused, readers may still hold the object.
@@ -741,8 +794,12 @@ pub const Zone = struct {
break true;
}
if (cpu.alloc_rounds == 0 and cpu.alloc != null and self.smr == null) {
// Free magazine is full. If the alloc magazine is completely empty, swap them.
if (cpu.alloc_rounds == 0 and
cpu.alloc != null and
self.smr == null)
{
// Free magazine is full.
// If the alloc magazine is completely empty, swap them.
// The empty alloc magazine becomes our new free magazine.
std.mem.swap(?*Magazine, &cpu.alloc, &cpu.free);
std.mem.swap(usize, &cpu.alloc_rounds, &cpu.free_rounds);
@@ -754,7 +811,11 @@ pub const Zone = struct {
if (Depot.alloc(&cpu.depot.empty_mags)) |empty_mag| {
// Put the free magazine on the full list.
if (cpu.free) |full_mag| {
Depot.free(full_mag, &cpu.depot.full_mags, self.reuse_policy);
Depot.free(
full_mag,
&cpu.depot.full_mags,
self.reuse_policy,
);
}
cpu.free = empty_mag;
@@ -772,7 +833,11 @@ pub const Zone = struct {
if (Depot.alloc(&cpu.depot.empty_mags)) |empty_mag| {
// Put the free magazine on the full list.
if (cpu.free) |full_mag| {
Depot.free(full_mag, &cpu.depot.full_mags, self.reuse_policy);
Depot.free(
full_mag,
&cpu.depot.full_mags,
self.reuse_policy,
);
}
cpu.free = empty_mag;
@@ -784,7 +849,12 @@ pub const Zone = struct {
// We need to drop IPL and the CPU lock here because we need to acquire
// the magazine's allocator's blocking lock.
cpu.lock.release(ipl);
const new_mag: ?*Magazine = @ptrCast(@alignCast(magazine_zone.alloc(.{ .policy = .DontWaitForMemory }) catch null));
const raw_mag = magazine_zone.alloc(
.{ .policy = .DontWaitForMemory },
) catch null;
const new_mag: ?*Magazine = @ptrCast(@alignCast(raw_mag));
// Re-grab our current context.
ipl = ke.ipl.raise(.Dispatch);
@@ -831,12 +901,18 @@ pub const Zone = struct {
// Find the slab for the buffer.
const slab: *Slab = if (self.obj_size > small_slab_size) blk: {
const page_va = std.mem.alignBackward(usize, @intFromPtr(obj), mm.page_size);
const page_va = std.mem.alignBackward(
usize,
@intFromPtr(obj),
mm.page_size,
);
const phys = mi.kernel_space.pmap.query(page_va).?;
const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys));
break :blk page.alloced.slab_data.slab;
} else blk: {
break :blk @ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(obj), mm.page_size));
break :blk @ptrFromInt(
std.mem.alignBackward(usize, @intFromPtr(obj), mm.page_size),
);
};
const was_full = slab.refcount == slab.capacity;
@@ -860,34 +936,46 @@ pub const Zone = struct {
}
/// Periodic updates on the zone.
/// Updates the working set and trims it if needed (for now, will do magazine resizing later...).
/// Updates the magazine working set and trims or grows them if needed.
fn update(self: *Self) void {
const ipl = self.depot_lock.acquire();
// Update the working set size.
// This tracks the minimum number of magazines required, we assume it is safe
// to reclaim `min` magazines, since the number of magazines never went below that, and
// those magazines sat unused. e.g if the number of magazines ranged between 47 and 37 in an update
// interval, then the working set size is 10 and we can reclaim 37 magazines. We keep track of
// this in a weighted moving average (WMA) to bias recent utilization when trimming.
self.depot.empty_mags.wma = wma_mix(self.depot.empty_mags.wma, self.depot.empty_mags.min);
// This tracks the minimum number of magazines required,
// we assume it is safe to reclaim `min` magazines, since the number of
// magazines never went below that, and those magazines sat unused. e.g
// if the number of magazines ranged between 47 and 37 in an update
// interval, then the working set size is 10 and we can
// reclaim 37 magazines. We keep track of this in a weighted moving
// average (WMA) to bias recent utilization when trimming.
self.depot.empty_mags.wma = wma_mix(
self.depot.empty_mags.wma,
self.depot.empty_mags.min,
);
self.depot.empty_mags.min = self.depot.empty_mags.num;
self.depot.full_mags.wma = wma_mix(self.depot.full_mags.wma, self.depot.full_mags.min);
self.depot.full_mags.wma = wma_mix(
self.depot.full_mags.wma,
self.depot.full_mags.min,
);
self.depot.full_mags.min = self.depot.full_mags.num;
// Calculate the number of contentions/second in fixed-point.
const old = self.depot_contention_wma;
var cur = self.depot_contention_cur * wma_unit / (ke.ncpus * update_interval_s);
var cur = self.depot_contention_cur * wma_unit /
(ke.ncpus * update_interval_s);
// WMA formula.
cur = (3 * old + cur) / 4;
if (self.use_magazines) {
if (self.cpu_depot_size < self.cpu_depot_limit and cur > depot_grow_level.load()) {
if (self.cpu_depot_size < self.cpu_depot_limit and
cur > depot_grow_level.load())
{
// We have room to grow the depot and we should.
// Put the new WMA at around midpoint between shrink and growth, so that
// we have time to check whether what we just did is good or not.
// Put the new WMA at around midpoint between shrink and growth,
// so that we have time to check whether what we just did is
// good or not.
cur = (depot_grow_level.load() + depot_shrink_level.load()) / 2;
const size = if (self.cpu_depot_size == 0)
@@ -898,7 +986,9 @@ pub const Zone = struct {
// Clamp it.
self.cpu_depot_size = @min(size, self.cpu_depot_limit);
} else if (self.cpu_depot_size > 0 and cur <= depot_shrink_level.load()) {
} else if (self.cpu_depot_size > 0 and
cur <= depot_shrink_level.load())
{
// We should shrink the depot.
cur = (depot_grow_level.load() + depot_shrink_level.load()) / 2;
self.cpu_depot_size -= 1;
@@ -926,19 +1016,28 @@ pub const Zone = struct {
if (!self.use_magazines) return false;
if (self.trim_depot) return true;
const empty = @min(self.depot.empty_mags.wma, self.depot.empty_mags.min * wma_unit);
const empty = @min(
self.depot.empty_mags.wma,
self.depot.empty_mags.min * wma_unit,
);
if (empty > excess_magazines.load() * wma_unit) {
// Too many excess magazines.
return true;
}
const full = @min(self.depot.full_mags.wma, self.depot.full_mags.min * wma_unit);
const full = @min(
self.depot.full_mags.wma,
self.depot.full_mags.min * wma_unit,
);
const full_bytes = full * magazine_size.load() * self.chunk_size;
if (full >= 2 * wma_unit and full_bytes >= excess_memory.load() * wma_unit) {
// We have at least 2 excess full magazines and they take up at least 16 KiB.
if (full >= 2 * wma_unit and
full_bytes >= excess_memory.load() * wma_unit)
{
// We have at least 2 excess full magazines
// and they take up at least 16 KiB.
return true;
}
@@ -948,8 +1047,8 @@ pub const Zone = struct {
/// Called periodically to trim our magazines.
/// This frees the excess magazines.
fn trim(self: *Self, trim_depot: bool) void {
// Note: we do all of this under the zone lock,
// but it *should* be a fairly quick operation, even though it is O(ncpus).
// Note: we do all of this under the zone lock, but it *should* be a
// a fairly quick operation, even though it is O(ncpus).
self.lock.acquire();
self.trim_maglist(&self.depot.empty_mags, 0);
@@ -975,8 +1074,7 @@ pub const Zone = struct {
fn drain(self: *Self) void {
if (!self.use_magazines) return;
// Note: we do all of this under the zone lock,
// but it *should* be a fairly quick operation, even though it is O(ncpus).
// Note: we do all of this under the zone lock, same as above.
self.lock.acquire();
const ipl = self.depot_lock.acquire();
@@ -1055,7 +1153,11 @@ pub const Zone = struct {
// Move them to our depot.
for (0..n) |_| {
Depot.free(Depot.alloc(&cpu.depot.full_mags).?, &depot.full_mags, self.reuse_policy);
Depot.free(
Depot.alloc(&cpu.depot.full_mags).?,
&depot.full_mags,
self.reuse_policy,
);
}
}
@@ -1065,7 +1167,11 @@ pub const Zone = struct {
// Move them to our depot.
for (0..n) |_| {
Depot.free(Depot.alloc(&cpu.depot.empty_mags).?, &depot.empty_mags, self.reuse_policy);
Depot.free(
Depot.alloc(&cpu.depot.empty_mags).?,
&depot.empty_mags,
self.reuse_policy,
);
}
}
@@ -1108,7 +1214,8 @@ pub const Zone = struct {
}
fn alloc_page(policy: mm.WaitPolicy) !*anyopaque {
const alloc_ret = mi.phys.alloc_opts(.{ .policy = policy }) orelse return error.OutOfMemory;
const alloc_ret = mi.phys.alloc_opts(.{ .policy = policy }) orelse
return error.OutOfMemory;
return @ptrFromInt(mm.p2v(alloc_ret));
}
@@ -1129,7 +1236,8 @@ pub const Zone = struct {
slab.alloc_rr = 0;
slab.refcount = 0;
slab.capacity = @intCast(capacity);
const meta_end = @intFromPtr(slab) + @sizeOf(Slab) + Slab.bitmap_bytes(capacity);
const meta_end = @intFromPtr(slab) + @sizeOf(Slab) +
Slab.bitmap_bytes(capacity);
slab.base = std.mem.alignForward(usize, meta_end, self.alignment);
const bm = slab.bitmap();
@@ -1154,7 +1262,11 @@ pub const Zone = struct {
const capacity = (self.slab_size) / self.chunk_size;
const buf = try mi.heap.alloc(self.slab_size, policy);
var ret: *Slab = @ptrCast(@alignCast(try gpa.alloc(u8, @sizeOf(Slab) + Slab.bitmap_bytes(capacity))));
var ret: *Slab = @ptrCast(
@alignCast(
try gpa.alloc(u8, @sizeOf(Slab) + Slab.bitmap_bytes(capacity)),
),
);
// Mark the bitmap as all free.
const bm = ret.bitmap();
@@ -1171,7 +1283,9 @@ pub const Zone = struct {
ret.alloc_rr = 0;
for (0..self.slab_size / mm.page_size) |i| {
const phys_page = mi.kernel_space.pmap.query(@intFromPtr(buf) + i * mm.page_size) orelse @panic("Could not query page");
const phys_page = mi.kernel_space.pmap.query(
@intFromPtr(buf) + i * mm.page_size,
) orelse @panic("Could not query page");
const page = mm.pfn_to_struct_page(mm.page_to_pfn(phys_page));
page.alloced.slab_data.slab = ret;
}
@@ -1202,7 +1316,9 @@ pub const Zone = struct {
mm.heap.free(slab.base, self.slab_size);
} else {
free_page(@ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(slab), mm.page_size)));
free_page(@ptrFromInt(
std.mem.alignBackward(usize, @intFromPtr(slab), mm.page_size),
));
}
}
@@ -1257,9 +1373,10 @@ pub const Zone = struct {
ke.smr.deferred_advance_commit(smr, tail.seq);
}
/// Try to rebalance the zone and cpu depots w.r.t to each other after an allocation.
/// The CPU depot moves its extra empty magazines to the zone depot, and tries
/// to get full magazines from the zone depot up to `target`. CPU lock is held.
/// Try to rebalance the zone and cpu depots w.r.t to each other
/// after an allocation. The CPU depot moves its extra full magazines to the
/// zone depot, and tries to get full magazines from the zone depot up to
/// `target`. CPU lock is held.
fn alloc_depot_rebalance(self: *Self, target: usize, cpu: *Cpu) void {
if (!self.depot_lock.try_acquire_no_ipl()) {
// Register the contention.
@@ -1272,7 +1389,11 @@ pub const Zone = struct {
const n = cpu.depot.empty_mags.num - target / 2;
for (0..n) |_| {
Depot.free(Depot.alloc(&cpu.depot.empty_mags).?, &self.depot.empty_mags, self.reuse_policy);
Depot.free(
Depot.alloc(&cpu.depot.empty_mags).?,
&self.depot.empty_mags,
self.reuse_policy,
);
}
}
@@ -1280,27 +1401,35 @@ pub const Zone = struct {
// We got here because the local head hasn't expired or there is
// no local magazine at all, rotate the local magazines through the
// zone depot to let their sequences age.
// We then hope that the head of the zone depot has expired, otherwise we just return
// and fall back to the slab layer.
// We then hope that the head of the zone depot has expired,
// otherwise we just return and fall back to the slab layer.
self.smr_rotate_full(smr, cpu);
}
// Grab a batch of full magazines from the zone depot.
// Exchange at least one magazine even when the depot size is 0.
const n = @min(@max(target, 1) - cpu.depot.empty_mags.num, self.depot.full_mags.num);
const n = @min(
@max(target, 1) - cpu.depot.empty_mags.num,
self.depot.full_mags.num,
);
if (n != 0 and self.full_mags_ready(&self.depot.full_mags)) {
for (0..n) |_| {
Depot.free(Depot.alloc(&self.depot.full_mags).?, &cpu.depot.full_mags, self.reuse_policy);
Depot.free(
Depot.alloc(&self.depot.full_mags).?,
&cpu.depot.full_mags,
self.reuse_policy,
);
}
}
self.depot_lock.release_no_ipl();
}
/// Try to rebalance the zone and cpu depots w.r.t to each other after a free.
/// The CPU depot moves its extra full magazines to the zone depot, and tries
/// to get empty magazines from the zone depot up to `target`. CPU lock is held.
/// Try to rebalance the zone and cpu depots w.r.t to each other
/// after a free.The CPU depot moves its extra full magazines to the
/// zone depot, and tries to get empty magazines from the zone depot up to
/// `target`. CPU lock is held.
fn free_depot_rebalance(self: *Self, target: usize, cpu: *Cpu) void {
if (!self.depot_lock.try_acquire_no_ipl()) {
// Register the contention.
@@ -1318,7 +1447,11 @@ pub const Zone = struct {
if (n != 0 and self.full_mags_ready(&self.depot.full_mags)) {
for (0..n) |_| {
Depot.free(Depot.alloc(&self.depot.full_mags).?, &cpu.depot.full_mags, self.reuse_policy);
Depot.free(
Depot.alloc(&self.depot.full_mags).?,
&cpu.depot.full_mags,
self.reuse_policy,
);
}
}
} else {
@@ -1326,16 +1459,27 @@ pub const Zone = struct {
const n = cpu.depot.full_mags.num - target / 2;
for (0..n) |_| {
Depot.free(Depot.alloc(&cpu.depot.full_mags).?, &self.depot.full_mags, self.reuse_policy);
Depot.free(
Depot.alloc(&cpu.depot.full_mags).?,
&self.depot.full_mags,
self.reuse_policy,
);
}
}
}
// Grab a batch of empty magazines from the zone depot.
// Exchange at least one magazine even when the depot size is 0.
const n = @min(@max(target, 1) - cpu.depot.full_mags.num, self.depot.empty_mags.num);
const n = @min(
@max(target, 1) - cpu.depot.full_mags.num,
self.depot.empty_mags.num,
);
for (0..n) |_| {
Depot.free(Depot.alloc(&self.depot.empty_mags).?, &cpu.depot.empty_mags, self.reuse_policy);
Depot.free(
Depot.alloc(&self.depot.empty_mags).?,
&cpu.depot.empty_mags,
self.reuse_policy,
);
}
self.depot_lock.release_no_ipl();
@@ -1403,9 +1547,13 @@ pub fn early_init() linksection(r.init) void {
zone.init(generic_zone_names[i], @as(usize, 1) << @intCast(i + 3), .{});
}
magazine_zone.init("magazine", @sizeOf(Magazine) + magazine_size.load() * @sizeOf(*anyopaque), .{
magazine_zone.init(
"magazine",
@sizeOf(Magazine) + magazine_size.load() * @sizeOf(*anyopaque),
.{
.magazines = false,
});
},
);
}
/// Post-SMP initialization.
@@ -1415,7 +1563,8 @@ pub fn late_init() linksection(r.init) void {
.alignment = std.atomic.cache_line,
});
// Go through all caches with magazines enabled and initialize their per-CPU magazines.
// Go through all caches with magazines enabled and initialize
// their per-CPU magazines.
zone_list_lock.acquire();
var zone = all_zones;
@@ -1446,8 +1595,10 @@ pub fn late_init() linksection(r.init) void {
smr_zone.init("SMR", .{});
smr_cpu_zone.init("SMR CPU", @sizeOf(ke.smr.Cpu) * ke.ncpus, .{});
smr_sys_domain = smr_domain_create(false) catch @panic("Failed to create smr domain");
smr_sys_preempt_domain = smr_domain_create(true) catch @panic("Failed to create smr domain");
smr_sys_domain = smr_domain_create(false) catch
@panic("Failed to create smr domain");
smr_sys_preempt_domain = smr_domain_create(true) catch
@panic("Failed to create smr domain");
}
pub fn smr_domain_create(preempt: bool) !*ke.smr.Domain {
@@ -1528,7 +1679,8 @@ fn gpa_remap(
) ?[*]u8 {
if (gpa_resize(undefined, buf, alignment, new_len, ret_addr)) return buf.ptr;
const new_ptr = gpa_alloc(undefined, new_len, alignment, ret_addr) orelse return null;
const new_ptr = gpa_alloc(undefined, new_len, alignment, ret_addr) orelse
return null;
const copy_len = @min(buf.len, new_len);
@memcpy(new_ptr[0..copy_len], buf[0..copy_len]);
gpa_free(undefined, buf, alignment, ret_addr);
src/platform/acpi/hpet.zig
+2 -1
View File
@@ -21,7 +21,8 @@ const femtos_per_s = 1_000_000_000_000_000;
pub fn init(hpet: *acpi.HpetTable) linksection(r.init) void {
hpet_regs = @ptrFromInt(mm.p2v(hpet.base_address.address));
hpet_timer.frequency = femtos_per_s / (r.mmio_read(u64, @intFromPtr(&hpet_regs.general_capabilities)) >> 32);
hpet_timer.frequency = femtos_per_s /
(r.mmio_read(u64, @intFromPtr(&hpet_regs.general_capabilities)) >> 32);
// Disable timer.
r.mmio_write(u64, @intFromPtr(&hpet_regs.general_configuration), 0);
src/platform/acpi/madt.zig
+3 -1
View File
@@ -11,7 +11,9 @@ pub const MadtIterator = struct {
pub fn next(self: *MadtIterator) ?*const acpi.MadtEntryHeader {
const entries_size = self.madt.header.length - @sizeOf(acpi.Madt);
if (self.i >= entries_size) return null;
const entry: *const acpi.MadtEntryHeader = @ptrFromInt(@intFromPtr(&self.madt.entries) + self.i);
const entry: *const acpi.MadtEntryHeader = @ptrFromInt(
@intFromPtr(&self.madt.entries) + self.i,
);
self.i += @max(2, entry.length);
return entry;
}
src/platform/acpi/root.zig
+19 -5
View File
@@ -97,7 +97,8 @@ pub const Gas = extern struct {
register_bit_width: u8 align(1),
/// The bit offset of the given register at the given address.
register_bit_offset: u8 align(1),
/// The access size of the given register. (1 = byte, 2 = word, 3 = dword, 4 = qword).
/// The access size of the given register.
/// (1 = byte, 2 = word, 3 = dword, 4 = qword).
access_size: u8 align(1),
address: u64 align(1),
pub fn read(self: Gas) u64 {
@@ -142,9 +143,21 @@ pub const Gas = extern struct {
else => unreachable,
},
.SystemIo => switch (self.access_size) {
1 => arch.pio_write(u8, @truncate(self.address), @truncate(shifted_value)),
2 => arch.pio_write(u16, @truncate(self.address), @truncate(shifted_value)),
3 => arch.pio_write(u32, @truncate(self.address), @truncate(shifted_value)),
1 => arch.pio_write(
u8,
@truncate(self.address),
@truncate(shifted_value),
),
2 => arch.pio_write(
u16,
@truncate(self.address),
@truncate(shifted_value),
),
3 => arch.pio_write(
u32,
@truncate(self.address),
@truncate(shifted_value),
),
else => unreachable,
},
else => @panic("unsupported GAS address space"),
@@ -213,7 +226,8 @@ pub const Fadt = extern struct {
const version_sizes = [_]usize{ 116, 132, 244, 244, 268, 276 };
pub fn revision(self: Fadt) u8 {
// We can't rely on firmware to set the revision field correctly, so we infer it from the length field.
// We can't rely on firmware to set the revision field correctly,
// so we infer it from the length field.
const declared: usize = self.header.revision;
const len: u32 = self.header.length;
src/platform/pc/apic.zig
+24 -10
View File
@@ -90,11 +90,16 @@ fn xapic_read(register: ApicRegisters) u32 {
}
fn x2apic_write(register: ApicRegisters, value: u64) void {
amd64.wrmsr(@intFromEnum(amd64.Msr.X2ApicBase) + (@as(u32, @intFromEnum(register)) >> 4), value);
amd64.wrmsr(
@intFromEnum(amd64.Msr.X2ApicBase) + (@intFromEnum(register) >> 4),
value,
);
}
fn x2apic_read(register: ApicRegisters) u64 {
return amd64.rdmsr(@intFromEnum(amd64.Msr.X2ApicBase) + (@as(u32, @intFromEnum(register)) >> 4));
return amd64.rdmsr(
@intFromEnum(amd64.Msr.X2ApicBase) + (@intFromEnum(register) >> 4),
);
}
pub fn write(register: ApicRegisters, value: u32) void {
@@ -133,7 +138,10 @@ pub fn send_init(apic_id: u32) void {
pub fn send_sipi(apic_id: u32, startup_page: u8) void {
if (in_x2apic_mode.local().*) {
x2apic_write(.Icr, (@as(u64, apic_id) << 32) | 0x00004600 | startup_page);
x2apic_write(
.Icr,
(@as(u64, apic_id) << 32) | 0x00004600 | startup_page,
);
} else {
xapic_write(.Icr1, @as(u32, apic_id) << 24);
xapic_write(.Icr, @as(u32, 0x00004600) | startup_page);
@@ -143,7 +151,8 @@ pub fn send_sipi(apic_id: u32, startup_page: u8) void {
pub fn send_ipi(apic_id: u32, vector: u8, delivery_mode: u8) void {
if (in_x2apic_mode.local().*) {
const icr_value = (@as(u64, apic_id) << 32) | (@as(u64, delivery_mode) << 8) | vector;
const icr_value = (@as(u64, apic_id) << 32) |
(@as(u64, delivery_mode) << 8) | vector;
x2apic_write(.Icr, icr_value);
} else {
const icr_value = (@as(u32, delivery_mode) << 8) | vector;
@@ -200,10 +209,11 @@ pub fn arm_timer(ns: r.Nanoseconds) void {
if (amd64.cpu_features.tsc_deadline) {
const now = amd64.rdtsc();
const delta: u64 = @intCast((@as(u128, ns.value) * tsc.tsc_timer.frequency) / std.time.ns_per_s);
const delta: u64 = @intCast(
(@as(u128, ns.value) * tsc.tsc_timer.frequency) / std.time.ns_per_s,
);
const deadline = now +% delta;
amd64.write_msr(.TscDeadline, deadline);
return;
}
@@ -242,7 +252,8 @@ pub fn init() linksection(r.init) void {
const cur_cpu_id = get_id();
// First, we iterate to check if we find *any* APICs. If we do, then we must use them over x2APIC entries for IDs under 255.
// First, we iterate to check if we find *any* APICs.
// If we do, then we must use them over x2APIC entries for IDs under 255.
var has_lapics = false;
while (iter.next()) |entry| {
switch (entry.type) {
@@ -264,13 +275,16 @@ pub fn init() linksection(r.init) void {
.LocalApic => {
const lapic: *const MadtLapic = @ptrCast(entry);
if (lapic.flags & 1 == 0 or lapic.id == cur_cpu_id) continue;
apics.append(mm.zone.gpa, lapic.id) catch @panic("Could not append APIC ID to list");
apics.append(mm.zone.gpa, lapic.id) catch
@panic("Could not append APIC ID to list");
},
.X2LocalApic => {
const x2lapic: *const MadtX2Lapic = @ptrCast(entry);
if (has_lapics and x2lapic.id <= 0xFF) continue;
if (x2lapic.flags & 1 == 0 or x2lapic.id == cur_cpu_id) continue;
apics.append(mm.zone.gpa, x2lapic.id) catch @panic("Could not append APIC ID to list");
if (x2lapic.flags & 1 == 0 or x2lapic.id == cur_cpu_id)
continue;
apics.append(mm.zone.gpa, x2lapic.id) catch
@panic("Could not append APIC ID to list");
},
else => {},
}
src/platform/pc/smp.zig
+45 -13
View File
@@ -37,10 +37,15 @@ fn ap_entry(cpu_id: u32) callconv(.c) noreturn {
ki.impl.init.ap_entry(cpu_id, &aps_booted);
}
fn make_thread(entrypoint: *const fn (?*anyopaque) void, stack: usize) *ke.Thread {
const td = mm.zone.gpa.create(ke.Thread) catch @panic("Failed to allocate thread for AP");
fn make_thread(
entrypoint: *const fn (?*anyopaque) void,
stack: usize,
size: usize,
) *ke.Thread {
const td = mm.zone.gpa.create(ke.Thread) catch
@panic("Failed to allocate thread for AP");
td.init(stack, r.kib(32), ke.Thread.Priority.idle_low, entrypoint, null);
td.init(stack, size, ke.Thread.Priority.idle_low, entrypoint, null);
td.pinned = true;
return td;
@@ -65,8 +70,12 @@ pub fn init() linksection(r.init) void {
.Unknown => false,
};
const init_delay: r.Nanoseconds = .init(if (skip_delay) 0 else std.time.ns_per_ms * 10);
const sipi_delay: r.Nanoseconds = .init(if (skip_delay) std.time.ns_per_us * 10 else std.time.ns_per_us * 300);
const init_delay: r.Nanoseconds = .init(
if (skip_delay) 0 else std.time.ns_per_ms * 10,
);
const sipi_delay: r.Nanoseconds = .init(
if (skip_delay) std.time.ns_per_us * 10 else std.time.ns_per_us * 300,
);
const page: [*]u8 = @ptrFromInt(mm.p2v(0x8000));
@memcpy(page[0..trampoline_size], @as([*]u8, @ptrFromInt(trampoline_start)));
@@ -76,8 +85,11 @@ pub fn init() linksection(r.init) void {
const idtr = amd64.sidtr();
const offsets = mm.zone.gpa.alloc(usize, apic.apics.items.len + 1) catch
@panic("Failed to allocate AP local data offsets");
// Allocate per-cpu offsets for CPU-local data.
ki.impl.cpu_offsets = @ptrCast(mm.zone.gpa.alloc(usize, apic.apics.items.len + 1) catch @panic("Failed to allocate AP local data offsets"));
ki.impl.cpu_offsets = @ptrCast(offsets);
const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start);
log.info("per-CPU data size: {} bytes", .{percpu_size});
@@ -98,19 +110,39 @@ pub fn init() linksection(r.init) void {
cpu_id_to_apic_id[cpu_id] = apic_id;
// Allocate per-cpu data.
const cpu_data = mm.zone.gpa.alloc(u8, percpu_size) catch @panic("Failed to allocate per-cpu data");
@memcpy(cpu_data, @as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size]);
const cpu_data = mm.zone.gpa.alloc(u8, percpu_size) catch
@panic("Failed to allocate per-cpu data");
const self_offset_offset = @intFromPtr(&ki.impl.cpu_self_offset) - @intFromPtr(&__percpu_start);
ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -% @intFromPtr(&__percpu_start);
@memcpy(
cpu_data,
@as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size],
);
const self_offset_offset = @intFromPtr(&ki.impl.cpu_self_offset) -
@intFromPtr(&__percpu_start);
ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -%
@intFromPtr(&__percpu_start);
const off: *usize = @ptrCast(@alignCast(&cpu_data[self_offset_offset]));
// copy the offset into the AP's self_offset variable
@as(*usize, @ptrCast(@alignCast(&cpu_data[self_offset_offset]))).* = ki.impl.cpu_offsets[cpu_id];
off.* = ki.impl.cpu_offsets[cpu_id];
const stack_top = @intFromPtr(mm.heap.alloc(r.kib(32), .DontWaitForMemory) catch @panic("Failed to allocate AP stack")) + r.kib(32);
const stack_size = r.kib(16);
const stack_top = @intFromPtr(mm.heap.alloc(
stack_size,
.DontWaitForMemory,
) catch
@panic("Failed to allocate AP stack")) + stack_size;
start_stack.remote(@intCast(cpu_id)).* = stack_top & ~@as(usize, 15);
start_thread.remote(@intCast(cpu_id)).* = make_thread(@ptrCast(&ap_entry), stack_top - r.kib(32));
start_thread.remote(@intCast(cpu_id)).* = make_thread(
@ptrCast(&ap_entry),
stack_top - stack_size,
stack_size,
);
rtl.barrier.fence(.release);
src/proc/thread.zig
+5 -1
View File
@@ -55,7 +55,11 @@ pub fn init() void {
/// - `prio`: base priority of the thread.
/// - `entry`: entry point.
/// - `arg`: argument passed to `entry`.
pub fn create_kernel(prio: u8, entry: *const fn (arg: ?*anyopaque) void, arg: ?*anyopaque) !*Thread {
pub fn create_kernel(
prio: u8,
entry: *const fn (arg: ?*anyopaque) void,
arg: ?*anyopaque,
) !*Thread {
var td = try thread_zone.create();
const stack = try mm.heap.alloc(kernel_thread_stack_size, .WaitForMemory);
src/rcu/hashtable.zig
+41 -10
View File
@@ -133,7 +133,7 @@ const rehashing_reseed: u8 = 1 << 2;
const Core = struct {
/// The current state of the hash table.
state: std.atomic.Value(State),
/// One pointer for current buckets, another for new buckets (during resizing).
/// One pointer for current buckets, another for new buckets.
buckets: [2]std.atomic.Value([*]Bucket),
/// One seed for current table, another for the new table.
seeds: [2]std.atomic.Value(u32),
@@ -153,7 +153,10 @@ const Core = struct {
policy: ResizePolicy,
min_size: u32,
) !void {
const min: u32 = if (policy == .Compact) @max(2, min_size) else @max(16, min_size);
const min: u32 = if (policy == .Compact)
@max(2, min_size)
else
@max(16, min_size);
const start_size = min;
@@ -186,7 +189,13 @@ const Core = struct {
self.rehash_work.init(.Normal, rehash, self);
}
fn migrate_bucket(self: *Self, old: *Bucket, new_array: [*]Bucket, new_seed: u32, new_shift: u5) void {
fn migrate_bucket(
self: *Self,
old: *Bucket,
new_array: [*]Bucket,
new_seed: u32,
new_shift: u5,
) void {
// This is called in worker thread context, so we have ample stack size
// to work with.
const flat_size = 256;
@@ -331,7 +340,10 @@ const Core = struct {
/// Remove a given element from the table, SMR must be entered.
fn remove(self: *Self, link: *rcu.SList.Entry) void {
var state = self.state.load(.acquire);
var h = self.hash_link(link, self.seeds[state.current_index].load(.monotonic));
var h = self.hash_link(
link,
self.seeds[state.current_index].load(.monotonic),
);
var b = self.bucket(state, .Cur, h);
@@ -343,7 +355,10 @@ const Core = struct {
// Get the new state instead.
state = self.state.load(.acquire);
h = self.hash_link(link, self.seeds[state.new_index].load(.monotonic));
h = self.hash_link(
link,
self.seeds[state.new_index].load(.monotonic),
);
b = self.bucket(state, .New, h);
first = b.lock();
}
@@ -425,7 +440,11 @@ const Core = struct {
};
/// Scalable, lock-free hashtable backed by SMR.
pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context: type) type {
pub fn Table(
comptime T: type,
comptime link_field: []const u8,
comptime Context: type,
) type {
comptime rtl.assert_interface(Context, struct {
pub const Key = Context.Key;
pub fn hash(key: Key, seed: usize) u32 {
@@ -470,7 +489,10 @@ pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context
/// Get an element given a key, SMR must be entered.
pub fn get(self: *Self, key: Context.Key) ?*T {
const state = self.core.state.load(.acquire);
const h = Context.hash(key, self.core.seeds[state.current_index].load(.monotonic));
const h = Context.hash(
key,
self.core.seeds[state.current_index].load(.monotonic),
);
const b = self.core.bucket(state, .Cur, h);
var word = b.load();
@@ -504,7 +526,10 @@ pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context
value: *rcu.SList.Entry,
) ?*T {
var state = self.core.state.load(.acquire);
var h = Context.hash(key, self.core.seeds[state.current_index].load(.monotonic));
var h = Context.hash(
key,
self.core.seeds[state.current_index].load(.monotonic),
);
var b = self.core.bucket(state, .Cur, h);
var first = b.lock();
@@ -515,7 +540,10 @@ pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context
// Get the new state instead.
state = self.core.state.load(.acquire);
h = Context.hash(key, self.core.seeds[state.new_index].load(.monotonic));
h = Context.hash(
key,
self.core.seeds[state.new_index].load(.monotonic),
);
b = self.core.bucket(state, .New, h);
first = b.lock();
}
@@ -574,7 +602,10 @@ pub fn Table(comptime T: type, comptime link_field: []const u8, comptime Context
// Now do the same as in get(), but with the new state.
const state = self.core.state.load(.acquire);
const h = Context.hash(key, self.core.seeds[state.new_index].load(.monotonic));
const h = Context.hash(
key,
self.core.seeds[state.new_index].load(.monotonic),
);
const b = self.core.bucket(state, .New, h);
var word: usize = b.load();