mirror of
https://github.com/rdmsr/zag.git
synced 2026-10-27 13:58:20 +00:00
refactor: time -> clock
This commit is contained in:
@@ -4,9 +4,8 @@ const r = @import("root");
|
||||
const ke = r.ke;
|
||||
const ki = ke.private;
|
||||
|
||||
/// TimeCounter structure.
|
||||
/// Represents a time-keeping source upon which the kernel can rely.
|
||||
pub const TimeCounter = struct {
|
||||
/// Represents a clock source upon which the kernel can rely.
|
||||
pub const ClockSource = struct {
|
||||
/// Callback that reads the current count.
|
||||
read_count: *const fn () u64,
|
||||
/// Frequency of the timer in Hz.
|
||||
@@ -14,7 +13,7 @@ pub const TimeCounter = struct {
|
||||
/// Name of the timer.
|
||||
name: []const u8,
|
||||
/// Quality of the timer.
|
||||
/// Used to choose between two TimeCounters.
|
||||
/// Used to choose between two ClockSources.
|
||||
quality: i16,
|
||||
/// Mask to apply to the counter value.
|
||||
mask: u64,
|
||||
@@ -26,95 +25,103 @@ pub const TimeCounter = struct {
|
||||
n: u64,
|
||||
};
|
||||
|
||||
var best_tc: ?*TimeCounter = null;
|
||||
var best_clock: ?*ClockSource = null;
|
||||
|
||||
const TimeState = struct {
|
||||
// Counter value at the time the current TimeCounter was selected.
|
||||
const State = struct {
|
||||
// Counter value at the time the current ClockSource was selected.
|
||||
// Used as the baseline for elapsed time calculation.
|
||||
initial_count: u64,
|
||||
// Accumulated nanosecond offset from previous TimeCounters and overflow events.
|
||||
// Added to the current counter's contribution in `read_time_nano`.
|
||||
// Accumulated nanosecond offset from previous ClockSources and overflow events.
|
||||
// Added to the current counter's contribution in `read_time`.
|
||||
offset: u64,
|
||||
};
|
||||
|
||||
// SeqLock-protected time state.
|
||||
// Avoids a race on `initial_count` and `offset` with `update_overflow`.
|
||||
var state = rtl.SeqLock(TimeState).init(.{
|
||||
var state = rtl.SeqLock(State).init(.{
|
||||
.initial_count = 0,
|
||||
.offset = 0,
|
||||
});
|
||||
|
||||
// Convert a TimeCounter's count to nanoseconds.
|
||||
// Convert a ClockSource's count to nanoseconds.
|
||||
// For 64-bit counters (`mask == maxInt(u64)`), uses a fast multiply-shift path
|
||||
// via precomputed `p` and `n` to avoid 128-bit division at runtime.
|
||||
// Falls back to a division-based conversion for narrower counters.
|
||||
fn ticks_to_ns(tc: *TimeCounter, count: u64) u64 {
|
||||
if (tc.p != 0) {
|
||||
const val: u128 = @as(u128, @intCast(count)) * tc.n;
|
||||
return @intCast(val >> @intCast(tc.p));
|
||||
fn ticks_to_ns(cs: *ClockSource, count: u64) u64 {
|
||||
if (cs.p != 0) {
|
||||
const val: u128 = @as(u128, @intCast(count)) * cs.n;
|
||||
return @intCast(val >> @intCast(cs.p));
|
||||
}
|
||||
|
||||
return (count * std.time.ns_per_s) / tc.frequency;
|
||||
return (count * std.time.ns_per_s) / cs.frequency;
|
||||
}
|
||||
|
||||
/// Register a time-keeping source.
|
||||
/// The TimeCounter will be used if it is the highest quality one available.
|
||||
/// Register a clock source.
|
||||
/// The provided source will be used if it is the highest quality one available.
|
||||
/// This function cannot be safely called when the hardware timer has been started, as data races on `best_tc` are not avoided.
|
||||
pub fn register_source(tc: *TimeCounter) void {
|
||||
pub fn register_source(cs: *ClockSource) void {
|
||||
// Precompute scaling factors for the fast multiply-shift path.
|
||||
// This is meant to avoid 128-bit division and only use multiplication for 64-bit counters.
|
||||
if (tc.mask == std.math.maxInt(u64)) {
|
||||
tc.p = std.math.log2_int_ceil(u64, tc.frequency);
|
||||
tc.n = (@as(u64, std.time.ns_per_s) << @intCast(tc.p)) / tc.frequency;
|
||||
if (cs.mask == std.math.maxInt(u64)) {
|
||||
cs.p = std.math.log2_int_ceil(u64, cs.frequency);
|
||||
cs.n = (@as(u64, std.time.ns_per_s) << @intCast(cs.p)) / cs.frequency;
|
||||
} else {
|
||||
tc.p = 0;
|
||||
tc.n = 0;
|
||||
cs.p = 0;
|
||||
cs.n = 0;
|
||||
}
|
||||
|
||||
// If the best TimeCounter changed, pass the baton:
|
||||
// snapshot the old counter's elapsed time into `offset` and
|
||||
// reset `initial_count` to the new counter's current value.
|
||||
if (best_tc == null or tc.quality > best_tc.?.quality) {
|
||||
const old_best = best_tc;
|
||||
best_tc = tc;
|
||||
if (best_clock == null or cs.quality > best_clock.?.quality) {
|
||||
const old_best = best_clock;
|
||||
best_clock = cs;
|
||||
|
||||
if (old_best) |old| {
|
||||
const old_count = old.read_count() & old.mask;
|
||||
state.raw.offset = ticks_to_ns(old, old_count);
|
||||
}
|
||||
|
||||
state.raw.initial_count = tc.read_count() & tc.mask;
|
||||
state.raw.initial_count = cs.read_count() & cs.mask;
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the time elapsed since boot in nanoseconds.
|
||||
pub fn read_time_nano() r.Nanoseconds {
|
||||
const tc = best_tc orelse return 0;
|
||||
pub fn read_time() r.Nanoseconds {
|
||||
const cs = best_clock orelse return 0;
|
||||
|
||||
const s = state.load();
|
||||
const curr_count = tc.read_count() & tc.mask;
|
||||
const elapsed = (curr_count - s.initial_count) & tc.mask;
|
||||
return s.offset + ticks_to_ns(tc, elapsed);
|
||||
const curr_count = cs.read_count() & cs.mask;
|
||||
const elapsed = (curr_count - s.initial_count) & cs.mask;
|
||||
return s.offset + ticks_to_ns(cs, elapsed);
|
||||
}
|
||||
|
||||
/// Return the best TimeCounter.
|
||||
pub fn best() ?*TimeCounter {
|
||||
return best_tc;
|
||||
/// Return the best clock source.
|
||||
pub fn best() ?*ClockSource {
|
||||
return best_clock;
|
||||
}
|
||||
|
||||
/// Do a busy sleep of `ns` nanoseconds using TimeCounter.
|
||||
/// Return whether the current best clock is better than `cs`.
|
||||
pub fn is_better_than(cs: *ClockSource) bool {
|
||||
if (best_clock) |b| {
|
||||
return b.quality > cs.quality;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Do a busy sleep of `ns` nanoseconds.
|
||||
pub fn sleep(ns: r.Nanoseconds) void {
|
||||
const start = read_time_nano();
|
||||
const start = read_time();
|
||||
|
||||
while (true) {
|
||||
var cur = read_time_nano();
|
||||
var cur = read_time();
|
||||
|
||||
if (cur < start) continue;
|
||||
|
||||
if (cur - start >= ns) {
|
||||
cur = read_time_nano();
|
||||
cur = read_time();
|
||||
|
||||
// Check again as we may have overflowed and read a bogus value..
|
||||
// Check again as we may have overflowed and read a bogus value.
|
||||
// This is kinda hacky but oh well.
|
||||
if (cur - start >= ns) {
|
||||
break;
|
||||
@@ -127,17 +134,17 @@ pub fn sleep(ns: r.Nanoseconds) void {
|
||||
|
||||
/// Called periodically to handle counter overflows.
|
||||
pub fn update_overflow() void {
|
||||
const tc = best_tc orelse return;
|
||||
const cs = best_clock orelse return;
|
||||
const s = state.load();
|
||||
const cur_count = tc.read_count() & tc.mask;
|
||||
const cur_count = cs.read_count() & cs.mask;
|
||||
|
||||
if (s.initial_count > cur_count) {
|
||||
// Counter has wrapped, use wrapping arithmetic to figure out by how much,
|
||||
// then fold it into the offset and reset `initial_count`.
|
||||
const overflowed_by = (cur_count -% s.initial_count) & tc.mask;
|
||||
const overflowed_by = (cur_count -% s.initial_count) & cs.mask;
|
||||
state.store(.{
|
||||
.initial_count = cur_count,
|
||||
.offset = s.offset + ticks_to_ns(tc, overflowed_by),
|
||||
.offset = s.offset + ticks_to_ns(cs, overflowed_by),
|
||||
});
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -55,7 +55,7 @@ pub fn log(
|
||||
// Reserve space in the ring buffer.
|
||||
var res = ringbuffer.reserve(required_len) catch return;
|
||||
|
||||
res.info.timestamp = ke.time.read_time_nano();
|
||||
res.info.timestamp = ke.clock.read_time();
|
||||
res.info.length = @truncate(required_len);
|
||||
|
||||
// Format the log message into the reserved buffer.
|
||||
|
||||
@@ -20,7 +20,7 @@ pub const log = @import("log.zig");
|
||||
pub const cpu = @import("cpu.zig");
|
||||
pub const thread = @import("thread.zig");
|
||||
pub const dpc = @import("dpc.zig");
|
||||
pub const time = @import("time.zig");
|
||||
pub const clock = @import("clock.zig");
|
||||
pub const sched = @import("sched.zig");
|
||||
pub const wait = @import("wait.zig");
|
||||
pub const timer = @import("timer.zig");
|
||||
|
||||
+7
-6
@@ -10,7 +10,7 @@ pub const Ipl = p.ipl.Ipl;
|
||||
pub const SpinLock = p.spinlock.SpinLock;
|
||||
pub const Thread = p.thread.Thread;
|
||||
pub const Dpc = p.dpc.Dpc;
|
||||
pub const TimeCounter = p.time.TimeCounter;
|
||||
pub const ClockSource = p.clock.ClockSource;
|
||||
pub const Timer = p.timer.Timer;
|
||||
pub const Console = p.log.Console;
|
||||
pub const CpuLocal = p.cpu.CpuLocal;
|
||||
@@ -35,11 +35,12 @@ pub const dpc = struct {
|
||||
pub const enqueue = p.dpc.enqueue;
|
||||
};
|
||||
|
||||
pub const time = struct {
|
||||
pub const register_source = p.time.register_source;
|
||||
pub const read_time_nano = p.time.read_time_nano;
|
||||
pub const best = p.time.best;
|
||||
pub const sleep = p.time.sleep;
|
||||
pub const clock = struct {
|
||||
pub const register_source = p.clock.register_source;
|
||||
pub const read_time = p.clock.read_time;
|
||||
pub const best = p.clock.best;
|
||||
pub const sleep = p.clock.sleep;
|
||||
pub const is_better_than = p.clock.is_better_than;
|
||||
};
|
||||
|
||||
pub const sched = struct {
|
||||
|
||||
+2
-2
@@ -246,7 +246,7 @@ pub fn block() void {
|
||||
td.lock.acquire_no_ipl();
|
||||
|
||||
td.state = .Blocked;
|
||||
td.sleep_start = ke.time.read_time_nano();
|
||||
td.sleep_start = ke.clock.read_time();
|
||||
td.runq = null;
|
||||
|
||||
yield(ke.cpu.current());
|
||||
@@ -259,7 +259,7 @@ pub fn block() void {
|
||||
pub fn unblock(td: *ke.Thread) void {
|
||||
const ipl = td.lock.acquire();
|
||||
|
||||
td.sleep_time = (ke.time.read_time_nano() - td.sleep_start) / std.time.ns_per_ms;
|
||||
td.sleep_time = (ke.clock.read_time() - td.sleep_start) / std.time.ns_per_ms;
|
||||
|
||||
if (td.sleep_time >= config.CONFIG_SCHED_TIMESLICE and td.priority_class == .Batch) {
|
||||
// If we have slept for more than a tick, update interactivity.
|
||||
|
||||
+4
-4
@@ -75,7 +75,7 @@ pub fn set(timer: *Timer, time: r.Nanoseconds, dpc: ?*ke.Dpc) void {
|
||||
defer cpu.lock.release_no_ipl();
|
||||
|
||||
// Initialize the timer.
|
||||
timer.deadline = ke.time.read_time_nano() + time;
|
||||
timer.deadline = ke.clock.read_time() + time;
|
||||
|
||||
timer.cpu = cpu;
|
||||
timer.dpc = dpc;
|
||||
@@ -134,7 +134,7 @@ pub fn clock() void {
|
||||
ke.dpc.enqueue(&percpu.local().dpc, null);
|
||||
// Check for overflows.
|
||||
// This is fine to call a lot, as the function only gets expensive (i.e seqlock store) when an overflow actually happens.
|
||||
ki.time.update_overflow();
|
||||
ki.clock.update_overflow();
|
||||
}
|
||||
|
||||
// Called in a DPC when a timer has expired.
|
||||
@@ -143,7 +143,7 @@ fn handle_expiry(_: ?*anyopaque) void {
|
||||
const cpu = percpu.local();
|
||||
|
||||
while (true) {
|
||||
const curtime = ke.time.read_time_nano();
|
||||
const curtime = ke.clock.read_time();
|
||||
|
||||
cpu.lock.acquire_no_ipl();
|
||||
|
||||
@@ -162,7 +162,7 @@ fn handle_expiry(_: ?*anyopaque) void {
|
||||
cpu.lock.release_no_ipl();
|
||||
|
||||
// Re-check in case the timer expired while we were here.
|
||||
const now = ke.time.read_time_nano();
|
||||
const now = ke.clock.read_time();
|
||||
if (next.deadline <= now or next.deadline - now <= std.time.ns_per_ms) {
|
||||
continue; // expired in the meantime, loop again
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ const acpi = b.pl.acpi;
|
||||
const mm = b.mm;
|
||||
const ke = b.ke;
|
||||
|
||||
var hpet_timer: ke.TimeCounter = .{
|
||||
var hpet_cs: ke.ClockSource = .{
|
||||
.name = "HPET",
|
||||
.quality = 50,
|
||||
.frequency = 0, // Filled at runtime
|
||||
@@ -19,9 +19,10 @@ var hpet_regs: *volatile acpi.HpetRegs = undefined;
|
||||
const femtos_per_s = 1_000_000_000_000_000;
|
||||
|
||||
pub fn init(hpet: *acpi.HpetTable) linksection(b.init) void {
|
||||
hpet_regs = @ptrFromInt(mm.p2v(hpet.base_address.address));
|
||||
if (ke.clock.is_better_than(&hpet_cs)) return;
|
||||
|
||||
hpet_timer.frequency = femtos_per_s / (b.mmio_read(u64, @intFromPtr(&hpet_regs.general_capabilities)) >> 32);
|
||||
hpet_regs = @ptrFromInt(mm.p2v(hpet.base_address.address));
|
||||
hpet_cs.frequency = femtos_per_s / (b.mmio_read(u64, @intFromPtr(&hpet_regs.general_capabilities)) >> 32);
|
||||
|
||||
// Disable timer.
|
||||
b.mmio_write(u64, @intFromPtr(&hpet_regs.general_configuration), 0);
|
||||
@@ -32,7 +33,7 @@ pub fn init(hpet: *acpi.HpetTable) linksection(b.init) void {
|
||||
// Enable timer.
|
||||
b.mmio_write(u64, @intFromPtr(&hpet_regs.general_configuration), 1);
|
||||
|
||||
ke.time.register_source(&hpet_timer);
|
||||
ke.clock.register_source(&hpet_cs);
|
||||
}
|
||||
|
||||
fn read_hpet() u64 {
|
||||
|
||||
@@ -4,7 +4,7 @@ const acpi = b.pl.acpi;
|
||||
const ke = b.ke;
|
||||
const arch = @import("arch");
|
||||
|
||||
var pm_timer: ke.TimeCounter = .{
|
||||
var pm_timer: ke.ClockSource = .{
|
||||
.name = "ACPI",
|
||||
.quality = 25,
|
||||
.frequency = 3579545, // 3.579545 MHz
|
||||
@@ -27,6 +27,8 @@ pub fn init(fadt: *acpi.Fadt) linksection(b.init) void {
|
||||
return;
|
||||
}
|
||||
|
||||
if (ke.clock.is_better_than(&pm_timer)) return;
|
||||
|
||||
timer_gas.address_space_id = .SystemIo;
|
||||
timer_gas.address = fadt.pm_tmr_blk;
|
||||
timer_gas.access_size = 3;
|
||||
@@ -46,7 +48,7 @@ pub fn init(fadt: *acpi.Fadt) linksection(b.init) void {
|
||||
pm_timer.mask = std.math.maxInt(u32);
|
||||
}
|
||||
|
||||
ke.time.register_source(&pm_timer);
|
||||
ke.clock.register_source(&pm_timer);
|
||||
}
|
||||
|
||||
fn read_timer() u64 {
|
||||
|
||||
@@ -160,7 +160,7 @@ fn lapic_calibrate(ms: u64) u64 {
|
||||
write(.LvtTimer, (1 << 16));
|
||||
write(.TimerInitialCount, std.math.maxInt(u32));
|
||||
|
||||
ke.time.sleep(std.time.ns_per_ms * ms);
|
||||
ke.clock.sleep(std.time.ns_per_ms * ms);
|
||||
|
||||
const ticks = std.math.maxInt(u32) - read(.TimerCurrentCount);
|
||||
|
||||
@@ -198,7 +198,7 @@ fn timer_init() linksection(r.init) void {
|
||||
pub fn arm_timer(ns: r.Nanoseconds) void {
|
||||
if (amd64.cpu_features.tsc_deadline) {
|
||||
const now = amd64.rdtsc();
|
||||
const deadline = now + (ns * tsc.tsc_timer.frequency / std.time.ns_per_s);
|
||||
const deadline = now + (ns * tsc.tsc.frequency / std.time.ns_per_s);
|
||||
amd64.write_msr(.TscDeadline, deadline);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ const mm = r.mm;
|
||||
const ke = r.ke;
|
||||
|
||||
var pvclock_src: kvm.PVClockSource = undefined;
|
||||
var pvclock_tc: ke.TimeCounter = .{
|
||||
var pvclock: ke.ClockSource = .{
|
||||
.name = "pvclock",
|
||||
.quality = 75,
|
||||
.frequency = std.time.ns_per_s,
|
||||
@@ -17,9 +17,10 @@ var pvclock_tc: ke.TimeCounter = .{
|
||||
};
|
||||
|
||||
pub fn init() linksection(r.init) void {
|
||||
if (ke.clock.is_better_than(&pvclock)) return;
|
||||
const page = mm.phys.alloc();
|
||||
pvclock_src = kvm.PVClockSource.init(page, mm.p2v(page));
|
||||
ke.time.register_source(&pvclock_tc);
|
||||
ke.clock.register_source(&pvclock);
|
||||
}
|
||||
|
||||
fn read_pvclock() u64 {
|
||||
|
||||
@@ -103,9 +103,9 @@ pub fn init() linksection(r.init) void {
|
||||
// Send the INIT-SIPI-SIPI sequence to start the AP.
|
||||
apic.send_init(apic_id);
|
||||
|
||||
ke.time.sleep(init_delay);
|
||||
ke.clock.sleep(init_delay);
|
||||
apic.send_sipi(apic_id, 0x08);
|
||||
ke.time.sleep(sipi_delay);
|
||||
ke.clock.sleep(sipi_delay);
|
||||
apic.send_sipi(apic_id, 0x08);
|
||||
}
|
||||
|
||||
|
||||
+10
-10
@@ -6,7 +6,7 @@ const ke = r.ke;
|
||||
|
||||
const log = std.log.scoped(.@"amd64/tsc");
|
||||
|
||||
pub var tsc_timer: ke.TimeCounter = .{
|
||||
pub var tsc: ke.ClockSource = .{
|
||||
.name = "TSC",
|
||||
.quality = 100,
|
||||
.frequency = 0,
|
||||
@@ -21,7 +21,7 @@ pub fn init() linksection(r.init) void {
|
||||
|
||||
if (!features.invariant_tsc) {
|
||||
log.info("invariant TSC not supported (get a new PC), using fallback", .{});
|
||||
if (ke.time.best()) |best| {
|
||||
if (ke.clock.best()) |best| {
|
||||
log.info("using {s} as timecounter source", .{best.name});
|
||||
}
|
||||
return;
|
||||
@@ -33,14 +33,14 @@ pub fn init() linksection(r.init) void {
|
||||
if (cpuid_state.ebx != 0 and cpuid_state.ecx != 0 and cpuid_state.eax != 0) {
|
||||
const tsc_freq = (cpuid_state.ebx / cpuid_state.eax) * cpuid_state.ecx;
|
||||
log.info("TSC frequency determined via CPUID: {} Hz", .{tsc_freq});
|
||||
tsc_timer.frequency = tsc_freq;
|
||||
tsc.frequency = tsc_freq;
|
||||
|
||||
ke.time.register_source(&tsc_timer);
|
||||
ke.clock.register_source(&tsc);
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback to the other best time source to calibrate.
|
||||
const best = ke.time.best() orelse return;
|
||||
const best = ke.clock.best() orelse return;
|
||||
|
||||
const calib_cost_runs = 5;
|
||||
var calib_cost: u64 = 0;
|
||||
@@ -48,7 +48,7 @@ pub fn init() linksection(r.init) void {
|
||||
// Calculate how many ticks it costs to call tc_sleep.
|
||||
for (0..calib_cost_runs) |_| {
|
||||
const start = amd64.rdtsc();
|
||||
ke.time.sleep(0);
|
||||
ke.clock.sleep(0);
|
||||
const end = amd64.rdtsc();
|
||||
calib_cost += (end - start);
|
||||
}
|
||||
@@ -63,7 +63,7 @@ pub fn init() linksection(r.init) void {
|
||||
|
||||
for (0..runs) |i| {
|
||||
const start = amd64.rdtsc();
|
||||
ke.time.sleep(calib_time);
|
||||
ke.clock.sleep(calib_time);
|
||||
const end = amd64.rdtsc();
|
||||
freqs[i] = (end - start - calib_cost) * (std.time.ns_per_s / calib_time);
|
||||
}
|
||||
@@ -73,11 +73,11 @@ pub fn init() linksection(r.init) void {
|
||||
var sum: u64 = 0;
|
||||
for (freqs[1 .. runs - 1]) |f| sum += f;
|
||||
|
||||
tsc_timer.frequency = sum / (runs - 2);
|
||||
tsc.frequency = sum / (runs - 2);
|
||||
|
||||
ke.time.register_source(&tsc_timer);
|
||||
ke.clock.register_source(&tsc);
|
||||
|
||||
log.info("frequency calibrated using {s}: {}.{} MHz", .{ best.name, tsc_timer.frequency / 1_000_000, (tsc_timer.frequency % 1_000_000) / 1000 });
|
||||
log.info("frequency calibrated using {s}: {}.{} MHz", .{ best.name, tsc.frequency / 1_000_000, (tsc.frequency % 1_000_000) / 1000 });
|
||||
}
|
||||
|
||||
fn read_tsc() u64 {
|
||||
|
||||
@@ -39,7 +39,7 @@ fn tc_read() u64 {
|
||||
return @intCast(ts.sec * std.time.ns_per_s + ts.nsec);
|
||||
}
|
||||
|
||||
var tc: ke.TimeCounter = .{
|
||||
var tc: ke.ClockSource = .{
|
||||
.read_count = tc_read,
|
||||
.frequency = std.time.ns_per_s,
|
||||
.name = "CLOCK_MONOTONIC",
|
||||
@@ -59,7 +59,7 @@ pub fn init() void {
|
||||
|
||||
posix.sigaction(posix.SIG.ALRM, &act, null);
|
||||
|
||||
ke.time.register_source(&tc);
|
||||
ke.clock.register_source(&tc);
|
||||
}
|
||||
|
||||
pub fn init_cpu() void {
|
||||
|
||||
Reference in New Issue
Block a user