From a1db75746adf4e834fdac7b56ba4298fd39a27b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Gr=C3=A9goire?= Date: Sat, 18 Apr 2026 11:18:33 -0400 Subject: [PATCH] feat(pl/pc): add APIC timer support --- build.zig | 39 +++++++++++-------- src/arch/amd64/root.zig | 2 + src/ex/init.zig | 20 ++++++++-- src/kern/amd64/init.zig | 13 ++++--- src/kern/amd64/int.zig | 17 +++++++++ src/kern/sched.zig | 3 +- src/kern/um/impl.zig | 20 +++++----- src/platform/pc/apic.zig | 79 +++++++++++++++++++++++++++++++++++++++ src/platform/pc/impl.zig | 10 +++-- src/platform/pc/smp.zig | 9 +++-- src/platform/pc/tsc.zig | 4 +- src/platform/root.zig | 3 +- src/platform/um/timer.zig | 4 +- 13 files changed, 175 insertions(+), 48 deletions(-) diff --git a/build.zig b/build.zig index 911a5f1..61337f5 100644 --- a/build.zig +++ b/build.zig @@ -193,29 +193,36 @@ fn addKernel(b: *std.Build, plat: config.Platform, optimize: std.builtin.Optimiz kernel.root_module.omit_frame_pointer = false; + const modules: []const *std.Build.Module = &.{ config_module, ksyms_module, rtl, arch, kernel.root_module }; + if (plat.os == .freestanding) { kernel.linkage = .static; kernel.pie = false; - kernel.root_module.pic = false; - kernel.root_module.strip = false; - kernel.root_module.stack_check = false; - kernel.root_module.stack_protector = false; - kernel.root_module.unwind_tables = .none; + for (modules) |m| { + m.pic = false; + m.strip = false; + m.stack_check = false; + m.stack_protector = false; + m.unwind_tables = .none; + m.omit_frame_pointer = false; + + m.code_model = switch (plat.arch) { + .x86_64 => .kernel, + .riscv64 => .medium, + else => .large, + }; + + switch (plat.arch) { + .x86_64 => { + m.red_zone = false; + }, + else => {}, + } + } kernel.entry = .{ .symbol_name = "kmain" }; - kernel.root_module.code_model = switch (plat.arch) { - .x86_64 => .kernel, - .riscv64 => .medium, - else => .large, - }; - - switch (plat.arch) { - .x86_64 => kernel.root_module.red_zone = false, - else => {}, - } - switch (plat.arch) { .x86_64 => { kernel.root_module.addAssemblyFile(b.path("src/kern/amd64/locore.s")); diff --git a/src/arch/amd64/root.zig b/src/arch/amd64/root.zig index c8f69dc..a35e374 100644 --- a/src/arch/amd64/root.zig +++ b/src/arch/amd64/root.zig @@ -21,6 +21,8 @@ pub const Msr = enum(u32) { Star = 0xC000_0081, /// IA32_LSTAR LStar = 0xC000_0082, + /// IA32_TSC_DEADLINE + TscDeadline = 0x6E0, /// MSR_KVM_SYSTEM_TIME_NEW KvmSystemTimeNew = 0x4B564D01, _, diff --git a/src/ex/init.zig b/src/ex/init.zig index 4a71fe3..4839c38 100644 --- a/src/ex/init.zig +++ b/src/ex/init.zig @@ -1,8 +1,13 @@ const std = @import("std"); -const b = @import("root"); -const pl = b.pl; -const ke = b.ke; -const mm = b.mm; +const r = @import("root"); +const pl = r.pl; +const ke = r.ke; +const mm = r.mm; +const amd64 = @import("arch"); + +fn callback(_: ?*anyopaque) void { + std.log.info("Timer callback called", .{}); +} pub fn init(boot_info: *pl.BootInfo) void { mm.init(boot_info); @@ -10,6 +15,13 @@ pub fn init(boot_info: *pl.BootInfo) void { pl.late_init(boot_info); ke.sched.late_init(); + var timer: ke.Timer = undefined; + var dpc: ke.Dpc = .init(callback); + + timer.init(); + + ke.timer.set(&timer, std.time.ns_per_ms * 50, &dpc); + while (true) { std.atomic.spinLoopHint(); } diff --git a/src/kern/amd64/init.zig b/src/kern/amd64/init.zig index a56b631..a81d8b7 100644 --- a/src/kern/amd64/init.zig +++ b/src/kern/amd64/init.zig @@ -81,30 +81,33 @@ fn early_cpu_init() linksection(r.init) void { efer.nxe = true; amd64.write_msr(.Efer, @bitCast(efer)); + + amd64.sti(); } -pub fn ap_entry(cpu_id: u32) noreturn { +pub fn ap_entry(cpu_id: u32, booted: *std.atomic.Value(usize)) noreturn { early_cpu_init(); ki.cpu.init_cpu(cpu_id); pl.impl.init_ap(); + _ = booted.fetchAdd(1, .monotonic); + while (true) { std.atomic.spinLoopHint(); } } -var initial_offsets: [1]usize = undefined; +var initial_offsets: [1]usize = .{0}; extern var __percpu_start: u8; pub fn early_init() linksection(r.init) void { amd64.detect_cpu_features(); early_cpu_init(); - initial_offsets[0] = @intFromPtr(&__percpu_start); impl.cpu_offsets = &initial_offsets; - int.init(); - // Enable Per-CPU data for this CPU amd64.write_msr(.GsBase, 0); + + int.init(); } diff --git a/src/kern/amd64/int.zig b/src/kern/amd64/int.zig index 7a7ab29..8aca96b 100644 --- a/src/kern/amd64/int.zig +++ b/src/kern/amd64/int.zig @@ -3,6 +3,7 @@ const std = @import("std"); const r = @import("root"); const ke = r.ke; const ki = ke.private; +const pl = r.pl; extern const __interrupt_vectors: [256]usize; @@ -44,6 +45,8 @@ const exception_msg = [32][]const u8{ }; export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sysv = .{} }) void { + const ipl = ke.ipl.set_hardware(.Device); + if (frame.intno < 32) { std.log.err("Unhandled exception: 0x{x} ({s}), err=0x{x}, pc=0x{x}", .{ frame.intno, exception_msg[frame.intno], frame.errcode, frame.rip }); std.log.err("RAX=0x{x:0>16} RBX=0x{x:0>16} RCX=0x{x:0>16} RDX=0x{x:0>16}", .{ frame.rax, frame.rbx, frame.rcx, frame.rdx }); @@ -75,6 +78,20 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys ki.panic.panic_with_frame("Unhandled exception", frame.rbp); } + + if (frame.intno == 32) { + ki.timer.clock(); + } + + if (frame.intno >= 32) { + pl.impl.apic.eoi(); + } + + ki.impl.set_hardware_ipl(ipl); + + if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and ki.ipl.is_softint_pending(.Dispatch)) { + ki.dpc.dispatch(ke.cpu.current()); + } } extern fn idt_load(idt_ptr: *const amd64.Idtr) callconv(.{ .x86_64_sysv = .{} }) void; diff --git a/src/kern/sched.zig b/src/kern/sched.zig index f6acb10..feff9ed 100644 --- a/src/kern/sched.zig +++ b/src/kern/sched.zig @@ -100,7 +100,7 @@ const balance_interval = @as(u64, config.CONFIG_BALANCE_INTERVAL); const steal_threshold = 1; var balance_timer: ke.Timer = undefined; -var balance_dpc: ke.Dpc = undefined; +var balance_dpc: ke.Dpc = .init(balance); // LCG values taken from FreeBSD. var lcg = std.Random.lcg.Wrapping(u32).init(0, 69069, 5); @@ -313,7 +313,6 @@ fn init_cpu() linksection(r.init) callconv(.c) void { /// Called on CPU 0 to initialize load balancing mechanisms. pub fn late_init() linksection(r.init) void { - balance_dpc = ke.Dpc.init(balance); balance_timer.init(); ke.timer.set(&balance_timer, balance_interval * std.time.ns_per_ms, &balance_dpc); } diff --git a/src/kern/um/impl.zig b/src/kern/um/impl.zig index 6b7669e..910cdcb 100644 --- a/src/kern/um/impl.zig +++ b/src/kern/um/impl.zig @@ -8,7 +8,10 @@ fn set_signals(enabled: bool) bool { var newmask: c.sigset_t = undefined; var oldmask: c.sigset_t = undefined; - _ = c.sigfillset(&newmask); + _ = c.sigemptyset(&newmask); + _ = c.sigaddset(&newmask, c.SIGUSR1); + _ = c.sigaddset(&newmask, c.SIGUSR2); + _ = c.sigaddset(&newmask, c.SIGALRM); const how = if (enabled) c.SIG_UNBLOCK else c.SIG_BLOCK; @@ -19,12 +22,13 @@ fn set_signals(enabled: bool) bool { return c.sigismember(&oldmask, c.SIGUSR1) == 0; } -threadlocal var hard_ipl: ke.Ipl = .Passive; - /// Set the hardware IPL. pub fn set_hardware_ipl(ipl: ke.Ipl) void { - // This is a no-op since we handle this in enable_interrupts(). - hard_ipl = ipl; + if (@intFromEnum(ipl) > @intFromEnum(ke.Ipl.get_max_software())) { + _ = disable_interrupts(); + } else { + _ = enable_interrupts(); + } } /// Disable interrupts and return the previous state. @@ -34,14 +38,12 @@ pub fn disable_interrupts() bool { /// Enable interrupts and return the previous state. pub fn enable_interrupts() bool { - const ipl_allows_ints = @intFromEnum(hard_ipl) <= @intFromEnum(ke.Ipl.get_max_software()); - return set_signals(ipl_allows_ints); + return set_signals(true); } /// Restore an interrupt state. pub fn restore_interrupts(val: bool) void { - const ipl_allows_ints = @intFromEnum(hard_ipl) <= @intFromEnum(ke.Ipl.get_max_software()); - _ = set_signals(val and ipl_allows_ints); + _ = set_signals(val); } pub fn send_resched_ipi(cpu: u32) void { diff --git a/src/platform/pc/apic.zig b/src/platform/pc/apic.zig index f263d51..7c77852 100644 --- a/src/platform/pc/apic.zig +++ b/src/platform/pc/apic.zig @@ -1,6 +1,7 @@ const std = @import("std"); const amd64 = @import("arch"); const r = @import("root"); +const tsc = @import("tsc.zig"); const acpi = r.pl.acpi; const mm = r.mm; const ke = r.ke; @@ -74,6 +75,7 @@ pub var apics: std.ArrayList(u32) = .empty; var xapic_address: ?usize = null; pub var xapic_base_physical: usize = 0; const in_x2apic_mode = ke.CpuLocal(bool, false); +const timer_ticks_per_us = ke.CpuLocal(u64, 0); const log = std.log.scoped(.apic); @@ -154,6 +156,82 @@ pub fn send_ipi(apic_id: u32, vector: u8, delivery_mode: u8) void { } } +fn lapic_calibrate(ms: u64) u64 { + write(.LvtTimer, (1 << 16)); + write(.TimerInitialCount, std.math.maxInt(u32)); + + ke.time.sleep(std.time.ns_per_ms * ms); + + const ticks = std.math.maxInt(u32) - read(.TimerCurrentCount); + + write(.LvtTimer, (1 << 16)); + write(.TimerInitialCount, 0); + + return ticks; +} + +fn timer_init() linksection(r.init) void { + if (amd64.cpu_features.tsc_deadline) { + // TSC-deadline mode. + write(.LvtTimer, 32 | (2 << 17)); + asm volatile ("mfence" ::: .{ .memory = true }); + return; + } + + // Divide by 16. + write(.TimerDivideConfig, 0x3); + + const runs = 5; + const calib_ms = 10; + var total_ticks: u64 = 0; + + for (0..runs) |_| { + total_ticks += lapic_calibrate(calib_ms); + } + + const avg_ticks = total_ticks / runs; + const calib_us = calib_ms * std.time.us_per_ms; + + timer_ticks_per_us.local().* = (avg_ticks / calib_us); +} + +pub fn arm_timer(ns: r.Nanoseconds) void { + if (amd64.cpu_features.tsc_deadline) { + const now = amd64.rdtsc(); + const deadline = now + (ns * tsc.tsc_timer.frequency / std.time.ns_per_s); + amd64.write_msr(.TscDeadline, deadline); + return; + } + + // Masked. + write(.LvtTimer, 1 << 16); + write(.TimerInitialCount, 0); + + const us = ns / std.time.ns_per_us; + const ticks: u32 = @truncate(@max(us * timer_ticks_per_us.local().*, 1)); + + // Setup IRQ. + write(.LvtTimer, 32); + + // Divide by 16 + write(.TimerDivideConfig, 0x3); + write(.TimerInitialCount, @as(u32, ticks)); +} + +pub fn stop_timer() void { + if (amd64.cpu_features.tsc_deadline) { + amd64.write_msr(.TscDeadline, 0); + return; + } + + write(.LvtTimer, (1 << 16)); + write(.TimerInitialCount, 0); +} + +pub fn eoi() void { + write(.Eoi, 0); +} + pub fn init() linksection(r.init) void { var iter = acpi.madt.iterator(); @@ -210,6 +288,7 @@ pub fn init_local() void { write(.Svr, @intCast(read(.Svr) | 0x1FF)); write(.Tpr, 0); write(.Eoi, 0); + timer_init(); } pub fn enter_x2apic() void { diff --git a/src/platform/pc/impl.zig b/src/platform/pc/impl.zig index abea30b..eea73d3 100644 --- a/src/platform/pc/impl.zig +++ b/src/platform/pc/impl.zig @@ -4,7 +4,7 @@ const amd64 = r.arch; const std = @import("std"); const tsc = @import("tsc.zig"); const pvclock = @import("pvclock.zig"); -const apic = @import("apic.zig"); +pub const apic = @import("apic.zig"); const smp = @import("smp.zig"); const ki = r.ke.private; @@ -80,12 +80,12 @@ pub fn init_ap() void { } pub fn late_init(boot_info: *pl.BootInfo) linksection(r.init) void { - pl.acpi.init(boot_info); - if (amd64.hypervisor.info) |hv| { init_hypervisor(hv); } + pl.acpi.init(boot_info); + tsc.init(); apic.init(); @@ -102,4 +102,6 @@ pub fn debug_write(c: u8) void { com1_write(c); } -pub fn arm_timer(_: u64) void {} +pub fn arm_timer(deadline: r.Nanoseconds) void { + apic.arm_timer(deadline); +} diff --git a/src/platform/pc/smp.zig b/src/platform/pc/smp.zig index e7d4f6f..2ab031a 100644 --- a/src/platform/pc/smp.zig +++ b/src/platform/pc/smp.zig @@ -32,8 +32,7 @@ const ApData = extern struct { }; fn ap_entry(cpu_id: u32) callconv(.c) noreturn { - _ = aps_booted.fetchAdd(1, .monotonic); - ki.impl.init.ap_entry(cpu_id); + ki.impl.init.ap_entry(cpu_id, &aps_booted); } pub fn init() linksection(r.init) void { @@ -70,7 +69,7 @@ pub fn init() linksection(r.init) void { ki.impl.cpu_offsets = @ptrCast(mm.zone.gpa.alloc(usize, apic.apics.items.len + 1) catch @panic("Failed to allocate AP local data offsets")); const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start); - ki.impl.cpu_offsets[0] = @intFromPtr(&__percpu_start); + ki.impl.cpu_offsets[0] = 0; cpu_id_to_apic_id[0] = apic.get_id(); @@ -89,8 +88,12 @@ pub fn init() linksection(r.init) void { const cpu_data = mm.zone.gpa.alloc(u8, percpu_size) catch @panic("Failed to allocate per-cpu data"); @memcpy(cpu_data, @as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size]); + const self_offset_offset = @intFromPtr(&ki.impl.cpu_self_offset) - @intFromPtr(&__percpu_start); ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -% @intFromPtr(&__percpu_start); + // copy the offset into the AP's self_offset variable + @as(*usize, @ptrCast(@alignCast(&cpu_data[self_offset_offset]))).* = ki.impl.cpu_offsets[cpu_id]; + const stack_top = @intFromPtr(mm.heap.alloc(r.kib(16)) catch @panic("Failed to allocate AP stack")) + r.kib(16); start_stack.remote(@intCast(cpu_id)).* = stack_top & ~@as(usize, 15); diff --git a/src/platform/pc/tsc.zig b/src/platform/pc/tsc.zig index 2d79b4c..46952d9 100644 --- a/src/platform/pc/tsc.zig +++ b/src/platform/pc/tsc.zig @@ -6,7 +6,7 @@ const ke = r.ke; const log = std.log.scoped(.@"amd64/tsc"); -var tsc_timer: ke.TimeCounter = .{ +pub var tsc_timer: ke.TimeCounter = .{ .name = "TSC", .quality = 100, .frequency = 0, @@ -30,7 +30,7 @@ pub fn init() linksection(r.init) void { // Try using cpuid leaf 0x15 to get the TSC frequency. const cpuid_state = amd64.cpuid(0x15, 0); - if (cpuid_state.ebx != 0 and cpuid_state.ecx != 0) { + if (cpuid_state.ebx != 0 and cpuid_state.ecx != 0 and cpuid_state.eax != 0) { const tsc_freq = (cpuid_state.ebx / cpuid_state.eax) * cpuid_state.ecx; log.info("TSC frequency determined via CPUID: {} Hz", .{tsc_freq}); tsc_timer.frequency = tsc_freq; diff --git a/src/platform/root.zig b/src/platform/root.zig index 0f3b799..0c13db9 100644 --- a/src/platform/root.zig +++ b/src/platform/root.zig @@ -29,12 +29,11 @@ pub inline fn debug_write(c: u8) void { } /// Arm a one-shot timer to fire in `ns` nanoseconds. -/// The timer must call `ki_timer_expiry()` when it fires. pub inline fn arm_timer(ns: u64) void { return impl.arm_timer(ns); } -/// Pretty name for the platform +/// Pretty name for the platform. pub const name = impl.name; /// A generic struct representing information passed from the bootloader. diff --git a/src/platform/um/timer.zig b/src/platform/um/timer.zig index 1d9df87..8d4d1b7 100644 --- a/src/platform/um/timer.zig +++ b/src/platform/um/timer.zig @@ -50,11 +50,13 @@ var tc: ke.TimeCounter = .{ }; pub fn init() void { - var act: posix.Sigaction = undefined; + var act = std.mem.zeroes(posix.Sigaction); act.handler.sigaction = timer_handler; act.flags = posix.SA.RESTART | posix.SA.SIGINFO; + act.mask = posix.sigfillset(); + posix.sigaction(posix.SIG.ALRM, &act, null); ke.time.register_source(&tc);