feat(pl/pc): add APIC timer support

build.zig

@@ -193,29 +193,36 @@ fn addKernel(b: *std.Build, plat: config.Platform, optimize: std.builtin.Optimiz
 
     kernel.root_module.omit_frame_pointer = false;
 
+    const modules: []const *std.Build.Module = &.{ config_module, ksyms_module, rtl, arch, kernel.root_module };
+
     if (plat.os == .freestanding) {
         kernel.linkage = .static;
         kernel.pie = false;
 
-        kernel.root_module.pic = false;
+        for (modules) |m| {
-        kernel.root_module.strip = false;
+            m.pic = false;
-        kernel.root_module.stack_check = false;
+            m.strip = false;
-        kernel.root_module.stack_protector = false;
+            m.stack_check = false;
-        kernel.root_module.unwind_tables = .none;
+            m.stack_protector = false;
+            m.unwind_tables = .none;
+            m.omit_frame_pointer = false;
+
+            m.code_model = switch (plat.arch) {
+                .x86_64 => .kernel,
+                .riscv64 => .medium,
+                else => .large,
+            };
+
+            switch (plat.arch) {
+                .x86_64 => {
+                    m.red_zone = false;
+                },
+                else => {},
+            }
+        }
 
         kernel.entry = .{ .symbol_name = "kmain" };
 
-        kernel.root_module.code_model = switch (plat.arch) {
-            .x86_64 => .kernel,
-            .riscv64 => .medium,
-            else => .large,
-        };
-
-        switch (plat.arch) {
-            .x86_64 => kernel.root_module.red_zone = false,
-            else => {},
-        }
-
         switch (plat.arch) {
             .x86_64 => {
                 kernel.root_module.addAssemblyFile(b.path("src/kern/amd64/locore.s"));

src/arch/amd64/root.zig

@@ -21,6 +21,8 @@ pub const Msr = enum(u32) {
     Star = 0xC000_0081,
     /// IA32_LSTAR
     LStar = 0xC000_0082,
+    /// IA32_TSC_DEADLINE
+    TscDeadline = 0x6E0,
     /// MSR_KVM_SYSTEM_TIME_NEW
     KvmSystemTimeNew = 0x4B564D01,
     _,

src/ex/init.zig

@@ -1,8 +1,13 @@
 const std = @import("std");
-const b = @import("root");
+const r = @import("root");
-const pl = b.pl;
+const pl = r.pl;
-const ke = b.ke;
+const ke = r.ke;
-const mm = b.mm;
+const mm = r.mm;
+const amd64 = @import("arch");
+
+fn callback(_: ?*anyopaque) void {
+    std.log.info("Timer callback called", .{});
+}
 
 pub fn init(boot_info: *pl.BootInfo) void {
     mm.init(boot_info);
@@ -10,6 +15,13 @@ pub fn init(boot_info: *pl.BootInfo) void {
     pl.late_init(boot_info);
     ke.sched.late_init();
 
+    var timer: ke.Timer = undefined;
+    var dpc: ke.Dpc = .init(callback);
+
+    timer.init();
+
+    ke.timer.set(&timer, std.time.ns_per_ms * 50, &dpc);
+
     while (true) {
         std.atomic.spinLoopHint();
     }

src/kern/amd64/init.zig

@@ -81,30 +81,33 @@ fn early_cpu_init() linksection(r.init) void {
     efer.nxe = true;
 
     amd64.write_msr(.Efer, @bitCast(efer));
+
+    amd64.sti();
 }
 
-pub fn ap_entry(cpu_id: u32) noreturn {
+pub fn ap_entry(cpu_id: u32, booted: *std.atomic.Value(usize)) noreturn {
     early_cpu_init();
     ki.cpu.init_cpu(cpu_id);
     pl.impl.init_ap();
 
+    _ = booted.fetchAdd(1, .monotonic);
+
     while (true) {
         std.atomic.spinLoopHint();
     }
 }
 
-var initial_offsets: [1]usize = undefined;
+var initial_offsets: [1]usize = .{0};
 extern var __percpu_start: u8;
 
 pub fn early_init() linksection(r.init) void {
     amd64.detect_cpu_features();
     early_cpu_init();
 
-    initial_offsets[0] = @intFromPtr(&__percpu_start);
     impl.cpu_offsets = &initial_offsets;
 
-    int.init();
-
     // Enable Per-CPU data for this CPU
     amd64.write_msr(.GsBase, 0);
+
+    int.init();
 }

src/kern/amd64/int.zig

@@ -3,6 +3,7 @@ const std = @import("std");
 const r = @import("root");
 const ke = r.ke;
 const ki = ke.private;
+const pl = r.pl;
 
 extern const __interrupt_vectors: [256]usize;
 
@@ -44,6 +45,8 @@ const exception_msg = [32][]const u8{
 };
 
 export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sysv = .{} }) void {
+    const ipl = ke.ipl.set_hardware(.Device);
+
     if (frame.intno < 32) {
         std.log.err("Unhandled exception: 0x{x} ({s}), err=0x{x}, pc=0x{x}", .{ frame.intno, exception_msg[frame.intno], frame.errcode, frame.rip });
         std.log.err("RAX=0x{x:0>16} RBX=0x{x:0>16} RCX=0x{x:0>16} RDX=0x{x:0>16}", .{ frame.rax, frame.rbx, frame.rcx, frame.rdx });
@@ -75,6 +78,20 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
 
         ki.panic.panic_with_frame("Unhandled exception", frame.rbp);
     }
+
+    if (frame.intno == 32) {
+        ki.timer.clock();
+    }
+
+    if (frame.intno >= 32) {
+        pl.impl.apic.eoi();
+    }
+
+    ki.impl.set_hardware_ipl(ipl);
+
+    if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and ki.ipl.is_softint_pending(.Dispatch)) {
+        ki.dpc.dispatch(ke.cpu.current());
+    }
 }
 
 extern fn idt_load(idt_ptr: *const amd64.Idtr) callconv(.{ .x86_64_sysv = .{} }) void;

src/kern/sched.zig

@@ -100,7 +100,7 @@ const balance_interval = @as(u64, config.CONFIG_BALANCE_INTERVAL);
 const steal_threshold = 1;
 
 var balance_timer: ke.Timer = undefined;
-var balance_dpc: ke.Dpc = undefined;
+var balance_dpc: ke.Dpc = .init(balance);
 
 // LCG values taken from FreeBSD.
 var lcg = std.Random.lcg.Wrapping(u32).init(0, 69069, 5);
@@ -313,7 +313,6 @@ fn init_cpu() linksection(r.init) callconv(.c) void {
 
 /// Called on CPU 0 to initialize load balancing mechanisms.
 pub fn late_init() linksection(r.init) void {
-    balance_dpc = ke.Dpc.init(balance);
     balance_timer.init();
     ke.timer.set(&balance_timer, balance_interval * std.time.ns_per_ms, &balance_dpc);
 }

src/kern/um/impl.zig

@@ -8,7 +8,10 @@ fn set_signals(enabled: bool) bool {
     var newmask: c.sigset_t = undefined;
     var oldmask: c.sigset_t = undefined;
 
-    _ = c.sigfillset(&newmask);
+    _ = c.sigemptyset(&newmask);
+    _ = c.sigaddset(&newmask, c.SIGUSR1);
+    _ = c.sigaddset(&newmask, c.SIGUSR2);
+    _ = c.sigaddset(&newmask, c.SIGALRM);
 
     const how = if (enabled) c.SIG_UNBLOCK else c.SIG_BLOCK;
 
@@ -19,12 +22,13 @@ fn set_signals(enabled: bool) bool {
     return c.sigismember(&oldmask, c.SIGUSR1) == 0;
 }
 
-threadlocal var hard_ipl: ke.Ipl = .Passive;
-
 /// Set the hardware IPL.
 pub fn set_hardware_ipl(ipl: ke.Ipl) void {
-    // This is a no-op since we handle this in enable_interrupts().
+    if (@intFromEnum(ipl) > @intFromEnum(ke.Ipl.get_max_software())) {
-    hard_ipl = ipl;
+        _ = disable_interrupts();
+    } else {
+        _ = enable_interrupts();
+    }
 }
 
 /// Disable interrupts and return the previous state.
@@ -34,14 +38,12 @@ pub fn disable_interrupts() bool {
 
 /// Enable interrupts and return the previous state.
 pub fn enable_interrupts() bool {
-    const ipl_allows_ints = @intFromEnum(hard_ipl) <= @intFromEnum(ke.Ipl.get_max_software());
+    return set_signals(true);
-    return set_signals(ipl_allows_ints);
 }
 
 /// Restore an interrupt state.
 pub fn restore_interrupts(val: bool) void {
-    const ipl_allows_ints = @intFromEnum(hard_ipl) <= @intFromEnum(ke.Ipl.get_max_software());
+    _ = set_signals(val);
-    _ = set_signals(val and ipl_allows_ints);
 }
 
 pub fn send_resched_ipi(cpu: u32) void {

src/platform/pc/apic.zig

@@ -1,6 +1,7 @@
 const std = @import("std");
 const amd64 = @import("arch");
 const r = @import("root");
+const tsc = @import("tsc.zig");
 const acpi = r.pl.acpi;
 const mm = r.mm;
 const ke = r.ke;
@@ -74,6 +75,7 @@ pub var apics: std.ArrayList(u32) = .empty;
 var xapic_address: ?usize = null;
 pub var xapic_base_physical: usize = 0;
 const in_x2apic_mode = ke.CpuLocal(bool, false);
+const timer_ticks_per_us = ke.CpuLocal(u64, 0);
 
 const log = std.log.scoped(.apic);
 
@@ -154,6 +156,82 @@ pub fn send_ipi(apic_id: u32, vector: u8, delivery_mode: u8) void {
     }
 }
 
+fn lapic_calibrate(ms: u64) u64 {
+    write(.LvtTimer, (1 << 16));
+    write(.TimerInitialCount, std.math.maxInt(u32));
+
+    ke.time.sleep(std.time.ns_per_ms * ms);
+
+    const ticks = std.math.maxInt(u32) - read(.TimerCurrentCount);
+
+    write(.LvtTimer, (1 << 16));
+    write(.TimerInitialCount, 0);
+
+    return ticks;
+}
+
+fn timer_init() linksection(r.init) void {
+    if (amd64.cpu_features.tsc_deadline) {
+        // TSC-deadline mode.
+        write(.LvtTimer, 32 | (2 << 17));
+        asm volatile ("mfence" ::: .{ .memory = true });
+        return;
+    }
+
+    // Divide by 16.
+    write(.TimerDivideConfig, 0x3);
+
+    const runs = 5;
+    const calib_ms = 10;
+    var total_ticks: u64 = 0;
+
+    for (0..runs) |_| {
+        total_ticks += lapic_calibrate(calib_ms);
+    }
+
+    const avg_ticks = total_ticks / runs;
+    const calib_us = calib_ms * std.time.us_per_ms;
+
+    timer_ticks_per_us.local().* = (avg_ticks / calib_us);
+}
+
+pub fn arm_timer(ns: r.Nanoseconds) void {
+    if (amd64.cpu_features.tsc_deadline) {
+        const now = amd64.rdtsc();
+        const deadline = now + (ns * tsc.tsc_timer.frequency / std.time.ns_per_s);
+        amd64.write_msr(.TscDeadline, deadline);
+        return;
+    }
+
+    // Masked.
+    write(.LvtTimer, 1 << 16);
+    write(.TimerInitialCount, 0);
+
+    const us = ns / std.time.ns_per_us;
+    const ticks: u32 = @truncate(@max(us * timer_ticks_per_us.local().*, 1));
+
+    // Setup IRQ.
+    write(.LvtTimer, 32);
+
+    // Divide by 16
+    write(.TimerDivideConfig, 0x3);
+    write(.TimerInitialCount, @as(u32, ticks));
+}
+
+pub fn stop_timer() void {
+    if (amd64.cpu_features.tsc_deadline) {
+        amd64.write_msr(.TscDeadline, 0);
+        return;
+    }
+
+    write(.LvtTimer, (1 << 16));
+    write(.TimerInitialCount, 0);
+}
+
+pub fn eoi() void {
+    write(.Eoi, 0);
+}
+
 pub fn init() linksection(r.init) void {
     var iter = acpi.madt.iterator();
 
@@ -210,6 +288,7 @@ pub fn init_local() void {
     write(.Svr, @intCast(read(.Svr) | 0x1FF));
     write(.Tpr, 0);
     write(.Eoi, 0);
+    timer_init();
 }
 
 pub fn enter_x2apic() void {

src/platform/pc/impl.zig

@@ -4,7 +4,7 @@ const amd64 = r.arch;
 const std = @import("std");
 const tsc = @import("tsc.zig");
 const pvclock = @import("pvclock.zig");
-const apic = @import("apic.zig");
+pub const apic = @import("apic.zig");
 const smp = @import("smp.zig");
 
 const ki = r.ke.private;
@@ -80,12 +80,12 @@ pub fn init_ap() void {
 }
 
 pub fn late_init(boot_info: *pl.BootInfo) linksection(r.init) void {
-    pl.acpi.init(boot_info);
-
     if (amd64.hypervisor.info) |hv| {
         init_hypervisor(hv);
     }
 
+    pl.acpi.init(boot_info);
+
     tsc.init();
     apic.init();
 
@@ -102,4 +102,6 @@ pub fn debug_write(c: u8) void {
     com1_write(c);
 }
 
-pub fn arm_timer(_: u64) void {}
+pub fn arm_timer(deadline: r.Nanoseconds) void {
+    apic.arm_timer(deadline);
+}

src/platform/pc/smp.zig

@@ -32,8 +32,7 @@ const ApData = extern struct {
 };
 
 fn ap_entry(cpu_id: u32) callconv(.c) noreturn {
-    _ = aps_booted.fetchAdd(1, .monotonic);
+    ki.impl.init.ap_entry(cpu_id, &aps_booted);
-    ki.impl.init.ap_entry(cpu_id);
 }
 
 pub fn init() linksection(r.init) void {
@@ -70,7 +69,7 @@ pub fn init() linksection(r.init) void {
     ki.impl.cpu_offsets = @ptrCast(mm.zone.gpa.alloc(usize, apic.apics.items.len + 1) catch @panic("Failed to allocate AP local data offsets"));
     const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start);
 
-    ki.impl.cpu_offsets[0] = @intFromPtr(&__percpu_start);
+    ki.impl.cpu_offsets[0] = 0;
 
     cpu_id_to_apic_id[0] = apic.get_id();
 
@@ -89,8 +88,12 @@ pub fn init() linksection(r.init) void {
         const cpu_data = mm.zone.gpa.alloc(u8, percpu_size) catch @panic("Failed to allocate per-cpu data");
         @memcpy(cpu_data, @as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size]);
 
+        const self_offset_offset = @intFromPtr(&ki.impl.cpu_self_offset) - @intFromPtr(&__percpu_start);
         ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -% @intFromPtr(&__percpu_start);
 
+        // copy the offset into the AP's self_offset variable
+        @as(*usize, @ptrCast(@alignCast(&cpu_data[self_offset_offset]))).* = ki.impl.cpu_offsets[cpu_id];
+
         const stack_top = @intFromPtr(mm.heap.alloc(r.kib(16)) catch @panic("Failed to allocate AP stack")) + r.kib(16);
 
         start_stack.remote(@intCast(cpu_id)).* = stack_top & ~@as(usize, 15);

src/platform/pc/tsc.zig

@@ -6,7 +6,7 @@ const ke = r.ke;
 
 const log = std.log.scoped(.@"amd64/tsc");
 
-var tsc_timer: ke.TimeCounter = .{
+pub var tsc_timer: ke.TimeCounter = .{
     .name = "TSC",
     .quality = 100,
     .frequency = 0,
@@ -30,7 +30,7 @@ pub fn init() linksection(r.init) void {
     // Try using cpuid leaf 0x15 to get the TSC frequency.
     const cpuid_state = amd64.cpuid(0x15, 0);
 
-    if (cpuid_state.ebx != 0 and cpuid_state.ecx != 0) {
+    if (cpuid_state.ebx != 0 and cpuid_state.ecx != 0 and cpuid_state.eax != 0) {
         const tsc_freq = (cpuid_state.ebx / cpuid_state.eax) * cpuid_state.ecx;
         log.info("TSC frequency determined via CPUID: {} Hz", .{tsc_freq});
         tsc_timer.frequency = tsc_freq;

src/platform/root.zig

@@ -29,12 +29,11 @@ pub inline fn debug_write(c: u8) void {
 }
 
 /// Arm a one-shot timer to fire in `ns` nanoseconds.
-/// The timer must call `ki_timer_expiry()` when it fires.
 pub inline fn arm_timer(ns: u64) void {
     return impl.arm_timer(ns);
 }
 
-/// Pretty name for the platform
+/// Pretty name for the platform.
 pub const name = impl.name;
 
 /// A generic struct representing information passed from the bootloader.

src/platform/um/timer.zig

@@ -50,11 +50,13 @@ var tc: ke.TimeCounter = .{
 };
 
 pub fn init() void {
-    var act: posix.Sigaction = undefined;
+    var act = std.mem.zeroes(posix.Sigaction);
 
     act.handler.sigaction = timer_handler;
     act.flags = posix.SA.RESTART | posix.SA.SIGINFO;
 
+    act.mask = posix.sigfillset();
+
     posix.sigaction(posix.SIG.ALRM, &act, null);
 
     ke.time.register_source(&tc);