feat: fireworks!

build/run.zig

@@ -44,9 +44,12 @@ pub fn addRun(b: *std.Build, kernel: *std.Build.Step.Compile, plat: config.Platf
 
         if (debug) {
             qemu.addArgs(&.{
-                "-d",         "int",
+                "-d",
-                "-D",         "qemu.log",
+                "int",
-                "-no-reboot", "-no-shutdown",
+                "-D",
+                "qemu.log",
+                "-no-reboot",
+                "-no-shutdown",
             });
         }
 

src/ex/fireworks.zig

@@ -1,13 +1,10 @@
 //! Fireworks test ported from the Boron operating system.
 const std = @import("std");
-const base = @import("root");
+const r = @import("root");
-const ke = base.ke;
+const ke = r.ke;
+const mm = r.mm;
 const sintab = @import("sintab.zig");
 
-const c = @cImport({
-    @cInclude("stdlib.h");
-});
-
 var pixel_buffer: [*]u8 = undefined;
 var fb_width: usize = 0;
 var fb_height: usize = 0;
@@ -130,8 +127,6 @@ fn particle(param: ?*anyopaque) void {
     data.act_y = parent_data.act_y;
     const explosion_range = parent_data.explosion_range;
 
-    c.free(param);
-
     const angle = @rem(rand(), 65536);
     data.vel_x = mul_fp_fp(cos(angle), rand_fp_sign()) * explosion_range;
     data.vel_y = mul_fp_fp(sin(angle), rand_fp_sign()) * explosion_range;
@@ -169,10 +164,10 @@ fn particle(param: ?*anyopaque) void {
 }
 
 fn make_thread(entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) *ke.Thread {
-    var ret: *ke.Thread = @ptrCast(@alignCast(c.malloc(@sizeOf(ke.Thread))));
+    var ret: *ke.Thread = mm.zone.gpa.create(ke.Thread) catch @panic("oom");
-    const stack = std.heap.page_allocator.alloc(u8, 16384) catch @panic("wtf");
+    const stack = mm.heap.alloc(r.kib(16)) catch @panic("wtf");
 
-    ret.init(@intFromPtr(stack.ptr), 16384, entry, arg);
+    ret.init(@intFromPtr(stack), r.kib(16), entry, arg);
 
     ret.priority = ke.Thread.Priority.default;
     ret.priority_class = .Batch;
@@ -235,7 +230,7 @@ fn explodeable(_: ?*anyopaque) void {
     const part_count: usize = @intCast(@rem(rand(), 100) + 100);
 
     for (0..part_count) |_| {
-        const param: *FireworkData = @ptrCast(@alignCast(c.malloc(@sizeOf(FireworkData))));
+        const param: *FireworkData = mm.zone.gpa.create(FireworkData) catch @panic("oom");
         param.* = data;
         spawn_particle(param);
     }
@@ -244,7 +239,7 @@ fn explodeable(_: ?*anyopaque) void {
 }
 
 pub fn start(param: ?*anyopaque) void {
-    const boot_info: *base.pl.BootInfo = @ptrCast(@alignCast(param));
+    const boot_info: *r.pl.BootInfo = @ptrCast(@alignCast(param));
 
     rand_gen ^= rand_tsc_based();
 

src/ex/init.zig

@@ -3,11 +3,10 @@ const r = @import("root");
 const pl = r.pl;
 const ke = r.ke;
 const mm = r.mm;
+const ex = r.ex;
 const amd64 = @import("arch");
 
-fn callback(_: ?*anyopaque) void {
+var thread0: ke.Thread = undefined;
-    std.log.info("Timer callback called", .{});
-}
 
 pub fn init(boot_info: *pl.BootInfo) void {
     mm.init(boot_info);
@@ -16,12 +15,9 @@ pub fn init(boot_info: *pl.BootInfo) void {
     mm.late_init();
     ke.sched.late_init();
 
-    var timer: ke.Timer = undefined;
+    const sp = mm.heap.alloc(r.kib(16)) catch @panic("oom");
-    var dpc: ke.Dpc = .init(callback);
+    thread0.init(@intFromPtr(sp), r.kib(16), ex.fireworks.start, boot_info);
-
+    ke.sched.enqueue(&thread0);
-    timer.init();
-
-    ke.timer.set(&timer, std.time.ns_per_ms * 50, &dpc);
 
     while (true) {
         std.atomic.spinLoopHint();

src/kern/amd64/impl.zig

@@ -11,92 +11,54 @@ pub const early_init = init.early_init;
 pub export var cpu_self_offset: usize linksection(r.percpu) = 0;
 pub export var cpu_offsets: [*]usize = undefined;
 
-pub const ThreadContext = struct {
+const ThreadFrame = extern struct {
-    rdi: u64,
+    rbp: u64 align(1),
-    rsi: u64,
+    rbx: u64 align(1),
-    rbx: u64,
+    r12: u64 align(1),
-    rsp: u64,
+    r13: u64 align(1),
-    rbp: u64,
+    r14: u64 align(1),
-    r12: u64,
+    r15: u64 align(1),
-    r13: u64,
+    rip: u64 align(1),
-    r14: u64,
+};
-    r15: u64,
 
-    fn thread_trampoline(entry: usize, arg: usize) callconv(.c) void {
+extern fn asm_thread_entry() void;
+
+extern fn do_context_switch(old: *ThreadContext, new: *ThreadContext, lock: *u8) callconv(.c) void;
+
+pub const ThreadContext = extern struct {
+    rsp: u64 align(1),
+    frame: ThreadFrame,
+
+    export fn thread_entry(entry: usize, arg: usize) callconv(.c) void {
         const entry_fn: *const fn (?*anyopaque) void = @ptrFromInt(entry);
+        ke.ipl.lower(.Passive);
         entry_fn(@ptrFromInt(arg));
         // TODO: This shouldn't be reached!!!
     }
 
     pub fn init(stack: r.VAddr, stack_size: usize, entry: *const fn (?*anyopaque) void, arg: ?*anyopaque) @This() {
         var ctx: @This() = undefined;
-        ctx.rdi = @intFromPtr(entry);
+        var sp: usize = stack + stack_size;
-        ctx.rsi = @intFromPtr(arg);
 
-        var sp = stack + stack_size;
+        if (sp % 16 != 0) {
-        {
+            sp -= (sp % 16);
-            sp -= @sizeOf(usize);
-            @as(*usize, @ptrFromInt(sp)).* = 0;
-
-            sp -= @sizeOf(usize);
-            @as(*usize, @ptrFromInt(sp)).* = @intFromPtr(&thread_trampoline);
         }
-        ctx.rsp = sp;
+
+        const frame: *ThreadFrame = @ptrFromInt(sp - @sizeOf(ThreadFrame));
+
+        frame.* = std.mem.zeroes(ThreadFrame);
+
+        ctx.rsp = @intFromPtr(frame);
+        frame.rip = @intFromPtr(&asm_thread_entry);
+        frame.r12 = @intFromPtr(entry);
+        frame.r13 = @intFromPtr(arg);
 
         return ctx;
     }
 
-    export fn do_switch_to() callconv(.naked) void {
-        const asm_template = std.fmt.comptimePrint(
-            \\movq %%rbx, {d}(%%rdi)
-            \\movq %%rsp, {d}(%%rdi)
-            \\movq %%rbp, {d}(%%rdi)
-            \\movq %%r12, {d}(%%rdi)
-            \\movq %%r13, {d}(%%rdi)
-            \\movq %%r14, {d}(%%rdi)
-            \\movq %%r15, {d}(%%rdi)
-            \\
-            \\movq {d}(%%rsi), %%rbx
-            \\movq {d}(%%rsi), %%rsp
-            \\movq {d}(%%rsi), %%rbp
-            \\movq {d}(%%rsi), %%r12
-            \\movq {d}(%%rsi), %%r13
-            \\movq {d}(%%rsi), %%r14
-            \\movq {d}(%%rsi), %%r15
-            \\movq {d}(%%rsi), %%rdi
-            \\movq {d}(%%rsi), %%rsi
-            \\
-            \\ret
-        , .{
-            @offsetOf(ThreadContext, "rbx"),
-            @offsetOf(ThreadContext, "rsp"),
-            @offsetOf(ThreadContext, "rbp"),
-            @offsetOf(ThreadContext, "r12"),
-            @offsetOf(ThreadContext, "r13"),
-            @offsetOf(ThreadContext, "r14"),
-            @offsetOf(ThreadContext, "r15"),
-
-            @offsetOf(ThreadContext, "rbx"),
-            @offsetOf(ThreadContext, "rsp"),
-            @offsetOf(ThreadContext, "rbp"),
-            @offsetOf(ThreadContext, "r12"),
-            @offsetOf(ThreadContext, "r13"),
-            @offsetOf(ThreadContext, "r14"),
-            @offsetOf(ThreadContext, "r15"),
-            @offsetOf(ThreadContext, "rdi"),
-            @offsetOf(ThreadContext, "rsi"),
-        });
-
-        asm volatile (asm_template);
-    }
-
     pub fn switch_to(self: *ThreadContext, new: *ThreadContext) callconv(.c) void {
-        asm volatile (
+        const thread: *ke.Thread = @alignCast(@fieldParentPtr("context", self));
-            \\call do_switch_to
+        do_context_switch(self, new, &thread.lock.locked);
-            :
-            : [_] "{rdi}" (self),
-              [_] "{rsi}" (new),
-        );
     }
 };
 
@@ -118,9 +80,14 @@ pub inline fn percpu_ptr(variable: anytype) @TypeOf(variable) {
 }
 
 pub inline fn set_hardware_ipl(ipl: ke.Ipl) void {
+    const cr8_value: u64 = switch (ipl) {
+        .Passive, .Dispatch => 0,
+        .Device => 13,
+        .High => 15,
+    };
     asm volatile ("mov %[ipl], %%cr8"
         :
-        : [ipl] "r" (@as(u64, @intFromEnum(ipl))),
+        : [ipl] "r" (cr8_value),
     );
 }
 
@@ -144,6 +111,6 @@ pub inline fn restore_interrupts(state: bool) void {
     }
 }
 
-pub inline fn send_resched_ipi(_: u32) void {
+pub inline fn send_resched_ipi(target: u32) void {
-    @panic("TODO send_resched_ipi");
+    pl.impl.send_resched_ipi(target);
 }

src/kern/amd64/init.zig

@@ -86,11 +86,16 @@ fn early_cpu_init() linksection(r.init) void {
 }
 
 pub fn ap_entry(cpu_id: u32, booted: *std.atomic.Value(usize)) noreturn {
+    amd64.write_msr(.GsBase, ki.impl.cpu_offsets[cpu_id]);
+
     early_cpu_init();
     ki.cpu.init_cpu(cpu_id);
     pl.impl.init_ap();
 
+    ki.sched.percpu.local().current_thread = pl.impl.smp.start_thread.local().*;
+
     _ = booted.fetchAdd(1, .monotonic);
+    ke.ipl.lower(.Passive);
 
     while (true) {
         std.atomic.spinLoopHint();

src/kern/amd64/int.zig

@@ -87,7 +87,7 @@ export fn isr_handler_main(frame: *const amd64.IrqFrame) callconv(.{ .x86_64_sys
         pl.impl.apic.eoi();
     }
 
-    ki.impl.set_hardware_ipl(ipl);
+    _ = ke.ipl.set_hardware(ipl);
 
     if (@intFromEnum(ipl) < @intFromEnum(ke.Ipl.get_max_software()) and ki.ipl.is_softint_pending(.Dispatch)) {
         ki.dpc.dispatch(ke.cpu.current());

src/kern/amd64/locore.s

@@ -131,3 +131,42 @@ __interrupt_vectors:
 .rept 256
 .quad isr\()\+
 .endr
+
+
+.global do_context_switch
+do_context_switch:
+    /* Save callee-saved registers. */
+    push %r15
+    push %r14
+    push %r13
+    push %r12
+    push %rbx
+    push %rbp
+
+    /* Save stack pointer. */
+    mov %rsp, (%rdi)
+
+    /* Load new stack pointer. */
+    mov (%rsi), %rsp
+
+    /* Unlock passed lock. */
+    xorl %ecx, %ecx
+    xchgb %cl, (%rdx)
+
+    /* Restore callee-saved registers. */
+    pop %rbp
+    pop %rbx
+    pop %r12
+    pop %r13
+    pop %r14
+    pop %r15
+    ret
+
+
+.extern thread_entry
+.global asm_thread_entry
+asm_thread_entry:
+    mov %r12, %rdi
+    mov %r13, %rsi
+    call thread_entry
+    ud2

src/kern/init.zig

@@ -7,7 +7,7 @@ const ki = ke.private;
 const ex = r.ex;
 
 var thread0: ke.Thread = undefined;
-var stack: [8192]u8 align(16) = undefined;
+var stack: [r.kib(16)]u8 align(16) = undefined;
 
 pub fn init(boot_info: *pl.BootInfo) linksection(r.init) void {
     ki.impl.early_init();
@@ -15,10 +15,12 @@ pub fn init(boot_info: *pl.BootInfo) linksection(r.init) void {
     pl.early_init(boot_info);
     ki.cpu.init_cpu(0);
 
-    thread0.init(@intFromPtr(&stack), 8192, ki.sched.idle, null);
+    thread0.init(@intFromPtr(&stack), r.kib(16), ki.sched.idle, null);
     thread0.priority = 0;
     thread0.priority_class = .Idle;
 
+    ki.sched.percpu.local().current_thread = &thread0;
+
     std.log.info("Zag for {s} ({s}), cmdline is \"{?s}\"", .{ pl.name, arch.name, boot_info.cmdline });
 
     ex.private.init(boot_info);

src/kern/sched.zig

@@ -795,26 +795,26 @@ fn yield(cpu: u32) void {
 
     sched_cpu.queues_lock.acquire_no_ipl();
 
-    const cur = cpu.current_thread;
+    const cur = sched_cpu.current_thread;
-    var next = cpu.next_thread;
+    var next = sched_cpu.next_thread;
 
     if (next != null) {
-        cpu.next_thread = null;
+        sched_cpu.next_thread = null;
     } else {
         // Pick a new thread to run.
-        next = select_thread(null, cpu, false);
+        next = select_thread(null, sched_cpu, false);
     }
 
     if (next == null) {
         // Nothing to run, go idle.
-        next = cpu.idle_thread;
+        next = sched_cpu.idle_thread;
         sched_cpu.steal_work = true;
     }
 
     sched_cpu.queues_lock.release_no_ipl();
 
     // Switch into the thread.
-    do_switch(cpu, cur.?, next.?);
+    do_switch(sched_cpu, cur.?, next.?);
 
     // cur lock dropped
 }

src/kern/timer.zig

@@ -172,13 +172,16 @@ fn handle_expiry(_: ?*anyopaque) void {
             return;
         }
 
+        if (timer.state.cmpxchgStrong(.Pending, .Running, .acquire, .monotonic) != null) {
+            // Timer must've been canceled.
+            cpu.lock.release_no_ipl();
+            continue;
+        }
+
         _ = cpu.timers.pop();
         cpu.lock.release_no_ipl();
 
-        if (timer.state.cmpxchgStrong(.Pending, .Running, .acquire, .monotonic) != null) {
+        const maybe_dpc = timer.dpc;
-            // Timer must've been canceled.
-            continue;
-        }
 
         timer.hdr.lock.acquire_no_ipl();
 
@@ -186,12 +189,12 @@ fn handle_expiry(_: ?*anyopaque) void {
         timer.hdr.signaled = 1;
         timer.cpu = null;
 
-        // Wake whomever was waiting on the timer.
-        ki.wait.satisfy_wait(&timer.hdr);
-
         timer.state.store(.Stopped, .release);
 
-        if (timer.dpc) |dpc| ke.dpc.enqueue(dpc, null);
+        // Wake whomever was waiting on the timer.
+        ki.wait.satisfy_wait(&timer.hdr);
         timer.hdr.lock.release_no_ipl();
+
+        if (maybe_dpc) |dpc| ke.dpc.enqueue(dpc, null);
     }
 }

src/kern/wait.zig

@@ -5,6 +5,7 @@ const rtl = @import("rtl");
 const std = @import("std");
 const r = @import("root");
 const ke = r.ke;
+const ki = ke.private;
 
 /// Header for waitable objects.
 /// This must be added to any structured which is considered waitable.
@@ -86,7 +87,7 @@ pub fn wait_one(object: *DispatchHeader, timeout: ?r.Nanoseconds) !usize {
 pub fn wait_any(objects: []*DispatchHeader, timeout: ?r.Nanoseconds, waitblocks: ?[*]WaitBlock) !usize {
     const ipl = ke.ipl.raise(.Dispatch);
     defer ke.ipl.lower(ipl);
-    const curtd = ke.curcpu().current_thread.?;
+    const curtd = ki.sched.percpu.local().current_thread.?;
     const obj_count = objects.len;
     const has_timeout = timeout != null;
     const blocks = waitblocks orelse blk: {

src/mm/init.zig

@@ -28,6 +28,15 @@ pub const map_kernel = if (!is_um)
             const rodata_size = rodata_end - rodata_start;
             const data_size = data_end - data_start;
 
+            const start_phys = kaddr.physical_base;
+            const start_virt = kaddr.virtual_base;
+            const until_size = text_start - start_virt;
+            mi.kernel_pmap.map_contiguous_range(start_virt, start_phys, until_size, .{
+                .read = true,
+                .write = true,
+                .global = true,
+            });
+
             mi.kernel_pmap.map_contiguous_range(text_start, kaddr.physical_base + (text_start - kaddr.virtual_base), text_size, .{
                 .read = true,
                 .execute = true,

src/mm/phys.zig

@@ -51,6 +51,15 @@ var batch_pool: std.atomic.Value(GlobalHead) = .init(.{
     .tag = 0,
 });
 
+export const phys_percpu_init linksection(r.percpu_init) = &init_cpu;
+
+fn init_cpu() callconv(.c) void {
+    const cpu = percpu.local();
+    cpu.active_batch = null;
+    cpu.active_count = 0;
+    cpu.depot_count = 0;
+}
+
 /// Push a batch of free pages to the global pool.
 fn push_to_pool(batch: *mm.Page) void {
     var old_head = batch_pool.load(.acquire);

src/mm/zone.zig

@@ -766,12 +766,14 @@ pub fn late_init() linksection(r.init) void {
             z.cpus = gpa.alloc(Cpu, ke.ncpus) catch @panic("Failed to allocate per-CPU magazine state");
 
             for (0..ke.ncpus) |i| {
-                z.cpus[i].lock = .init();
+                z.cpus[i] = .{
-                z.cpus[i].alloc = null;
+                    .lock = .init(),
-                z.cpus[i].free = null;
+                    .alloc = null,
-                z.cpus[i].alloc_rounds = 0;
+                    .free = null,
-                z.cpus[i].free_rounds = 0;
+                    .alloc_rounds = 0,
-                z.cpus[i].magazine_size = z.magtype.?.rounds;
+                    .free_rounds = 0,
+                    .magazine_size = z.magtype.?.rounds,
+                };
             }
         }
         zone = z.next;

src/platform/pc/apic.zig

@@ -142,10 +142,11 @@ pub fn send_sipi(apic_id: u32, startup_page: u8) void {
 }
 
 pub fn send_ipi(apic_id: u32, vector: u8, delivery_mode: u8) void {
-    const icr_value = (@as(u64, apic_id) << 32) | (@as(u64, delivery_mode) << 8) | vector;
     if (in_x2apic_mode.local().*) {
+        const icr_value = (@as(u64, apic_id) << 32) | (@as(u64, delivery_mode) << 8) | vector;
         x2apic_write(.Icr, icr_value);
     } else {
+        const icr_value = (@as(u32, delivery_mode) << 8) | vector;
         // Write to ICR1 first, because writing to the low word causes
         // the IPI to be sent.
         xapic_write(.Icr1, @as(u32, apic_id) << 24);

src/platform/pc/impl.zig

@@ -5,7 +5,7 @@ const std = @import("std");
 const tsc = @import("tsc.zig");
 const pvclock = @import("pvclock.zig");
 pub const apic = @import("apic.zig");
-const smp = @import("smp.zig");
+pub const smp = @import("smp.zig");
 
 const ki = r.ke.private;
 
@@ -105,3 +105,7 @@ pub fn debug_write(c: u8) void {
 pub fn arm_timer(deadline: r.Nanoseconds) void {
     apic.arm_timer(deadline);
 }
+
+pub fn send_resched_ipi(target_cpu: u32) void {
+    apic.send_ipi(smp.cpu_id_to_apic_id[target_cpu], 240, 0);
+}

src/platform/pc/smp.zig

@@ -17,10 +17,12 @@ extern var AP_TRAMPOLINE_DATA: u8;
 extern var __percpu_start: u8;
 extern var __percpu_end: u8;
 
-export var cpu_id_to_apic_id: [config.CONFIG_NCPUS]u32 = undefined;
+pub export var cpu_id_to_apic_id: [config.CONFIG_NCPUS]u32 = undefined;
 
 const start_stack = ke.ExportedCpuLocal(usize, 0, "ap_start_stack");
 
+pub const start_thread = ke.CpuLocal(*ke.Thread, undefined);
+
 var aps_booted: std.atomic.Value(usize) = .init(0);
 
 const ApData = extern struct {
@@ -35,6 +37,16 @@ fn ap_entry(cpu_id: u32) callconv(.c) noreturn {
     ki.impl.init.ap_entry(cpu_id, &aps_booted);
 }
 
+fn make_thread(entrypoint: *const fn (?*anyopaque) void, stack: usize) *ke.Thread {
+    const td = mm.zone.gpa.create(ke.Thread) catch @panic("Failed to allocate thread for AP");
+
+    td.init(stack, r.kib(32), entrypoint, null);
+
+    td.priority = 0;
+    td.priority_class = .Idle;
+    return td;
+}
+
 pub fn init() linksection(r.init) void {
     const trampoline_start = @intFromPtr(&AP_TRAMPOLINE_START);
     const trampoline_size = @intFromPtr(&AP_TRAMPOLINE_END) - trampoline_start;
@@ -94,10 +106,12 @@ pub fn init() linksection(r.init) void {
         // copy the offset into the AP's self_offset variable
         @as(*usize, @ptrCast(@alignCast(&cpu_data[self_offset_offset]))).* = ki.impl.cpu_offsets[cpu_id];
 
-        const stack_top = @intFromPtr(mm.heap.alloc(r.kib(16)) catch @panic("Failed to allocate AP stack")) + r.kib(16);
+        const stack_top = @intFromPtr(mm.heap.alloc(r.kib(32)) catch @panic("Failed to allocate AP stack")) + r.kib(32);
 
         start_stack.remote(@intCast(cpu_id)).* = stack_top & ~@as(usize, 15);
 
+        start_thread.remote(@intCast(cpu_id)).* = make_thread(@ptrCast(&ap_entry), stack_top - r.kib(32));
+
         rtl.barrier.wmb();
 
         // Send the INIT-SIPI-SIPI sequence to start the AP.
@@ -109,6 +123,8 @@ pub fn init() linksection(r.init) void {
         apic.send_sipi(apic_id, 0x08);
     }
 
+    log.info("Waiting for APs...", .{});
+
     while (aps_booted.load(.acquire) < apic.apics.items.len) {
         std.atomic.spinLoopHint();
     }

src/platform/um/impl.zig

@@ -38,6 +38,7 @@ pub const GlobalState = struct {
 
 pub const CpuState = struct {
     pthread: c.pthread_t,
+    td: *ke.Thread,
 };
 
 pub var global_state: GlobalState = undefined;
@@ -122,6 +123,8 @@ fn other_cpu_entry(_: ?*anyopaque) callconv(.c) ?*anyopaque {
 
     ki.cpu.init_cpu(@intCast(my_cpu_id));
 
+    ki.sched.percpu.local().current_thread = percpu.local().td;
+
     timer.init_cpu();
 
     while (true) {}
@@ -174,6 +177,10 @@ pub fn late_init(_: *pl.BootInfo) linksection(r.init) void {
 
         @memcpy(block, @as([*]u8, @ptrCast(&__percpu_start))[0..size]);
 
+        const td = allocator.create(ke.Thread) catch @panic("oom");
+        make_thread(idle, td);
+        percpu.remote(@intCast(i)).td = td;
+
         var attr: c.pthread_attr_t = undefined;
 
         _ = c.pthread_attr_init(&attr);