feat(pc): add AP startup

src/platform/acpi/madt.zig

@@ -0,0 +1,25 @@
+const std = @import("std");
+const b = @import("base");
+const acpi = b.pl.acpi;
+
+pub var madt_ptr: ?*acpi.Madt = null;
+
+pub const MadtIterator = struct {
+    madt: *const acpi.Madt,
+    i: usize,
+
+    pub fn next(self: *MadtIterator) ?*const acpi.MadtEntryHeader {
+        const entries_size = self.madt.header.length - @sizeOf(acpi.Madt);
+        if (self.i >= entries_size) return null;
+        const entry: *const acpi.MadtEntryHeader = @ptrFromInt(@intFromPtr(&self.madt.entries) + self.i);
+        self.i += @max(2, entry.length);
+        return entry;
+    }
+};
+
+pub fn iterator() MadtIterator {
+    return .{
+        .madt = madt_ptr orelse @panic("MADT not initialized"),
+        .i = 0,
+    };
+}

src/platform/acpi/root.zig

@@ -5,6 +5,7 @@ const arch = @import("arch");
 const std = @import("std");
 const timer = @import("timer.zig");
 const hpet = @import("hpet.zig");
+pub const madt = @import("madt.zig");
 
 const log = std.log.scoped(.acpi);
 
@@ -67,11 +68,25 @@ pub const Xsdt = extern struct {
     entries: [0]u64,
 };
 
+pub const MadtEntryHeader = extern struct {
+    const Type = enum(u8) {
+        LocalApic = 0,
+        IoApic = 1,
+        Iso = 2,
+        NmiSource = 3,
+        LocalApicNmi = 4,
+        X2LocalApic = 9,
+        X2LocalApicNmi = 10,
+    };
+    type: Type align(1),
+    length: u8 align(1),
+};
+
 pub const Madt = extern struct {
     header: SdtHeader,
     lapic_address: u32 align(1),
     flags: u32 align(1),
-    entries: [0]u8 align(1),
+    entries: [0]MadtEntryHeader align(1),
 };
 
 /// Generic address structure.
@@ -334,4 +349,10 @@ pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void {
         const hpet_struct: *HpetTable = @ptrCast(h);
         hpet.init(hpet_struct);
     }
+
+    const madt_t = find_table(madt_signature);
+
+    if (madt_t) |m| {
+        madt.madt_ptr = @ptrCast(m);
+    }
 }

src/platform/pc/ap.s

@@ -0,0 +1,139 @@
+.global AP_TRAMPOLINE_START
+.global AP_TRAMPOLINE_END
+.global AP_TRAMPOLINE_DATA
+.section .rodata
+
+AP_TRAMPOLINE_START:
+.set BASE, 0x8000
+.set DATA, BASE + (AP_TRAMPOLINE_DATA - AP_TRAMPOLINE_START)
+
+.extern cpu_id_to_apic_id
+.extern ap_start_stack
+.extern cpu_offsets
+.extern cpu_self_offset
+
+.code16
+.startup:
+    cli
+    cld
+    xor %ax, %ax
+    mov %ax, %ds
+    mov %ax, %es
+    mov %ax, %fs
+    mov %ax, %gs
+    mov %ax, %ss
+    lgdt (BASE + (gdt - AP_TRAMPOLINE_START))
+    mov %cr0, %eax
+    or $0x1, %eax
+    mov %eax, %cr0
+    ljmp $0x8, $(BASE + (.pmode - AP_TRAMPOLINE_START))
+
+.code32
+.pmode:
+    mov $0x18, %ax
+    mov %ax, %ds
+    mov %ax, %es
+    mov %ax, %fs
+    mov %ax, %gs
+    mov %ax, %ss
+    mov (DATA + 8), %eax   /* page table */
+    mov %eax, %cr3
+    mov %cr4, %eax
+    or $(1 << 5), %eax
+    mov %eax, %cr4
+    mov $0xC0000080, %ecx
+    rdmsr
+    /* NXE and LME */
+    or $(1 << 8) | (1 << 11), %eax
+    wrmsr
+    mov %cr0, %eax
+    or $(1 << 31) | (1 << 16) | (1 << 5) | 1, %eax
+    mov %eax, %cr0
+    ljmp $0x10, $(BASE + (.lmode - AP_TRAMPOLINE_START))
+.code64
+.lmode:
+    mov $0x18, %ax
+    mov %ax, %ds
+    mov %ax, %es
+    mov %ax, %ss
+    xor %ax, %ax
+    mov %ax, %fs
+    mov %ax, %gs
+    mov (DATA + 16), %rax   /* idtr */
+    lidt (%rax)
+
+    /* Get xAPIC base address from data */
+    mov (DATA + 24), %rax
+    test %rax, %rax
+    jz 1f                   /* apic_base = 0 means x2apic */
+
+    /* xAPIC mode */
+    /* Set xAPIC base to the BSP xAPIC address */
+    mov %rax, %rdx
+    shr $32, %rdx           /* high 32 bits of address */
+    /* %eax already has the lower 32 bits */
+    or $0x800, %eax        /* Global enable */
+    mov $0x1B, %ecx         /* IA32_APIC_BASE MSR */
+    wrmsr
+
+    /* Read APIC ID from memory mapped IO */
+    mov (DATA + 32), %rax
+    mov 0x20(%rax), %edx
+    shr $24, %edx
+    jmp 2f
+
+1:
+    /* x2APIC mode */
+    mov $0x802, %ecx        /* x2APIC ID MSR */
+    rdmsr
+    mov %eax, %edx          /* x2APIC ID is in EAX */
+2:
+    /* edx = our APIC ID */
+    /* Scan cpu_id_to_apic_id[] to find our cpu_id */
+    xor %rcx, %rcx
+    mov $cpu_id_to_apic_id, %rbx
+3:
+    mov (%rbx, %rcx, 4), %eax
+    cmp %eax, %edx
+    je 4f
+    inc %rcx
+    jmp 3b
+4:
+    /* rcx = our cpu_id, save it */
+    mov %rcx, %r15
+    /* Set GS base to offsets[cpu_id] */
+    mov $cpu_offsets, %rdx
+    mov (%rdx), %rdx /* get the actual array (since it's a [*]) */
+    mov (%rdx, %r15, 8), %rax   /* offsets[cpu_id] */
+    mov %rax, %rdx
+    shr $32, %rdx
+    mov $0xC0000101, %ecx       /* MSR_GS_BASE */
+    wrmsr
+
+    mov %rax, %gs:cpu_self_offset
+
+    /* Load stack from start_stack */
+    mov %gs:ap_start_stack, %rsp
+
+    /* Call entry with cpu_id as argument */
+    mov %r15, %rdi
+    mov (DATA), %rax        /* entry */
+    mfence
+    jmp *%rax
+.align 4
+gdt:
+    .word gdt_end - gdt_start - 1
+    .long BASE + (gdt_start - AP_TRAMPOLINE_START)
+gdt_start:
+    .quad 0 /* null segment */
+    .quad 0x00cf9b000000ffff /* 0x08: 32-bit code segment */
+    .quad 0x00af9b000000ffff /* 0x10: 64-bit code segment */
+    .quad 0x00cf93000000ffff /* 0x18: Data segment */
+gdt_end:
+AP_TRAMPOLINE_DATA:
+    .quad 0 /* entry  +0  */
+    .quad 0 /* cr3    +8  */
+    .quad 0 /* idtr   +16 */
+    .quad 0 /* xapic  +24 */
+    .quad 0 /* xapic_virt +32 */
+AP_TRAMPOLINE_END:

src/platform/pc/apic.zig

@@ -0,0 +1,209 @@
+const std = @import("std");
+const amd64 = @import("arch");
+const b = @import("base");
+const acpi = b.pl.acpi;
+const mm = b.mm;
+
+const MadtLapic = extern struct {
+    header: acpi.MadtEntryHeader,
+    id: u8 align(1),
+    version: u8 align(1),
+    flags: u32 align(1),
+};
+
+const MadtIoapic = extern struct {
+    header: acpi.MadtEntryHeader,
+    id: u8 align(1),
+    version: u8 align(1),
+    flags: u32 align(1),
+    address: u32 align(1),
+    global_system_interrupt_base: u32 align(1),
+};
+
+const MadtIso = extern struct {
+    header: acpi.MadtEntryHeader,
+    bus_source: u8 align(1),
+    irq_source: u8 align(1),
+    global_system_interrupt: u32 align(1),
+    flags: u16 align(1),
+};
+
+const MadtX2Lapic = extern struct {
+    header: acpi.MadtEntryHeader,
+    reserved: u16 align(1),
+    id: u32 align(1),
+    flags: u32 align(1),
+    acpi_uid: u32 align(1),
+};
+
+pub const ApicRegisters = enum(u32) {
+    Id = 0x20,
+    Version = 0x30,
+
+    Tpr = 0x80,
+    Apr = 0x90,
+    Ppr = 0xA0,
+    Eoi = 0xB0,
+    RemoteRead = 0xC0,
+    Ldr = 0xD0,
+    Dfr = 0xE0,
+    Svr = 0xF0,
+
+    Isr = 0x100,
+    Tmr = 0x180,
+    Irr = 0x200,
+    Esr = 0x280,
+
+    LvtCmci = 0x2F0,
+    Icr = 0x300,
+    Icr1 = 0x310,
+    LvtTimer = 0x320,
+    LvtThermal = 0x330,
+    LvtPerf = 0x340,
+    LvtLint0 = 0x350,
+    LvtLint1 = 0x360,
+    LvtError = 0x370,
+
+    TimerInitialCount = 0x380,
+    TimerCurrentCount = 0x390,
+    TimerDivideConfig = 0x3E0,
+};
+
+pub var apics: std.ArrayList(u32) = .empty;
+var xapic_address: ?usize = null;
+pub var xapic_base_physical: usize = 0;
+pub var in_x2apic_mode: bool = false;
+
+const log = std.log.scoped(.apic);
+
+fn xapic_write(register: ApicRegisters, value: u32) void {
+    const lapic_base = xapic_address orelse return;
+    b.mmio_write(u32, lapic_base + @intFromEnum(register), value);
+}
+
+fn xapic_read(register: ApicRegisters) u32 {
+    const lapic_base = xapic_address orelse return 0;
+    return b.mmio_read(u32, lapic_base + @intFromEnum(register));
+}
+
+fn x2apic_write(register: ApicRegisters, value: u64) void {
+    amd64.wrmsr(@intFromEnum(amd64.Msr.X2ApicBase) + (@as(u32, @intFromEnum(register)) >> 4), value);
+}
+
+fn x2apic_read(register: ApicRegisters) u64 {
+    return amd64.rdmsr(@intFromEnum(amd64.Msr.X2ApicBase) + (@as(u32, @intFromEnum(register)) >> 4));
+}
+
+pub fn write(register: ApicRegisters, value: u32) void {
+    if (in_x2apic_mode) {
+        x2apic_write(register, value);
+    } else {
+        xapic_write(register, value);
+    }
+}
+
+pub fn read(register: ApicRegisters) u64 {
+    if (in_x2apic_mode) {
+        return x2apic_read(register);
+    } else {
+        return @intCast(xapic_read(register));
+    }
+}
+
+pub fn get_id() u32 {
+    if (in_x2apic_mode) {
+        return @truncate(x2apic_read(.Id));
+    } else {
+        return xapic_read(.Id) >> 24;
+    }
+}
+
+pub fn send_init(apic_id: u32) void {
+    if (in_x2apic_mode) {
+        x2apic_write(.Icr, (@as(u64, apic_id) << 32) | 0x0000C500);
+    } else {
+        xapic_write(.Icr1, @as(u32, apic_id) << 24);
+        xapic_write(.Icr, 0x0000C500);
+        while (xapic_read(.Icr) & (1 << 12) != 0) {}
+    }
+}
+
+pub fn send_sipi(apic_id: u32, startup_page: u8) void {
+    if (in_x2apic_mode) {
+        x2apic_write(.Icr, (@as(u64, apic_id) << 32) | 0x00004600 | startup_page);
+    } else {
+        xapic_write(.Icr1, @as(u32, apic_id) << 24);
+        xapic_write(.Icr, @as(u32, 0x00004600) | startup_page);
+        while (xapic_read(.Icr) & (1 << 12) != 0) {}
+    }
+}
+
+pub fn send_ipi(apic_id: u32, vector: u8, delivery_mode: u8) void {
+    const icr_value = (@as(u64, apic_id) << 32) | (@as(u64, delivery_mode) << 8) | vector;
+    if (in_x2apic_mode) {
+        x2apic_write(.Icr, icr_value);
+    } else {
+        // Write to ICR1 first, because writing to the low word causes
+        // the IPI to be sent.
+        xapic_write(.Icr1, @as(u32, apic_id) << 24);
+        xapic_write(.Icr, @intCast(icr_value));
+
+        // Wait for the delivery status bit to clear.
+        while (xapic_read(.Icr) & (1 << 12) != 0) {}
+    }
+}
+
+pub fn init() linksection(b.init) void {
+    var iter = acpi.madt.iterator();
+
+    const cur_cpu_id = get_id();
+
+    // First, we iterate to check if we find *any* APICs. If we do, then we must use them over x2APIC entries for IDs under 255.
+    var has_lapics = false;
+    while (iter.next()) |entry| {
+        switch (entry.type) {
+            .LocalApic => {
+                const lapic: *const MadtLapic = @ptrCast(entry);
+                if (lapic.flags & 1 == 0 or lapic.id == cur_cpu_id) continue;
+                has_lapics = true;
+                break;
+            },
+            else => {},
+        }
+    }
+
+    // Now, actually iterate and add the APIC IDs to the list.
+    iter = acpi.madt.iterator();
+
+    while (iter.next()) |entry| {
+        switch (entry.type) {
+            .LocalApic => {
+                const lapic: *const MadtLapic = @ptrCast(entry);
+                if (lapic.flags & 1 == 0 or lapic.id == cur_cpu_id) continue;
+                apics.append(mm.zone.gpa, lapic.id) catch @panic("Could not append APIC ID to list");
+            },
+            .X2LocalApic => {
+                const x2lapic: *const MadtX2Lapic = @ptrCast(entry);
+                if (has_lapics and x2lapic.id <= 0xFF) continue;
+                if (x2lapic.flags & 1 == 0 or x2lapic.id == cur_cpu_id) continue;
+                apics.append(mm.zone.gpa, x2lapic.id) catch @panic("Could not append APIC ID to list");
+            },
+            else => {},
+        }
+    }
+
+    // If in xapic mode, set the xapic base address.
+    const base = amd64.read_msr(.LapicBase);
+
+    if (base & (1 << 10) == 0) {
+        xapic_address = mm.p2v(base & 0xFFFFF000);
+        xapic_base_physical = base & 0xFFFFF000;
+    }
+}
+
+pub fn enter_x2apic() void {
+    if (in_x2apic_mode) return;
+
+    const apic_base = amd64.read_msr(.LapicBase);
+    amd64.wrmsr(.LapicBase, apic_base | (1 << 10) | (1 << 11));
+}

src/platform/pc/impl.zig

@@ -4,6 +4,8 @@ const amd64 = b.arch;
 const std = @import("std");
 const tsc = @import("tsc.zig");
 const pvclock = @import("pvclock.zig");
+const apic = @import("apic.zig");
+const smp = @import("smp.zig");
 
 const ki = b.ke.private;
 
@@ -33,13 +35,12 @@ fn com1_write(c: u8) void {
 pub fn early_init(boot_info: *pl.BootInfo) linksection(b.init) void {
     com1_init();
 
-    amd64.detect_cpu_features();
-
     const f = amd64.cpu_features;
 
     log.info("CPU: {s}", .{&f.brand_string});
-    log.info("vendor={s} features={}{}{}{}{}{}{}{}{}{}{}{}", .{
-        &f.vendor_string,
+    log.info("vendor={s} family={x} features={}{}{}{}{}{}{}{}{}{}{}{}", .{
+        @tagName(f.vendor),
+        f.family,
         @intFromBool(f.x2apic),
         @intFromBool(f.five_level_paging),
         @intFromBool(f.nx),
@@ -54,26 +55,36 @@ pub fn early_init(boot_info: *pl.BootInfo) linksection(b.init) void {
         @intFromBool(f.tsc_deadline),
     });
 
-    if (amd64.hypervisor_info) |h| {
+    if (amd64.hypervisor.info) |h| {
         log.info("running on hypervisor {s}", .{@tagName(h.vendor)});
     }
     _ = boot_info;
 }
 
+fn init_hypervisor(hv: amd64.hypervisor.Info) void {
+    switch (hv.data) {
+        .KVM => |kvm_info| {
+            if (kvm_info.features.clocksource2 == true) {
+                pvclock.init();
+            }
+        },
+        else => {},
+    }
+}
+
 pub fn late_init(boot_info: *pl.BootInfo) linksection(b.init) void {
     pl.acpi.init(boot_info);
 
-    if (amd64.hypervisor_info) |h| {
-        if (h.vendor == .KVM) {
-            const features = amd64.cpuid(0x40000001, 0);
-
-            if (features.eax & (1 << 3) != 0) {
-                pvclock.init();
-            }
-        }
+    if (amd64.hypervisor.info) |hv| {
+        init_hypervisor(hv);
     }
 
     tsc.init();
+    apic.init();
+
+    if (apic.apics.items.len != 0) {
+        smp.init();
+    }
 }
 
 pub fn debug_write(c: u8) void {

src/platform/pc/smp.zig

@@ -0,0 +1,115 @@
+const apic = @import("apic.zig");
+const b = @import("base");
+const std = @import("std");
+const rtl = @import("rtl");
+const config = @import("config");
+const mm = b.mm;
+const ke = b.ke;
+const ki = ke.private;
+const amd64 = @import("arch");
+
+const log = std.log.scoped(.smp);
+
+extern var AP_TRAMPOLINE_START: u8;
+extern var AP_TRAMPOLINE_END: u8;
+extern var AP_TRAMPOLINE_DATA: u8;
+
+extern var __percpu_start: u8;
+extern var __percpu_end: u8;
+
+export var cpu_id_to_apic_id: [config.CONFIG_NCPUS]u32 = undefined;
+
+const start_stack = ke.ExportedCpuLocal(usize, 0, "ap_start_stack");
+
+var aps_booted: std.atomic.Value(usize) = .init(0);
+
+const ApData = extern struct {
+    entry: usize align(1),
+    cr3: usize align(1),
+    idtr: usize align(1),
+    xapic_base: usize align(1),
+    xapic_virt_base: usize align(1),
+};
+
+fn ap_entry(cpu_id: u32) callconv(.c) noreturn {
+    _ = aps_booted.fetchAdd(1, .monotonic);
+    ki.impl.init.ap_entry(cpu_id);
+}
+
+pub fn init() linksection(b.init) void {
+    const trampoline_start = @intFromPtr(&AP_TRAMPOLINE_START);
+    const trampoline_size = @intFromPtr(&AP_TRAMPOLINE_END) - trampoline_start;
+
+    // Map the trampoline page.
+    mm.private.kernel_pmap.map_contiguous_range(0x8000, 0x8000, 0x1000, .{
+        .read = true,
+        .write = true,
+        .execute = true,
+    });
+
+    // From Linux, on modern CPUs we can skip the long delay after INIT.
+    const skip_delay = switch (amd64.cpu_features.vendor) {
+        .Intel => amd64.cpu_features.family >= 0x06,
+        .Amd => amd64.cpu_features.family >= 0x0f,
+        .Hygon => amd64.cpu_features.family >= 0x18,
+        .Unknown => false,
+    };
+
+    const init_delay: usize = if (skip_delay) 0 else std.time.ns_per_ms * 10;
+    const sipi_delay: usize = if (skip_delay) std.time.ns_per_us * 10 else std.time.ns_per_us * 300;
+
+    const page: [*]u8 = @ptrFromInt(mm.p2v(0x8000));
+    @memcpy(page[0..trampoline_size], @as([*]u8, @ptrFromInt(trampoline_start)));
+
+    const data_offset = @intFromPtr(&AP_TRAMPOLINE_DATA) - trampoline_start;
+    const data_phys: *ApData = @ptrFromInt(mm.p2v(0x8000 + data_offset));
+
+    const idtr = amd64.sidtr();
+
+    // Allocate per-cpu offsets for CPU-local data.
+    ki.impl.cpu_offsets = @ptrCast(mm.zone.gpa.alloc(usize, apic.apics.items.len + 1) catch @panic("Failed to allocate AP local data offsets"));
+    const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start);
+
+    ki.impl.cpu_offsets[0] = @intFromPtr(&__percpu_start);
+
+    cpu_id_to_apic_id[0] = apic.get_id();
+
+    // Set up the AP data block.
+    data_phys.entry = @intFromPtr(&ap_entry);
+    data_phys.cr3 = amd64.read_cr(3);
+    data_phys.idtr = @intFromPtr(&idtr);
+    data_phys.xapic_base = apic.xapic_base_physical;
+    data_phys.xapic_virt_base = mm.p2v(apic.xapic_base_physical);
+
+    for (0..apic.apics.items.len, apic.apics.items) |i, apic_id| {
+        const cpu_id = i + 1;
+        cpu_id_to_apic_id[cpu_id] = apic_id;
+
+        // Allocate per-cpu data.
+        const cpu_data = mm.zone.gpa.alloc(u8, percpu_size) catch @panic("Failed to allocate per-cpu data");
+        @memcpy(cpu_data, @as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size]);
+
+        ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -% @intFromPtr(&__percpu_start);
+
+        const stack_top = @intFromPtr(mm.heap.alloc(b.kib(16)) catch @panic("Failed to allocate AP stack")) + b.kib(16);
+
+        start_stack.remote(@intCast(cpu_id)).* = stack_top & ~@as(usize, 15);
+
+        rtl.barrier.wmb();
+
+        // Send the INIT-SIPI-SIPI sequence to start the AP.
+        apic.send_init(apic_id);
+
+        ke.time.sleep(init_delay);
+        apic.send_sipi(apic_id, 0x08);
+        ke.time.sleep(sipi_delay);
+        apic.send_sipi(apic_id, 0x08);
+    }
+
+    while (aps_booted.load(.acquire) < apic.apics.items.len) {
+        std.atomic.spinLoopHint();
+    }
+
+    log.info("Booted all APs, total {} CPUs", .{apic.apics.items.len + 1});
+    ke.ncpus = apic.apics.items.len + 1;
+}