From a09c78474dffde8f8b104aa76cc087c05d8817a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Gr=C3=A9goire?= Date: Wed, 15 Apr 2026 19:38:00 -0400 Subject: [PATCH] feat(pc): add AP startup --- build.zig | 1 + src/kern/amd64/impl.zig | 15 +-- src/kern/amd64/init.zig | 72 ++++++++++++- src/kern/cpu.zig | 25 +++-- src/kern/init.zig | 2 +- src/kern/log.zig | 7 +- src/kern/root.zig | 1 + src/platform/acpi/madt.zig | 25 +++++ src/platform/acpi/root.zig | 23 +++- src/platform/pc/ap.s | 139 ++++++++++++++++++++++++ src/platform/pc/apic.zig | 209 +++++++++++++++++++++++++++++++++++++ src/platform/pc/impl.zig | 37 ++++--- src/platform/pc/smp.zig | 115 ++++++++++++++++++++ 13 files changed, 636 insertions(+), 35 deletions(-) create mode 100644 src/platform/acpi/madt.zig create mode 100644 src/platform/pc/ap.s create mode 100644 src/platform/pc/apic.zig create mode 100644 src/platform/pc/smp.zig diff --git a/build.zig b/build.zig index 14e8222..0297fc3 100644 --- a/build.zig +++ b/build.zig @@ -239,6 +239,7 @@ fn addKernel(b: *std.Build, plat: config.Platform, optimize: std.builtin.Optimiz switch (plat.arch) { .x86_64 => { kernel.root_module.addAssemblyFile(b.path("src/kern/amd64/locore.s")); + kernel.root_module.addAssemblyFile(b.path("src/platform/pc/ap.s")); }, else => {}, } diff --git a/src/kern/amd64/impl.zig b/src/kern/amd64/impl.zig index 835ce5d..b776d39 100644 --- a/src/kern/amd64/impl.zig +++ b/src/kern/amd64/impl.zig @@ -5,14 +5,11 @@ const amd64 = b.arch; const std = @import("std"); -pub const early_init = @import("init.zig").early_init; +pub const init = @import("init.zig"); +pub const early_init = init.early_init; -pub const Cpu = struct { - self_ptr: *Cpu, - percpu_offset: usize, -}; - -pub var cpu_self_offset: usize linksection(b.percpu) = 0; +pub export var cpu_self_offset: usize linksection(b.percpu) = 0; +pub export var cpu_offsets: [*]usize = undefined; pub const ThreadContext = struct { rdi: u64, @@ -104,9 +101,7 @@ pub const ThreadContext = struct { }; inline fn percpu_ptr_for(variable: anytype, cpu: u32) @TypeOf(variable) { - _ = cpu; // TODO - @panic("TODO percpu_ptr_for"); - // return @ptrFromInt(@intFromPtr(variable) +% cpu.impl.percpu_offset); + return @ptrFromInt(@intFromPtr(variable) +% cpu_offsets[cpu]); } pub inline fn percpu_ptr_other(variable: anytype, id: u32) @TypeOf(variable) { diff --git a/src/kern/amd64/init.zig b/src/kern/amd64/init.zig index 4f3837e..f56919a 100644 --- a/src/kern/amd64/init.zig +++ b/src/kern/amd64/init.zig @@ -1,6 +1,10 @@ const amd64 = @import("arch"); const int = @import("int.zig"); const b = @import("base"); +const std = @import("std"); +const impl = @import("impl.zig"); +const ke = b.ke; +const ki = b.ke.private; var gdt = extern struct { entries: [9]u64 align(1), @@ -23,7 +27,7 @@ var gdt = extern struct { extern fn gdt_load(gdtr: *const amd64.Gdtr) callconv(.{ .x86_64_sysv = .{} }) void; -pub fn early_init() linksection(b.init) void { +fn early_cpu_init() linksection(b.init) void { const gdtr: amd64.Gdtr = .{ .limit = @sizeOf(@TypeOf(gdt)) - 1, .base = @intFromPtr(&gdt), @@ -31,6 +35,72 @@ pub fn early_init() linksection(b.init) void { gdt_load(&gdtr); + var cr0: amd64.Cr0 = @bitCast(amd64.read_cr(0)); + + // Disable x87 emulation. + cr0.em = false; + // Monitor co-processor. + cr0.mp = true; + // Numeric error. + cr0.ne = true; + // Write-protection (cannot write to read-only pages). + cr0.wp = true; + + amd64.write_cr(0, @bitCast(cr0)); + + // Enable CPU features we might want through cr4. + const f = amd64.cpu_features; + var cr4: amd64.Cr4 = @bitCast(amd64.read_cr(4)); + + // Allow userspace to read TSC. + cr4.tsd = false; + // Enable global pages. + cr4.pge = f.pge; + + if (f.fxsave == true) { + // Enable FXSAVE/FXRSTOR and SSE. + cr4.osfxsr = true; + cr4.osxmmexcpt = true; + } + + // Enable XSAVE. + cr4.osxsave = f.xsave; + + // Enable UMIP. + cr4.umip = f.umip; + + amd64.write_cr(4, @bitCast(cr4)); + + // Configure the EFER MSR. + var efer: amd64.Efer = @bitCast(amd64.read_msr(.Efer)); + + // Enable syscall if supported. + efer.sce = f.syscall; + // Enable NX. + efer.nxe = true; + + amd64.write_msr(.Efer, @bitCast(efer)); +} + +pub fn ap_entry(cpu_id: u32) noreturn { + early_cpu_init(); + ki.cpu.init_cpu(cpu_id); + + while (true) { + std.atomic.spinLoopHint(); + } +} + +var initial_offsets: [1]usize = undefined; +extern var __percpu_start: u8; + +pub fn early_init() linksection(b.init) void { + amd64.detect_cpu_features(); + early_cpu_init(); + + initial_offsets[0] = @intFromPtr(&__percpu_start); + impl.cpu_offsets = &initial_offsets; + int.init(); // Enable Per-CPU data for this CPU diff --git a/src/kern/cpu.zig b/src/kern/cpu.zig index 5a926d5..cfec78e 100644 --- a/src/kern/cpu.zig +++ b/src/kern/cpu.zig @@ -9,20 +9,15 @@ extern var __init_array_percpu_start: u8; extern var __init_array_percpu_end: u8; const id = CpuLocal(u32, 0); -var curr_id: std.atomic.Value(u32) = .init(0); /// Initialize a CPU. Must be called on all CPUs. -pub fn init_cpu() void { +pub fn init_cpu(cpu_id: u32) void { const start = @intFromPtr(&__init_array_percpu_start); const end = @intFromPtr(&__init_array_percpu_end); const count = (end - start) / @sizeOf(*const fn () void); const funcs: [*]const *const fn () callconv(.c) void = @ptrFromInt(start); - if (curr_id.load(.monotonic) >= config.CONFIG_NCPUS) { - std.debug.panic("cpu: Too many CPUs booted! Maximum supported is {}", .{config.CONFIG_NCPUS}); - } - - id.local().* = curr_id.fetchAdd(1, .monotonic); + id.local().* = cpu_id; for (0..count) |i| { funcs[i](); @@ -55,6 +50,22 @@ pub fn CpuLocal(comptime T: type, comptime init: T) type { }; } +/// Wraps around CPU-local data with a designated symbol name. +pub fn ExportedCpuLocal(comptime T: type, comptime init: T, comptime name: []const u8) type { + const S = struct { + var storage: T linksection(".data.percpu") = init; + + pub fn local() @TypeOf(ki.impl.percpu_ptr(&storage)) { + return ki.impl.percpu_ptr(&storage); + } + pub fn remote(cpu: u32) *T { + return ki.impl.percpu_ptr_other(&storage, cpu); + } + }; + @export(&S.storage, .{ .name = name, .linkage = .strong }); + return S; +} + /// Bitmask of CPUs. pub const CpuMask = struct { const bits_per_word = @bitSizeOf(usize); diff --git a/src/kern/init.zig b/src/kern/init.zig index 7106692..c41dc81 100644 --- a/src/kern/init.zig +++ b/src/kern/init.zig @@ -13,7 +13,7 @@ pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void { std.log.info("hello, world", .{}); ki.impl.early_init(); pl.early_init(boot_info); - ki.cpu.init_cpu(); + ki.cpu.init_cpu(0); thread0.init(@intFromPtr(&stack), 8192, ki.sched.idle, null); thread0.priority = 0; diff --git a/src/kern/log.zig b/src/kern/log.zig index be32eb3..a204bde 100644 --- a/src/kern/log.zig +++ b/src/kern/log.zig @@ -6,6 +6,8 @@ const ke = b.ke; const ki = ke.private; const config = @import("config"); +var out_lock: ke.SpinLock = .init(); + const DebugWriter = struct { interface: std.Io.Writer, @@ -45,7 +47,6 @@ pub fn log( ) void { _ = level; - const scope_str = if (scope == .default) "" else @tagName(scope) ++ ": "; // Calculate the length required. @@ -57,7 +58,6 @@ pub fn log( res.info.timestamp = ke.time.read_time_nano(); res.info.length = @truncate(required_len); - // Format the log message into the reserved buffer. res.buf = std.fmt.bufPrint(res.buf, scope_str ++ fmt ++ "\n", args) catch return; @@ -68,6 +68,9 @@ pub fn log( // which would consume the ringbuffer from multiple consoles in a separate thread. var writer = DebugWriter.init(); + const ipl = out_lock.acquire(); + defer out_lock.release(ipl); + writer.interface.print("[{:>5}.{:06}] ", .{ res.info.timestamp / std.time.ns_per_s, res.info.timestamp / std.time.ns_per_us }) catch return; _ = writer.interface.writeAll(res.buf[0..res.info.length]) catch return; } diff --git a/src/kern/root.zig b/src/kern/root.zig index d055310..a9fe9fa 100644 --- a/src/kern/root.zig +++ b/src/kern/root.zig @@ -14,6 +14,7 @@ pub const TimeCounter = p.time.TimeCounter; pub const Timer = p.timer.Timer; pub const Console = p.log.Console; pub const CpuLocal = p.cpu.CpuLocal; +pub const ExportedCpuLocal = p.cpu.ExportedCpuLocal; pub const CpuMask = p.cpu.CpuMask; // === Exported Interfaces === diff --git a/src/platform/acpi/madt.zig b/src/platform/acpi/madt.zig new file mode 100644 index 0000000..987661a --- /dev/null +++ b/src/platform/acpi/madt.zig @@ -0,0 +1,25 @@ +const std = @import("std"); +const b = @import("base"); +const acpi = b.pl.acpi; + +pub var madt_ptr: ?*acpi.Madt = null; + +pub const MadtIterator = struct { + madt: *const acpi.Madt, + i: usize, + + pub fn next(self: *MadtIterator) ?*const acpi.MadtEntryHeader { + const entries_size = self.madt.header.length - @sizeOf(acpi.Madt); + if (self.i >= entries_size) return null; + const entry: *const acpi.MadtEntryHeader = @ptrFromInt(@intFromPtr(&self.madt.entries) + self.i); + self.i += @max(2, entry.length); + return entry; + } +}; + +pub fn iterator() MadtIterator { + return .{ + .madt = madt_ptr orelse @panic("MADT not initialized"), + .i = 0, + }; +} diff --git a/src/platform/acpi/root.zig b/src/platform/acpi/root.zig index 6b62ca2..ea335b3 100644 --- a/src/platform/acpi/root.zig +++ b/src/platform/acpi/root.zig @@ -5,6 +5,7 @@ const arch = @import("arch"); const std = @import("std"); const timer = @import("timer.zig"); const hpet = @import("hpet.zig"); +pub const madt = @import("madt.zig"); const log = std.log.scoped(.acpi); @@ -67,11 +68,25 @@ pub const Xsdt = extern struct { entries: [0]u64, }; +pub const MadtEntryHeader = extern struct { + const Type = enum(u8) { + LocalApic = 0, + IoApic = 1, + Iso = 2, + NmiSource = 3, + LocalApicNmi = 4, + X2LocalApic = 9, + X2LocalApicNmi = 10, + }; + type: Type align(1), + length: u8 align(1), +}; + pub const Madt = extern struct { header: SdtHeader, lapic_address: u32 align(1), flags: u32 align(1), - entries: [0]u8 align(1), + entries: [0]MadtEntryHeader align(1), }; /// Generic address structure. @@ -334,4 +349,10 @@ pub fn init(boot_info: *pl.BootInfo) linksection(b.init) void { const hpet_struct: *HpetTable = @ptrCast(h); hpet.init(hpet_struct); } + + const madt_t = find_table(madt_signature); + + if (madt_t) |m| { + madt.madt_ptr = @ptrCast(m); + } } diff --git a/src/platform/pc/ap.s b/src/platform/pc/ap.s new file mode 100644 index 0000000..b34a4d8 --- /dev/null +++ b/src/platform/pc/ap.s @@ -0,0 +1,139 @@ +.global AP_TRAMPOLINE_START +.global AP_TRAMPOLINE_END +.global AP_TRAMPOLINE_DATA +.section .rodata + +AP_TRAMPOLINE_START: +.set BASE, 0x8000 +.set DATA, BASE + (AP_TRAMPOLINE_DATA - AP_TRAMPOLINE_START) + +.extern cpu_id_to_apic_id +.extern ap_start_stack +.extern cpu_offsets +.extern cpu_self_offset + +.code16 +.startup: + cli + cld + xor %ax, %ax + mov %ax, %ds + mov %ax, %es + mov %ax, %fs + mov %ax, %gs + mov %ax, %ss + lgdt (BASE + (gdt - AP_TRAMPOLINE_START)) + mov %cr0, %eax + or $0x1, %eax + mov %eax, %cr0 + ljmp $0x8, $(BASE + (.pmode - AP_TRAMPOLINE_START)) + +.code32 +.pmode: + mov $0x18, %ax + mov %ax, %ds + mov %ax, %es + mov %ax, %fs + mov %ax, %gs + mov %ax, %ss + mov (DATA + 8), %eax /* page table */ + mov %eax, %cr3 + mov %cr4, %eax + or $(1 << 5), %eax + mov %eax, %cr4 + mov $0xC0000080, %ecx + rdmsr + /* NXE and LME */ + or $(1 << 8) | (1 << 11), %eax + wrmsr + mov %cr0, %eax + or $(1 << 31) | (1 << 16) | (1 << 5) | 1, %eax + mov %eax, %cr0 + ljmp $0x10, $(BASE + (.lmode - AP_TRAMPOLINE_START)) +.code64 +.lmode: + mov $0x18, %ax + mov %ax, %ds + mov %ax, %es + mov %ax, %ss + xor %ax, %ax + mov %ax, %fs + mov %ax, %gs + mov (DATA + 16), %rax /* idtr */ + lidt (%rax) + + /* Get xAPIC base address from data */ + mov (DATA + 24), %rax + test %rax, %rax + jz 1f /* apic_base = 0 means x2apic */ + + /* xAPIC mode */ + /* Set xAPIC base to the BSP xAPIC address */ + mov %rax, %rdx + shr $32, %rdx /* high 32 bits of address */ + /* %eax already has the lower 32 bits */ + or $0x800, %eax /* Global enable */ + mov $0x1B, %ecx /* IA32_APIC_BASE MSR */ + wrmsr + + /* Read APIC ID from memory mapped IO */ + mov (DATA + 32), %rax + mov 0x20(%rax), %edx + shr $24, %edx + jmp 2f + +1: + /* x2APIC mode */ + mov $0x802, %ecx /* x2APIC ID MSR */ + rdmsr + mov %eax, %edx /* x2APIC ID is in EAX */ +2: + /* edx = our APIC ID */ + /* Scan cpu_id_to_apic_id[] to find our cpu_id */ + xor %rcx, %rcx + mov $cpu_id_to_apic_id, %rbx +3: + mov (%rbx, %rcx, 4), %eax + cmp %eax, %edx + je 4f + inc %rcx + jmp 3b +4: + /* rcx = our cpu_id, save it */ + mov %rcx, %r15 + /* Set GS base to offsets[cpu_id] */ + mov $cpu_offsets, %rdx + mov (%rdx), %rdx /* get the actual array (since it's a [*]) */ + mov (%rdx, %r15, 8), %rax /* offsets[cpu_id] */ + mov %rax, %rdx + shr $32, %rdx + mov $0xC0000101, %ecx /* MSR_GS_BASE */ + wrmsr + + mov %rax, %gs:cpu_self_offset + + /* Load stack from start_stack */ + mov %gs:ap_start_stack, %rsp + + /* Call entry with cpu_id as argument */ + mov %r15, %rdi + mov (DATA), %rax /* entry */ + mfence + jmp *%rax +.align 4 +gdt: + .word gdt_end - gdt_start - 1 + .long BASE + (gdt_start - AP_TRAMPOLINE_START) +gdt_start: + .quad 0 /* null segment */ + .quad 0x00cf9b000000ffff /* 0x08: 32-bit code segment */ + .quad 0x00af9b000000ffff /* 0x10: 64-bit code segment */ + .quad 0x00cf93000000ffff /* 0x18: Data segment */ +gdt_end: +AP_TRAMPOLINE_DATA: + .quad 0 /* entry +0 */ + .quad 0 /* cr3 +8 */ + .quad 0 /* idtr +16 */ + .quad 0 /* xapic +24 */ + .quad 0 /* xapic_virt +32 */ +AP_TRAMPOLINE_END: diff --git a/src/platform/pc/apic.zig b/src/platform/pc/apic.zig new file mode 100644 index 0000000..a930452 --- /dev/null +++ b/src/platform/pc/apic.zig @@ -0,0 +1,209 @@ +const std = @import("std"); +const amd64 = @import("arch"); +const b = @import("base"); +const acpi = b.pl.acpi; +const mm = b.mm; + +const MadtLapic = extern struct { + header: acpi.MadtEntryHeader, + id: u8 align(1), + version: u8 align(1), + flags: u32 align(1), +}; + +const MadtIoapic = extern struct { + header: acpi.MadtEntryHeader, + id: u8 align(1), + version: u8 align(1), + flags: u32 align(1), + address: u32 align(1), + global_system_interrupt_base: u32 align(1), +}; + +const MadtIso = extern struct { + header: acpi.MadtEntryHeader, + bus_source: u8 align(1), + irq_source: u8 align(1), + global_system_interrupt: u32 align(1), + flags: u16 align(1), +}; + +const MadtX2Lapic = extern struct { + header: acpi.MadtEntryHeader, + reserved: u16 align(1), + id: u32 align(1), + flags: u32 align(1), + acpi_uid: u32 align(1), +}; + +pub const ApicRegisters = enum(u32) { + Id = 0x20, + Version = 0x30, + + Tpr = 0x80, + Apr = 0x90, + Ppr = 0xA0, + Eoi = 0xB0, + RemoteRead = 0xC0, + Ldr = 0xD0, + Dfr = 0xE0, + Svr = 0xF0, + + Isr = 0x100, + Tmr = 0x180, + Irr = 0x200, + Esr = 0x280, + + LvtCmci = 0x2F0, + Icr = 0x300, + Icr1 = 0x310, + LvtTimer = 0x320, + LvtThermal = 0x330, + LvtPerf = 0x340, + LvtLint0 = 0x350, + LvtLint1 = 0x360, + LvtError = 0x370, + + TimerInitialCount = 0x380, + TimerCurrentCount = 0x390, + TimerDivideConfig = 0x3E0, +}; + +pub var apics: std.ArrayList(u32) = .empty; +var xapic_address: ?usize = null; +pub var xapic_base_physical: usize = 0; +pub var in_x2apic_mode: bool = false; + +const log = std.log.scoped(.apic); + +fn xapic_write(register: ApicRegisters, value: u32) void { + const lapic_base = xapic_address orelse return; + b.mmio_write(u32, lapic_base + @intFromEnum(register), value); +} + +fn xapic_read(register: ApicRegisters) u32 { + const lapic_base = xapic_address orelse return 0; + return b.mmio_read(u32, lapic_base + @intFromEnum(register)); +} + +fn x2apic_write(register: ApicRegisters, value: u64) void { + amd64.wrmsr(@intFromEnum(amd64.Msr.X2ApicBase) + (@as(u32, @intFromEnum(register)) >> 4), value); +} + +fn x2apic_read(register: ApicRegisters) u64 { + return amd64.rdmsr(@intFromEnum(amd64.Msr.X2ApicBase) + (@as(u32, @intFromEnum(register)) >> 4)); +} + +pub fn write(register: ApicRegisters, value: u32) void { + if (in_x2apic_mode) { + x2apic_write(register, value); + } else { + xapic_write(register, value); + } +} + +pub fn read(register: ApicRegisters) u64 { + if (in_x2apic_mode) { + return x2apic_read(register); + } else { + return @intCast(xapic_read(register)); + } +} + +pub fn get_id() u32 { + if (in_x2apic_mode) { + return @truncate(x2apic_read(.Id)); + } else { + return xapic_read(.Id) >> 24; + } +} + +pub fn send_init(apic_id: u32) void { + if (in_x2apic_mode) { + x2apic_write(.Icr, (@as(u64, apic_id) << 32) | 0x0000C500); + } else { + xapic_write(.Icr1, @as(u32, apic_id) << 24); + xapic_write(.Icr, 0x0000C500); + while (xapic_read(.Icr) & (1 << 12) != 0) {} + } +} + +pub fn send_sipi(apic_id: u32, startup_page: u8) void { + if (in_x2apic_mode) { + x2apic_write(.Icr, (@as(u64, apic_id) << 32) | 0x00004600 | startup_page); + } else { + xapic_write(.Icr1, @as(u32, apic_id) << 24); + xapic_write(.Icr, @as(u32, 0x00004600) | startup_page); + while (xapic_read(.Icr) & (1 << 12) != 0) {} + } +} + +pub fn send_ipi(apic_id: u32, vector: u8, delivery_mode: u8) void { + const icr_value = (@as(u64, apic_id) << 32) | (@as(u64, delivery_mode) << 8) | vector; + if (in_x2apic_mode) { + x2apic_write(.Icr, icr_value); + } else { + // Write to ICR1 first, because writing to the low word causes + // the IPI to be sent. + xapic_write(.Icr1, @as(u32, apic_id) << 24); + xapic_write(.Icr, @intCast(icr_value)); + + // Wait for the delivery status bit to clear. + while (xapic_read(.Icr) & (1 << 12) != 0) {} + } +} + +pub fn init() linksection(b.init) void { + var iter = acpi.madt.iterator(); + + const cur_cpu_id = get_id(); + + // First, we iterate to check if we find *any* APICs. If we do, then we must use them over x2APIC entries for IDs under 255. + var has_lapics = false; + while (iter.next()) |entry| { + switch (entry.type) { + .LocalApic => { + const lapic: *const MadtLapic = @ptrCast(entry); + if (lapic.flags & 1 == 0 or lapic.id == cur_cpu_id) continue; + has_lapics = true; + break; + }, + else => {}, + } + } + + // Now, actually iterate and add the APIC IDs to the list. + iter = acpi.madt.iterator(); + + while (iter.next()) |entry| { + switch (entry.type) { + .LocalApic => { + const lapic: *const MadtLapic = @ptrCast(entry); + if (lapic.flags & 1 == 0 or lapic.id == cur_cpu_id) continue; + apics.append(mm.zone.gpa, lapic.id) catch @panic("Could not append APIC ID to list"); + }, + .X2LocalApic => { + const x2lapic: *const MadtX2Lapic = @ptrCast(entry); + if (has_lapics and x2lapic.id <= 0xFF) continue; + if (x2lapic.flags & 1 == 0 or x2lapic.id == cur_cpu_id) continue; + apics.append(mm.zone.gpa, x2lapic.id) catch @panic("Could not append APIC ID to list"); + }, + else => {}, + } + } + + // If in xapic mode, set the xapic base address. + const base = amd64.read_msr(.LapicBase); + + if (base & (1 << 10) == 0) { + xapic_address = mm.p2v(base & 0xFFFFF000); + xapic_base_physical = base & 0xFFFFF000; + } +} + +pub fn enter_x2apic() void { + if (in_x2apic_mode) return; + + const apic_base = amd64.read_msr(.LapicBase); + amd64.wrmsr(.LapicBase, apic_base | (1 << 10) | (1 << 11)); +} diff --git a/src/platform/pc/impl.zig b/src/platform/pc/impl.zig index c1648ef..c535ab7 100644 --- a/src/platform/pc/impl.zig +++ b/src/platform/pc/impl.zig @@ -4,6 +4,8 @@ const amd64 = b.arch; const std = @import("std"); const tsc = @import("tsc.zig"); const pvclock = @import("pvclock.zig"); +const apic = @import("apic.zig"); +const smp = @import("smp.zig"); const ki = b.ke.private; @@ -33,13 +35,12 @@ fn com1_write(c: u8) void { pub fn early_init(boot_info: *pl.BootInfo) linksection(b.init) void { com1_init(); - amd64.detect_cpu_features(); - const f = amd64.cpu_features; log.info("CPU: {s}", .{&f.brand_string}); - log.info("vendor={s} features={}{}{}{}{}{}{}{}{}{}{}{}", .{ - &f.vendor_string, + log.info("vendor={s} family={x} features={}{}{}{}{}{}{}{}{}{}{}{}", .{ + @tagName(f.vendor), + f.family, @intFromBool(f.x2apic), @intFromBool(f.five_level_paging), @intFromBool(f.nx), @@ -54,26 +55,36 @@ pub fn early_init(boot_info: *pl.BootInfo) linksection(b.init) void { @intFromBool(f.tsc_deadline), }); - if (amd64.hypervisor_info) |h| { + if (amd64.hypervisor.info) |h| { log.info("running on hypervisor {s}", .{@tagName(h.vendor)}); } _ = boot_info; } +fn init_hypervisor(hv: amd64.hypervisor.Info) void { + switch (hv.data) { + .KVM => |kvm_info| { + if (kvm_info.features.clocksource2 == true) { + pvclock.init(); + } + }, + else => {}, + } +} + pub fn late_init(boot_info: *pl.BootInfo) linksection(b.init) void { pl.acpi.init(boot_info); - if (amd64.hypervisor_info) |h| { - if (h.vendor == .KVM) { - const features = amd64.cpuid(0x40000001, 0); - - if (features.eax & (1 << 3) != 0) { - pvclock.init(); - } - } + if (amd64.hypervisor.info) |hv| { + init_hypervisor(hv); } tsc.init(); + apic.init(); + + if (apic.apics.items.len != 0) { + smp.init(); + } } pub fn debug_write(c: u8) void { diff --git a/src/platform/pc/smp.zig b/src/platform/pc/smp.zig new file mode 100644 index 0000000..e6d532c --- /dev/null +++ b/src/platform/pc/smp.zig @@ -0,0 +1,115 @@ +const apic = @import("apic.zig"); +const b = @import("base"); +const std = @import("std"); +const rtl = @import("rtl"); +const config = @import("config"); +const mm = b.mm; +const ke = b.ke; +const ki = ke.private; +const amd64 = @import("arch"); + +const log = std.log.scoped(.smp); + +extern var AP_TRAMPOLINE_START: u8; +extern var AP_TRAMPOLINE_END: u8; +extern var AP_TRAMPOLINE_DATA: u8; + +extern var __percpu_start: u8; +extern var __percpu_end: u8; + +export var cpu_id_to_apic_id: [config.CONFIG_NCPUS]u32 = undefined; + +const start_stack = ke.ExportedCpuLocal(usize, 0, "ap_start_stack"); + +var aps_booted: std.atomic.Value(usize) = .init(0); + +const ApData = extern struct { + entry: usize align(1), + cr3: usize align(1), + idtr: usize align(1), + xapic_base: usize align(1), + xapic_virt_base: usize align(1), +}; + +fn ap_entry(cpu_id: u32) callconv(.c) noreturn { + _ = aps_booted.fetchAdd(1, .monotonic); + ki.impl.init.ap_entry(cpu_id); +} + +pub fn init() linksection(b.init) void { + const trampoline_start = @intFromPtr(&AP_TRAMPOLINE_START); + const trampoline_size = @intFromPtr(&AP_TRAMPOLINE_END) - trampoline_start; + + // Map the trampoline page. + mm.private.kernel_pmap.map_contiguous_range(0x8000, 0x8000, 0x1000, .{ + .read = true, + .write = true, + .execute = true, + }); + + // From Linux, on modern CPUs we can skip the long delay after INIT. + const skip_delay = switch (amd64.cpu_features.vendor) { + .Intel => amd64.cpu_features.family >= 0x06, + .Amd => amd64.cpu_features.family >= 0x0f, + .Hygon => amd64.cpu_features.family >= 0x18, + .Unknown => false, + }; + + const init_delay: usize = if (skip_delay) 0 else std.time.ns_per_ms * 10; + const sipi_delay: usize = if (skip_delay) std.time.ns_per_us * 10 else std.time.ns_per_us * 300; + + const page: [*]u8 = @ptrFromInt(mm.p2v(0x8000)); + @memcpy(page[0..trampoline_size], @as([*]u8, @ptrFromInt(trampoline_start))); + + const data_offset = @intFromPtr(&AP_TRAMPOLINE_DATA) - trampoline_start; + const data_phys: *ApData = @ptrFromInt(mm.p2v(0x8000 + data_offset)); + + const idtr = amd64.sidtr(); + + // Allocate per-cpu offsets for CPU-local data. + ki.impl.cpu_offsets = @ptrCast(mm.zone.gpa.alloc(usize, apic.apics.items.len + 1) catch @panic("Failed to allocate AP local data offsets")); + const percpu_size = @intFromPtr(&__percpu_end) - @intFromPtr(&__percpu_start); + + ki.impl.cpu_offsets[0] = @intFromPtr(&__percpu_start); + + cpu_id_to_apic_id[0] = apic.get_id(); + + // Set up the AP data block. + data_phys.entry = @intFromPtr(&ap_entry); + data_phys.cr3 = amd64.read_cr(3); + data_phys.idtr = @intFromPtr(&idtr); + data_phys.xapic_base = apic.xapic_base_physical; + data_phys.xapic_virt_base = mm.p2v(apic.xapic_base_physical); + + for (0..apic.apics.items.len, apic.apics.items) |i, apic_id| { + const cpu_id = i + 1; + cpu_id_to_apic_id[cpu_id] = apic_id; + + // Allocate per-cpu data. + const cpu_data = mm.zone.gpa.alloc(u8, percpu_size) catch @panic("Failed to allocate per-cpu data"); + @memcpy(cpu_data, @as([*]u8, @ptrCast(&__percpu_start))[0..percpu_size]); + + ki.impl.cpu_offsets[cpu_id] = @intFromPtr(cpu_data.ptr) -% @intFromPtr(&__percpu_start); + + const stack_top = @intFromPtr(mm.heap.alloc(b.kib(16)) catch @panic("Failed to allocate AP stack")) + b.kib(16); + + start_stack.remote(@intCast(cpu_id)).* = stack_top & ~@as(usize, 15); + + rtl.barrier.wmb(); + + // Send the INIT-SIPI-SIPI sequence to start the AP. + apic.send_init(apic_id); + + ke.time.sleep(init_delay); + apic.send_sipi(apic_id, 0x08); + ke.time.sleep(sipi_delay); + apic.send_sipi(apic_id, 0x08); + } + + while (aps_booted.load(.acquire) < apic.apics.items.len) { + std.atomic.spinLoopHint(); + } + + log.info("Booted all APs, total {} CPUs", .{apic.apics.items.len + 1}); + ke.ncpus = apic.apics.items.len + 1; +}