lib: Add epoch-based SMR

Kconfig

@@ -5,5 +5,8 @@ config KERNEL_FLAVOR
   default "vanilla"
 
 source "arch/$(ARCH)/Kconfig"
-source "core/Kconfig"
+source "boot/Kconfig"
+source "mem/Kconfig"
+source "sched/Kconfig"
 source "dev/Kconfig"
+source "tests/Kconfig"

arch/x86_64/Kconfig

@@ -4,8 +4,10 @@ config X86_64
   bool
   default y
   select ACPI
+  select HAVE_EFISTUB
   select HAVE_MMU
   select HAVE_MMU_GENERIC
+  select HAVE_TLB_SHOOTDOWN_IPI
   help
     64-bit x86 CPU support for AMD and Intel CPUs.
 

arch/x86_64/acpi.c

@@ -0,0 +1,36 @@
+#include <stdckdint.h>
+#include <stdint.h>
+#include <uacpi/kernel_api.h>
+#include <uacpi/status.h>
+#include <x86_64/asm.h>
+
+uacpi_status uacpi_kernel_io_map(uacpi_io_addr base, uacpi_size len, uacpi_handle* out_handle) {
+    uintptr_t end;
+    if (ckd_add(&end, base, len) || end > 0x10000)
+        return UACPI_STATUS_INVALID_ARGUMENT;
+    *out_handle = (uacpi_handle)(uintptr_t)base;
+    return UACPI_STATUS_OK;
+}
+
+void uacpi_kernel_io_unmap(uacpi_handle handle) {
+    // Left intentionally blank.
+}
+
+#define IO_READ(width, suffix) \
+    uacpi_status uacpi_kernel_io_read##width(uacpi_handle handle, uacpi_size offset, uacpi_u##width* out_value) { \
+        *out_value = asm_in##suffix((uintptr_t)handle + offset); \
+        return UACPI_STATUS_OK; \
+    }
+
+#define IO_WRITE(width, suffix) \
+    uacpi_status uacpi_kernel_io_write##width(uacpi_handle handle, uacpi_size offset, uacpi_u##width in_value) { \
+        asm_out##suffix((uintptr_t)handle + offset, in_value); \
+        return UACPI_STATUS_OK; \
+    }
+
+IO_READ(8, b)
+IO_READ(16, w)
+IO_READ(32, l)
+IO_WRITE(8, b)
+IO_WRITE(16, w)
+IO_WRITE(32, l)

arch/x86_64/apic.c

@@ -4,37 +4,50 @@
 #include <zinnia/mmio.h>
 #include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
+#include <zinnia/utils.h>
 #include <zinnia/vma.h>
+#include <arch/cpu.h>
+#include <arch/timer.h>
 #include <stdint.h>
 #include <x86_64/apic.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
+#include <x86_64/percpu.h>
 
-static uint64_t lapic_read_reg(struct local_apic* lapic, uint32_t reg) {
+static uint32_t lapic_msr(size_t offset) {
+    return 0x800 + (offset >> 4);
+}
+
+static uint32_t lapic_read_reg(struct local_apic* lapic, mmio_reg32 reg) {
+    if (lapic->xapic_regs)
+        return mmio_reg_read(lapic->xapic_regs, reg);
+    else
+        return asm_rdmsr(lapic_msr(reg.offset));
+}
+
+static void lapic_write_reg(struct local_apic* lapic, mmio_reg32 reg, uint32_t value) {
+    if (lapic->xapic_regs)
+        mmio_reg_write(lapic->xapic_regs, reg, value);
+    else
+        asm_wrmsr(lapic_msr(reg.offset), value);
+}
+
+static uint64_t lapic_read_icr(struct local_apic* lapic) {
     if (lapic->xapic_regs) {
-        if (reg == APIC_REG_ICR) {
-            uint64_t lo, hi;
-            lo = mmio_read32(lapic->xapic_regs + APIC_REG_ICR);
-            hi = mmio_read32(lapic->xapic_regs + APIC_REG_ICR_HI);
-            return (hi << 32) | lo;
-        } else {
-            return mmio_read32(lapic->xapic_regs + reg);
-        }
+        uint64_t lo = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_LO);
+        uint64_t hi = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_HI);
+        return (hi << 32) | lo;
     } else {
-        return asm_rdmsr(0x800 + (reg >> 4));
+        return asm_rdmsr(lapic_msr(APIC_REG_ICR.offset));
     }
 }
 
-static void lapic_write_reg(struct local_apic* lapic, uint32_t reg, uint64_t value) {
+static void lapic_write_icr(struct local_apic* lapic, uint64_t value) {
     if (lapic->xapic_regs) {
-        if (reg == APIC_REG_ICR) {
-            mmio_write32(lapic->xapic_regs + APIC_REG_ICR_HI, value >> 32);
-            mmio_write32(lapic->xapic_regs + APIC_REG_ICR, (uint32_t)value);
-        } else {
-            mmio_write32(lapic->xapic_regs + reg, (uint32_t)value);
-        }
+        mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_HI, (uint32_t)(value >> 32));
+        mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_LO, (uint32_t)value);
     } else {
-        asm_wrmsr(0x800 + (reg >> 4), value);
+        asm_wrmsr(lapic_msr(APIC_REG_ICR.offset), value);
     }
 }
 
@@ -49,15 +62,22 @@ void lapic_init(struct local_apic* lapic) {
         uintptr_t vaddr = vm_alloc(0x1000);
         pmap_map(&kernel_vma.pmap, vaddr, apic_msr & 0xFFFF'F000, PTE_READ | PTE_WRITE, CACHE_MMIO);
         lapic->xapic_regs = (void*)vaddr;
-        assert(lapic->xapic_regs, "Failed to allocate virtual memory!\n");
+        assert(lapic->xapic_regs, "Failed to allocate virtual memory!");
     }
 
     asm_wrmsr(0x1B, apic_msr);
 
+    lapic->id = lapic_read_reg(lapic, APIC_REG_ID);
+    if (lapic->xapic_regs)
+        lapic->id = mmio_field_get(APIC_ID_XAPIC, lapic->id);
+
     // Reset the TPR.
     lapic_write_reg(lapic, APIC_REG_TPR, 0);
     // Enable APIC bit in the SIVR.
-    lapic_write_reg(lapic, APIC_REG_SIVR, (lapic_read_reg(lapic, APIC_REG_SIVR) & ~0xFF) | 0x100 | IDT_SPURIOUS);
+    uint32_t sivr = lapic_read_reg(lapic, APIC_REG_SIVR);
+    sivr |= mmio_field(APIC_SIVR_VECTOR, IDT_SPURIOUS);
+    sivr |= mmio_field(APIC_SIVR_ENABLE, 1);
+    lapic_write_reg(lapic, APIC_REG_SIVR, sivr);
 
     if (lapic->xapic_regs) {
         lapic_write_reg(lapic, APIC_REG_DFR, 0xF000'0000);
@@ -76,14 +96,50 @@ void lapic_init(struct local_apic* lapic) {
     clock_spin_ns(10'000'000);
 
     lapic->ticks_per_10ms = 0xFFFF'FFFF - lapic_read_reg(lapic, APIC_REG_CCR);
-    lapic_write_reg(lapic, APIC_REG_LVT_TR, IDT_IPI_RESCHED | 0x20000);
-    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, lapic->ticks_per_10ms);
+    lapic_write_reg(lapic, APIC_REG_LVT_TR, mmio_field(APIC_LVT_TR_VECTOR, IDT_TIMER));
+    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, 0);
+}
+
+void arch_timer_program(uint64_t deadline_ns) {
+    struct local_apic* lapic = &percpu_ref(arch_data)->lapic;
+    const uint64_t now = clock_get_elapsed_ns();
+    const uint64_t delta = MIN(deadline_ns > now ? deadline_ns - now : 0, 1'000'000'000);
+    const uint64_t ticks = delta * lapic->ticks_per_10ms / 10'000'000;
+    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, MIN(MAX(ticks, 1), 0xFFFF'FFFF));
 }
 
 void lapic_eoi(struct local_apic* lapic) {
     lapic_write_reg(lapic, APIC_REG_EOI, 0);
 }
 
+void lapic_send_ipi(
+    struct local_apic* lapic,
+    uint32_t target,
+    uint8_t vector,
+    enum apic_delivery_mode delivery_mode,
+    enum apic_dest_mode dest_mode,
+    enum apic_level level,
+    enum apic_trigger_mode trigger_mode
+) {
+    uint64_t icr = mmio_field(APIC_ICR_VECTOR, vector);
+    icr |= mmio_field(APIC_ICR_DELIVERY_MODE, delivery_mode);
+    icr |= mmio_field(APIC_ICR_DEST_MODE, dest_mode);
+    icr |= mmio_field(APIC_ICR_LEVEL, level);
+    icr |= mmio_field(APIC_ICR_TRIGGER_MODE, trigger_mode);
+
+    if (lapic->xapic_regs)
+        icr |= mmio_field(APIC_ICR_DEST_XAPIC, target);
+    else
+        icr |= mmio_field(APIC_ICR_DEST_X2APIC, target);
+
+    lapic_write_icr(lapic, icr);
+
+    if (lapic->xapic_regs) {
+        while (mmio_field_get(APIC_ICR_DELIVERY_STATUS, lapic_read_icr(lapic)) == APIC_DELIVERY_STATUS_PENDING)
+            arch_cpu_pause();
+    }
+}
+
 constexpr uint16_t PIC1_COMMAND_PORT = 0x20;
 constexpr uint16_t PIC1_DATA_PORT = 0x21;
 constexpr uint16_t PIC2_COMMAND_PORT = 0xA0;

arch/x86_64/archctl.c

@@ -12,21 +12,21 @@ struct syscall_ret arch_archctl(uintptr_t op, uintptr_t arg) {
     switch (op) {
     case __ARCHCTL_SET_FSBASE:
         if (!arch_mem_addr_is_user(arg)) {
-            ret.errno = EINVAL;
+            ret.error = EINVAL;
             return ret;
         }
         asm_wrmsr(MSR_FS_BASE, arg);
         return ret;
     case __ARCHCTL_SET_GSBASE:
         if (!arch_mem_addr_is_user(arg)) {
-            ret.errno = EINVAL;
+            ret.error = EINVAL;
             return ret;
         }
         // Update the inactive GSBASE MSR, not ours.
         asm_wrmsr(MSR_KERNEL_GS_BASE, arg);
         return ret;
     default:
-        ret.errno = EINVAL;
+        ret.error = EINVAL;
         return ret;
     }
 }

arch/x86_64/gdt.c

@@ -1,13 +1,13 @@
 #include <zinnia/percpu.h>
 #include <stddef.h>
-#include <zinnia/string.h>
+#include <string.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
 #include <x86_64/percpu.h>
 #include <x86_64/tss.h>
 
-static const struct gdt main_gdt = {
+const struct gdt gdt_base = {
     .null = {0},
     .kernel_code32 = {.val = 0x00cf9b000000ffff},
     .kernel_data32 = {.val = 0x00cf93000000ffff},
@@ -26,7 +26,7 @@ static const struct gdt main_gdt = {
 
 void gdt_load() {
     struct gdt* gdt = &percpu_ref(arch_data)->gdt;
-    memcpy(gdt, &main_gdt, sizeof(main_gdt));
+    memcpy(gdt, &gdt_base, sizeof(gdt_base));
 
     struct tss* tss = &percpu_ref(arch_data)->tss;
     tss->iopb = sizeof(struct tss);

arch/x86_64/hpet.c

@@ -1,7 +1,10 @@
+#include <zinnia/acpi.h>
 #include <zinnia/assert.h>
 #include <zinnia/clock.h>
 #include <zinnia/cmdline.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/mmio.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/utils.h>
@@ -23,19 +26,21 @@ struct hpet {
     uint32_t period;
 };
 
-constexpr uint32_t HPET_CAP = 0x0;
-constexpr uint32_t HPET_CFG = 0x10;
-constexpr uint32_t HPET_COUNTER = 0xF0;
+constexpr mmio_reg64 HPET_REG_CAP = {.offset = 0x0};
+constexpr mmio_field64 HPET_CAP_PERIOD = {HPET_REG_CAP, 32, 32};
+constexpr mmio_reg64 HPET_REG_CFG = {.offset = 0x10};
+constexpr mmio_field64 HPET_CFG_ENABLE = {HPET_REG_CFG, 0, 1};
+constexpr mmio_reg64 HPET_REG_COUNTER = {.offset = 0xF0};
 
 static void hpet_reset(struct clock* c) {
     struct hpet* hpet = containerof(c, struct hpet, clock);
-    mmio_write64(hpet->regs + HPET_COUNTER, 0);
+    mmio_reg_write(hpet->regs, HPET_REG_COUNTER, 0);
 }
 
 static uint64_t hpet_get_elapsed_ns(struct clock* c) {
     struct hpet* hpet = containerof(c, struct hpet, clock);
-    uint64_t counter = mmio_read64(hpet->regs + HPET_COUNTER);
-    return (counter * (uint64_t)hpet->period / 1'000'000);
+    uint64_t counter = mmio_reg_read(hpet->regs, HPET_REG_COUNTER);
+    return (uint64_t)((__uint128_t)counter * (__uint128_t)hpet->period / 1'000'000);
 }
 
 static struct hpet hpet_clock = {
@@ -47,15 +52,15 @@ static struct hpet hpet_clock = {
     },
 };
 
-static char table_buf[0x1000];
-
-void hpet_init() {
+static void hpet_init(struct boot_info*) {
     if (!use_hpet)
         return;
 
-    uacpi_setup_early_table_access(table_buf, sizeof(table_buf));
     uacpi_table table = {0};
-    assert(!uacpi_table_find_by_signature("HPET", &table), "HPET table not found!\n");
+    if (uacpi_table_find_by_signature("HPET", &table) != UACPI_STATUS_OK) {
+        log("x86_64/hpet: No HPET table found\n");
+        return;
+    }
 
     struct acpi_hpet* hpet = table.ptr;
 
@@ -66,10 +71,15 @@ void hpet_init() {
     uacpi_table_unref(&table);
     log("x86_64/hpet: HPET registers mapped at %p\n", hpet_clock.regs);
 
-    mmio_write64(hpet_clock.regs + HPET_CFG, mmio_read64(hpet_clock.regs + HPET_CFG) | 1);
+    uint64_t cfg = mmio_reg_read(hpet_clock.regs, HPET_REG_CFG);
+    cfg |= mmio_field(HPET_CFG_ENABLE, 1);
+    mmio_reg_write(hpet_clock.regs, HPET_REG_CFG, cfg);
 
-    uint64_t caps = mmio_read64(hpet_clock.regs + HPET_CAP);
-    hpet_clock.period = (caps >> 32) & 0xFFFF'FFFF;
+    uint64_t cap = mmio_reg_read(hpet_clock.regs, HPET_REG_CAP);
+    hpet_clock.period = mmio_field_get(HPET_CAP_PERIOD, cap);
 
-    assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!\n");
+    assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!");
 }
+INIT_TASK(hpet_stage, "arch.x86_64.hpet", hpet_init);
+INIT_DEPENDS(hpet_stage, acpi_stage);
+INIT_ENTAILS(hpet_stage, percpu_stage);

arch/x86_64/idt.c

@@ -1,10 +1,17 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/assert.h>
 #include <zinnia/compiler.h>
 #include <zinnia/irq.h>
 #include <zinnia/panic.h>
 #include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
 #include <zinnia/print.h>
 #include <zinnia/sched.h>
+#include <zinnia/timer.h>
+#include <zinnia/usercopy.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
 #include <arch/sched.h>
 #include <bits/sched.h>
 #include <stddef.h>
@@ -36,29 +43,82 @@ static_assert(sizeof(struct idtr) == 10);
 void interrupt_handler(struct arch_context* ctx) {
     struct percpu* cpu = percpu_get();
     struct arch_percpu* arch_cpu = percpu_ref(arch_data);
-    bool old = irq_set_interrupted(true);
-
     uint64_t isr = ctx->isr;
 
     switch (isr) {
     case IDT_PF:
         uint64_t cr2;
         asm volatile("mov %0, cr2" : "=r"(cr2));
-        panic("Page fault at %#lx, accessed: %#lx\n", ctx->rip, cr2);
-    case IDT_IPI_RESCHED:
-        arch_sched_preempt_disable();
-        if (arch_sched_preempt_enable()) {
-            lapic_eoi(&arch_cpu->lapic);
-            sched_reschedule(&cpu->sched);
+
+        const struct vma_fault_info info = {
+            .ip = ctx->rip,
+            .addr = cr2,
+            .caused_by_user = (ctx->cs & CPL_USER) == CPL_USER,
+            .caused_by_write = ctx->error & (1 << 1),
+            .caused_by_fetch = ctx->error & (1 << 4),
+            .page_was_present = ctx->error & (1 << 0),
+        };
+
+        struct task* current = cpu->sched.current;
+        const bool irqs_were_on = ctx->rflags & RFLAGS_IF;
+        if (irqs_were_on)
+            irq_set_state(true);
+
+        const bool handled = current && current->space && vma_page_fault(current->space, &info);
+
+        if (irqs_were_on)
+            irq_set_state(false);
+
+        if (handled)
+            break;
+
+        if (current && current->usercopy_region) {
+            struct usercopy_region* region = current->usercopy_region;
+            if (ctx->rip >= (uintptr_t)region->start_ip && ctx->rip < (uintptr_t)region->end_ip) {
+                ctx->rip = (uintptr_t)region->fault_ip;
+                break;
+            }
         }
+
+        panic(
+            "Unrecoverable page fault. Attempted to %s a %s page at %#lx (IP: %#lx)\n",
+            info.caused_by_write       ? "write to"
+                : info.caused_by_fetch ? "execute on"
+                                       : "read from",
+            info.page_was_present ? "present" : "non-present",
+            info.addr,
+            info.ip
+        );
+    case IDT_SPURIOUS:
+        break;
+    case IDT_IPI_RESCHED:
+        lapic_eoi(&arch_cpu->lapic);
+        arch_sched_preempt_disable();
+        if (arch_sched_preempt_enable())
+            sched_reschedule(&cpu->sched);
+        break;
+    case IDT_IPI_SHOOTDOWN:
+        lapic_eoi(&arch_cpu->lapic);
+        pmap_shootdown_handle();
+        break;
+    case IDT_TIMER:
+        lapic_eoi(&arch_cpu->lapic);
+        arch_sched_preempt_disable();
+        timer_interrupt();
+        if (arch_sched_preempt_enable() && (cpu->sched.need_resched || cpu->sched.owe_preempt))
+            sched_reschedule(&cpu->sched);
         break;
     default:
+        if (isr < 0x20)
+            panic("Unhandled exception %lu at %#lx, error: %#lx\n", isr, ctx->rip, ctx->error);
+        assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu", isr);
         struct irq_line* line = arch_cpu->irq_lines[isr - 0x20];
-        assert(line, "Unhandled interrupt on ISR %lu\n", isr);
-        // TODO: irq_raise(line);
+        assert(line, "Unhandled interrupt on ISR %lu", isr);
+        arch_sched_preempt_disable();
+        irq_line_raise(line);
+        if (arch_sched_preempt_enable() && cpu->sched.owe_preempt)
+            sched_reschedule(&cpu->sched);
     }
-
-    irq_set_interrupted(old);
 }
 
 #define CS_OFFSET (sizeof(struct arch_context) - sizeof(uint64_t) - offsetof(struct arch_context, cs))
@@ -198,6 +258,7 @@ static struct idt_entry idt[256] = {};
 
 void idt_init() {
     REPEAT_256(INTERRUPT_SETUP)
+    idt[IDT_BP].attrib |= CPL_USER << 5;
 }
 
 void idt_load() {

arch/x86_64/include/bits/cpu.h

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+static inline void arch_cpu_pause() {
+    asm volatile("pause");
+}
+
+#define ARCH_CACHE_LINE_SIZE 64

arch/x86_64/include/bits/elf.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/elf.h>
@@ -11,19 +13,21 @@
 #define ELF_ST_BIND ELF64_ST_BIND
 #define ELF_ST_TYPE ELF64_ST_TYPE
 
-#define elf_ehdr elf64_ehdr
-#define elf_phdr elf64_phdr
-#define elf_dyn  elf64_dyn
-#define elf_addr elf64_addr
-#define elf_off  elf64_off
-#define elf_nhdr elf64_nhdr
-#define elf_auxv elf64_auxv
-#define elf_sym  elf64_sym
-#define elf_rela elf64_rela
-#define elf_rel  elf64_rel
+typedef elf64_ehdr_t elf_ehdr_t;
+typedef elf64_phdr_t elf_phdr_t;
+typedef elf64_dyn_t elf_dyn_t;
+typedef elf64_addr_t elf_addr_t;
+typedef elf64_off_t elf_off_t;
+typedef elf64_nhdr_t elf_nhdr_t;
+typedef elf64_auxv_t elf_auxv_t;
+typedef elf64_sym_t elf_sym_t;
+typedef elf64_rela_t elf_rela_t;
+typedef elf64_rel_t elf_rel_t;
+typedef elf64_relr_t elf_relr_t;
 
 enum {
     R_NONE = 0,
+    R_64 = 1,
     R_COPY = 5,
     R_GLOB_DAT = 6,
     R_JUMP_SLOT = 7,

arch/x86_64/include/bits/pmap.h

@@ -1,5 +1,24 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>
 
 typedef uint64_t pte_t;
+
+enum : pte_t {
+    ARCH_FLAG_PRESENT = 1 << 0,
+    ARCH_FLAG_READ_WRITE = 1 << 1,
+    ARCH_FLAG_USER_MODE = 1 << 2,
+    ARCH_FLAG_WRITE_THROUGH = 1 << 3,
+    ARCH_FLAG_CACHE_DISABLE = 1 << 4,
+    ARCH_FLAG_ACCESSED = 1 << 5,
+    ARCH_FLAG_DIRTY = 1 << 6,
+    ARCH_FLAG_SIZE = 1 << 7,
+    ARCH_FLAG_GLOBAL = 1 << 8,
+    ARCH_FLAG_AVAILABLE = 1 << 9,
+    ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 7,
+    ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
+
+    ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
+};

arch/x86_64/include/bits/sched.h

@@ -1,10 +1,14 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
+#include <zinnia/types.h>
 #include <stdint.h>
 
 struct arch_task_context {
     uint64_t rsp;
     uint8_t* fpu_region;
+    phys_t fpu_phys;
     uint16_t ds, es, fs, gs;
     uint64_t fs_base, gs_base;
 };
@@ -35,6 +39,7 @@ struct arch_context {
 };
 
 #define ARCH_CTX_NUM  rax
+#define ARCH_CTX_SP   rsp
 #define ARCH_CTX_A0   rdi
 #define ARCH_CTX_A1   rsi
 #define ARCH_CTX_A2   rdx
@@ -43,3 +48,8 @@ struct arch_context {
 #define ARCH_CTX_A5   r9
 #define ARCH_CTX_RET0 rax
 #define ARCH_CTX_RET1 rdx
+
+struct arch_syscall_restart {
+    uint64_t rax;
+    uint64_t rdx;
+};

arch/x86_64/include/bits/types.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>

arch/x86_64/include/x86_64/apic.h

@@ -1,5 +1,11 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
+#include <zinnia/errno.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/irq.h>
+#include <zinnia/mmio.h>
 #include <stdint.h>
 
 struct local_apic {
@@ -37,22 +43,49 @@ enum apic_trigger_mode {
     APIC_TRIGGER_LEVEL = 1,
 };
 
-enum apic_regs {
-    APIC_REG_ID = 0x20,
-    APIC_REG_TPR = 0x80,
-    APIC_REG_EOI = 0xB0,
-    APIC_REG_LDR = 0xD0,
-    APIC_REG_DFR = 0xE0,
-    APIC_REG_SIVR = 0xF0,
-    APIC_REG_ESR = 0x280,
-    APIC_REG_ICR = 0x300,
-    APIC_REG_ICR_HI = 0x310,
-    APIC_REG_LVT_TR = 0x320,
-    APIC_REG_ICR_TIMER = 0x380,
-    APIC_REG_CCR = 0x390,
-    APIC_REG_DCR = 0x3E0,
-};
+constexpr mmio_reg32 APIC_REG_ID = {.offset = 0x20};
+constexpr mmio_field32 APIC_ID_XAPIC = {APIC_REG_ID, 24, 8};
+constexpr mmio_reg32 APIC_REG_TPR = {.offset = 0x80};
+constexpr mmio_reg32 APIC_REG_EOI = {.offset = 0xB0};
+constexpr mmio_reg32 APIC_REG_LDR = {.offset = 0xD0};
+constexpr mmio_reg32 APIC_REG_DFR = {.offset = 0xE0};
+constexpr mmio_reg32 APIC_REG_SIVR = {.offset = 0xF0};
+constexpr mmio_field32 APIC_SIVR_VECTOR = {APIC_REG_SIVR, 0, 8};
+constexpr mmio_field32 APIC_SIVR_ENABLE = {APIC_REG_SIVR, 8, 1};
+constexpr mmio_reg32 APIC_REG_ESR = {.offset = 0x280};
+constexpr mmio_reg64 APIC_REG_ICR = {.offset = 0x300};
+constexpr mmio_field64 APIC_ICR_VECTOR = {APIC_REG_ICR, 0, 8};
+constexpr mmio_field64 APIC_ICR_DELIVERY_MODE = {APIC_REG_ICR, 8, 3};
+constexpr mmio_field64 APIC_ICR_DEST_MODE = {APIC_REG_ICR, 11, 1};
+constexpr mmio_field64 APIC_ICR_DELIVERY_STATUS = {APIC_REG_ICR, 12, 1};
+constexpr mmio_field64 APIC_ICR_LEVEL = {APIC_REG_ICR, 14, 1};
+constexpr mmio_field64 APIC_ICR_TRIGGER_MODE = {APIC_REG_ICR, 15, 1};
+constexpr mmio_field64 APIC_ICR_DEST_XAPIC = {APIC_REG_ICR, 56, 8};
+constexpr mmio_field64 APIC_ICR_DEST_X2APIC = {APIC_REG_ICR, 32, 32};
+constexpr mmio_reg32 APIC_REG_ICR_LO = {.offset = 0x300};
+constexpr mmio_reg32 APIC_REG_ICR_HI = {.offset = 0x310};
+constexpr mmio_reg32 APIC_REG_LVT_TR = {.offset = 0x320};
+constexpr mmio_field32 APIC_LVT_TR_VECTOR = {APIC_REG_LVT_TR, 0, 8};
+constexpr mmio_reg32 APIC_REG_ICR_TIMER = {.offset = 0x380};
+constexpr mmio_reg32 APIC_REG_CCR = {.offset = 0x390};
+constexpr mmio_reg32 APIC_REG_DCR = {.offset = 0x3E0};
 
 void lapic_init(struct local_apic* lapic);
 void lapic_eoi(struct local_apic* lapic);
+
+void lapic_send_ipi(
+    struct local_apic* lapic,
+    uint32_t target,
+    uint8_t vector,
+    enum apic_delivery_mode delivery_mode,
+    enum apic_dest_mode dest_mode,
+    enum apic_level level,
+    enum apic_trigger_mode trigger_mode
+);
+
 void pic_disable();
+
+errno_t apic_alloc_vector(struct irq_line* line, uint8_t* out_vector, uint32_t* out_lapic);
+void apic_free_vector(uint8_t vector);
+
+INIT_EXTERN(ioapic_stage);

arch/x86_64/include/x86_64/asm.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>
@@ -40,6 +42,12 @@ static inline void asm_wrxcr(uint32_t reg, uint64_t val) {
     asm volatile("xsetbv" ::"a"(eax), "d"(edx), "c"(reg) : "memory");
 }
 
+static inline uint64_t asm_rdxcr(uint32_t reg) {
+    uint32_t eax, edx;
+    asm volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(reg));
+    return ((uint64_t)edx << 32) | eax;
+}
+
 // Saves the FPU state to a 512-byte region of memory using FXSAVE.
 // Pointer must be 16-byte aligned.
 static inline void asm_fxsave(void* mem) {
@@ -55,13 +63,13 @@ static inline void asm_fxrstor(const void* mem) {
 // Saves the FPU state to a region of memory using XSAVE.
 // Pointer must be 16-byte aligned.
 static inline void asm_xsave(void* mem) {
-    asm volatile("xsave [%0]" ::"r"(mem) : "memory");
+    asm volatile("xsave [%0]" ::"r"(mem), "a"(0xFFFF'FFFF), "d"(0xFFFF'FFFF) : "memory");
 }
 
 // Restores the FPU state from a region of memory using XRSTOR.
 // Pointer must be 16-byte aligned.
 static inline void asm_xrstor(const void* mem) {
-    asm volatile("xrstor [%0]" ::"r"(mem) : "memory");
+    asm volatile("xrstor [%0]" ::"r"(mem), "a"(0xFFFF'FFFF), "d"(0xFFFF'FFFF) : "memory");
 }
 
 static inline uint8_t asm_inb(uint16_t port) {

arch/x86_64/include/x86_64/defs.h

@@ -1,202 +1,258 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
-#define CPUID_1C_SSE3       (1 << 0)
-#define CPUID_1C_PCLMUL     (1 << 1)
-#define CPUID_1C_DTES64     (1 << 2)
-#define CPUID_1C_MONITOR    (1 << 3)
-#define CPUID_1C_DS_CPL     (1 << 4)
-#define CPUID_1C_VMX        (1 << 5)
-#define CPUID_1C_SMX        (1 << 6)
-#define CPUID_1C_EST        (1 << 7)
-#define CPUID_1C_TM2        (1 << 8)
-#define CPUID_1C_SSSE3      (1 << 9)
-#define CPUID_1C_CID        (1 << 10)
-#define CPUID_1C_SDBG       (1 << 11)
-#define CPUID_1C_FMA        (1 << 12)
-#define CPUID_1C_CX16       (1 << 13)
-#define CPUID_1C_XTPR       (1 << 14)
-#define CPUID_1C_PDCM       (1 << 15)
-#define CPUID_1C_PCID       (1 << 17)
-#define CPUID_1C_DCA        (1 << 18)
-#define CPUID_1C_SSE4_1     (1 << 19)
-#define CPUID_1C_SSE4_2     (1 << 20)
-#define CPUID_1C_X2APIC     (1 << 21)
-#define CPUID_1C_MOVBE      (1 << 22)
-#define CPUID_1C_POPCNT     (1 << 23)
-#define CPUID_1C_TSC        (1 << 24)
-#define CPUID_1C_AES        (1 << 25)
-#define CPUID_1C_XSAVE      (1 << 26)
-#define CPUID_1C_OSXSAVE    (1 << 27)
-#define CPUID_1C_AVX        (1 << 28)
-#define CPUID_1C_F16C       (1 << 29)
-#define CPUID_1C_RDRAND     (1 << 30)
-#define CPUID_1C_HYPERVISOR (1 << 31)
+#include <stdint.h>
 
-#define CPUID_1D_FPU     (1 << 0)
-#define CPUID_1D_VME     (1 << 1)
-#define CPUID_1D_DE      (1 << 2)
-#define CPUID_1D_PSE     (1 << 3)
-#define CPUID_1D_TSC     (1 << 4)
-#define CPUID_1D_MSR     (1 << 5)
-#define CPUID_1D_PAE     (1 << 6)
-#define CPUID_1D_MCE     (1 << 7)
-#define CPUID_1D_CX8     (1 << 8)
-#define CPUID_1D_APIC    (1 << 9)
-#define CPUID_1D_SEP     (1 << 11)
-#define CPUID_1D_MTRR    (1 << 12)
-#define CPUID_1D_PGE     (1 << 13)
-#define CPUID_1D_MCA     (1 << 14)
-#define CPUID_1D_CMOV    (1 << 15)
-#define CPUID_1D_PAT     (1 << 16)
-#define CPUID_1D_PSE36   (1 << 17)
-#define CPUID_1D_PSN     (1 << 18)
-#define CPUID_1D_CLFLUSH (1 << 19)
-#define CPUID_1D_DS      (1 << 21)
-#define CPUID_1D_ACPI    (1 << 22)
-#define CPUID_1D_MMX     (1 << 23)
-#define CPUID_1D_FXSR    (1 << 24)
-#define CPUID_1D_SSE     (1 << 25)
-#define CPUID_1D_SSE2    (1 << 26)
-#define CPUID_1D_SS      (1 << 27)
-#define CPUID_1D_HTT     (1 << 28)
-#define CPUID_1D_TM      (1 << 29)
-#define CPUID_1D_IA64    (1 << 30)
-#define CPUID_1D_PBE     (1 << 31)
+enum : uint32_t {
+    CPUID_1C_SSE3 = 1u << 0,
+    CPUID_1C_PCLMUL = 1u << 1,
+    CPUID_1C_DTES64 = 1u << 2,
+    CPUID_1C_MONITOR = 1u << 3,
+    CPUID_1C_DS_CPL = 1u << 4,
+    CPUID_1C_VMX = 1u << 5,
+    CPUID_1C_SMX = 1u << 6,
+    CPUID_1C_EST = 1u << 7,
+    CPUID_1C_TM2 = 1u << 8,
+    CPUID_1C_SSSE3 = 1u << 9,
+    CPUID_1C_CID = 1u << 10,
+    CPUID_1C_SDBG = 1u << 11,
+    CPUID_1C_FMA = 1u << 12,
+    CPUID_1C_CX16 = 1u << 13,
+    CPUID_1C_XTPR = 1u << 14,
+    CPUID_1C_PDCM = 1u << 15,
+    CPUID_1C_PCID = 1u << 17,
+    CPUID_1C_DCA = 1u << 18,
+    CPUID_1C_SSE4_1 = 1u << 19,
+    CPUID_1C_SSE4_2 = 1u << 20,
+    CPUID_1C_X2APIC = 1u << 21,
+    CPUID_1C_MOVBE = 1u << 22,
+    CPUID_1C_POPCNT = 1u << 23,
+    CPUID_1C_TSC = 1u << 24,
+    CPUID_1C_AES = 1u << 25,
+    CPUID_1C_XSAVE = 1u << 26,
+    CPUID_1C_OSXSAVE = 1u << 27,
+    CPUID_1C_AVX = 1u << 28,
+    CPUID_1C_F16C = 1u << 29,
+    CPUID_1C_RDRAND = 1u << 30,
+    CPUID_1C_HYPERVISOR = 1u << 31,
+};
 
-#define CPUID_7B_FSGSBASE   (1 << 0)
-#define CPUID_7B_SGX        (1 << 2)
-#define CPUID_7B_BMI        (1 << 3)
-#define CPUID_7B_HLE        (1 << 4)
-#define CPUID_7B_AVX2       (1 << 5)
-#define CPUID_7B_SMEP       (1 << 7)
-#define CPUID_7B_BMI2       (1 << 8)
-#define CPUID_7B_RTM        (1 << 11)
-#define CPUID_7B_AVX512F    (1 << 16)
-#define CPUID_7B_AVX512DQ   (1 << 17)
-#define CPUID_7B_RDSEED     (1 << 18)
-#define CPUID_7B_ADX        (1 << 19)
-#define CPUID_7B_SMAP       (1 << 20)
-#define CPUID_7B_AVX512IFMA (1 << 21)
-#define CPUID_7B_CLFLUSHOPT (1 << 23)
-#define CPUID_7B_CLWB       (1 << 24)
-#define CPUID_7B_AVX512CD   (1 << 28)
-#define CPUID_7B_SHA        (1 << 29)
-#define CPUID_7B_AVX512BW   (1 << 30)
-#define CPUID_7B_AVX512VL   (1 << 31)
+enum : uint32_t {
+    CPUID_1D_FPU = 1u << 0,
+    CPUID_1D_VME = 1u << 1,
+    CPUID_1D_DE = 1u << 2,
+    CPUID_1D_PSE = 1u << 3,
+    CPUID_1D_TSC = 1u << 4,
+    CPUID_1D_MSR = 1u << 5,
+    CPUID_1D_PAE = 1u << 6,
+    CPUID_1D_MCE = 1u << 7,
+    CPUID_1D_CX8 = 1u << 8,
+    CPUID_1D_APIC = 1u << 9,
+    CPUID_1D_SEP = 1u << 11,
+    CPUID_1D_MTRR = 1u << 12,
+    CPUID_1D_PGE = 1u << 13,
+    CPUID_1D_MCA = 1u << 14,
+    CPUID_1D_CMOV = 1u << 15,
+    CPUID_1D_PAT = 1u << 16,
+    CPUID_1D_PSE36 = 1u << 17,
+    CPUID_1D_PSN = 1u << 18,
+    CPUID_1D_CLFLUSH = 1u << 19,
+    CPUID_1D_DS = 1u << 21,
+    CPUID_1D_ACPI = 1u << 22,
+    CPUID_1D_MMX = 1u << 23,
+    CPUID_1D_FXSR = 1u << 24,
+    CPUID_1D_SSE = 1u << 25,
+    CPUID_1D_SSE2 = 1u << 26,
+    CPUID_1D_SS = 1u << 27,
+    CPUID_1D_HTT = 1u << 28,
+    CPUID_1D_TM = 1u << 29,
+    CPUID_1D_IA64 = 1u << 30,
+    CPUID_1D_PBE = 1u << 31,
+};
 
-#define CPUID_7C_AVX512VBMI      (1 << 1)
-#define CPUID_7C_UMIP            (1 << 2)
-#define CPUID_7C_PKU             (1 << 3)
-#define CPUID_7C_OSPKE           (1 << 4)
-#define CPUID_7C_WAITPKG         (1 << 5)
-#define CPUID_7C_AVX512VBMI2     (1 << 6)
-#define CPUID_7C_SHSTK           (1 << 7)
-#define CPUID_7C_GFNI            (1 << 8)
-#define CPUID_7C_VAES            (1 << 9)
-#define CPUID_7C_VPCLMULQDQ      (1 << 10)
-#define CPUID_7C_AVX512VNNI      (1 << 11)
-#define CPUID_7C_AVX512BITALG    (1 << 12)
-#define CPUID_7C_TME_EN          (1 << 13)
-#define CPUID_7C_AVX512VPOPCNTDQ (1 << 14)
-#define CPUID_7C_RDPID           (1 << 22)
-#define CPUID_7C_KL              (1 << 23)
-#define CPUID_7C_CLDEMOTE        (1 << 25)
-#define CPUID_7C_MOVDIRI         (1 << 27)
-#define CPUID_7C_MOVDIR64B       (1 << 28)
-#define CPUID_7C_ENQCMD          (1 << 29)
+enum : uint32_t {
+    CPUID_7B_FSGSBASE = 1u << 0,
+    CPUID_7B_SGX = 1u << 2,
+    CPUID_7B_BMI = 1u << 3,
+    CPUID_7B_HLE = 1u << 4,
+    CPUID_7B_AVX2 = 1u << 5,
+    CPUID_7B_SMEP = 1u << 7,
+    CPUID_7B_BMI2 = 1u << 8,
+    CPUID_7B_RTM = 1u << 11,
+    CPUID_7B_AVX512F = 1u << 16,
+    CPUID_7B_AVX512DQ = 1u << 17,
+    CPUID_7B_RDSEED = 1u << 18,
+    CPUID_7B_ADX = 1u << 19,
+    CPUID_7B_SMAP = 1u << 20,
+    CPUID_7B_AVX512IFMA = 1u << 21,
+    CPUID_7B_CLFLUSHOPT = 1u << 23,
+    CPUID_7B_CLWB = 1u << 24,
+    CPUID_7B_AVX512CD = 1u << 28,
+    CPUID_7B_SHA = 1u << 29,
+    CPUID_7B_AVX512BW = 1u << 30,
+    CPUID_7B_AVX512VL = 1u << 31,
+};
 
-#define CPUID_7D_UINTR              (1 << 5)
-#define CPUID_7D_AVX512VP2INTERSECT (1 << 8)
-#define CPUID_7D_SERIALIZE          (1 << 14)
-#define CPUID_7D_TSXLDTRK           (1 << 16)
-#define CPUID_7D_PCONFIG            (1 << 18)
-#define CPUID_7D_IBT                (1 << 20)
-#define CPUID_7D_AMX_BF16           (1 << 22)
-#define CPUID_7D_AVX512FP16         (1 << 23)
-#define CPUID_7D_AMX_TILE           (1 << 24)
-#define CPUID_7D_AMX_INT8           (1 << 25)
+enum : uint32_t {
+    CPUID_7C_AVX512VBMI = 1u << 1,
+    CPUID_7C_UMIP = 1u << 2,
+    CPUID_7C_PKU = 1u << 3,
+    CPUID_7C_OSPKE = 1u << 4,
+    CPUID_7C_WAITPKG = 1u << 5,
+    CPUID_7C_AVX512VBMI2 = 1u << 6,
+    CPUID_7C_SHSTK = 1u << 7,
+    CPUID_7C_GFNI = 1u << 8,
+    CPUID_7C_VAES = 1u << 9,
+    CPUID_7C_VPCLMULQDQ = 1u << 10,
+    CPUID_7C_AVX512VNNI = 1u << 11,
+    CPUID_7C_AVX512BITALG = 1u << 12,
+    CPUID_7C_TME_EN = 1u << 13,
+    CPUID_7C_AVX512VPOPCNTDQ = 1u << 14,
+    CPUID_7C_RDPID = 1u << 22,
+    CPUID_7C_KL = 1u << 23,
+    CPUID_7C_CLDEMOTE = 1u << 25,
+    CPUID_7C_MOVDIRI = 1u << 27,
+    CPUID_7C_MOVDIR64B = 1u << 28,
+    CPUID_7C_ENQCMD = 1u << 29,
+};
 
-#define RFLAGS_CF   (1u << 0)
-#define RFLAGS_PF   (1u << 2)
-#define RFLAGS_AF   (1u << 4)
-#define RFLAGS_ZF   (1u << 6)
-#define RFLAGS_SF   (1u << 7)
-#define RFLAGS_TF   (1u << 8)
-#define RFLAGS_IF   (1u << 9)
-#define RFLAGS_DF   (1u << 10)
-#define RFLAGS_OF   (1u << 11)
-#define RFLAGS_IOPL (3u << 12)
-#define RFLAGS_NT   (1u << 14)
-#define RFLAGS_RF   (1u << 16)
-#define RFLAGS_VM   (1u << 17)
-#define RFLAGS_AC   (1u << 18)
-#define RFLAGS_VIF  (1u << 19)
-#define RFLAGS_VIP  (1u << 20)
-#define RFLAGS_ID   (1u << 21)
+enum : uint32_t {
+    CPUID_7D_UINTR = 1u << 5,
+    CPUID_7D_AVX512VP2INTERSECT = 1u << 8,
+    CPUID_7D_SERIALIZE = 1u << 14,
+    CPUID_7D_TSXLDTRK = 1u << 16,
+    CPUID_7D_PCONFIG = 1u << 18,
+    CPUID_7D_IBT = 1u << 20,
+    CPUID_7D_AMX_BF16 = 1u << 22,
+    CPUID_7D_AVX512FP16 = 1u << 23,
+    CPUID_7D_AMX_TILE = 1u << 24,
+    CPUID_7D_AMX_INT8 = 1u << 25,
+};
 
-#define XCR0_X87              (1 << 0)
-#define XCR0_SSE              (1 << 1)
-#define XCR0_AVX              (1 << 2)
-#define XCR0_AXV512_OPMASK    (1 << 5)
-#define XCR0_AVX512_ZMM_HI256 (1 << 6)
-#define XCR0_AVX512_HI16_ZMM  (1 << 7)
+enum : uint64_t {
+    RFLAGS_CF = 1ull << 0,
+    RFLAGS_BRKI = 1ull << 1,
+    RFLAGS_PF = 1ull << 2,
+    RFLAGS_AF = 1ull << 4,
+    RFLAGS_ZF = 1ull << 6,
+    RFLAGS_SF = 1ull << 7,
+    RFLAGS_TF = 1ull << 8,
+    RFLAGS_IF = 1ull << 9,
+    RFLAGS_DF = 1ull << 10,
+    RFLAGS_OF = 1ull << 11,
+    RFLAGS_IOPL = 3ull << 12,
+    RFLAGS_NT = 1ull << 14,
+    RFLAGS_RF = 1ull << 16,
+    RFLAGS_VM = 1ull << 17,
+    RFLAGS_AC = 1ull << 18,
+    RFLAGS_VIF = 1ull << 19,
+    RFLAGS_VIP = 1ull << 20,
+    RFLAGS_ID = 1ull << 21,
+};
 
-#define CR0_PE (1 << 0)
-#define CR0_MP (1 << 1)
-#define CR0_EM (1 << 2)
-#define CR0_TS (1 << 3)
-#define CR0_ET (1 << 4)
-#define CR0_NE (1 << 5)
-#define CR0_WP (1 << 16)
-#define CR0_AM (1 << 18)
-#define CR0_NW (1 << 29)
-#define CR0_CD (1 << 30)
-#define CR0_PG (1 << 31)
+enum : uint64_t {
+    XCR0_X87 = 1ull << 0,
+    XCR0_SSE = 1ull << 1,
+    XCR0_AVX = 1ull << 2,
+    XCR0_AXV512_OPMASK = 1ull << 5,
+    XCR0_AVX512_ZMM_HI256 = 1ull << 6,
+    XCR0_AVX512_HI16_ZMM = 1ull << 7,
+};
 
-#define CR4_VME        (1 << 0)
-#define CR4_PVI        (1 << 1)
-#define CR4_TSD        (1 << 2)
-#define CR4_DE         (1 << 3)
-#define CR4_PSE        (1 << 4)
-#define CR4_PAE        (1 << 5)
-#define CR4_MCE        (1 << 6)
-#define CR4_PGE        (1 << 7)
-#define CR4_PCE        (1 << 8)
-#define CR4_OSFXSR     (1 << 9)
-#define CR4_OSXMMEXCPT (1 << 10)
-#define CR4_UMIP       (1 << 11)
-#define CR4_VMXE       (1 << 13)
-#define CR4_SMXE       (1 << 14)
-#define CR4_FSGSBASE   (1 << 16)
-#define CR4_PCIDE      (1 << 17)
-#define CR4_OSXSAVE    (1 << 18)
-#define CR4_SMEP       (1 << 20)
-#define CR4_SMAP       (1 << 21)
-#define CR4_PKE        (1 << 22)
-#define CR4_CET        (1 << 23)
-#define CR4_PKS        (1 << 24)
+enum : uint64_t {
+    CR0_PE = 1ull << 0,
+    CR0_MP = 1ull << 1,
+    CR0_EM = 1ull << 2,
+    CR0_TS = 1ull << 3,
+    CR0_ET = 1ull << 4,
+    CR0_NE = 1ull << 5,
+    CR0_WP = 1ull << 16,
+    CR0_AM = 1ull << 18,
+    CR0_NW = 1ull << 29,
+    CR0_CD = 1ull << 30,
+    CR0_PG = 1ull << 31,
+};
 
-#define MSR_EFER_SCE   (1 << 0)
-#define MSR_EFER_LME   (1 << 8)
-#define MSR_EFER_LMA   (1 << 10)
-#define MSR_EFER_NXE   (1 << 11)
-#define MSR_EFER_SVME  (1 << 12)
-#define MSR_EFER_LMSLE (1 << 13)
-#define MSR_EFER_FFXSR (1 << 14)
-#define MSR_EFER_TCE   (1 << 15)
+enum : uint64_t {
+    CR4_VME = 1ull << 0,
+    CR4_PVI = 1ull << 1,
+    CR4_TSD = 1ull << 2,
+    CR4_DE = 1ull << 3,
+    CR4_PSE = 1ull << 4,
+    CR4_PAE = 1ull << 5,
+    CR4_MCE = 1ull << 6,
+    CR4_PGE = 1ull << 7,
+    CR4_PCE = 1ull << 8,
+    CR4_OSFXSR = 1ull << 9,
+    CR4_OSXMMEXCPT = 1ull << 10,
+    CR4_UMIP = 1ull << 11,
+    CR4_LA57 = 1ull << 12,
+    CR4_VMXE = 1ull << 13,
+    CR4_SMXE = 1ull << 14,
+    CR4_FSGSBASE = 1ull << 16,
+    CR4_PCIDE = 1ull << 17,
+    CR4_OSXSAVE = 1ull << 18,
+    CR4_SMEP = 1ull << 20,
+    CR4_SMAP = 1ull << 21,
+    CR4_PKE = 1ull << 22,
+    CR4_CET = 1ull << 23,
+    CR4_PKS = 1ull << 24,
+};
 
-#define MSR_EFER           (0xC0000080)
-#define MSR_STAR           (0xC0000081)
-#define MSR_LSTAR          (0xC0000082)
-#define MSR_CSTAR          (0xC0000083)
-#define MSR_SFMASK         (0xC0000084)
-#define MSR_FS_BASE        (0xC0000100)
-#define MSR_GS_BASE        (0xC0000101)
-#define MSR_KERNEL_GS_BASE (0xC0000102)
+enum : uint64_t {
+    MSR_EFER_SCE = 1ull << 0,
+    MSR_EFER_LME = 1ull << 8,
+    MSR_EFER_LMA = 1ull << 10,
+    MSR_EFER_NXE = 1ull << 11,
+    MSR_EFER_SVME = 1ull << 12,
+    MSR_EFER_LMSLE = 1ull << 13,
+    MSR_EFER_FFXSR = 1ull << 14,
+    MSR_EFER_TCE = 1ull << 15,
+};
 
-#define CPL_USER   (0x3)
-#define CPL_KERNEL (0x0)
+enum : uint32_t {
+    MSR_PAT = 0x277,
+    MSR_EFER = 0xC0000080,
+    MSR_STAR = 0xC0000081,
+    MSR_LSTAR = 0xC0000082,
+    MSR_CSTAR = 0xC0000083,
+    MSR_SFMASK = 0xC0000084,
+    MSR_FS_BASE = 0xC0000100,
+    MSR_GS_BASE = 0xC0000101,
+    MSR_KERNEL_GS_BASE = 0xC0000102,
+};
 
-#define IDT_PF          (0xE)
-#define IDT_SPURIOUS    (0xFE)
-#define IDT_IPI_RESCHED (0xFF)
+enum : uint8_t {
+    PAT_UC = 0x0,
+    PAT_WC = 0x1,
+    PAT_WT = 0x4,
+    PAT_WP = 0x5,
+    PAT_WB = 0x6,
+    PAT_UCM = 0x7,
+};
+
+enum : uint8_t {
+    CPL_USER = 0x3,
+    CPL_KERNEL = 0x0,
+};
+
+enum : uint8_t {
+    IDT_DE = 0x0,
+    IDT_DB = 0x1,
+    IDT_BP = 0x3,
+    IDT_OF = 0x4,
+    IDT_BR = 0x5,
+    IDT_UD = 0x6,
+    IDT_NM = 0x7,
+    IDT_SS = 0xC,
+    IDT_GP = 0xD,
+    IDT_PF = 0xE,
+    IDT_MF = 0x10,
+    IDT_AC = 0x11,
+    IDT_XM = 0x13,
+    IDT_IPI_SHOOTDOWN = 0xFC,
+    IDT_TIMER = 0xFD,
+    IDT_SPURIOUS = 0xFE,
+    IDT_IPI_RESCHED = 0xFF,
+};

arch/x86_64/include/x86_64/gdt.h

@@ -1,7 +1,9 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
-#include <stdint.h>
 #include <zinnia/compiler.h>
+#include <stdint.h>
 
 union gdt_desc {
     uint64_t val;
@@ -50,7 +52,7 @@ struct [[__packed]] gdtr {
     struct gdt* base;
 };
 
-struct percpu;
+extern const struct gdt gdt_base;
 
 // Initializes a GDT on the local CPU.
 void gdt_load();

arch/x86_64/include/x86_64/hpet.h

@@ -1,3 +0,0 @@
-#pragma once
-
-void hpet_init();

arch/x86_64/include/x86_64/idt.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/init.h>

arch/x86_64/include/x86_64/percpu.h

@@ -1,6 +1,9 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <zinnia/irq.h>
+#include <zinnia/percpu.h>
 #include <stddef.h>
 #include <x86_64/apic.h>
 #include <x86_64/gdt.h>
@@ -18,3 +21,6 @@ struct arch_percpu {
 };
 
 extern struct arch_percpu arch_data;
+
+// Initializes the calling CPU and makes `cpu` its per-CPU block.
+void x86_64_cpu_setup(struct percpu* cpu);

arch/x86_64/include/x86_64/pmap.h

@@ -0,0 +1,4 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+#include <zinnia/pmap.h>

arch/x86_64/include/x86_64/smp.h

@@ -1,23 +1,12 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
-#define SMP_GDTR_OFFSET       80
-#define SMP_FARJMP_OFFSET     90
-#define SMP_TEMP_STACK_OFFSET 98
-#define SMP_TEMP_CR3_OFFSET   102
-#define SMP_ENTRY_OFFSET      106
-#define SMP_HHDM_OFFSET       114
-#define SMP_KERNEL32_DS       16
-#define SMP_KERNEL64_CS       24
-#define SMP_KERNEL64_DS       32
-#define SMP_INFO_SIZE         127
-
-#ifndef __ASSEMBLER__
-
-#include <stddef.h>
-#include <stdint.h>
-#include <x86_64/gdt.h>
+#include <zinnia/atomic.h>
 #include <zinnia/compiler.h>
 #include <zinnia/utils.h>
+#include <stdint.h>
+#include <x86_64/gdt.h>
 
 struct [[__packed]] smp_info {
     struct gdt gdt;
@@ -28,22 +17,10 @@ struct [[__packed]] smp_info {
     uint32_t temp_cr3;
     uint64_t entry;
     uint64_t hhdm_offset;
+    uint64_t percpu;
     uint32_t lapic_id;
-    uint8_t booted;
+    atomic_bool booted;
 };
 
-static_assert(SMP_GDTR_OFFSET == offsetof(struct smp_info, gdtr));
-static_assert(SMP_FARJMP_OFFSET == offsetof(struct smp_info, farjmp_offset));
-static_assert(SMP_TEMP_STACK_OFFSET == offsetof(struct smp_info, temp_stack));
-static_assert(SMP_TEMP_CR3_OFFSET == offsetof(struct smp_info, temp_cr3));
-static_assert(SMP_ENTRY_OFFSET == offsetof(struct smp_info, entry));
-static_assert(SMP_HHDM_OFFSET == offsetof(struct smp_info, hhdm_offset));
-static_assert(SMP_KERNEL32_DS == offsetof(struct gdt, kernel_data32));
-static_assert(SMP_KERNEL64_CS == offsetof(struct gdt, kernel_code64));
-static_assert(SMP_KERNEL64_DS == offsetof(struct gdt, kernel_data64));
-static_assert(SMP_INFO_SIZE == sizeof(struct smp_info));
-
-// Initializes a LAPIC.
-void x86_64_smp_init(uint32_t id);
-
-#endif
+// Discovers all APs and starts them.
+void x86_64_smp_init();

arch/x86_64/include/x86_64/syscall.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 void arch_syscall_stub();

arch/x86_64/include/x86_64/tsc.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 void tsc_init();

arch/x86_64/include/x86_64/tss.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>

arch/x86_64/ioapic.c

@@ -0,0 +1,281 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "ioapic: " fmt
+
+#include <zinnia/acpi.h>
+#include <zinnia/alloc.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/irq.h>
+#include <zinnia/mmio.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/irq.h>
+#include <arch/mem.h>
+#include <uacpi/acpi.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+#include <x86_64/apic.h>
+#include <x86_64/percpu.h>
+
+constexpr size_t ISA_IRQS = 16;
+
+constexpr mmio_reg32 IOAPIC_REG_SELECT = {.offset = 0x00};
+constexpr mmio_reg32 IOAPIC_REG_WINDOW = {.offset = 0x10};
+
+constexpr uint32_t IOAPIC_VERSION = 0x01;
+constexpr uint32_t IOAPIC_REDIRECT = 0x10;
+
+constexpr uint64_t IOAPIC_RTE_POLARITY_LOW = 1 << 13;
+constexpr uint64_t IOAPIC_RTE_TRIGGER_LEVEL = 1 << 15;
+constexpr uint64_t IOAPIC_RTE_MASKED = 1 << 16;
+
+struct ioapic_line {
+    struct irq_line line;
+    struct ioapic* ioapic;
+    uint32_t pin;
+    uint32_t lapic;
+    uint8_t vector;
+};
+
+struct ioapic {
+    void* regs;
+    struct ioapic_line** lines;
+    struct spinlock lock;
+    uint32_t gsi_base;
+    uint32_t num_pins;
+};
+
+struct isa_override {
+    uint32_t gsi;
+    uint16_t flags;
+    bool present;
+};
+
+static struct ioapic* ioapics;
+static size_t num_ioapics;
+static struct isa_override isa_overrides[ISA_IRQS];
+static struct spinlock lines_lock;
+
+static uint32_t ioapic_read(struct ioapic* ioapic, uint32_t reg) {
+    mmio_reg_write(ioapic->regs, IOAPIC_REG_SELECT, reg);
+    return mmio_reg_read(ioapic->regs, IOAPIC_REG_WINDOW);
+}
+
+static void ioapic_write(struct ioapic* ioapic, uint32_t reg, uint32_t value) {
+    mmio_reg_write(ioapic->regs, IOAPIC_REG_SELECT, reg);
+    mmio_reg_write(ioapic->regs, IOAPIC_REG_WINDOW, value);
+}
+
+static void write_rte(struct ioapic* ioapic, uint32_t pin, uint64_t rte) {
+    ioapic_write(ioapic, IOAPIC_REDIRECT + pin * 2, IOAPIC_RTE_MASKED);
+    ioapic_write(ioapic, IOAPIC_REDIRECT + pin * 2 + 1, rte >> 32);
+    ioapic_write(ioapic, IOAPIC_REDIRECT + pin * 2, rte);
+}
+
+static void update_mask(struct ioapic_line* line, bool masked) {
+    struct ioapic* ioapic = line->ioapic;
+    spin_lock(&ioapic->lock);
+    uint32_t low = ioapic_read(ioapic, IOAPIC_REDIRECT + line->pin * 2);
+    low = masked ? low | IOAPIC_RTE_MASKED : low & ~IOAPIC_RTE_MASKED;
+    ioapic_write(ioapic, IOAPIC_REDIRECT + line->pin * 2, low);
+    spin_unlock(&ioapic->lock);
+}
+
+static enum irq_mode ioapic_set_config(
+    struct irq_line* self,
+    enum irq_trigger_mode trigger,
+    enum irq_polarity polarity
+) {
+    struct ioapic_line* line = containerof(self, struct ioapic_line, line);
+    uint64_t rte = line->vector | ((uint64_t)line->lapic << 56);
+    if (polarity == IRQ_POLARITY_LOW)
+        rte |= IOAPIC_RTE_POLARITY_LOW;
+    if (trigger == IRQ_TRIGGER_LEVEL)
+        rte |= IOAPIC_RTE_TRIGGER_LEVEL;
+
+    spin_lock(&line->ioapic->lock);
+    write_rte(line->ioapic, line->pin, rte);
+    spin_unlock(&line->ioapic->lock);
+    return IRQ_MODE_EOI | IRQ_MODE_MASKABLE;
+}
+
+static void ioapic_mask(struct irq_line* self) {
+    update_mask(containerof(self, struct ioapic_line, line), true);
+}
+
+static void ioapic_unmask(struct irq_line* self) {
+    update_mask(containerof(self, struct ioapic_line, line), false);
+}
+
+static void ioapic_eoi(struct irq_line* self) {
+    lapic_eoi(&percpu_ref(arch_data)->lapic);
+}
+
+errno_t arch_irq_get_gsi(uint32_t gsi, struct irq_line** out) {
+    // Find the IOAPIC that handles this GSI.
+    struct ioapic* ioapic = nullptr;
+    for (size_t i = 0; i < num_ioapics; i++) {
+        if (gsi >= ioapics[i].gsi_base && gsi < ioapics[i].gsi_base + ioapics[i].num_pins) {
+            ioapic = &ioapics[i];
+            break;
+        }
+    }
+    if (!ioapic)
+        return ENOENT;
+    const uint32_t pin = gsi - ioapic->gsi_base;
+
+    spin_lock(&lines_lock);
+    struct ioapic_line* line = ioapic->lines[pin];
+    if (line) {
+        spin_unlock(&lines_lock);
+        *out = &line->line;
+        return 0;
+    }
+    spin_unlock(&lines_lock);
+
+    line = mem_alloc(sizeof(*line), 0);
+    if (!line)
+        return ENOMEM;
+    line->line.set_config = ioapic_set_config;
+    line->line.mask = ioapic_mask;
+    line->line.unmask = ioapic_unmask;
+    line->line.eoi = ioapic_eoi;
+    line->ioapic = ioapic;
+    line->pin = pin;
+
+    errno_t status = apic_alloc_vector(&line->line, &line->vector, &line->lapic);
+    if (status) {
+        mem_free(line);
+        return status;
+    }
+
+    spin_lock(&lines_lock);
+    if (ioapic->lines[pin]) {
+        spin_unlock(&lines_lock);
+        apic_free_vector(line->vector);
+        mem_free(line);
+        return arch_irq_get_gsi(gsi, out);
+    }
+    ioapic->lines[pin] = line;
+    spin_unlock(&lines_lock);
+
+    *out = &line->line;
+    return 0;
+}
+
+bool arch_irq_isa_override(
+    uint8_t isa_irq,
+    uint32_t* out_gsi,
+    enum irq_trigger_mode* out_trigger,
+    enum irq_polarity* out_polarity
+) {
+    if (isa_irq >= ISA_IRQS || !isa_overrides[isa_irq].present)
+        return false;
+
+    const struct isa_override* iso = &isa_overrides[isa_irq];
+    *out_gsi = iso->gsi;
+    switch (iso->flags & ACPI_MADT_POLARITY_MASK) {
+    case ACPI_MADT_POLARITY_ACTIVE_HIGH:
+        *out_polarity = IRQ_POLARITY_HIGH;
+        break;
+    case ACPI_MADT_POLARITY_ACTIVE_LOW:
+        *out_polarity = IRQ_POLARITY_LOW;
+        break;
+    }
+    switch (iso->flags & ACPI_MADT_TRIGGERING_MASK) {
+    case ACPI_MADT_TRIGGERING_EDGE:
+        *out_trigger = IRQ_TRIGGER_EDGE;
+        break;
+    case ACPI_MADT_TRIGGERING_LEVEL:
+        *out_trigger = IRQ_TRIGGER_LEVEL;
+        break;
+    }
+    return true;
+}
+
+static uacpi_iteration_decision count_ioapics(uacpi_handle user, struct acpi_entry_hdr* hdr) {
+    size_t* count = user;
+    if (hdr->type == ACPI_MADT_ENTRY_TYPE_IOAPIC)
+        (*count)++;
+    return UACPI_ITERATION_DECISION_CONTINUE;
+}
+
+static uacpi_iteration_decision parse_madt_entry(uacpi_handle, struct acpi_entry_hdr* hdr) {
+    switch (hdr->type) {
+    case ACPI_MADT_ENTRY_TYPE_IOAPIC: {
+        // Map the IOAPIC and mask all of its pins.
+        const struct acpi_madt_ioapic* entry = (const struct acpi_madt_ioapic*)hdr;
+        uintptr_t vaddr = vm_alloc(arch_mem_page_size());
+        if (!vaddr)
+            return UACPI_ITERATION_DECISION_CONTINUE;
+        pmap_map(&kernel_vma.pmap, vaddr, entry->address, PTE_READ | PTE_WRITE, CACHE_MMIO);
+
+        struct ioapic* ioapic = &ioapics[num_ioapics];
+        ioapic->regs = (void*)vaddr;
+        ioapic->gsi_base = entry->gsi_base;
+        ioapic->num_pins = ((ioapic_read(ioapic, IOAPIC_VERSION) >> 16) & 0xFF) + 1;
+        spin_init(&ioapic->lock);
+        ioapic->lines = mem_alloc(ioapic->num_pins * sizeof(*ioapic->lines), 0);
+        if (!ioapic->lines) {
+            pmap_unmap(&kernel_vma.pmap, vaddr);
+            vm_free(vaddr, arch_mem_page_size());
+            return UACPI_ITERATION_DECISION_CONTINUE;
+        }
+
+        for (uint32_t pin = 0; pin < ioapic->num_pins; pin++)
+            write_rte(ioapic, pin, IOAPIC_RTE_MASKED);
+
+        num_ioapics++;
+        log("IOAPIC %u at %#x handles GSIs %u-%u\n",
+            entry->id,
+            entry->address,
+            ioapic->gsi_base,
+            ioapic->gsi_base + ioapic->num_pins - 1);
+        break;
+    }
+    case ACPI_MADT_ENTRY_TYPE_INTERRUPT_SOURCE_OVERRIDE:
+        auto iso = (const struct acpi_madt_interrupt_source_override*)hdr;
+        if (iso->bus == 0 && iso->source < ISA_IRQS) {
+            isa_overrides[iso->source] = (struct isa_override){
+                .gsi = iso->gsi,
+                .flags = iso->flags,
+                .present = true,
+            };
+        }
+        break;
+    }
+    return UACPI_ITERATION_DECISION_CONTINUE;
+}
+
+static void ioapic_init(struct boot_info*) {
+    uacpi_table table;
+    if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) {
+        log_warn("No MADT found\n");
+        return;
+    }
+
+    size_t capacity = 0;
+    uacpi_status status = uacpi_for_each_subtable(table.hdr, sizeof(struct acpi_madt), count_ioapics, &capacity);
+    if (status != UACPI_STATUS_OK)
+        log_warn("Failed to parse the MADT: %s\n", uacpi_status_to_string(status));
+
+    if (capacity) {
+        ioapics = mem_alloc(capacity * sizeof(*ioapics), 0);
+        if (!ioapics) {
+            log_err("Unable to allocate %zu controllers\n", capacity);
+            uacpi_table_unref(&table);
+            return;
+        }
+    }
+
+    uacpi_for_each_subtable(table.hdr, sizeof(struct acpi_madt), parse_madt_entry, nullptr);
+    uacpi_table_unref(&table);
+}
+INIT_TASK(ioapic_stage, "arch.x86_64.ioapic", ioapic_init);
+INIT_DEPENDS(ioapic_stage, acpi_stage, percpu_stage);
+INIT_ENTAILS(ioapic_stage, acpi_namespace_stage);

arch/x86_64/irq.c

@@ -1,6 +1,14 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/percpu.h>
+#include <zinnia/utils.h>
 #include <arch/irq.h>
 #include <stdint.h>
+#include <x86_64/apic.h>
 #include <x86_64/defs.h>
+#include <x86_64/percpu.h>
 
 bool arch_irq_get_state() {
     uint64_t flags;
@@ -8,13 +16,95 @@ bool arch_irq_get_state() {
     return flags & RFLAGS_IF;
 }
 
-void arch_irq_set_state(bool state) {
+bool arch_irq_set_state(bool state) {
+    bool old = arch_irq_get_state();
     if (state)
-        asm volatile("sti");
+        asm volatile("sti" ::: "memory");
     else
-        asm volatile("cli");
+        asm volatile("cli" ::: "memory");
+    return old;
 }
 
 void arch_irq_wait() {
     asm volatile("hlt");
 }
+
+struct apic_msi_line {
+    struct msi_line msi;
+    uint8_t vector;
+    uint32_t lapic;
+};
+
+static phys_t apic_msi_msg_addr(struct msi_line* self) {
+    struct apic_msi_line* line = containerof(self, struct apic_msi_line, msi);
+    return 0xFEE0'0000 | ((phys_t)line->lapic << 12);
+}
+
+static uint32_t apic_msi_msg_data(struct msi_line* self) {
+    return containerof(self, struct apic_msi_line, msi)->vector;
+}
+
+static enum irq_mode apic_msi_set_config(
+    struct irq_line* self,
+    enum irq_trigger_mode trigger,
+    enum irq_polarity polarity
+) {
+    assert(trigger == IRQ_TRIGGER_EDGE, "MSIs must be edge triggered");
+    return IRQ_MODE_EOI;
+}
+
+static void apic_msi_mask(struct irq_line* self) {}
+
+static void apic_msi_unmask(struct irq_line* self) {}
+
+static void apic_msi_eoi(struct irq_line* self) {
+    lapic_eoi(&percpu_ref(arch_data)->lapic);
+}
+
+errno_t apic_alloc_vector(struct irq_line* line, uint8_t* out_vector, uint32_t* out_lapic) {
+    irq_lock();
+    struct arch_percpu* arch_cpu = percpu_ref(arch_data);
+    size_t slot;
+    for (slot = 0; slot < ARRAY_SIZE(arch_cpu->irq_lines); slot++) {
+        if (!arch_cpu->irq_lines[slot])
+            break;
+    }
+    if (slot == ARRAY_SIZE(arch_cpu->irq_lines)) {
+        irq_unlock();
+        return ENOSPC;
+    }
+
+    *out_vector = slot + 0x20;
+    *out_lapic = arch_cpu->lapic.id;
+    arch_cpu->irq_lines[slot] = line;
+    irq_unlock();
+    return 0;
+}
+
+void apic_free_vector(uint8_t vector) {
+    irq_lock();
+    percpu_ref(arch_data)->irq_lines[vector - 0x20] = nullptr;
+    irq_unlock();
+}
+
+errno_t arch_irq_alloc_msi(struct msi_line** out) {
+    struct apic_msi_line* line = mem_alloc(sizeof(*line), 0);
+    if (!line)
+        return ENOMEM;
+
+    line->msi.line.set_config = apic_msi_set_config;
+    line->msi.line.mask = apic_msi_mask;
+    line->msi.line.unmask = apic_msi_unmask;
+    line->msi.line.eoi = apic_msi_eoi;
+    line->msi.msg_addr = apic_msi_msg_addr;
+    line->msi.msg_data = apic_msi_msg_data;
+
+    errno_t status = apic_alloc_vector(&line->msi.line, &line->vector, &line->lapic);
+    if (status) {
+        mem_free(line);
+        return status;
+    }
+
+    *out = &line->msi;
+    return 0;
+}

arch/x86_64/mem.c

@@ -1,4 +1,3 @@
-#include <zinnia/types.h>
 #include <arch/mem.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -32,5 +31,5 @@ uintptr_t arch_mem_mapping_addr() {
 }
 
 bool arch_mem_addr_is_user(uintptr_t addr) {
-    return addr < 0x0000'8000'0000'0000;
+    return addr < 0x0000'7FFF'FFFF'F000;
 }

arch/x86_64/meson.build

@@ -10,17 +10,25 @@ kernel_sources += files(
     'gdt.c',
     'hpet.c',
     'idt.c',
+    'ioapic.c',
     'irq.c',
     'mem.c',
     'percpu.c',
     'pmap.c',
-    'pmap.c',
+    'random.c',
     'sched.c',
     'serial.c',
     'smp.c',
-    'sys.c',
+    'start.c',
     'syscall.c',
-    'trampoline.S',
     'tsc.c',
     'usercopy.c',
 )
+
+if kconfig.get('CONFIG_PCI', 'n') == 'y'
+    kernel_sources += files('pci.c')
+endif
+
+if kconfig.get('CONFIG_ACPI', 'n') == 'y'
+    kernel_sources += files('acpi.c')
+endif

arch/x86_64/pci.c

@@ -0,0 +1,50 @@
+#include <zinnia/init.h>
+#include <zinnia/pci.h>
+#include <zinnia/spin.h>
+#include <x86_64/asm.h>
+
+constexpr uint16_t PCI_CONFIG_ADDRESS = 0xcf8;
+constexpr uint16_t PCI_CONFIG_DATA = 0xcfc;
+
+static struct spinlock port_lock = {};
+
+static void port_select(const struct pci_address* addr, uint16_t offset) {
+    asm_outl(
+        PCI_CONFIG_ADDRESS,
+        1u << 31 | (uint32_t)addr->bus << 16 | (uint32_t)(addr->device & 0x1f) << 11 |
+            (uint32_t)(addr->function & 0x7) << 8 | (offset & 0xfc)
+    );
+}
+
+static uint32_t port_read32(struct pci_access*, const struct pci_address* addr, uint16_t offset) {
+    spin_lock(&port_lock);
+    port_select(addr, offset);
+    uint32_t value = asm_inl(PCI_CONFIG_DATA);
+    spin_unlock(&port_lock);
+    return value;
+}
+
+static void port_write32(struct pci_access*, const struct pci_address* addr, uint16_t offset, uint32_t value) {
+    spin_lock(&port_lock);
+    port_select(addr, offset);
+    asm_outl(PCI_CONFIG_DATA, value);
+    spin_unlock(&port_lock);
+}
+
+static struct pci_access port_access = {
+    .name = "port",
+    .domain = 0,
+    .start_bus = 0,
+    .end_bus = 255,
+    .priority = PCI_ACCESS_PRIO_LEGACY,
+    .read32 = port_read32,
+    .write32 = port_write32,
+};
+
+[[__init]]
+static void pci_x86_stage_init(struct boot_info*) {
+    pci_access_register(&port_access);
+}
+INIT_TASK(pci_x86_stage, "arch.x86_64.pci", pci_x86_stage_init);
+INIT_DEPENDS(pci_x86_stage, mem_stage);
+INIT_ENTAILS(pci_x86_stage, pci_stage);

arch/x86_64/percpu.c

@@ -1,3 +1,4 @@
+#include <zinnia/initgraph.h>
 #include <zinnia/percpu.h>
 #include <zinnia/print.h>
 #include <config.h>
@@ -6,9 +7,9 @@
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
-#include <x86_64/hpet.h>
 #include <x86_64/idt.h>
 #include <x86_64/percpu.h>
+#include <x86_64/smp.h>
 #include <x86_64/syscall.h>
 #include <x86_64/tsc.h>
 
@@ -43,7 +44,7 @@ static_assert(sizeof(union ia32_star) == 8);
 [[__percpu]]
 struct arch_percpu arch_data = {0};
 
-static void setup_cpu(struct percpu* cpu) {
+void x86_64_cpu_setup(struct percpu* cpu) {
     asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
     asm_wrmsr(MSR_GS_BASE, (uint64_t)cpu);
     asm_wrmsr(MSR_FS_BASE, 0);
@@ -55,14 +56,21 @@ static void setup_cpu(struct percpu* cpu) {
     // TODO: not if FRED
     idt_load();
 
+    asm_wrmsr(
+        MSR_PAT,
+        PAT_WB | PAT_WT << 8 | PAT_UCM << 16 | PAT_UC << 24 | (uint64_t)PAT_WP << 32 | (uint64_t)PAT_WC << 40 |
+            (uint64_t)PAT_UCM << 48 | (uint64_t)PAT_UC << 56
+    );
+
     // Syscall extension.
     asm_wrmsr(MSR_EFER, asm_rdmsr(MSR_EFER) | MSR_EFER_SCE);
     union ia32_star star = {0};
     star.kernel_cs_ss = offsetof(struct gdt, kernel_code64);
-    star.user_cs_ss = offsetof(struct gdt, user_code32);
+    star.user_cs_ss = offsetof(struct gdt, user_code32) | CPL_USER;
     asm_wrmsr(MSR_STAR, star.value);
     asm_wrmsr(MSR_LSTAR, (uint64_t)arch_syscall_stub);
-    asm_wrmsr(MSR_SFMASK, RFLAGS_AC | RFLAGS_DF | RFLAGS_IF);
+    // Mask all RFLAGS except for BRKI which is reserved.
+    asm_wrmsr(MSR_SFMASK, ~RFLAGS_BRKI);
 
     // TODO: FRED
 
@@ -76,7 +84,7 @@ static void setup_cpu(struct percpu* cpu) {
 
     // Enable SSE.
     cr0 &= ~CR0_EM;
-    cr0 |= CR0_MP;
+    cr0 |= CR0_MP | CR0_WP;
     cr4 |= CR4_OSFXSR | CR4_OSXMMEXCPT;
 
     if (cpuid1.ecx & CPUID_1C_XSAVE) {
@@ -125,13 +133,13 @@ static void setup_cpu(struct percpu* cpu) {
 }
 
 void arch_percpu_init() {
-    hpet_init();
 #if CONFIG_TSC
     tsc_init();
 #endif
 
     // Init the BSP.
-    setup_cpu(&percpu_bsp);
+    x86_64_cpu_setup(&percpu_bsp);
 
     // Init any APs.
+    x86_64_smp_init();
 }

arch/x86_64/pmap.c

@@ -1,27 +1,44 @@
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/types.h>
-
-enum : pte_t {
-    ARCH_FLAG_PRESENT = 1 << 0,
-    ARCH_FLAG_READ_WRITE = 1 << 1,
-    ARCH_FLAG_USER_MODE = 1 << 2,
-    ARCH_FLAG_WRITE_THROUGH = 1 << 3,
-    ARCH_FLAG_CACHE_DISABLE = 1 << 4,
-    ARCH_FLAG_ACCESSED = 1 << 5,
-    ARCH_FLAG_DIRTY = 1 << 6,
-    ARCH_FLAG_SIZE = 1 << 7,
-    ARCH_FLAG_GLOBAL = 1 << 8,
-    ARCH_FLAG_AVAILABLE = 1 << 9,
-    ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 10,
-    ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
-
-    ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
-};
+#include <x86_64/apic.h>
+#include <x86_64/defs.h>
+#include <x86_64/percpu.h>
+#include <x86_64/pmap.h>
 
 void pte_clear(pte_t* pte) {
     *pte = 0;
 }
 
+void pte_write(pte_t* pte, pte_t val) {
+    *pte = val;
+}
+
+void pte_flush(uintptr_t vaddr) {
+    asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory");
+}
+
+void pte_flush_all() {
+    uint64_t cr3;
+    asm volatile("mov %0, cr3" : "=r"(cr3));
+    asm volatile("mov cr3, %0" ::"r"(cr3) : "memory");
+}
+
+void pte_shootdown_kick(struct percpu* cpu) {
+    irq_lock();
+    lapic_send_ipi(
+        &percpu_ref(arch_data)->lapic,
+        percpu_ref_on(cpu, arch_data)->lapic.id,
+        IDT_IPI_SHOOTDOWN,
+        APIC_DELIVERY_FIXED,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    irq_unlock();
+}
+
 pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
     pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT;
 
@@ -35,18 +52,40 @@ pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
             result |= ARCH_FLAG_READ_WRITE;
         if (!(flags & PTE_EXEC))
             result |= ARCH_FLAG_EXECUTE_DISABLE;
+
+        switch (cache) {
+        case CACHE_ANY:
+        case CACHE_WRITE_BACK:
+            break;
+        case CACHE_WRITE_THROUGH:
+            result |= ARCH_FLAG_WRITE_THROUGH;
+            break;
+        case CACHE_MMIO:
+            result |= ARCH_FLAG_CACHE_DISABLE | ARCH_FLAG_WRITE_THROUGH;
+            break;
+        case CACHE_WRITE_COMBINE:
+            result |= ARCH_FLAG_ATTRIBUTE_TABLE | ARCH_FLAG_WRITE_THROUGH;
+            break;
+        }
     }
     return result;
 }
 
+pte_t pte_protect(pte_t old, enum pte_flags flags) {
+    pte_t result = old & ~(ARCH_FLAG_READ_WRITE | ARCH_FLAG_USER_MODE | ARCH_FLAG_EXECUTE_DISABLE);
+    if (flags & PTE_USER)
+        result |= ARCH_FLAG_USER_MODE;
+    if (flags & PTE_WRITE)
+        result |= ARCH_FLAG_READ_WRITE;
+    if (!(flags & PTE_EXEC))
+        result |= ARCH_FLAG_EXECUTE_DISABLE;
+    return result;
+}
+
 bool pte_is_present(pte_t* pte) {
     return *pte & ARCH_FLAG_PRESENT;
 }
 
-bool pte_is_dir(pte_t* pte) {
-    return *pte & ARCH_FLAG_SIZE;
-}
-
 phys_t pte_address(pte_t* pte) {
     return (phys_t)(*pte & ARCH_PTE_ADDR_MASK);
 }

arch/x86_64/random.c

@@ -0,0 +1,36 @@
+#include <zinnia/initgraph.h>
+#include <zinnia/print.h>
+#include <zinnia/random.h>
+#include <x86_64/asm.h>
+
+#define CPUID_1C_RDRAND (1u << 30)
+#define RDRAND_RETRIES  10
+
+static bool rdrand_get_u64(struct random_source*, uint64_t* out) {
+    for (int i = 0; i < RDRAND_RETRIES; i++) {
+        uint64_t value;
+        uint8_t ok;
+        asm volatile("rdrand %0\n setc %1" : "=r"(value), "=qm"(ok) : : "cc");
+        if (ok) {
+            *out = value;
+            return true;
+        }
+    }
+    return false;
+}
+
+static struct random_source rdrand_source = {
+    .name = "x86_64.rdrand",
+    .quality = 100,
+    .get_u64 = rdrand_get_u64,
+};
+
+static void rdrand_init(struct boot_info*) {
+    if (asm_cpuid(1, 0).ecx & CPUID_1C_RDRAND) {
+        random_register(&rdrand_source);
+    } else {
+        log_warn("random: This CPU doesn't support RDRAND, ignoring\n");
+    }
+}
+INIT_TASK(rdrand_stage, "arch.x86_64.rdrand", rdrand_init);
+INIT_ENTAILS(rdrand_stage, random_stage);

arch/x86_64/sched.c

@@ -1,6 +1,8 @@
 #include <zinnia/alloc.h>
+#include <zinnia/assert.h>
 #include <zinnia/compiler.h>
 #include <zinnia/errno.h>
+#include <zinnia/irq.h>
 #include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/sched.h>
@@ -9,6 +11,8 @@
 #include <bits/sched.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <string.h>
+#include <x86_64/apic.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
@@ -25,30 +29,30 @@ struct task_frame {
 static struct task* perform_switch(uint64_t* old_rsp, uint64_t new_rsp, struct task* prev) {
     asm volatile(
         "sub rsp, 0x30\n" // Make room for all regs (except RIP).
-        "mov [rsp + %c0], rbx\n"
-        "mov [rsp + %c1], rbp\n"
-        "mov [rsp + %c2], r12\n"
-        "mov [rsp + %c3], r13\n"
-        "mov [rsp + %c4], r14\n"
-        "mov [rsp + %c5], r15\n"
+        "mov [rsp + %c[rbx]], rbx\n"
+        "mov [rsp + %c[rbp]], rbp\n"
+        "mov [rsp + %c[r12]], r12\n"
+        "mov [rsp + %c[r13]], r13\n"
+        "mov [rsp + %c[r14]], r14\n"
+        "mov [rsp + %c[r15]], r15\n"
         "mov [rdi], rsp\n" // rdi = old_rsp
         "mov rsp, rsi\n"   // rsi = new_rsp
-        "mov rbx, [rsp + %c0]\n"
-        "mov rbp, [rsp + %c1]\n"
-        "mov r12, [rsp + %c2]\n"
-        "mov r13, [rsp + %c3]\n"
-        "mov r14, [rsp + %c4]\n"
-        "mov r15, [rsp + %c5]\n"
+        "mov rbx, [rsp + %c[rbx]]\n"
+        "mov rbp, [rsp + %c[rbp]]\n"
+        "mov r12, [rsp + %c[r12]]\n"
+        "mov r13, [rsp + %c[r13]]\n"
+        "mov r14, [rsp + %c[r14]]\n"
+        "mov r15, [rsp + %c[r15]]\n"
         "add rsp, 0x30\n"
         "mov rax, rdx\n"
         "ret" // This will conveniently move us to the RIP we put at this stack entry.
         :
-        : "i"(offsetof(struct task_frame, rbx)),
-          "i"(offsetof(struct task_frame, rbp)),
-          "i"(offsetof(struct task_frame, r12)),
-          "i"(offsetof(struct task_frame, r13)),
-          "i"(offsetof(struct task_frame, r14)),
-          "i"(offsetof(struct task_frame, r15))
+        : [rbx] "i"(offsetof(struct task_frame, rbx)),
+          [rbp] "i"(offsetof(struct task_frame, rbp)),
+          [r12] "i"(offsetof(struct task_frame, r12)),
+          [r13] "i"(offsetof(struct task_frame, r13)),
+          [r14] "i"(offsetof(struct task_frame, r14)),
+          [r15] "i"(offsetof(struct task_frame, r15))
         : "memory"
     );
 }
@@ -58,9 +62,9 @@ struct task* arch_sched_switch(struct task* from, struct task* to) {
     struct arch_task_context* from_ctx = &from->context;
     struct arch_task_context* to_ctx = &to->context;
 
-    cpu->tss.rsp0 = to->kernel_stack + KERNEL_STACK_SIZE;
+    cpu->tss.rsp0 = kernel_stack_top(&to->kernel_stack);
 
-    if (from->user_stack) {
+    if (from_ctx->fpu_region != nullptr) {
         cpu->fpu_save(from_ctx->fpu_region);
         from_ctx->ds = asm_read_ds();
         from_ctx->es = asm_read_es();
@@ -70,7 +74,7 @@ struct task* arch_sched_switch(struct task* from, struct task* to) {
         from_ctx->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
     }
 
-    if (to->user_stack) {
+    if (to_ctx->fpu_region != nullptr) {
         cpu->fpu_restore(to_ctx->fpu_region);
         asm_write_ds(to_ctx->ds);
         asm_write_es(to_ctx->es);
@@ -90,15 +94,22 @@ struct task* arch_sched_switch(struct task* from, struct task* to) {
     return perform_switch(&from_ctx->rsp, to_ctx->rsp, from);
 }
 
+constexpr size_t FPU_FCW_OFFSET = 0;
+constexpr size_t FPU_MXCSR_OFFSET = 24;
+constexpr uint16_t FPU_DEFAULT_FCW = 0x37F;
+constexpr uint32_t FPU_DEFAULT_MXCSR = 0x1F80;
+
+// Calls task_entry with the correct parameters.
 [[__naked]]
 void task_entry_thunk() {
     asm volatile(
         "mov rdi, rbx\n"
         "mov rsi, r12\n"
-        "jmp %0"
+        "mov rdx, rax\n"
+        "jmp task_entry"
         :
-        : "r"(task_entry)
-        : "memory"
+        :
+        : "memory", "rdi", "rsi", "rdx"
     );
 }
 
@@ -111,8 +122,10 @@ errno_t arch_task_init(
 ) {
     struct arch_percpu* cpu = percpu_ref(arch_data);
 
+    assert((stack_start & 0xF) == 0, "Task stack is not 16-byte aligned!");
+
     // Prepare a dummy stack with an entry point function to return to.
-    struct task_frame* frame = (struct task_frame*)stack_start - 1;
+    struct task_frame* frame = (struct task_frame*)(stack_start - sizeof(uint64_t)) - 1;
     frame->rbx = (uint64_t)entry;
     frame->r12 = arg0;
     frame->rip = (uint64_t)task_entry_thunk;
@@ -124,17 +137,21 @@ errno_t arch_task_init(
         uintptr_t fpu_base = vm_alloc(fpu_size);
         if (!fpu_base)
             return ENOMEM;
-        for (size_t i = 0; i < fpu_size; i += arch_mem_page_size()) {
-            phys_t page;
-            errno_t s = mem_phys_alloc(1, 0, &page);
-            if (s) {
-                vm_free(fpu_base, fpu_size);
-                return s;
-            }
-
-            pmap_map(&kernel_vma.pmap, (uintptr_t)fpu_base, page, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
+        phys_t pages;
+        errno_t s = mem_phys_alloc(ALIGN_UP(fpu_size, arch_mem_page_size()) / arch_mem_page_size(), 0, &pages);
+        if (s) {
+            vm_free(fpu_base, fpu_size);
+            return s;
         }
+        for (size_t i = 0; i < fpu_size; i += arch_mem_page_size())
+            pmap_map(&kernel_vma.pmap, fpu_base + i, pages + i, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
         context->fpu_region = (void*)fpu_base;
+        context->fpu_phys = pages;
+
+        const uint16_t fcw = FPU_DEFAULT_FCW;
+        const uint32_t mxcsr = FPU_DEFAULT_MXCSR;
+        memcpy((uint8_t*)fpu_base + FPU_FCW_OFFSET, &fcw, sizeof(fcw));
+        memcpy((uint8_t*)fpu_base + FPU_MXCSR_OFFSET, &mxcsr, sizeof(mxcsr));
 
         context->ds = asm_read_ds();
         context->es = asm_read_es();
@@ -147,6 +164,37 @@ errno_t arch_task_init(
     return 0;
 }
 
+void arch_task_fini(struct arch_task_context* context) {
+    if (context->fpu_region == nullptr)
+        return;
+
+    const size_t page_size = arch_mem_page_size();
+    const size_t fpu_size = percpu_ref(arch_data)->fpu_size;
+    const uintptr_t fpu_base = (uintptr_t)context->fpu_region;
+    for (size_t i = 0; i < fpu_size; i += page_size)
+        pmap_unmap(&kernel_vma.pmap, fpu_base + i);
+    pmap_shootdown(&kernel_vma.pmap);
+    mem_phys_free(context->fpu_phys, ALIGN_UP(fpu_size, page_size) / page_size);
+    vm_free(fpu_base, fpu_size);
+
+    context->fpu_region = nullptr;
+}
+
+void arch_task_fork(struct arch_task_context* dst) {
+    if (dst->fpu_region)
+        percpu_ref(arch_data)->fpu_save(dst->fpu_region);
+}
+
+void arch_context_init_user(struct arch_context* ctx, uintptr_t ip, uintptr_t sp) {
+    *ctx = (struct arch_context){
+        .rip = ip,
+        .rsp = sp,
+        .rflags = 0x202,
+        .cs = offsetof(struct gdt, user_code64) | CPL_USER,
+        .ss = offsetof(struct gdt, user_data) | CPL_USER,
+    };
+}
+
 [[__naked]]
 void arch_sched_preempt_disable() {
     asm volatile(
@@ -170,6 +218,23 @@ bool arch_sched_preempt_enable() {
     );
 }
 
+void arch_sched_kick(struct percpu* cpu) {
+    if (!cpu->online)
+        return;
+    // The xAPIC ICR takes two writes, which must not be interleaved with an IPI sent by an interrupt handler.
+    irq_lock();
+    lapic_send_ipi(
+        &percpu_ref(arch_data)->lapic,
+        percpu_ref_on(cpu, arch_data)->lapic.id,
+        IDT_IPI_RESCHED,
+        APIC_DELIVERY_FIXED,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    irq_unlock();
+}
+
 [[__naked]]
 void arch_sched_jump_to_context(struct arch_context* context) {
     asm volatile(
@@ -189,14 +254,14 @@ void arch_sched_jump_to_user(uintptr_t ip, uintptr_t sp, uintptr_t arg) {
     };
 
     uintptr_t percpu = asm_rdmsr(MSR_GS_BASE);
-    const uint16_t zero = 0;
     asm volatile(
         "mov ds, %0\n"
         "mov es, %0\n"
         "mov fs, %0\n"
         "mov gs, %0\n"
         :
-        : "r"(zero)
+        : "a"(0)
+        : "memory"
     );
 
     asm_wrmsr(MSR_FS_BASE, 0);

arch/x86_64/serial.c

@@ -1,4 +1,9 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/console.h>
+#include <zinnia/init.h>
+#include <zinnia/initgraph.h>
+#include <arch/cpu.h>
 #include <stdint.h>
 #include <x86_64/asm.h>
 
@@ -11,7 +16,7 @@ static bool is_ready() {
 static void serial_write(struct console* con, const char* buf, size_t count) {
     for (size_t i = 0; i < count; i++) {
         while (!is_ready())
-            asm volatile("pause");
+            arch_cpu_pause();
 
         asm_outb(COM1_BASE, buf[i]);
     }
@@ -44,4 +49,10 @@ static struct console serial_con = {
     .write = serial_write,
     .init = serial_setup,
 };
-DEFINE_CONSOLE(&serial_con);
+
+[[__init]]
+static void serial_init(struct boot_info*) {
+    console_register(&serial_con);
+}
+INIT_TASK(serial_stage, "arch.x86_64.serial", serial_init);
+INIT_ENTAILS(serial_stage, console_stage);

arch/x86_64/smp.c

@@ -1,17 +1,359 @@
+#define pr_fmt(fmt) "smp: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/atomic.h>
+#include <zinnia/clock.h>
 #include <zinnia/compiler.h>
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
 #include <zinnia/types.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <uacpi/acpi.h>
+#include <uacpi/tables.h>
+#include <x86_64/apic.h>
+#include <x86_64/defs.h>
+#include <x86_64/percpu.h>
+#include <x86_64/pmap.h>
 #include <x86_64/smp.h>
 
 extern uint8_t smp_trampoline_start[];
+extern uint8_t smp_trampoline_data[];
 extern uint8_t smp_trampoline_end[];
 
-[[noreturn]]
-static void ap_entry(phys_t info) {
-    // TODO
+[[__naked, __used]]
+static void smp_trampoline() {
+    asm volatile(
+        ".pushsection .rodata\n"
+        ".code16\n"
+        ".global smp_trampoline_start\n"
+        "smp_trampoline_start:\n"
+        "    cli\n"
+        "    cld\n"
+        "    jmp .Lmode16\n"
 
-    while (1) {
-        asm volatile("hlt");
-    }
+        ".org smp_trampoline_start + %c[data_off]\n"
+        ".global smp_trampoline_data\n"
+        "smp_trampoline_data:\n"
+        "    .skip %c[info_size]\n"
+
+        ".set .Ldata_offset, %c[data_off]\n"
+        ".set .Lgdtr_offset, (.Ldata_offset + %c[gdtr])\n"
+        ".set .Lfarjmp_offset, (.Ldata_offset + %c[farjmp])\n"
+        ".set .Ltemp_stack_offset, (.Ldata_offset + %c[temp_stack])\n"
+        ".set .Ltemp_cr3_offset, (.Ldata_offset + %c[temp_cr3])\n"
+        ".set .Lentry_offset, (.Ldata_offset + %c[entry])\n"
+        ".set .Lhhdm_offset, (.Ldata_offset + %c[hhdm])\n"
+        ".set .Linvalid_idtr_offset, (.Linvalid_idtr - smp_trampoline_start)\n"
+        ".set .Lmode32_offset, (.Lmode32 - smp_trampoline_start)\n"
+        ".set .Lmode64_offset, (.Lmode64 - smp_trampoline_start)\n"
+
+        ".Lmode16:\n"
+        "    mov bx, cs\n"
+        "    shl ebx, 4\n"
+
+        "    lidtd cs:.Linvalid_idtr_offset\n"
+        "    lgdtd cs:.Lgdtr_offset\n"
+
+        "    lea eax, [ebx + .Lmode32_offset]\n"
+        "    mov dword ptr cs:.Lfarjmp_offset, eax\n"
+
+        "    mov eax, %c[cr0]\n"
+        "    mov cr0, eax\n"
+        "    jmp fword ptr cs:.Lfarjmp_offset\n"
+
+        ".code32\n"
+        ".Lmode32:\n"
+        "    mov ax, %c[kernel32_ds]\n"
+        "    mov ds, ax\n"
+        "    mov es, ax\n"
+        "    mov fs, ax\n"
+        "    mov gs, ax\n"
+        "    mov ss, ax\n"
+
+        "    xor eax, eax\n"
+        "    lldt ax\n"
+
+        "    mov esp, dword ptr [ebx + .Ltemp_stack_offset]\n"
+
+        "    xor eax, eax\n"
+        "    mov cr4, eax\n"
+
+        "    or eax, %c[cr4_pae]\n"
+        "    mov cr4, eax\n"
+
+        "    mov ecx, %c[msr_efer]\n"
+        "    mov eax, %c[efer_lme]\n"
+        "    xor edx, edx\n"
+        "    wrmsr\n"
+
+        "    mov eax, dword ptr [ebx + .Ltemp_cr3_offset]\n"
+        "    mov cr3, eax\n"
+
+        "    mov eax, cr0\n"
+        "    or eax, %c[cr0_pg]\n"
+        "    mov cr0, eax\n"
+
+        "    lea eax, [ebx + .Lmode64_offset]\n"
+        "    push %c[kernel64_cs]\n"
+        "    push eax\n"
+        "    retf\n"
+
+        ".code64\n"
+        ".Lmode64:\n"
+        "    mov ax, %c[kernel64_ds]\n"
+        "    mov ds, ax\n"
+        "    mov es, ax\n"
+        "    mov fs, ax\n"
+        "    mov gs, ax\n"
+        "    mov ss, ax\n"
+
+        "    mov ecx, %c[msr_efer]\n"
+        "    rdmsr\n"
+        "    or eax, %c[efer_nxe]\n"
+        "    wrmsr\n"
+
+        "    xor ebp, ebp\n"
+
+        "    lea rdi, [rbx + .Ldata_offset]\n"
+        "    mov rax, [rbx + .Lentry_offset]\n"
+        "    add rsp, [rbx + .Lhhdm_offset]\n"
+        "    jmp rax\n"
+
+        ".Linvalid_idtr:\n"
+        "    .word 0\n"
+        "    .quad 0\n"
+
+        ".global smp_trampoline_end\n"
+        "smp_trampoline_end:\n"
+        ".popsection\n"
+        "    ud2\n"
+        :
+        : [data_off] "i"(8),
+          [info_size] "i"(sizeof(struct smp_info)),
+          [gdtr] "i"(offsetof(struct smp_info, gdtr)),
+          [farjmp] "i"(offsetof(struct smp_info, farjmp_offset)),
+          [temp_stack] "i"(offsetof(struct smp_info, temp_stack)),
+          [temp_cr3] "i"(offsetof(struct smp_info, temp_cr3)),
+          [entry] "i"(offsetof(struct smp_info, entry)),
+          [hhdm] "i"(offsetof(struct smp_info, hhdm_offset)),
+          [kernel32_ds] "i"(offsetof(struct gdt, kernel_data32)),
+          [kernel64_cs] "i"(offsetof(struct gdt, kernel_code64)),
+          [kernel64_ds] "i"(offsetof(struct gdt, kernel_data64)),
+          [cr0] "i"(CR0_PE | CR0_ET),
+          [cr0_pg] "i"((uint32_t)CR0_PG),
+          [cr4_pae] "i"(CR4_PAE),
+          [msr_efer] "i"(MSR_EFER),
+          [efer_lme] "i"(MSR_EFER_LME),
+          [efer_nxe] "i"(MSR_EFER_NXE)
+    );
 }
 
-void x86_64_smp_init(uint32_t id) {}
+constexpr size_t AP_BOOT_STACK_SIZE = 0x8000;
+constexpr uint64_t AP_BOOT_TIMEOUT_NS = 1'000'000'000;
+
+[[noreturn]]
+static void ap_entry(phys_t info_addr) {
+    // We're still running on the temporary page table.
+    pmap_set(&kernel_vma.pmap);
+
+    struct smp_info* info = HHDM_PTR(info_addr);
+    struct percpu* cpu = (struct percpu*)info->percpu;
+
+    x86_64_cpu_setup(cpu);
+    irq_lock();
+
+    // This task keeps running on the boot stack, but is never scheduled again after the first yield.
+    sched_init(&cpu->sched);
+
+    // Let the BSP know that we're alive.
+    // `info` must not be touched after this point.
+    atomic_store(&info->booted, 1, ATOMIC_RELEASE);
+
+    log("CPU %zu online\n", cpu->id);
+
+    sched_yield(&cpu->sched);
+    unreachable();
+}
+
+static bool start_ap(phys_t trampoline, uint32_t temp_cr3, uint32_t lapic_id) {
+    const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
+
+    phys_t stack;
+    if (mem_phys_alloc(AP_BOOT_STACK_SIZE / arch_mem_page_size(), ALLOC_MEM32 | ALLOC_NOZERO, &stack)) {
+        log_err("Failed to allocate a boot stack for LAPIC %u\n", lapic_id);
+        return false;
+    }
+
+    struct percpu* cpu = percpu_new();
+    if (!cpu) {
+        log_err("Failed to allocate per-CPU data for LAPIC %u\n", lapic_id);
+        mem_phys_free(stack, AP_BOOT_STACK_SIZE / arch_mem_page_size());
+        return false;
+    }
+
+    log("Starting CPU %zu\n", cpu->id);
+
+    memcpy(HHDM_PTR(trampoline), smp_trampoline_start, trampoline_size);
+
+    const phys_t info_addr = trampoline + (smp_trampoline_data - smp_trampoline_start);
+    struct smp_info* info = HHDM_PTR(info_addr);
+    memcpy(&info->gdt, &gdt_base, sizeof(info->gdt));
+    info->gdtr.limit = sizeof(struct gdt) - 1;
+    info->gdtr.base = (struct gdt*)(info_addr + offsetof(struct smp_info, gdt));
+    info->farjmp_offset = 0; // Filled in by the trampoline.
+    info->farjmp_segment = offsetof(struct gdt, kernel_code32);
+    info->temp_stack = stack + AP_BOOT_STACK_SIZE;
+    info->temp_cr3 = temp_cr3;
+    info->entry = (uint64_t)ap_entry;
+    info->hhdm_offset = mem_hhdm_base;
+    info->percpu = (uint64_t)cpu;
+    info->lapic_id = lapic_id;
+    atomic_init(&info->booted, 0);
+
+    struct local_apic* lapic = &percpu_ref(arch_data)->lapic;
+
+    lapic_send_ipi(lapic, lapic_id, 0, APIC_DELIVERY_INIT, APIC_DEST_PHYSICAL, APIC_LEVEL_ASSERT, APIC_TRIGGER_EDGE);
+    clock_spin_ns(10'000'000);
+
+    lapic_send_ipi(
+        lapic,
+        lapic_id,
+        trampoline >> 12,
+        APIC_DELIVERY_STARTUP,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    clock_spin_ns(10'000'000);
+
+    for (uint64_t waited = 0; !atomic_load(&info->booted, ATOMIC_ACQUIRE); waited += 1'000'000) {
+        if (waited >= AP_BOOT_TIMEOUT_NS) {
+            // The per-CPU data and boot stack are leaked on purpose, in case the AP still comes up.
+            log_err("CPU %zu (LAPIC %u) did not respond\n", cpu->id, lapic_id);
+            return false;
+        }
+        clock_spin_ns(1'000'000);
+    }
+
+    return true;
+}
+
+static errno_t build_temp_table(phys_t* out) {
+    const size_t pgsz = arch_mem_page_size();
+
+    phys_t mem;
+    errno_t status = mem_phys_alloc(6, ALLOC_MEM32, &mem);
+    if (status)
+        return status;
+
+    uint64_t* l4 = HHDM_PTR(mem);
+    uint64_t* l3 = HHDM_PTR(mem + pgsz);
+    const phys_t l2_base = mem + 2 * pgsz;
+
+    // Use 2 MiB pages.
+    l4[0] = (mem + pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+    for (size_t i = 0; i < 4; i++) {
+        uint64_t* l2 = HHDM_PTR(l2_base + i * pgsz);
+        l3[i] = (l2_base + i * pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+        for (size_t j = 0; j < 512; j++)
+            l2[j] = ((i << 30) + (j << 21)) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE | ARCH_FLAG_SIZE;
+    }
+
+    // Copy over the higher half from the kernel page table.
+    const uint64_t* kernel_l4 = HHDM_PTR(kernel_vma.pmap.root);
+    for (size_t i = 256; i < 512; i++)
+        l4[i] = kernel_l4[i];
+
+    *out = mem;
+    return 0;
+}
+
+void x86_64_smp_init() {
+    const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
+    assert(trampoline_size <= arch_mem_page_size(), "SMP trampoline doesn't fit in a page");
+
+    const size_t max_cpus = percpu_max_cpus();
+    if (max_cpus <= 1)
+        return;
+
+    uacpi_table table;
+    if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) {
+        log_warn("No MADT found, not starting any APs\n");
+        return;
+    }
+
+    phys_t trampoline;
+    if (mem_phys_alloc(1, ALLOC_MEM20, &trampoline)) {
+        log_err("Failed to allocate 20-bit memory for the AP trampoline\n");
+        uacpi_table_unref(&table);
+        return;
+    }
+
+    phys_t temp_table;
+    if (build_temp_table(&temp_table)) {
+        log_err("Failed to allocate a temporary page table\n");
+        mem_phys_free(trampoline, 1);
+        uacpi_table_unref(&table);
+        return;
+    }
+
+    const uint32_t bsp_id = percpu_ref(arch_data)->lapic.id;
+    bool all_booted = true;
+    size_t num_cpus = 1;
+
+    const struct acpi_madt* madt = table.ptr;
+    const uint8_t* entry = (const uint8_t*)madt->entries;
+    const uint8_t* end = (const uint8_t*)madt + madt->hdr.length;
+
+    while (entry + sizeof(struct acpi_entry_hdr) <= end && num_cpus < max_cpus) {
+        const struct acpi_entry_hdr* hdr = (const struct acpi_entry_hdr*)entry;
+
+        // Don't trust malformed firmware tables.
+        if (hdr->length < sizeof(struct acpi_entry_hdr) || entry + hdr->length > end) {
+            log_warn("MADT contains a malformed entry\n");
+            break;
+        }
+
+        uint32_t lapic_id = 0;
+        bool enabled = false;
+        switch (hdr->type) {
+        case ACPI_MADT_ENTRY_TYPE_LAPIC:
+            const struct acpi_madt_lapic* lapic = (const struct acpi_madt_lapic*)hdr;
+            lapic_id = lapic->id;
+            enabled = lapic->flags & ACPI_PIC_ENABLED;
+            break;
+        case ACPI_MADT_ENTRY_TYPE_LOCAL_X2APIC:
+            const struct acpi_madt_x2apic* x2apic = (const struct acpi_madt_x2apic*)hdr;
+            lapic_id = x2apic->id;
+            enabled = x2apic->flags & ACPI_PIC_ENABLED;
+            break;
+        }
+
+        if (enabled && lapic_id != bsp_id) {
+            if (start_ap(trampoline, temp_table, lapic_id))
+                num_cpus++;
+            else
+                all_booted = false;
+        }
+
+        entry += hdr->length;
+    }
+
+    uacpi_table_unref(&table);
+
+    // An AP which didn't respond might still be using these.
+    if (all_booted) {
+        mem_phys_free(temp_table, 6);
+        mem_phys_free(trampoline, 1);
+    }
+
+    log("%zu CPU(s) online\n", num_cpus);
+}

arch/x86_64/syscall.c

@@ -12,17 +12,17 @@
 void arch_syscall_stub() {
     asm volatile(
         "    swapgs\n"
-        "    mov gs:%c0, rsp\n" // user_stack
-        "    mov rsp, gs:%c1\n" // kernel_stack
+        "    mov gs:%c[user_stack], rsp\n"
+        "    mov rsp, gs:%c[kernel_stack]\n"
         "    cld\n"
         "1:\n"
-        "    push %c3\n" // ss
-        "    push gs:%c0\n"
-        "    push r11\n"  // rflags
-        "    push %c2\n"  // cs
-        "    push rcx\n"  // rip
-        "    push 0x00\n" // arch_context::error
-        "    push 0x00\n" // arch_context::isr
+        "    push %c[user_data]\n" // ss
+        "    push gs:%c[user_stack]\n"
+        "    push r11\n"             // rflags
+        "    push %c[user_code64]\n" // cs
+        "    push rcx\n"             // rip
+        "    push 0x00\n"            // arch_context::error
+        "    push 0x00\n"            // arch_context::isr
         "    push rax\n"
         "    push rbx\n"
         "    push rcx\n"
@@ -39,8 +39,9 @@ void arch_syscall_stub() {
         "    push r14\n"
         "    push r15\n"
         "    xor rbp, rbp\n"
-        "    mov rdi, rsp\n" // SysV ABI first argument.
-        "    call %4\n"      // Call syscall handler.
+        "    mov rdi, rsp\n"          // SysV ABI first argument.
+        "    sti\n"
+        "    call syscall_dispatch\n" // Call syscall handler.
         "    cli\n"
         "    pop r15\n"
         "    pop r14\n"
@@ -58,15 +59,16 @@ void arch_syscall_stub() {
         "    pop rbx\n"
         "    pop rax\n"
         "    add rsp, 0x10\n" // Skip .error and .isr fields
-        "2:\n"
-        "    mov rsp, gs:%c0\n" // user_stack
+        "    pop rcx\n"       // rip
+        "    add rsp, 0x08\n" // Skip cs
+        "    pop r11\n"       // rflags
+        "    pop rsp\n"       // rsp
         "    swapgs\n"
         "    sysretq\n"
         :
-        : "i"(offsetof(struct percpu, user_stack)),          // 0
-          "i"(offsetof(struct percpu, kernel_stack)),        // 1
-          "i"(offsetof(struct gdt, user_code64) | CPL_USER), // 2
-          "i"(offsetof(struct gdt, user_data) | CPL_USER),   // 3
-          "r"(syscall_dispatch)                              // 4
+        : [user_stack] "i"(offsetof(struct percpu, user_stack)),
+          [kernel_stack] "i"(offsetof(struct percpu, kernel_stack)),
+          [user_code64] "i"(offsetof(struct gdt, user_code64) | CPL_USER),
+          [user_data] "i"(offsetof(struct gdt, user_data) | CPL_USER)
     );
 }

arch/x86_64/trampoline.S

@@ -1,107 +0,0 @@
-#include <x86_64/smp.h>
-#include <x86_64/defs.h>
-
-.intel_syntax noprefix
-
-.global smp_trampoline_start
-.global smp_trampoline_end
-
-.section .rodata
-.code16
-
-smp_trampoline_start:
-    cli
-    cld
-    jmp 1f
-
-smp_trampoline_data:
-    .skip SMP_INFO_SIZE
-
-.set data_offset, (smp_trampoline_data - smp_trampoline_start)
-.set gdtr_offset, (data_offset + SMP_GDTR_OFFSET)
-.set farjmp_offset, (data_offset + SMP_FARJMP_OFFSET)
-.set temp_stack_offset, (data_offset + SMP_TEMP_STACK_OFFSET)
-.set temp_cr3_offset, (data_offset + SMP_TEMP_CR3_OFFSET)
-.set entry_offset, (data_offset + SMP_ENTRY_OFFSET)
-.set hhdm_offset, (data_offset + SMP_HHDM_OFFSET)
-
-1:
-    mov bx, cs
-    shl ebx, 4
-
-.set idtr_offset, (invalid_idtr - smp_trampoline_start)
-    lidtd cs:idtr_offset
-    lgdtd cs:gdtr_offset
-
-.set mode32_offset, (mode32 - smp_trampoline_start)
-    lea eax, [ebx + mode32_offset]
-    mov dword ptr cs:farjmp_offset, eax
-
-    mov eax, CR0_PE | CR0_ET
-    mov cr0, eax
-    jmp fword ptr cs:farjmp_offset
-
-.code32
-mode32:
-    mov ax, SMP_KERNEL32_DS
-    mov ds, ax
-    mov es, ax
-    mov fs, ax
-    mov gs, ax
-    mov ss, ax
-
-    xor  eax, eax
-    lldt ax
-
-    mov esp, dword ptr [ebx + temp_stack_offset]
-
-    xor eax, eax
-    mov cr4, eax
-
-    or  eax, CR4_PAE
-    mov cr4, eax
-
-    mov ecx, MSR_EFER
-    mov eax, MSR_EFER_LME
-    xor edx, edx
-    wrmsr
-
-    mov eax, dword ptr [ebx + temp_cr3_offset]
-    mov cr3, eax
-
-    mov eax, cr0
-    or  eax, CR0_PG
-    mov cr0, eax
-
-.set mode64_offset, (mode64 - smp_trampoline_start)
-    lea eax, [ebx + mode64_offset]
-    push SMP_KERNEL64_CS
-    push eax
-    retf
-
-.code64
-mode64:
-    mov ax, SMP_KERNEL64_DS
-    mov ds, ax
-    mov es, ax
-    mov fs, ax
-    mov gs, ax
-    mov ss, ax
-
-    mov ecx, MSR_EFER
-    rdmsr
-    or eax, MSR_EFER_NXE
-    wrmsr
-
-    xor ebp, ebp
-
-    lea rdi, [rbx + data_offset]
-    mov rax, [rbx + entry_offset]
-    add rsp, [rbx + hhdm_offset]
-    jmp rax
-
-invalid_idtr:
-    .word 0
-    .quad 0
-
-smp_trampoline_end:

arch/x86_64/tsc.c

@@ -1,3 +1,4 @@
+#define pr_fmt(fmt) "x86_64/tsc: " fmt
 #include <zinnia/clock.h>
 #include <zinnia/cmdline.h>
 #include <zinnia/print.h>
@@ -42,19 +43,22 @@ static struct clock tsc_clock = {
 };
 
 void tsc_init() {
-    if (!use_tsc)
+    if (!use_tsc) {
+        log_warn("TSC disabled on the command line");
         return;
+    }
 
     if (!(asm_cpuid(1, 0).edx & CPUID_1D_TSC) || !(asm_cpuid(0x8000'0007, 0).edx & (1 << 8))) {
-        log("x86_64/tsc: No invariant TSC detected!\n");
+        log_warn("No invariant TSC detected!\n");
         return;
     }
 
     // Check for the TSC info leaf.
     bool has_tsc_leaf = false;
     struct cpuid tsc_leaf = {0};
-    if (asm_cpuid(0x8000'0000, 0).eax >= 0x15) {
+    if (asm_cpuid(0, 0).eax >= 0x15) {
         tsc_leaf = asm_cpuid(0x15, 0);
+        has_tsc_leaf = true;
     }
 
     if (clock_available()) {
@@ -68,17 +72,17 @@ void tsc_init() {
         // On a normal system, this should usually never be called and is a last resort
         // since at this point we have at least the HPET timer.
         if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) {
-            log("x86_64/tsc: Calibrating TSC using CPUID 0x15");
-            tsc_freq = tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
+            log("Calibrating TSC using CPUID 0x15");
+            tsc_freq = (uint64_t)tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
         } else {
-            log("x86_64/tsc: Unable to calibrate TSC using CPUID frequency information");
+            log_err("Unable to calibrate TSC using CPUID frequency information");
             return;
         }
     } else {
-        log("x86_64/tsc: Unable to calibrate the TSC without a clock or static frequency info!\n");
+        log_err("Unable to calibrate the TSC without a clock or static frequency info!\n");
         return;
     }
 
-    log("x86_64/tsc: Timer frequency is %lu MHz\n", tsc_freq / 1'000'000);
+    log("Timer frequency is %lu MHz\n", tsc_freq / 1'000'000);
     clock_switch(&tsc_clock);
 }

arch/x86_64/usercopy.c

@@ -1,7 +1,12 @@
 #include <zinnia/compiler.h>
+#include <zinnia/percpu.h>
 #include <arch/usercopy.h>
+#include <x86_64/percpu.h>
 
 void arch_usercopy_allow(bool state) {
+    if (!percpu_ref(arch_data)->can_smap)
+        return;
+
     if (state) {
         asm volatile("stac" ::: "memory");
     } else {
@@ -24,7 +29,7 @@ bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct us
     asm volatile(
         // Setup regs for `rep movsb`.
         "    xchg rcx, rdx\n"
-        "    mov rax, [rip + %0]\n"
+        "    lea rax, [rip + %0]\n"
         "    xchg [rdx], rax\n"
 
         ".global x86_64_read_start\n"
@@ -41,7 +46,7 @@ bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct us
         ".global x86_64_read_fault\n"
         "x86_64_read_fault:\n"
         "    xor rax, rax\n"
-        "    xchg [rdx], rax\n"
+        "    mov [rdx], rax\n"
         "    ret\n"
         :
         : "i"(&read_region)
@@ -64,7 +69,7 @@ bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct u
     asm volatile(
         // Setup regs for `rep movsb`.
         "    xchg rcx, rdx\n"
-        "    mov rax, [rip + %0]\n"
+        "    lea rax, [rip + %0]\n"
         "    mov [rdx], rax\n"
 
         ".global x86_64_write_start\n"
@@ -81,7 +86,7 @@ bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct u
         ".global x86_64_write_fault\n"
         "x86_64_write_fault:\n"
         "    xor rax, rax\n"
-        "    xchg [rdx], rax\n"
+        "    mov [rdx], rax\n"
         "    ret\n"
         :
         : "i"(&write_region)
@@ -102,18 +107,19 @@ static const struct usercopy_region strlen_region = {
 [[__naked]]
 bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region) {
     asm volatile(
-        "    mov rax, [rip + %0]\n"
+        "    lea rax, [rip + %0]\n"
         "    xchg [rcx], rax\n"
 
         ".global x86_64_strlen_start\n"
         "    x86_64_strlen_start:\n"
         "    xor r8, r8\n"
         ".Lloop:\n"
+        "    cmp r8, rsi\n"
+        "    je .Lleave\n"
         "    cmp byte ptr [rdi + r8], 0\n"
         "    je .Lleave\n"
         "    inc r8\n"
-        "    cmp rsi, r8\n"
-        "    jne .Lloop\n"
+        "    jmp .Lloop\n"
         ".Lleave:\n"
         "    mov [rdx], r8\n"
 
@@ -127,7 +133,7 @@ bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struc
         ".global x86_64_strlen_fault\n"
         "x86_64_strlen_fault:\n"
         "    xor rax, rax\n"
-        "    xchg [rcx], rax\n"
+        "    mov [rcx], rax\n"
         "    ret\n"
         :
         : "i"(&strlen_region)

boot/Kconfig

@@ -0,0 +1,19 @@
+config HAVE_EFISTUB
+  bool
+
+menu "Boot loaders"
+
+config BOOT_EFISTUB
+  bool "EFI stub"
+  depends on HAVE_EFISTUB
+  default y
+  help
+    Kernel loader for UEFI systems.
+
+config BOOT_LIMINE
+  bool "Limine"
+  default n
+  help
+    ELF loader with the Limine protocol.
+
+endmenu

boot/efi/efi.h

@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "efi_arch.h"
+
+typedef uint64_t efi_status;
+typedef void* efi_handle;
+typedef uint16_t char16;
+
+constexpr efi_status EFI_SUCCESS = 0;
+constexpr efi_status EFI_ERROR = 1ull << 63;
+constexpr efi_status EFI_LOAD_ERROR = EFI_ERROR | 1;
+constexpr efi_status EFI_INVALID_PARAMETER = EFI_ERROR | 2;
+constexpr efi_status EFI_UNSUPPORTED = EFI_ERROR | 3;
+constexpr efi_status EFI_BUFFER_TOO_SMALL = EFI_ERROR | 5;
+constexpr efi_status EFI_OUT_OF_RESOURCES = EFI_ERROR | 9;
+constexpr efi_status EFI_NOT_FOUND = EFI_ERROR | 14;
+
+struct efi_guid {
+    uint32_t data1;
+    uint16_t data2;
+    uint16_t data3;
+    uint8_t data4[8];
+};
+
+enum efi_memory_type : uint32_t {
+    EFI_RESERVED_MEMORY_TYPE,
+    EFI_LOADER_CODE,
+    EFI_LOADER_DATA,
+    EFI_BOOT_SERVICES_CODE,
+    EFI_BOOT_SERVICES_DATA,
+    EFI_RUNTIME_SERVICES_CODE,
+    EFI_RUNTIME_SERVICES_DATA,
+    EFI_CONVENTIONAL_MEMORY,
+    EFI_UNUSABLE_MEMORY,
+    EFI_ACPI_RECLAIM_MEMORY,
+    EFI_ACPI_MEMORY_NVS,
+    EFI_MEMORY_MAPPED_IO,
+    EFI_MEMORY_MAPPED_IO_PORT_SPACE,
+    EFI_PAL_CODE,
+    EFI_PERSISTENT_MEMORY,
+    EFI_UNACCEPTED_MEMORY,
+};
+
+enum efi_allocate_type : uint32_t {
+    EFI_ALLOCATE_ANY_PAGES,
+    EFI_ALLOCATE_MAX_ADDRESS,
+    EFI_ALLOCATE_ADDRESS,
+};
+
+enum efi_locate_search_type : uint32_t {
+    EFI_LOCATE_ALL_HANDLES,
+    EFI_LOCATE_BY_REGISTER_NOTIFY,
+    EFI_LOCATE_BY_PROTOCOL,
+};
+
+struct efi_memory_descriptor {
+    uint32_t type;
+    uint64_t physical_start;
+    uint64_t virtual_start;
+    uint64_t number_of_pages;
+    uint64_t attribute;
+};
+
+struct efi_table_header {
+    uint64_t signature;
+    uint32_t revision;
+    uint32_t header_size;
+    uint32_t crc32;
+    uint32_t reserved;
+};
+
+struct efi_simple_text_output_protocol {
+    void* reset;
+    efi_status(EFIAPI* output_string)(struct efi_simple_text_output_protocol* self, const char16* string);
+};
+
+struct efi_boot_services {
+    struct efi_table_header hdr;
+    void* raise_tpl;
+    void* restore_tpl;
+    efi_status(EFIAPI* allocate_pages)(
+        enum efi_allocate_type type,
+        enum efi_memory_type memory_type,
+        size_t pages,
+        uint64_t* memory
+    );
+    efi_status(EFIAPI* free_pages)(uint64_t memory, size_t pages);
+    efi_status(EFIAPI* get_memory_map)(
+        size_t* map_size,
+        struct efi_memory_descriptor* map,
+        size_t* map_key,
+        size_t* descriptor_size,
+        uint32_t* descriptor_version
+    );
+    efi_status(EFIAPI* allocate_pool)(enum efi_memory_type pool_type, size_t size, void** buffer);
+    efi_status(EFIAPI* free_pool)(void* buffer);
+    void* create_event;
+    void* set_timer;
+    void* wait_for_event;
+    void* signal_event;
+    void* close_event;
+    void* check_event;
+    void* install_protocol_interface;
+    void* reinstall_protocol_interface;
+    void* uninstall_protocol_interface;
+    efi_status(EFIAPI* handle_protocol)(efi_handle handle, const struct efi_guid* protocol, void** interface);
+    void* reserved;
+    void* register_protocol_notify;
+    void* locate_handle;
+    void* locate_device_path;
+    void* install_configuration_table;
+    void* load_image;
+    void* start_image;
+    void* exit;
+    void* unload_image;
+    efi_status(EFIAPI* exit_boot_services)(efi_handle image_handle, size_t map_key);
+    void* get_next_monotonic_count;
+    void* stall;
+    void* set_watchdog_timer;
+    void* connect_controller;
+    void* disconnect_controller;
+    void* open_protocol;
+    void* close_protocol;
+    void* open_protocol_information;
+    void* protocols_per_handle;
+    efi_status(EFIAPI* locate_handle_buffer)(
+        enum efi_locate_search_type search_type,
+        const struct efi_guid* protocol,
+        void* search_key,
+        size_t* no_handles,
+        efi_handle** buffer
+    );
+    efi_status(EFIAPI* locate_protocol)(const struct efi_guid* protocol, void* registration, void** interface);
+};
+
+struct efi_configuration_table {
+    struct efi_guid vendor_guid;
+    void* vendor_table;
+};
+
+struct efi_system_table {
+    struct efi_table_header hdr;
+    char16* firmware_vendor;
+    uint32_t firmware_revision;
+    efi_handle console_in_handle;
+    void* con_in;
+    efi_handle console_out_handle;
+    struct efi_simple_text_output_protocol* con_out;
+    efi_handle standard_error_handle;
+    void* std_err;
+    void* runtime_services;
+    struct efi_boot_services* boot_services;
+    size_t number_of_table_entries;
+    struct efi_configuration_table* configuration_table;
+};
+
+struct efi_loaded_image_protocol {
+    uint32_t revision;
+    efi_handle parent_handle;
+    struct efi_system_table* system_table;
+    efi_handle device_handle;
+    void* file_path;
+    void* reserved;
+    uint32_t load_options_size;
+    void* load_options;
+    void* image_base;
+    uint64_t image_size;
+    enum efi_memory_type image_code_type;
+    enum efi_memory_type image_data_type;
+    void* unload;
+};
+
+constexpr uint64_t EFI_FILE_MODE_READ = 1;
+
+struct efi_file_protocol {
+    uint64_t revision;
+    efi_status(EFIAPI* open)(
+        struct efi_file_protocol* self,
+        struct efi_file_protocol** new_handle,
+        const char16* file_name,
+        uint64_t open_mode,
+        uint64_t attributes
+    );
+    efi_status(EFIAPI* close)(struct efi_file_protocol* self);
+    void* delete;
+    efi_status(EFIAPI* read)(struct efi_file_protocol* self, size_t* buffer_size, void* buffer);
+    void* write;
+    efi_status(EFIAPI* get_position)(struct efi_file_protocol* self, uint64_t* position);
+    efi_status(EFIAPI* set_position)(struct efi_file_protocol* self, uint64_t position);
+};
+
+struct efi_simple_file_system_protocol {
+    uint64_t revision;
+    efi_status(EFIAPI* open_volume)(struct efi_simple_file_system_protocol* self, struct efi_file_protocol** root);
+};
+
+enum efi_pixel_format : uint32_t {
+    EFI_PIXEL_RGBX_8,
+    EFI_PIXEL_BGRX_8,
+    EFI_PIXEL_BIT_MASK,
+    EFI_PIXEL_BLT_ONLY,
+};
+
+struct efi_pixel_bitmask {
+    uint32_t red;
+    uint32_t green;
+    uint32_t blue;
+    uint32_t reserved;
+};
+
+struct efi_gop_mode_info {
+    uint32_t version;
+    uint32_t horizontal_resolution;
+    uint32_t vertical_resolution;
+    enum efi_pixel_format pixel_format;
+    struct efi_pixel_bitmask pixel_information;
+    uint32_t pixels_per_scan_line;
+};
+
+struct efi_gop_mode {
+    uint32_t max_mode;
+    uint32_t mode;
+    struct efi_gop_mode_info* info;
+    size_t size_of_info;
+    uint64_t frame_buffer_base;
+    size_t frame_buffer_size;
+};
+
+struct efi_graphics_output_protocol {
+    void* query_mode;
+    void* set_mode;
+    void* blt;
+    struct efi_gop_mode* mode;
+};
+
+static const struct efi_guid EFI_LOADED_IMAGE_PROTOCOL_GUID =
+    {0x5b1b'31a1, 0x9562, 0x11d2, {0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}};
+static const struct efi_guid EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID =
+    {0x964e'5b22, 0x6459, 0x11d2, {0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}};
+static const struct efi_guid EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID =
+    {0x9042'a9de, 0x23dc, 0x4a38, {0x96, 0xfb, 0x7a, 0xde, 0xd0, 0x80, 0x51, 0x6a}};
+static const struct efi_guid EFI_ACPI_20_TABLE_GUID =
+    {0x8868'e871, 0xe4f1, 0x11d3, {0xbc, 0x22, 0x00, 0x80, 0xc7, 0x3c, 0x88, 0x81}};
+static const struct efi_guid EFI_ACPI_10_TABLE_GUID =
+    {0xeb9d'2d30, 0x2d88, 0x11d3, {0x9a, 0x16, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d}};

boot/efi/header.c

@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/compiler.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "efi_arch.h"
+#include "pe.h"
+
+extern const char __pe_code_size[];
+extern const char __pe_data_sizes[];
+extern const char __pe_entry[];
+extern const char __pe_image_sizes[];
+extern const char __pe_base_relocs[];
+extern const char __pe_text_virtual[];
+extern const char __pe_text_raw[];
+extern const char __pe_reloc_virtual[];
+extern const char __pe_reloc_raw[];
+extern const char __pe_rodata_virtual[];
+extern const char __pe_rodata_raw[];
+extern const char __pe_data_virtual[];
+extern const char __pe_data_raw[];
+
+struct efistub_pe_header {
+    struct pe_dos_header dos;
+    uint32_t signature;
+    struct pe_coff_header coff;
+    struct pe_optional_header64 optional;
+    struct pe_section_header sections[4];
+};
+
+static_assert(offsetof(struct efistub_pe_header, optional) % 8 == 0);
+static_assert(offsetof(struct efistub_pe_header, sections) % 8 == 0);
+
+[[__used, __section(".header")]]
+const struct efistub_pe_header efistub_pe_header = {
+    .dos =
+        {
+            .e_magic = PE_DOS_MAGIC,
+            .e_lfanew = offsetof(struct efistub_pe_header, signature),
+        },
+    .signature = PE_SIGNATURE,
+    .coff =
+        {
+            .machine = EFI_ARCH_PE_MACHINE,
+            .number_of_sections = 4,
+            .size_of_optional_header = sizeof(struct pe_optional_header64),
+            .characteristics = PE_FILE_EXECUTABLE_IMAGE | PE_FILE_LINE_NUMS_STRIPPED | PE_FILE_DEBUG_STRIPPED,
+        },
+    .optional =
+        {
+            .magic_and_code_size = (uintptr_t)__pe_code_size + PE_OPTIONAL_MAGIC_PE32_PLUS,
+            .data_sizes = (uintptr_t)__pe_data_sizes,
+            .entry_and_code_base = (uintptr_t)__pe_entry,
+            .section_alignment = 0x1000,
+            .file_alignment = 0x1000,
+            .image_and_header_sizes = (uintptr_t)__pe_image_sizes,
+            .subsystem = PE_SUBSYSTEM_EFI_APPLICATION,
+            .dll_characteristics = PE_DLL_NX_COMPAT,
+            .number_of_rva_and_sizes = PE_NUM_DATA_DIRECTORIES,
+            .data_directories =
+                {
+                    [PE_DIRECTORY_BASE_RELOC] = {.layout = (uintptr_t)__pe_base_relocs},
+                },
+        },
+    .sections = {
+        {
+            .name = ".text",
+            .virtual_layout = (uintptr_t)__pe_text_virtual,
+            .raw_layout = (uintptr_t)__pe_text_raw,
+            .characteristics = PE_SCN_CNT_CODE | PE_SCN_MEM_EXECUTE | PE_SCN_MEM_READ,
+        },
+        {
+            .name = ".reloc",
+            .virtual_layout = (uintptr_t)__pe_reloc_virtual,
+            .raw_layout = (uintptr_t)__pe_reloc_raw,
+            .characteristics = PE_SCN_CNT_INITIALIZED_DATA | PE_SCN_MEM_DISCARDABLE | PE_SCN_MEM_READ,
+        },
+        {
+            .name = ".rdata",
+            .virtual_layout = (uintptr_t)__pe_rodata_virtual,
+            .raw_layout = (uintptr_t)__pe_rodata_raw,
+            .characteristics = PE_SCN_CNT_INITIALIZED_DATA | PE_SCN_MEM_READ,
+        },
+        {
+            .name = ".data",
+            .virtual_layout = (uintptr_t)__pe_data_virtual,
+            .raw_layout = (uintptr_t)__pe_data_raw,
+            .characteristics = PE_SCN_CNT_INITIALIZED_DATA | PE_SCN_MEM_READ | PE_SCN_MEM_WRITE,
+        },
+    },
+};
+
+[[__used, __section(".reloc")]]
+const struct pe_base_reloc_block efistub_base_relocs = {
+    .page_rva = 0,
+    .block_size = sizeof(struct pe_base_reloc_block),
+};

boot/efi/main.c

@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: ISC
+
+#include <string.h>
+#include "stub.h"
+
+constexpr size_t MEM_MAP_SLACK = 64;
+
+struct efi_boot_services* bs;
+
+bool loader_alloc(enum phys_mem_usage usage, size_t size, uint64_t* out) {
+    enum efi_memory_type efi_type = EFI_LOADER_DATA;
+    if (usage == PHYS_STATIC)
+        efi_type = MEM_STATIC;
+    else if (usage == PHYS_RECLAIMABLE)
+        efi_type = MEM_RECLAIMABLE;
+
+    const size_t pages = ALIGN_UP(size, PAGE_SIZE) / PAGE_SIZE;
+    if (bs->allocate_pages(EFI_ALLOCATE_ANY_PAGES, efi_type, pages, out))
+        return false;
+    memset((void*)*out, 0, pages * PAGE_SIZE);
+    return true;
+}
+
+void* loader_ptr(uint64_t phys) {
+    return (void*)phys;
+}
+
+static efi_status open_file_on(efi_handle handle, const char16* path, struct efi_file_protocol** out) {
+    struct efi_simple_file_system_protocol* fs;
+    efi_status status = bs->handle_protocol(handle, &EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID, (void**)&fs);
+    if (status)
+        return status;
+
+    struct efi_file_protocol* root;
+    status = fs->open_volume(fs, &root);
+    if (status)
+        return status;
+
+    status = root->open(root, out, path, EFI_FILE_MODE_READ, 0);
+    if (status && path[0] == '\\')
+        status = root->open(root, out, path + 1, EFI_FILE_MODE_READ, 0);
+    root->close(root);
+    return status;
+}
+
+static efi_status read_file(struct efi_file_protocol* file, enum phys_mem_usage usage, uint64_t* base, uint64_t* size) {
+    uint64_t total;
+    uint64_t done = 0;
+    efi_status status = file->set_position(file, UINT64_MAX);
+    if (!status)
+        status = file->get_position(file, &total);
+    if (!status)
+        status = file->set_position(file, 0);
+    if (!status && !loader_alloc(usage, total ? total : 1, base))
+        status = EFI_OUT_OF_RESOURCES;
+    while (!status && done < total) {
+        size_t chunk = total - done;
+        status = file->read(file, &chunk, (void*)(*base + done));
+        if (!chunk)
+            break;
+        done += chunk;
+    }
+    file->close(file);
+    *size = done;
+    return status;
+}
+
+static void color_from_mask(uint32_t mask, struct boot_color* out) {
+    if (!mask)
+        return;
+    out->shift = __builtin_ctz(mask);
+    out->size = __builtin_ctzll(~(uint64_t)(mask >> out->shift));
+}
+
+static enum phys_mem_usage convert_mem_type(uint32_t type) {
+    switch (type) {
+    case EFI_LOADER_CODE:
+    case EFI_LOADER_DATA:
+    case EFI_BOOT_SERVICES_CODE:
+    case EFI_BOOT_SERVICES_DATA:
+    case EFI_CONVENTIONAL_MEMORY:
+        return PHYS_USABLE;
+    case EFI_ACPI_RECLAIM_MEMORY:
+    case MEM_RECLAIMABLE:
+        return PHYS_RECLAIMABLE;
+    case MEM_STATIC:
+        return PHYS_STATIC;
+    default:
+        return PHYS_RESERVED;
+    }
+}
+
+efi_status efi_main(efi_handle image_handle, struct efi_system_table* st) {
+    bs = st->boot_services;
+
+    // Check if we support this configuration.
+    if (!arch_check_cpu())
+        return EFI_UNSUPPORTED;
+
+    struct efi_loaded_image_protocol* image;
+    efi_status status = bs->handle_protocol(image_handle, &EFI_LOADED_IMAGE_PROTOCOL_GUID, (void**)&image);
+    if (status)
+        return status;
+
+    struct kernel_image kernel;
+    if (!loader_load_kernel(&kernel, "EFISTUB"))
+        return EFI_LOAD_ERROR;
+    struct boot_info* info = kernel.info;
+
+    // Reserve room for the memory map.
+    size_t map_size = 0;
+    size_t map_key;
+    size_t desc_size;
+    uint32_t desc_version;
+    status = bs->get_memory_map(&map_size, nullptr, &map_key, &desc_size, &desc_version);
+    if (status != EFI_BUFFER_TOO_SMALL)
+        return status;
+
+    const size_t map_capacity = map_size + MEM_MAP_SLACK * desc_size;
+    uint64_t map;
+    if (!loader_alloc(PHYS_USABLE, map_capacity, &map))
+        return EFI_OUT_OF_RESOURCES;
+
+    uint64_t mem;
+    if (!loader_alloc(PHYS_USABLE, map_capacity / desc_size * sizeof(struct phys_mem), &mem))
+        return EFI_OUT_OF_RESOURCES;
+
+    // Use the load options as the command line.
+    const char16* opts = image->load_options;
+    const size_t opts_len = image->load_options_size / sizeof(char16);
+    size_t cmdline_len = 0;
+    for (size_t i = 0; opts && i < opts_len && opts[i]; i++) {
+        if (opts[i] < 0x20 || opts[i] > 0x7e || cmdline_len + 1 >= sizeof(info->cmdline)) {
+            cmdline_len = 0;
+            break;
+        }
+        info->cmdline[cmdline_len++] = (char)opts[i];
+    }
+    info->cmdline[cmdline_len] = 0;
+
+    // Without load options, fall back to a command line file on the boot device.
+    struct efi_file_protocol* cmdline_file;
+    if (!cmdline_len && !open_file_on(image->device_handle, u"\\EFI\\zinnia\\cmdline", &cmdline_file)) {
+        uint64_t base;
+        uint64_t size;
+        status = read_file(cmdline_file, PHYS_USABLE, &base, &size);
+        if (status)
+            return status;
+
+        const char* text = (const char*)base;
+        while (cmdline_len < size && text[cmdline_len] && text[cmdline_len] != '\n' && text[cmdline_len] != '\r') {
+            if (text[cmdline_len] < 0x20 || text[cmdline_len] > 0x7e)
+                return EFI_INVALID_PARAMETER;
+            if (cmdline_len + 1 >= sizeof(info->cmdline))
+                return EFI_BUFFER_TOO_SMALL;
+            info->cmdline[cmdline_len] = text[cmdline_len];
+            cmdline_len++;
+        }
+        while (cmdline_len && info->cmdline[cmdline_len - 1] == ' ')
+            cmdline_len--;
+        info->cmdline[cmdline_len] = 0;
+    }
+
+    // Load every initrd= file.
+    const char* cmdline = info->cmdline;
+    for (size_t i = 0; cmdline[i];) {
+        if (cmdline[i] == ' ') {
+            i++;
+            continue;
+        }
+
+        const size_t start = i;
+        bool quoted = false;
+        while (cmdline[i] && (quoted || cmdline[i] != ' ')) {
+            if (cmdline[i] == '"')
+                quoted = !quoted;
+            i++;
+        }
+        const char* token = cmdline + start;
+        const size_t len = i - start;
+        if (len <= 7 || memcmp(token, "initrd=", 7))
+            continue;
+
+        const size_t path_len = len - 7;
+        if (path_len >= BOOT_PATH_MAX)
+            return EFI_INVALID_PARAMETER;
+        char path[BOOT_PATH_MAX];
+        char16 wide[BOOT_PATH_MAX];
+        for (size_t j = 0; j < path_len; j++) {
+            path[j] = token[7 + j];
+            wide[j] = path[j] == '/' ? '\\' : (char16)path[j];
+        }
+        path[path_len] = 0;
+        wide[path_len] = 0;
+
+        // Try the boot device first, then all other file systems.
+        struct efi_file_protocol* file;
+        status = open_file_on(image->device_handle, wide, &file);
+        size_t num_handles;
+        efi_handle* handles;
+        if (status &&
+            !bs->locate_handle_buffer(
+                EFI_LOCATE_BY_PROTOCOL,
+                &EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID,
+                nullptr,
+                &num_handles,
+                &handles
+            )) {
+            for (size_t j = 0; j < num_handles && status; j++) {
+                if (handles[j] != image->device_handle)
+                    status = open_file_on(handles[j], wide, &file);
+            }
+            bs->free_pool(handles);
+        }
+        if (status)
+            return status;
+
+        uint64_t base;
+        uint64_t size;
+        status = read_file(file, PHYS_STATIC, &base, &size);
+        if (status)
+            return status;
+
+        if (!loader_add_file(&kernel, path, base, size))
+            return EFI_OUT_OF_RESOURCES;
+    }
+
+    // Framebuffer
+    struct efi_graphics_output_protocol* gop;
+    if (!bs->locate_protocol(&EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID, nullptr, (void**)&gop) && gop && gop->mode &&
+        gop->mode->info) {
+        const struct efi_gop_mode_info* mode = gop->mode->info;
+        struct boot_framebuffer* fb = &info->fb;
+        switch (mode->pixel_format) {
+        case EFI_PIXEL_RGBX_8:
+            fb->bytes_per_pixel = 4;
+            fb->red = (struct boot_color){0, 8};
+            fb->green = (struct boot_color){8, 8};
+            fb->blue = (struct boot_color){16, 8};
+            break;
+        case EFI_PIXEL_BGRX_8:
+            fb->bytes_per_pixel = 4;
+            fb->red = (struct boot_color){16, 8};
+            fb->green = (struct boot_color){8, 8};
+            fb->blue = (struct boot_color){0, 8};
+            break;
+        case EFI_PIXEL_BIT_MASK: {
+            const struct efi_pixel_bitmask* m = &mode->pixel_information;
+            const uint32_t all = m->red | m->green | m->blue | m->reserved;
+            if (!all)
+                break;
+            fb->bytes_per_pixel = (32 - __builtin_clz(all) + 7) / 8;
+            color_from_mask(m->red, &fb->red);
+            color_from_mask(m->green, &fb->green);
+            color_from_mask(m->blue, &fb->blue);
+            break;
+        }
+        default:
+            break;
+        }
+
+        if (fb->bytes_per_pixel) {
+            fb->base = gop->mode->frame_buffer_base;
+            fb->width = mode->horizontal_resolution;
+            fb->height = mode->vertical_resolution;
+            fb->pitch = mode->pixels_per_scan_line * fb->bytes_per_pixel;
+        }
+    }
+
+    // Find the RSDP.
+    for (size_t i = 0; i < st->number_of_table_entries; i++) {
+        const struct efi_configuration_table* table = &st->configuration_table[i];
+        if (!memcmp(&table->vendor_guid, &EFI_ACPI_20_TABLE_GUID, sizeof(struct efi_guid))) {
+            info->rsdp = (uint64_t)table->vendor_table;
+            break;
+        }
+        if (!memcmp(&table->vendor_guid, &EFI_ACPI_10_TABLE_GUID, sizeof(struct efi_guid)))
+            info->rsdp = (uint64_t)table->vendor_table;
+    }
+
+    // Map everything up to the highest non-reserved address.
+    map_size = map_capacity;
+    status = bs->get_memory_map(&map_size, (struct efi_memory_descriptor*)map, &map_key, &desc_size, &desc_version);
+    if (status)
+        return status;
+
+    uint64_t top = 0;
+    for (size_t off = 0; off + desc_size <= map_size; off += desc_size) {
+        const struct efi_memory_descriptor* desc = (const struct efi_memory_descriptor*)(map + off);
+        if (convert_mem_type(desc->type) != PHYS_RESERVED)
+            top = MAX(top, desc->physical_start + desc->number_of_pages * PAGE_SIZE);
+    }
+
+    uint64_t root;
+    if (!loader_page_tables(&kernel, top, (uint64_t)arch_enter_kernel, &root))
+        return EFI_OUT_OF_RESOURCES;
+
+    // Exit boot services.
+    for (size_t attempt = 0;; attempt++) {
+        map_size = map_capacity;
+        status = bs->get_memory_map(&map_size, (struct efi_memory_descriptor*)map, &map_key, &desc_size, &desc_version);
+        if (status)
+            return status;
+        status = bs->exit_boot_services(image_handle, map_key);
+        if (!status)
+            break;
+        if (attempt == 8)
+            return status;
+    }
+
+    // Hand the final memory map to the kernel.
+    struct phys_mem* mem_map = (struct phys_mem*)mem;
+    size_t num_mem = 0;
+    for (size_t off = 0; off + desc_size <= map_size; off += desc_size) {
+        const struct efi_memory_descriptor* desc = (const struct efi_memory_descriptor*)(map + off);
+        mem_map[num_mem++] = (struct phys_mem){
+            .address = desc->physical_start,
+            .length = desc->number_of_pages * PAGE_SIZE,
+            .usage = convert_mem_type(desc->type),
+        };
+    }
+    loader_set_mem_map(&kernel, mem_map, num_mem);
+
+    arch_enter_kernel(root, kernel.entry);
+}

boot/efi/meson.build

@@ -0,0 +1,42 @@
+efistub_sources = files(
+    'header.c',
+    'main.c',
+    'reloc.c',
+)
+
+efistub_lds = meson.current_source_dir() / 'stub.ld'
+
+subdir(target_machine.cpu_family())
+
+efistub_elf = executable(
+    'efistub.elf',
+    sources: [loader_sources, efistub_sources, loader_kernel],
+    include_directories: [
+        common_includes,
+        loader_includes,
+        include_directories('.', target_machine.cpu_family()),
+    ],
+    c_args: loader_c_args,
+    link_args: [
+        '-nostdlib',
+        '-Wl,-T,' + efistub_lds,
+        '-Wl,--no-dynamic-linker',
+        '-Wl,-z,text',
+        '-Wl,-z,norelro',
+        '-Wl,-z,noexecstack',
+        '-Wl,-z,max-page-size=0x1000',
+        '-Wl,--build-id=none',
+    ],
+    link_depends: efistub_lds,
+    pie: true,
+)
+
+zinnia_efi = custom_target(
+    'zinnia.efi',
+    input: efistub_elf,
+    output: 'zinnia.efi',
+    command: [objcopy, '-O', 'binary', '@INPUT@', '@OUTPUT@'],
+    build_by_default: true,
+    install: true,
+    install_dir: zinnia_bootdir,
+)

boot/efi/pe.h

@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+constexpr uint16_t PE_DOS_MAGIC = 0x5a4d;
+constexpr uint32_t PE_SIGNATURE = 0x0000'4550;
+constexpr uint16_t PE_OPTIONAL_MAGIC_PE32_PLUS = 0x020b;
+constexpr uint16_t PE_SUBSYSTEM_EFI_APPLICATION = 10;
+constexpr size_t PE_NUM_DATA_DIRECTORIES = 16;
+constexpr size_t PE_DIRECTORY_BASE_RELOC = 5;
+
+enum pe_file_characteristics : uint16_t {
+    PE_FILE_EXECUTABLE_IMAGE = 0x0002,
+    PE_FILE_LINE_NUMS_STRIPPED = 0x0004,
+    PE_FILE_DEBUG_STRIPPED = 0x0200,
+};
+
+enum pe_dll_characteristics : uint16_t {
+    PE_DLL_NX_COMPAT = 0x0100,
+};
+
+enum pe_section_characteristics : uint32_t {
+    PE_SCN_CNT_CODE = 0x0000'0020,
+    PE_SCN_CNT_INITIALIZED_DATA = 0x0000'0040,
+    PE_SCN_MEM_DISCARDABLE = 0x0200'0000,
+    PE_SCN_MEM_EXECUTE = 0x2000'0000,
+    PE_SCN_MEM_READ = 0x4000'0000,
+    PE_SCN_MEM_WRITE = 0x8000'0000,
+};
+
+struct pe_dos_header {
+    uint16_t e_magic;
+    uint16_t e_cblp;
+    uint16_t e_cp;
+    uint16_t e_crlc;
+    uint16_t e_cparhdr;
+    uint16_t e_minalloc;
+    uint16_t e_maxalloc;
+    uint16_t e_ss;
+    uint16_t e_sp;
+    uint16_t e_csum;
+    uint16_t e_ip;
+    uint16_t e_cs;
+    uint16_t e_lfarlc;
+    uint16_t e_ovno;
+    uint16_t e_res[4];
+    uint16_t e_oemid;
+    uint16_t e_oeminfo;
+    uint16_t e_res2[10];
+    uint32_t e_lfanew;
+};
+
+struct pe_coff_header {
+    uint16_t machine;
+    uint16_t number_of_sections;
+    uint32_t time_date_stamp;
+    uint32_t pointer_to_symbol_table;
+    uint32_t number_of_symbols;
+    uint16_t size_of_optional_header;
+    uint16_t characteristics;
+};
+
+struct pe_data_directory {
+    union {
+        struct {
+            uint32_t virtual_address;
+            uint32_t size;
+        };
+        uint64_t layout;
+    };
+};
+
+struct pe_optional_header64 {
+    union {
+        struct {
+            uint16_t magic;
+            uint8_t major_linker_version;
+            uint8_t minor_linker_version;
+            uint32_t size_of_code;
+        };
+        uint64_t magic_and_code_size;
+    };
+    union {
+        struct {
+            uint32_t size_of_initialized_data;
+            uint32_t size_of_uninitialized_data;
+        };
+        uint64_t data_sizes;
+    };
+    union {
+        struct {
+            uint32_t address_of_entry_point;
+            uint32_t base_of_code;
+        };
+        uint64_t entry_and_code_base;
+    };
+    uint64_t image_base;
+    uint32_t section_alignment;
+    uint32_t file_alignment;
+    uint16_t major_operating_system_version;
+    uint16_t minor_operating_system_version;
+    uint16_t major_image_version;
+    uint16_t minor_image_version;
+    uint16_t major_subsystem_version;
+    uint16_t minor_subsystem_version;
+    uint32_t win32_version_value;
+    union {
+        struct {
+            uint32_t size_of_image;
+            uint32_t size_of_headers;
+        };
+        uint64_t image_and_header_sizes;
+    };
+    uint32_t checksum;
+    uint16_t subsystem;
+    uint16_t dll_characteristics;
+    uint64_t size_of_stack_reserve;
+    uint64_t size_of_stack_commit;
+    uint64_t size_of_heap_reserve;
+    uint64_t size_of_heap_commit;
+    uint32_t loader_flags;
+    uint32_t number_of_rva_and_sizes;
+    struct pe_data_directory data_directories[PE_NUM_DATA_DIRECTORIES];
+};
+
+struct pe_section_header {
+    char name[8];
+    union {
+        struct {
+            uint32_t virtual_size;
+            uint32_t virtual_address;
+        };
+        uint64_t virtual_layout;
+    };
+    union {
+        struct {
+            uint32_t size_of_raw_data;
+            uint32_t pointer_to_raw_data;
+        };
+        uint64_t raw_layout;
+    };
+    uint32_t pointer_to_relocations;
+    uint32_t pointer_to_line_numbers;
+    uint16_t number_of_relocations;
+    uint16_t number_of_line_numbers;
+    uint32_t characteristics;
+};
+
+struct pe_base_reloc_block {
+    uint32_t page_rva;
+    uint32_t block_size;
+    uint16_t entries[2];
+};
+
+static_assert(sizeof(struct pe_dos_header) == 64);
+static_assert(sizeof(struct pe_coff_header) == 20);
+static_assert(sizeof(struct pe_optional_header64) == 240);
+static_assert(offsetof(struct pe_optional_header64, size_of_image) == 56);
+static_assert(offsetof(struct pe_optional_header64, data_directories) == 112);
+static_assert(sizeof(struct pe_section_header) == 40);
+static_assert(sizeof(struct pe_base_reloc_block) == 12);

boot/efi/reloc.c

@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: ISC
+
+#include <bits/elf.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "efi.h"
+
+efi_status efistub_relocate(uintptr_t base, const elf64_dyn_t* dynamic) {
+    uintptr_t rela = 0;
+    size_t rela_size = 0;
+    size_t rela_ent = sizeof(elf64_rela_t);
+    uintptr_t relr = 0;
+    size_t relr_size = 0;
+
+    for (const elf64_dyn_t* dyn = dynamic; dyn->d_tag != DT_NULL; dyn++) {
+        switch (dyn->d_tag) {
+        case DT_RELA:
+            rela = base + dyn->d_un.d_val;
+            break;
+        case DT_RELASZ:
+            rela_size = dyn->d_un.d_val;
+            break;
+        case DT_RELAENT:
+            rela_ent = dyn->d_un.d_val;
+            break;
+        case DT_RELR:
+            relr = base + dyn->d_un.d_val;
+            break;
+        case DT_RELRSZ:
+            relr_size = dyn->d_un.d_val;
+            break;
+        }
+    }
+
+    for (size_t off = 0; rela && off + rela_ent <= rela_size; off += rela_ent) {
+        const elf64_rela_t* r = (const elf64_rela_t*)(rela + off);
+        const uint32_t type = ELF64_R_TYPE(r->r_info);
+        if (type == R_NONE)
+            continue;
+        if (type != R_RELATIVE)
+            return EFI_LOAD_ERROR;
+        *(uint64_t*)(base + r->r_offset) = base + r->r_addend;
+    }
+
+    uint64_t* where = nullptr;
+    for (size_t off = 0; relr && off + sizeof(uint64_t) <= relr_size; off += sizeof(uint64_t)) {
+        uint64_t entry = *(const uint64_t*)(relr + off);
+        if (!(entry & 1)) {
+            where = (uint64_t*)(base + entry);
+            *where++ += base;
+            continue;
+        }
+        for (size_t i = 0; (entry >>= 1) != 0; i++) {
+            if (entry & 1)
+                where[i] += base;
+        }
+        where += 63;
+    }
+
+    return EFI_SUCCESS;
+}

boot/efi/stub.h

@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "efi.h"
+#include "loader.h"
+
+constexpr enum efi_memory_type MEM_STATIC = (enum efi_memory_type)0x8000'0000;
+constexpr enum efi_memory_type MEM_RECLAIMABLE = (enum efi_memory_type)0x8000'0001;
+
+extern struct efi_boot_services* bs;

boot/efi/stub.ld

@@ -0,0 +1,100 @@
+ENTRY(_start)
+
+PHDRS
+{
+    text    PT_LOAD    FLAGS(5);
+    rodata  PT_LOAD    FLAGS(4);
+    data    PT_LOAD    FLAGS(6);
+    dynamic PT_DYNAMIC FLAGS(6);
+}
+
+SECTIONS
+{
+    . = 0;
+
+    .header : {
+        __image_base = .;
+        KEEP(*(.header))
+    } :text
+
+    . = ALIGN(0x1000);
+    .text : {
+        __text_start = .;
+        *(.text .text.*)
+        . = ALIGN(0x1000);
+        __text_end = .;
+    } :text
+
+    .reloc : {
+        __reloc_start = .;
+        KEEP(*(.reloc))
+        __reloc_data_end = .;
+        . = ALIGN(0x1000);
+        __reloc_end = .;
+    } :rodata
+
+    .rodata : {
+        __rodata_start = .;
+        *(.rodata .rodata.*)
+        . = ALIGN(16);
+        __kernel_start = .;
+        KEEP(*(.kernel))
+        __kernel_end = .;
+    } :rodata
+
+    .hash : { *(.hash) } :rodata
+    .gnu.hash : { *(.gnu.hash) } :rodata
+    .dynsym : { *(.dynsym) } :rodata
+    .dynstr : { *(.dynstr) } :rodata
+    .rela.dyn : { *(.rela.*) } :rodata
+    .relr.dyn : { *(.relr.dyn) } :rodata
+
+    .rodata.end : {
+        . = ALIGN(0x1000);
+        __rodata_end = .;
+    } :rodata
+
+    .dynamic : {
+        __data_start = .;
+        *(.dynamic)
+    } :data :dynamic
+
+    .data : {
+        *(.data .data.*)
+        *(.got .got.*)
+        . = ALIGN(0x1000);
+        __data_raw_end = .;
+    } :data
+
+    .bss (NOLOAD) : {
+        __bss_start = .;
+        *(.bss .bss.*)
+        *(COMMON)
+        . = ALIGN(0x1000);
+        __bss_end = .;
+    } :data
+
+    __image_end = .;
+
+    __pe_code_size = ABSOLUTE((__text_end - __text_start) << 32);
+    __pe_data_sizes = ABSOLUTE(((__reloc_end - __reloc_start) + (__rodata_end - __rodata_start) +
+                                (__data_raw_end - __data_start)) | ((__bss_end - __bss_start) << 32));
+    __pe_entry = ABSOLUTE(_start | (__text_start << 32));
+    __pe_image_sizes = ABSOLUTE(__image_end | (__text_start << 32));
+    __pe_base_relocs = ABSOLUTE(__reloc_start | ((__reloc_data_end - __reloc_start) << 32));
+    __pe_text_virtual = ABSOLUTE((__text_end - __text_start) | (__text_start << 32));
+    __pe_text_raw = ABSOLUTE((__text_end - __text_start) | (__text_start << 32));
+    __pe_reloc_virtual = ABSOLUTE((__reloc_data_end - __reloc_start) | (__reloc_start << 32));
+    __pe_reloc_raw = ABSOLUTE((__reloc_end - __reloc_start) | (__reloc_start << 32));
+    __pe_rodata_virtual = ABSOLUTE((__rodata_end - __rodata_start) | (__rodata_start << 32));
+    __pe_rodata_raw = ABSOLUTE((__rodata_end - __rodata_start) | (__rodata_start << 32));
+    __pe_data_virtual = ABSOLUTE((__bss_end - __data_start) | (__data_start << 32));
+    __pe_data_raw = ABSOLUTE((__data_raw_end - __data_start) | (__data_start << 32));
+
+    /DISCARD/ : {
+        *(.comment*)
+        *(.eh_frame*)
+        *(.note*)
+        *(.interp)
+    }
+}

boot/efi/x86_64/efi_arch.h

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stdint.h>
+
+#define EFIAPI __attribute__((ms_abi))
+
+constexpr uint16_t EFI_ARCH_PE_MACHINE = 0x8664;

boot/efi/x86_64/entry.S

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: ISC
+
+    .intel_syntax noprefix
+    .text
+
+    .globl _start
+    .type _start, @function
+_start:
+    // Align the stack pointer
+    push rbp
+    mov rbp, rsp
+    push rdi
+    push rsi
+    push rbx
+    push r12
+    and rsp, -16
+    cld
+
+    mov rbx, rcx
+    mov r12, rdx
+
+    // Clear .bss
+    lea rdi, [rip + __bss_start]
+    lea rcx, [rip + __bss_end]
+    sub rcx, rdi
+    xor eax, eax
+    rep stosb
+
+    // Self-relocate
+    lea rdi, [rip + __image_base]
+    lea rsi, [rip + _DYNAMIC]
+    call efistub_relocate
+    test rax, rax
+    jnz 1f
+
+    mov rdi, rbx
+    mov rsi, r12
+    call efi_main
+
+1:
+    lea rsp, [rbp - 32]
+    pop r12
+    pop rbx
+    pop rsi
+    pop rdi
+    pop rbp
+    ret
+    .size _start, . - _start

boot/efi/x86_64/meson.build

@@ -0,0 +1 @@
+efistub_sources += files('entry.S')

boot/kernel.ld

@@ -6,6 +6,8 @@ PHDRS {
     data    PT_LOAD     FLAGS(6);
     dynamic PT_DYNAMIC  FLAGS(6);
     percpu  PT_LOAD     FLAGS(6);
+    cmdline 0x60000005;
+    boot    0x60000008;
 }
 
 SECTIONS
@@ -28,11 +30,31 @@ SECTIONS
         __ld_cmdline_start = .;
         KEEP(*(.cmdline))
         __ld_cmdline_end = .;
-
-    } :rodata
+    } :rodata :cmdline
 
     .rodata : {
         *(.rodata .rodata.*)
+
+        /* Builtin module entry points */
+        . = ALIGN(8);
+        __ld_mod_entry_start = .;
+        KEEP(*(.mod.entry .mod.entry.*))
+        __ld_mod_entry_end = .;
+
+        . = ALIGN(8);
+        __ld_initgraph_nodes_start = .;
+        KEEP(*(.initgraph.nodes))
+        __ld_initgraph_nodes_end = .;
+
+        __ld_initgraph_edges_start = .;
+        KEEP(*(.initgraph.edges))
+        __ld_initgraph_edges_end = .;
+
+        . = ALIGN(8);
+        __ld_ktest_start = .;
+        KEEP(*(.ktest))
+        __ld_ktest_end = .;
+
         *(.got .got.*)
 
         __ld_rodata_end = .;
@@ -46,11 +68,9 @@ SECTIONS
         *(.data .data.*)
     } :data
 
-    .console : {
-        __ld_console_start = .;
-        KEEP(*(.console))
-        __ld_console_end = .;
-    } :data
+    .boot : {
+        KEEP(*(.boot.info))
+    } :data :boot
 
     .dynsym : {
         __ld_dynsym_start = .;

boot/limine/entry.c

@@ -1,91 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/cmdline.h>
-#include <zinnia/init.h>
-#include <zinnia/print.h>
-#include <zinnia/utils.h>
-#include <zinnia/string.h>
-#include "limine.h"
-
-[[__initdata_sorted("limine.0")]]
-static volatile uint64_t limine_start[] = LIMINE_REQUESTS_START_MARKER;
-[[__initdata_sorted("limine.1")]]
-static volatile uint64_t limine_revision[] = LIMINE_BASE_REVISION(6);
-[[__initdata_sorted("limine.2")]]
-static volatile uint64_t limine_end[] = LIMINE_REQUESTS_END_MARKER;
-
-#define LIMINE_REQUEST(request, tag, rev) \
-    [[__initdata_sorted("limine.1")]] \
-    static volatile struct limine_##request request = { \
-        .id = tag, \
-        .revision = rev, \
-        .response = nullptr, \
-    }
-
-LIMINE_REQUEST(memmap_request, LIMINE_MEMMAP_REQUEST_ID, 0);
-LIMINE_REQUEST(hhdm_request, LIMINE_HHDM_REQUEST_ID, 0);
-LIMINE_REQUEST(executable_address_request, LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID, 0);
-LIMINE_REQUEST(executable_file_request, LIMINE_EXECUTABLE_FILE_REQUEST_ID, 0);
-LIMINE_REQUEST(framebuffer_request, LIMINE_FRAMEBUFFER_REQUEST_ID, 1);
-LIMINE_REQUEST(module_request, LIMINE_MODULE_REQUEST_ID, 0);
-LIMINE_REQUEST(dtb_request, LIMINE_DTB_REQUEST_ID, 0);
-LIMINE_REQUEST(rsdp_request, LIMINE_RSDP_REQUEST_ID, 0);
-
-[[__initdata]]
-static struct phys_mem mem[128];
-[[__initdata]]
-static struct boot_file files[8];
-
-[[__init]]
-void kernel_entry() {
-    kernel_early_init();
-
-    struct boot_info info;
-
-    struct limine_memmap_response* mm_res = memmap_request.response;
-    struct limine_module_response* module_res = module_request.response;
-    struct limine_executable_address_response* exec_res = executable_address_request.response;
-    struct limine_executable_file_response* exec_file_res = executable_file_request.response;
-
-    info.cmdline = exec_file_res->executable_file->string;
-
-    info.num_mem_maps = MIN(ARRAY_SIZE(mem), mm_res->entry_count);
-    for (size_t i = 0; i < info.num_mem_maps; i++) {
-        mem[i].address = mm_res->entries[i]->base;
-        mem[i].length = mm_res->entries[i]->length;
-
-        switch (mm_res->entries[i]->type) {
-        case LIMINE_MEMMAP_USABLE:
-            mem[i].usage = PHYS_USABLE;
-            break;
-        case LIMINE_MEMMAP_ACPI_RECLAIMABLE:
-        case LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE:
-            mem[i].usage = PHYS_RECLAIMABLE;
-            break;
-        case LIMINE_MEMMAP_EXECUTABLE_AND_MODULES:
-            mem[i].usage = PHYS_STATIC;
-            break;
-        default:
-            mem[i].usage = PHYS_RESERVED;
-            break;
-        }
-    }
-
-    if (module_res) {
-        info.files = files;
-        info.num_files = MIN(ARRAY_SIZE(files), module_res->module_count);
-        for (size_t i = 0; i < info.num_files; i++) {
-            files[i].data = (phys_t)module_res->modules[i]->address - hhdm_request.response->offset;
-            files[i].length = module_res->modules[i]->size;
-            strncpy(files[i].path, module_res->modules[i]->path, sizeof(files[i].path));
-        }
-    }
-
-    info.mem_map = mem;
-    info.phys_base = (phys_t)exec_res->physical_base;
-    info.virt_base = (uintptr_t)exec_res->virtual_base;
-    info.hhdm_base = (uintptr_t)hhdm_request.response->offset;
-
-    info.rsdp = (phys_t)rsdp_request.response->address - info.hhdm_base;
-
-    kernel_main(&info);
-}

boot/limine/loader.ld

@@ -0,0 +1,46 @@
+ENTRY(loader_main)
+
+PHDRS
+{
+    text    PT_LOAD FLAGS(5);
+    rodata  PT_LOAD FLAGS(4);
+    data    PT_LOAD FLAGS(6);
+}
+
+SECTIONS
+{
+    . = 0xffffffffc0000000;
+
+    .text : {
+        *(.text .text.*)
+    } :text
+
+    . = ALIGN(0x1000);
+    .rodata : {
+        *(.rodata .rodata.*)
+        . = ALIGN(16);
+        __kernel_start = .;
+        KEEP(*(.kernel))
+        __kernel_end = .;
+    } :rodata
+
+    . = ALIGN(0x1000);
+    .data : {
+        KEEP(*(.limine_requests_start))
+        KEEP(*(.limine_requests))
+        KEEP(*(.limine_requests_end))
+        *(.data .data.*)
+    } :data
+
+    .bss : {
+        *(.bss .bss.*)
+        *(COMMON)
+    } :data
+
+    /DISCARD/ : {
+        *(.comment*)
+        *(.eh_frame*)
+        *(.note*)
+        *(.interp)
+    }
+}

boot/limine/main.c

@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/compiler.h>
+#include <string.h>
+#include "limine.h"
+#include "loader.h"
+
+[[__used, __section(".limine_requests_start")]]
+static volatile uint64_t limine_start[] = LIMINE_REQUESTS_START_MARKER;
+[[__used, __section(".limine_requests")]]
+static volatile uint64_t limine_revision[] = LIMINE_BASE_REVISION(6);
+[[__used, __section(".limine_requests_end")]]
+static volatile uint64_t limine_end[] = LIMINE_REQUESTS_END_MARKER;
+
+#define LIMINE_REQUEST(request, tag, rev) \
+    [[__used, __section(".limine_requests")]] \
+    static volatile struct limine_##request request = { \
+        .id = tag, \
+        .revision = rev, \
+        .response = nullptr, \
+    }
+
+LIMINE_REQUEST(bootloader_info_request, LIMINE_BOOTLOADER_INFO_REQUEST_ID, 0);
+LIMINE_REQUEST(memmap_request, LIMINE_MEMMAP_REQUEST_ID, 0);
+LIMINE_REQUEST(hhdm_request, LIMINE_HHDM_REQUEST_ID, 0);
+LIMINE_REQUEST(executable_address_request, LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID, 0);
+LIMINE_REQUEST(executable_file_request, LIMINE_EXECUTABLE_FILE_REQUEST_ID, 0);
+LIMINE_REQUEST(framebuffer_request, LIMINE_FRAMEBUFFER_REQUEST_ID, 1);
+LIMINE_REQUEST(module_request, LIMINE_MODULE_REQUEST_ID, 0);
+LIMINE_REQUEST(dtb_request, LIMINE_DTB_REQUEST_ID, 0);
+LIMINE_REQUEST(rsdp_request, LIMINE_RSDP_REQUEST_ID, 0);
+
+static uint64_t hhdm_offset;
+static const struct limine_memmap_entry* pool;
+static uint64_t pool_static_end;
+static uint64_t pool_reclaimable_start;
+
+void* loader_ptr(uint64_t phys) {
+    return (void*)(phys + hhdm_offset);
+}
+
+static uint64_t to_phys(const void* ptr) {
+    const uint64_t addr = (uint64_t)ptr;
+    return addr >= hhdm_offset ? addr - hhdm_offset : addr;
+}
+
+bool loader_alloc(enum phys_mem_usage usage, size_t size, uint64_t* out) {
+    size = ALIGN_UP(size, PAGE_SIZE);
+    if (pool_reclaimable_start - pool_static_end < size)
+        return false;
+
+    if (usage == PHYS_STATIC) {
+        *out = pool_static_end;
+        pool_static_end += size;
+    } else {
+        pool_reclaimable_start -= size;
+        *out = pool_reclaimable_start;
+    }
+    memset(loader_ptr(*out), 0, size);
+    return true;
+}
+
+static enum phys_mem_usage convert_mem_type(uint64_t type) {
+    switch (type) {
+    case LIMINE_MEMMAP_USABLE:
+        return PHYS_USABLE;
+    case LIMINE_MEMMAP_ACPI_RECLAIMABLE:
+    case LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE:
+        return PHYS_RECLAIMABLE;
+    case LIMINE_MEMMAP_EXECUTABLE_AND_MODULES:
+        return PHYS_STATIC;
+    default:
+        return PHYS_RESERVED;
+    }
+}
+
+static bool init_pool(const struct limine_memmap_response* memmap) {
+    for (size_t i = 0; i < memmap->entry_count; i++) {
+        const struct limine_memmap_entry* entry = memmap->entries[i];
+        if (entry->type == LIMINE_MEMMAP_USABLE && (!pool || entry->length > pool->length))
+            pool = entry;
+    }
+    if (!pool)
+        return false;
+
+    pool_static_end = pool->base;
+    pool_reclaimable_start = pool->base + pool->length;
+    return true;
+}
+
+static uint64_t mem_map_top(const struct limine_memmap_response* memmap) {
+    uint64_t top = 0;
+    for (size_t i = 0; i < memmap->entry_count; i++) {
+        const struct limine_memmap_entry* entry = memmap->entries[i];
+        if (convert_mem_type(entry->type) != PHYS_RESERVED)
+            top = MAX(top, entry->base + entry->length);
+    }
+    return top;
+}
+
+static size_t convert_mem_map(const struct limine_memmap_response* memmap, struct phys_mem* out) {
+    size_t n = 0;
+    for (size_t i = 0; i < memmap->entry_count; i++) {
+        const struct limine_memmap_entry* entry = memmap->entries[i];
+        if (entry != pool) {
+            out[n++] = (struct phys_mem){entry->base, entry->length, convert_mem_type(entry->type)};
+            continue;
+        }
+
+        const uint64_t end = entry->base + entry->length;
+        out[n++] = (struct phys_mem){entry->base, pool_static_end - entry->base, PHYS_STATIC};
+        out[n++] = (struct phys_mem){pool_static_end, pool_reclaimable_start - pool_static_end, PHYS_USABLE};
+        out[n++] = (struct phys_mem){pool_reclaimable_start, end - pool_reclaimable_start, PHYS_RECLAIMABLE};
+    }
+    return n;
+}
+
+static void find_framebuffer(struct boot_framebuffer* out) {
+    const struct limine_framebuffer_response* res = framebuffer_request.response;
+    if (!res || !res->framebuffer_count)
+        return;
+
+    const struct limine_framebuffer* fb = res->framebuffers[0];
+    out->base = to_phys(fb->address);
+    out->width = fb->width;
+    out->height = fb->height;
+    out->pitch = fb->pitch;
+    out->bytes_per_pixel = fb->bpp / 8;
+    out->red = (struct boot_color){fb->red_mask_shift, fb->red_mask_size};
+    out->green = (struct boot_color){fb->green_mask_shift, fb->green_mask_size};
+    out->blue = (struct boot_color){fb->blue_mask_shift, fb->blue_mask_size};
+}
+
+static bool load() {
+    const struct limine_memmap_response* memmap = memmap_request.response;
+    const struct limine_executable_address_response* exec = executable_address_request.response;
+    if (!LIMINE_BASE_REVISION_SUPPORTED(limine_revision) || !memmap || !hhdm_request.response || !exec)
+        return false;
+    hhdm_offset = hhdm_request.response->offset;
+
+    if (!arch_check_cpu() || !init_pool(memmap))
+        return false;
+
+    const struct limine_bootloader_info_response* info = bootloader_info_request.response;
+    struct kernel_image kernel;
+    if (!loader_load_kernel(&kernel, info ? info->name : "Limine"))
+        return false;
+
+    uint64_t mem;
+    if (!loader_alloc(PHYS_RECLAIMABLE, (memmap->entry_count + 2) * sizeof(struct phys_mem), &mem))
+        return false;
+
+    const struct limine_executable_file_response* file = executable_file_request.response;
+    if (file && file->executable_file->string) {
+        const char* cmdline = file->executable_file->string;
+        memcpy(kernel.info->cmdline, cmdline, MIN(strlen(cmdline), sizeof(kernel.info->cmdline) - 1));
+    }
+
+    const struct limine_module_response* modules = module_request.response;
+    for (size_t i = 0; modules && i < modules->module_count; i++) {
+        const struct limine_file* module = modules->modules[i];
+        if (!loader_add_file(&kernel, module->path, to_phys(module->address), module->size))
+            return false;
+    }
+
+    find_framebuffer(&kernel.info->fb);
+    if (rsdp_request.response)
+        kernel.info->rsdp = to_phys(rsdp_request.response->address);
+    if (dtb_request.response && dtb_request.response->dtb_ptr)
+        kernel.info->dtb = to_phys(dtb_request.response->dtb_ptr);
+
+    const uint64_t trampoline = (uint64_t)arch_enter_kernel;
+    uint64_t root;
+    if (!loader_page_tables(
+            &kernel,
+            mem_map_top(memmap),
+            exec->physical_base + (trampoline - exec->virtual_base),
+            &root
+        ))
+        return false;
+
+    struct phys_mem* mem_map = loader_ptr(mem);
+    loader_set_mem_map(&kernel, mem_map, convert_mem_map(memmap, mem_map));
+    arch_enter_kernel(root, kernel.entry);
+}
+
+[[noreturn]]
+void loader_main() {
+    load();
+    for (;;)
+        asm volatile("");
+}

boot/limine/meson.build

@@ -1 +1,25 @@
-kernel_sources += files('entry.c')
+limine_lds = meson.current_source_dir() / 'loader.ld'
+
+limine_c_args = [loader_c_args, '-fno-pic']
+if target_machine.cpu_family() == 'x86_64'
+    limine_c_args += '-mcmodel=kernel'
+endif
+
+executable(
+    'zinnia-limine',
+    sources: [loader_sources, files('main.c'), loader_kernel],
+    include_directories: [common_includes, loader_includes],
+    c_args: limine_c_args,
+    link_args: [
+        '-nostdlib',
+        '-static',
+        '-Wl,-T,' + limine_lds,
+        '-Wl,-z,noexecstack',
+        '-Wl,-z,max-page-size=0x1000',
+        '-Wl,--build-id=none',
+    ],
+    link_depends: limine_lds,
+    pie: false,
+    install: true,
+    install_dir: zinnia_bootdir,
+)

boot/loader/elf.c

@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: ISC
+
+#include <bits/elf.h>
+#include <string.h>
+#include "loader.h"
+
+static bool in_image(const struct kernel_image* image, uint64_t vaddr, uint64_t len) {
+    return vaddr >= image->virt && len <= image->size && vaddr - image->virt <= image->size - len;
+}
+
+void* kernel_ptr(const struct kernel_image* image, uint64_t vaddr) {
+    return loader_ptr(image->phys + (vaddr - image->virt));
+}
+
+static bool apply_relocations(const struct kernel_image* image, uint64_t dynamic_vaddr) {
+    if (!in_image(image, dynamic_vaddr, sizeof(elf64_dyn_t)))
+        return false;
+
+    uint64_t rela = 0;
+    uint64_t rela_size = 0;
+    uint64_t rela_ent = sizeof(elf64_rela_t);
+    for (const elf64_dyn_t* dyn = kernel_ptr(image, dynamic_vaddr); dyn->d_tag != DT_NULL; dyn++) {
+        switch (dyn->d_tag) {
+        case DT_RELA:
+            rela = dyn->d_un.d_val;
+            break;
+        case DT_RELASZ:
+            rela_size = dyn->d_un.d_val;
+            break;
+        case DT_RELAENT:
+            rela_ent = dyn->d_un.d_val;
+            break;
+        }
+    }
+
+    if (!rela)
+        return true;
+    if (rela_ent < sizeof(elf64_rela_t) || !in_image(image, rela, rela_size))
+        return false;
+
+    for (uint64_t off = 0; off + rela_ent <= rela_size; off += rela_ent) {
+        const elf64_rela_t* r = kernel_ptr(image, rela + off);
+        const uint32_t type = ELF64_R_TYPE(r->r_info);
+        if (type == R_NONE)
+            continue;
+        if (type != R_RELATIVE)
+            return false;
+        if (!in_image(image, r->r_offset, sizeof(uint64_t)))
+            return false;
+        *(uint64_t*)kernel_ptr(image, r->r_offset) = (uint64_t)r->r_addend;
+    }
+
+    return true;
+}
+
+bool elf_load(const uint8_t* elf, size_t size, struct kernel_image* out) {
+    elf64_ehdr_t ehdr;
+    if (size < sizeof(ehdr))
+        return false;
+    memcpy(&ehdr, elf, sizeof(ehdr));
+
+    if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) || ehdr.e_ident[EI_CLASS] != ELF_ARCH_CLASS ||
+        ehdr.e_ident[EI_DATA] != ELF_ARCH_DATA)
+        return false;
+    if (ehdr.e_machine != ELF_ARCH_MACHINE)
+        return false;
+    if (ehdr.e_phentsize != sizeof(elf64_phdr_t) || ehdr.e_phoff > size ||
+        (size - ehdr.e_phoff) / sizeof(elf64_phdr_t) < ehdr.e_phnum)
+        return false;
+
+    const elf64_phdr_t* phdrs = (const elf64_phdr_t*)(elf + ehdr.e_phoff);
+
+    uint64_t lo = UINT64_MAX;
+    uint64_t hi = 0;
+    uint64_t dynamic_vaddr = 0;
+    uint64_t info_vaddr = 0;
+    uint64_t info_size = 0;
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf64_phdr_t* ph = &phdrs[i];
+        if (ph->p_type == PT_DYNAMIC)
+            dynamic_vaddr = ph->p_vaddr;
+        if (ph->p_type == PT_BOOT_INFO) {
+            info_vaddr = ph->p_vaddr;
+            info_size = ph->p_memsz;
+        }
+        if (ph->p_type != PT_LOAD || ph->p_memsz == 0)
+            continue;
+        if (ph->p_filesz > ph->p_memsz || ph->p_offset > size || size - ph->p_offset < ph->p_filesz)
+            return false;
+        if (ALIGN_DOWN(ph->p_vaddr, PAGE_SIZE) < lo)
+            lo = ALIGN_DOWN(ph->p_vaddr, PAGE_SIZE);
+        if (ALIGN_UP(ph->p_vaddr + ph->p_memsz, PAGE_SIZE) > hi)
+            hi = ALIGN_UP(ph->p_vaddr + ph->p_memsz, PAGE_SIZE);
+    }
+    if (lo >= hi)
+        return false;
+
+    out->virt = lo;
+    out->size = hi - lo;
+    out->entry = ehdr.e_entry;
+    if (!in_image(out, out->entry, 1))
+        return false;
+    if (info_size != sizeof(struct boot_info) || !in_image(out, info_vaddr, info_size))
+        return false;
+
+    if (!loader_alloc(PHYS_STATIC, out->size, &out->phys))
+        return false;
+
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf64_phdr_t* ph = &phdrs[i];
+        if (ph->p_type != PT_LOAD || ph->p_memsz == 0)
+            continue;
+        memcpy(kernel_ptr(out, ph->p_vaddr), elf + ph->p_offset, ph->p_filesz);
+    }
+    out->info = kernel_ptr(out, info_vaddr);
+
+    if (dynamic_vaddr)
+        return apply_relocations(out, dynamic_vaddr);
+    return true;
+}

boot/loader/lib.c

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: ISC
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+size_t strlen(const char* str) {
+    if (!str)
+        return 0;
+
+    size_t len = 0;
+    while (str[len])
+        len++;
+    return len;
+}
+
+int memcmp(const void* s1, const void* s2, size_t size) {
+    const uint8_t* a = s1;
+    const uint8_t* b = s2;
+    for (size_t i = 0; i < size; i++) {
+        if (a[i] != b[i])
+            return a[i] < b[i] ? -1 : 1;
+    }
+    return 0;
+}
+
+void* memcpy(void* restrict dest, const void* restrict src, size_t n) {
+    uint8_t* d = dest;
+    const uint8_t* s = src;
+    for (size_t i = 0; i < n; i++)
+        d[i] = s[i];
+    return dest;
+}
+
+void* memmove(void* dstptr, const void* srcptr, size_t size) {
+    uint8_t* d = dstptr;
+    const uint8_t* s = srcptr;
+    if (d < s) {
+        for (size_t i = 0; i < size; i++)
+            d[i] = s[i];
+    } else {
+        for (size_t i = size; i > 0; i--)
+            d[i - 1] = s[i - 1];
+    }
+    return dstptr;
+}
+
+void* memset(void* dest, int value, size_t n) {
+    uint8_t* d = dest;
+    for (size_t i = 0; i < n; i++)
+        d[i] = (uint8_t)value;
+    return dest;
+}

boot/loader/loader.c

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: ISC
+
+#include "loader.h"
+#include <string.h>
+#include "loader_arch.h"
+
+extern const uint8_t __kernel_start[];
+extern const uint8_t __kernel_end[];
+
+bool loader_load_kernel(struct kernel_image* out, const char* name) {
+    if (!elf_load(__kernel_start, __kernel_end - __kernel_start, out))
+        return false;
+
+    struct boot_info* info = out->info;
+    memcpy(info->loader, name, MIN(strlen(name), sizeof(info->loader) - 1));
+    info->hhdm_base = ARCH_HHDM_BASE;
+    info->phys_base = out->phys;
+    info->virt_base = out->virt;
+    return true;
+}
+
+bool loader_add_file(const struct kernel_image* kernel, const char* path, uint64_t base, uint64_t length) {
+    struct boot_info* info = kernel->info;
+    if (info->num_files == ARRAY_SIZE(info->files))
+        return false;
+
+    struct boot_file* file = &info->files[info->num_files++];
+    file->data = base;
+    file->length = length;
+    const size_t path_len = MIN(strlen(path), sizeof(file->path) - 1);
+    memcpy(file->path, path, path_len);
+    file->path[path_len] = 0;
+    return true;
+}
+
+void loader_set_mem_map(const struct kernel_image* kernel, struct phys_mem* map, size_t len) {
+    for (size_t i = 1; i < len; i++) {
+        const struct phys_mem entry = map[i];
+        size_t j = i;
+        while (j > 0 && map[j - 1].address > entry.address) {
+            map[j] = map[j - 1];
+            j--;
+        }
+        map[j] = entry;
+    }
+
+    size_t merged = 0;
+    for (size_t i = 0; i < len; i++) {
+        if (!map[i].length)
+            continue;
+        struct phys_mem* prev = merged ? &map[merged - 1] : nullptr;
+        if (prev && prev->usage == map[i].usage && prev->address + prev->length == map[i].address)
+            prev->length += map[i].length;
+        else
+            map[merged++] = map[i];
+    }
+
+    struct boot_info* info = kernel->info;
+    info->num_mem_maps = MIN(merged, ARRAY_SIZE(info->mem_map));
+    memcpy(info->mem_map, map, info->num_mem_maps * sizeof(*map));
+}
+
+bool loader_page_tables(
+    const struct kernel_image* kernel,
+    uint64_t mem_top,
+    uint64_t trampoline_phys,
+    uint64_t* out_root
+) {
+    uint64_t root;
+    if (!loader_alloc(PHYS_RECLAIMABLE, PAGE_SIZE, &root))
+        return false;
+
+    const uint64_t trampoline_virt = ALIGN_DOWN((uint64_t)arch_enter_kernel, PAGE_SIZE);
+    if (!arch_paging_map(root, ARCH_HHDM_BASE, 0, ALIGN_UP(mem_top, ARCH_HUGE_PAGE_SIZE), ARCH_HUGE_PAGE_SIZE) ||
+        !arch_paging_map(root, kernel->virt, kernel->phys, kernel->size, PAGE_SIZE) ||
+        !arch_paging_map(root, trampoline_virt, ALIGN_DOWN(trampoline_phys, PAGE_SIZE), 2 * PAGE_SIZE, PAGE_SIZE))
+        return false;
+
+    *out_root = root;
+    return true;
+}

boot/loader/loader.h

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/boot.h>
+#include <zinnia/utils.h>
+#include <stddef.h>
+#include <stdint.h>
+
+constexpr uint64_t PAGE_SIZE = 0x1000;
+
+struct kernel_image {
+    uint64_t phys;
+    uint64_t virt;
+    uint64_t size;
+    uint64_t entry;
+    struct boot_info* info;
+};
+
+bool loader_alloc(enum phys_mem_usage usage, size_t size, uint64_t* out);
+void* loader_ptr(uint64_t phys);
+
+bool elf_load(const uint8_t* elf, size_t size, struct kernel_image* out);
+void* kernel_ptr(const struct kernel_image* image, uint64_t vaddr);
+
+bool loader_load_kernel(struct kernel_image* out, const char* name);
+bool loader_add_file(const struct kernel_image* kernel, const char* path, uint64_t base, uint64_t length);
+void loader_set_mem_map(const struct kernel_image* kernel, struct phys_mem* map, size_t len);
+
+bool loader_page_tables(
+    const struct kernel_image* kernel,
+    uint64_t mem_top,
+    uint64_t trampoline_phys,
+    uint64_t* out_root
+);
+
+bool arch_check_cpu();
+bool arch_paging_map(uint64_t root, uint64_t virt, uint64_t phys, uint64_t size, uint64_t page_size);
+
+[[noreturn]]
+void arch_enter_kernel(uint64_t root, uint64_t entry);

boot/loader/x86_64/enter.S

@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: ISC
+
+    .intel_syntax noprefix
+    .text
+
+    .globl arch_enter_kernel
+    .type arch_enter_kernel, @function
+    .balign 16
+arch_enter_kernel:
+    cli
+    mov cr3, rdi
+    jmp rsi
+    .size arch_enter_kernel, . - arch_enter_kernel

boot/loader/x86_64/loader_arch.h

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stdint.h>
+
+constexpr uint64_t ARCH_HHDM_BASE = 0xffff'8000'0000'0000;
+constexpr uint64_t ARCH_HUGE_PAGE_SIZE = 0x20'0000;

boot/loader/x86_64/meson.build

@@ -0,0 +1,11 @@
+loader_sources += files(
+    'enter.S',
+    'paging.c',
+)
+
+loader_c_args += [
+    '-mno-red-zone',
+    '-masm=intel',
+]
+
+loader_objcopy_format = 'elf64-x86-64'

boot/loader/x86_64/paging.c

@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: ISC
+
+#include <bits/pmap.h>
+#include <x86_64/defs.h>
+#include "loader.h"
+#include "loader_arch.h"
+
+static bool next_table(uint64_t* table, size_t index, uint64_t** out) {
+    if (!(table[index] & ARCH_FLAG_PRESENT)) {
+        uint64_t page;
+        if (!loader_alloc(PHYS_RECLAIMABLE, PAGE_SIZE, &page))
+            return false;
+        table[index] = page | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+    }
+    *out = loader_ptr(table[index] & ARCH_PTE_ADDR_MASK);
+    return true;
+}
+
+bool arch_check_cpu() {
+    uint64_t cr4;
+    asm volatile("mov %0, cr4" : "=r"(cr4));
+    if (cr4 & CR4_LA57)
+        return false;
+    return true;
+}
+
+bool arch_paging_map(uint64_t root, uint64_t virt, uint64_t phys, uint64_t size, uint64_t page_size) {
+    const int leaf_level = page_size == ARCH_HUGE_PAGE_SIZE ? 1 : 0;
+
+    for (uint64_t off = 0; off < size; off += page_size) {
+        const uint64_t va = virt + off;
+        uint64_t* table = loader_ptr(root);
+        for (int level = 3; level > leaf_level; level--) {
+            if (!next_table(table, (va >> (12 + 9 * level)) & 0x1ff, &table))
+                return false;
+        }
+
+        uint64_t entry = (phys + off) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+        if (leaf_level == 1)
+            entry |= ARCH_FLAG_SIZE;
+        table[(va >> (12 + 9 * leaf_level)) & 0x1ff] = entry;
+    }
+
+    return true;
+}

boot/main.c

@@ -1,97 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/assert.h>
-#include <zinnia/clock.h>
-#include <zinnia/cmdline.h>
-#include <zinnia/compiler.h>
-#include <zinnia/console.h>
-#include <zinnia/elf.h>
-#include <zinnia/init.h>
-#include <zinnia/irq.h>
-#include <zinnia/percpu.h>
-#include <zinnia/print.h>
-#include <zinnia/sched.h>
-#include <zinnia/vma.h>
-#include <zinnia/vmo.h>
-#include <config.h>
-#include <stdint.h>
-
-#if CONFIG_ACPI
-#include <zinnia/acpi.h>
-#endif
-
-static const char zinnia_banner[] = "Zinnia " ZINNIA_VERSION " " CONFIG_KERNEL_FLAVOR " (" ZINNIA_ARCH
-                                    ", " ZINNIA_COMPILER_ID ", " ZINNIA_LINKER_ID ")";
-
-static void kernel_main_task(uintptr_t arg0) {
-    struct boot_info* info = (struct boot_info*)arg0;
-
-    log("main: Hello from main task!\n");
-
-    assert(info->num_files >= 1, "No init executable provided\n");
-
-    struct vma* init_space;
-    assert(vma_new(&init_space) == EOK, "");
-
-    assert(false, "TODO!");
-
-    struct boot_file* file_data = &info->files[0];
-    struct paged_vmo* init_file;
-    assert(vmo_new_phys(&init_file) == EOK, "");
-    vmo_write(&init_file->object, 0, HHDM_PTR(file_data->data), file_data->length, nullptr);
-
-    log("main: %s:%i\n", __FUNCTION__, __LINE__);
-
-    // Load init.
-    const char* init_argv[] = {"init", nullptr};
-    const char* init_envp[] = {nullptr};
-
-    struct task* init_task = nullptr;
-    errno_t status = 0;
-
-    assert(false, "Not implemented!");
-
-    assert(status == 0, "Failed to load init process (%i)\n", status);
-    sched_add_task(&percpu_get()->sched, init_task);
-
-    log("main: %s:%i\n", __FUNCTION__, __LINE__);
-
-    while (true) {
-        irq_wait();
-    }
-}
-
-[[__init]]
-void kernel_early_init() {
-    percpu_bsp_init();
-    percpu_get()->online = true;
-    irq_lock();
-}
-
-[[noreturn]]
-void kernel_main(struct boot_info* info) {
-    // TODO: Initialize in-memory log ring.
-    kprintf("%s\n", zinnia_banner); // Say hello!
-    kprintf("Command line: %s\n", info->cmdline);
-
-    cmdline_parse(info->cmdline);
-    console_init();
-    kprintf("\ec");
-
-    mem_init(info->mem_map, info->num_mem_maps, info->virt_base, info->phys_base, info->hhdm_base);
-
-    percpu_init();
-    sched_init(&percpu_get()->sched);
-
-#if CONFIG_ACPI
-    acpi_init(info);
-#endif
-
-    struct task* main_task;
-    task_create("main", &kernel_vma, kernel_main_task, (uintptr_t)info, &main_task);
-    sched_add_task(&percpu_get()->sched, main_task);
-
-    irq_unlock();
-    sched_yield(&percpu_get()->sched);
-
-    unreachable();
-}

boot/meson.build

@@ -1,6 +1,44 @@
-kernel_sources += files(
-    'cmdline.c',
-    'main.c',
+objcopy = find_program('objcopy')
+zinnia_bootdir = zinnia_datadir / 'boot'
+
+loader_sources = files(
+    'loader/elf.c',
+    'loader/lib.c',
+    'loader/loader.c',
 )
 
-subdir('limine')
+loader_c_args = [
+    '-ffreestanding',
+    '-nostdinc',
+    '-fno-stack-protector',
+    '-fno-omit-frame-pointer',
+    '-mgeneral-regs-only',
+]
+
+loader_includes = include_directories('loader', 'loader' / target_machine.cpu_family())
+
+loader_objcopy_format = 'unknown'
+
+subdir('loader' / target_machine.cpu_family())
+
+loader_kernel = custom_target(
+    'loader-kernel.o',
+    input: kernel,
+    output: 'loader-kernel.o',
+    command: [
+        objcopy,
+        '-I', 'binary',
+        '-O', loader_objcopy_format,
+        '--rename-section', '.data=.kernel,alloc,load,readonly,data,contents',
+        '@INPUT@',
+        '@OUTPUT@',
+    ],
+)
+
+if kconfig.get('CONFIG_BOOT_EFISTUB', 'n') == 'y'
+    subdir('efi')
+endif
+
+if kconfig.get('CONFIG_BOOT_LIMINE', 'n') == 'y'
+    subdir('limine')
+endif

core/clock.c

@@ -1,54 +0,0 @@
-#include <zinnia/atomic.h>
-#include <zinnia/clock.h>
-#include <zinnia/compiler.h>
-#include <zinnia/print.h>
-#include <zinnia/spin.h>
-#include <stdint.h>
-
-static struct clock* active_clock = nullptr;
-static uint64_t counter_base = 0;
-static struct spinlock lock = {};
-
-bool clock_available() {
-    return active_clock != nullptr;
-}
-
-uint64_t clock_get_elapsed_ns() {
-    struct clock* active = atomic_load(&active_clock, ATOMIC_RELAXED);
-    if (__unlikely(!active))
-        return 0;
-
-    return atomic_load(&counter_base, ATOMIC_RELAXED) + active->get_elapsed_ns(active);
-}
-
-bool clock_switch(struct clock* clock) {
-    spin_lock(&lock);
-
-    if (active_clock) {
-        if (clock->priority <= active_clock->priority) {
-            spin_unlock(&lock);
-            return false;
-        }
-    }
-
-    log("clock: Switching to clock source \"%s\"\n", clock->name);
-
-    uint64_t elapsed = clock_get_elapsed_ns();
-    counter_base += elapsed;
-
-    active_clock = clock;
-    spin_unlock(&lock);
-    return true;
-}
-
-void clock_spin_ns(uint64_t ns) {
-    if (active_clock == nullptr) {
-        log("clock: Unable to sleep for %lu ns, no clock available!\n", ns);
-        return;
-    }
-
-    uint64_t target = clock_get_elapsed_ns() + ns;
-    while (clock_get_elapsed_ns() < target) {
-        // TODO: Maybe pause?
-    }
-}

core/console.c

@@ -1,43 +0,0 @@
-#include <zinnia/cmdline.h>
-#include <zinnia/console.h>
-#include <zinnia/print.h>
-#include <zinnia/string.h>
-
-extern struct console* const __ld_console_start[];
-extern struct console* const __ld_console_end[];
-
-struct console* active = nullptr;
-
-void console_write(const char* buf, size_t len) {
-    if (__likely(active))
-        active->write(active, buf, len);
-}
-
-static void console_setup(const char* value) {
-    if (!value)
-        return;
-
-    // Find a suitable boot console.
-    struct console* const* cur = __ld_console_start;
-    while (cur < __ld_console_end) {
-        if (!strcmp(value, (*cur)->name)) {
-            active = *cur;
-        }
-        cur++;
-    }
-}
-
-CMDLINE_OPTION("console", "Set a default console", console_setup);
-
-void console_init() {
-    // If we have no console set and at least one set, use that as a default.
-    if (!active && &__ld_console_start[0] != &__ld_console_end[0])
-        active = __ld_console_start[0];
-
-    if (active) {
-        if (active->init(active) == false)
-            active = nullptr;
-
-        log("console: Logging on console \"%s\"\n", active->name);
-    }
-}

core/elf.c

@@ -1,198 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/assert.h>
-#include <zinnia/elf.h>
-#include <zinnia/errno.h>
-#include <zinnia/mman.h>
-#include <zinnia/percpu.h>
-#include <zinnia/pmap.h>
-#include <zinnia/print.h>
-#include <zinnia/sched.h>
-#include <zinnia/utils.h>
-#include <zinnia/vma.h>
-#include <zinnia/vmo.h>
-#include <bits/elf.h>
-#include <zinnia/string.h>
-
-struct exec_info {
-    struct vma* space;
-    struct vmo* file_obj;
-    const char** argv;
-    const char** envp;
-};
-
-errno_t elf_load(struct exec_info* info, struct task** out) {
-    errno_t status;
-
-    size_t read;
-    struct elf_ehdr ehdr = {};
-    vmo_read(info->file_obj, 0, &ehdr, sizeof(ehdr), &read);
-
-    // Check if the file is an ELF.
-    if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) != 0)
-        return EINVAL;
-    if (ehdr.e_ident[EI_VERSION] != EV_CURRENT)
-        return EINVAL;
-    if (ehdr.e_ident[EI_CLASS] != ELF_ARCH_CLASS)
-        return EINVAL;
-    if (ehdr.e_ident[EI_DATA] != ELF_ARCH_DATA)
-        return EINVAL;
-    if (ehdr.e_machine != ELF_ARCH_MACHINE)
-        return EINVAL;
-    if (ehdr.e_type != ET_EXEC)
-        return EINVAL;
-
-    const size_t page_size = arch_mem_page_size();
-
-    for (size_t i = 0; i < ehdr.e_phnum; i++) {
-        struct elf_phdr phdr = {};
-        vmo_read(info->file_obj, (ehdr.e_phoff + (i * ehdr.e_phentsize)), &phdr, sizeof(phdr), &read);
-        if (phdr.p_type == PT_LOAD) {
-            enum prot_flags prot = 0;
-            if (phdr.p_flags & PF_R)
-                prot |= PROT_READ;
-            if (phdr.p_flags & PF_W)
-                prot |= PROT_WRITE;
-            if (phdr.p_flags & PF_X)
-                prot |= PROT_EXEC;
-
-            assert(phdr.p_offset % phdr.p_align == phdr.p_vaddr % phdr.p_align, "");
-
-            const uintptr_t misalign = phdr.p_vaddr & (page_size - 1);
-            const uintptr_t map_address = phdr.p_vaddr - misalign;
-            const size_t backed_map_size = (phdr.p_filesz + misalign + page_size - 1) & ~(page_size - 1);
-            const size_t total_map_size = (phdr.p_memsz + misalign + page_size - 1) & ~(page_size - 1);
-
-            // Copy the file data into its own mapping.
-            struct paged_vmo* phdr_obj;
-            status = vmo_new_phys(&phdr_obj);
-            if (status)
-                return status;
-
-            status = vmo_copy(&phdr_obj->object, phdr.p_offset, info->file_obj, phdr.p_offset, phdr.p_filesz, nullptr);
-            if (status)
-                return status;
-
-            // We map more than we copied so the rest is filled with zeroed pages.
-            status = vma_map(info->space, &phdr_obj->object, phdr.p_vaddr, phdr.p_memsz, prot, phdr.p_offset);
-            if (status)
-                return status;
-        }
-    }
-
-    const uintptr_t highest = ((uintptr_t)1 << (mem_high_shift() - 1)) - page_size;
-    const size_t stack_size = 2 * 1024 * 1024; // 2MiB
-    const uintptr_t stack_start = highest - stack_size;
-
-    // Allocate stack.
-    struct paged_vmo* stack;
-    status = vmo_new_phys(&stack);
-    if (status)
-        return status;
-
-    // Fill in stack info.
-    uintptr_t stack_off = stack_size;
-
-    // TODO: Instead of allocating, calculate it on the fly.
-    size_t num_envp = 0;
-    for (const char** ptr = info->envp; *ptr != nullptr; ptr++) {
-        num_envp++;
-    }
-    size_t num_argv = 0;
-    for (const char** ptr = info->argv; *ptr != nullptr; ptr++) {
-        num_argv++;
-    }
-    uintptr_t* envp_offsets = mem_alloc(num_envp * sizeof(uintptr_t), 0);
-    uintptr_t* argv_offsets = mem_alloc(num_argv * sizeof(uintptr_t), 0);
-
-    for (size_t env = 0; env < num_envp; env++) {
-        const char nul = 0;
-        stack_off -= 1;
-        vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr);
-
-        const size_t len = strlen(info->envp[env]);
-        stack_off -= len;
-        vmo_write(&stack->object, stack_off, info->envp[env], len, nullptr);
-
-        envp_offsets[env] = stack_start + stack_off;
-    }
-
-    for (size_t arg = 0; arg < num_argv; arg++) {
-        const char nul = 0;
-        stack_off -= 1;
-        vmo_write(&stack->object, stack_off, &nul, sizeof(nul), nullptr);
-
-        const size_t len = strlen(info->argv[arg]);
-        stack_off -= len;
-        vmo_write(&stack->object, stack_off, info->argv[arg], len, nullptr);
-
-        argv_offsets[arg] = stack_start + stack_off;
-    }
-
-    stack_off = ALIGN_DOWN(stack_off, 16);
-
-    // Align the stack if argc + argv + envp does not add up to 16 byte alignment.
-    if ((1 + num_argv + num_envp) & 1) {
-        stack_off -= sizeof(uintptr_t);
-        uintptr_t zero = 0;
-        vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
-    }
-
-#define WRITE_AUXV(auxv, value) \
-    do { \
-        stack_off -= sizeof(uintptr_t); \
-        uintptr_t auxv_val = value; \
-        vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \
-        stack_off -= sizeof(uintptr_t); \
-        auxv_val = auxv; \
-        vmo_write(&stack->object, stack_off, &auxv_val, sizeof(uintptr_t), nullptr); \
-    } while (0)
-
-    // Write auxiliary values.
-    WRITE_AUXV(AT_NULL, 0);
-    WRITE_AUXV(AT_SECURE, 0);
-    WRITE_AUXV(AT_PHNUM, ehdr.e_phnum);
-    WRITE_AUXV(AT_PHENT, ehdr.e_phentsize);
-    WRITE_AUXV(AT_ENTRY, ehdr.e_entry);
-
-    // envp pointers.
-    stack_off -= sizeof(uintptr_t);
-    const uintptr_t zero = 0;
-    vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
-    for (size_t env = 0; env < num_envp; env++) {
-        stack_off -= sizeof(uintptr_t);
-        vmo_write(&stack->object, stack_off, &envp_offsets[env], sizeof(envp_offsets[env]), nullptr);
-    }
-
-    // argv pointers.
-    stack_off -= sizeof(uintptr_t);
-    vmo_write(&stack->object, stack_off, &zero, sizeof(zero), nullptr);
-    for (size_t arg = 0; arg < num_argv; arg++) {
-        stack_off -= sizeof(uintptr_t);
-        vmo_write(&stack->object, stack_off, &argv_offsets[arg], sizeof(argv_offsets[arg]), nullptr);
-    }
-
-    mem_free(envp_offsets);
-    mem_free(argv_offsets);
-
-    // argc
-    stack_off -= sizeof(uintptr_t);
-    uintptr_t argc = num_argv;
-    vmo_write(&stack->object, stack_off, &argc, sizeof(argc), nullptr);
-
-    status = vma_map(info->space, &stack->object, stack_start, stack_size, PROT_READ | PROT_WRITE, 0);
-    if (status)
-        return status;
-
-    struct task_start_info* start_info = mem_alloc(sizeof(struct task_start_info), 0);
-    start_info->ip = ehdr.e_entry;
-    start_info->sp = highest - stack_size + stack_off;
-    start_info->arg = 0;
-
-    struct task* result;
-    status = task_create(info->argv[0], info->space, sched_to_user, (uintptr_t)start_info, &result);
-    if (status)
-        return status;
-
-    *out = result;
-    return 0;
-}

core/irq.c

@@ -1,41 +0,0 @@
-#include <zinnia/atomic.h>
-#include <zinnia/irq.h>
-#include <zinnia/percpu.h>
-#include <arch/irq.h>
-
-void irq_set_state(bool state) {
-    arch_irq_set_state(state);
-}
-
-bool irq_get_state() {
-    return arch_irq_get_state();
-}
-
-void irq_wait() {
-    arch_irq_wait();
-}
-
-void irq_lock() {
-    struct irq_percpu* cpu = &percpu_get()->irq;
-    if (atomic_load(&cpu->in_interrupt, ATOMIC_ACQUIRE))
-        return;
-
-    arch_irq_set_state(false);
-    atomic_fetch_add(&percpu_get()->irq.level, 1, ATOMIC_ACQ_REL);
-}
-
-void irq_unlock() {
-    struct irq_percpu* cpu = &percpu_get()->irq;
-    if (atomic_load(&cpu->in_interrupt, ATOMIC_ACQUIRE))
-        return;
-
-    uint32_t old_level = atomic_fetch_sub(&cpu->level, 1, ATOMIC_ACQ_REL);
-    // If it was 1, the new IRQ level is now 0.
-    if (old_level == 1) {
-        arch_irq_set_state(true);
-    }
-}
-
-bool irq_set_interrupted(bool is_interrupted) {
-    return atomic_exchange(&percpu_get()->irq.in_interrupt, is_interrupted, ATOMIC_RELEASE);
-}

core/lib/meson.build

@@ -1,6 +0,0 @@
-# Freestanding libc functions that are needed in the kernel
-
-kernel_sources += files(
-    'stdlib.c',
-    'string.c',
-)

core/mem/pmap.c

@@ -1,108 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/errno.h>
-#include <zinnia/pmap.h>
-#include <zinnia/spin.h>
-#include <zinnia/vma.h>
-#include <config.h>
-#include <zinnia/string.h>
-
-// TODO: For now, we assume that we can do generic PTE lookups on every arch that has a MMU.
-static_assert(CONFIG_HAVE_MMU_GENERIC);
-
-errno_t pmap_new_kernel(struct pmap* pt, enum alloc_flags flags) {
-    flags &= ~ALLOC_NOZERO;
-
-    phys_t addr;
-    errno_t status = mem_phys_alloc(1, flags, &addr);
-    if (status)
-        return status;
-
-    struct pmap result = {
-        .root = addr,
-        .lock = (struct spinlock){0},
-        .is_user = false,
-    };
-
-    *pt = result;
-    return 0;
-}
-
-errno_t pmap_new_user(struct pmap* pt, enum alloc_flags flags) {
-    phys_t user_l1;
-    errno_t status = mem_phys_alloc(1, 0, &user_l1);
-    if (status)
-        return status;
-
-    void* user_l1_ptr = HHDM_PTR(user_l1);
-    void* kernel_l1_ptr = HHDM_PTR(kernel_vma.pmap.root);
-    memcpy(user_l1_ptr, kernel_l1_ptr, arch_mem_page_size());
-
-    struct pmap result = {
-        .root = user_l1,
-        .lock = (struct spinlock){0},
-        .is_user = true,
-    };
-
-    *pt = result;
-    return 0;
-}
-
-// Gets a reference to the PTE at the given virtual address.
-// If `check_only` is set, only checks if the PTE exists,
-// and doesn't allocate new levels if they don't already exist.
-// If it can't allocate a page if it has to, returns `nullptr`.
-static pte_t* get_pte(struct pmap* pt, uintptr_t vaddr, bool is_user, bool check_only) {
-    pte_t* current_head = HHDM_PTR(pt->root);
-    size_t index = 0;
-
-    for (int8_t level = arch_mem_num_levels() - 1; level >= 0; level--) {
-        const size_t addr_mask = (1 << arch_mem_level_bits()) - 1;
-        const size_t addr_shift = arch_mem_page_bits() + (arch_mem_level_bits() * level);
-        const enum pte_flags level_flags = PTE_DIR | (is_user ? PTE_USER : 0);
-
-        index = (vaddr >> addr_shift) & addr_mask;
-
-        // The last level is used to access the actual PTE, so break the loop then.
-        // We still need to update the index beforehand, that's why we can't just end early.
-        if (level == 0)
-            break;
-
-        pte_t* pte = &current_head[index];
-
-        if (pte_is_present(pte)) {
-            // Get the next level.
-            *pte = pte_build(pte_address(pte), level_flags, CACHE_ANY);
-            current_head = HHDM_PTR(pte_address(pte));
-        } else {
-            // If the current level isn't present, we can skip the rest.
-            if (check_only)
-                return nullptr;
-
-            phys_t addr;
-            if (mem_phys_alloc(1, 0, &addr) != 0)
-                return nullptr;
-
-            *pte = pte_build(addr, level_flags, CACHE_ANY);
-            current_head = HHDM_PTR(addr);
-        }
-    }
-
-    return &current_head[index];
-}
-
-errno_t pmap_map(struct pmap* pt, uintptr_t vaddr, phys_t paddr, enum pte_flags flags, enum cache_mode cache) {
-    spin_lock(&pt->lock);
-
-    errno_t status = 0;
-    pte_t* pte = get_pte(pt, vaddr, flags & PTE_USER, false);
-    if (pte == nullptr) {
-        status = ENOMEM;
-        goto fail;
-    }
-
-    *pte = pte_build(paddr, flags, cache);
-
-fail:
-    spin_unlock(&pt->lock);
-    return status;
-}

core/mem/vma.c

@@ -1,99 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/assert.h>
-#include <zinnia/errno.h>
-#include <zinnia/mman.h>
-#include <zinnia/pmap.h>
-#include <zinnia/utils.h>
-#include <zinnia/vma.h>
-
-// Try to create a new, empty virtual address space.
-errno_t vma_new(struct vma** out) {
-    debug_assert(out != nullptr, "Missing out");
-
-    errno_t res;
-    if (!out)
-        return EINVAL;
-
-    struct vma* vmspace = mem_alloc(sizeof(struct vma), 0);
-    if (!vmspace)
-        return ENOMEM;
-
-    res = pmap_new_user(&vmspace->pmap, 0);
-    if (res)
-        goto err1;
-
-    *out = vmspace;
-    return 0;
-
-err1:
-    mem_free(vmspace);
-err0:
-    return res;
-}
-
-// Delete a virtual address space.
-void vma_free(struct vma* v) {}
-
-// Remove any mappings in the given address range.
-// May fail if there is no memory and a mapping needs to be split.
-errno_t vma_unmap(struct vma* v, uintptr_t vaddr, size_t size);
-
-// Change the protection of an existing mapping.
-// May fail if there is no memory and a mapping needs to be split.
-errno_t vma_protect(struct vma* v, uintptr_t vaddr, size_t size, enum prot_flags flags);
-
-// Try to handle a page fault.
-bool vma_page_fault(struct vma* v, const struct vma_fault_info* info) {
-    // TODO
-    return false;
-}
-
-errno_t vma_map(
-    struct vma* vma,
-    struct vmo* vmo,
-    uintptr_t addr,
-    size_t len,
-    enum prot_flags flags,
-    uintptr_t vmo_offset
-) {
-    const size_t page_size = arch_mem_page_size();
-
-    if (addr % page_size != vmo_offset % page_size)
-        return EINVAL;
-
-    size_t start_page = addr / page_size;
-    const size_t addr_offset = addr % page_size;
-    const size_t pages = ALIGN_UP(addr_offset + len, page_size) / page_size;
-    size_t end_page = start_page + pages;
-
-    if (!vma || !vmo)
-        return EINVAL;
-
-    const size_t first_vmo_page = vmo_offset / page_size;
-
-    for (size_t p = start_page; p < end_page; ++p) {
-        const size_t page_index = p - start_page;
-        struct page* page_obj;
-        errno_t status = vmo->get_page(vmo, first_vmo_page + page_index, &page_obj);
-        if (status != 0)
-            return status;
-
-        phys_t paddr = (phys_t)vm_page_addr(page_obj);
-
-        enum pte_flags pf = 0;
-        if (flags & PROT_READ)
-            pf |= PTE_READ;
-        if (flags & PROT_WRITE)
-            pf |= PTE_WRITE;
-        if (flags & PROT_EXEC)
-            pf |= PTE_EXEC;
-        if (vma->pmap.is_user)
-            pf |= PTE_USER;
-
-        status = pmap_map(&vma->pmap, p * page_size, paddr, pf, CACHE_ANY);
-        if (status != 0)
-            return status;
-    }
-
-    return 0;
-}

core/mem/vmo.c

@@ -1,195 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/errno.h>
-#include <zinnia/list.h>
-#include <zinnia/utils.h>
-#include <zinnia/vmo.h>
-#include <zinnia/string.h>
-
-errno_t vmo_read(struct vmo* obj, uintptr_t offset, void* buf, size_t len, size_t* actual_read) {
-    const size_t page_size = arch_mem_page_size();
-    size_t progress = 0;
-    errno_t status = 0;
-
-    while (progress < len) {
-        const size_t misalign = (progress + offset) % page_size;
-        const size_t page_index = (progress + offset) / page_size;
-        const size_t copy_size = MIN(page_size - misalign, len - progress);
-
-        struct page* p;
-        status = obj->get_page(obj, page_index, &p);
-        if (status != 0)
-            break;
-
-        uintptr_t page_addr = vm_page_addr(p);
-
-        memcpy(buf + progress, HHDM_PTR(page_addr) + misalign, copy_size);
-        progress += copy_size;
-    }
-
-    if (actual_read)
-        *actual_read = progress;
-    return status;
-}
-
-errno_t vmo_write(struct vmo* obj, uintptr_t offset, const void* buf, size_t len, size_t* actual_written) {
-    const size_t page_size = arch_mem_page_size();
-    size_t progress = 0;
-    errno_t status = 0;
-
-    while (progress < len) {
-        const size_t misalign = (progress + offset) % page_size;
-        const size_t page_index = (progress + offset) / page_size;
-        const size_t copy_size = MIN(page_size - misalign, len - progress);
-
-        struct page* p;
-        status = obj->get_page(obj, page_index, &p);
-        if (status != 0)
-            break;
-
-        uintptr_t page_addr = vm_page_addr(p);
-
-        memcpy(HHDM_PTR(page_addr) + misalign, buf + progress, copy_size);
-        progress += copy_size;
-    }
-
-    if (actual_written)
-        *actual_written = progress;
-    return status;
-}
-
-errno_t vmo_copy(
-    struct vmo* target,
-    uintptr_t target_offset,
-    struct vmo* src,
-    uintptr_t src_offset,
-    size_t len,
-    size_t* actual_copied
-) {
-    const size_t page_size = arch_mem_page_size();
-    size_t progress = 0;
-    errno_t status = 0;
-
-    while (progress < len) {
-        const size_t target_misalign = (progress + target_offset) % page_size;
-        const size_t src_misalign = (progress + src_offset) % page_size;
-        const size_t target_page_index = (progress + target_offset) / page_size;
-        const size_t src_page_index = (progress + src_offset) / page_size;
-
-        const size_t copy_size = MIN(MIN(page_size - target_misalign, page_size - src_misalign), len - progress);
-
-        struct page* tpage;
-        status = target->get_page(target, target_page_index, &tpage);
-        if (status != 0)
-            break;
-
-        struct page* spage;
-        status = src->get_page(src, src_page_index, &spage);
-        if (status != 0)
-            break;
-
-        uintptr_t taddr = vm_page_addr(tpage);
-        uintptr_t saddr = vm_page_addr(spage);
-
-        memcpy(HHDM_PTR(taddr) + target_misalign, HHDM_PTR(saddr) + src_misalign, copy_size);
-        progress += copy_size;
-    }
-
-    if (actual_copied)
-        *actual_copied = progress;
-    return status;
-}
-
-static errno_t paged_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
-    struct paged_vmo* paged = containerof(vmo, struct paged_vmo, object);
-
-    // Try to find the requested page in the cache.
-    struct page_list* iter;
-    list_foreach(iter, &paged->cache, next) {
-        if (iter->offset != offset_idx)
-            continue;
-
-        // We found a cached page!
-        *out = iter->value;
-        return 0;
-    }
-
-    // Get page from out pager if we don't have one already.
-    struct page* new_page;
-    errno_t status = paged->source.get_page(&paged->source, offset_idx, &new_page);
-    if (status)
-        return status;
-
-    struct page_list* entry = mem_alloc(sizeof(struct page_list), 0);
-    if (!entry)
-        return ENOMEM;
-
-    entry->offset = offset_idx;
-    entry->value = new_page;
-
-    // Add page to the object cache.
-    list_insert_head(&paged->cache, entry, next);
-
-    *out = new_page;
-    return 0;
-}
-
-errno_t vmo_new_paged(struct pager_ops pager, struct paged_vmo** out) {
-    struct paged_vmo* result = mem_alloc(sizeof(struct paged_vmo), 0);
-    if (!result)
-        return ENOMEM;
-
-    result->source = pager;
-    result->object.get_page = paged_get_page;
-
-    *out = result;
-    return 0;
-}
-
-static errno_t phys_get_page(struct pager_ops* pager, uintptr_t offset, struct page** out) {
-    phys_t addr;
-    errno_t status = mem_phys_alloc(1, 0, &addr);
-    if (status != 0)
-        return status;
-    size_t idx = addr / arch_mem_page_size();
-    *out = &vm_pfndb[idx];
-    return 0;
-}
-
-static errno_t phys_put_page(struct pager_ops* pager, uintptr_t offset, struct page* page) {
-    // We do nothing here.
-    return 0;
-}
-
-static const struct pager_ops phys_pager = {
-    .get_page = phys_get_page,
-    .put_page = phys_put_page,
-};
-
-errno_t vmo_new_phys(struct paged_vmo** out) {
-    return vmo_new_paged(phys_pager, out);
-}
-
-struct phys_range_vmo {
-    struct vmo object;
-    uintptr_t start;
-    size_t len;
-};
-
-static errno_t phys_range_get_page(struct vmo* vmo, uintptr_t offset_idx, struct page** out) {
-    struct phys_range_vmo* p = containerof(vmo, struct phys_range_vmo, object);
-    *out = &vm_pfndb[(p->start / arch_mem_page_size()) + offset_idx];
-    return 0;
-}
-
-errno_t vmo_new_phys_range(uintptr_t addr, size_t length, struct vmo** out) {
-    struct phys_range_vmo* vmo = mem_alloc(sizeof(struct phys_range_vmo), 0);
-    if (!vmo)
-        return ENOMEM;
-
-    vmo->object.get_page = phys_range_get_page;
-    vmo->start = ALIGN_DOWN(addr, arch_mem_page_size());
-    vmo->len = ALIGN_UP(length, arch_mem_page_size());
-
-    *out = &vmo->object;
-    return 0;
-}

core/meson.build

@@ -1,15 +0,0 @@
-kernel_sources += files(
-    'clock.c',
-    'console.c',
-    'elf.c',
-    'irq.c',
-    'panic.c',
-    'percpu.c',
-    'print.c',
-    'spin.c',
-    'wait.c',
-)
-
-subdir('lib')
-subdir('mem')
-subdir('sched')

core/panic.c

@@ -1,36 +0,0 @@
-#include <zinnia/irq.h>
-#include <zinnia/panic.h>
-#include <zinnia/percpu.h>
-#include <zinnia/print.h>
-#include <stdarg.h>
-
-#define panic_log(msg, ...) kprintf("[    !!!!    ] \e[31m" msg "\e[0m", ##__VA_ARGS__)
-
-[[noreturn]]
-void panic(const char* msg, ...) {
-    // TODO: Stop all other CPUs with IPI.
-    irq_set_state(false);
-
-    struct percpu* cpu = percpu_get();
-    struct task* current = cpu->sched.current;
-    kprintf(
-        "[    !!!!    ] \e[31m"
-        "Kernel panic on CPU %zu (T%zu, \"%s\"): ",
-        cpu->id,
-        current ? current->id : 0,
-        current ? current->name : nullptr
-    );
-
-    va_list args;
-    va_start(args, msg);
-    kvprintf(msg, args);
-    va_end(args);
-
-    kprintf("\e[0m");
-
-    panic_log("End of panic message\n");
-
-    while (1) {
-        irq_wait();
-    }
-}

core/percpu.c

@@ -1,27 +0,0 @@
-#include <zinnia/compiler.h>
-#include <zinnia/percpu.h>
-#include <arch/percpu.h>
-
-[[__used, __section(".percpu.init")]]
-struct percpu percpu_bsp = {
-    .self = &percpu_bsp,
-    .id = 0,
-    .online = false,
-};
-
-struct percpu* percpu_new() {
-    // TODO
-    return nullptr;
-}
-
-struct percpu* percpu_get() {
-    return arch_percpu_get();
-}
-
-void percpu_bsp_init() {
-    arch_percpu_bsp_init();
-}
-
-void percpu_init() {
-    arch_percpu_init();
-}

core/print.c

@@ -1,301 +0,0 @@
-#include <zinnia/clock.h>
-#include <zinnia/console.h>
-#include <zinnia/panic.h>
-#include <zinnia/print.h>
-#include <stdarg.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <zinnia/stdlib.h>
-#include <zinnia/string.h>
-
-typedef void (*log_fn_t)(void* ctx, const char* buf, size_t len);
-
-static void do_print(log_fn_t callback, void* ctx, const char* restrict fmt, va_list args) {
-    // Amount of bytes written.
-    int32_t written = 0;
-
-    while (*fmt != '\0') {
-        size_t maxrem = INT32_MAX - written;
-
-        if (fmt[0] != '%' || fmt[1] == '%') {
-            if (fmt[0] == '%')
-                fmt++;
-            size_t amount = 1;
-            while (fmt[amount] && fmt[amount] != '%')
-                amount++;
-            if (maxrem < amount) {
-                return;
-            }
-            callback(ctx, fmt, amount);
-            fmt += amount;
-            written += amount;
-            continue;
-        }
-
-        const char* format_begun_at = fmt++;
-        bool force_sign = false;
-        bool blank_sign = false;
-        bool write_prefix = false;
-        bool pad_zero = false;
-        bool has_width = false;
-        size_t width = 0;
-        bool has_precision = false;
-        size_t precision = 0;
-        size_t size = sizeof(uint32_t);
-        char number[32];
-check_fmt:
-        memset(number, 0, sizeof(number));
-
-        switch (*fmt) {
-        // Flags
-        case '+': {
-            force_sign = true;
-            fmt++;
-            goto check_fmt;
-        }
-        case ' ': {
-            blank_sign = true;
-            fmt++;
-            goto check_fmt;
-        }
-        case '#': {
-            write_prefix = true;
-            fmt++;
-            goto check_fmt;
-        }
-        case '0': {
-            pad_zero = true;
-            fmt++;
-            goto check_fmt;
-        }
-        // Width
-        case '1':
-        case '2':
-        case '3':
-        case '4':
-        case '5':
-        case '6':
-        case '7':
-        case '8':
-        case '9': {
-            has_width = true;
-            char number[10 + 1] = {0};
-            size_t idx = 0;
-            while (*fmt >= '0' && *fmt <= '9') {
-                number[idx++] = *fmt;
-                fmt++;
-            }
-            width = atolu(number, 10);
-
-            goto check_fmt;
-        }
-        // Precision
-        case '.': {
-            has_precision = true;
-            fmt++;
-            if (*fmt == '*') {
-                precision = (char)va_arg(args, int32_t);
-            } else {
-                char number[10 + 1] = {0};
-                size_t idx = 0;
-                while (*fmt >= '0' && *fmt <= '9') {
-                    number[idx++] = *fmt;
-                    fmt++;
-                }
-                precision = atolu(number, 10);
-            }
-            goto check_fmt;
-        }
-        // Length
-        case 'h': {
-            if (fmt[1] == 'h') {
-                size = sizeof(uint8_t);
-                fmt++;
-            } else
-                size = sizeof(uint16_t);
-            fmt++;
-            goto check_fmt;
-        }
-        case 'l': {
-            // There is no difference between `long int` and `long long int`.
-            if (fmt[1] == 'l')
-                fmt++;
-            fmt++;
-            size = sizeof(uint64_t);
-
-            goto check_fmt;
-        }
-        case 'z': {
-            size = sizeof(size_t);
-            fmt++;
-            goto check_fmt;
-        }
-        // Character
-        case 'c': {
-            char c = (char)va_arg(args, int32_t);
-
-            callback(ctx, &c, sizeof(c));
-
-            written++;
-            fmt++;
-            break;
-        }
-        // String of characters
-        case 's': {
-            const char* str = va_arg(args, const char*);
-            if (!str)
-                str = "(null)";
-            size_t len = 0;
-            if (has_precision)
-                len = strnlen(str, precision);
-            else
-                len = strlen(str);
-
-            callback(ctx, str, len);
-
-            written += len;
-            fmt++;
-            break;
-        }
-        // Signed decimal integer
-        case 'i':
-        case 'd': {
-            switch (size) {
-            case sizeof(int8_t):
-                ltoa((int8_t)va_arg(args, int32_t), number, 10);
-                break;
-            case sizeof(int16_t):
-                ltoa((int16_t)va_arg(args, int32_t), number, 10);
-                break;
-            case sizeof(int32_t):
-                ltoa((int32_t)va_arg(args, int32_t), number, 10);
-                break;
-            default:
-                ltoa(va_arg(args, int64_t), number, 10);
-                break;
-            }
-            goto print_num;
-        }
-        // Unsigned decimal integer
-        case 'u': {
-            switch (size) {
-            case sizeof(uint8_t):
-                lutoa((uint8_t)va_arg(args, uint32_t), number, 10);
-                break;
-            case sizeof(uint16_t):
-                lutoa((uint16_t)va_arg(args, uint32_t), number, 10);
-                break;
-            case sizeof(uint32_t):
-                lutoa(va_arg(args, uint32_t), number, 10);
-                break;
-            case sizeof(uint64_t):
-                lutoa(va_arg(args, uint64_t), number, 10);
-                break;
-            }
-            goto print_num;
-        }
-        // Unsigned hexadecimal integer
-        case 'X':
-        case 'x': {
-            switch (size) {
-            case sizeof(uint8_t):
-                lutoa((uint8_t)va_arg(args, uint32_t), number, 16);
-                break;
-            case sizeof(uint16_t):
-                lutoa((uint16_t)va_arg(args, uint32_t), number, 16);
-                break;
-            case sizeof(uint32_t):
-                lutoa(va_arg(args, uint32_t), number, 16);
-                break;
-            case sizeof(uint64_t):
-                lutoa(va_arg(args, uint64_t), number, 16);
-                break;
-            }
-print_num:
-            size_t len = strlen(number);
-            if (has_precision) {
-                for (size_t i = 0; i < precision - len; i++)
-                    callback(ctx, " ", 1);
-            }
-            if (has_width) {
-                char c = pad_zero ? '0' : ' ';
-
-                // We don't have to pad anything.
-                if (len < width) {
-                    for (size_t i = 0; i < width - len; i++)
-                        callback(ctx, &c, 1);
-                }
-            }
-            if (write_prefix) {
-                callback(ctx, "0x", 2);
-            }
-            if (force_sign) {
-                char c = '+';
-                if (number[0] != '-')
-                    callback(ctx, &c, 1);
-            }
-            if (blank_sign) {
-                char c = ' ';
-                if (number[0] != '-')
-                    callback(ctx, &c, 1);
-            }
-            callback(ctx, number, len);
-
-            written += len;
-            fmt++;
-            break;
-        }
-        // Pointer address
-        case 'p': {
-            const size_t num = va_arg(args, size_t);
-            const size_t buf_size = sizeof(size_t) * 2 + 1;
-            char str[buf_size];
-            lutoa(num, number, 0x10);
-            const size_t len = strlen(number); // Get the length of the final number.
-            for (size_t i = 0; i < buf_size; i++)
-                str[i] = '0';
-            size_t offset = sizeof(str) - len - 1;
-
-            callback(ctx, str, offset);
-            callback(ctx, number, len);
-
-            fmt++;
-            written += len;
-            break;
-        }
-        // The number of characters written so far is stored in the pointed location.
-        case 'n': {
-            int32_t* const ptr = va_arg(args, int32_t*);
-            *ptr = written;
-            fmt++;
-            break;
-        }
-        // No format, just normal text.
-        default: {
-            fmt = format_begun_at;
-            size_t len = strlen(fmt);
-
-            callback(ctx, fmt, len);
-
-            written += len;
-            fmt += len;
-            break;
-        }
-        }
-    }
-}
-
-static void printf_console(void*, const char* msg, size_t len) {
-    console_write(msg, len);
-}
-
-void kprintf(const char* message, ...) {
-    va_list args;
-    va_start(args, message);
-    do_print(printf_console, nullptr, message, args);
-    va_end(args);
-}
-
-void kvprintf(const char* message, va_list args) {
-    do_print(printf_console, nullptr, message, args);
-}

core/sched/core.c

@@ -1 +0,0 @@
-#include <zinnia/sched.h>

core/sched/sched_dumb.c

@@ -1,134 +0,0 @@
-#include <zinnia/alloc.h>
-#include <zinnia/assert.h>
-#include <zinnia/atomic.h>
-#include <zinnia/errno.h>
-#include <zinnia/irq.h>
-#include <zinnia/list.h>
-#include <zinnia/percpu.h>
-#include <zinnia/pmap.h>
-#include <zinnia/print.h>
-#include <zinnia/sched.h>
-#include <zinnia/spin.h>
-#include <zinnia/tailq.h>
-#include <zinnia/vma.h>
-#include <arch/sched.h>
-#include <stdint.h>
-#include <zinnia/string.h>
-
-static void idle_func(uintptr_t) {
-    while (1) {
-        irq_set_state(true);
-        irq_wait();
-    }
-}
-
-void sched_to_user(uintptr_t info_ptr) {
-    struct task_start_info* info = (struct task_start_info*)info_ptr;
-    uintptr_t ip = info->ip;
-    uintptr_t sp = info->sp;
-    uintptr_t arg = info->arg;
-    mem_free(info);
-    arch_sched_jump_to_user(ip, sp, arg);
-}
-
-void sched_to_user_context(uintptr_t ctx) {
-    arch_sched_jump_to_context((struct arch_context*)ctx);
-}
-
-void sched_init(struct scheduler* sched) {
-    struct task* idle_task;
-    assert(task_create("idle", &kernel_vma, idle_func, 0, &idle_task) == 0, "Unable to create idle task!\n");
-    sched->idle_task = idle_task;
-    sched->current = idle_task;
-
-    tailq_init(&sched->run_queue);
-
-    log("sched: Scheduler initialized\n");
-}
-
-void sched_add_task(struct scheduler* sched, struct task* task) {
-    assert(task != nullptr, "Tried to add NULL to the run queue\n");
-    tailq_insert_tail(&sched->run_queue, task, next);
-}
-
-static struct task* next_task(struct scheduler* sched) {
-    struct task* t = tailq_first(&sched->run_queue);
-
-    if (t == nullptr)
-        return sched->idle_task;
-
-    tailq_remove(&sched->run_queue, t, next);
-
-    return t;
-}
-
-static void do_reschedule(struct scheduler* sched) {
-    struct task* from = sched->current;
-    struct task* to = next_task(sched);
-
-    // We'd end up on the same task, skip switching altogether.
-    if (from == to) {
-        irq_unlock();
-        return;
-    }
-
-    sched->current = to;
-    pmap_set(&to->space->pmap);
-
-    struct percpu* cpu = percpu_get();
-    from->kernel_stack = cpu->kernel_stack;
-    from->user_stack = cpu->user_stack;
-    cpu->kernel_stack = to->kernel_stack;
-    cpu->user_stack = to->user_stack;
-
-    struct task* old = arch_sched_switch(from, to);
-}
-
-void sched_reschedule(struct scheduler* sched) {
-    irq_lock();
-    if (sched->current->state != TASK_STATE_DEAD)
-        sched_add_task(sched, sched->current);
-    do_reschedule(sched);
-}
-
-void sched_yield(struct scheduler* sched) {
-    irq_lock();
-    do_reschedule(sched);
-}
-
-errno_t task_create(const char* name, struct vma* space, task_fn_t entry, uintptr_t arg0, struct task** out) {
-    if (!out)
-        return EINVAL;
-
-    struct task* new_task = mem_alloc(sizeof(struct task), 0);
-    if (new_task == nullptr)
-        return ENOMEM;
-
-    static size_t last_tid = 0;
-    new_task->id = atomic_fetch_add(&last_tid, 1, ATOMIC_ACQUIRE);
-    new_task->space = space;
-    strncpy(new_task->name, name, sizeof(new_task->name));
-
-    // Allocate a kernel stack.
-    new_task->kernel_stack = (uintptr_t)mem_alloc(KERNEL_STACK_SIZE, 0);
-    if (!new_task->kernel_stack)
-        return ENOMEM;
-    new_task->user_stack = 0;
-
-    // Initialize the arch-dependent parts.
-    errno_t s = arch_task_init(&new_task->context, entry, arg0, new_task->kernel_stack + KERNEL_STACK_SIZE, true);
-    if (s != 0)
-        return s;
-
-    *out = new_task;
-    return 0;
-}
-
-void task_entry(task_fn_t entry, uintptr_t arg0) {
-    entry(arg0);
-
-    struct scheduler* sched = &percpu_get()->sched;
-    sched->current->state = TASK_STATE_DEAD;
-    sched_yield(sched);
-    panic("unreachable");
-}

core/spin.c

@@ -1,15 +0,0 @@
-#include <zinnia/atomic.h>
-#include <zinnia/spin.h>
-
-void spin_lock(struct spinlock* spin) {
-    uint32_t my = atomic_fetch_add(&spin->next, 1, ATOMIC_RELAXED);
-
-    while (atomic_load(&spin->owner, ATOMIC_ACQUIRE) != my) {
-        asm volatile("pause");
-    }
-}
-
-void spin_unlock(struct spinlock* spin) {
-    uint32_t val = atomic_load(&spin->owner, ATOMIC_RELAXED);
-    atomic_store(&spin->owner, val + 1, ATOMIC_RELEASE);
-}

core/wait.c

@@ -1,17 +0,0 @@
-#include <zinnia/waitq.h>
-
-void waitq_init(struct waitq* wq) {
-    tailq_init(&wq->waiters);
-}
-
-void waitq_wait(struct waitq* wq) {
-    // TODO
-}
-
-void waitq_wake_one(struct waitq* wq) {
-    // TODO
-}
-
-void waitq_wake_all(struct waitq* wq) {
-    // TODO
-}

dev/Kconfig

@@ -14,5 +14,6 @@ config ACPI
 
 source "dev/block/Kconfig"
 source "dev/pci/Kconfig"
+source "dev/video/Kconfig"
 
 endmenu

dev/acpi/include/uacpi_types.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdarg.h>

dev/acpi/include/zinnia/acpi.h

@@ -1,8 +1,26 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
+#include <zinnia/errno.h>
 #include <zinnia/init.h>
+#include <zinnia/initgraph.h>
 #include <zinnia/types.h>
+#include <zinnia/workqueue.h>
+#include <stdint.h>
 
-void acpi_init(struct boot_info* info);
+INIT_EXTERN(acpi_stage);
+INIT_EXTERN(acpi_namespace_stage);
 
 extern phys_t acpi_rsdp_addr;
+extern struct workqueue* acpi_workqueue;
+
+errno_t acpi_reboot();
+errno_t acpi_power_off();
+
+struct acpi_match {
+    const char* hid;
+    uintptr_t data;
+};
+
+#define ACPI_MATCH_HID(id) .hid = (id)

dev/acpi/init.c

@@ -1,13 +1,117 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "acpi: " fmt
+
 #include <zinnia/acpi.h>
+#include <zinnia/alloc.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/irq.h>
+#include <zinnia/pci.h>
+#include <zinnia/percpu.h>
 #include <zinnia/print.h>
+#include <zinnia/sched.h>
 #include <zinnia/types.h>
+#include <zinnia/workqueue.h>
+#include <uacpi/event.h>
+#include <uacpi/sleep.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+#include <uacpi/uacpi.h>
 
 phys_t acpi_rsdp_addr = 0;
+struct workqueue* acpi_workqueue;
+
+static char table_buf[0x1000];
+
+static bool use_acpi = true;
+void use_acpi_option(bool use) {
+    use_acpi = use;
+}
+CMDLINE_OPTION("acpi", "Enable ACPI firmware subsystem", use_acpi_option);
+
+[[__init]]
+static bool acpi_stage_init(struct boot_info* info) {
+    if (!use_acpi) {
+        log_warn("ACPI disabled on the command line!\n");
+        return false;
+    }
 
-void acpi_init(struct boot_info* info) {
     if (!info->rsdp) {
-        log("acpi: No RSDP was passed, unable to start ACPI subsystem!\n");
-        return;
+        log_warn("No RSDP was passed, unable to start ACPI subsystem!\n");
+        return false;
     }
     acpi_rsdp_addr = info->rsdp;
+
+    if (uacpi_setup_early_table_access(table_buf, sizeof(table_buf)) != UACPI_STATUS_OK) {
+        log_err("Failed to set up early table access!\n");
+        return false;
+    }
+
+    return true;
+}
+INIT_TASK(acpi_stage, "dev.acpi", acpi_stage_init);
+INIT_DEPENDS(acpi_stage, mem_stage);
+INIT_ENTAILS(acpi_stage, percpu_stage);
+
+static bool namespace_done;
+
+[[__init]]
+static void acpi_namespace_init(struct boot_info*) {
+    errno_t error = workqueue_create("acpi", SCHED_PRIO_WORK_HIGH, &acpi_workqueue);
+    if (error) {
+        log_err("Failed to create the ACPI workqueue (%i)\n", error);
+        return;
+    }
+
+    uacpi_status status = uacpi_initialize(0);
+    if (status != UACPI_STATUS_OK) {
+        log_err("Initialization failed with error \"%s\"!\n", uacpi_status_to_string(status));
+        return;
+    }
+
+    status = uacpi_namespace_load();
+    if (status != UACPI_STATUS_OK) {
+        log_err("Namespace loading failed with error \"%s\"!\n", uacpi_status_to_string(status));
+        return;
+    }
+
+    status = uacpi_namespace_initialize();
+    if (status != UACPI_STATUS_OK) {
+        log_err("Namespace initialization failed with error \"%s\"!\n", uacpi_status_to_string(status));
+        return;
+    }
+
+    status = uacpi_finalize_gpe_initialization();
+    if (status != UACPI_STATUS_OK)
+        log_warn("GPE initialization failed with error \"%s\"!\n", uacpi_status_to_string(status));
+
+    namespace_done = true;
+}
+INIT_TASK(acpi_namespace_stage, "dev.acpi.namespace", acpi_namespace_init);
+INIT_DEPENDS(acpi_namespace_stage, acpi_stage, workqueue_stage);
+#if CONFIG_PCI
+INIT_DEPENDS(acpi_namespace_stage, pci_stage);
+#endif
+
+errno_t acpi_reboot() {
+    if (!namespace_done)
+        return ENOSYS;
+
+    uacpi_status status = uacpi_reboot();
+    log_err("Failed to reset the system: %s\n", uacpi_status_to_string(status));
+    return EIO;
+}
+
+errno_t acpi_power_off() {
+    if (!namespace_done)
+        return ENOSYS;
+
+    uacpi_status status = uacpi_prepare_for_sleep_state(UACPI_SLEEP_STATE_S5);
+    if (status == UACPI_STATUS_OK) {
+        irq_lock();
+        status = uacpi_enter_sleep_state(UACPI_SLEEP_STATE_S5);
+        irq_unlock();
+    }
+    log_err("Failed to power off the system: %s\n", uacpi_status_to_string(status));
+    return EIO;
 }

dev/acpi/mcfg.c

@@ -0,0 +1,32 @@
+#define pr_fmt(fmt) "acpi: " fmt
+
+#include <zinnia/acpi.h>
+#include <zinnia/pci.h>
+#include <zinnia/print.h>
+#include <uacpi/acpi.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+
+[[__init]]
+static void acpi_mcfg_stage_init(struct boot_info*) {
+    uacpi_table table = {};
+    if (uacpi_table_find_by_signature(ACPI_MCFG_SIGNATURE, &table) != UACPI_STATUS_OK)
+        return;
+
+    struct acpi_mcfg* mcfg = table.ptr;
+    size_t count = 0;
+    if (mcfg->hdr.length > sizeof(*mcfg))
+        count = (mcfg->hdr.length - sizeof(*mcfg)) / sizeof(struct acpi_mcfg_allocation);
+
+    for (size_t i = 0; i < count; i++) {
+        struct acpi_mcfg_allocation* entry = &mcfg->entries[i];
+        errno_t status = pci_ecam_register(entry->address, entry->segment, entry->start_bus, entry->end_bus);
+        if (status != 0)
+            log_warn("Failed to register ECAM for segment %hu: %i\n", entry->segment, status);
+    }
+
+    uacpi_table_unref(&table);
+}
+INIT_TASK(acpi_mcfg_stage, "dev.acpi.mcfg", acpi_mcfg_stage_init);
+INIT_DEPENDS(acpi_mcfg_stage, acpi_stage);
+INIT_ENTAILS(acpi_mcfg_stage, pci_stage);

dev/acpi/meson.build

@@ -6,11 +6,14 @@ kernel_sources += files(
     'uacpi_glue.c',
 )
 
+if kconfig.get('CONFIG_PCI', 'n') == 'y'
+    kernel_sources += files('mcfg.c')
+endif
+
 kernel_includes += uacpi.get_variable('includes')
 kernel_includes += include_directories('include')
 
 kernel_c_args += [
     '-DUACPI_FORMATTED_LOGGING',
-    '-DUACPI_BAREBONES_MODE', # TODO: Remove
     '-DUACPI_OVERRIDE_TYPES',
 ]

dev/acpi/uacpi_glue.c

@@ -1,12 +1,24 @@
+// SPDX-License-Identifier: ISC
+
 #include <zinnia/acpi.h>
+#include <zinnia/alloc.h>
+#include <zinnia/clock.h>
 #include <zinnia/console.h>
 #include <zinnia/init.h>
+#include <zinnia/irq.h>
 #include <zinnia/mmio.h>
+#include <zinnia/pci.h>
+#include <zinnia/percpu.h>
 #include <zinnia/pmap.h>
 #include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/semaphore.h>
+#include <zinnia/spin.h>
 #include <zinnia/utils.h>
 #include <zinnia/virt.h>
 #include <zinnia/vma.h>
+#include <zinnia/workqueue.h>
+#include <arch/irq.h>
 #include <arch/mem.h>
 #include <stdarg.h>
 #include <uacpi/kernel_api.h>
@@ -23,29 +35,297 @@ void* uacpi_kernel_map(uacpi_phys_addr addr, uacpi_size len) {
     uintptr_t difference = addr - aligned_addr;
     size_t aligned_len = ALIGN_UP(len + difference, arch_mem_page_size());
 
-    uintptr_t vaddr = vm_alloc(len);
-    for (size_t i = 0; i < len; i += arch_mem_page_size()) {
-        pmap_map(&kernel_vma.pmap, vaddr + i, addr + i, PROT_READ | PROT_WRITE, CACHE_ANY);
+    uintptr_t vaddr = vm_alloc(aligned_len);
+    if (!vaddr)
+        return nullptr;
+
+    for (size_t i = 0; i < aligned_len; i += arch_mem_page_size()) {
+        pmap_map(&kernel_vma.pmap, vaddr + i, aligned_addr + i, PTE_READ | PTE_WRITE, CACHE_ANY);
     }
 
-    return (void*)vaddr;
+    return (void*)(vaddr + difference);
 }
 
 void uacpi_kernel_unmap(void* addr, uacpi_size len) {
-    // TODO
+    uintptr_t aligned_addr = ALIGN_DOWN((uintptr_t)addr, arch_mem_page_size());
+    uintptr_t difference = (uintptr_t)addr - aligned_addr;
+    size_t aligned_len = ALIGN_UP(len + difference, arch_mem_page_size());
+
+    for (size_t i = 0; i < aligned_len; i += arch_mem_page_size())
+        pmap_unmap(&kernel_vma.pmap, aligned_addr + i);
+    vm_free(aligned_addr, aligned_len);
+}
+
+static enum log_level acpi_log_level(uacpi_log_level lvl) {
+    switch (lvl) {
+    case UACPI_LOG_ERROR:
+        return LOG_ERR;
+
+    case UACPI_LOG_WARN:
+        return LOG_WARN;
+
+    case UACPI_LOG_INFO:
+        return LOG_INFO;
+
+    default:
+        return LOG_DEBUG;
+    }
 }
 
 void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) {
-    log("acpi: ");
-
     va_list args;
     va_start(args, msg);
-
-    kvprintf(msg, args);
-
+    log_vwrite_prefixed(acpi_log_level(lvl), "acpi: ", msg, args);
     va_end(args);
 }
 
 void uacpi_kernel_vlog(uacpi_log_level lvl, const uacpi_char* msg, uacpi_va_list args) {
-    kvprintf(msg, args);
+    log_vwrite_prefixed(acpi_log_level(lvl), "acpi: ", msg, args);
+}
+
+uacpi_status uacpi_kernel_pci_device_open(uacpi_pci_address address, uacpi_handle* out_handle) {
+#if CONFIG_PCI
+    struct pci_address addr = {
+        .domain = address.segment,
+        .bus = address.bus,
+        .device = address.device,
+        .function = address.function,
+    };
+    struct pci_access* access = pci_access_find(&addr);
+    if (!access)
+        return UACPI_STATUS_NOT_FOUND;
+
+    struct pci_device* dev = mem_alloc(sizeof(*dev), 0);
+    if (!dev)
+        return UACPI_STATUS_OUT_OF_MEMORY;
+    dev->address = addr;
+    dev->access = access;
+    *out_handle = dev;
+    return UACPI_STATUS_OK;
+#else
+    return UACPI_STATUS_NOT_FOUND;
+#endif
+}
+
+void uacpi_kernel_pci_device_close(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+#if CONFIG_PCI
+#define PCI_READ(width) \
+    uacpi_status uacpi_kernel_pci_read##width(uacpi_handle device, uacpi_size offset, uacpi_u##width* value) { \
+        *value = pci_read##width(device, offset); \
+        return UACPI_STATUS_OK; \
+    }
+#define PCI_WRITE(width) \
+    uacpi_status uacpi_kernel_pci_write##width(uacpi_handle device, uacpi_size offset, uacpi_u##width value) { \
+        pci_write##width(device, offset, value); \
+        return UACPI_STATUS_OK; \
+    }
+#else
+#define PCI_READ(width) \
+    uacpi_status uacpi_kernel_pci_read##width(uacpi_handle device, uacpi_size offset, uacpi_u##width* value) { \
+        return UACPI_STATUS_UNIMPLEMENTED; \
+    }
+#define PCI_WRITE(width) \
+    uacpi_status uacpi_kernel_pci_write##width(uacpi_handle device, uacpi_size offset, uacpi_u##width value) { \
+        return UACPI_STATUS_UNIMPLEMENTED; \
+    }
+#endif
+
+PCI_READ(8)
+PCI_READ(16)
+PCI_READ(32)
+PCI_WRITE(8)
+PCI_WRITE(16)
+PCI_WRITE(32)
+
+void* uacpi_kernel_alloc(uacpi_size size) {
+    return mem_alloc(size, ALLOC_NOZERO);
+}
+
+void uacpi_kernel_free(void* mem) {
+    if (mem)
+        mem_free(mem);
+}
+
+uacpi_u64 uacpi_kernel_get_nanoseconds_since_boot() {
+    return clock_get_elapsed_ns();
+}
+
+void uacpi_kernel_stall(uacpi_u8 usec) {
+    clock_spin_ns(usec * 1000ull);
+}
+
+void uacpi_kernel_sleep(uacpi_u64 msec) {
+    sched_sleep_until(clock_get_elapsed_ns() + msec * 1'000'000);
+}
+
+uacpi_handle uacpi_kernel_create_semaphore(uacpi_u32 initial_units) {
+    sem_t* sem = mem_alloc(sizeof(*sem), 0);
+    if (sem)
+        sem_init(sem, initial_units);
+    return sem;
+}
+
+void uacpi_kernel_free_semaphore(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+uacpi_status uacpi_kernel_wait_for_semaphore(uacpi_handle handle, uacpi_u16 timeout) {
+    sem_t* sem = handle;
+    if (timeout == 0)
+        return sem_trywait(sem) ? UACPI_STATUS_OK : UACPI_STATUS_TIMEOUT;
+    if (timeout == 0xFFFF) {
+        sem_wait(sem);
+        return UACPI_STATUS_OK;
+    }
+
+    const uint64_t deadline = clock_get_elapsed_ns() + timeout * 1'000'000ull;
+    return sem_timedwait(sem, &deadline) ? UACPI_STATUS_OK : UACPI_STATUS_TIMEOUT;
+}
+
+void uacpi_kernel_signal_semaphore(uacpi_handle handle) {
+    sem_post(handle);
+}
+
+uacpi_thread_id uacpi_kernel_get_thread_id() {
+    return percpu_get()->sched.current;
+}
+
+uacpi_interrupt_state uacpi_kernel_disable_interrupts() {
+    irq_lock();
+    return 0;
+}
+
+void uacpi_kernel_restore_interrupts(uacpi_interrupt_state) {
+    irq_unlock();
+}
+
+uacpi_status uacpi_kernel_handle_firmware_request(uacpi_firmware_request* req) {
+    switch (req->type) {
+    case UACPI_FIRMWARE_REQUEST_TYPE_BREAKPOINT:
+        log_warn("acpi: Ignoring AML breakpoint\n");
+        break;
+    case UACPI_FIRMWARE_REQUEST_TYPE_FATAL:
+        log_err(
+            "acpi: Fatal firmware error: type %u, code %#x, arg %#llx\n",
+            req->fatal.type,
+            req->fatal.code,
+            (unsigned long long)req->fatal.arg
+        );
+        break;
+    }
+    return UACPI_STATUS_OK;
+}
+
+struct acpi_irq {
+    struct irq_handler handler;
+    uacpi_interrupt_handler func;
+    uacpi_handle ctx;
+};
+
+static enum irq_status acpi_irq_raise(struct irq_handler* self) {
+    struct acpi_irq* irq = containerof(self, struct acpi_irq, handler);
+    return irq->func(irq->ctx) == UACPI_INTERRUPT_HANDLED ? IRQ_STATUS_HANDLED : IRQ_STATUS_IGNORED;
+}
+
+uacpi_status uacpi_kernel_install_interrupt_handler(
+    uacpi_u32 irq,
+    uacpi_interrupt_handler handler,
+    uacpi_handle ctx,
+    uacpi_handle* out_irq_handle
+) {
+    uint32_t gsi = irq;
+    enum irq_trigger_mode trigger = IRQ_TRIGGER_LEVEL;
+    enum irq_polarity polarity = IRQ_POLARITY_LOW;
+    if (irq <= UINT8_MAX)
+        arch_irq_isa_override(irq, &gsi, &trigger, &polarity);
+
+    struct irq_line* line;
+    errno_t status = arch_irq_get_gsi(gsi, &line);
+    if (status) {
+        log_err("acpi: Unable to get GSI %u for IRQ %u (%i)\n", gsi, irq, status);
+        return UACPI_STATUS_UNIMPLEMENTED;
+    }
+
+    struct acpi_irq* acpi_irq = mem_alloc(sizeof(*acpi_irq), 0);
+    if (!acpi_irq)
+        return UACPI_STATUS_OUT_OF_MEMORY;
+    acpi_irq->handler.raise = acpi_irq_raise;
+    acpi_irq->func = handler;
+    acpi_irq->ctx = ctx;
+
+    irq_line_attach(line, &acpi_irq->handler);
+    irq_line_program(line, trigger, polarity);
+    log("acpi: Installed handler for IRQ %u on GSI %u\n", irq, gsi);
+
+    *out_irq_handle = acpi_irq;
+    return UACPI_STATUS_OK;
+}
+
+uacpi_status uacpi_kernel_uninstall_interrupt_handler(uacpi_interrupt_handler handler, uacpi_handle irq_handle) {
+    struct acpi_irq* acpi_irq = irq_handle;
+    struct irq_line* line = acpi_irq->handler.line;
+    irq_line_detach(line, &acpi_irq->handler);
+    if (list_is_empty(&line->handlers) && line->mask)
+        line->mask(line);
+    mem_free(acpi_irq);
+    return UACPI_STATUS_OK;
+}
+
+uacpi_handle uacpi_kernel_create_spinlock() {
+    struct spinlock* lock = mem_alloc(sizeof(*lock), 0);
+    if (lock)
+        spin_init(lock);
+    return lock;
+}
+
+void uacpi_kernel_free_spinlock(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+uacpi_cpu_flags uacpi_kernel_lock_spinlock(uacpi_handle handle) {
+    spin_lock(handle);
+    return 0;
+}
+
+void uacpi_kernel_unlock_spinlock(uacpi_handle handle, uacpi_cpu_flags) {
+    spin_unlock(handle);
+}
+
+struct acpi_work {
+    struct work work;
+    uacpi_work_handler handler;
+    uacpi_handle ctx;
+};
+
+static void acpi_work_fn(struct work* work) {
+    struct acpi_work* item = containerof(work, struct acpi_work, work);
+    const uacpi_work_handler handler = item->handler;
+    const uacpi_handle ctx = item->ctx;
+    handler(ctx);
+}
+
+uacpi_handle uacpi_kernel_create_work_item() {
+    struct acpi_work* item = mem_alloc(sizeof(*item), 0);
+    if (item)
+        work_init(&item->work, acpi_work_fn);
+    return item;
+}
+
+void uacpi_kernel_free_work_item(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+void uacpi_kernel_schedule_work(uacpi_work_type, uacpi_handle work_item, uacpi_work_handler handler, uacpi_handle ctx) {
+    struct acpi_work* item = work_item;
+    item->handler = handler;
+    item->ctx = ctx;
+    work_queue(acpi_workqueue, &item->work);
+}
+
+uacpi_status uacpi_kernel_wait_for_work_completion() {
+    workqueue_flush(acpi_workqueue);
+    return UACPI_STATUS_OK;
 }

dev/block/bio.c

@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/utils.h>
+
+struct bio* bio_alloc(enum bio_op op, size_t num_segs) {
+    struct bio* bio = mem_alloc(sizeof(struct bio) + num_segs * sizeof(struct bio_seg), 0);
+    if (!bio)
+        return nullptr;
+    bio->op = op;
+    bio->num_segs = num_segs;
+    event_init(&bio->event);
+    atomic_init(&bio->pending, 0);
+    atomic_init(&bio->short_offset, UINT64_MAX);
+    atomic_init(&bio->child_status, 0);
+    return bio;
+}
+
+void bio_free(struct bio* bio) {
+    mem_free(bio);
+}
+
+static void record_short(struct bio* parent, uint64_t offset, errno_t status) {
+    uint64_t current = atomic_load(&parent->short_offset, ATOMIC_RELAXED);
+    while (offset < current) {
+        if (atomic_compare_exchange_weak(&parent->short_offset, &current, offset, ATOMIC_RELAXED, ATOMIC_RELAXED))
+            break;
+    }
+    if (status) {
+        errno_t expected = 0;
+        atomic_compare_exchange(&parent->child_status, &expected, status, ATOMIC_RELAXED, ATOMIC_RELAXED);
+    }
+}
+
+static void parent_put(struct bio* parent) {
+    if (atomic_fetch_sub(&parent->pending, 1, ATOMIC_ACQ_REL) != 1)
+        return;
+    const uint64_t offset = atomic_load(&parent->short_offset, ATOMIC_RELAXED);
+    const size_t done = offset == UINT64_MAX ? parent->num_lbas : offset;
+    const errno_t status = atomic_load(&parent->child_status, ATOMIC_RELAXED);
+    bio_complete(parent, done == 0 ? status : 0, done);
+}
+
+void bio_complete(struct bio* bio, errno_t status, size_t done_lbas) {
+    bio->status = status;
+    bio->done_lbas = done_lbas;
+
+    struct bio* parent = bio->parent;
+    if (parent) {
+        if (status || done_lbas < bio->num_lbas)
+            record_short(parent, bio->parent_offset + done_lbas, status);
+        bio_free(bio);
+        parent_put(parent);
+        return;
+    }
+
+    if (bio->end)
+        bio->end(bio);
+    else
+        event_set(&bio->event);
+}
+
+errno_t bio_wait(struct bio* bio, size_t* out_lbas) {
+    event_wait(&bio->event);
+    if (out_lbas)
+        *out_lbas = bio->done_lbas;
+    return bio->status;
+}
+
+// Submits one bio and completes early if it failed.
+static void submit_one(struct blockdev* dev, struct bio* bio) {
+    errno_t status = dev->ops->submit(dev, bio);
+    if (status)
+        bio_complete(bio, status, 0);
+}
+
+void bio_submit(struct blockdev* dev, struct bio* bio) {
+    bio->dev = dev;
+    if (bio->op == BIO_FLUSH) {
+        submit_one(dev, bio);
+        return;
+    }
+
+    if (bio->op == BIO_READ) {
+        if (bio->lba >= dev->lba_count) {
+            bio_complete(bio, 0, 0);
+            return;
+        }
+        bio->num_lbas = MIN(bio->num_lbas, dev->lba_count - bio->lba);
+    }
+    if (bio->num_lbas == 0) {
+        bio_complete(bio, 0, 0);
+        return;
+    }
+
+    const size_t lba_size = dev->lba_size;
+    const size_t lbas_per_page = MAX(arch_mem_page_size() / lba_size, 1);
+    size_t max_lbas = dev->limits.max_lbas ? dev->limits.max_lbas : SIZE_MAX;
+    if (max_lbas > lbas_per_page)
+        max_lbas -= max_lbas % lbas_per_page;
+    const size_t max_segs = dev->limits.max_segs ? dev->limits.max_segs : SIZE_MAX;
+
+    if (bio->num_lbas <= max_lbas && bio->num_segs <= max_segs) {
+        submit_one(dev, bio);
+        return;
+    }
+
+    atomic_store(&bio->pending, 1, ATOMIC_RELAXED);
+    atomic_store(&bio->short_offset, UINT64_MAX, ATOMIC_RELAXED);
+    atomic_store(&bio->child_status, 0, ATOMIC_RELAXED);
+
+    size_t seg = 0;
+    size_t seg_off = 0;
+    uint64_t consumed = 0;
+    while (consumed < bio->num_lbas) {
+        const size_t cap = MIN(max_segs, bio->num_segs - seg);
+        struct bio* child = cap ? bio_alloc(bio->op, cap) : nullptr;
+        if (!child) {
+            record_short(bio, consumed, cap ? ENOMEM : EINVAL);
+            break;
+        }
+
+        size_t count = 0;
+        size_t lbas = 0;
+        while (lbas < max_lbas && count < cap && consumed + lbas < bio->num_lbas) {
+            const struct bio_seg* src = &bio->segs[seg];
+            const size_t take =
+                MIN(MIN(src->len - seg_off, (max_lbas - lbas) * lba_size),
+                    (bio->num_lbas - consumed - lbas) * lba_size);
+            if (take < lba_size)
+                break;
+            child->segs[count++] = (struct bio_seg){.addr = src->addr + seg_off, .len = take};
+            lbas += take / lba_size;
+            seg_off += take;
+            if (seg_off == src->len) {
+                seg++;
+                seg_off = 0;
+            }
+        }
+
+        if (lbas == 0) {
+            bio_free(child);
+            record_short(bio, consumed, EINVAL);
+            break;
+        }
+
+        child->num_segs = count;
+        child->num_lbas = lbas;
+        child->lba = bio->lba + consumed;
+        child->parent = bio;
+        child->parent_offset = consumed;
+        child->dev = dev;
+        consumed += lbas;
+        atomic_fetch_add(&bio->pending, 1, ATOMIC_RELAXED);
+        submit_one(dev, child);
+    }
+
+    parent_put(bio);
+}

dev/block/blockdev.c

@@ -0,0 +1,490 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "block: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/print.h>
+#include <zinnia/tailq.h>
+#include <zinnia/usercopy.h>
+#include <zinnia/utils.h>
+#include <zinnia/vec.h>
+#include <zinnia/vma.h>
+#include <string.h>
+#include <uapi/block.h>
+
+static size_t lbas_per_page(struct blockdev* dev) {
+    return arch_mem_page_size() / dev->lba_size;
+}
+
+static errno_t cache_get_pages(struct vmo* vmo, uint64_t index, struct page** pages, size_t count, size_t* out_valid) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    const size_t page_size = arch_mem_page_size();
+    const size_t per_page = lbas_per_page(dev);
+    const uint64_t lba = index * per_page;
+
+    if (lba >= dev->lba_count) {
+        *out_valid = 0;
+        return 0;
+    }
+
+    const size_t num_lbas = MIN((uint64_t)count * per_page, dev->lba_count - lba);
+    size_t done;
+    errno_t status = blockdev_rw_pages(dev, BIO_READ, lba, num_lbas, pages, count, &done);
+    if (status)
+        return status;
+
+    const size_t valid = done / per_page + (done % per_page != 0);
+    if (done % per_page != 0) {
+        const size_t tail = (done % per_page) * dev->lba_size;
+        memset((uint8_t*)HHDM_PTR(vm_page_addr(pages[valid - 1])) + tail, 0, page_size - tail);
+    }
+    *out_valid = valid;
+    return valid == 0 && num_lbas > 0 ? EIO : 0;
+}
+
+static errno_t cache_put_pages(struct vmo* vmo, uint64_t index, struct page** pages, size_t count) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    const size_t per_page = lbas_per_page(dev);
+    const uint64_t lba = index * per_page;
+    if (lba >= dev->lba_count)
+        return 0;
+
+    const size_t num_lbas = MIN((uint64_t)count * per_page, dev->lba_count - lba);
+    size_t done;
+    errno_t status = blockdev_rw_pages(dev, BIO_WRITE, lba, num_lbas, pages, count, &done);
+    if (status)
+        return status;
+    return done < num_lbas ? EIO : 0;
+}
+
+static void cache_destroy(struct vmo* vmo) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    if (dev->ops->destroy)
+        dev->ops->destroy(dev);
+}
+
+static const struct vmo_pager_ops cache_pager = {
+    .get_pages = cache_get_pages,
+    .put_pages = cache_put_pages,
+    .max_cluster = 64,
+    .destroy = cache_destroy,
+};
+
+void blockdev_init(
+    struct blockdev* dev,
+    const struct blockdev_ops* ops,
+    uint32_t major,
+    uint32_t minor,
+    size_t lba_size,
+    uint64_t lba_count
+) {
+    vmo_init(&dev->cache, &cache_pager);
+    dev->ops = ops;
+    dev->dev = dev_make(major, minor);
+    dev->lba_size = lba_size;
+    dev->lba_count = lba_count;
+    atomic_init(&dev->claimed, false);
+}
+
+void blockdev_ref(struct blockdev* dev) {
+    vmo_ref(&dev->cache);
+}
+
+void blockdev_unref(struct blockdev* dev) {
+    vmo_unref(&dev->cache);
+}
+
+errno_t blockdev_rw(
+    struct blockdev* dev,
+    enum bio_op op,
+    uint64_t lba,
+    size_t num_lbas,
+    const struct bio_seg* segs,
+    size_t num_segs,
+    size_t* out_lbas
+) {
+    struct bio* bio = bio_alloc(op, num_segs);
+    if (!bio)
+        return ENOMEM;
+    memcpy(bio->segs, segs, num_segs * sizeof(struct bio_seg));
+    bio->lba = lba;
+    bio->num_lbas = num_lbas;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, out_lbas);
+    bio_free(bio);
+    return status;
+}
+
+errno_t blockdev_rw_pages(
+    struct blockdev* dev,
+    enum bio_op op,
+    uint64_t lba,
+    size_t num_lbas,
+    struct page** pages,
+    size_t count,
+    size_t* out_lbas
+) {
+    const size_t page_size = arch_mem_page_size();
+    size_t bytes = num_lbas * dev->lba_size;
+    if (bytes > count * page_size)
+        return EINVAL;
+
+    struct bio* bio = bio_alloc(op, count);
+    if (!bio)
+        return ENOMEM;
+
+    size_t num_segs = 0;
+    for (size_t i = 0; i < count && bytes > 0; i++) {
+        const size_t len = MIN(bytes, page_size);
+        bio->segs[num_segs++] = (struct bio_seg){.addr = vm_page_addr(pages[i]), .len = len};
+        bytes -= len;
+    }
+    bio->num_segs = num_segs;
+    bio->lba = lba;
+    bio->num_lbas = num_lbas;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, out_lbas);
+    bio_free(bio);
+    return status;
+}
+
+errno_t blockdev_flush(struct blockdev* dev) {
+    struct bio* bio = bio_alloc(BIO_FLUSH, 0);
+    if (!bio)
+        return ENOMEM;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, nullptr);
+    bio_free(bio);
+    return status;
+}
+
+static struct blockdev* file_dev(struct vfs_file* file) {
+    return file->priv;
+}
+
+static uint64_t dev_size(struct blockdev* dev) {
+    return dev->lba_count * dev->lba_size;
+}
+
+static errno_t blockdev_file_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    struct blockdev* dev = file_dev(file);
+    *out_read = 0;
+    if (offset >= dev_size(dev))
+        return 0;
+
+    size_t progress;
+    errno_t status =
+        vmo_read_iter(&dev->cache, offset, buffer, MIN(iovec_iter_len(buffer), dev_size(dev) - offset), &progress);
+    *out_read = progress;
+    return progress > 0 ? 0 : status;
+}
+
+static errno_t blockdev_file_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    struct blockdev* dev = file_dev(file);
+    *out_written = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (offset >= dev_size(dev))
+        return ENOSPC;
+
+    size_t progress;
+    errno_t status =
+        vmo_write_iter(&dev->cache, offset, buffer, MIN(iovec_iter_len(buffer), dev_size(dev) - offset), &progress);
+    *out_written = progress;
+    return progress > 0 ? 0 : status;
+}
+
+static errno_t blockdev_file_ioctl(struct vfs_file* file, uintptr_t request, uintptr_t arg, uintptr_t* out_result) {
+    struct blockdev* dev = file_dev(file);
+    switch (request) {
+    case __BLKGETSIZE64: {
+        const uint64_t size = dev_size(dev);
+        if (!usercopy_write((__user void*)arg, &size, sizeof(size)))
+            return EFAULT;
+        *out_result = 0;
+        return 0;
+    }
+    case __BLKSSZGET: {
+        const unsigned int lba_size = dev->lba_size;
+        if (!usercopy_write((__user void*)arg, &lba_size, sizeof(lba_size)))
+            return EFAULT;
+        *out_result = 0;
+        return 0;
+    }
+    case __BLKRRPART: {
+        errno_t status = blockdev_rescan(dev);
+        if (!status)
+            *out_result = 0;
+        return status;
+    }
+    default:
+        if (dev->ops->ioctl)
+            return dev->ops->ioctl(dev, file, request, arg, out_result);
+        return ENOTTY;
+    }
+}
+
+static errno_t blockdev_file_sync(struct vfs_file* file, bool data_only) {
+    struct blockdev* dev = file_dev(file);
+    errno_t status = vmo_sync(&dev->cache, 0, UINT64_MAX);
+    if (status)
+        return status;
+    return blockdev_flush(dev);
+}
+
+static errno_t blockdev_file_mmap(
+    struct vfs_file* file,
+    struct vma* vma,
+    uintptr_t addr,
+    size_t len,
+    enum prot_flags prot,
+    enum map_flags flags,
+    off_t offset,
+    uintptr_t* out_addr
+) {
+    errno_t status =
+        vma_map(vma, &file_dev(file)->cache, addr, len, prot, (flags & MAP_SHARED) ? VMA_SHARED : 0, offset);
+    if (status)
+        return status;
+    *out_addr = addr;
+    return 0;
+}
+
+static void blockdev_file_release(struct vfs_file* file) {
+    blockdev_unref(file_dev(file));
+}
+
+static const struct vfs_file_ops blockdev_file_ops = {
+    .read = blockdev_file_read,
+    .write = blockdev_file_write,
+    .ioctl = blockdev_file_ioctl,
+    .sync = blockdev_file_sync,
+    .mmap = blockdev_file_mmap,
+    .release = blockdev_file_release,
+};
+
+static errno_t blockdev_node_open(struct vfs_node* node, struct vfs_file* file) {
+    struct blockdev* dev = node->priv;
+    blockdev_ref(dev);
+    file->priv = dev;
+    file->ops = &blockdev_file_ops;
+    return 0;
+}
+
+static void blockdev_node_destroy(struct vfs_node* node) {
+    blockdev_unref(node->priv);
+}
+
+static const struct vfs_node_ops blockdev_node_ops = {
+    .destroy = blockdev_node_destroy,
+    .device = {
+        .open = blockdev_node_open,
+    },
+};
+
+errno_t devtmpfs_add_blockdev(const char* name, struct blockdev* dev, mode_t mode) {
+    struct vfs_mknod_target target = {
+        .type = VNODE_BLOCKDEV,
+        .ops = &blockdev_node_ops,
+        .priv = dev,
+        .rdev = dev->dev,
+    };
+
+    blockdev_ref(dev);
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mknod(root, root, name, mode, &target, ident_kernel(), nullptr);
+    vfs_path_drop(root);
+    if (status) {
+        blockdev_unref(dev);
+        return status;
+    }
+    devctl_notify_create(name);
+    return 0;
+}
+
+errno_t blockdev_from_node(struct vfs_node* node, struct blockdev** out) {
+    if (node->type != VNODE_BLOCKDEV || node->ops != &blockdev_node_ops)
+        return ENOTBLK;
+
+    struct blockdev* dev = node->priv;
+    blockdev_ref(dev);
+    *out = dev;
+    return 0;
+}
+
+errno_t blockdev_claim(struct blockdev* dev) {
+    bool expected = false;
+    if (!atomic_compare_exchange(&dev->claimed, &expected, true, ATOMIC_ACQUIRE, ATOMIC_RELAXED))
+        return EBUSY;
+    return 0;
+}
+
+void blockdev_release(struct blockdev* dev) {
+    atomic_store(&dev->claimed, false, ATOMIC_RELEASE);
+}
+
+struct disk_node {
+    char name[64];
+    struct blockdev* dev;
+};
+
+struct disk {
+    char name[32];
+    struct blockdev* dev;
+    vec(struct disk_node) nodes;
+    tailq_link(struct disk) link;
+};
+
+static tailq_head(struct disk) disks;
+static mutex_t disks_lock;
+
+static void add_node(struct disk* disk, const char* name, struct blockdev* dev) {
+    struct disk_node node = {.dev = dev};
+    ksnprintf(node.name, sizeof(node.name), "%s", name);
+    if (vec_push(&disk->nodes, node) && dev)
+        blockdev_unref(dev);
+}
+
+static void scan_partitions(struct disk* disk) {
+    struct gpt_partition* parts;
+    size_t count;
+    if (gpt_scan(disk->dev, &parts, &count))
+        return;
+
+    for (size_t i = 0; i < count; i++) {
+        const struct gpt_partition* part = &parts[i];
+        if (part->end_lba < part->start_lba || part->end_lba >= disk->dev->lba_count) {
+            log_warn("Ignoring partition %zu of \"%s\" with invalid range\n", i + 1, disk->name);
+            continue;
+        }
+
+        char part_name[48];
+        ksnprintf(part_name, sizeof(part_name), "%sp%zu", disk->name, i + 1);
+
+        struct blockdev* dev;
+        errno_t status = partition_new(disk->dev, part->start_lba, part->end_lba - part->start_lba + 1, &dev);
+        if (status) {
+            log_warn("Unable to create partition \"%s\" (%i)\n", part_name, status);
+            continue;
+        }
+
+        char path[64];
+        ksnprintf(path, sizeof(path), "block/%s", part_name);
+        status = devtmpfs_add_blockdev(path, dev, 0660);
+        if (status) {
+            blockdev_unref(dev);
+            log_warn("Unable to register \"%s\" (%i)\n", path, status);
+            continue;
+        }
+        add_node(disk, path, dev);
+
+        char type_str[GUID_STR_LEN];
+        char uuid_str[GUID_STR_LEN];
+        guid_format(&part->type_guid, type_str);
+        guid_format(&part->unique_guid, uuid_str);
+
+        ksnprintf(path, sizeof(path), "block/parttype-%s", type_str);
+        status = devtmpfs_symlink(path, part_name);
+        if (!status)
+            add_node(disk, path, nullptr);
+        else if (status == EEXIST)
+            log("Partition type %s already claimed, no \"%s\" link for \"%s\"\n", type_str, path, part_name);
+
+        ksnprintf(path, sizeof(path), "block/partuuid-%s", uuid_str);
+        status = devtmpfs_symlink(path, part_name);
+        if (!status)
+            add_node(disk, path, nullptr);
+
+        log("Partition %zu: \"%s\" Type: %s UUID: %s\n", i + 1, part_name, type_str, uuid_str);
+    }
+
+    mem_free(parts);
+}
+
+errno_t blockdev_register(const char* name, struct blockdev* dev) {
+    struct disk* disk = mem_alloc(sizeof(struct disk), 0);
+    if (!disk)
+        return ENOMEM;
+    ksnprintf(disk->name, sizeof(disk->name), "%s", name);
+    vec_init(&disk->nodes);
+
+    char path[64];
+    ksnprintf(path, sizeof(path), "block/%s", name);
+    errno_t status = devtmpfs_add_blockdev(path, dev, 0660);
+    if (status) {
+        mem_free(disk);
+        return status;
+    }
+
+    blockdev_ref(dev);
+    disk->dev = dev;
+    log("Registered block device \"%s\"\n", name);
+
+    mutex_lock(&disks_lock);
+    tailq_insert_tail(&disks, disk, link);
+    scan_partitions(disk);
+    mutex_unlock(&disks_lock);
+    return 0;
+}
+
+errno_t blockdev_rescan(struct blockdev* dev) {
+    mutex_lock(&disks_lock);
+    struct disk* disk;
+    for (disk = tailq_first(&disks); disk; disk = tailq_next(disk, link)) {
+        if (disk->dev == dev)
+            break;
+    }
+    if (!disk) {
+        mutex_unlock(&disks_lock);
+        return ENOTTY;
+    }
+
+    for (size_t i = 0; i < vec_length(&disk->nodes); i++) {
+        struct disk_node* node = &vec_elem_get(&disk->nodes, i);
+        errno_t status = devtmpfs_remove(node->name);
+        if (status)
+            log_warn("Unable to remove stale node \"%s\" (%i)\n", node->name, status);
+        if (node->dev)
+            blockdev_unref(node->dev);
+    }
+    disk->nodes.length = 0;
+
+    scan_partitions(disk);
+    mutex_unlock(&disks_lock);
+    return 0;
+}
+
+errno_t blockdev_sync_all() {
+    errno_t result = 0;
+    mutex_lock(&disks_lock);
+    for (struct disk* disk = tailq_first(&disks); disk; disk = tailq_next(disk, link)) {
+        for (size_t i = 0; i < vec_length(&disk->nodes); i++) {
+            struct blockdev* part = vec_elem_get(&disk->nodes, i).dev;
+            errno_t status = part ? vmo_sync(&part->cache, 0, UINT64_MAX) : 0;
+            if (status && !result)
+                result = status;
+        }
+        errno_t status = vmo_sync(&disk->dev->cache, 0, UINT64_MAX);
+        if (!status)
+            status = blockdev_flush(disk->dev);
+        if (status && !result)
+            result = status;
+    }
+    mutex_unlock(&disks_lock);
+    return result;
+}
+
+static void block_stage_init(struct boot_info* info) {
+    tailq_init(&disks);
+    mutex_init(&disks_lock);
+    errno_t status = devtmpfs_mkdir("block", 0755);
+    if (status && status != EEXIST)
+        log_warn("Unable to create /dev/block (%i)\n", status);
+}
+
+INIT_TASK(block_stage, "dev.block", block_stage_init);
+INIT_DEPENDS(block_stage, devtmpfs_stage);

dev/block/gpt.c

@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <string.h>
+
+struct [[__packed]] gpt_header {
+    char signature[8];
+    uint32_t revision;
+    uint32_t header_size;
+    uint32_t header_crc32;
+    uint32_t reserved;
+    uint64_t my_lba;
+    uint64_t alternate_lba;
+    uint64_t first_usable_lba;
+    uint64_t last_usable_lba;
+    struct guid disk_guid;
+    uint64_t partition_entry_lba;
+    uint32_t num_partition_entries;
+    uint32_t partition_entry_size;
+    uint32_t partition_entry_array_crc32;
+};
+
+struct [[__packed]] gpt_entry {
+    struct guid type_guid;
+    struct guid unique_guid;
+    uint64_t start_lba;
+    uint64_t end_lba;
+    uint64_t attributes;
+    uint16_t name[36];
+};
+
+constexpr size_t GPT_MAX_ENTRIES = 1024;
+
+errno_t gpt_scan(struct blockdev* dev, struct gpt_partition** out, size_t* out_count) {
+    struct gpt_header header;
+    size_t read;
+    errno_t status = vmo_read(&dev->cache, dev->lba_size, &header, sizeof(header), &read);
+    if (status)
+        return status;
+    if (read != sizeof(header) || memcmp(header.signature, "EFI PART", 8) != 0)
+        return ENODATA;
+    if (header.partition_entry_size < sizeof(struct gpt_entry) || header.num_partition_entries > GPT_MAX_ENTRIES)
+        return EINVAL;
+
+    struct gpt_partition* parts = mem_alloc(MAX(header.num_partition_entries, 1) * sizeof(struct gpt_partition), 0);
+    if (!parts)
+        return ENOMEM;
+
+    size_t count = 0;
+    const uint64_t base = header.partition_entry_lba * dev->lba_size;
+    for (size_t i = 0; i < header.num_partition_entries; i++) {
+        struct gpt_entry entry;
+        status = vmo_read(&dev->cache, base + i * header.partition_entry_size, &entry, sizeof(entry), &read);
+        if (status || read != sizeof(entry))
+            break;
+        if (guid_is_zero(&entry.type_guid))
+            continue;
+
+        struct gpt_partition* part = &parts[count++];
+        part->type_guid = entry.type_guid;
+        part->unique_guid = entry.unique_guid;
+        part->start_lba = entry.start_lba;
+        part->end_lba = entry.end_lba;
+    }
+
+    *out = parts;
+    *out_count = count;
+    return 0;
+}

dev/block/meson.build

@@ -1,3 +1,11 @@
-if kconfig.get('CONFIG_NVME_CORE', 'n') != 'n'
+kernel_sources += files(
+    'bio.c',
+    'blockdev.c',
+    'gpt.c',
+    'partition.c',
+    'ram.c',
+)
+
+if kconfig.get('CONFIG_NVME', 'n') != 'n'
     subdir('nvme')
 endif

dev/block/nvme/Kconfig

@@ -1,13 +1,4 @@
-menu "NVMe Support"
-
-config NVME_CORE
-  bool
-  default y
-
-config NVME_PCI
-  bool "NVMe via PCI transport"
+config NVME
+  bool "NVMe block devices"
   default y
   depends on PCI
-  depends on NVME_CORE
-
-endmenu

dev/block/nvme/command.c

@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/mmio.h>
+#include "nvme.h"
+
+void nvme_rw_cmd_write(const struct nvme_rw_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, cmd->do_write ? NVME_CMD_WRITE : NVME_CMD_READ) |
+        mmio_field(NVME_SQE_CID, cmd->cid) | mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_RW_SLBA_LOW, (uint32_t)cmd->start_lba);
+
+    uint32_t cdw11 = mmio_field(NVME_RW_SLBA_HIGH, (uint32_t)(cmd->start_lba >> 32));
+
+    uint32_t cdw12 = mmio_field(NVME_RW_NLB, (uint16_t)(cmd->num_lbas - 1));
+    uint32_t cdw13 = 0;
+    uint32_t cdw14 = 0;
+    uint32_t cdw15 = 0;
+
+    mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->prp1);
+    mmio_reg_write(sqe, NVME_SQE_DPTR1, cmd->prp2);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+    mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
+    mmio_reg_write(sqe, NVME_SQE_CDW12, cdw12);
+    mmio_reg_write(sqe, NVME_SQE_CDW13, cdw13);
+    mmio_reg_write(sqe, NVME_SQE_CDW14, cdw14);
+    mmio_reg_write(sqe, NVME_SQE_CDW15, cdw15);
+}
+
+void nvme_flush_cmd_write(const struct nvme_flush_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_CMD_FLUSH) | mmio_field(NVME_SQE_CID, cmd->cid);
+
+    mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+}
+
+void nvme_create_cq_cmd_write(const struct nvme_create_cq_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_CREATE_CQ) |
+        mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_CREATE_CQ_QSIZE, (uint16_t)(cmd->depth - 1)) |
+        mmio_field(NVME_CREATE_CQ_QID, (uint16_t)cmd->qid);
+
+    uint32_t cdw11 = mmio_field(NVME_CREATE_CQ_IV, cmd->irq_vector) |
+        mmio_field(NVME_CREATE_CQ_IEN, cmd->irqs_enabled ? 1 : 0) |
+        mmio_field(NVME_CREATE_CQ_PC, 1); // Our buffer is physically contiguous.
+
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->cq_addr);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+    mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
+}
+
+void nvme_create_sq_cmd_write(const struct nvme_create_sq_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_CREATE_SQ) |
+        mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_CREATE_SQ_QSIZE, (uint16_t)(cmd->depth - 1)) |
+        mmio_field(NVME_CREATE_SQ_QID, (uint16_t)cmd->qid);
+
+    uint32_t cdw11 = mmio_field(NVME_CREATE_SQ_CQID, (uint16_t)cmd->cqid) |
+        mmio_field(NVME_CREATE_SQ_QPRIO, 0) | // Priority is the highest.
+        mmio_field(NVME_CREATE_SQ_PC, 1);     // Our buffer is physically contiguous.
+
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->sq_addr);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+    mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
+}
+
+void nvme_identify_cmd_write(const struct nvme_identify_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_IDENTIFY) |
+        mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_IDENTIFY_CNTID, cmd->controller_id) | mmio_field(NVME_IDENTIFY_CNS, cmd->cns);
+
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->buffer);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+}

dev/block/nvme/controller.c

@@ -0,0 +1,388 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/clock.h>
+#include <zinnia/mmio.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <arch/mem.h>
+#include <string.h>
+#include "nvme.h"
+
+constexpr size_t ADMIN_QUEUE_SIZE = 32;
+constexpr size_t IO_QUEUE_SIZE_MAX = 1024;
+constexpr size_t IDENTIFY_SIZE = 4096;
+constexpr size_t NS_LIST_LEN = 1024;
+
+constexpr mmio_reg8 ID_MDTS = {77, MMIO_LE};
+constexpr mmio_reg32 ID_NN = {516, MMIO_LE};
+constexpr mmio_reg64 ID_NSZE = {0, MMIO_LE};
+constexpr mmio_reg8 ID_FLBAS = {26, MMIO_LE};
+constexpr mmio_field8 ID_LBAF_IDX = {ID_FLBAS, 0, 4};
+constexpr mmio_reg32 ID_LBAF0 = {128, MMIO_LE};
+constexpr mmio_field32 ID_LBADS = {ID_LBAF0, 16, 8};
+
+size_t nvme_ctrl_max_transfer_bytes(struct nvme_ctrl* ctrl) {
+    const size_t cap = 512 * arch_mem_page_size();
+    if (ctrl->mdts != 0)
+        return MIN(ctrl->mdts, cap);
+    return cap;
+}
+
+errno_t nvme_ctrl_new_pci(void* regs, struct dma_tag* dma_tag, struct nvme_ctrl** out) {
+    // Read controller version.
+    uint32_t vs = mmio_reg_read(regs, NVME_REG_VS);
+    log("NVMe controller version %u.%u.%u\n",
+        mmio_field_get(NVME_VS_MJR, vs),
+        mmio_field_get(NVME_VS_MNR, vs),
+        mmio_field_get(NVME_VS_TER, vs));
+
+    uint64_t cap = mmio_reg_read(regs, NVME_REG_CAP);
+
+    // Check if our host page size is supported.
+    const size_t mps_max = (size_t)1 << (mmio_field_get(NVME_CAP_MPSMAX, cap) + 12);
+    const size_t mps_min = (size_t)1 << (mmio_field_get(NVME_CAP_MPSMIN, cap) + 12);
+    const size_t page_size = arch_mem_page_size();
+    if (page_size < mps_min || page_size > mps_max) {
+        log_err("Host page size is not supported on this NVMe!\n");
+        return ENOTSUP;
+    }
+
+    struct nvme_ctrl* ctrl = mem_alloc(sizeof(*ctrl), 0);
+    if (!ctrl)
+        return ENOMEM;
+
+    ctrl->regs = regs;
+    ctrl->dma_tag = dma_tag;
+    ctrl->doorbell_stride = (size_t)4 << mmio_field_get(NVME_CAP_DSTRD, cap);
+    ctrl->mps_min = mps_min;
+    ctrl->mps = __builtin_ctzl(page_size) - 12;
+    mutex_init(&ctrl->admin_lock);
+    spin_init(&ctrl->io_lock);
+    atomic_init(&ctrl->ns_count, 0);
+
+    *out = ctrl;
+    return 0;
+}
+
+// Sets the CC.EN flag.
+static errno_t set_status(struct nvme_ctrl* ctrl, bool enable) {
+    uint64_t cap = mmio_reg_read(ctrl->regs, NVME_REG_CAP);
+    // CAP.TO is in 500ms units.
+    const uint64_t timeout = MAX((uint64_t)mmio_field_get(NVME_CAP_TO, cap), 1) * 500'000'000;
+    const uint64_t deadline = clock_get_elapsed_ns() + timeout;
+
+    uint32_t cc = mmio_reg_read(ctrl->regs, NVME_REG_CC);
+    cc &= ~mmio_field(NVME_CC_EN, UINT32_MAX);
+    cc |= mmio_field(NVME_CC_EN, enable ? 1 : 0);
+    mmio_reg_write(ctrl->regs, NVME_REG_CC, cc);
+
+    while (true) {
+        uint32_t csts = mmio_reg_read(ctrl->regs, NVME_REG_CSTS);
+
+        if (mmio_field_get(NVME_CSTS_CFS, csts) != 0)
+            return EIO;
+
+        bool ready = mmio_field_get(NVME_CSTS_RDY, csts) != 0;
+        if (ready == enable)
+            break;
+
+        if (clock_get_elapsed_ns() >= deadline)
+            return ETIMEDOUT;
+    }
+
+    return 0;
+}
+
+static errno_t admin_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe, struct nvme_completion* out) {
+    errno_t status = nvme_queue_submit_cmd(queue, sqe);
+    if (status)
+        return status;
+    return nvme_queue_next_completion(queue, out);
+}
+
+static errno_t identify(struct nvme_queue* queue, const struct nvme_identify_cmd* cmd) {
+    struct nvme_sqe sqe = {};
+    nvme_identify_cmd_write(cmd, &sqe);
+    struct nvme_completion comp;
+    errno_t status = admin_cmd(queue, &sqe, &comp);
+    if (status)
+        return status;
+    if (!nvme_status_is_success(comp.status)) {
+        log_err("Identification failed with status %x\n", comp.status);
+        return EIO;
+    }
+    return 0;
+}
+
+errno_t nvme_ctrl_reset(struct nvme_ctrl* ctrl, bool irqs_enabled, uint16_t io_irq_vector) {
+    // Disable the controller.
+    errno_t status = set_status(ctrl, false);
+    if (status)
+        return status;
+
+    uint64_t cap = mmio_reg_read(ctrl->regs, NVME_REG_CAP);
+
+    const size_t queue_depth = MIN((size_t)mmio_field_get(NVME_CAP_MQES, cap) + 1, IO_QUEUE_SIZE_MAX);
+
+    // Create an admin queue first so we can create more queues.
+    struct nvme_queue* admin_queue;
+    status = nvme_queue_new(ctrl->dma_tag, ctrl->regs, ctrl->doorbell_stride, 0, ADMIN_QUEUE_SIZE, &admin_queue);
+    if (status)
+        return status;
+
+    // Set the admin queue sizes.
+    uint32_t aqa = 0;
+    aqa |= mmio_field(NVME_AQA_ACQS, admin_queue->depth - 1);
+    aqa |= mmio_field(NVME_AQA_ASQS, admin_queue->depth - 1);
+    mmio_reg_write(ctrl->regs, NVME_REG_AQA, aqa);
+
+    // Set the addresses to the admin submission and completion queue.
+    mmio_reg_write(ctrl->regs, NVME_REG_ASQ, admin_queue->sq.addr);
+    mmio_reg_write(ctrl->regs, NVME_REG_ACQ, admin_queue->cq.addr);
+
+    uint32_t cc = mmio_reg_read(ctrl->regs, NVME_REG_CC);
+    cc &=
+        ~(mmio_field(NVME_CC_MPS, UINT32_MAX) | mmio_field(NVME_CC_IOCQES, UINT32_MAX) |
+          mmio_field(NVME_CC_IOSQES, UINT32_MAX));
+
+    cc |= mmio_field(NVME_CC_MPS, ctrl->mps);
+    cc |= mmio_field(NVME_CC_IOCQES, __builtin_ctzl(sizeof(struct nvme_cqe)));
+    cc |= mmio_field(NVME_CC_IOSQES, __builtin_ctzl(sizeof(struct nvme_sqe)));
+
+    mmio_reg_write(ctrl->regs, NVME_REG_CC, cc);
+
+    // Re-enable the controller.
+    status = set_status(ctrl, true);
+    if (status)
+        goto fail_admin;
+
+    struct nvme_io_queue* io_queue;
+    status =
+        nvme_io_queue_new(ctrl->dma_tag, ctrl->regs, ctrl->doorbell_stride, 1, queue_depth, !irqs_enabled, &io_queue);
+    if (status)
+        goto fail_admin;
+
+    struct nvme_sqe sqe = {};
+    struct nvme_completion comp;
+    nvme_create_cq_cmd_write(
+        &(struct nvme_create_cq_cmd){
+            .cq_addr = io_queue->cq.addr,
+            .depth = io_queue->depth,
+            .qid = io_queue->queue_id,
+            .irqs_enabled = irqs_enabled,
+            .irq_vector = irqs_enabled ? io_irq_vector : 0,
+        },
+        &sqe
+    );
+    status = admin_cmd(admin_queue, &sqe, &comp);
+    if (!status && !nvme_status_is_success(comp.status))
+        status = EIO;
+    if (status)
+        goto fail_io;
+
+    sqe = (struct nvme_sqe){};
+    nvme_create_sq_cmd_write(
+        &(struct nvme_create_sq_cmd){
+            .sq_addr = io_queue->sq.addr,
+            .depth = io_queue->depth,
+            .qid = io_queue->queue_id,
+            .cqid = io_queue->queue_id,
+        },
+        &sqe
+    );
+    status = admin_cmd(admin_queue, &sqe, &comp);
+    if (!status && !nvme_status_is_success(comp.status))
+        status = EIO;
+    if (status)
+        goto fail_io;
+
+    mutex_lock(&ctrl->admin_lock);
+    struct nvme_queue* old_admin = ctrl->admin_queue;
+    ctrl->admin_queue = admin_queue;
+    mutex_unlock(&ctrl->admin_lock);
+    if (old_admin)
+        nvme_queue_free(old_admin);
+
+    spin_lock(&ctrl->io_lock);
+    ctrl->io_queue = io_queue;
+    spin_unlock(&ctrl->io_lock);
+
+    return 0;
+
+fail_io:
+    nvme_io_queue_free(io_queue);
+fail_admin:
+    nvme_queue_free(admin_queue);
+    return status;
+}
+
+static void trim_ascii_end(char* str, size_t len) {
+    while (len > 0 &&
+           (str[len - 1] == ' ' || str[len - 1] == '\t' || str[len - 1] == '\n' || str[len - 1] == '\r' ||
+            str[len - 1] == '\f'))
+        len--;
+    str[len] = '\0';
+}
+
+errno_t nvme_ctrl_identify(struct nvme_ctrl* ctrl) {
+    struct dma_buffer identify_buffer;
+    errno_t status = dma_buffer_alloc(ctrl->dma_tag, IDENTIFY_SIZE, &identify_buffer);
+    if (status)
+        return ENOMEM;
+
+    mutex_lock(&ctrl->admin_lock);
+    if (!ctrl->admin_queue) {
+        status = ENXIO;
+        goto out;
+    }
+
+    status = identify(
+        ctrl->admin_queue,
+        &(struct nvme_identify_cmd){
+            .buffer = identify_buffer.addr,
+            .controller_id = 0, // TODO: Use a real number here
+            .cns = 1,
+            .nsid = 0,
+        }
+    );
+    // Check if the identification was successful.
+    if (status)
+        goto out;
+
+    dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
+
+    // Read the model strings.
+    const uint8_t* view = identify_buffer.virt;
+    memcpy(ctrl->serial, view + 4, sizeof(ctrl->serial) - 1);
+    memcpy(ctrl->model, view + 24, sizeof(ctrl->model) - 1);
+    memcpy(ctrl->revision, view + 64, sizeof(ctrl->revision) - 1);
+    trim_ascii_end(ctrl->serial, sizeof(ctrl->serial) - 1);
+    trim_ascii_end(ctrl->model, sizeof(ctrl->model) - 1);
+    trim_ascii_end(ctrl->revision, sizeof(ctrl->revision) - 1);
+    log("Model \"%s\"\n", ctrl->model);
+
+    atomic_store(&ctrl->ns_count, mmio_reg_read(identify_buffer.virt, ID_NN), ATOMIC_RELEASE);
+
+    uint8_t mdts = mmio_reg_read(identify_buffer.virt, ID_MDTS);
+    if (mdts != 0 && mdts < sizeof(size_t) * 8)
+        ctrl->mdts = ((size_t)1 << mdts) * ctrl->mps_min;
+
+out:
+    mutex_unlock(&ctrl->admin_lock);
+    dma_buffer_free(&identify_buffer);
+    return status;
+}
+
+// Scans all existing namespaces.
+errno_t nvme_ctrl_scan_namespaces(struct nvme_ctrl* ctrl, nvme_ns_vec_t* out) {
+    vec_init(out);
+
+    uint32_t* ns_list = mem_alloc(NS_LIST_LEN * sizeof(uint32_t), 0);
+    if (!ns_list)
+        return ENOMEM;
+
+    struct dma_buffer identify_buffer;
+    errno_t status = dma_buffer_alloc(ctrl->dma_tag, IDENTIFY_SIZE, &identify_buffer);
+    if (status) {
+        mem_free(ns_list);
+        return ENOMEM;
+    }
+
+    mutex_lock(&ctrl->admin_lock);
+    struct nvme_queue* aq = ctrl->admin_queue;
+    if (!aq) {
+        status = ENXIO;
+        goto out;
+    }
+
+    uint32_t nsid = 0;
+    while (true) {
+        const struct nvme_identify_cmd admin_ident_cmd = {
+            .buffer = identify_buffer.addr,
+            .controller_id = 0,
+            .cns = 2,
+            .nsid = nsid,
+        };
+        status = identify(aq, &admin_ident_cmd);
+
+        if (status)
+            goto out;
+
+        dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
+
+        // Parse the identifier list until we hit an namespace value of 0.
+        size_t ns_list_len = 0;
+        const uint32_t* raw_list = identify_buffer.virt;
+        while (ns_list_len < NS_LIST_LEN && raw_list[ns_list_len] != 0) {
+            ns_list[ns_list_len] = raw_list[ns_list_len];
+            ns_list_len++;
+        }
+
+        for (size_t i = 0; i < ns_list_len; i++) {
+            const uint32_t id = ns_list[i];
+            const struct nvme_identify_cmd ns_ident_cmd = {
+                .buffer = identify_buffer.addr,
+                .controller_id = 0,
+                .cns = 0,
+                .nsid = id,
+            };
+            status = identify(aq, &ns_ident_cmd);
+
+            if (status)
+                goto out;
+
+            dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
+
+            const uint64_t lba_count = mmio_reg_read(identify_buffer.virt, ID_NSZE);
+
+            // To get the block size of a namespace, we need to read a descriptor (see LBAF0-15).
+            // Get the format descriptor index used by this namespace.
+            const uint8_t lba_format_idx = mmio_field_get(ID_LBAF_IDX, mmio_reg_read(identify_buffer.virt, ID_FLBAS));
+
+            const mmio_reg32 lbaf = {ID_LBAF0.offset + lba_format_idx * sizeof(uint32_t), MMIO_LE};
+            const uint32_t lba_format = mmio_reg_read(identify_buffer.virt, lbaf);
+
+            uint8_t lba_shift = mmio_field_get(ID_LBADS, lba_format);
+            // If LBADS doesn't contain a non-zero value, we need to assume a default. Use 512 bytes here.
+            if (lba_shift == 0)
+                lba_shift = 9;
+
+            struct nvme_ns* ns;
+            status = nvme_ns_new(ctrl, id, lba_shift, lba_count, nvme_ctrl_max_transfer_bytes(ctrl), &ns);
+            if (status)
+                goto out;
+            status = vec_push(out, ns);
+            if (status) {
+                blockdev_unref(&ns->blk);
+                goto out;
+            }
+        }
+
+        // If there is no 0 in the buffer, there's more entries than can fit in `identify_buffer`.
+        if (ns_list_len < NS_LIST_LEN)
+            break;
+        nsid = ns_list[ns_list_len - 1];
+    }
+
+out:
+    mutex_unlock(&ctrl->admin_lock);
+    dma_buffer_free(&identify_buffer);
+    mem_free(ns_list);
+    if (status) {
+        vec_foreach(out, i) {
+            blockdev_unref(&vec_elem_get(out, i)->blk);
+        }
+        vec_destroy(out);
+    }
+    return status;
+}
+
+struct nvme_io_queue* nvme_ctrl_io_queue(struct nvme_ctrl* ctrl) {
+    spin_lock(&ctrl->io_lock);
+    struct nvme_io_queue* queue = ctrl->io_queue;
+    spin_unlock(&ctrl->io_lock);
+    return queue;
+}

dev/block/nvme/core.c

@@ -1,12 +0,0 @@
-#include <zinnia/cmdline.h>
-#include <zinnia/module.h>
-#include <zinnia/print.h>
-
-void nvme_init(struct module* mod) {
-    log("Hello world from NVMe!\n");
-}
-
-void do_nvme(bool on) {}
-CMDLINE_OPTION("nvme_on", "My description", do_nvme);
-
-MODULE("NVME", "Marvin Friedrich", "ISC", "1.0", nvme_init);