dev/block: Add NVMe driver

.clangd

@@ -1,2 +1,2 @@
 Style:
-  AngledHeaders: [ "zinnia/.*", " kernel/.*" ]
+  AngledHeaders: [ "zinnia/.*", "uapi/.*" ]

.gitignore

@@ -0,0 +1,8 @@
+.config*
+build/
+.cache/
+.wraplock
+*.o
+*.a
+*.so
+__pycache__/

Kconfig

@@ -0,0 +1,12 @@
+mainmenu "Zinnia Kernel Configuration"
+
+config KERNEL_FLAVOR
+  string "Kernel flavor name"
+  default "vanilla"
+
+source "arch/$(ARCH)/Kconfig"
+source "boot/Kconfig"
+source "mem/Kconfig"
+source "sched/Kconfig"
+source "dev/Kconfig"
+source "tests/Kconfig"

LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2026 Marvin Friedrich
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

LICENSE

@@ -0,0 +1 @@
+LICENSES/ISC.txt

LICENSES/ISC.txt

@@ -0,0 +1,7 @@
+ISC License
+
+Copyright 2026 Marvin Friedrich and contributors
+
+Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

LICENSES/Unlicense.txt

@@ -0,0 +1,10 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means.
+
+In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>

README.md

@@ -1,20 +1,27 @@
+<img src="logo.svg" width="10%"/>
+
 # Zinnia
 
-![GitHub License](https://img.shields.io/github/license/zinnia-os/zinnia?style=flat&color=blue)
+![GitHub License](https://img.shields.io/github/license/zinnia-os/zinnia?style=flat&color=green)
+![GitHub top language](https://img.shields.io/github/languages/top/zinnia-os/zinnia)
 ![GitHub Repo stars](https://img.shields.io/github/stars/zinnia-os/zinnia?style=flat)
+![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/zinnia-os/zinnia/ci.yml)
 ![GitHub Issues or Pull Requests](https://img.shields.io/github/issues/zinnia-os/zinnia?style=flat)
+![Discord](https://img.shields.io/discord/1305893510704267327?logo=discord&logoColor=white&label=discord)
 
-Zinnia is a lightweight Unix-like microkernel targeting 64-bit devices.
+Zinnia is a lightweight Unix-like monolithic kernel targeting 64-bit devices.
 
 ## Building Zinnia
 
-To build the kernel and the servers you need `meson` and a C23-compatible
+To build the kernel you need:
-GNU-like compiler targeting Zinnia.
+- `meson`
-Currently supported are GCC and Clang toolchains.
+- a `kconfig` frontend (like `kconfiglib` via `pip`)
+- a GNU-like C compiler with support for C23 (e.g. `gcc` or `clang`)
 
 To configure, run:
 ```sh
-meson setup -Dbuild_kernel=true -Dbuild_servers=true $build_dir
+alldefconfig
+meson setup $build_dir
 ```
 
 And to build:

arch/x86_64/Kconfig

@@ -0,0 +1,18 @@
+menu "x86_64 Architecture options"
+
+config X86_64
+  bool
+  default y
+  select ACPI
+  select HAVE_EFISTUB
+  select HAVE_MMU
+  select HAVE_MMU_GENERIC
+  select HAVE_TLB_SHOOTDOWN_IPI
+  help
+    64-bit x86 CPU support for AMD and Intel CPUs.
+
+config TSC
+  bool "Invariant Timestamp Counter support"
+  default y
+
+endmenu

arch/x86_64/acpi.c

@@ -0,0 +1,36 @@
+#include <stdckdint.h>
+#include <stdint.h>
+#include <uacpi/kernel_api.h>
+#include <uacpi/status.h>
+#include <x86_64/asm.h>
+
+uacpi_status uacpi_kernel_io_map(uacpi_io_addr base, uacpi_size len, uacpi_handle* out_handle) {
+    uintptr_t end;
+    if (ckd_add(&end, base, len) || end > 0x10000)
+        return UACPI_STATUS_INVALID_ARGUMENT;
+    *out_handle = (uacpi_handle)(uintptr_t)base;
+    return UACPI_STATUS_OK;
+}
+
+void uacpi_kernel_io_unmap(uacpi_handle handle) {
+    // Left intentionally blank.
+}
+
+#define IO_READ(width, suffix) \
+    uacpi_status uacpi_kernel_io_read##width(uacpi_handle handle, uacpi_size offset, uacpi_u##width* out_value) { \
+        *out_value = asm_in##suffix((uintptr_t)handle + offset); \
+        return UACPI_STATUS_OK; \
+    }
+
+#define IO_WRITE(width, suffix) \
+    uacpi_status uacpi_kernel_io_write##width(uacpi_handle handle, uacpi_size offset, uacpi_u##width in_value) { \
+        asm_out##suffix((uintptr_t)handle + offset, in_value); \
+        return UACPI_STATUS_OK; \
+    }
+
+IO_READ(8, b)
+IO_READ(16, w)
+IO_READ(32, l)
+IO_WRITE(8, b)
+IO_WRITE(16, w)
+IO_WRITE(32, l)

arch/x86_64/apic.c

@@ -0,0 +1,163 @@
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/clock.h>
+#include <zinnia/mmio.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <arch/cpu.h>
+#include <arch/timer.h>
+#include <stdint.h>
+#include <x86_64/apic.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+#include <x86_64/percpu.h>
+
+static uint32_t lapic_msr(size_t offset) {
+    return 0x800 + (offset >> 4);
+}
+
+static uint32_t lapic_read_reg(struct local_apic* lapic, mmio_reg32 reg) {
+    if (lapic->xapic_regs)
+        return mmio_reg_read(lapic->xapic_regs, reg);
+    else
+        return asm_rdmsr(lapic_msr(reg.offset));
+}
+
+static void lapic_write_reg(struct local_apic* lapic, mmio_reg32 reg, uint32_t value) {
+    if (lapic->xapic_regs)
+        mmio_reg_write(lapic->xapic_regs, reg, value);
+    else
+        asm_wrmsr(lapic_msr(reg.offset), value);
+}
+
+static uint64_t lapic_read_icr(struct local_apic* lapic) {
+    if (lapic->xapic_regs) {
+        uint64_t lo = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_LO);
+        uint64_t hi = mmio_reg_read(lapic->xapic_regs, APIC_REG_ICR_HI);
+        return (hi << 32) | lo;
+    } else {
+        return asm_rdmsr(lapic_msr(APIC_REG_ICR.offset));
+    }
+}
+
+static void lapic_write_icr(struct local_apic* lapic, uint64_t value) {
+    if (lapic->xapic_regs) {
+        mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_HI, (uint32_t)(value >> 32));
+        mmio_reg_write(lapic->xapic_regs, APIC_REG_ICR_LO, (uint32_t)value);
+    } else {
+        asm_wrmsr(lapic_msr(APIC_REG_ICR.offset), value);
+    }
+}
+
+void lapic_init(struct local_apic* lapic) {
+    uint64_t apic_msr = asm_rdmsr(0x1B);
+    apic_msr |= 1 << 11; // Enable APIC.
+
+    struct cpuid cpuid = asm_cpuid(1, 0);
+    if (cpuid.ecx & CPUID_1C_X2APIC)
+        apic_msr |= 1 << 10;
+    else {
+        uintptr_t vaddr = vm_alloc(0x1000);
+        pmap_map(&kernel_vma.pmap, vaddr, apic_msr & 0xFFFF'F000, PTE_READ | PTE_WRITE, CACHE_MMIO);
+        lapic->xapic_regs = (void*)vaddr;
+        assert(lapic->xapic_regs, "Failed to allocate virtual memory!");
+    }
+
+    asm_wrmsr(0x1B, apic_msr);
+
+    lapic->id = lapic_read_reg(lapic, APIC_REG_ID);
+    if (lapic->xapic_regs)
+        lapic->id = mmio_field_get(APIC_ID_XAPIC, lapic->id);
+
+    // Reset the TPR.
+    lapic_write_reg(lapic, APIC_REG_TPR, 0);
+    // Enable APIC bit in the SIVR.
+    uint32_t sivr = lapic_read_reg(lapic, APIC_REG_SIVR);
+    sivr |= mmio_field(APIC_SIVR_VECTOR, IDT_SPURIOUS);
+    sivr |= mmio_field(APIC_SIVR_ENABLE, 1);
+    lapic_write_reg(lapic, APIC_REG_SIVR, sivr);
+
+    if (lapic->xapic_regs) {
+        lapic_write_reg(lapic, APIC_REG_DFR, 0xF000'0000);
+        // Logical destination = LAPIC ID.
+        lapic_write_reg(lapic, APIC_REG_LDR, lapic_read_reg(lapic, APIC_REG_ID));
+    }
+
+    // TODO: Parse MADT and setup NMI sources.
+
+    // Tell the APIC timer to divide by 16.
+    lapic_write_reg(lapic, APIC_REG_DCR, 3);
+    // Set the timer to the highest value possible.
+    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, 0xFFFF'FFFF);
+
+    // Sleep for 10 milliseconds.
+    clock_spin_ns(10'000'000);
+
+    lapic->ticks_per_10ms = 0xFFFF'FFFF - lapic_read_reg(lapic, APIC_REG_CCR);
+    lapic_write_reg(lapic, APIC_REG_LVT_TR, mmio_field(APIC_LVT_TR_VECTOR, IDT_TIMER));
+    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, 0);
+}
+
+void arch_timer_program(uint64_t deadline_ns) {
+    struct local_apic* lapic = &percpu_ref(arch_data)->lapic;
+    const uint64_t now = clock_get_elapsed_ns();
+    const uint64_t delta = MIN(deadline_ns > now ? deadline_ns - now : 0, 1'000'000'000);
+    const uint64_t ticks = delta * lapic->ticks_per_10ms / 10'000'000;
+    lapic_write_reg(lapic, APIC_REG_ICR_TIMER, MIN(MAX(ticks, 1), 0xFFFF'FFFF));
+}
+
+void lapic_eoi(struct local_apic* lapic) {
+    lapic_write_reg(lapic, APIC_REG_EOI, 0);
+}
+
+void lapic_send_ipi(
+    struct local_apic* lapic,
+    uint32_t target,
+    uint8_t vector,
+    enum apic_delivery_mode delivery_mode,
+    enum apic_dest_mode dest_mode,
+    enum apic_level level,
+    enum apic_trigger_mode trigger_mode
+) {
+    uint64_t icr = mmio_field(APIC_ICR_VECTOR, vector);
+    icr |= mmio_field(APIC_ICR_DELIVERY_MODE, delivery_mode);
+    icr |= mmio_field(APIC_ICR_DEST_MODE, dest_mode);
+    icr |= mmio_field(APIC_ICR_LEVEL, level);
+    icr |= mmio_field(APIC_ICR_TRIGGER_MODE, trigger_mode);
+
+    if (lapic->xapic_regs)
+        icr |= mmio_field(APIC_ICR_DEST_XAPIC, target);
+    else
+        icr |= mmio_field(APIC_ICR_DEST_X2APIC, target);
+
+    lapic_write_icr(lapic, icr);
+
+    if (lapic->xapic_regs) {
+        while (mmio_field_get(APIC_ICR_DELIVERY_STATUS, lapic_read_icr(lapic)) == APIC_DELIVERY_STATUS_PENDING)
+            arch_cpu_pause();
+    }
+}
+
+constexpr uint16_t PIC1_COMMAND_PORT = 0x20;
+constexpr uint16_t PIC1_DATA_PORT = 0x21;
+constexpr uint16_t PIC2_COMMAND_PORT = 0xA0;
+constexpr uint16_t PIC2_DATA_PORT = 0xA1;
+
+void pic_disable() {
+    // We initialize the PIC properly, but completely disable it and use the APIC in favor of it.
+    // Remap IRQs so they start at 0x20 since interrupts 0x00..0x1F are used by CPU exceptions.
+    asm_outb(PIC1_COMMAND_PORT, 0x11);
+    asm_outb(PIC1_DATA_PORT, 0x20);
+    asm_outb(PIC1_DATA_PORT, 0x04);
+    asm_outb(PIC1_DATA_PORT, 0x01);
+    asm_outb(PIC1_DATA_PORT, 0xFF);
+
+    // Same for the slave PIC.
+    asm_outb(PIC2_COMMAND_PORT, 0x11);
+    asm_outb(PIC2_DATA_PORT, 0x28);
+    asm_outb(PIC2_DATA_PORT, 0x02);
+    asm_outb(PIC2_DATA_PORT, 0x01);
+    asm_outb(PIC2_DATA_PORT, 0xFF);
+}

arch/x86_64/archctl.c

@@ -0,0 +1,32 @@
+#include <zinnia/errno.h>
+#include <zinnia/syscall.h>
+#include <arch/archctl.h>
+#include <arch/mem.h>
+#include <uapi/archctl.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+
+struct syscall_ret arch_archctl(uintptr_t op, uintptr_t arg) {
+    struct syscall_ret ret = {};
+
+    switch (op) {
+    case __ARCHCTL_SET_FSBASE:
+        if (!arch_mem_addr_is_user(arg)) {
+            ret.error = EINVAL;
+            return ret;
+        }
+        asm_wrmsr(MSR_FS_BASE, arg);
+        return ret;
+    case __ARCHCTL_SET_GSBASE:
+        if (!arch_mem_addr_is_user(arg)) {
+            ret.error = EINVAL;
+            return ret;
+        }
+        // Update the inactive GSBASE MSR, not ours.
+        asm_wrmsr(MSR_KERNEL_GS_BASE, arg);
+        return ret;
+    default:
+        ret.error = EINVAL;
+        return ret;
+    }
+}

arch/x86_64/gdt.c

@@ -1,14 +1,14 @@
-#include <kernel/percpu.h>
+#include <zinnia/percpu.h>
-#include <kernel/syscalls.h>
 #include <stddef.h>
 #include <string.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
+#include <x86_64/percpu.h>
 #include <x86_64/tss.h>
 
-static const struct gdt main_gdt = {
+const struct gdt gdt_base = {
-    .null = {.val = 0},
+    .null = {0},
     .kernel_code32 = {.val = 0x00cf9b000000ffff},
     .kernel_data32 = {.val = 0x00cf93000000ffff},
     .kernel_code64 = {.val = 0x00a09b0000000000},
@@ -18,17 +18,17 @@ static const struct gdt main_gdt = {
     .user_code64 = {.val = 0x00a0fb0000000000},
     .tss = {
         .limit0 = sizeof(struct tss) - 1,
-        .access = 0x89, // Present | DPL = 0 | System | 64-bit TSS
+        .access = 0x89, // Present | DPL=0 | System | 64-bit TSS
         .limit1_flags = ((sizeof(struct tss) - 1) >> 16) & 0x0F,
         .reserved = 0,
     },
 };
 
 void gdt_load() {
-    struct gdt* gdt = &percpu_get()->arch.gdt;
+    struct gdt* gdt = &percpu_ref(arch_data)->gdt;
-    memcpy(gdt, &main_gdt, sizeof(main_gdt));
+    memcpy(gdt, &gdt_base, sizeof(gdt_base));
 
-    struct tss* tss = &percpu_get()->arch.tss;
+    struct tss* tss = &percpu_ref(arch_data)->tss;
     tss->iopb = sizeof(struct tss);
 
     uintptr_t tss_addr = (uintptr_t)tss;

arch/x86_64/hpet.c

@@ -0,0 +1,85 @@
+#include <zinnia/acpi.h>
+#include <zinnia/assert.h>
+#include <zinnia/clock.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/mmio.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <uacpi/acpi.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+#include <uacpi/uacpi.h>
+
+static bool use_hpet = true;
+static void hpet_option(bool use) {
+    use_hpet = use;
+}
+CMDLINE_OPTION("x86_64.hpet", "Enable High Precision Event Timer clock", hpet_option);
+
+struct hpet {
+    struct clock clock;
+    void* regs;
+    uint32_t period;
+};
+
+constexpr mmio_reg64 HPET_REG_CAP = {.offset = 0x0};
+constexpr mmio_field64 HPET_CAP_PERIOD = {HPET_REG_CAP, 32, 32};
+constexpr mmio_reg64 HPET_REG_CFG = {.offset = 0x10};
+constexpr mmio_field64 HPET_CFG_ENABLE = {HPET_REG_CFG, 0, 1};
+constexpr mmio_reg64 HPET_REG_COUNTER = {.offset = 0xF0};
+
+static void hpet_reset(struct clock* c) {
+    struct hpet* hpet = containerof(c, struct hpet, clock);
+    mmio_reg_write(hpet->regs, HPET_REG_COUNTER, 0);
+}
+
+static uint64_t hpet_get_elapsed_ns(struct clock* c) {
+    struct hpet* hpet = containerof(c, struct hpet, clock);
+    uint64_t counter = mmio_reg_read(hpet->regs, HPET_REG_COUNTER);
+    return (uint64_t)((__uint128_t)counter * (__uint128_t)hpet->period / 1'000'000);
+}
+
+static struct hpet hpet_clock = {
+    .clock = {
+        .name = "hpet",
+        .priority = 128,
+        .reset = hpet_reset,
+        .get_elapsed_ns = hpet_get_elapsed_ns,
+    },
+};
+
+static void hpet_init(struct boot_info*) {
+    if (!use_hpet)
+        return;
+
+    uacpi_table table = {0};
+    if (uacpi_table_find_by_signature("HPET", &table) != UACPI_STATUS_OK) {
+        log("x86_64/hpet: No HPET table found\n");
+        return;
+    }
+
+    struct acpi_hpet* hpet = table.ptr;
+
+    uintptr_t vaddr = vm_alloc(0x1000);
+    pmap_map(&kernel_vma.pmap, vaddr, hpet->address.address, PTE_READ | PTE_WRITE, CACHE_MMIO);
+    hpet_clock.regs = (void*)vaddr;
+
+    uacpi_table_unref(&table);
+    log("x86_64/hpet: HPET registers mapped at %p\n", hpet_clock.regs);
+
+    uint64_t cfg = mmio_reg_read(hpet_clock.regs, HPET_REG_CFG);
+    cfg |= mmio_field(HPET_CFG_ENABLE, 1);
+    mmio_reg_write(hpet_clock.regs, HPET_REG_CFG, cfg);
+
+    uint64_t cap = mmio_reg_read(hpet_clock.regs, HPET_REG_CAP);
+    hpet_clock.period = mmio_field_get(HPET_CAP_PERIOD, cap);
+
+    assert(clock_switch(&hpet_clock.clock), "Unable to switch to HPET!");
+}
+INIT_TASK(hpet_stage, "arch.x86_64.hpet", hpet_init);
+INIT_DEPENDS(hpet_stage, acpi_stage);
+INIT_ENTAILS(hpet_stage, percpu_stage);

arch/x86_64/idt.c

@@ -1,16 +1,25 @@
-#include <common/compiler.h>
+// SPDX-License-Identifier: ISC
-#include <kernel/irq.h>
+
-#include <kernel/panic.h>
+#include <zinnia/assert.h>
-#include <kernel/percpu.h>
+#include <zinnia/compiler.h>
-#include <kernel/print.h>
+#include <zinnia/irq.h>
-#include <kernel/sched.h>
+#include <zinnia/panic.h>
-#include <assert.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/timer.h>
+#include <zinnia/usercopy.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <arch/sched.h>
 #include <bits/sched.h>
 #include <stddef.h>
 #include <stdint.h>
 #include <x86_64/apic.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
+#include <x86_64/percpu.h>
 
 struct [[__packed]] idt_entry {
     uint16_t base0;
@@ -33,31 +42,82 @@ static_assert(sizeof(struct idtr) == 10);
 
 void interrupt_handler(struct arch_context* ctx) {
     struct percpu* cpu = percpu_get();
-    bool old = irq_set_interrupted(true);
+    struct arch_percpu* arch_cpu = percpu_ref(arch_data);
-
     uint64_t isr = ctx->isr;
+
     switch (isr) {
     case IDT_PF:
         uint64_t cr2;
         asm volatile("mov %0, cr2" : "=r"(cr2));
-        panic("Page fault at %#lx, accessed: %#lx\n", ctx->rip, cr2);
+
+        const struct vma_fault_info info = {
+            .ip = ctx->rip,
+            .addr = cr2,
+            .caused_by_user = (ctx->cs & CPL_USER) == CPL_USER,
+            .caused_by_write = ctx->error & (1 << 1),
+            .caused_by_fetch = ctx->error & (1 << 4),
+            .page_was_present = ctx->error & (1 << 0),
+        };
+
+        struct task* current = cpu->sched.current;
+        const bool irqs_were_on = ctx->rflags & RFLAGS_IF;
+        if (irqs_were_on)
+            irq_set_state(true);
+
+        const bool handled = current && current->space && vma_page_fault(current->space, &info);
+
+        if (irqs_were_on)
+            irq_set_state(false);
+
+        if (handled)
+            break;
+
+        if (current && current->usercopy_region) {
+            struct usercopy_region* region = current->usercopy_region;
+            if (ctx->rip >= (uintptr_t)region->start_ip && ctx->rip < (uintptr_t)region->end_ip) {
+                ctx->rip = (uintptr_t)region->fault_ip;
+                break;
+            }
+        }
+
+        panic(
+            "Unrecoverable page fault. Attempted to %s a %s page at %#lx (IP: %#lx)\n",
+            info.caused_by_write       ? "write to"
+                : info.caused_by_fetch ? "execute on"
+                                       : "read from",
+            info.page_was_present ? "present" : "non-present",
+            info.addr,
+            info.ip
+        );
+    case IDT_SPURIOUS:
+        break;
     case IDT_IPI_RESCHED:
+        lapic_eoi(&arch_cpu->lapic);
         arch_sched_preempt_disable();
-        if (arch_sched_preempt_enable()) {
+        if (arch_sched_preempt_enable())
-            lapic_eoi(&cpu->arch.lapic);
+            sched_reschedule(&cpu->sched);
+        break;
+    case IDT_IPI_SHOOTDOWN:
+        lapic_eoi(&arch_cpu->lapic);
+        pmap_shootdown_handle();
+        break;
+    case IDT_TIMER:
+        lapic_eoi(&arch_cpu->lapic);
+        timer_interrupt();
+        if (cpu->sched.need_resched) {
+            arch_sched_preempt_disable();
+            if (arch_sched_preempt_enable())
                 sched_reschedule(&cpu->sched);
         }
         break;
-    case 0x20 ...(0x20 + ARRAY_SIZE(cpu->arch.irq_lines)):
-        struct irq_line* line = cpu->arch.irq_lines[isr - 0x20];
-        assert(line, "Unhandled interrupt on ISR %lu\n", isr);
-        // TODO: irq_raise(line);
-        break;
     default:
-        panic("Unhandled exception on ISR %lu\n", isr);
+        if (isr < 0x20)
+            panic("Unhandled exception %lu at %#lx, error: %#lx\n", isr, ctx->rip, ctx->error);
+        assert(isr - 0x20 < ARRAY_SIZE(arch_cpu->irq_lines), "Unhandled interrupt on ISR %lu", isr);
+        struct irq_line* line = arch_cpu->irq_lines[isr - 0x20];
+        assert(line, "Unhandled interrupt on ISR %lu", isr);
+        irq_line_raise(line);
     }
-
-    irq_set_interrupted(old);
 }
 
 #define CS_OFFSET (sizeof(struct arch_context) - sizeof(uint64_t) - offsetof(struct arch_context, cs))
@@ -197,6 +257,7 @@ static struct idt_entry idt[256] = {};
 
 void idt_init() {
     REPEAT_256(INTERRUPT_SETUP)
+    idt[IDT_BP].attrib |= CPL_USER << 5;
 }
 
 void idt_load() {

arch/x86_64/include/bits/cpu.h

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+static inline void arch_cpu_pause() {
+    asm volatile("pause");
+}
+
+#define ARCH_CACHE_LINE_SIZE 64

arch/x86_64/include/bits/elf.h

@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/elf.h>
+
+#define ELF_ARCH_CLASS   ELFCLASS64
+#define ELF_ARCH_DATA    ELFDATA2LSB
+#define ELF_ARCH_MACHINE EM_X86_64
+
+#define ELF_R_SYM   ELF64_R_SYM
+#define ELF_R_TYPE  ELF64_R_TYPE
+#define ELF_ST_BIND ELF64_ST_BIND
+#define ELF_ST_TYPE ELF64_ST_TYPE
+
+typedef elf64_ehdr_t elf_ehdr_t;
+typedef elf64_phdr_t elf_phdr_t;
+typedef elf64_dyn_t elf_dyn_t;
+typedef elf64_addr_t elf_addr_t;
+typedef elf64_off_t elf_off_t;
+typedef elf64_nhdr_t elf_nhdr_t;
+typedef elf64_auxv_t elf_auxv_t;
+typedef elf64_sym_t elf_sym_t;
+typedef elf64_rela_t elf_rela_t;
+typedef elf64_rel_t elf_rel_t;
+typedef elf64_relr_t elf_relr_t;
+
+enum {
+    R_NONE = 0,
+    R_64 = 1,
+    R_COPY = 5,
+    R_GLOB_DAT = 6,
+    R_JUMP_SLOT = 7,
+    R_RELATIVE = 8,
+    R_IRELATIVE = 37,
+};

arch/x86_64/include/bits/pmap.h

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stdint.h>
+
+typedef uint64_t pte_t;
+
+enum : pte_t {
+    ARCH_FLAG_PRESENT = 1 << 0,
+    ARCH_FLAG_READ_WRITE = 1 << 1,
+    ARCH_FLAG_USER_MODE = 1 << 2,
+    ARCH_FLAG_WRITE_THROUGH = 1 << 3,
+    ARCH_FLAG_CACHE_DISABLE = 1 << 4,
+    ARCH_FLAG_ACCESSED = 1 << 5,
+    ARCH_FLAG_DIRTY = 1 << 6,
+    ARCH_FLAG_SIZE = 1 << 7,
+    ARCH_FLAG_GLOBAL = 1 << 8,
+    ARCH_FLAG_AVAILABLE = 1 << 9,
+    ARCH_FLAG_ATTRIBUTE_TABLE = 1 << 7,
+    ARCH_FLAG_EXECUTE_DISABLE = 1LU << 63,
+
+    ARCH_PTE_ADDR_MASK = 0x000F'FFFF'FFFF'F000,
+};

arch/x86_64/include/bits/sched.h

@@ -1,36 +1,34 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
-#include <bits/types.h>
+#include <zinnia/types.h>
 #include <stdint.h>
 
 struct arch_task_context {
     uint64_t rsp;
     uint8_t* fpu_region;
+    phys_t fpu_phys;
     uint16_t ds, es, fs, gs;
     uint64_t fs_base, gs_base;
 };
 
-#define ARCH_ABI(typ, reg, ...) \
-    union { \
-        typ reg, __VA_ARGS__; \
-    }
-
 struct arch_context {
     uint64_t r15;
     uint64_t r14;
-    ARCH_ABI(uint64_t, r13, a7);
+    uint64_t r13;
-    ARCH_ABI(uint64_t, r12, a6);
+    uint64_t r12;
     uint64_t r11;
-    ARCH_ABI(uint64_t, r10, a5);
+    uint64_t r10;
-    ARCH_ABI(uint64_t, r9, a3);
+    uint64_t r9;
-    ARCH_ABI(uint64_t, r8, a4);
+    uint64_t r8;
-    ARCH_ABI(uint64_t, rsi, a1);
+    uint64_t rsi;
-    ARCH_ABI(uint64_t, rdi, a0);
+    uint64_t rdi;
     uint64_t rbp;
-    ARCH_ABI(uint64_t, rdx, a2, ret1);
+    uint64_t rdx;
     uint64_t rcx;
     uint64_t rbx;
-    ARCH_ABI(uint64_t, rax, num, ret0);
+    uint64_t rax;
     uint64_t isr;   // Pushed onto the stack by the interrupt handler stubs.
     uint64_t error; // Pushed onto the stack by the CPU if the interrupt has an error code.
     uint64_t rip;   // The rest is pushed onto the stack by the CPU during an interrupt.
@@ -40,16 +38,18 @@ struct arch_context {
     uint64_t ss;
 };
 
-static_assert(sizeof(struct arch_context) == 22 * sizeof(uint64_t));
-
 #define ARCH_CTX_NUM  rax
+#define ARCH_CTX_SP   rsp
 #define ARCH_CTX_A0   rdi
 #define ARCH_CTX_A1   rsi
 #define ARCH_CTX_A2   rdx
-#define ARCH_CTX_A3   r9
+#define ARCH_CTX_A3   r10
 #define ARCH_CTX_A4   r8
-#define ARCH_CTX_A5   r10
+#define ARCH_CTX_A5   r9
-#define ARCH_CTX_A6   r12
-#define ARCH_CTX_A7   r13
 #define ARCH_CTX_RET0 rax
 #define ARCH_CTX_RET1 rdx
+
+struct arch_syscall_restart {
+    uint64_t rax;
+    uint64_t rdx;
+};

arch/x86_64/include/bits/types.h

@@ -1,6 +1,7 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>
 
 typedef uint64_t phys_t;
-typedef uint64_t reg_t;

arch/x86_64/include/x86_64/apic.h

@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/errno.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/irq.h>
+#include <zinnia/mmio.h>
+#include <stdint.h>
+
+struct local_apic {
+    uint32_t id;
+    uint32_t ticks_per_10ms;
+    void* xapic_regs;
+};
+
+enum apic_delivery_mode {
+    APIC_DELIVERY_FIXED = 0b000,
+    APIC_DELIVERY_LOWESTPRIO = 0b001,
+    APIC_DELIVERY_SMI = 0b010,
+    APIC_DELIVERY_NMI = 0b100,
+    APIC_DELIVERY_INIT = 0b101,
+    APIC_DELIVERY_STARTUP = 0b110,
+};
+
+enum apic_dest_mode {
+    APIC_DEST_PHYSICAL = 0,
+    APIC_DEST_LOGICAL = 1,
+};
+
+enum apic_delivery_status {
+    APIC_DELIVERY_STATUS_IDLE = 0,
+    APIC_DELIVERY_STATUS_PENDING = 1,
+};
+
+enum apic_level {
+    APIC_LEVEL_DEASSERT = 0,
+    APIC_LEVEL_ASSERT = 1,
+};
+
+enum apic_trigger_mode {
+    APIC_TRIGGER_EDGE = 0,
+    APIC_TRIGGER_LEVEL = 1,
+};
+
+constexpr mmio_reg32 APIC_REG_ID = {.offset = 0x20};
+constexpr mmio_field32 APIC_ID_XAPIC = {APIC_REG_ID, 24, 8};
+constexpr mmio_reg32 APIC_REG_TPR = {.offset = 0x80};
+constexpr mmio_reg32 APIC_REG_EOI = {.offset = 0xB0};
+constexpr mmio_reg32 APIC_REG_LDR = {.offset = 0xD0};
+constexpr mmio_reg32 APIC_REG_DFR = {.offset = 0xE0};
+constexpr mmio_reg32 APIC_REG_SIVR = {.offset = 0xF0};
+constexpr mmio_field32 APIC_SIVR_VECTOR = {APIC_REG_SIVR, 0, 8};
+constexpr mmio_field32 APIC_SIVR_ENABLE = {APIC_REG_SIVR, 8, 1};
+constexpr mmio_reg32 APIC_REG_ESR = {.offset = 0x280};
+constexpr mmio_reg64 APIC_REG_ICR = {.offset = 0x300};
+constexpr mmio_field64 APIC_ICR_VECTOR = {APIC_REG_ICR, 0, 8};
+constexpr mmio_field64 APIC_ICR_DELIVERY_MODE = {APIC_REG_ICR, 8, 3};
+constexpr mmio_field64 APIC_ICR_DEST_MODE = {APIC_REG_ICR, 11, 1};
+constexpr mmio_field64 APIC_ICR_DELIVERY_STATUS = {APIC_REG_ICR, 12, 1};
+constexpr mmio_field64 APIC_ICR_LEVEL = {APIC_REG_ICR, 14, 1};
+constexpr mmio_field64 APIC_ICR_TRIGGER_MODE = {APIC_REG_ICR, 15, 1};
+constexpr mmio_field64 APIC_ICR_DEST_XAPIC = {APIC_REG_ICR, 56, 8};
+constexpr mmio_field64 APIC_ICR_DEST_X2APIC = {APIC_REG_ICR, 32, 32};
+constexpr mmio_reg32 APIC_REG_ICR_LO = {.offset = 0x300};
+constexpr mmio_reg32 APIC_REG_ICR_HI = {.offset = 0x310};
+constexpr mmio_reg32 APIC_REG_LVT_TR = {.offset = 0x320};
+constexpr mmio_field32 APIC_LVT_TR_VECTOR = {APIC_REG_LVT_TR, 0, 8};
+constexpr mmio_reg32 APIC_REG_ICR_TIMER = {.offset = 0x380};
+constexpr mmio_reg32 APIC_REG_CCR = {.offset = 0x390};
+constexpr mmio_reg32 APIC_REG_DCR = {.offset = 0x3E0};
+
+void lapic_init(struct local_apic* lapic);
+void lapic_eoi(struct local_apic* lapic);
+
+void lapic_send_ipi(
+    struct local_apic* lapic,
+    uint32_t target,
+    uint8_t vector,
+    enum apic_delivery_mode delivery_mode,
+    enum apic_dest_mode dest_mode,
+    enum apic_level level,
+    enum apic_trigger_mode trigger_mode
+);
+
+void pic_disable();
+
+errno_t apic_alloc_vector(struct irq_line* line, uint8_t* out_vector, uint32_t* out_lapic);
+void apic_free_vector(uint8_t vector);
+
+INIT_EXTERN(ioapic_stage);

arch/x86_64/include/x86_64/asm.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdint.h>
@@ -40,6 +42,12 @@ static inline void asm_wrxcr(uint32_t reg, uint64_t val) {
     asm volatile("xsetbv" ::"a"(eax), "d"(edx), "c"(reg) : "memory");
 }
 
+static inline uint64_t asm_rdxcr(uint32_t reg) {
+    uint32_t eax, edx;
+    asm volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(reg));
+    return ((uint64_t)edx << 32) | eax;
+}
+
 // Saves the FPU state to a 512-byte region of memory using FXSAVE.
 // Pointer must be 16-byte aligned.
 static inline void asm_fxsave(void* mem) {
@@ -55,13 +63,13 @@ static inline void asm_fxrstor(const void* mem) {
 // Saves the FPU state to a region of memory using XSAVE.
 // Pointer must be 16-byte aligned.
 static inline void asm_xsave(void* mem) {
-    asm volatile("xsave [%0]" ::"r"(mem) : "memory");
+    asm volatile("xsave [%0]" ::"r"(mem), "a"(0xFFFF'FFFF), "d"(0xFFFF'FFFF) : "memory");
 }
 
 // Restores the FPU state from a region of memory using XRSTOR.
 // Pointer must be 16-byte aligned.
 static inline void asm_xrstor(const void* mem) {
-    asm volatile("xrstor [%0]" ::"r"(mem) : "memory");
+    asm volatile("xrstor [%0]" ::"r"(mem), "a"(0xFFFF'FFFF), "d"(0xFFFF'FFFF) : "memory");
 }
 
 static inline uint8_t asm_inb(uint16_t port) {

arch/x86_64/include/x86_64/defs.h

@@ -0,0 +1,258 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stdint.h>
+
+enum : uint32_t {
+    CPUID_1C_SSE3 = 1u << 0,
+    CPUID_1C_PCLMUL = 1u << 1,
+    CPUID_1C_DTES64 = 1u << 2,
+    CPUID_1C_MONITOR = 1u << 3,
+    CPUID_1C_DS_CPL = 1u << 4,
+    CPUID_1C_VMX = 1u << 5,
+    CPUID_1C_SMX = 1u << 6,
+    CPUID_1C_EST = 1u << 7,
+    CPUID_1C_TM2 = 1u << 8,
+    CPUID_1C_SSSE3 = 1u << 9,
+    CPUID_1C_CID = 1u << 10,
+    CPUID_1C_SDBG = 1u << 11,
+    CPUID_1C_FMA = 1u << 12,
+    CPUID_1C_CX16 = 1u << 13,
+    CPUID_1C_XTPR = 1u << 14,
+    CPUID_1C_PDCM = 1u << 15,
+    CPUID_1C_PCID = 1u << 17,
+    CPUID_1C_DCA = 1u << 18,
+    CPUID_1C_SSE4_1 = 1u << 19,
+    CPUID_1C_SSE4_2 = 1u << 20,
+    CPUID_1C_X2APIC = 1u << 21,
+    CPUID_1C_MOVBE = 1u << 22,
+    CPUID_1C_POPCNT = 1u << 23,
+    CPUID_1C_TSC = 1u << 24,
+    CPUID_1C_AES = 1u << 25,
+    CPUID_1C_XSAVE = 1u << 26,
+    CPUID_1C_OSXSAVE = 1u << 27,
+    CPUID_1C_AVX = 1u << 28,
+    CPUID_1C_F16C = 1u << 29,
+    CPUID_1C_RDRAND = 1u << 30,
+    CPUID_1C_HYPERVISOR = 1u << 31,
+};
+
+enum : uint32_t {
+    CPUID_1D_FPU = 1u << 0,
+    CPUID_1D_VME = 1u << 1,
+    CPUID_1D_DE = 1u << 2,
+    CPUID_1D_PSE = 1u << 3,
+    CPUID_1D_TSC = 1u << 4,
+    CPUID_1D_MSR = 1u << 5,
+    CPUID_1D_PAE = 1u << 6,
+    CPUID_1D_MCE = 1u << 7,
+    CPUID_1D_CX8 = 1u << 8,
+    CPUID_1D_APIC = 1u << 9,
+    CPUID_1D_SEP = 1u << 11,
+    CPUID_1D_MTRR = 1u << 12,
+    CPUID_1D_PGE = 1u << 13,
+    CPUID_1D_MCA = 1u << 14,
+    CPUID_1D_CMOV = 1u << 15,
+    CPUID_1D_PAT = 1u << 16,
+    CPUID_1D_PSE36 = 1u << 17,
+    CPUID_1D_PSN = 1u << 18,
+    CPUID_1D_CLFLUSH = 1u << 19,
+    CPUID_1D_DS = 1u << 21,
+    CPUID_1D_ACPI = 1u << 22,
+    CPUID_1D_MMX = 1u << 23,
+    CPUID_1D_FXSR = 1u << 24,
+    CPUID_1D_SSE = 1u << 25,
+    CPUID_1D_SSE2 = 1u << 26,
+    CPUID_1D_SS = 1u << 27,
+    CPUID_1D_HTT = 1u << 28,
+    CPUID_1D_TM = 1u << 29,
+    CPUID_1D_IA64 = 1u << 30,
+    CPUID_1D_PBE = 1u << 31,
+};
+
+enum : uint32_t {
+    CPUID_7B_FSGSBASE = 1u << 0,
+    CPUID_7B_SGX = 1u << 2,
+    CPUID_7B_BMI = 1u << 3,
+    CPUID_7B_HLE = 1u << 4,
+    CPUID_7B_AVX2 = 1u << 5,
+    CPUID_7B_SMEP = 1u << 7,
+    CPUID_7B_BMI2 = 1u << 8,
+    CPUID_7B_RTM = 1u << 11,
+    CPUID_7B_AVX512F = 1u << 16,
+    CPUID_7B_AVX512DQ = 1u << 17,
+    CPUID_7B_RDSEED = 1u << 18,
+    CPUID_7B_ADX = 1u << 19,
+    CPUID_7B_SMAP = 1u << 20,
+    CPUID_7B_AVX512IFMA = 1u << 21,
+    CPUID_7B_CLFLUSHOPT = 1u << 23,
+    CPUID_7B_CLWB = 1u << 24,
+    CPUID_7B_AVX512CD = 1u << 28,
+    CPUID_7B_SHA = 1u << 29,
+    CPUID_7B_AVX512BW = 1u << 30,
+    CPUID_7B_AVX512VL = 1u << 31,
+};
+
+enum : uint32_t {
+    CPUID_7C_AVX512VBMI = 1u << 1,
+    CPUID_7C_UMIP = 1u << 2,
+    CPUID_7C_PKU = 1u << 3,
+    CPUID_7C_OSPKE = 1u << 4,
+    CPUID_7C_WAITPKG = 1u << 5,
+    CPUID_7C_AVX512VBMI2 = 1u << 6,
+    CPUID_7C_SHSTK = 1u << 7,
+    CPUID_7C_GFNI = 1u << 8,
+    CPUID_7C_VAES = 1u << 9,
+    CPUID_7C_VPCLMULQDQ = 1u << 10,
+    CPUID_7C_AVX512VNNI = 1u << 11,
+    CPUID_7C_AVX512BITALG = 1u << 12,
+    CPUID_7C_TME_EN = 1u << 13,
+    CPUID_7C_AVX512VPOPCNTDQ = 1u << 14,
+    CPUID_7C_RDPID = 1u << 22,
+    CPUID_7C_KL = 1u << 23,
+    CPUID_7C_CLDEMOTE = 1u << 25,
+    CPUID_7C_MOVDIRI = 1u << 27,
+    CPUID_7C_MOVDIR64B = 1u << 28,
+    CPUID_7C_ENQCMD = 1u << 29,
+};
+
+enum : uint32_t {
+    CPUID_7D_UINTR = 1u << 5,
+    CPUID_7D_AVX512VP2INTERSECT = 1u << 8,
+    CPUID_7D_SERIALIZE = 1u << 14,
+    CPUID_7D_TSXLDTRK = 1u << 16,
+    CPUID_7D_PCONFIG = 1u << 18,
+    CPUID_7D_IBT = 1u << 20,
+    CPUID_7D_AMX_BF16 = 1u << 22,
+    CPUID_7D_AVX512FP16 = 1u << 23,
+    CPUID_7D_AMX_TILE = 1u << 24,
+    CPUID_7D_AMX_INT8 = 1u << 25,
+};
+
+enum : uint64_t {
+    RFLAGS_CF = 1ull << 0,
+    RFLAGS_BRKI = 1ull << 1,
+    RFLAGS_PF = 1ull << 2,
+    RFLAGS_AF = 1ull << 4,
+    RFLAGS_ZF = 1ull << 6,
+    RFLAGS_SF = 1ull << 7,
+    RFLAGS_TF = 1ull << 8,
+    RFLAGS_IF = 1ull << 9,
+    RFLAGS_DF = 1ull << 10,
+    RFLAGS_OF = 1ull << 11,
+    RFLAGS_IOPL = 3ull << 12,
+    RFLAGS_NT = 1ull << 14,
+    RFLAGS_RF = 1ull << 16,
+    RFLAGS_VM = 1ull << 17,
+    RFLAGS_AC = 1ull << 18,
+    RFLAGS_VIF = 1ull << 19,
+    RFLAGS_VIP = 1ull << 20,
+    RFLAGS_ID = 1ull << 21,
+};
+
+enum : uint64_t {
+    XCR0_X87 = 1ull << 0,
+    XCR0_SSE = 1ull << 1,
+    XCR0_AVX = 1ull << 2,
+    XCR0_AXV512_OPMASK = 1ull << 5,
+    XCR0_AVX512_ZMM_HI256 = 1ull << 6,
+    XCR0_AVX512_HI16_ZMM = 1ull << 7,
+};
+
+enum : uint64_t {
+    CR0_PE = 1ull << 0,
+    CR0_MP = 1ull << 1,
+    CR0_EM = 1ull << 2,
+    CR0_TS = 1ull << 3,
+    CR0_ET = 1ull << 4,
+    CR0_NE = 1ull << 5,
+    CR0_WP = 1ull << 16,
+    CR0_AM = 1ull << 18,
+    CR0_NW = 1ull << 29,
+    CR0_CD = 1ull << 30,
+    CR0_PG = 1ull << 31,
+};
+
+enum : uint64_t {
+    CR4_VME = 1ull << 0,
+    CR4_PVI = 1ull << 1,
+    CR4_TSD = 1ull << 2,
+    CR4_DE = 1ull << 3,
+    CR4_PSE = 1ull << 4,
+    CR4_PAE = 1ull << 5,
+    CR4_MCE = 1ull << 6,
+    CR4_PGE = 1ull << 7,
+    CR4_PCE = 1ull << 8,
+    CR4_OSFXSR = 1ull << 9,
+    CR4_OSXMMEXCPT = 1ull << 10,
+    CR4_UMIP = 1ull << 11,
+    CR4_LA57 = 1ull << 12,
+    CR4_VMXE = 1ull << 13,
+    CR4_SMXE = 1ull << 14,
+    CR4_FSGSBASE = 1ull << 16,
+    CR4_PCIDE = 1ull << 17,
+    CR4_OSXSAVE = 1ull << 18,
+    CR4_SMEP = 1ull << 20,
+    CR4_SMAP = 1ull << 21,
+    CR4_PKE = 1ull << 22,
+    CR4_CET = 1ull << 23,
+    CR4_PKS = 1ull << 24,
+};
+
+enum : uint64_t {
+    MSR_EFER_SCE = 1ull << 0,
+    MSR_EFER_LME = 1ull << 8,
+    MSR_EFER_LMA = 1ull << 10,
+    MSR_EFER_NXE = 1ull << 11,
+    MSR_EFER_SVME = 1ull << 12,
+    MSR_EFER_LMSLE = 1ull << 13,
+    MSR_EFER_FFXSR = 1ull << 14,
+    MSR_EFER_TCE = 1ull << 15,
+};
+
+enum : uint32_t {
+    MSR_PAT = 0x277,
+    MSR_EFER = 0xC0000080,
+    MSR_STAR = 0xC0000081,
+    MSR_LSTAR = 0xC0000082,
+    MSR_CSTAR = 0xC0000083,
+    MSR_SFMASK = 0xC0000084,
+    MSR_FS_BASE = 0xC0000100,
+    MSR_GS_BASE = 0xC0000101,
+    MSR_KERNEL_GS_BASE = 0xC0000102,
+};
+
+enum : uint8_t {
+    PAT_UC = 0x0,
+    PAT_WC = 0x1,
+    PAT_WT = 0x4,
+    PAT_WP = 0x5,
+    PAT_WB = 0x6,
+    PAT_UCM = 0x7,
+};
+
+enum : uint8_t {
+    CPL_USER = 0x3,
+    CPL_KERNEL = 0x0,
+};
+
+enum : uint8_t {
+    IDT_DE = 0x0,
+    IDT_DB = 0x1,
+    IDT_BP = 0x3,
+    IDT_OF = 0x4,
+    IDT_BR = 0x5,
+    IDT_UD = 0x6,
+    IDT_NM = 0x7,
+    IDT_SS = 0xC,
+    IDT_GP = 0xD,
+    IDT_PF = 0xE,
+    IDT_MF = 0x10,
+    IDT_AC = 0x11,
+    IDT_XM = 0x13,
+    IDT_IPI_SHOOTDOWN = 0xFC,
+    IDT_TIMER = 0xFD,
+    IDT_SPURIOUS = 0xFE,
+    IDT_IPI_RESCHED = 0xFF,
+};

arch/x86_64/include/x86_64/gdt.h

@@ -1,6 +1,8 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
-#include <common/compiler.h>
+#include <zinnia/compiler.h>
 #include <stdint.h>
 
 union gdt_desc {
@@ -50,7 +52,7 @@ struct [[__packed]] gdtr {
     struct gdt* base;
 };
 
-struct percpu;
+extern const struct gdt gdt_base;
 
 // Initializes a GDT on the local CPU.
 void gdt_load();

arch/x86_64/include/x86_64/idt.h

@@ -1,6 +1,8 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
-#include <kernel/init.h>
+#include <zinnia/init.h>
 
 [[__init]]
 void idt_init();

arch/x86_64/include/x86_64/percpu.h

@@ -1,6 +1,9 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
-#include <kernel/irq.h>
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
 #include <stddef.h>
 #include <x86_64/apic.h>
 #include <x86_64/gdt.h>
@@ -16,3 +19,8 @@ struct arch_percpu {
     void (*fpu_restore)(const void*);
     bool can_smap;
 };
+
+extern struct arch_percpu arch_data;
+
+// Initializes the calling CPU and makes `cpu` its per-CPU block.
+void x86_64_cpu_setup(struct percpu* cpu);

arch/x86_64/include/x86_64/pmap.h

@@ -0,0 +1,4 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+#include <zinnia/pmap.h>

arch/x86_64/include/x86_64/smp.h

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/compiler.h>
+#include <zinnia/utils.h>
+#include <stdint.h>
+#include <x86_64/gdt.h>
+
+struct [[__packed]] smp_info {
+    struct gdt gdt;
+    struct gdtr gdtr;
+    uint32_t farjmp_offset;
+    uint32_t farjmp_segment;
+    uint32_t temp_stack;
+    uint32_t temp_cr3;
+    uint64_t entry;
+    uint64_t hhdm_offset;
+    uint64_t percpu;
+    uint32_t lapic_id;
+    atomic_bool booted;
+};
+
+// Discovers all APs and starts them.
+void x86_64_smp_init();

arch/x86_64/include/x86_64/syscall.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 void arch_syscall_stub();

arch/x86_64/include/x86_64/tsc.h

@@ -0,0 +1,5 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+void tsc_init();

arch/x86_64/include/x86_64/tss.h

@@ -1,7 +1,9 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
-#include <common/compiler.h>
 #include <stdint.h>
+#include <zinnia/compiler.h>
 
 struct [[__packed]] tss {
     uint32_t reserved0;

arch/x86_64/ioapic.c

@@ -0,0 +1,281 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "ioapic: " fmt
+
+#include <zinnia/acpi.h>
+#include <zinnia/alloc.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/irq.h>
+#include <zinnia/mmio.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/spin.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/irq.h>
+#include <arch/mem.h>
+#include <uacpi/acpi.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+#include <x86_64/apic.h>
+#include <x86_64/percpu.h>
+
+constexpr size_t ISA_IRQS = 16;
+
+constexpr mmio_reg32 IOAPIC_REG_SELECT = {.offset = 0x00};
+constexpr mmio_reg32 IOAPIC_REG_WINDOW = {.offset = 0x10};
+
+constexpr uint32_t IOAPIC_VERSION = 0x01;
+constexpr uint32_t IOAPIC_REDIRECT = 0x10;
+
+constexpr uint64_t IOAPIC_RTE_POLARITY_LOW = 1 << 13;
+constexpr uint64_t IOAPIC_RTE_TRIGGER_LEVEL = 1 << 15;
+constexpr uint64_t IOAPIC_RTE_MASKED = 1 << 16;
+
+struct ioapic_line {
+    struct irq_line line;
+    struct ioapic* ioapic;
+    uint32_t pin;
+    uint32_t lapic;
+    uint8_t vector;
+};
+
+struct ioapic {
+    void* regs;
+    struct ioapic_line** lines;
+    struct spinlock lock;
+    uint32_t gsi_base;
+    uint32_t num_pins;
+};
+
+struct isa_override {
+    uint32_t gsi;
+    uint16_t flags;
+    bool present;
+};
+
+static struct ioapic* ioapics;
+static size_t num_ioapics;
+static struct isa_override isa_overrides[ISA_IRQS];
+static struct spinlock lines_lock;
+
+static uint32_t ioapic_read(struct ioapic* ioapic, uint32_t reg) {
+    mmio_reg_write(ioapic->regs, IOAPIC_REG_SELECT, reg);
+    return mmio_reg_read(ioapic->regs, IOAPIC_REG_WINDOW);
+}
+
+static void ioapic_write(struct ioapic* ioapic, uint32_t reg, uint32_t value) {
+    mmio_reg_write(ioapic->regs, IOAPIC_REG_SELECT, reg);
+    mmio_reg_write(ioapic->regs, IOAPIC_REG_WINDOW, value);
+}
+
+static void write_rte(struct ioapic* ioapic, uint32_t pin, uint64_t rte) {
+    ioapic_write(ioapic, IOAPIC_REDIRECT + pin * 2, IOAPIC_RTE_MASKED);
+    ioapic_write(ioapic, IOAPIC_REDIRECT + pin * 2 + 1, rte >> 32);
+    ioapic_write(ioapic, IOAPIC_REDIRECT + pin * 2, rte);
+}
+
+static void update_mask(struct ioapic_line* line, bool masked) {
+    struct ioapic* ioapic = line->ioapic;
+    spin_lock(&ioapic->lock);
+    uint32_t low = ioapic_read(ioapic, IOAPIC_REDIRECT + line->pin * 2);
+    low = masked ? low | IOAPIC_RTE_MASKED : low & ~IOAPIC_RTE_MASKED;
+    ioapic_write(ioapic, IOAPIC_REDIRECT + line->pin * 2, low);
+    spin_unlock(&ioapic->lock);
+}
+
+static enum irq_mode ioapic_set_config(
+    struct irq_line* self,
+    enum irq_trigger_mode trigger,
+    enum irq_polarity polarity
+) {
+    struct ioapic_line* line = containerof(self, struct ioapic_line, line);
+    uint64_t rte = line->vector | ((uint64_t)line->lapic << 56);
+    if (polarity == IRQ_POLARITY_LOW)
+        rte |= IOAPIC_RTE_POLARITY_LOW;
+    if (trigger == IRQ_TRIGGER_LEVEL)
+        rte |= IOAPIC_RTE_TRIGGER_LEVEL;
+
+    spin_lock(&line->ioapic->lock);
+    write_rte(line->ioapic, line->pin, rte);
+    spin_unlock(&line->ioapic->lock);
+    return IRQ_MODE_EOI | IRQ_MODE_MASKABLE;
+}
+
+static void ioapic_mask(struct irq_line* self) {
+    update_mask(containerof(self, struct ioapic_line, line), true);
+}
+
+static void ioapic_unmask(struct irq_line* self) {
+    update_mask(containerof(self, struct ioapic_line, line), false);
+}
+
+static void ioapic_eoi(struct irq_line* self) {
+    lapic_eoi(&percpu_ref(arch_data)->lapic);
+}
+
+errno_t arch_irq_get_gsi(uint32_t gsi, struct irq_line** out) {
+    // Find the IOAPIC that handles this GSI.
+    struct ioapic* ioapic = nullptr;
+    for (size_t i = 0; i < num_ioapics; i++) {
+        if (gsi >= ioapics[i].gsi_base && gsi < ioapics[i].gsi_base + ioapics[i].num_pins) {
+            ioapic = &ioapics[i];
+            break;
+        }
+    }
+    if (!ioapic)
+        return ENOENT;
+    const uint32_t pin = gsi - ioapic->gsi_base;
+
+    spin_lock(&lines_lock);
+    struct ioapic_line* line = ioapic->lines[pin];
+    if (line) {
+        spin_unlock(&lines_lock);
+        *out = &line->line;
+        return 0;
+    }
+    spin_unlock(&lines_lock);
+
+    line = mem_alloc(sizeof(*line), 0);
+    if (!line)
+        return ENOMEM;
+    line->line.set_config = ioapic_set_config;
+    line->line.mask = ioapic_mask;
+    line->line.unmask = ioapic_unmask;
+    line->line.eoi = ioapic_eoi;
+    line->ioapic = ioapic;
+    line->pin = pin;
+
+    errno_t status = apic_alloc_vector(&line->line, &line->vector, &line->lapic);
+    if (status) {
+        mem_free(line);
+        return status;
+    }
+
+    spin_lock(&lines_lock);
+    if (ioapic->lines[pin]) {
+        spin_unlock(&lines_lock);
+        apic_free_vector(line->vector);
+        mem_free(line);
+        return arch_irq_get_gsi(gsi, out);
+    }
+    ioapic->lines[pin] = line;
+    spin_unlock(&lines_lock);
+
+    *out = &line->line;
+    return 0;
+}
+
+bool arch_irq_isa_override(
+    uint8_t isa_irq,
+    uint32_t* out_gsi,
+    enum irq_trigger_mode* out_trigger,
+    enum irq_polarity* out_polarity
+) {
+    if (isa_irq >= ISA_IRQS || !isa_overrides[isa_irq].present)
+        return false;
+
+    const struct isa_override* iso = &isa_overrides[isa_irq];
+    *out_gsi = iso->gsi;
+    switch (iso->flags & ACPI_MADT_POLARITY_MASK) {
+    case ACPI_MADT_POLARITY_ACTIVE_HIGH:
+        *out_polarity = IRQ_POLARITY_HIGH;
+        break;
+    case ACPI_MADT_POLARITY_ACTIVE_LOW:
+        *out_polarity = IRQ_POLARITY_LOW;
+        break;
+    }
+    switch (iso->flags & ACPI_MADT_TRIGGERING_MASK) {
+    case ACPI_MADT_TRIGGERING_EDGE:
+        *out_trigger = IRQ_TRIGGER_EDGE;
+        break;
+    case ACPI_MADT_TRIGGERING_LEVEL:
+        *out_trigger = IRQ_TRIGGER_LEVEL;
+        break;
+    }
+    return true;
+}
+
+static uacpi_iteration_decision count_ioapics(uacpi_handle user, struct acpi_entry_hdr* hdr) {
+    size_t* count = user;
+    if (hdr->type == ACPI_MADT_ENTRY_TYPE_IOAPIC)
+        (*count)++;
+    return UACPI_ITERATION_DECISION_CONTINUE;
+}
+
+static uacpi_iteration_decision parse_madt_entry(uacpi_handle, struct acpi_entry_hdr* hdr) {
+    switch (hdr->type) {
+    case ACPI_MADT_ENTRY_TYPE_IOAPIC: {
+        // Map the IOAPIC and mask all of its pins.
+        const struct acpi_madt_ioapic* entry = (const struct acpi_madt_ioapic*)hdr;
+        uintptr_t vaddr = vm_alloc(arch_mem_page_size());
+        if (!vaddr)
+            return UACPI_ITERATION_DECISION_CONTINUE;
+        pmap_map(&kernel_vma.pmap, vaddr, entry->address, PTE_READ | PTE_WRITE, CACHE_MMIO);
+
+        struct ioapic* ioapic = &ioapics[num_ioapics];
+        ioapic->regs = (void*)vaddr;
+        ioapic->gsi_base = entry->gsi_base;
+        ioapic->num_pins = ((ioapic_read(ioapic, IOAPIC_VERSION) >> 16) & 0xFF) + 1;
+        spin_init(&ioapic->lock);
+        ioapic->lines = mem_alloc(ioapic->num_pins * sizeof(*ioapic->lines), 0);
+        if (!ioapic->lines) {
+            pmap_unmap(&kernel_vma.pmap, vaddr);
+            vm_free(vaddr, arch_mem_page_size());
+            return UACPI_ITERATION_DECISION_CONTINUE;
+        }
+
+        for (uint32_t pin = 0; pin < ioapic->num_pins; pin++)
+            write_rte(ioapic, pin, IOAPIC_RTE_MASKED);
+
+        num_ioapics++;
+        log("IOAPIC %u at %#x handles GSIs %u-%u\n",
+            entry->id,
+            entry->address,
+            ioapic->gsi_base,
+            ioapic->gsi_base + ioapic->num_pins - 1);
+        break;
+    }
+    case ACPI_MADT_ENTRY_TYPE_INTERRUPT_SOURCE_OVERRIDE:
+        auto iso = (const struct acpi_madt_interrupt_source_override*)hdr;
+        if (iso->bus == 0 && iso->source < ISA_IRQS) {
+            isa_overrides[iso->source] = (struct isa_override){
+                .gsi = iso->gsi,
+                .flags = iso->flags,
+                .present = true,
+            };
+        }
+        break;
+    }
+    return UACPI_ITERATION_DECISION_CONTINUE;
+}
+
+static void ioapic_init(struct boot_info*) {
+    uacpi_table table;
+    if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) {
+        log_warn("No MADT found\n");
+        return;
+    }
+
+    size_t capacity = 0;
+    uacpi_status status = uacpi_for_each_subtable(table.hdr, sizeof(struct acpi_madt), count_ioapics, &capacity);
+    if (status != UACPI_STATUS_OK)
+        log_warn("Failed to parse the MADT: %s\n", uacpi_status_to_string(status));
+
+    if (capacity) {
+        ioapics = mem_alloc(capacity * sizeof(*ioapics), 0);
+        if (!ioapics) {
+            log_err("Unable to allocate %zu controllers\n", capacity);
+            uacpi_table_unref(&table);
+            return;
+        }
+    }
+
+    uacpi_for_each_subtable(table.hdr, sizeof(struct acpi_madt), parse_madt_entry, nullptr);
+    uacpi_table_unref(&table);
+}
+INIT_TASK(ioapic_stage, "arch.x86_64.ioapic", ioapic_init);
+INIT_DEPENDS(ioapic_stage, acpi_stage, percpu_stage);
+INIT_ENTAILS(ioapic_stage, acpi_namespace_stage);

arch/x86_64/irq.c

@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/percpu.h>
+#include <zinnia/utils.h>
+#include <arch/irq.h>
+#include <stdint.h>
+#include <x86_64/apic.h>
+#include <x86_64/defs.h>
+#include <x86_64/percpu.h>
+
+bool arch_irq_get_state() {
+    uint64_t flags;
+    asm volatile("pushf; pop %0" : "=r"(flags)::"memory");
+    return flags & RFLAGS_IF;
+}
+
+bool arch_irq_set_state(bool state) {
+    bool old = arch_irq_get_state();
+    if (state)
+        asm volatile("sti" ::: "memory");
+    else
+        asm volatile("cli" ::: "memory");
+    return old;
+}
+
+void arch_irq_wait() {
+    asm volatile("hlt");
+}
+
+struct apic_msi_line {
+    struct msi_line msi;
+    uint8_t vector;
+    uint32_t lapic;
+};
+
+static phys_t apic_msi_msg_addr(struct msi_line* self) {
+    struct apic_msi_line* line = containerof(self, struct apic_msi_line, msi);
+    return 0xFEE0'0000 | ((phys_t)line->lapic << 12);
+}
+
+static uint32_t apic_msi_msg_data(struct msi_line* self) {
+    return containerof(self, struct apic_msi_line, msi)->vector;
+}
+
+static enum irq_mode apic_msi_set_config(
+    struct irq_line* self,
+    enum irq_trigger_mode trigger,
+    enum irq_polarity polarity
+) {
+    assert(trigger == IRQ_TRIGGER_EDGE, "MSIs must be edge triggered");
+    return IRQ_MODE_EOI;
+}
+
+static void apic_msi_mask(struct irq_line* self) {}
+
+static void apic_msi_unmask(struct irq_line* self) {}
+
+static void apic_msi_eoi(struct irq_line* self) {
+    lapic_eoi(&percpu_ref(arch_data)->lapic);
+}
+
+errno_t apic_alloc_vector(struct irq_line* line, uint8_t* out_vector, uint32_t* out_lapic) {
+    irq_lock();
+    struct arch_percpu* arch_cpu = percpu_ref(arch_data);
+    size_t slot;
+    for (slot = 0; slot < ARRAY_SIZE(arch_cpu->irq_lines); slot++) {
+        if (!arch_cpu->irq_lines[slot])
+            break;
+    }
+    if (slot == ARRAY_SIZE(arch_cpu->irq_lines)) {
+        irq_unlock();
+        return ENOSPC;
+    }
+
+    *out_vector = slot + 0x20;
+    *out_lapic = arch_cpu->lapic.id;
+    arch_cpu->irq_lines[slot] = line;
+    irq_unlock();
+    return 0;
+}
+
+void apic_free_vector(uint8_t vector) {
+    irq_lock();
+    percpu_ref(arch_data)->irq_lines[vector - 0x20] = nullptr;
+    irq_unlock();
+}
+
+errno_t arch_irq_alloc_msi(struct msi_line** out) {
+    struct apic_msi_line* line = mem_alloc(sizeof(*line), 0);
+    if (!line)
+        return ENOMEM;
+
+    line->msi.line.set_config = apic_msi_set_config;
+    line->msi.line.mask = apic_msi_mask;
+    line->msi.line.unmask = apic_msi_unmask;
+    line->msi.line.eoi = apic_msi_eoi;
+    line->msi.msg_addr = apic_msi_msg_addr;
+    line->msi.msg_data = apic_msi_msg_data;
+
+    errno_t status = apic_alloc_vector(&line->msi.line, &line->vector, &line->lapic);
+    if (status) {
+        mem_free(line);
+        return status;
+    }
+
+    *out = &line->msi;
+    return 0;
+}

arch/x86_64/mem.c

@@ -1,4 +1,4 @@
-#include <kernel/types.h>
+#include <arch/mem.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -29,3 +29,7 @@ uintptr_t arch_mem_pfndb_addr() {
 uintptr_t arch_mem_mapping_addr() {
     return 0xFFFF'C000'0000'0000;
 }
+
+bool arch_mem_addr_is_user(uintptr_t addr) {
+    return addr < 0x0000'7FFF'FFFF'F000;
+}

arch/x86_64/meson.build

@@ -1,24 +1,34 @@
 kernel_c_args += [
     '-mgeneral-regs-only',
     '-mno-red-zone',
-    '-mcmodel=kernel',
+    '-masm=intel',
 ]
 
 kernel_sources += files(
     'apic.c',
+    'archctl.c',
     'gdt.c',
     'hpet.c',
     'idt.c',
+    'ioapic.c',
     'irq.c',
     'mem.c',
     'percpu.c',
     'pmap.c',
+    'random.c',
     'sched.c',
     'serial.c',
     'smp.c',
-    'sys.c',
+    'start.c',
     'syscall.c',
-    'trampoline.S',
     'tsc.c',
     'usercopy.c',
 )
+
+if kconfig.get('CONFIG_PCI', 'n') == 'y'
+    kernel_sources += files('pci.c')
+endif
+
+if kconfig.get('CONFIG_ACPI', 'n') == 'y'
+    kernel_sources += files('acpi.c')
+endif

arch/x86_64/pci.c

@@ -0,0 +1,50 @@
+#include <zinnia/init.h>
+#include <zinnia/pci.h>
+#include <zinnia/spin.h>
+#include <x86_64/asm.h>
+
+constexpr uint16_t PCI_CONFIG_ADDRESS = 0xcf8;
+constexpr uint16_t PCI_CONFIG_DATA = 0xcfc;
+
+static struct spinlock port_lock = {};
+
+static void port_select(const struct pci_address* addr, uint16_t offset) {
+    asm_outl(
+        PCI_CONFIG_ADDRESS,
+        1u << 31 | (uint32_t)addr->bus << 16 | (uint32_t)(addr->device & 0x1f) << 11 |
+            (uint32_t)(addr->function & 0x7) << 8 | (offset & 0xfc)
+    );
+}
+
+static uint32_t port_read32(struct pci_access*, const struct pci_address* addr, uint16_t offset) {
+    spin_lock(&port_lock);
+    port_select(addr, offset);
+    uint32_t value = asm_inl(PCI_CONFIG_DATA);
+    spin_unlock(&port_lock);
+    return value;
+}
+
+static void port_write32(struct pci_access*, const struct pci_address* addr, uint16_t offset, uint32_t value) {
+    spin_lock(&port_lock);
+    port_select(addr, offset);
+    asm_outl(PCI_CONFIG_DATA, value);
+    spin_unlock(&port_lock);
+}
+
+static struct pci_access port_access = {
+    .name = "port",
+    .domain = 0,
+    .start_bus = 0,
+    .end_bus = 255,
+    .priority = PCI_ACCESS_PRIO_LEGACY,
+    .read32 = port_read32,
+    .write32 = port_write32,
+};
+
+[[__init]]
+static void pci_x86_stage_init(struct boot_info*) {
+    pci_access_register(&port_access);
+}
+INIT_TASK(pci_x86_stage, "arch.x86_64.pci", pci_x86_stage_init);
+INIT_DEPENDS(pci_x86_stage, mem_stage);
+INIT_ENTAILS(pci_x86_stage, pci_stage);

arch/x86_64/percpu.c

@@ -1,12 +1,15 @@
-#include <kernel/percpu.h>
+#include <zinnia/initgraph.h>
-#include <kernel/print.h>
+#include <zinnia/percpu.h>
+#include <zinnia/print.h>
+#include <config.h>
 #include <stddef.h>
 #include <x86_64/apic.h>
 #include <x86_64/asm.h>
 #include <x86_64/defs.h>
 #include <x86_64/gdt.h>
-#include <x86_64/hpet.h>
 #include <x86_64/idt.h>
+#include <x86_64/percpu.h>
+#include <x86_64/smp.h>
 #include <x86_64/syscall.h>
 #include <x86_64/tsc.h>
 
@@ -38,22 +41,36 @@ union ia32_star {
 
 static_assert(sizeof(union ia32_star) == 8);
 
-static void setup_cpu(struct percpu* cpu) {
+[[__percpu]]
+struct arch_percpu arch_data = {0};
+
+void x86_64_cpu_setup(struct percpu* cpu) {
     asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
     asm_wrmsr(MSR_GS_BASE, (uint64_t)cpu);
     asm_wrmsr(MSR_FS_BASE, 0);
 
+    struct arch_percpu* arch_cpu = percpu_ref(arch_data);
+
     gdt_load();
+
+    // TODO: not if FRED
     idt_load();
 
+    asm_wrmsr(
+        MSR_PAT,
+        PAT_WB | PAT_WT << 8 | PAT_UCM << 16 | PAT_UC << 24 | (uint64_t)PAT_WP << 32 | (uint64_t)PAT_WC << 40 |
+            (uint64_t)PAT_UCM << 48 | (uint64_t)PAT_UC << 56
+    );
+
     // Syscall extension.
     asm_wrmsr(MSR_EFER, asm_rdmsr(MSR_EFER) | MSR_EFER_SCE);
     union ia32_star star = {0};
     star.kernel_cs_ss = offsetof(struct gdt, kernel_code64);
-    star.user_cs_ss = offsetof(struct gdt, user_code32);
+    star.user_cs_ss = offsetof(struct gdt, user_code32) | CPL_USER;
     asm_wrmsr(MSR_STAR, star.value);
     asm_wrmsr(MSR_LSTAR, (uint64_t)arch_syscall_stub);
-    asm_wrmsr(MSR_SFMASK, RFLAGS_AC | RFLAGS_DF | RFLAGS_IF);
+    // Mask all RFLAGS except for BRKI which is reserved.
+    asm_wrmsr(MSR_SFMASK, ~RFLAGS_BRKI);
 
     // TODO: FRED
 
@@ -67,7 +84,7 @@ static void setup_cpu(struct percpu* cpu) {
 
     // Enable SSE.
     cr0 &= ~CR0_EM;
-    cr0 |= CR0_MP;
+    cr0 |= CR0_MP | CR0_WP;
     cr4 |= CR4_OSFXSR | CR4_OSXMMEXCPT;
 
     if (cpuid1.ecx & CPUID_1C_XSAVE) {
@@ -85,13 +102,13 @@ static void setup_cpu(struct percpu* cpu) {
 
         asm_wrxcr(0, xcr0);
 
-        cpu->arch.fpu_size = cpuid13.ecx;
+        arch_cpu->fpu_size = cpuid13.ecx;
-        cpu->arch.fpu_save = asm_xsave;
+        arch_cpu->fpu_save = asm_xsave;
-        cpu->arch.fpu_restore = asm_xrstor;
+        arch_cpu->fpu_restore = asm_xrstor;
     } else {
-        cpu->arch.fpu_size = 512;
+        arch_cpu->fpu_size = 512;
-        cpu->arch.fpu_save = asm_fxsave;
+        arch_cpu->fpu_save = asm_fxsave;
-        cpu->arch.fpu_restore = asm_fxrstor;
+        arch_cpu->fpu_restore = asm_fxrstor;
     }
 
     if (cpuid7.ecx & CPUID_7C_UMIP)
@@ -102,25 +119,27 @@ static void setup_cpu(struct percpu* cpu) {
 
     if (cpuid7.ebx & CPUID_7B_SMAP) {
         cr4 |= CR4_SMAP;
-        cpu->arch.can_smap = true;
+        arch_cpu->can_smap = true;
     }
 
     if (cpuid7.ebx & CPUID_7B_FSGSBASE)
         cr4 |= CR4_FSGSBASE;
 
-    asm volatile("mov %0, cr0" : "=r"(cr0));
+    asm volatile("mov cr0, %0" ::"r"(cr0));
-    asm volatile("mov %0, cr4" : "=r"(cr4));
+    asm volatile("mov cr4, %0" ::"r"(cr4));
 
-    lapic_init(&cpu->arch.lapic);
+    lapic_init(&arch_cpu->lapic);
     cpu->online = true;
 }
 
 void arch_percpu_init() {
-    hpet_init();
+#if CONFIG_TSC
     tsc_init();
+#endif
 
     // Init the BSP.
-    setup_cpu(&percpu_bsp);
+    x86_64_cpu_setup(&percpu_bsp);
 
     // Init any APs.
+    x86_64_smp_init();
 }

arch/x86_64/pmap.c

@@ -0,0 +1,95 @@
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/types.h>
+#include <x86_64/apic.h>
+#include <x86_64/defs.h>
+#include <x86_64/percpu.h>
+#include <x86_64/pmap.h>
+
+void pte_clear(pte_t* pte) {
+    *pte = 0;
+}
+
+void pte_write(pte_t* pte, pte_t val) {
+    *pte = val;
+}
+
+void pte_flush(uintptr_t vaddr) {
+    asm volatile("invlpg [%0]" ::"r"(vaddr) : "memory");
+}
+
+void pte_flush_all() {
+    uint64_t cr3;
+    asm volatile("mov %0, cr3" : "=r"(cr3));
+    asm volatile("mov cr3, %0" ::"r"(cr3) : "memory");
+}
+
+void pte_shootdown_kick(struct percpu* cpu) {
+    irq_lock();
+    lapic_send_ipi(
+        &percpu_ref(arch_data)->lapic,
+        percpu_ref_on(cpu, arch_data)->lapic.id,
+        IDT_IPI_SHOOTDOWN,
+        APIC_DELIVERY_FIXED,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    irq_unlock();
+}
+
+pte_t pte_build(phys_t addr, enum pte_flags flags, enum cache_mode cache) {
+    pte_t result = ((pte_t)addr & ARCH_PTE_ADDR_MASK) | ARCH_FLAG_PRESENT;
+
+    if (flags & PTE_USER)
+        result |= ARCH_FLAG_USER_MODE;
+
+    if (flags & PTE_DIR)
+        result |= ARCH_FLAG_READ_WRITE;
+    else {
+        if (flags & PTE_WRITE)
+            result |= ARCH_FLAG_READ_WRITE;
+        if (!(flags & PTE_EXEC))
+            result |= ARCH_FLAG_EXECUTE_DISABLE;
+
+        switch (cache) {
+        case CACHE_ANY:
+        case CACHE_WRITE_BACK:
+            break;
+        case CACHE_WRITE_THROUGH:
+            result |= ARCH_FLAG_WRITE_THROUGH;
+            break;
+        case CACHE_MMIO:
+            result |= ARCH_FLAG_CACHE_DISABLE | ARCH_FLAG_WRITE_THROUGH;
+            break;
+        case CACHE_WRITE_COMBINE:
+            result |= ARCH_FLAG_ATTRIBUTE_TABLE | ARCH_FLAG_WRITE_THROUGH;
+            break;
+        }
+    }
+    return result;
+}
+
+pte_t pte_protect(pte_t old, enum pte_flags flags) {
+    pte_t result = old & ~(ARCH_FLAG_READ_WRITE | ARCH_FLAG_USER_MODE | ARCH_FLAG_EXECUTE_DISABLE);
+    if (flags & PTE_USER)
+        result |= ARCH_FLAG_USER_MODE;
+    if (flags & PTE_WRITE)
+        result |= ARCH_FLAG_READ_WRITE;
+    if (!(flags & PTE_EXEC))
+        result |= ARCH_FLAG_EXECUTE_DISABLE;
+    return result;
+}
+
+bool pte_is_present(pte_t* pte) {
+    return *pte & ARCH_FLAG_PRESENT;
+}
+
+phys_t pte_address(pte_t* pte) {
+    return (phys_t)(*pte & ARCH_PTE_ADDR_MASK);
+}
+
+void pmap_set(struct pmap* pt) {
+    asm volatile("mov cr3, %0" ::"r"(pt->root) : "memory");
+}

arch/x86_64/random.c

@@ -0,0 +1,36 @@
+#include <zinnia/initgraph.h>
+#include <zinnia/print.h>
+#include <zinnia/random.h>
+#include <x86_64/asm.h>
+
+#define CPUID_1C_RDRAND (1u << 30)
+#define RDRAND_RETRIES  10
+
+static bool rdrand_get_u64(struct random_source*, uint64_t* out) {
+    for (int i = 0; i < RDRAND_RETRIES; i++) {
+        uint64_t value;
+        uint8_t ok;
+        asm volatile("rdrand %0\n setc %1" : "=r"(value), "=qm"(ok) : : "cc");
+        if (ok) {
+            *out = value;
+            return true;
+        }
+    }
+    return false;
+}
+
+static struct random_source rdrand_source = {
+    .name = "x86_64.rdrand",
+    .quality = 100,
+    .get_u64 = rdrand_get_u64,
+};
+
+static void rdrand_init(struct boot_info*) {
+    if (asm_cpuid(1, 0).ecx & CPUID_1C_RDRAND) {
+        random_register(&rdrand_source);
+    } else {
+        log_warn("random: This CPU doesn't support RDRAND, ignoring\n");
+    }
+}
+INIT_TASK(rdrand_stage, "arch.x86_64.rdrand", rdrand_init);
+INIT_ENTAILS(rdrand_stage, random_stage);

arch/x86_64/sched.c

@@ -0,0 +1,272 @@
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/compiler.h>
+#include <zinnia/errno.h>
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/sched.h>
+#include <zinnia/vma.h>
+#include <arch/sched.h>
+#include <bits/sched.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <x86_64/apic.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+#include <x86_64/gdt.h>
+#include <x86_64/idt.h>
+#include <x86_64/percpu.h>
+
+// The task frame consists of registers that the C ABI marks as callee-saved.
+// If we don't save them, these registers are lost during a context switch.
+struct task_frame {
+    uint64_t rbx, rbp, r12, r13, r14, r15, rip;
+};
+
+[[__naked]]
+static struct task* perform_switch(uint64_t* old_rsp, uint64_t new_rsp, struct task* prev) {
+    asm volatile(
+        "sub rsp, 0x30\n" // Make room for all regs (except RIP).
+        "mov [rsp + %c[rbx]], rbx\n"
+        "mov [rsp + %c[rbp]], rbp\n"
+        "mov [rsp + %c[r12]], r12\n"
+        "mov [rsp + %c[r13]], r13\n"
+        "mov [rsp + %c[r14]], r14\n"
+        "mov [rsp + %c[r15]], r15\n"
+        "mov [rdi], rsp\n" // rdi = old_rsp
+        "mov rsp, rsi\n"   // rsi = new_rsp
+        "mov rbx, [rsp + %c[rbx]]\n"
+        "mov rbp, [rsp + %c[rbp]]\n"
+        "mov r12, [rsp + %c[r12]]\n"
+        "mov r13, [rsp + %c[r13]]\n"
+        "mov r14, [rsp + %c[r14]]\n"
+        "mov r15, [rsp + %c[r15]]\n"
+        "add rsp, 0x30\n"
+        "mov rax, rdx\n"
+        "ret" // This will conveniently move us to the RIP we put at this stack entry.
+        :
+        : [rbx] "i"(offsetof(struct task_frame, rbx)),
+          [rbp] "i"(offsetof(struct task_frame, rbp)),
+          [r12] "i"(offsetof(struct task_frame, r12)),
+          [r13] "i"(offsetof(struct task_frame, r13)),
+          [r14] "i"(offsetof(struct task_frame, r14)),
+          [r15] "i"(offsetof(struct task_frame, r15))
+        : "memory"
+    );
+}
+
+struct task* arch_sched_switch(struct task* from, struct task* to) {
+    struct arch_percpu* cpu = percpu_ref(arch_data);
+    struct arch_task_context* from_ctx = &from->context;
+    struct arch_task_context* to_ctx = &to->context;
+
+    cpu->tss.rsp0 = kernel_stack_top(&to->kernel_stack);
+
+    if (from_ctx->fpu_region != nullptr) {
+        cpu->fpu_save(from_ctx->fpu_region);
+        from_ctx->ds = asm_read_ds();
+        from_ctx->es = asm_read_es();
+        from_ctx->fs = asm_read_fs();
+        from_ctx->gs = asm_read_gs();
+        from_ctx->fs_base = asm_rdmsr(MSR_FS_BASE);
+        from_ctx->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
+    }
+
+    if (to_ctx->fpu_region != nullptr) {
+        cpu->fpu_restore(to_ctx->fpu_region);
+        asm_write_ds(to_ctx->ds);
+        asm_write_es(to_ctx->es);
+        asm_write_fs(to_ctx->fs);
+
+        // If we have to change the GS segment we need to reload the MSR, otherwise we lose its value.
+        if (to_ctx->gs != asm_read_gs()) {
+            uint64_t percpu = asm_rdmsr(MSR_GS_BASE);
+            asm_write_gs(to_ctx->gs);
+            asm_wrmsr(MSR_GS_BASE, percpu);
+        }
+
+        asm_wrmsr(MSR_FS_BASE, to_ctx->fs_base);
+        asm_wrmsr(MSR_KERNEL_GS_BASE, to_ctx->gs_base);
+    }
+
+    return perform_switch(&from_ctx->rsp, to_ctx->rsp, from);
+}
+
+constexpr size_t FPU_FCW_OFFSET = 0;
+constexpr size_t FPU_MXCSR_OFFSET = 24;
+constexpr uint16_t FPU_DEFAULT_FCW = 0x37F;
+constexpr uint32_t FPU_DEFAULT_MXCSR = 0x1F80;
+
+// Calls task_entry with the correct parameters.
+[[__naked]]
+void task_entry_thunk() {
+    asm volatile(
+        "mov rdi, rbx\n"
+        "mov rsi, r12\n"
+        "mov rdx, rax\n"
+        "jmp task_entry"
+        :
+        :
+        : "memory", "rdi", "rsi", "rdx"
+    );
+}
+
+errno_t arch_task_init(
+    struct arch_task_context* context,
+    void* entry,
+    uintptr_t arg0,
+    uintptr_t stack_start,
+    bool is_user
+) {
+    struct arch_percpu* cpu = percpu_ref(arch_data);
+
+    assert((stack_start & 0xF) == 0, "Task stack is not 16-byte aligned!");
+
+    // Prepare a dummy stack with an entry point function to return to.
+    struct task_frame* frame = (struct task_frame*)(stack_start - sizeof(uint64_t)) - 1;
+    frame->rbx = (uint64_t)entry;
+    frame->r12 = arg0;
+    frame->rip = (uint64_t)task_entry_thunk;
+    context->rsp = (uint64_t)frame;
+
+    if (is_user) {
+        // Allocate FPU space.
+        const size_t fpu_size = cpu->fpu_size;
+        uintptr_t fpu_base = vm_alloc(fpu_size);
+        if (!fpu_base)
+            return ENOMEM;
+        phys_t pages;
+        errno_t s = mem_phys_alloc(ALIGN_UP(fpu_size, arch_mem_page_size()) / arch_mem_page_size(), 0, &pages);
+        if (s) {
+            vm_free(fpu_base, fpu_size);
+            return s;
+        }
+        for (size_t i = 0; i < fpu_size; i += arch_mem_page_size())
+            pmap_map(&kernel_vma.pmap, fpu_base + i, pages + i, PTE_WRITE | PTE_READ, CACHE_WRITE_BACK);
+        context->fpu_region = (void*)fpu_base;
+        context->fpu_phys = pages;
+
+        const uint16_t fcw = FPU_DEFAULT_FCW;
+        const uint32_t mxcsr = FPU_DEFAULT_MXCSR;
+        memcpy((uint8_t*)fpu_base + FPU_FCW_OFFSET, &fcw, sizeof(fcw));
+        memcpy((uint8_t*)fpu_base + FPU_MXCSR_OFFSET, &mxcsr, sizeof(mxcsr));
+
+        context->ds = asm_read_ds();
+        context->es = asm_read_es();
+        context->fs = asm_read_fs();
+        context->gs = asm_read_gs();
+        context->fs_base = asm_rdmsr(MSR_FS_BASE);
+        context->gs_base = asm_rdmsr(MSR_KERNEL_GS_BASE);
+    }
+
+    return 0;
+}
+
+void arch_task_fini(struct arch_task_context* context) {
+    if (context->fpu_region == nullptr)
+        return;
+
+    const size_t page_size = arch_mem_page_size();
+    const size_t fpu_size = percpu_ref(arch_data)->fpu_size;
+    const uintptr_t fpu_base = (uintptr_t)context->fpu_region;
+    for (size_t i = 0; i < fpu_size; i += page_size)
+        pmap_unmap(&kernel_vma.pmap, fpu_base + i);
+    pmap_shootdown(&kernel_vma.pmap);
+    mem_phys_free(context->fpu_phys, ALIGN_UP(fpu_size, page_size) / page_size);
+    vm_free(fpu_base, fpu_size);
+
+    context->fpu_region = nullptr;
+}
+
+void arch_task_fork(struct arch_task_context* dst) {
+    if (dst->fpu_region)
+        percpu_ref(arch_data)->fpu_save(dst->fpu_region);
+}
+
+void arch_context_init_user(struct arch_context* ctx, uintptr_t ip, uintptr_t sp) {
+    *ctx = (struct arch_context){
+        .rip = ip,
+        .rsp = sp,
+        .rflags = 0x202,
+        .cs = offsetof(struct gdt, user_code64) | CPL_USER,
+        .ss = offsetof(struct gdt, user_data) | CPL_USER,
+    };
+}
+
+[[__naked]]
+void arch_sched_preempt_disable() {
+    asm volatile(
+        "inc qword ptr gs:%c0\n"
+        "ret"
+        :
+        : "i"(offsetof(struct percpu, sched.preempt_level))
+    );
+}
+
+[[__naked]]
+bool arch_sched_preempt_enable() {
+    asm volatile(
+        "dec qword ptr gs:%c0\n"
+        "setz al\n"
+        "movzx eax, al\n"
+        "ret\n"
+        :
+        : "i"(offsetof(struct percpu, sched.preempt_level))
+        : "cc"
+    );
+}
+
+void arch_sched_kick(struct percpu* cpu) {
+    if (!cpu->online)
+        return;
+    // The xAPIC ICR takes two writes, which must not be interleaved with an IPI sent by an interrupt handler.
+    irq_lock();
+    lapic_send_ipi(
+        &percpu_ref(arch_data)->lapic,
+        percpu_ref_on(cpu, arch_data)->lapic.id,
+        IDT_IPI_RESCHED,
+        APIC_DELIVERY_FIXED,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    irq_unlock();
+}
+
+[[__naked]]
+void arch_sched_jump_to_context(struct arch_context* context) {
+    asm volatile(
+        "mov rsp, rdi\n"
+        "jmp %0\n" ::"r"(interrupt_return)
+    );
+}
+
+void arch_sched_jump_to_user(uintptr_t ip, uintptr_t sp, uintptr_t arg) {
+    struct arch_context context = {
+        .rip = ip,
+        .rsp = sp,
+        .rdi = arg,
+        .rflags = 0x202,
+        .cs = offsetof(struct gdt, user_code64) | CPL_USER,
+        .ss = offsetof(struct gdt, user_data) | CPL_USER,
+    };
+
+    uintptr_t percpu = asm_rdmsr(MSR_GS_BASE);
+    asm volatile(
+        "mov ds, %0\n"
+        "mov es, %0\n"
+        "mov fs, %0\n"
+        "mov gs, %0\n"
+        :
+        : "a"(0)
+        : "memory"
+    );
+
+    asm_wrmsr(MSR_FS_BASE, 0);
+    asm_wrmsr(MSR_GS_BASE, percpu);
+    asm_wrmsr(MSR_KERNEL_GS_BASE, 0);
+
+    arch_sched_jump_to_context(&context);
+}

arch/x86_64/serial.c

@@ -1,4 +1,9 @@
-#include <kernel/console.h>
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/console.h>
+#include <zinnia/init.h>
+#include <zinnia/initgraph.h>
+#include <arch/cpu.h>
 #include <stdint.h>
 #include <x86_64/asm.h>
 
@@ -11,7 +16,7 @@ static bool is_ready() {
 static void serial_write(struct console* con, const char* buf, size_t count) {
     for (size_t i = 0; i < count; i++) {
         while (!is_ready())
-            asm volatile("pause");
+            arch_cpu_pause();
 
         asm_outb(COM1_BASE, buf[i]);
     }
@@ -44,4 +49,10 @@ static struct console serial_con = {
     .write = serial_write,
     .init = serial_setup,
 };
-DEFINE_CONSOLE(&serial_con);
+
+[[__init]]
+static void serial_init(struct boot_info*) {
+    console_register(&serial_con);
+}
+INIT_TASK(serial_stage, "arch.x86_64.serial", serial_init);
+INIT_ENTAILS(serial_stage, console_stage);

arch/x86_64/smp.c

@@ -0,0 +1,359 @@
+#define pr_fmt(fmt) "smp: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/atomic.h>
+#include <zinnia/clock.h>
+#include <zinnia/compiler.h>
+#include <zinnia/irq.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/types.h>
+#include <zinnia/utils.h>
+#include <zinnia/vma.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <uacpi/acpi.h>
+#include <uacpi/tables.h>
+#include <x86_64/apic.h>
+#include <x86_64/defs.h>
+#include <x86_64/percpu.h>
+#include <x86_64/pmap.h>
+#include <x86_64/smp.h>
+
+extern uint8_t smp_trampoline_start[];
+extern uint8_t smp_trampoline_data[];
+extern uint8_t smp_trampoline_end[];
+
+[[__naked, __used]]
+static void smp_trampoline() {
+    asm volatile(
+        ".pushsection .rodata\n"
+        ".code16\n"
+        ".global smp_trampoline_start\n"
+        "smp_trampoline_start:\n"
+        "    cli\n"
+        "    cld\n"
+        "    jmp .Lmode16\n"
+
+        ".org smp_trampoline_start + %c[data_off]\n"
+        ".global smp_trampoline_data\n"
+        "smp_trampoline_data:\n"
+        "    .skip %c[info_size]\n"
+
+        ".set .Ldata_offset, %c[data_off]\n"
+        ".set .Lgdtr_offset, (.Ldata_offset + %c[gdtr])\n"
+        ".set .Lfarjmp_offset, (.Ldata_offset + %c[farjmp])\n"
+        ".set .Ltemp_stack_offset, (.Ldata_offset + %c[temp_stack])\n"
+        ".set .Ltemp_cr3_offset, (.Ldata_offset + %c[temp_cr3])\n"
+        ".set .Lentry_offset, (.Ldata_offset + %c[entry])\n"
+        ".set .Lhhdm_offset, (.Ldata_offset + %c[hhdm])\n"
+        ".set .Linvalid_idtr_offset, (.Linvalid_idtr - smp_trampoline_start)\n"
+        ".set .Lmode32_offset, (.Lmode32 - smp_trampoline_start)\n"
+        ".set .Lmode64_offset, (.Lmode64 - smp_trampoline_start)\n"
+
+        ".Lmode16:\n"
+        "    mov bx, cs\n"
+        "    shl ebx, 4\n"
+
+        "    lidtd cs:.Linvalid_idtr_offset\n"
+        "    lgdtd cs:.Lgdtr_offset\n"
+
+        "    lea eax, [ebx + .Lmode32_offset]\n"
+        "    mov dword ptr cs:.Lfarjmp_offset, eax\n"
+
+        "    mov eax, %c[cr0]\n"
+        "    mov cr0, eax\n"
+        "    jmp fword ptr cs:.Lfarjmp_offset\n"
+
+        ".code32\n"
+        ".Lmode32:\n"
+        "    mov ax, %c[kernel32_ds]\n"
+        "    mov ds, ax\n"
+        "    mov es, ax\n"
+        "    mov fs, ax\n"
+        "    mov gs, ax\n"
+        "    mov ss, ax\n"
+
+        "    xor eax, eax\n"
+        "    lldt ax\n"
+
+        "    mov esp, dword ptr [ebx + .Ltemp_stack_offset]\n"
+
+        "    xor eax, eax\n"
+        "    mov cr4, eax\n"
+
+        "    or eax, %c[cr4_pae]\n"
+        "    mov cr4, eax\n"
+
+        "    mov ecx, %c[msr_efer]\n"
+        "    mov eax, %c[efer_lme]\n"
+        "    xor edx, edx\n"
+        "    wrmsr\n"
+
+        "    mov eax, dword ptr [ebx + .Ltemp_cr3_offset]\n"
+        "    mov cr3, eax\n"
+
+        "    mov eax, cr0\n"
+        "    or eax, %c[cr0_pg]\n"
+        "    mov cr0, eax\n"
+
+        "    lea eax, [ebx + .Lmode64_offset]\n"
+        "    push %c[kernel64_cs]\n"
+        "    push eax\n"
+        "    retf\n"
+
+        ".code64\n"
+        ".Lmode64:\n"
+        "    mov ax, %c[kernel64_ds]\n"
+        "    mov ds, ax\n"
+        "    mov es, ax\n"
+        "    mov fs, ax\n"
+        "    mov gs, ax\n"
+        "    mov ss, ax\n"
+
+        "    mov ecx, %c[msr_efer]\n"
+        "    rdmsr\n"
+        "    or eax, %c[efer_nxe]\n"
+        "    wrmsr\n"
+
+        "    xor ebp, ebp\n"
+
+        "    lea rdi, [rbx + .Ldata_offset]\n"
+        "    mov rax, [rbx + .Lentry_offset]\n"
+        "    add rsp, [rbx + .Lhhdm_offset]\n"
+        "    jmp rax\n"
+
+        ".Linvalid_idtr:\n"
+        "    .word 0\n"
+        "    .quad 0\n"
+
+        ".global smp_trampoline_end\n"
+        "smp_trampoline_end:\n"
+        ".popsection\n"
+        "    ud2\n"
+        :
+        : [data_off] "i"(8),
+          [info_size] "i"(sizeof(struct smp_info)),
+          [gdtr] "i"(offsetof(struct smp_info, gdtr)),
+          [farjmp] "i"(offsetof(struct smp_info, farjmp_offset)),
+          [temp_stack] "i"(offsetof(struct smp_info, temp_stack)),
+          [temp_cr3] "i"(offsetof(struct smp_info, temp_cr3)),
+          [entry] "i"(offsetof(struct smp_info, entry)),
+          [hhdm] "i"(offsetof(struct smp_info, hhdm_offset)),
+          [kernel32_ds] "i"(offsetof(struct gdt, kernel_data32)),
+          [kernel64_cs] "i"(offsetof(struct gdt, kernel_code64)),
+          [kernel64_ds] "i"(offsetof(struct gdt, kernel_data64)),
+          [cr0] "i"(CR0_PE | CR0_ET),
+          [cr0_pg] "i"((uint32_t)CR0_PG),
+          [cr4_pae] "i"(CR4_PAE),
+          [msr_efer] "i"(MSR_EFER),
+          [efer_lme] "i"(MSR_EFER_LME),
+          [efer_nxe] "i"(MSR_EFER_NXE)
+    );
+}
+
+constexpr size_t AP_BOOT_STACK_SIZE = 0x8000;
+constexpr uint64_t AP_BOOT_TIMEOUT_NS = 1'000'000'000;
+
+[[noreturn]]
+static void ap_entry(phys_t info_addr) {
+    // We're still running on the temporary page table.
+    pmap_set(&kernel_vma.pmap);
+
+    struct smp_info* info = HHDM_PTR(info_addr);
+    struct percpu* cpu = (struct percpu*)info->percpu;
+
+    x86_64_cpu_setup(cpu);
+    irq_lock();
+
+    // This task keeps running on the boot stack, but is never scheduled again after the first yield.
+    sched_init(&cpu->sched);
+
+    // Let the BSP know that we're alive.
+    // `info` must not be touched after this point.
+    atomic_store(&info->booted, 1, ATOMIC_RELEASE);
+
+    log("CPU %zu online\n", cpu->id);
+
+    sched_yield(&cpu->sched);
+    unreachable();
+}
+
+static bool start_ap(phys_t trampoline, uint32_t temp_cr3, uint32_t lapic_id) {
+    const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
+
+    phys_t stack;
+    if (mem_phys_alloc(AP_BOOT_STACK_SIZE / arch_mem_page_size(), ALLOC_MEM32 | ALLOC_NOZERO, &stack)) {
+        log_err("Failed to allocate a boot stack for LAPIC %u\n", lapic_id);
+        return false;
+    }
+
+    struct percpu* cpu = percpu_new();
+    if (!cpu) {
+        log_err("Failed to allocate per-CPU data for LAPIC %u\n", lapic_id);
+        mem_phys_free(stack, AP_BOOT_STACK_SIZE / arch_mem_page_size());
+        return false;
+    }
+
+    log("Starting CPU %zu\n", cpu->id);
+
+    memcpy(HHDM_PTR(trampoline), smp_trampoline_start, trampoline_size);
+
+    const phys_t info_addr = trampoline + (smp_trampoline_data - smp_trampoline_start);
+    struct smp_info* info = HHDM_PTR(info_addr);
+    memcpy(&info->gdt, &gdt_base, sizeof(info->gdt));
+    info->gdtr.limit = sizeof(struct gdt) - 1;
+    info->gdtr.base = (struct gdt*)(info_addr + offsetof(struct smp_info, gdt));
+    info->farjmp_offset = 0; // Filled in by the trampoline.
+    info->farjmp_segment = offsetof(struct gdt, kernel_code32);
+    info->temp_stack = stack + AP_BOOT_STACK_SIZE;
+    info->temp_cr3 = temp_cr3;
+    info->entry = (uint64_t)ap_entry;
+    info->hhdm_offset = mem_hhdm_base;
+    info->percpu = (uint64_t)cpu;
+    info->lapic_id = lapic_id;
+    atomic_init(&info->booted, 0);
+
+    struct local_apic* lapic = &percpu_ref(arch_data)->lapic;
+
+    lapic_send_ipi(lapic, lapic_id, 0, APIC_DELIVERY_INIT, APIC_DEST_PHYSICAL, APIC_LEVEL_ASSERT, APIC_TRIGGER_EDGE);
+    clock_spin_ns(10'000'000);
+
+    lapic_send_ipi(
+        lapic,
+        lapic_id,
+        trampoline >> 12,
+        APIC_DELIVERY_STARTUP,
+        APIC_DEST_PHYSICAL,
+        APIC_LEVEL_ASSERT,
+        APIC_TRIGGER_EDGE
+    );
+    clock_spin_ns(10'000'000);
+
+    for (uint64_t waited = 0; !atomic_load(&info->booted, ATOMIC_ACQUIRE); waited += 1'000'000) {
+        if (waited >= AP_BOOT_TIMEOUT_NS) {
+            // The per-CPU data and boot stack are leaked on purpose, in case the AP still comes up.
+            log_err("CPU %zu (LAPIC %u) did not respond\n", cpu->id, lapic_id);
+            return false;
+        }
+        clock_spin_ns(1'000'000);
+    }
+
+    return true;
+}
+
+static errno_t build_temp_table(phys_t* out) {
+    const size_t pgsz = arch_mem_page_size();
+
+    phys_t mem;
+    errno_t status = mem_phys_alloc(6, ALLOC_MEM32, &mem);
+    if (status)
+        return status;
+
+    uint64_t* l4 = HHDM_PTR(mem);
+    uint64_t* l3 = HHDM_PTR(mem + pgsz);
+    const phys_t l2_base = mem + 2 * pgsz;
+
+    // Use 2 MiB pages.
+    l4[0] = (mem + pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+    for (size_t i = 0; i < 4; i++) {
+        uint64_t* l2 = HHDM_PTR(l2_base + i * pgsz);
+        l3[i] = (l2_base + i * pgsz) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+        for (size_t j = 0; j < 512; j++)
+            l2[j] = ((i << 30) + (j << 21)) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE | ARCH_FLAG_SIZE;
+    }
+
+    // Copy over the higher half from the kernel page table.
+    const uint64_t* kernel_l4 = HHDM_PTR(kernel_vma.pmap.root);
+    for (size_t i = 256; i < 512; i++)
+        l4[i] = kernel_l4[i];
+
+    *out = mem;
+    return 0;
+}
+
+void x86_64_smp_init() {
+    const size_t trampoline_size = smp_trampoline_end - smp_trampoline_start;
+    assert(trampoline_size <= arch_mem_page_size(), "SMP trampoline doesn't fit in a page");
+
+    const size_t max_cpus = percpu_max_cpus();
+    if (max_cpus <= 1)
+        return;
+
+    uacpi_table table;
+    if (uacpi_table_find_by_signature(ACPI_MADT_SIGNATURE, &table) != UACPI_STATUS_OK) {
+        log_warn("No MADT found, not starting any APs\n");
+        return;
+    }
+
+    phys_t trampoline;
+    if (mem_phys_alloc(1, ALLOC_MEM20, &trampoline)) {
+        log_err("Failed to allocate 20-bit memory for the AP trampoline\n");
+        uacpi_table_unref(&table);
+        return;
+    }
+
+    phys_t temp_table;
+    if (build_temp_table(&temp_table)) {
+        log_err("Failed to allocate a temporary page table\n");
+        mem_phys_free(trampoline, 1);
+        uacpi_table_unref(&table);
+        return;
+    }
+
+    const uint32_t bsp_id = percpu_ref(arch_data)->lapic.id;
+    bool all_booted = true;
+    size_t num_cpus = 1;
+
+    const struct acpi_madt* madt = table.ptr;
+    const uint8_t* entry = (const uint8_t*)madt->entries;
+    const uint8_t* end = (const uint8_t*)madt + madt->hdr.length;
+
+    while (entry + sizeof(struct acpi_entry_hdr) <= end && num_cpus < max_cpus) {
+        const struct acpi_entry_hdr* hdr = (const struct acpi_entry_hdr*)entry;
+
+        // Don't trust malformed firmware tables.
+        if (hdr->length < sizeof(struct acpi_entry_hdr) || entry + hdr->length > end) {
+            log_warn("MADT contains a malformed entry\n");
+            break;
+        }
+
+        uint32_t lapic_id = 0;
+        bool enabled = false;
+        switch (hdr->type) {
+        case ACPI_MADT_ENTRY_TYPE_LAPIC:
+            const struct acpi_madt_lapic* lapic = (const struct acpi_madt_lapic*)hdr;
+            lapic_id = lapic->id;
+            enabled = lapic->flags & ACPI_PIC_ENABLED;
+            break;
+        case ACPI_MADT_ENTRY_TYPE_LOCAL_X2APIC:
+            const struct acpi_madt_x2apic* x2apic = (const struct acpi_madt_x2apic*)hdr;
+            lapic_id = x2apic->id;
+            enabled = x2apic->flags & ACPI_PIC_ENABLED;
+            break;
+        }
+
+        if (enabled && lapic_id != bsp_id) {
+            if (start_ap(trampoline, temp_table, lapic_id))
+                num_cpus++;
+            else
+                all_booted = false;
+        }
+
+        entry += hdr->length;
+    }
+
+    uacpi_table_unref(&table);
+
+    // An AP which didn't respond might still be using these.
+    if (all_booted) {
+        mem_phys_free(temp_table, 6);
+        mem_phys_free(trampoline, 1);
+    }
+
+    log("%zu CPU(s) online\n", num_cpus);
+}

arch/x86_64/start.c

@@ -0,0 +1,19 @@
+#include <zinnia/compiler.h>
+#include <zinnia/errno.h>
+#include <zinnia/init.h>
+#include <zinnia/percpu.h>
+#include <uapi/archctl.h>
+#include <x86_64/apic.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+#include <x86_64/gdt.h>
+
+[[__init, __naked, __used]]
+void _start() {
+    asm volatile(
+        "mov rsp, %0\n"
+        "jmp %1"
+        :
+        : "r"(__ld_stack_top), "r"(kernel_entry)
+    );
+}

arch/x86_64/syscall.c

@@ -0,0 +1,73 @@
+#include <zinnia/compiler.h>
+#include <zinnia/percpu.h>
+#include <zinnia/syscall.h>
+#include <bits/sched.h>
+#include <stddef.h>
+#include <x86_64/defs.h>
+#include <x86_64/gdt.h>
+
+// TODO: Refactor to only push callee-saved registers.
+
+[[__naked]]
+void arch_syscall_stub() {
+    asm volatile(
+        "    swapgs\n"
+        "    mov gs:%c[user_stack], rsp\n"
+        "    mov rsp, gs:%c[kernel_stack]\n"
+        "    cld\n"
+        "1:\n"
+        "    push %c[user_data]\n" // ss
+        "    push gs:%c[user_stack]\n"
+        "    push r11\n"             // rflags
+        "    push %c[user_code64]\n" // cs
+        "    push rcx\n"             // rip
+        "    push 0x00\n"            // arch_context::error
+        "    push 0x00\n"            // arch_context::isr
+        "    push rax\n"
+        "    push rbx\n"
+        "    push rcx\n"
+        "    push rdx\n"
+        "    push rbp\n"
+        "    push rdi\n"
+        "    push rsi\n"
+        "    push r8\n"
+        "    push r9\n"
+        "    push r10\n"
+        "    push r11\n"
+        "    push r12\n"
+        "    push r13\n"
+        "    push r14\n"
+        "    push r15\n"
+        "    xor rbp, rbp\n"
+        "    mov rdi, rsp\n"          // SysV ABI first argument.
+        "    call syscall_dispatch\n" // Call syscall handler.
+        "    cli\n"
+        "    pop r15\n"
+        "    pop r14\n"
+        "    pop r13\n"
+        "    pop r12\n"
+        "    pop r11\n"
+        "    pop r10\n"
+        "    pop r9\n"
+        "    pop r8\n"
+        "    pop rsi\n"
+        "    pop rdi\n"
+        "    pop rbp\n"
+        "    pop rdx\n"
+        "    pop rcx\n"
+        "    pop rbx\n"
+        "    pop rax\n"
+        "    add rsp, 0x10\n" // Skip .error and .isr fields
+        "    pop rcx\n"       // rip
+        "    add rsp, 0x08\n" // Skip cs
+        "    pop r11\n"       // rflags
+        "    pop rsp\n"       // rsp
+        "    swapgs\n"
+        "    sysretq\n"
+        :
+        : [user_stack] "i"(offsetof(struct percpu, user_stack)),
+          [kernel_stack] "i"(offsetof(struct percpu, kernel_stack)),
+          [user_code64] "i"(offsetof(struct gdt, user_code64) | CPL_USER),
+          [user_data] "i"(offsetof(struct gdt, user_data) | CPL_USER)
+    );
+}

arch/x86_64/tsc.c

@@ -0,0 +1,88 @@
+#define pr_fmt(fmt) "x86_64/tsc: " fmt
+#include <zinnia/clock.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <stdint.h>
+#include <x86_64/asm.h>
+#include <x86_64/defs.h>
+
+static bool use_tsc = true;
+static void tsc_option(bool opt) {
+    use_tsc = opt;
+}
+CMDLINE_OPTION("x86_64.tsc", "Enable timestamp counter", tsc_option);
+
+constexpr uint64_t nanos_per_sec = 1'000'000'000;
+static uint64_t tsc_freq = 0;
+static uint64_t tsc_base_cycles = 0;
+
+static uint64_t cycles_to_ns(uint64_t cycles, uint64_t freq) {
+    uint64_t seconds = cycles / freq;
+    uint64_t remaining_cycles = cycles % freq;
+    uint64_t fractional_ns = (uint64_t)((__uint128_t)remaining_cycles * (__uint128_t)nanos_per_sec / (__uint128_t)freq);
+    uint64_t ns = seconds * nanos_per_sec + fractional_ns;
+
+    return MIN(ns, UINT64_MAX);
+}
+
+static void tsc_reset(struct clock* clock) {
+    tsc_base_cycles = asm_rdtsc();
+}
+
+static uint64_t tsc_get_elapsed_ns(struct clock* clock) {
+    uint64_t cycles = asm_rdtsc() - tsc_base_cycles;
+    return cycles_to_ns(cycles, tsc_freq);
+}
+
+static struct clock tsc_clock = {
+    .name = "tsc",
+    .priority = 255,
+    .reset = tsc_reset,
+    .get_elapsed_ns = tsc_get_elapsed_ns,
+};
+
+void tsc_init() {
+    if (!use_tsc) {
+        log_warn("TSC disabled on the command line");
+        return;
+    }
+
+    if (!(asm_cpuid(1, 0).edx & CPUID_1D_TSC) || !(asm_cpuid(0x8000'0007, 0).edx & (1 << 8))) {
+        log_warn("No invariant TSC detected!\n");
+        return;
+    }
+
+    // Check for the TSC info leaf.
+    bool has_tsc_leaf = false;
+    struct cpuid tsc_leaf = {0};
+    if (asm_cpuid(0, 0).eax >= 0x15) {
+        tsc_leaf = asm_cpuid(0x15, 0);
+        has_tsc_leaf = true;
+    }
+
+    if (clock_available()) {
+        // Calibrate using existing clock.
+        uint64_t t1 = asm_rdtsc();
+        clock_spin_ns(10'000'000);
+        uint64_t t2 = asm_rdtsc();
+        tsc_freq = (t2 - t1) * 100;
+    } else if (has_tsc_leaf) {
+        // If we have no timer (yet), the only way we can calibrate the TSC is if CPUID gives us the frequency.
+        // On a normal system, this should usually never be called and is a last resort
+        // since at this point we have at least the HPET timer.
+        if (tsc_leaf.ecx != 0 && tsc_leaf.ebx != 0 && tsc_leaf.eax != 0) {
+            log("Calibrating TSC using CPUID 0x15");
+            tsc_freq = (uint64_t)tsc_leaf.ecx * tsc_leaf.ebx / tsc_leaf.eax;
+        } else {
+            log_err("Unable to calibrate TSC using CPUID frequency information");
+            return;
+        }
+    } else {
+        log_err("Unable to calibrate the TSC without a clock or static frequency info!\n");
+        return;
+    }
+
+    log("Timer frequency is %lu MHz\n", tsc_freq / 1'000'000);
+    clock_switch(&tsc_clock);
+}

arch/x86_64/usercopy.c

@@ -0,0 +1,142 @@
+#include <zinnia/compiler.h>
+#include <zinnia/percpu.h>
+#include <arch/usercopy.h>
+#include <x86_64/percpu.h>
+
+void arch_usercopy_allow(bool state) {
+    if (!percpu_ref(arch_data)->can_smap)
+        return;
+
+    if (state) {
+        asm volatile("stac" ::: "memory");
+    } else {
+        asm volatile("clac" ::: "memory");
+    }
+}
+
+extern void x86_64_read_start();
+extern void x86_64_read_end();
+extern void x86_64_read_fault();
+
+static const struct usercopy_region read_region = {
+    .start_ip = x86_64_read_start,
+    .end_ip = x86_64_read_end,
+    .fault_ip = x86_64_read_fault,
+};
+
+[[__naked]]
+bool arch_usercopy_read(void* dst, const __user void* src, size_t len, struct usercopy_region** region) {
+    asm volatile(
+        // Setup regs for `rep movsb`.
+        "    xchg rcx, rdx\n"
+        "    lea rax, [rip + %0]\n"
+        "    xchg [rdx], rax\n"
+
+        ".global x86_64_read_start\n"
+        "    x86_64_read_start:\n"
+        "    rep movsb\n"
+
+        ".global x86_64_read_end\n"
+        "x86_64_read_end:\n"
+        "    xor rax, rax\n"
+        "    xchg [rdx], rax\n"
+        "    mov rax, 1\n"
+        "    ret\n"
+
+        ".global x86_64_read_fault\n"
+        "x86_64_read_fault:\n"
+        "    xor rax, rax\n"
+        "    mov [rdx], rax\n"
+        "    ret\n"
+        :
+        : "i"(&read_region)
+        : "memory"
+    );
+}
+
+extern void x86_64_write_start();
+extern void x86_64_write_end();
+extern void x86_64_write_fault();
+
+static const struct usercopy_region write_region = {
+    .start_ip = x86_64_write_start,
+    .end_ip = x86_64_write_end,
+    .fault_ip = x86_64_write_fault,
+};
+
+[[__naked]]
+bool arch_usercopy_write(__user void* dst, const void* src, size_t len, struct usercopy_region** region) {
+    asm volatile(
+        // Setup regs for `rep movsb`.
+        "    xchg rcx, rdx\n"
+        "    lea rax, [rip + %0]\n"
+        "    mov [rdx], rax\n"
+
+        ".global x86_64_write_start\n"
+        "    x86_64_write_start:\n"
+        "    rep movsb\n"
+
+        ".global x86_64_write_end\n"
+        "    x86_64_write_end:\n"
+        "    xor rax, rax\n"
+        "    xchg [rdx], rax\n"
+        "    mov rax, 1\n"
+        "    ret\n"
+
+        ".global x86_64_write_fault\n"
+        "x86_64_write_fault:\n"
+        "    xor rax, rax\n"
+        "    mov [rdx], rax\n"
+        "    ret\n"
+        :
+        : "i"(&write_region)
+        : "memory"
+    );
+}
+
+extern void x86_64_strlen_start();
+extern void x86_64_strlen_end();
+extern void x86_64_strlen_fault();
+
+static const struct usercopy_region strlen_region = {
+    .start_ip = x86_64_strlen_start,
+    .end_ip = x86_64_strlen_end,
+    .fault_ip = x86_64_strlen_fault,
+};
+
+[[__naked]]
+bool arch_usercopy_strlen(const __user char* str, size_t max, size_t* len, struct usercopy_region** region) {
+    asm volatile(
+        "    lea rax, [rip + %0]\n"
+        "    xchg [rcx], rax\n"
+
+        ".global x86_64_strlen_start\n"
+        "    x86_64_strlen_start:\n"
+        "    xor r8, r8\n"
+        ".Lloop:\n"
+        "    cmp r8, rsi\n"
+        "    je .Lleave\n"
+        "    cmp byte ptr [rdi + r8], 0\n"
+        "    je .Lleave\n"
+        "    inc r8\n"
+        "    jmp .Lloop\n"
+        ".Lleave:\n"
+        "    mov [rdx], r8\n"
+
+        ".global x86_64_strlen_end\n"
+        "x86_64_strlen_end:\n"
+        "    xor rax, rax\n"
+        "    xchg [rcx], rax\n"
+        "    mov rax, 1\n"
+        "    ret\n"
+
+        ".global x86_64_strlen_fault\n"
+        "x86_64_strlen_fault:\n"
+        "    xor rax, rax\n"
+        "    mov [rcx], rax\n"
+        "    ret\n"
+        :
+        : "i"(&strlen_region)
+        : "memory"
+    );
+}

boot/Kconfig

@@ -0,0 +1,19 @@
+config HAVE_EFISTUB
+  bool
+
+menu "Boot loaders"
+
+config BOOT_EFISTUB
+  bool "EFI stub"
+  depends on HAVE_EFISTUB
+  default y
+  help
+    Kernel loader for UEFI systems.
+
+config BOOT_LIMINE
+  bool "Limine"
+  default n
+  help
+    ELF loader with the Limine protocol.
+
+endmenu

boot/efi/efi.h

@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "efi_arch.h"
+
+typedef uint64_t efi_status;
+typedef void* efi_handle;
+typedef uint16_t char16;
+
+constexpr efi_status EFI_SUCCESS = 0;
+constexpr efi_status EFI_ERROR = 1ull << 63;
+constexpr efi_status EFI_LOAD_ERROR = EFI_ERROR | 1;
+constexpr efi_status EFI_INVALID_PARAMETER = EFI_ERROR | 2;
+constexpr efi_status EFI_UNSUPPORTED = EFI_ERROR | 3;
+constexpr efi_status EFI_BUFFER_TOO_SMALL = EFI_ERROR | 5;
+constexpr efi_status EFI_OUT_OF_RESOURCES = EFI_ERROR | 9;
+constexpr efi_status EFI_NOT_FOUND = EFI_ERROR | 14;
+
+struct efi_guid {
+    uint32_t data1;
+    uint16_t data2;
+    uint16_t data3;
+    uint8_t data4[8];
+};
+
+enum efi_memory_type : uint32_t {
+    EFI_RESERVED_MEMORY_TYPE,
+    EFI_LOADER_CODE,
+    EFI_LOADER_DATA,
+    EFI_BOOT_SERVICES_CODE,
+    EFI_BOOT_SERVICES_DATA,
+    EFI_RUNTIME_SERVICES_CODE,
+    EFI_RUNTIME_SERVICES_DATA,
+    EFI_CONVENTIONAL_MEMORY,
+    EFI_UNUSABLE_MEMORY,
+    EFI_ACPI_RECLAIM_MEMORY,
+    EFI_ACPI_MEMORY_NVS,
+    EFI_MEMORY_MAPPED_IO,
+    EFI_MEMORY_MAPPED_IO_PORT_SPACE,
+    EFI_PAL_CODE,
+    EFI_PERSISTENT_MEMORY,
+    EFI_UNACCEPTED_MEMORY,
+};
+
+enum efi_allocate_type : uint32_t {
+    EFI_ALLOCATE_ANY_PAGES,
+    EFI_ALLOCATE_MAX_ADDRESS,
+    EFI_ALLOCATE_ADDRESS,
+};
+
+enum efi_locate_search_type : uint32_t {
+    EFI_LOCATE_ALL_HANDLES,
+    EFI_LOCATE_BY_REGISTER_NOTIFY,
+    EFI_LOCATE_BY_PROTOCOL,
+};
+
+struct efi_memory_descriptor {
+    uint32_t type;
+    uint64_t physical_start;
+    uint64_t virtual_start;
+    uint64_t number_of_pages;
+    uint64_t attribute;
+};
+
+struct efi_table_header {
+    uint64_t signature;
+    uint32_t revision;
+    uint32_t header_size;
+    uint32_t crc32;
+    uint32_t reserved;
+};
+
+struct efi_simple_text_output_protocol {
+    void* reset;
+    efi_status(EFIAPI* output_string)(struct efi_simple_text_output_protocol* self, const char16* string);
+};
+
+struct efi_boot_services {
+    struct efi_table_header hdr;
+    void* raise_tpl;
+    void* restore_tpl;
+    efi_status(EFIAPI* allocate_pages)(
+        enum efi_allocate_type type,
+        enum efi_memory_type memory_type,
+        size_t pages,
+        uint64_t* memory
+    );
+    efi_status(EFIAPI* free_pages)(uint64_t memory, size_t pages);
+    efi_status(EFIAPI* get_memory_map)(
+        size_t* map_size,
+        struct efi_memory_descriptor* map,
+        size_t* map_key,
+        size_t* descriptor_size,
+        uint32_t* descriptor_version
+    );
+    efi_status(EFIAPI* allocate_pool)(enum efi_memory_type pool_type, size_t size, void** buffer);
+    efi_status(EFIAPI* free_pool)(void* buffer);
+    void* create_event;
+    void* set_timer;
+    void* wait_for_event;
+    void* signal_event;
+    void* close_event;
+    void* check_event;
+    void* install_protocol_interface;
+    void* reinstall_protocol_interface;
+    void* uninstall_protocol_interface;
+    efi_status(EFIAPI* handle_protocol)(efi_handle handle, const struct efi_guid* protocol, void** interface);
+    void* reserved;
+    void* register_protocol_notify;
+    void* locate_handle;
+    void* locate_device_path;
+    void* install_configuration_table;
+    void* load_image;
+    void* start_image;
+    void* exit;
+    void* unload_image;
+    efi_status(EFIAPI* exit_boot_services)(efi_handle image_handle, size_t map_key);
+    void* get_next_monotonic_count;
+    void* stall;
+    void* set_watchdog_timer;
+    void* connect_controller;
+    void* disconnect_controller;
+    void* open_protocol;
+    void* close_protocol;
+    void* open_protocol_information;
+    void* protocols_per_handle;
+    efi_status(EFIAPI* locate_handle_buffer)(
+        enum efi_locate_search_type search_type,
+        const struct efi_guid* protocol,
+        void* search_key,
+        size_t* no_handles,
+        efi_handle** buffer
+    );
+    efi_status(EFIAPI* locate_protocol)(const struct efi_guid* protocol, void* registration, void** interface);
+};
+
+struct efi_configuration_table {
+    struct efi_guid vendor_guid;
+    void* vendor_table;
+};
+
+struct efi_system_table {
+    struct efi_table_header hdr;
+    char16* firmware_vendor;
+    uint32_t firmware_revision;
+    efi_handle console_in_handle;
+    void* con_in;
+    efi_handle console_out_handle;
+    struct efi_simple_text_output_protocol* con_out;
+    efi_handle standard_error_handle;
+    void* std_err;
+    void* runtime_services;
+    struct efi_boot_services* boot_services;
+    size_t number_of_table_entries;
+    struct efi_configuration_table* configuration_table;
+};
+
+struct efi_loaded_image_protocol {
+    uint32_t revision;
+    efi_handle parent_handle;
+    struct efi_system_table* system_table;
+    efi_handle device_handle;
+    void* file_path;
+    void* reserved;
+    uint32_t load_options_size;
+    void* load_options;
+    void* image_base;
+    uint64_t image_size;
+    enum efi_memory_type image_code_type;
+    enum efi_memory_type image_data_type;
+    void* unload;
+};
+
+constexpr uint64_t EFI_FILE_MODE_READ = 1;
+
+struct efi_file_protocol {
+    uint64_t revision;
+    efi_status(EFIAPI* open)(
+        struct efi_file_protocol* self,
+        struct efi_file_protocol** new_handle,
+        const char16* file_name,
+        uint64_t open_mode,
+        uint64_t attributes
+    );
+    efi_status(EFIAPI* close)(struct efi_file_protocol* self);
+    void* delete;
+    efi_status(EFIAPI* read)(struct efi_file_protocol* self, size_t* buffer_size, void* buffer);
+    void* write;
+    efi_status(EFIAPI* get_position)(struct efi_file_protocol* self, uint64_t* position);
+    efi_status(EFIAPI* set_position)(struct efi_file_protocol* self, uint64_t position);
+};
+
+struct efi_simple_file_system_protocol {
+    uint64_t revision;
+    efi_status(EFIAPI* open_volume)(struct efi_simple_file_system_protocol* self, struct efi_file_protocol** root);
+};
+
+enum efi_pixel_format : uint32_t {
+    EFI_PIXEL_RGBX_8,
+    EFI_PIXEL_BGRX_8,
+    EFI_PIXEL_BIT_MASK,
+    EFI_PIXEL_BLT_ONLY,
+};
+
+struct efi_pixel_bitmask {
+    uint32_t red;
+    uint32_t green;
+    uint32_t blue;
+    uint32_t reserved;
+};
+
+struct efi_gop_mode_info {
+    uint32_t version;
+    uint32_t horizontal_resolution;
+    uint32_t vertical_resolution;
+    enum efi_pixel_format pixel_format;
+    struct efi_pixel_bitmask pixel_information;
+    uint32_t pixels_per_scan_line;
+};
+
+struct efi_gop_mode {
+    uint32_t max_mode;
+    uint32_t mode;
+    struct efi_gop_mode_info* info;
+    size_t size_of_info;
+    uint64_t frame_buffer_base;
+    size_t frame_buffer_size;
+};
+
+struct efi_graphics_output_protocol {
+    void* query_mode;
+    void* set_mode;
+    void* blt;
+    struct efi_gop_mode* mode;
+};
+
+static const struct efi_guid EFI_LOADED_IMAGE_PROTOCOL_GUID =
+    {0x5b1b'31a1, 0x9562, 0x11d2, {0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}};
+static const struct efi_guid EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID =
+    {0x964e'5b22, 0x6459, 0x11d2, {0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}};
+static const struct efi_guid EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID =
+    {0x9042'a9de, 0x23dc, 0x4a38, {0x96, 0xfb, 0x7a, 0xde, 0xd0, 0x80, 0x51, 0x6a}};
+static const struct efi_guid EFI_ACPI_20_TABLE_GUID =
+    {0x8868'e871, 0xe4f1, 0x11d3, {0xbc, 0x22, 0x00, 0x80, 0xc7, 0x3c, 0x88, 0x81}};
+static const struct efi_guid EFI_ACPI_10_TABLE_GUID =
+    {0xeb9d'2d30, 0x2d88, 0x11d3, {0x9a, 0x16, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d}};

boot/efi/header.c

@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/compiler.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "efi_arch.h"
+#include "pe.h"
+
+extern const char __pe_code_size[];
+extern const char __pe_data_sizes[];
+extern const char __pe_entry[];
+extern const char __pe_image_sizes[];
+extern const char __pe_base_relocs[];
+extern const char __pe_text_virtual[];
+extern const char __pe_text_raw[];
+extern const char __pe_reloc_virtual[];
+extern const char __pe_reloc_raw[];
+extern const char __pe_rodata_virtual[];
+extern const char __pe_rodata_raw[];
+extern const char __pe_data_virtual[];
+extern const char __pe_data_raw[];
+
+struct efistub_pe_header {
+    struct pe_dos_header dos;
+    uint32_t signature;
+    struct pe_coff_header coff;
+    struct pe_optional_header64 optional;
+    struct pe_section_header sections[4];
+};
+
+static_assert(offsetof(struct efistub_pe_header, optional) % 8 == 0);
+static_assert(offsetof(struct efistub_pe_header, sections) % 8 == 0);
+
+[[__used, __section(".header")]]
+const struct efistub_pe_header efistub_pe_header = {
+    .dos =
+        {
+            .e_magic = PE_DOS_MAGIC,
+            .e_lfanew = offsetof(struct efistub_pe_header, signature),
+        },
+    .signature = PE_SIGNATURE,
+    .coff =
+        {
+            .machine = EFI_ARCH_PE_MACHINE,
+            .number_of_sections = 4,
+            .size_of_optional_header = sizeof(struct pe_optional_header64),
+            .characteristics = PE_FILE_EXECUTABLE_IMAGE | PE_FILE_LINE_NUMS_STRIPPED | PE_FILE_DEBUG_STRIPPED,
+        },
+    .optional =
+        {
+            .magic_and_code_size = (uintptr_t)__pe_code_size + PE_OPTIONAL_MAGIC_PE32_PLUS,
+            .data_sizes = (uintptr_t)__pe_data_sizes,
+            .entry_and_code_base = (uintptr_t)__pe_entry,
+            .section_alignment = 0x1000,
+            .file_alignment = 0x1000,
+            .image_and_header_sizes = (uintptr_t)__pe_image_sizes,
+            .subsystem = PE_SUBSYSTEM_EFI_APPLICATION,
+            .dll_characteristics = PE_DLL_NX_COMPAT,
+            .number_of_rva_and_sizes = PE_NUM_DATA_DIRECTORIES,
+            .data_directories =
+                {
+                    [PE_DIRECTORY_BASE_RELOC] = {.layout = (uintptr_t)__pe_base_relocs},
+                },
+        },
+    .sections = {
+        {
+            .name = ".text",
+            .virtual_layout = (uintptr_t)__pe_text_virtual,
+            .raw_layout = (uintptr_t)__pe_text_raw,
+            .characteristics = PE_SCN_CNT_CODE | PE_SCN_MEM_EXECUTE | PE_SCN_MEM_READ,
+        },
+        {
+            .name = ".reloc",
+            .virtual_layout = (uintptr_t)__pe_reloc_virtual,
+            .raw_layout = (uintptr_t)__pe_reloc_raw,
+            .characteristics = PE_SCN_CNT_INITIALIZED_DATA | PE_SCN_MEM_DISCARDABLE | PE_SCN_MEM_READ,
+        },
+        {
+            .name = ".rdata",
+            .virtual_layout = (uintptr_t)__pe_rodata_virtual,
+            .raw_layout = (uintptr_t)__pe_rodata_raw,
+            .characteristics = PE_SCN_CNT_INITIALIZED_DATA | PE_SCN_MEM_READ,
+        },
+        {
+            .name = ".data",
+            .virtual_layout = (uintptr_t)__pe_data_virtual,
+            .raw_layout = (uintptr_t)__pe_data_raw,
+            .characteristics = PE_SCN_CNT_INITIALIZED_DATA | PE_SCN_MEM_READ | PE_SCN_MEM_WRITE,
+        },
+    },
+};
+
+[[__used, __section(".reloc")]]
+const struct pe_base_reloc_block efistub_base_relocs = {
+    .page_rva = 0,
+    .block_size = sizeof(struct pe_base_reloc_block),
+};

boot/efi/main.c

@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: ISC
+
+#include <string.h>
+#include "stub.h"
+
+constexpr size_t MEM_MAP_SLACK = 64;
+
+struct efi_boot_services* bs;
+
+bool loader_alloc(enum phys_mem_usage usage, size_t size, uint64_t* out) {
+    enum efi_memory_type efi_type = EFI_LOADER_DATA;
+    if (usage == PHYS_STATIC)
+        efi_type = MEM_STATIC;
+    else if (usage == PHYS_RECLAIMABLE)
+        efi_type = MEM_RECLAIMABLE;
+
+    const size_t pages = ALIGN_UP(size, PAGE_SIZE) / PAGE_SIZE;
+    if (bs->allocate_pages(EFI_ALLOCATE_ANY_PAGES, efi_type, pages, out))
+        return false;
+    memset((void*)*out, 0, pages * PAGE_SIZE);
+    return true;
+}
+
+void* loader_ptr(uint64_t phys) {
+    return (void*)phys;
+}
+
+static efi_status open_file_on(efi_handle handle, const char16* path, struct efi_file_protocol** out) {
+    struct efi_simple_file_system_protocol* fs;
+    efi_status status = bs->handle_protocol(handle, &EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID, (void**)&fs);
+    if (status)
+        return status;
+
+    struct efi_file_protocol* root;
+    status = fs->open_volume(fs, &root);
+    if (status)
+        return status;
+
+    status = root->open(root, out, path, EFI_FILE_MODE_READ, 0);
+    if (status && path[0] == '\\')
+        status = root->open(root, out, path + 1, EFI_FILE_MODE_READ, 0);
+    root->close(root);
+    return status;
+}
+
+static efi_status read_file(struct efi_file_protocol* file, enum phys_mem_usage usage, uint64_t* base, uint64_t* size) {
+    uint64_t total;
+    uint64_t done = 0;
+    efi_status status = file->set_position(file, UINT64_MAX);
+    if (!status)
+        status = file->get_position(file, &total);
+    if (!status)
+        status = file->set_position(file, 0);
+    if (!status && !loader_alloc(usage, total ? total : 1, base))
+        status = EFI_OUT_OF_RESOURCES;
+    while (!status && done < total) {
+        size_t chunk = total - done;
+        status = file->read(file, &chunk, (void*)(*base + done));
+        if (!chunk)
+            break;
+        done += chunk;
+    }
+    file->close(file);
+    *size = done;
+    return status;
+}
+
+static void color_from_mask(uint32_t mask, struct boot_color* out) {
+    if (!mask)
+        return;
+    out->shift = __builtin_ctz(mask);
+    out->size = __builtin_ctzll(~(uint64_t)(mask >> out->shift));
+}
+
+static enum phys_mem_usage convert_mem_type(uint32_t type) {
+    switch (type) {
+    case EFI_LOADER_CODE:
+    case EFI_LOADER_DATA:
+    case EFI_BOOT_SERVICES_CODE:
+    case EFI_BOOT_SERVICES_DATA:
+    case EFI_CONVENTIONAL_MEMORY:
+        return PHYS_USABLE;
+    case EFI_ACPI_RECLAIM_MEMORY:
+    case MEM_RECLAIMABLE:
+        return PHYS_RECLAIMABLE;
+    case MEM_STATIC:
+        return PHYS_STATIC;
+    default:
+        return PHYS_RESERVED;
+    }
+}
+
+efi_status efi_main(efi_handle image_handle, struct efi_system_table* st) {
+    bs = st->boot_services;
+
+    // Check if we support this configuration.
+    if (!arch_check_cpu())
+        return EFI_UNSUPPORTED;
+
+    struct efi_loaded_image_protocol* image;
+    efi_status status = bs->handle_protocol(image_handle, &EFI_LOADED_IMAGE_PROTOCOL_GUID, (void**)&image);
+    if (status)
+        return status;
+
+    struct kernel_image kernel;
+    if (!loader_load_kernel(&kernel, "EFISTUB"))
+        return EFI_LOAD_ERROR;
+    struct boot_info* info = kernel.info;
+
+    // Reserve room for the memory map.
+    size_t map_size = 0;
+    size_t map_key;
+    size_t desc_size;
+    uint32_t desc_version;
+    status = bs->get_memory_map(&map_size, nullptr, &map_key, &desc_size, &desc_version);
+    if (status != EFI_BUFFER_TOO_SMALL)
+        return status;
+
+    const size_t map_capacity = map_size + MEM_MAP_SLACK * desc_size;
+    uint64_t map;
+    if (!loader_alloc(PHYS_USABLE, map_capacity, &map))
+        return EFI_OUT_OF_RESOURCES;
+
+    uint64_t mem;
+    if (!loader_alloc(PHYS_USABLE, map_capacity / desc_size * sizeof(struct phys_mem), &mem))
+        return EFI_OUT_OF_RESOURCES;
+
+    // Use the load options as the command line.
+    const char16* opts = image->load_options;
+    const size_t opts_len = image->load_options_size / sizeof(char16);
+    size_t cmdline_len = 0;
+    for (size_t i = 0; opts && i < opts_len && opts[i]; i++) {
+        if (opts[i] < 0x20 || opts[i] > 0x7e || cmdline_len + 1 >= sizeof(info->cmdline)) {
+            cmdline_len = 0;
+            break;
+        }
+        info->cmdline[cmdline_len++] = (char)opts[i];
+    }
+    info->cmdline[cmdline_len] = 0;
+
+    // Without load options, fall back to a command line file on the boot device.
+    struct efi_file_protocol* cmdline_file;
+    if (!cmdline_len && !open_file_on(image->device_handle, u"\\EFI\\zinnia\\cmdline", &cmdline_file)) {
+        uint64_t base;
+        uint64_t size;
+        status = read_file(cmdline_file, PHYS_USABLE, &base, &size);
+        if (status)
+            return status;
+
+        const char* text = (const char*)base;
+        while (cmdline_len < size && text[cmdline_len] && text[cmdline_len] != '\n' && text[cmdline_len] != '\r') {
+            if (text[cmdline_len] < 0x20 || text[cmdline_len] > 0x7e)
+                return EFI_INVALID_PARAMETER;
+            if (cmdline_len + 1 >= sizeof(info->cmdline))
+                return EFI_BUFFER_TOO_SMALL;
+            info->cmdline[cmdline_len] = text[cmdline_len];
+            cmdline_len++;
+        }
+        while (cmdline_len && info->cmdline[cmdline_len - 1] == ' ')
+            cmdline_len--;
+        info->cmdline[cmdline_len] = 0;
+    }
+
+    // Load every initrd= file.
+    const char* cmdline = info->cmdline;
+    for (size_t i = 0; cmdline[i];) {
+        if (cmdline[i] == ' ') {
+            i++;
+            continue;
+        }
+
+        const size_t start = i;
+        bool quoted = false;
+        while (cmdline[i] && (quoted || cmdline[i] != ' ')) {
+            if (cmdline[i] == '"')
+                quoted = !quoted;
+            i++;
+        }
+        const char* token = cmdline + start;
+        const size_t len = i - start;
+        if (len <= 7 || memcmp(token, "initrd=", 7))
+            continue;
+
+        const size_t path_len = len - 7;
+        if (path_len >= BOOT_PATH_MAX)
+            return EFI_INVALID_PARAMETER;
+        char path[BOOT_PATH_MAX];
+        char16 wide[BOOT_PATH_MAX];
+        for (size_t j = 0; j < path_len; j++) {
+            path[j] = token[7 + j];
+            wide[j] = path[j] == '/' ? '\\' : (char16)path[j];
+        }
+        path[path_len] = 0;
+        wide[path_len] = 0;
+
+        // Try the boot device first, then all other file systems.
+        struct efi_file_protocol* file;
+        status = open_file_on(image->device_handle, wide, &file);
+        size_t num_handles;
+        efi_handle* handles;
+        if (status &&
+            !bs->locate_handle_buffer(
+                EFI_LOCATE_BY_PROTOCOL,
+                &EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID,
+                nullptr,
+                &num_handles,
+                &handles
+            )) {
+            for (size_t j = 0; j < num_handles && status; j++) {
+                if (handles[j] != image->device_handle)
+                    status = open_file_on(handles[j], wide, &file);
+            }
+            bs->free_pool(handles);
+        }
+        if (status)
+            return status;
+
+        uint64_t base;
+        uint64_t size;
+        status = read_file(file, PHYS_STATIC, &base, &size);
+        if (status)
+            return status;
+
+        if (!loader_add_file(&kernel, path, base, size))
+            return EFI_OUT_OF_RESOURCES;
+    }
+
+    // Framebuffer
+    struct efi_graphics_output_protocol* gop;
+    if (!bs->locate_protocol(&EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID, nullptr, (void**)&gop) && gop && gop->mode &&
+        gop->mode->info) {
+        const struct efi_gop_mode_info* mode = gop->mode->info;
+        struct boot_framebuffer* fb = &info->fb;
+        switch (mode->pixel_format) {
+        case EFI_PIXEL_RGBX_8:
+            fb->bytes_per_pixel = 4;
+            fb->red = (struct boot_color){0, 8};
+            fb->green = (struct boot_color){8, 8};
+            fb->blue = (struct boot_color){16, 8};
+            break;
+        case EFI_PIXEL_BGRX_8:
+            fb->bytes_per_pixel = 4;
+            fb->red = (struct boot_color){16, 8};
+            fb->green = (struct boot_color){8, 8};
+            fb->blue = (struct boot_color){0, 8};
+            break;
+        case EFI_PIXEL_BIT_MASK: {
+            const struct efi_pixel_bitmask* m = &mode->pixel_information;
+            const uint32_t all = m->red | m->green | m->blue | m->reserved;
+            if (!all)
+                break;
+            fb->bytes_per_pixel = (32 - __builtin_clz(all) + 7) / 8;
+            color_from_mask(m->red, &fb->red);
+            color_from_mask(m->green, &fb->green);
+            color_from_mask(m->blue, &fb->blue);
+            break;
+        }
+        default:
+            break;
+        }
+
+        if (fb->bytes_per_pixel) {
+            fb->base = gop->mode->frame_buffer_base;
+            fb->width = mode->horizontal_resolution;
+            fb->height = mode->vertical_resolution;
+            fb->pitch = mode->pixels_per_scan_line * fb->bytes_per_pixel;
+        }
+    }
+
+    // Find the RSDP.
+    for (size_t i = 0; i < st->number_of_table_entries; i++) {
+        const struct efi_configuration_table* table = &st->configuration_table[i];
+        if (!memcmp(&table->vendor_guid, &EFI_ACPI_20_TABLE_GUID, sizeof(struct efi_guid))) {
+            info->rsdp = (uint64_t)table->vendor_table;
+            break;
+        }
+        if (!memcmp(&table->vendor_guid, &EFI_ACPI_10_TABLE_GUID, sizeof(struct efi_guid)))
+            info->rsdp = (uint64_t)table->vendor_table;
+    }
+
+    // Map everything up to the highest non-reserved address.
+    map_size = map_capacity;
+    status = bs->get_memory_map(&map_size, (struct efi_memory_descriptor*)map, &map_key, &desc_size, &desc_version);
+    if (status)
+        return status;
+
+    uint64_t top = 0;
+    for (size_t off = 0; off + desc_size <= map_size; off += desc_size) {
+        const struct efi_memory_descriptor* desc = (const struct efi_memory_descriptor*)(map + off);
+        if (convert_mem_type(desc->type) != PHYS_RESERVED)
+            top = MAX(top, desc->physical_start + desc->number_of_pages * PAGE_SIZE);
+    }
+
+    uint64_t root;
+    if (!loader_page_tables(&kernel, top, (uint64_t)arch_enter_kernel, &root))
+        return EFI_OUT_OF_RESOURCES;
+
+    // Exit boot services.
+    for (size_t attempt = 0;; attempt++) {
+        map_size = map_capacity;
+        status = bs->get_memory_map(&map_size, (struct efi_memory_descriptor*)map, &map_key, &desc_size, &desc_version);
+        if (status)
+            return status;
+        status = bs->exit_boot_services(image_handle, map_key);
+        if (!status)
+            break;
+        if (attempt == 8)
+            return status;
+    }
+
+    // Hand the final memory map to the kernel.
+    struct phys_mem* mem_map = (struct phys_mem*)mem;
+    size_t num_mem = 0;
+    for (size_t off = 0; off + desc_size <= map_size; off += desc_size) {
+        const struct efi_memory_descriptor* desc = (const struct efi_memory_descriptor*)(map + off);
+        mem_map[num_mem++] = (struct phys_mem){
+            .address = desc->physical_start,
+            .length = desc->number_of_pages * PAGE_SIZE,
+            .usage = convert_mem_type(desc->type),
+        };
+    }
+    loader_set_mem_map(&kernel, mem_map, num_mem);
+
+    arch_enter_kernel(root, kernel.entry);
+}

boot/efi/meson.build

@@ -0,0 +1,42 @@
+efistub_sources = files(
+    'header.c',
+    'main.c',
+    'reloc.c',
+)
+
+efistub_lds = meson.current_source_dir() / 'stub.ld'
+
+subdir(target_machine.cpu_family())
+
+efistub_elf = executable(
+    'efistub.elf',
+    sources: [loader_sources, efistub_sources, loader_kernel],
+    include_directories: [
+        common_includes,
+        loader_includes,
+        include_directories('.', target_machine.cpu_family()),
+    ],
+    c_args: loader_c_args,
+    link_args: [
+        '-nostdlib',
+        '-Wl,-T,' + efistub_lds,
+        '-Wl,--no-dynamic-linker',
+        '-Wl,-z,text',
+        '-Wl,-z,norelro',
+        '-Wl,-z,noexecstack',
+        '-Wl,-z,max-page-size=0x1000',
+        '-Wl,--build-id=none',
+    ],
+    link_depends: efistub_lds,
+    pie: true,
+)
+
+zinnia_efi = custom_target(
+    'zinnia.efi',
+    input: efistub_elf,
+    output: 'zinnia.efi',
+    command: [objcopy, '-O', 'binary', '@INPUT@', '@OUTPUT@'],
+    build_by_default: true,
+    install: true,
+    install_dir: zinnia_bootdir,
+)

boot/efi/pe.h

@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+constexpr uint16_t PE_DOS_MAGIC = 0x5a4d;
+constexpr uint32_t PE_SIGNATURE = 0x0000'4550;
+constexpr uint16_t PE_OPTIONAL_MAGIC_PE32_PLUS = 0x020b;
+constexpr uint16_t PE_SUBSYSTEM_EFI_APPLICATION = 10;
+constexpr size_t PE_NUM_DATA_DIRECTORIES = 16;
+constexpr size_t PE_DIRECTORY_BASE_RELOC = 5;
+
+enum pe_file_characteristics : uint16_t {
+    PE_FILE_EXECUTABLE_IMAGE = 0x0002,
+    PE_FILE_LINE_NUMS_STRIPPED = 0x0004,
+    PE_FILE_DEBUG_STRIPPED = 0x0200,
+};
+
+enum pe_dll_characteristics : uint16_t {
+    PE_DLL_NX_COMPAT = 0x0100,
+};
+
+enum pe_section_characteristics : uint32_t {
+    PE_SCN_CNT_CODE = 0x0000'0020,
+    PE_SCN_CNT_INITIALIZED_DATA = 0x0000'0040,
+    PE_SCN_MEM_DISCARDABLE = 0x0200'0000,
+    PE_SCN_MEM_EXECUTE = 0x2000'0000,
+    PE_SCN_MEM_READ = 0x4000'0000,
+    PE_SCN_MEM_WRITE = 0x8000'0000,
+};
+
+struct pe_dos_header {
+    uint16_t e_magic;
+    uint16_t e_cblp;
+    uint16_t e_cp;
+    uint16_t e_crlc;
+    uint16_t e_cparhdr;
+    uint16_t e_minalloc;
+    uint16_t e_maxalloc;
+    uint16_t e_ss;
+    uint16_t e_sp;
+    uint16_t e_csum;
+    uint16_t e_ip;
+    uint16_t e_cs;
+    uint16_t e_lfarlc;
+    uint16_t e_ovno;
+    uint16_t e_res[4];
+    uint16_t e_oemid;
+    uint16_t e_oeminfo;
+    uint16_t e_res2[10];
+    uint32_t e_lfanew;
+};
+
+struct pe_coff_header {
+    uint16_t machine;
+    uint16_t number_of_sections;
+    uint32_t time_date_stamp;
+    uint32_t pointer_to_symbol_table;
+    uint32_t number_of_symbols;
+    uint16_t size_of_optional_header;
+    uint16_t characteristics;
+};
+
+struct pe_data_directory {
+    union {
+        struct {
+            uint32_t virtual_address;
+            uint32_t size;
+        };
+        uint64_t layout;
+    };
+};
+
+struct pe_optional_header64 {
+    union {
+        struct {
+            uint16_t magic;
+            uint8_t major_linker_version;
+            uint8_t minor_linker_version;
+            uint32_t size_of_code;
+        };
+        uint64_t magic_and_code_size;
+    };
+    union {
+        struct {
+            uint32_t size_of_initialized_data;
+            uint32_t size_of_uninitialized_data;
+        };
+        uint64_t data_sizes;
+    };
+    union {
+        struct {
+            uint32_t address_of_entry_point;
+            uint32_t base_of_code;
+        };
+        uint64_t entry_and_code_base;
+    };
+    uint64_t image_base;
+    uint32_t section_alignment;
+    uint32_t file_alignment;
+    uint16_t major_operating_system_version;
+    uint16_t minor_operating_system_version;
+    uint16_t major_image_version;
+    uint16_t minor_image_version;
+    uint16_t major_subsystem_version;
+    uint16_t minor_subsystem_version;
+    uint32_t win32_version_value;
+    union {
+        struct {
+            uint32_t size_of_image;
+            uint32_t size_of_headers;
+        };
+        uint64_t image_and_header_sizes;
+    };
+    uint32_t checksum;
+    uint16_t subsystem;
+    uint16_t dll_characteristics;
+    uint64_t size_of_stack_reserve;
+    uint64_t size_of_stack_commit;
+    uint64_t size_of_heap_reserve;
+    uint64_t size_of_heap_commit;
+    uint32_t loader_flags;
+    uint32_t number_of_rva_and_sizes;
+    struct pe_data_directory data_directories[PE_NUM_DATA_DIRECTORIES];
+};
+
+struct pe_section_header {
+    char name[8];
+    union {
+        struct {
+            uint32_t virtual_size;
+            uint32_t virtual_address;
+        };
+        uint64_t virtual_layout;
+    };
+    union {
+        struct {
+            uint32_t size_of_raw_data;
+            uint32_t pointer_to_raw_data;
+        };
+        uint64_t raw_layout;
+    };
+    uint32_t pointer_to_relocations;
+    uint32_t pointer_to_line_numbers;
+    uint16_t number_of_relocations;
+    uint16_t number_of_line_numbers;
+    uint32_t characteristics;
+};
+
+struct pe_base_reloc_block {
+    uint32_t page_rva;
+    uint32_t block_size;
+    uint16_t entries[2];
+};
+
+static_assert(sizeof(struct pe_dos_header) == 64);
+static_assert(sizeof(struct pe_coff_header) == 20);
+static_assert(sizeof(struct pe_optional_header64) == 240);
+static_assert(offsetof(struct pe_optional_header64, size_of_image) == 56);
+static_assert(offsetof(struct pe_optional_header64, data_directories) == 112);
+static_assert(sizeof(struct pe_section_header) == 40);
+static_assert(sizeof(struct pe_base_reloc_block) == 12);

boot/efi/reloc.c

@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: ISC
+
+#include <bits/elf.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "efi.h"
+
+efi_status efistub_relocate(uintptr_t base, const elf64_dyn_t* dynamic) {
+    uintptr_t rela = 0;
+    size_t rela_size = 0;
+    size_t rela_ent = sizeof(elf64_rela_t);
+    uintptr_t relr = 0;
+    size_t relr_size = 0;
+
+    for (const elf64_dyn_t* dyn = dynamic; dyn->d_tag != DT_NULL; dyn++) {
+        switch (dyn->d_tag) {
+        case DT_RELA:
+            rela = base + dyn->d_un.d_val;
+            break;
+        case DT_RELASZ:
+            rela_size = dyn->d_un.d_val;
+            break;
+        case DT_RELAENT:
+            rela_ent = dyn->d_un.d_val;
+            break;
+        case DT_RELR:
+            relr = base + dyn->d_un.d_val;
+            break;
+        case DT_RELRSZ:
+            relr_size = dyn->d_un.d_val;
+            break;
+        }
+    }
+
+    for (size_t off = 0; rela && off + rela_ent <= rela_size; off += rela_ent) {
+        const elf64_rela_t* r = (const elf64_rela_t*)(rela + off);
+        const uint32_t type = ELF64_R_TYPE(r->r_info);
+        if (type == R_NONE)
+            continue;
+        if (type != R_RELATIVE)
+            return EFI_LOAD_ERROR;
+        *(uint64_t*)(base + r->r_offset) = base + r->r_addend;
+    }
+
+    uint64_t* where = nullptr;
+    for (size_t off = 0; relr && off + sizeof(uint64_t) <= relr_size; off += sizeof(uint64_t)) {
+        uint64_t entry = *(const uint64_t*)(relr + off);
+        if (!(entry & 1)) {
+            where = (uint64_t*)(base + entry);
+            *where++ += base;
+            continue;
+        }
+        for (size_t i = 0; (entry >>= 1) != 0; i++) {
+            if (entry & 1)
+                where[i] += base;
+        }
+        where += 63;
+    }
+
+    return EFI_SUCCESS;
+}

boot/efi/stub.h

@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "efi.h"
+#include "loader.h"
+
+constexpr enum efi_memory_type MEM_STATIC = (enum efi_memory_type)0x8000'0000;
+constexpr enum efi_memory_type MEM_RECLAIMABLE = (enum efi_memory_type)0x8000'0001;
+
+extern struct efi_boot_services* bs;

boot/efi/stub.ld

@@ -0,0 +1,100 @@
+ENTRY(_start)
+
+PHDRS
+{
+    text    PT_LOAD    FLAGS(5);
+    rodata  PT_LOAD    FLAGS(4);
+    data    PT_LOAD    FLAGS(6);
+    dynamic PT_DYNAMIC FLAGS(6);
+}
+
+SECTIONS
+{
+    . = 0;
+
+    .header : {
+        __image_base = .;
+        KEEP(*(.header))
+    } :text
+
+    . = ALIGN(0x1000);
+    .text : {
+        __text_start = .;
+        *(.text .text.*)
+        . = ALIGN(0x1000);
+        __text_end = .;
+    } :text
+
+    .reloc : {
+        __reloc_start = .;
+        KEEP(*(.reloc))
+        __reloc_data_end = .;
+        . = ALIGN(0x1000);
+        __reloc_end = .;
+    } :rodata
+
+    .rodata : {
+        __rodata_start = .;
+        *(.rodata .rodata.*)
+        . = ALIGN(16);
+        __kernel_start = .;
+        KEEP(*(.kernel))
+        __kernel_end = .;
+    } :rodata
+
+    .hash : { *(.hash) } :rodata
+    .gnu.hash : { *(.gnu.hash) } :rodata
+    .dynsym : { *(.dynsym) } :rodata
+    .dynstr : { *(.dynstr) } :rodata
+    .rela.dyn : { *(.rela.*) } :rodata
+    .relr.dyn : { *(.relr.dyn) } :rodata
+
+    .rodata.end : {
+        . = ALIGN(0x1000);
+        __rodata_end = .;
+    } :rodata
+
+    .dynamic : {
+        __data_start = .;
+        *(.dynamic)
+    } :data :dynamic
+
+    .data : {
+        *(.data .data.*)
+        *(.got .got.*)
+        . = ALIGN(0x1000);
+        __data_raw_end = .;
+    } :data
+
+    .bss (NOLOAD) : {
+        __bss_start = .;
+        *(.bss .bss.*)
+        *(COMMON)
+        . = ALIGN(0x1000);
+        __bss_end = .;
+    } :data
+
+    __image_end = .;
+
+    __pe_code_size = ABSOLUTE((__text_end - __text_start) << 32);
+    __pe_data_sizes = ABSOLUTE(((__reloc_end - __reloc_start) + (__rodata_end - __rodata_start) +
+                                (__data_raw_end - __data_start)) | ((__bss_end - __bss_start) << 32));
+    __pe_entry = ABSOLUTE(_start | (__text_start << 32));
+    __pe_image_sizes = ABSOLUTE(__image_end | (__text_start << 32));
+    __pe_base_relocs = ABSOLUTE(__reloc_start | ((__reloc_data_end - __reloc_start) << 32));
+    __pe_text_virtual = ABSOLUTE((__text_end - __text_start) | (__text_start << 32));
+    __pe_text_raw = ABSOLUTE((__text_end - __text_start) | (__text_start << 32));
+    __pe_reloc_virtual = ABSOLUTE((__reloc_data_end - __reloc_start) | (__reloc_start << 32));
+    __pe_reloc_raw = ABSOLUTE((__reloc_end - __reloc_start) | (__reloc_start << 32));
+    __pe_rodata_virtual = ABSOLUTE((__rodata_end - __rodata_start) | (__rodata_start << 32));
+    __pe_rodata_raw = ABSOLUTE((__rodata_end - __rodata_start) | (__rodata_start << 32));
+    __pe_data_virtual = ABSOLUTE((__bss_end - __data_start) | (__data_start << 32));
+    __pe_data_raw = ABSOLUTE((__data_raw_end - __data_start) | (__data_start << 32));
+
+    /DISCARD/ : {
+        *(.comment*)
+        *(.eh_frame*)
+        *(.note*)
+        *(.interp)
+    }
+}

boot/efi/x86_64/efi_arch.h

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stdint.h>
+
+#define EFIAPI __attribute__((ms_abi))
+
+constexpr uint16_t EFI_ARCH_PE_MACHINE = 0x8664;

boot/efi/x86_64/entry.S

@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: ISC
+
+    .intel_syntax noprefix
+    .text
+
+    .globl _start
+    .type _start, @function
+_start:
+    // Align the stack pointer
+    push rbp
+    mov rbp, rsp
+    push rdi
+    push rsi
+    push rbx
+    push r12
+    and rsp, -16
+    cld
+
+    mov rbx, rcx
+    mov r12, rdx
+
+    // Clear .bss
+    lea rdi, [rip + __bss_start]
+    lea rcx, [rip + __bss_end]
+    sub rcx, rdi
+    xor eax, eax
+    rep stosb
+
+    // Self-relocate
+    lea rdi, [rip + __image_base]
+    lea rsi, [rip + _DYNAMIC]
+    call efistub_relocate
+    test rax, rax
+    jnz 1f
+
+    mov rdi, rbx
+    mov rsi, r12
+    call efi_main
+
+1:
+    lea rsp, [rbp - 32]
+    pop r12
+    pop rbx
+    pop rsi
+    pop rdi
+    pop rbp
+    ret
+    .size _start, . - _start

boot/efi/x86_64/meson.build

@@ -0,0 +1 @@
+efistub_sources += files('entry.S')

boot/kernel.ld

@@ -1,4 +1,4 @@
-ENTRY(arch_start)
+ENTRY(_start)
 
 PHDRS {
     text    PT_LOAD     FLAGS(5);
@@ -6,6 +6,8 @@ PHDRS {
     data    PT_LOAD     FLAGS(6);
     dynamic PT_DYNAMIC  FLAGS(6);
     percpu  PT_LOAD     FLAGS(6);
+    cmdline 0x60000005;
+    boot    0x60000008;
 }
 
 SECTIONS
@@ -23,18 +25,36 @@ SECTIONS
         __ld_text_end = .;
     } :text
 
-    .rodata ALIGN(CONSTANT(MAXPAGESIZE)) : {
+    .cmdline ALIGN(CONSTANT(MAXPAGESIZE)) : {
         __ld_rodata_start = .;
-
         __ld_cmdline_start = .;
         KEEP(*(.cmdline))
         __ld_cmdline_end = .;
+    } :rodata :cmdline
 
-        __ld_vdso_start = .;
+    .rodata : {
-        KEEP(*(.vdso))
-        __ld_vdso_end = .;
-
         *(.rodata .rodata.*)
+
+        /* Builtin module entry points */
+        . = ALIGN(8);
+        __ld_mod_entry_start = .;
+        KEEP(*(.mod.entry .mod.entry.*))
+        __ld_mod_entry_end = .;
+
+        . = ALIGN(8);
+        __ld_initgraph_nodes_start = .;
+        KEEP(*(.initgraph.nodes))
+        __ld_initgraph_nodes_end = .;
+
+        __ld_initgraph_edges_start = .;
+        KEEP(*(.initgraph.edges))
+        __ld_initgraph_edges_end = .;
+
+        . = ALIGN(8);
+        __ld_ktest_start = .;
+        KEEP(*(.ktest))
+        __ld_ktest_end = .;
+
         *(.got .got.*)
 
         __ld_rodata_end = .;
@@ -43,44 +63,47 @@ SECTIONS
     .data ALIGN(CONSTANT(MAXPAGESIZE)) : {
         __ld_data_start = .;
 
-        __ld_console_start = .;
-        KEEP(*(.console))
-        __ld_console_end = .;
-
         KEEP(*(SORT_BY_NAME(.init.data.*)))
         KEEP(*(.init.data))
         *(.data .data.*)
     } :data
 
+    .boot : {
+        KEEP(*(.boot.info))
+    } :data :boot
+
     .dynsym : {
         __ld_dynsym_start = .;
-        *(.dynsym)
+        KEEP(*(.dynsym*))
         __ld_dynsym_end = .;
     } :data
 
     .dynstr : {
         __ld_dynstr_start = .;
-        *(.dynstr)
+        KEEP(*(.dynstr*))
         __ld_dynstr_end = .;
     } :data
 
     .dynamic : {
-        *(.dynamic)
+        *(.dynamic*)
     } :data :dynamic
 
-    .bss : {
+    .bss ALIGN(CONSTANT(MAXPAGESIZE)) (NOLOAD) : {
+        *(.*bss .*bss.*)
+        *(COMMON)
+
         /* Small stack to bootstrap the allocator */
+        . = ALIGN(16);
         __ld_stack_bottom = .;
         . += 0x2000;
         __ld_stack_top = .;
-        *(.*bss .*bss.*)
-        *(COMMON)
     } :data
 
     /* Per-CPU data block. MUST always come last. */
     .percpu ALIGN(CONSTANT(MAXPAGESIZE)) : {
         __ld_percpu_start = .;
-        KEEP(*(.percpu))
+        KEEP(*(.percpu.init .percpu.init*))
+        KEEP(*(.percpu.data .percpu.data*))
         . = ALIGN(CONSTANT(MAXPAGESIZE));
         __ld_percpu_end = .;
 

boot/limine/limine.h

@@ -0,0 +1,669 @@
+/* SPDX-License-Identifier: 0BSD */
+
+/* Copyright (C) 2022-2026 Mintsuki and contributors.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef LIMINE_H
+#define LIMINE_H 1
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Misc */
+
+#ifdef LIMINE_NO_POINTERS
+#  define LIMINE_PTR(TYPE) uint64_t
+#else
+#  define LIMINE_PTR(TYPE) TYPE
+#endif
+
+#define LIMINE_REQUESTS_START_MARKER { 0xf6b8f4b39de7d1ae, 0xfab91a6940fcb9cf, \
+                                       0x785c6ed015d3e316, 0x181e920a7852b9d9 }
+#define LIMINE_REQUESTS_END_MARKER { 0xadc0e0531bb10d03, 0x9572709f31764c62 }
+
+#define LIMINE_BASE_REVISION(N) { 0xf9562b2d5c95a6c8, 0x6a7b384944536bdc, (N) }
+
+#define LIMINE_BASE_REVISION_SUPPORTED(VAR) ((VAR)[2] == 0)
+
+#define LIMINE_LOADED_BASE_REVISION_VALID(VAR) ((VAR)[1] != 0x6a7b384944536bdc)
+#define LIMINE_LOADED_BASE_REVISION(VAR) ((VAR)[1])
+
+#define LIMINE_COMMON_MAGIC 0xc7b1dd30df4c8b88, 0x0a82e883a194f07b
+
+struct limine_uuid {
+    uint32_t a;
+    uint16_t b;
+    uint16_t c;
+    uint8_t d[8];
+};
+
+#define LIMINE_MEDIA_TYPE_GENERIC 0
+#define LIMINE_MEDIA_TYPE_OPTICAL 1
+#define LIMINE_MEDIA_TYPE_TFTP 2
+
+struct limine_file {
+    uint64_t revision;
+    LIMINE_PTR(void *) address;
+    uint64_t size;
+    LIMINE_PTR(char *) path;
+    LIMINE_PTR(char *) string;
+    uint32_t media_type;
+    uint32_t unused;
+    uint8_t tftp_ipv4[4];
+    uint32_t tftp_port;
+    uint32_t partition_index;
+    uint32_t mbr_disk_id;
+    struct limine_uuid gpt_disk_uuid;
+    struct limine_uuid gpt_part_uuid;
+    struct limine_uuid part_uuid;
+};
+
+/* Boot info */
+
+#define LIMINE_BOOTLOADER_INFO_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xf55038d8e2a1202f, 0x279426fcf5f59740 }
+
+struct limine_bootloader_info_response {
+    uint64_t revision;
+    LIMINE_PTR(char *) name;
+    LIMINE_PTR(char *) version;
+};
+
+struct limine_bootloader_info_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_bootloader_info_response *) response;
+};
+
+/* Executable command line */
+
+#define LIMINE_EXECUTABLE_CMDLINE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x4b161536e598651e, 0xb390ad4a2f1f303a }
+
+struct limine_executable_cmdline_response {
+    uint64_t revision;
+    LIMINE_PTR(char *) cmdline;
+};
+
+struct limine_executable_cmdline_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_executable_cmdline_response *) response;
+};
+
+/* Firmware type */
+
+#define LIMINE_FIRMWARE_TYPE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x8c2f75d90bef28a8, 0x7045a4688eac00c3 }
+
+#define LIMINE_FIRMWARE_TYPE_X86BIOS 0
+#define LIMINE_FIRMWARE_TYPE_EFI32 1
+#define LIMINE_FIRMWARE_TYPE_EFI64 2
+#define LIMINE_FIRMWARE_TYPE_SBI 3
+
+struct limine_firmware_type_response {
+    uint64_t revision;
+    uint64_t firmware_type;
+};
+
+struct limine_firmware_type_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_firmware_type_response *) response;
+};
+
+/* Stack size */
+
+#define LIMINE_STACK_SIZE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x224ef0460a8e8926, 0xe1cb0fc25f46ea3d }
+
+struct limine_stack_size_response {
+    uint64_t revision;
+};
+
+struct limine_stack_size_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_stack_size_response *) response;
+    uint64_t stack_size;
+};
+
+/* HHDM */
+
+#define LIMINE_HHDM_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x48dcf1cb8ad2b852, 0x63984e959a98244b }
+
+struct limine_hhdm_response {
+    uint64_t revision;
+    uint64_t offset;
+};
+
+struct limine_hhdm_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_hhdm_response *) response;
+};
+
+/* Framebuffer */
+
+#define LIMINE_FRAMEBUFFER_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x9d5827dcd881dd75, 0xa3148604f6fab11b }
+
+#define LIMINE_FRAMEBUFFER_RGB 1
+
+struct limine_video_mode {
+    uint64_t pitch;
+    uint64_t width;
+    uint64_t height;
+    uint16_t bpp;
+    uint8_t memory_model;
+    uint8_t red_mask_size;
+    uint8_t red_mask_shift;
+    uint8_t green_mask_size;
+    uint8_t green_mask_shift;
+    uint8_t blue_mask_size;
+    uint8_t blue_mask_shift;
+};
+
+struct limine_framebuffer {
+    LIMINE_PTR(void *) address;
+    uint64_t width;
+    uint64_t height;
+    uint64_t pitch;
+    uint16_t bpp;
+    uint8_t memory_model;
+    uint8_t red_mask_size;
+    uint8_t red_mask_shift;
+    uint8_t green_mask_size;
+    uint8_t green_mask_shift;
+    uint8_t blue_mask_size;
+    uint8_t blue_mask_shift;
+    uint8_t unused[7];
+    uint64_t edid_size;
+    LIMINE_PTR(void *) edid;
+    /* Response revision 1 */
+    uint64_t mode_count;
+    LIMINE_PTR(struct limine_video_mode **) modes;
+};
+
+struct limine_framebuffer_response {
+    uint64_t revision;
+    uint64_t framebuffer_count;
+    LIMINE_PTR(struct limine_framebuffer **) framebuffers;
+};
+
+struct limine_framebuffer_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_framebuffer_response *) response;
+};
+
+/* Flanterm FB init params */
+
+#define LIMINE_FLANTERM_FB_INIT_PARAMS_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x3259399fe7c5f126, 0xe01c1c8c5db9d1a9 }
+
+#define LIMINE_FLANTERM_FB_ROTATE_0 0
+#define LIMINE_FLANTERM_FB_ROTATE_90 1
+#define LIMINE_FLANTERM_FB_ROTATE_180 2
+#define LIMINE_FLANTERM_FB_ROTATE_270 3
+
+struct limine_flanterm_fb_init_params {
+    LIMINE_PTR(uint32_t *) canvas;
+    uint64_t canvas_size;
+    uint32_t ansi_colours[8];
+    uint32_t ansi_bright_colours[8];
+    uint32_t default_bg;
+    uint32_t default_fg;
+    uint32_t default_bg_bright;
+    uint32_t default_fg_bright;
+    LIMINE_PTR(void *) font;
+    uint64_t font_width;
+    uint64_t font_height;
+    uint64_t font_spacing;
+    uint64_t font_scale_x;
+    uint64_t font_scale_y;
+    uint64_t margin;
+    uint64_t rotation;
+};
+
+struct limine_flanterm_fb_init_params_response {
+    uint64_t revision;
+    uint64_t entry_count;
+    LIMINE_PTR(struct limine_flanterm_fb_init_params **) entries;
+};
+
+struct limine_flanterm_fb_init_params_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_flanterm_fb_init_params_response *) response;
+};
+
+/* Paging mode */
+
+#define LIMINE_PAGING_MODE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x95c1a0edab0944cb, 0xa4e5cb3842f7488a }
+
+#define LIMINE_PAGING_MODE_X86_64_4LVL 0
+#define LIMINE_PAGING_MODE_X86_64_5LVL 1
+#define LIMINE_PAGING_MODE_X86_64_MIN LIMINE_PAGING_MODE_X86_64_4LVL
+#define LIMINE_PAGING_MODE_X86_64_DEFAULT LIMINE_PAGING_MODE_X86_64_4LVL
+
+#define LIMINE_PAGING_MODE_AARCH64_4LVL 0
+#define LIMINE_PAGING_MODE_AARCH64_5LVL 1
+#define LIMINE_PAGING_MODE_AARCH64_MIN LIMINE_PAGING_MODE_AARCH64_4LVL
+#define LIMINE_PAGING_MODE_AARCH64_DEFAULT LIMINE_PAGING_MODE_AARCH64_4LVL
+
+#define LIMINE_PAGING_MODE_RISCV_SV39 0
+#define LIMINE_PAGING_MODE_RISCV_SV48 1
+#define LIMINE_PAGING_MODE_RISCV_SV57 2
+#define LIMINE_PAGING_MODE_RISCV_MIN LIMINE_PAGING_MODE_RISCV_SV39
+#define LIMINE_PAGING_MODE_RISCV_DEFAULT LIMINE_PAGING_MODE_RISCV_SV48
+
+#define LIMINE_PAGING_MODE_LOONGARCH_4LVL 0
+#define LIMINE_PAGING_MODE_LOONGARCH_MIN LIMINE_PAGING_MODE_LOONGARCH_4LVL
+#define LIMINE_PAGING_MODE_LOONGARCH_DEFAULT LIMINE_PAGING_MODE_LOONGARCH_4LVL
+
+struct limine_paging_mode_response {
+    uint64_t revision;
+    uint64_t mode;
+};
+
+struct limine_paging_mode_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_paging_mode_response *) response;
+    uint64_t mode;
+    uint64_t max_mode;
+    uint64_t min_mode;
+};
+
+/* MP */
+
+#define LIMINE_MP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x95a67b819a1b857e, 0xa0b61b723b6a73e0 }
+
+struct limine_mp_info;
+
+typedef void (*limine_goto_address)(struct limine_mp_info *);
+
+#if defined (__x86_64__) || defined (__i386__)
+
+#define LIMINE_MP_RESPONSE_X86_64_X2APIC (1 << 0)
+
+struct limine_mp_info {
+    uint32_t processor_id;
+    uint32_t lapic_id;
+    uint64_t reserved;
+    LIMINE_PTR(limine_goto_address) goto_address;
+    uint64_t extra_argument;
+};
+
+struct limine_mp_response {
+    uint64_t revision;
+    uint32_t flags;
+    uint32_t bsp_lapic_id;
+    uint64_t cpu_count;
+    LIMINE_PTR(struct limine_mp_info **) cpus;
+};
+
+#elif defined (__aarch64__)
+
+struct limine_mp_info {
+    uint32_t processor_id;
+    uint32_t reserved1;
+    uint64_t mpidr;
+    uint64_t reserved;
+    LIMINE_PTR(limine_goto_address) goto_address;
+    uint64_t extra_argument;
+};
+
+struct limine_mp_response {
+    uint64_t revision;
+    uint64_t flags;
+    uint64_t bsp_mpidr;
+    uint64_t cpu_count;
+    LIMINE_PTR(struct limine_mp_info **) cpus;
+};
+
+#elif defined (__riscv) && (__riscv_xlen == 64)
+
+struct limine_mp_info {
+    uint64_t processor_id;
+    uint64_t hartid;
+    uint64_t reserved;
+    LIMINE_PTR(limine_goto_address) goto_address;
+    uint64_t extra_argument;
+};
+
+struct limine_mp_response {
+    uint64_t revision;
+    uint64_t flags;
+    uint64_t bsp_hartid;
+    uint64_t cpu_count;
+    LIMINE_PTR(struct limine_mp_info **) cpus;
+};
+
+#elif defined (__loongarch__) && (__loongarch_grlen == 64)
+
+struct limine_mp_info {
+    uint64_t processor_id;
+    uint64_t phys_id;
+    uint64_t reserved;
+    LIMINE_PTR(limine_goto_address) goto_address;
+    uint64_t extra_argument;
+};
+
+struct limine_mp_response {
+    uint64_t revision;
+    uint64_t flags;
+    uint64_t bsp_phys_id;
+    uint64_t cpu_count;
+    LIMINE_PTR(struct limine_mp_info **) cpus;
+};
+
+#else
+#error Unknown architecture
+#endif
+
+#define LIMINE_MP_REQUEST_X86_64_X2APIC (1 << 0)
+
+struct limine_mp_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_mp_response *) response;
+    uint64_t flags;
+};
+
+/* Memory map */
+
+#define LIMINE_MEMMAP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x67cf3d9d378a806f, 0xe304acdfc50c3c62 }
+
+#define LIMINE_MEMMAP_USABLE                 0
+#define LIMINE_MEMMAP_RESERVED               1
+#define LIMINE_MEMMAP_ACPI_RECLAIMABLE       2
+#define LIMINE_MEMMAP_ACPI_NVS               3
+#define LIMINE_MEMMAP_BAD_MEMORY             4
+#define LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE 5
+#define LIMINE_MEMMAP_EXECUTABLE_AND_MODULES 6
+#define LIMINE_MEMMAP_FRAMEBUFFER            7
+#define LIMINE_MEMMAP_RESERVED_MAPPED        8
+
+struct limine_memmap_entry {
+    uint64_t base;
+    uint64_t length;
+    uint64_t type;
+};
+
+struct limine_memmap_response {
+    uint64_t revision;
+    uint64_t entry_count;
+    LIMINE_PTR(struct limine_memmap_entry **) entries;
+};
+
+struct limine_memmap_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_memmap_response *) response;
+};
+
+/* Entry point */
+
+#define LIMINE_ENTRY_POINT_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x13d86c035a1cd3e1, 0x2b0caa89d8f3026a }
+
+typedef void (*limine_entry_point)(void);
+
+struct limine_entry_point_response {
+    uint64_t revision;
+};
+
+struct limine_entry_point_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_entry_point_response *) response;
+    LIMINE_PTR(limine_entry_point) entry;
+};
+
+/* Executable File */
+
+#define LIMINE_EXECUTABLE_FILE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xad97e90e83f1ed67, 0x31eb5d1c5ff23b69 }
+
+struct limine_executable_file_response {
+    uint64_t revision;
+    LIMINE_PTR(struct limine_file *) executable_file;
+};
+
+struct limine_executable_file_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_executable_file_response *) response;
+};
+
+/* Module */
+
+#define LIMINE_MODULE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x3e7e279702be32af, 0xca1c4f3bd1280cee }
+
+#define LIMINE_INTERNAL_MODULE_REQUIRED (1 << 0)
+#define LIMINE_INTERNAL_MODULE_COMPRESSED (1 << 1)
+
+struct limine_internal_module {
+    LIMINE_PTR(const char *) path;
+    LIMINE_PTR(const char *) string;
+    uint64_t flags;
+};
+
+struct limine_module_response {
+    uint64_t revision;
+    uint64_t module_count;
+    LIMINE_PTR(struct limine_file **) modules;
+};
+
+struct limine_module_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_module_response *) response;
+
+    /* Request revision 1 */
+    uint64_t internal_module_count;
+    LIMINE_PTR(struct limine_internal_module **) internal_modules;
+};
+
+/* RSDP */
+
+#define LIMINE_RSDP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xc5e77b6b397e7b43, 0x27637845accdcf3c }
+
+struct limine_rsdp_response {
+    uint64_t revision;
+    LIMINE_PTR(void *) address;
+};
+
+struct limine_rsdp_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_rsdp_response *) response;
+};
+
+/* SMBIOS */
+
+#define LIMINE_SMBIOS_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x9e9046f11e095391, 0xaa4a520fefbde5ee }
+
+struct limine_smbios_response {
+    uint64_t revision;
+    LIMINE_PTR(void *) entry_32;
+    LIMINE_PTR(void *) entry_64;
+};
+
+struct limine_smbios_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_smbios_response *) response;
+};
+
+/* EFI system table */
+
+#define LIMINE_EFI_SYSTEM_TABLE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x5ceba5163eaaf6d6, 0x0a6981610cf65fcc }
+
+struct limine_efi_system_table_response {
+    uint64_t revision;
+    LIMINE_PTR(void *) address;
+};
+
+struct limine_efi_system_table_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_efi_system_table_response *) response;
+};
+
+/* TPM event log */
+
+#define LIMINE_TPM_EVENT_LOG_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x98e094fc7e76e979, 0xee8d8775c54e1d1f }
+
+#define LIMINE_TPM_EVENT_LOG_FORMAT_TCG_1_2 1
+#define LIMINE_TPM_EVENT_LOG_FORMAT_TCG_2   2
+
+struct limine_tpm_event_log_response {
+    uint64_t revision;
+    uint64_t format;
+    uint64_t size;
+    LIMINE_PTR(void *) address;
+};
+
+struct limine_tpm_event_log_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_tpm_event_log_response *) response;
+};
+
+/* EFI memory map */
+
+#define LIMINE_EFI_MEMMAP_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x7df62a431d6872d5, 0xa4fcdfb3e57306c8 }
+
+struct limine_efi_memmap_response {
+    uint64_t revision;
+    LIMINE_PTR(void *) memmap;
+    uint64_t memmap_size;
+    uint64_t desc_size;
+    uint64_t desc_version;
+};
+
+struct limine_efi_memmap_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_efi_memmap_response *) response;
+};
+
+/* Date at boot */
+
+#define LIMINE_DATE_AT_BOOT_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x502746e184c088aa, 0xfbc5ec83e6327893 }
+
+struct limine_date_at_boot_response {
+    uint64_t revision;
+    int64_t timestamp;
+};
+
+struct limine_date_at_boot_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_date_at_boot_response *) response;
+};
+
+/* Executable address */
+
+#define LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x71ba76863cc55f63, 0xb2644a48c516a487 }
+
+struct limine_executable_address_response {
+    uint64_t revision;
+    uint64_t physical_base;
+    uint64_t virtual_base;
+};
+
+struct limine_executable_address_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_executable_address_response *) response;
+};
+
+/* Device Tree Blob */
+
+#define LIMINE_DTB_REQUEST_ID { LIMINE_COMMON_MAGIC, 0xb40ddb48fb54bac7, 0x545081493f81ffb7 }
+
+struct limine_dtb_response {
+    uint64_t revision;
+    LIMINE_PTR(void *) dtb_ptr;
+};
+
+struct limine_dtb_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_dtb_response *) response;
+};
+
+/* RISC-V Boot Hart ID */
+
+#define LIMINE_RISCV_BSP_HARTID_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x1369359f025525f9, 0x2ff2a56178391bb6 }
+
+struct limine_riscv_bsp_hartid_response {
+    uint64_t revision;
+    uint64_t bsp_hartid;
+};
+
+struct limine_riscv_bsp_hartid_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_riscv_bsp_hartid_response *) response;
+};
+
+/* Bootloader Performance */
+
+#define LIMINE_BOOTLOADER_PERFORMANCE_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x6b50ad9bf36d13ad, 0xdc4c7e88fc759e17 }
+
+struct limine_bootloader_performance_response {
+    uint64_t revision;
+    uint64_t reset_usec;
+    uint64_t init_usec;
+    uint64_t exec_usec;
+};
+
+struct limine_bootloader_performance_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_bootloader_performance_response *) response;
+};
+
+#define LIMINE_X86_64_KEEP_IOMMU_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x8ebaabe51f490179, 0x2aa86a59ffb4ab0f }
+
+struct limine_x86_64_keep_iommu_response {
+    uint64_t revision;
+};
+
+struct limine_x86_64_keep_iommu_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_x86_64_keep_iommu_response *) response;
+};
+
+/* TSC (Timestamp Counter) Frequency */
+
+#define LIMINE_TSC_FREQUENCY_REQUEST_ID { LIMINE_COMMON_MAGIC, 0x10f2ee1d87d195e4, 0xf747a2b78f6ddb31 }
+
+struct limine_tsc_frequency_response {
+    uint64_t revision;
+    uint64_t frequency;
+};
+
+struct limine_tsc_frequency_request {
+    uint64_t id[4];
+    uint64_t revision;
+    LIMINE_PTR(struct limine_tsc_frequency_response *) response;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

boot/limine/loader.ld

@@ -0,0 +1,46 @@
+ENTRY(loader_main)
+
+PHDRS
+{
+    text    PT_LOAD FLAGS(5);
+    rodata  PT_LOAD FLAGS(4);
+    data    PT_LOAD FLAGS(6);
+}
+
+SECTIONS
+{
+    . = 0xffffffffc0000000;
+
+    .text : {
+        *(.text .text.*)
+    } :text
+
+    . = ALIGN(0x1000);
+    .rodata : {
+        *(.rodata .rodata.*)
+        . = ALIGN(16);
+        __kernel_start = .;
+        KEEP(*(.kernel))
+        __kernel_end = .;
+    } :rodata
+
+    . = ALIGN(0x1000);
+    .data : {
+        KEEP(*(.limine_requests_start))
+        KEEP(*(.limine_requests))
+        KEEP(*(.limine_requests_end))
+        *(.data .data.*)
+    } :data
+
+    .bss : {
+        *(.bss .bss.*)
+        *(COMMON)
+    } :data
+
+    /DISCARD/ : {
+        *(.comment*)
+        *(.eh_frame*)
+        *(.note*)
+        *(.interp)
+    }
+}

boot/limine/main.c

@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/compiler.h>
+#include <string.h>
+#include "limine.h"
+#include "loader.h"
+
+[[__used, __section(".limine_requests_start")]]
+static volatile uint64_t limine_start[] = LIMINE_REQUESTS_START_MARKER;
+[[__used, __section(".limine_requests")]]
+static volatile uint64_t limine_revision[] = LIMINE_BASE_REVISION(6);
+[[__used, __section(".limine_requests_end")]]
+static volatile uint64_t limine_end[] = LIMINE_REQUESTS_END_MARKER;
+
+#define LIMINE_REQUEST(request, tag, rev) \
+    [[__used, __section(".limine_requests")]] \
+    static volatile struct limine_##request request = { \
+        .id = tag, \
+        .revision = rev, \
+        .response = nullptr, \
+    }
+
+LIMINE_REQUEST(bootloader_info_request, LIMINE_BOOTLOADER_INFO_REQUEST_ID, 0);
+LIMINE_REQUEST(memmap_request, LIMINE_MEMMAP_REQUEST_ID, 0);
+LIMINE_REQUEST(hhdm_request, LIMINE_HHDM_REQUEST_ID, 0);
+LIMINE_REQUEST(executable_address_request, LIMINE_EXECUTABLE_ADDRESS_REQUEST_ID, 0);
+LIMINE_REQUEST(executable_file_request, LIMINE_EXECUTABLE_FILE_REQUEST_ID, 0);
+LIMINE_REQUEST(framebuffer_request, LIMINE_FRAMEBUFFER_REQUEST_ID, 1);
+LIMINE_REQUEST(module_request, LIMINE_MODULE_REQUEST_ID, 0);
+LIMINE_REQUEST(dtb_request, LIMINE_DTB_REQUEST_ID, 0);
+LIMINE_REQUEST(rsdp_request, LIMINE_RSDP_REQUEST_ID, 0);
+
+static uint64_t hhdm_offset;
+static const struct limine_memmap_entry* pool;
+static uint64_t pool_static_end;
+static uint64_t pool_reclaimable_start;
+
+void* loader_ptr(uint64_t phys) {
+    return (void*)(phys + hhdm_offset);
+}
+
+static uint64_t to_phys(const void* ptr) {
+    const uint64_t addr = (uint64_t)ptr;
+    return addr >= hhdm_offset ? addr - hhdm_offset : addr;
+}
+
+bool loader_alloc(enum phys_mem_usage usage, size_t size, uint64_t* out) {
+    size = ALIGN_UP(size, PAGE_SIZE);
+    if (pool_reclaimable_start - pool_static_end < size)
+        return false;
+
+    if (usage == PHYS_STATIC) {
+        *out = pool_static_end;
+        pool_static_end += size;
+    } else {
+        pool_reclaimable_start -= size;
+        *out = pool_reclaimable_start;
+    }
+    memset(loader_ptr(*out), 0, size);
+    return true;
+}
+
+static enum phys_mem_usage convert_mem_type(uint64_t type) {
+    switch (type) {
+    case LIMINE_MEMMAP_USABLE:
+        return PHYS_USABLE;
+    case LIMINE_MEMMAP_ACPI_RECLAIMABLE:
+    case LIMINE_MEMMAP_BOOTLOADER_RECLAIMABLE:
+        return PHYS_RECLAIMABLE;
+    case LIMINE_MEMMAP_EXECUTABLE_AND_MODULES:
+        return PHYS_STATIC;
+    default:
+        return PHYS_RESERVED;
+    }
+}
+
+static bool init_pool(const struct limine_memmap_response* memmap) {
+    for (size_t i = 0; i < memmap->entry_count; i++) {
+        const struct limine_memmap_entry* entry = memmap->entries[i];
+        if (entry->type == LIMINE_MEMMAP_USABLE && (!pool || entry->length > pool->length))
+            pool = entry;
+    }
+    if (!pool)
+        return false;
+
+    pool_static_end = pool->base;
+    pool_reclaimable_start = pool->base + pool->length;
+    return true;
+}
+
+static uint64_t mem_map_top(const struct limine_memmap_response* memmap) {
+    uint64_t top = 0;
+    for (size_t i = 0; i < memmap->entry_count; i++) {
+        const struct limine_memmap_entry* entry = memmap->entries[i];
+        if (convert_mem_type(entry->type) != PHYS_RESERVED)
+            top = MAX(top, entry->base + entry->length);
+    }
+    return top;
+}
+
+static size_t convert_mem_map(const struct limine_memmap_response* memmap, struct phys_mem* out) {
+    size_t n = 0;
+    for (size_t i = 0; i < memmap->entry_count; i++) {
+        const struct limine_memmap_entry* entry = memmap->entries[i];
+        if (entry != pool) {
+            out[n++] = (struct phys_mem){entry->base, entry->length, convert_mem_type(entry->type)};
+            continue;
+        }
+
+        const uint64_t end = entry->base + entry->length;
+        out[n++] = (struct phys_mem){entry->base, pool_static_end - entry->base, PHYS_STATIC};
+        out[n++] = (struct phys_mem){pool_static_end, pool_reclaimable_start - pool_static_end, PHYS_USABLE};
+        out[n++] = (struct phys_mem){pool_reclaimable_start, end - pool_reclaimable_start, PHYS_RECLAIMABLE};
+    }
+    return n;
+}
+
+static void find_framebuffer(struct boot_framebuffer* out) {
+    const struct limine_framebuffer_response* res = framebuffer_request.response;
+    if (!res || !res->framebuffer_count)
+        return;
+
+    const struct limine_framebuffer* fb = res->framebuffers[0];
+    out->base = to_phys(fb->address);
+    out->width = fb->width;
+    out->height = fb->height;
+    out->pitch = fb->pitch;
+    out->bytes_per_pixel = fb->bpp / 8;
+    out->red = (struct boot_color){fb->red_mask_shift, fb->red_mask_size};
+    out->green = (struct boot_color){fb->green_mask_shift, fb->green_mask_size};
+    out->blue = (struct boot_color){fb->blue_mask_shift, fb->blue_mask_size};
+}
+
+static bool load() {
+    const struct limine_memmap_response* memmap = memmap_request.response;
+    const struct limine_executable_address_response* exec = executable_address_request.response;
+    if (!LIMINE_BASE_REVISION_SUPPORTED(limine_revision) || !memmap || !hhdm_request.response || !exec)
+        return false;
+    hhdm_offset = hhdm_request.response->offset;
+
+    if (!arch_check_cpu() || !init_pool(memmap))
+        return false;
+
+    const struct limine_bootloader_info_response* info = bootloader_info_request.response;
+    struct kernel_image kernel;
+    if (!loader_load_kernel(&kernel, info ? info->name : "Limine"))
+        return false;
+
+    uint64_t mem;
+    if (!loader_alloc(PHYS_RECLAIMABLE, (memmap->entry_count + 2) * sizeof(struct phys_mem), &mem))
+        return false;
+
+    const struct limine_executable_file_response* file = executable_file_request.response;
+    if (file && file->executable_file->string) {
+        const char* cmdline = file->executable_file->string;
+        memcpy(kernel.info->cmdline, cmdline, MIN(strlen(cmdline), sizeof(kernel.info->cmdline) - 1));
+    }
+
+    const struct limine_module_response* modules = module_request.response;
+    for (size_t i = 0; modules && i < modules->module_count; i++) {
+        const struct limine_file* module = modules->modules[i];
+        if (!loader_add_file(&kernel, module->path, to_phys(module->address), module->size))
+            return false;
+    }
+
+    find_framebuffer(&kernel.info->fb);
+    if (rsdp_request.response)
+        kernel.info->rsdp = to_phys(rsdp_request.response->address);
+    if (dtb_request.response && dtb_request.response->dtb_ptr)
+        kernel.info->dtb = to_phys(dtb_request.response->dtb_ptr);
+
+    const uint64_t trampoline = (uint64_t)arch_enter_kernel;
+    uint64_t root;
+    if (!loader_page_tables(
+            &kernel,
+            mem_map_top(memmap),
+            exec->physical_base + (trampoline - exec->virtual_base),
+            &root
+        ))
+        return false;
+
+    struct phys_mem* mem_map = loader_ptr(mem);
+    loader_set_mem_map(&kernel, mem_map, convert_mem_map(memmap, mem_map));
+    arch_enter_kernel(root, kernel.entry);
+}
+
+[[noreturn]]
+void loader_main() {
+    load();
+    for (;;)
+        asm volatile("");
+}

boot/limine/meson.build

@@ -0,0 +1,25 @@
+limine_lds = meson.current_source_dir() / 'loader.ld'
+
+limine_c_args = [loader_c_args, '-fno-pic']
+if target_machine.cpu_family() == 'x86_64'
+    limine_c_args += '-mcmodel=kernel'
+endif
+
+executable(
+    'zinnia-limine',
+    sources: [loader_sources, files('main.c'), loader_kernel],
+    include_directories: [common_includes, loader_includes],
+    c_args: limine_c_args,
+    link_args: [
+        '-nostdlib',
+        '-static',
+        '-Wl,-T,' + limine_lds,
+        '-Wl,-z,noexecstack',
+        '-Wl,-z,max-page-size=0x1000',
+        '-Wl,--build-id=none',
+    ],
+    link_depends: limine_lds,
+    pie: false,
+    install: true,
+    install_dir: zinnia_bootdir,
+)

boot/loader/elf.c

@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: ISC
+
+#include <bits/elf.h>
+#include <string.h>
+#include "loader.h"
+
+static bool in_image(const struct kernel_image* image, uint64_t vaddr, uint64_t len) {
+    return vaddr >= image->virt && len <= image->size && vaddr - image->virt <= image->size - len;
+}
+
+void* kernel_ptr(const struct kernel_image* image, uint64_t vaddr) {
+    return loader_ptr(image->phys + (vaddr - image->virt));
+}
+
+static bool apply_relocations(const struct kernel_image* image, uint64_t dynamic_vaddr) {
+    if (!in_image(image, dynamic_vaddr, sizeof(elf64_dyn_t)))
+        return false;
+
+    uint64_t rela = 0;
+    uint64_t rela_size = 0;
+    uint64_t rela_ent = sizeof(elf64_rela_t);
+    for (const elf64_dyn_t* dyn = kernel_ptr(image, dynamic_vaddr); dyn->d_tag != DT_NULL; dyn++) {
+        switch (dyn->d_tag) {
+        case DT_RELA:
+            rela = dyn->d_un.d_val;
+            break;
+        case DT_RELASZ:
+            rela_size = dyn->d_un.d_val;
+            break;
+        case DT_RELAENT:
+            rela_ent = dyn->d_un.d_val;
+            break;
+        }
+    }
+
+    if (!rela)
+        return true;
+    if (rela_ent < sizeof(elf64_rela_t) || !in_image(image, rela, rela_size))
+        return false;
+
+    for (uint64_t off = 0; off + rela_ent <= rela_size; off += rela_ent) {
+        const elf64_rela_t* r = kernel_ptr(image, rela + off);
+        const uint32_t type = ELF64_R_TYPE(r->r_info);
+        if (type == R_NONE)
+            continue;
+        if (type != R_RELATIVE)
+            return false;
+        if (!in_image(image, r->r_offset, sizeof(uint64_t)))
+            return false;
+        *(uint64_t*)kernel_ptr(image, r->r_offset) = (uint64_t)r->r_addend;
+    }
+
+    return true;
+}
+
+bool elf_load(const uint8_t* elf, size_t size, struct kernel_image* out) {
+    elf64_ehdr_t ehdr;
+    if (size < sizeof(ehdr))
+        return false;
+    memcpy(&ehdr, elf, sizeof(ehdr));
+
+    if (memcmp(ehdr.e_ident, ELF_MAG, sizeof(ELF_MAG)) || ehdr.e_ident[EI_CLASS] != ELF_ARCH_CLASS ||
+        ehdr.e_ident[EI_DATA] != ELF_ARCH_DATA)
+        return false;
+    if (ehdr.e_machine != ELF_ARCH_MACHINE)
+        return false;
+    if (ehdr.e_phentsize != sizeof(elf64_phdr_t) || ehdr.e_phoff > size ||
+        (size - ehdr.e_phoff) / sizeof(elf64_phdr_t) < ehdr.e_phnum)
+        return false;
+
+    const elf64_phdr_t* phdrs = (const elf64_phdr_t*)(elf + ehdr.e_phoff);
+
+    uint64_t lo = UINT64_MAX;
+    uint64_t hi = 0;
+    uint64_t dynamic_vaddr = 0;
+    uint64_t info_vaddr = 0;
+    uint64_t info_size = 0;
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf64_phdr_t* ph = &phdrs[i];
+        if (ph->p_type == PT_DYNAMIC)
+            dynamic_vaddr = ph->p_vaddr;
+        if (ph->p_type == PT_BOOT_INFO) {
+            info_vaddr = ph->p_vaddr;
+            info_size = ph->p_memsz;
+        }
+        if (ph->p_type != PT_LOAD || ph->p_memsz == 0)
+            continue;
+        if (ph->p_filesz > ph->p_memsz || ph->p_offset > size || size - ph->p_offset < ph->p_filesz)
+            return false;
+        if (ALIGN_DOWN(ph->p_vaddr, PAGE_SIZE) < lo)
+            lo = ALIGN_DOWN(ph->p_vaddr, PAGE_SIZE);
+        if (ALIGN_UP(ph->p_vaddr + ph->p_memsz, PAGE_SIZE) > hi)
+            hi = ALIGN_UP(ph->p_vaddr + ph->p_memsz, PAGE_SIZE);
+    }
+    if (lo >= hi)
+        return false;
+
+    out->virt = lo;
+    out->size = hi - lo;
+    out->entry = ehdr.e_entry;
+    if (!in_image(out, out->entry, 1))
+        return false;
+    if (info_size != sizeof(struct boot_info) || !in_image(out, info_vaddr, info_size))
+        return false;
+
+    if (!loader_alloc(PHYS_STATIC, out->size, &out->phys))
+        return false;
+
+    for (size_t i = 0; i < ehdr.e_phnum; i++) {
+        const elf64_phdr_t* ph = &phdrs[i];
+        if (ph->p_type != PT_LOAD || ph->p_memsz == 0)
+            continue;
+        memcpy(kernel_ptr(out, ph->p_vaddr), elf + ph->p_offset, ph->p_filesz);
+    }
+    out->info = kernel_ptr(out, info_vaddr);
+
+    if (dynamic_vaddr)
+        return apply_relocations(out, dynamic_vaddr);
+    return true;
+}

boot/loader/lib.c

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: ISC
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+size_t strlen(const char* str) {
+    if (!str)
+        return 0;
+
+    size_t len = 0;
+    while (str[len])
+        len++;
+    return len;
+}
+
+int memcmp(const void* s1, const void* s2, size_t size) {
+    const uint8_t* a = s1;
+    const uint8_t* b = s2;
+    for (size_t i = 0; i < size; i++) {
+        if (a[i] != b[i])
+            return a[i] < b[i] ? -1 : 1;
+    }
+    return 0;
+}
+
+void* memcpy(void* restrict dest, const void* restrict src, size_t n) {
+    uint8_t* d = dest;
+    const uint8_t* s = src;
+    for (size_t i = 0; i < n; i++)
+        d[i] = s[i];
+    return dest;
+}
+
+void* memmove(void* dstptr, const void* srcptr, size_t size) {
+    uint8_t* d = dstptr;
+    const uint8_t* s = srcptr;
+    if (d < s) {
+        for (size_t i = 0; i < size; i++)
+            d[i] = s[i];
+    } else {
+        for (size_t i = size; i > 0; i--)
+            d[i - 1] = s[i - 1];
+    }
+    return dstptr;
+}
+
+void* memset(void* dest, int value, size_t n) {
+    uint8_t* d = dest;
+    for (size_t i = 0; i < n; i++)
+        d[i] = (uint8_t)value;
+    return dest;
+}

boot/loader/loader.c

@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: ISC
+
+#include "loader.h"
+#include <string.h>
+#include "loader_arch.h"
+
+extern const uint8_t __kernel_start[];
+extern const uint8_t __kernel_end[];
+
+bool loader_load_kernel(struct kernel_image* out, const char* name) {
+    if (!elf_load(__kernel_start, __kernel_end - __kernel_start, out))
+        return false;
+
+    struct boot_info* info = out->info;
+    memcpy(info->loader, name, MIN(strlen(name), sizeof(info->loader) - 1));
+    info->hhdm_base = ARCH_HHDM_BASE;
+    info->phys_base = out->phys;
+    info->virt_base = out->virt;
+    return true;
+}
+
+bool loader_add_file(const struct kernel_image* kernel, const char* path, uint64_t base, uint64_t length) {
+    struct boot_info* info = kernel->info;
+    if (info->num_files == ARRAY_SIZE(info->files))
+        return false;
+
+    struct boot_file* file = &info->files[info->num_files++];
+    file->data = base;
+    file->length = length;
+    const size_t path_len = MIN(strlen(path), sizeof(file->path) - 1);
+    memcpy(file->path, path, path_len);
+    file->path[path_len] = 0;
+    return true;
+}
+
+void loader_set_mem_map(const struct kernel_image* kernel, struct phys_mem* map, size_t len) {
+    for (size_t i = 1; i < len; i++) {
+        const struct phys_mem entry = map[i];
+        size_t j = i;
+        while (j > 0 && map[j - 1].address > entry.address) {
+            map[j] = map[j - 1];
+            j--;
+        }
+        map[j] = entry;
+    }
+
+    size_t merged = 0;
+    for (size_t i = 0; i < len; i++) {
+        if (!map[i].length)
+            continue;
+        struct phys_mem* prev = merged ? &map[merged - 1] : nullptr;
+        if (prev && prev->usage == map[i].usage && prev->address + prev->length == map[i].address)
+            prev->length += map[i].length;
+        else
+            map[merged++] = map[i];
+    }
+
+    struct boot_info* info = kernel->info;
+    info->num_mem_maps = MIN(merged, ARRAY_SIZE(info->mem_map));
+    memcpy(info->mem_map, map, info->num_mem_maps * sizeof(*map));
+}
+
+bool loader_page_tables(
+    const struct kernel_image* kernel,
+    uint64_t mem_top,
+    uint64_t trampoline_phys,
+    uint64_t* out_root
+) {
+    uint64_t root;
+    if (!loader_alloc(PHYS_RECLAIMABLE, PAGE_SIZE, &root))
+        return false;
+
+    const uint64_t trampoline_virt = ALIGN_DOWN((uint64_t)arch_enter_kernel, PAGE_SIZE);
+    if (!arch_paging_map(root, ARCH_HHDM_BASE, 0, ALIGN_UP(mem_top, ARCH_HUGE_PAGE_SIZE), ARCH_HUGE_PAGE_SIZE) ||
+        !arch_paging_map(root, kernel->virt, kernel->phys, kernel->size, PAGE_SIZE) ||
+        !arch_paging_map(root, trampoline_virt, ALIGN_DOWN(trampoline_phys, PAGE_SIZE), 2 * PAGE_SIZE, PAGE_SIZE))
+        return false;
+
+    *out_root = root;
+    return true;
+}

boot/loader/loader.h

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/boot.h>
+#include <zinnia/utils.h>
+#include <stddef.h>
+#include <stdint.h>
+
+constexpr uint64_t PAGE_SIZE = 0x1000;
+
+struct kernel_image {
+    uint64_t phys;
+    uint64_t virt;
+    uint64_t size;
+    uint64_t entry;
+    struct boot_info* info;
+};
+
+bool loader_alloc(enum phys_mem_usage usage, size_t size, uint64_t* out);
+void* loader_ptr(uint64_t phys);
+
+bool elf_load(const uint8_t* elf, size_t size, struct kernel_image* out);
+void* kernel_ptr(const struct kernel_image* image, uint64_t vaddr);
+
+bool loader_load_kernel(struct kernel_image* out, const char* name);
+bool loader_add_file(const struct kernel_image* kernel, const char* path, uint64_t base, uint64_t length);
+void loader_set_mem_map(const struct kernel_image* kernel, struct phys_mem* map, size_t len);
+
+bool loader_page_tables(
+    const struct kernel_image* kernel,
+    uint64_t mem_top,
+    uint64_t trampoline_phys,
+    uint64_t* out_root
+);
+
+bool arch_check_cpu();
+bool arch_paging_map(uint64_t root, uint64_t virt, uint64_t phys, uint64_t size, uint64_t page_size);
+
+[[noreturn]]
+void arch_enter_kernel(uint64_t root, uint64_t entry);

boot/loader/x86_64/enter.S

@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: ISC
+
+    .intel_syntax noprefix
+    .text
+
+    .globl arch_enter_kernel
+    .type arch_enter_kernel, @function
+    .balign 16
+arch_enter_kernel:
+    cli
+    mov cr3, rdi
+    jmp rsi
+    .size arch_enter_kernel, . - arch_enter_kernel

boot/loader/x86_64/loader_arch.h

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <stdint.h>
+
+constexpr uint64_t ARCH_HHDM_BASE = 0xffff'8000'0000'0000;
+constexpr uint64_t ARCH_HUGE_PAGE_SIZE = 0x20'0000;

boot/loader/x86_64/meson.build

@@ -0,0 +1,11 @@
+loader_sources += files(
+    'enter.S',
+    'paging.c',
+)
+
+loader_c_args += [
+    '-mno-red-zone',
+    '-masm=intel',
+]
+
+loader_objcopy_format = 'elf64-x86-64'

boot/loader/x86_64/paging.c

@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: ISC
+
+#include <bits/pmap.h>
+#include <x86_64/defs.h>
+#include "loader.h"
+#include "loader_arch.h"
+
+static bool next_table(uint64_t* table, size_t index, uint64_t** out) {
+    if (!(table[index] & ARCH_FLAG_PRESENT)) {
+        uint64_t page;
+        if (!loader_alloc(PHYS_RECLAIMABLE, PAGE_SIZE, &page))
+            return false;
+        table[index] = page | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+    }
+    *out = loader_ptr(table[index] & ARCH_PTE_ADDR_MASK);
+    return true;
+}
+
+bool arch_check_cpu() {
+    uint64_t cr4;
+    asm volatile("mov %0, cr4" : "=r"(cr4));
+    if (cr4 & CR4_LA57)
+        return false;
+    return true;
+}
+
+bool arch_paging_map(uint64_t root, uint64_t virt, uint64_t phys, uint64_t size, uint64_t page_size) {
+    const int leaf_level = page_size == ARCH_HUGE_PAGE_SIZE ? 1 : 0;
+
+    for (uint64_t off = 0; off < size; off += page_size) {
+        const uint64_t va = virt + off;
+        uint64_t* table = loader_ptr(root);
+        for (int level = 3; level > leaf_level; level--) {
+            if (!next_table(table, (va >> (12 + 9 * level)) & 0x1ff, &table))
+                return false;
+        }
+
+        uint64_t entry = (phys + off) | ARCH_FLAG_PRESENT | ARCH_FLAG_READ_WRITE;
+        if (leaf_level == 1)
+            entry |= ARCH_FLAG_SIZE;
+        table[(va >> (12 + 9 * leaf_level)) & 0x1ff] = entry;
+    }
+
+    return true;
+}

boot/meson.build

@@ -0,0 +1,44 @@
+objcopy = find_program('objcopy')
+zinnia_bootdir = zinnia_datadir / 'boot'
+
+loader_sources = files(
+    'loader/elf.c',
+    'loader/lib.c',
+    'loader/loader.c',
+)
+
+loader_c_args = [
+    '-ffreestanding',
+    '-nostdinc',
+    '-fno-stack-protector',
+    '-fno-omit-frame-pointer',
+    '-mgeneral-regs-only',
+]
+
+loader_includes = include_directories('loader', 'loader' / target_machine.cpu_family())
+
+loader_objcopy_format = 'unknown'
+
+subdir('loader' / target_machine.cpu_family())
+
+loader_kernel = custom_target(
+    'loader-kernel.o',
+    input: kernel,
+    output: 'loader-kernel.o',
+    command: [
+        objcopy,
+        '-I', 'binary',
+        '-O', loader_objcopy_format,
+        '--rename-section', '.data=.kernel,alloc,load,readonly,data,contents',
+        '@INPUT@',
+        '@OUTPUT@',
+    ],
+)
+
+if kconfig.get('CONFIG_BOOT_EFISTUB', 'n') == 'y'
+    subdir('efi')
+endif
+
+if kconfig.get('CONFIG_BOOT_LIMINE', 'n') == 'y'
+    subdir('limine')
+endif

dev/Kconfig

@@ -0,0 +1,19 @@
+menu "Device Drivers"
+
+config OF
+  bool "Support for OpenFirmware"
+  default y
+  help
+    Adds support for flattened device trees and OF platform drivers.
+
+config ACPI
+  bool "Support for ACPI firmware"
+  default y
+  help
+    Adds support for ACPI firmware, tables and devices.
+
+source "dev/block/Kconfig"
+source "dev/pci/Kconfig"
+source "dev/video/Kconfig"
+
+endmenu

dev/acpi/include/uacpi_types.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: ISC
+
 #pragma once
 
 #include <stdarg.h>

dev/acpi/include/zinnia/acpi.h

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/errno.h>
+#include <zinnia/init.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/types.h>
+#include <zinnia/workqueue.h>
+#include <stdint.h>
+
+INIT_EXTERN(acpi_stage);
+INIT_EXTERN(acpi_namespace_stage);
+
+extern phys_t acpi_rsdp_addr;
+extern struct workqueue* acpi_workqueue;
+
+errno_t acpi_reboot();
+errno_t acpi_power_off();
+
+struct acpi_match {
+    const char* hid;
+    uintptr_t data;
+};
+
+#define ACPI_MATCH_HID(id) .hid = (id)

dev/acpi/init.c

@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "acpi: " fmt
+
+#include <zinnia/acpi.h>
+#include <zinnia/alloc.h>
+#include <zinnia/cmdline.h>
+#include <zinnia/irq.h>
+#include <zinnia/pci.h>
+#include <zinnia/percpu.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/types.h>
+#include <zinnia/workqueue.h>
+#include <uacpi/event.h>
+#include <uacpi/sleep.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+#include <uacpi/uacpi.h>
+
+phys_t acpi_rsdp_addr = 0;
+struct workqueue* acpi_workqueue;
+
+static char table_buf[0x1000];
+
+static bool use_acpi = true;
+void use_acpi_option(bool use) {
+    use_acpi = use;
+}
+CMDLINE_OPTION("acpi", "Enable ACPI firmware subsystem", use_acpi_option);
+
+[[__init]]
+static bool acpi_stage_init(struct boot_info* info) {
+    if (!use_acpi) {
+        log_warn("ACPI disabled on the command line!\n");
+        return false;
+    }
+
+    if (!info->rsdp) {
+        log_warn("No RSDP was passed, unable to start ACPI subsystem!\n");
+        return false;
+    }
+    acpi_rsdp_addr = info->rsdp;
+
+    if (uacpi_setup_early_table_access(table_buf, sizeof(table_buf)) != UACPI_STATUS_OK) {
+        log_err("Failed to set up early table access!\n");
+        return false;
+    }
+
+    return true;
+}
+INIT_TASK(acpi_stage, "dev.acpi", acpi_stage_init);
+INIT_DEPENDS(acpi_stage, mem_stage);
+INIT_ENTAILS(acpi_stage, percpu_stage);
+
+static bool namespace_done;
+
+[[__init]]
+static void acpi_namespace_init(struct boot_info*) {
+    errno_t error = workqueue_create("acpi", SCHED_PRIO_WORK_HIGH, &acpi_workqueue);
+    if (error) {
+        log_err("Failed to create the ACPI workqueue (%i)\n", error);
+        return;
+    }
+
+    uacpi_status status = uacpi_initialize(0);
+    if (status != UACPI_STATUS_OK) {
+        log_err("Initialization failed with error \"%s\"!\n", uacpi_status_to_string(status));
+        return;
+    }
+
+    status = uacpi_namespace_load();
+    if (status != UACPI_STATUS_OK) {
+        log_err("Namespace loading failed with error \"%s\"!\n", uacpi_status_to_string(status));
+        return;
+    }
+
+    status = uacpi_namespace_initialize();
+    if (status != UACPI_STATUS_OK) {
+        log_err("Namespace initialization failed with error \"%s\"!\n", uacpi_status_to_string(status));
+        return;
+    }
+
+    status = uacpi_finalize_gpe_initialization();
+    if (status != UACPI_STATUS_OK)
+        log_warn("GPE initialization failed with error \"%s\"!\n", uacpi_status_to_string(status));
+
+    namespace_done = true;
+}
+INIT_TASK(acpi_namespace_stage, "dev.acpi.namespace", acpi_namespace_init);
+INIT_DEPENDS(acpi_namespace_stage, acpi_stage, workqueue_stage);
+#if CONFIG_PCI
+INIT_DEPENDS(acpi_namespace_stage, pci_stage);
+#endif
+
+errno_t acpi_reboot() {
+    if (!namespace_done)
+        return ENOSYS;
+
+    uacpi_status status = uacpi_reboot();
+    log_err("Failed to reset the system: %s\n", uacpi_status_to_string(status));
+    return EIO;
+}
+
+errno_t acpi_power_off() {
+    if (!namespace_done)
+        return ENOSYS;
+
+    uacpi_status status = uacpi_prepare_for_sleep_state(UACPI_SLEEP_STATE_S5);
+    if (status == UACPI_STATUS_OK) {
+        irq_lock();
+        status = uacpi_enter_sleep_state(UACPI_SLEEP_STATE_S5);
+        irq_unlock();
+    }
+    log_err("Failed to power off the system: %s\n", uacpi_status_to_string(status));
+    return EIO;
+}

dev/acpi/mcfg.c

@@ -0,0 +1,32 @@
+#define pr_fmt(fmt) "acpi: " fmt
+
+#include <zinnia/acpi.h>
+#include <zinnia/pci.h>
+#include <zinnia/print.h>
+#include <uacpi/acpi.h>
+#include <uacpi/status.h>
+#include <uacpi/tables.h>
+
+[[__init]]
+static void acpi_mcfg_stage_init(struct boot_info*) {
+    uacpi_table table = {};
+    if (uacpi_table_find_by_signature(ACPI_MCFG_SIGNATURE, &table) != UACPI_STATUS_OK)
+        return;
+
+    struct acpi_mcfg* mcfg = table.ptr;
+    size_t count = 0;
+    if (mcfg->hdr.length > sizeof(*mcfg))
+        count = (mcfg->hdr.length - sizeof(*mcfg)) / sizeof(struct acpi_mcfg_allocation);
+
+    for (size_t i = 0; i < count; i++) {
+        struct acpi_mcfg_allocation* entry = &mcfg->entries[i];
+        errno_t status = pci_ecam_register(entry->address, entry->segment, entry->start_bus, entry->end_bus);
+        if (status != 0)
+            log_warn("Failed to register ECAM for segment %hu: %i\n", entry->segment, status);
+    }
+
+    uacpi_table_unref(&table);
+}
+INIT_TASK(acpi_mcfg_stage, "dev.acpi.mcfg", acpi_mcfg_stage_init);
+INIT_DEPENDS(acpi_mcfg_stage, acpi_stage);
+INIT_ENTAILS(acpi_mcfg_stage, pci_stage);

dev/acpi/meson.build

@@ -0,0 +1,19 @@
+uacpi = subproject('uacpi')
+
+kernel_sources += uacpi.get_variable('sources')
+kernel_sources += files(
+    'init.c',
+    'uacpi_glue.c',
+)
+
+if kconfig.get('CONFIG_PCI', 'n') == 'y'
+    kernel_sources += files('mcfg.c')
+endif
+
+kernel_includes += uacpi.get_variable('includes')
+kernel_includes += include_directories('include')
+
+kernel_c_args += [
+    '-DUACPI_FORMATTED_LOGGING',
+    '-DUACPI_OVERRIDE_TYPES',
+]

dev/acpi/uacpi_glue.c

@@ -0,0 +1,332 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/acpi.h>
+#include <zinnia/alloc.h>
+#include <zinnia/clock.h>
+#include <zinnia/console.h>
+#include <zinnia/init.h>
+#include <zinnia/irq.h>
+#include <zinnia/mmio.h>
+#include <zinnia/pci.h>
+#include <zinnia/percpu.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/sched.h>
+#include <zinnia/semaphore.h>
+#include <zinnia/spin.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <zinnia/workqueue.h>
+#include <arch/irq.h>
+#include <arch/mem.h>
+#include <stdarg.h>
+#include <uacpi/kernel_api.h>
+#include <uacpi/status.h>
+#include <uacpi/types.h>
+
+uacpi_status uacpi_kernel_get_rsdp(uacpi_phys_addr* out_rsdp_address) {
+    *out_rsdp_address = (uacpi_phys_addr)acpi_rsdp_addr;
+    return UACPI_STATUS_OK;
+}
+
+void* uacpi_kernel_map(uacpi_phys_addr addr, uacpi_size len) {
+    uintptr_t aligned_addr = ALIGN_DOWN(addr, arch_mem_page_size());
+    uintptr_t difference = addr - aligned_addr;
+    size_t aligned_len = ALIGN_UP(len + difference, arch_mem_page_size());
+
+    uintptr_t vaddr = vm_alloc(aligned_len);
+    if (!vaddr)
+        return nullptr;
+
+    for (size_t i = 0; i < aligned_len; i += arch_mem_page_size()) {
+        pmap_map(&kernel_vma.pmap, vaddr + i, aligned_addr + i, PTE_READ | PTE_WRITE, CACHE_ANY);
+    }
+
+    return (void*)(vaddr + difference);
+}
+
+void uacpi_kernel_unmap(void* addr, uacpi_size len) {
+    uintptr_t aligned_addr = ALIGN_DOWN((uintptr_t)addr, arch_mem_page_size());
+    uintptr_t difference = (uintptr_t)addr - aligned_addr;
+    size_t aligned_len = ALIGN_UP(len + difference, arch_mem_page_size());
+
+    for (size_t i = 0; i < aligned_len; i += arch_mem_page_size())
+        pmap_unmap(&kernel_vma.pmap, aligned_addr + i);
+    vm_free(aligned_addr, aligned_len);
+}
+
+static enum log_level acpi_log_level(uacpi_log_level lvl) {
+    switch (lvl) {
+    case UACPI_LOG_ERROR:
+        return LOG_ERR;
+
+    case UACPI_LOG_WARN:
+        return LOG_WARN;
+
+    case UACPI_LOG_INFO:
+        return LOG_INFO;
+
+    default:
+        return LOG_DEBUG;
+    }
+}
+
+void uacpi_kernel_log(uacpi_log_level lvl, const uacpi_char* msg, ...) {
+    va_list args;
+    va_start(args, msg);
+    log_vwrite_prefixed(acpi_log_level(lvl), "acpi: ", msg, args);
+    va_end(args);
+}
+
+void uacpi_kernel_vlog(uacpi_log_level lvl, const uacpi_char* msg, uacpi_va_list args) {
+    log_vwrite_prefixed(acpi_log_level(lvl), "acpi: ", msg, args);
+}
+
+uacpi_status uacpi_kernel_pci_device_open(uacpi_pci_address address, uacpi_handle* out_handle) {
+#if CONFIG_PCI
+    struct pci_address addr = {
+        .domain = address.segment,
+        .bus = address.bus,
+        .device = address.device,
+        .function = address.function,
+    };
+    struct pci_access* access = pci_access_find(&addr);
+    if (!access)
+        return UACPI_STATUS_NOT_FOUND;
+
+    struct pci_device* dev = mem_alloc(sizeof(*dev), 0);
+    if (!dev)
+        return UACPI_STATUS_OUT_OF_MEMORY;
+    dev->address = addr;
+    dev->access = access;
+    *out_handle = dev;
+    return UACPI_STATUS_OK;
+#else
+    return UACPI_STATUS_NOT_FOUND;
+#endif
+}
+
+void uacpi_kernel_pci_device_close(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+#if CONFIG_PCI
+#define PCI_READ(width) \
+    uacpi_status uacpi_kernel_pci_read##width(uacpi_handle device, uacpi_size offset, uacpi_u##width* value) { \
+        *value = pci_read##width(device, offset); \
+        return UACPI_STATUS_OK; \
+    }
+#define PCI_WRITE(width) \
+    uacpi_status uacpi_kernel_pci_write##width(uacpi_handle device, uacpi_size offset, uacpi_u##width value) { \
+        pci_write##width(device, offset, value); \
+        return UACPI_STATUS_OK; \
+    }
+#else
+#define PCI_READ(width) \
+    uacpi_status uacpi_kernel_pci_read##width(uacpi_handle device, uacpi_size offset, uacpi_u##width* value) { \
+        return UACPI_STATUS_UNIMPLEMENTED; \
+    }
+#define PCI_WRITE(width) \
+    uacpi_status uacpi_kernel_pci_write##width(uacpi_handle device, uacpi_size offset, uacpi_u##width value) { \
+        return UACPI_STATUS_UNIMPLEMENTED; \
+    }
+#endif
+
+PCI_READ(8)
+PCI_READ(16)
+PCI_READ(32)
+PCI_WRITE(8)
+PCI_WRITE(16)
+PCI_WRITE(32)
+
+void* uacpi_kernel_alloc(uacpi_size size) {
+    return mem_alloc(size, ALLOC_NOZERO);
+}
+
+void uacpi_kernel_free(void* mem) {
+    if (mem)
+        mem_free(mem);
+}
+
+uacpi_u64 uacpi_kernel_get_nanoseconds_since_boot() {
+    return clock_get_elapsed_ns();
+}
+
+void uacpi_kernel_stall(uacpi_u8 usec) {
+    clock_spin_ns(usec * 1000ull);
+}
+
+void uacpi_kernel_sleep(uacpi_u64 msec) {
+    sched_sleep_until(clock_get_elapsed_ns() + msec * 1'000'000);
+}
+
+uacpi_handle uacpi_kernel_create_semaphore(uacpi_u32 initial_units) {
+    sem_t* sem = mem_alloc(sizeof(*sem), 0);
+    if (sem)
+        sem_init(sem, initial_units);
+    return sem;
+}
+
+void uacpi_kernel_free_semaphore(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+uacpi_status uacpi_kernel_wait_for_semaphore(uacpi_handle handle, uacpi_u16 timeout) {
+    sem_t* sem = handle;
+    if (timeout == 0)
+        return sem_trywait(sem) ? UACPI_STATUS_OK : UACPI_STATUS_TIMEOUT;
+    if (timeout == 0xFFFF) {
+        sem_wait(sem);
+        return UACPI_STATUS_OK;
+    }
+
+    const uint64_t deadline = clock_get_elapsed_ns() + timeout * 1'000'000ull;
+    return sem_timedwait(sem, &deadline) ? UACPI_STATUS_OK : UACPI_STATUS_TIMEOUT;
+}
+
+void uacpi_kernel_signal_semaphore(uacpi_handle handle) {
+    sem_post(handle);
+}
+
+uacpi_thread_id uacpi_kernel_get_thread_id() {
+    return percpu_get()->sched.current;
+}
+
+uacpi_interrupt_state uacpi_kernel_disable_interrupts() {
+    irq_lock();
+    return 0;
+}
+
+void uacpi_kernel_restore_interrupts(uacpi_interrupt_state) {
+    irq_unlock();
+}
+
+uacpi_status uacpi_kernel_handle_firmware_request(uacpi_firmware_request* req) {
+    switch (req->type) {
+    case UACPI_FIRMWARE_REQUEST_TYPE_BREAKPOINT:
+        log_warn("acpi: Ignoring AML breakpoint\n");
+        break;
+    case UACPI_FIRMWARE_REQUEST_TYPE_FATAL:
+        log_err(
+            "acpi: Fatal firmware error: type %u, code %#x, arg %#llx\n",
+            req->fatal.type,
+            req->fatal.code,
+            (unsigned long long)req->fatal.arg
+        );
+        break;
+    }
+    return UACPI_STATUS_OK;
+}
+
+struct acpi_irq {
+    struct irq_handler handler;
+    uacpi_interrupt_handler func;
+    uacpi_handle ctx;
+};
+
+static enum irq_status acpi_irq_raise(struct irq_handler* self) {
+    struct acpi_irq* irq = containerof(self, struct acpi_irq, handler);
+    return irq->func(irq->ctx) == UACPI_INTERRUPT_HANDLED ? IRQ_STATUS_HANDLED : IRQ_STATUS_IGNORED;
+}
+
+uacpi_status uacpi_kernel_install_interrupt_handler(
+    uacpi_u32 irq,
+    uacpi_interrupt_handler handler,
+    uacpi_handle ctx,
+    uacpi_handle* out_irq_handle
+) {
+    uint32_t gsi = irq;
+    enum irq_trigger_mode trigger = IRQ_TRIGGER_LEVEL;
+    enum irq_polarity polarity = IRQ_POLARITY_LOW;
+    if (irq <= UINT8_MAX)
+        arch_irq_isa_override(irq, &gsi, &trigger, &polarity);
+
+    struct irq_line* line;
+    errno_t status = arch_irq_get_gsi(gsi, &line);
+    if (status) {
+        log_err("acpi: Unable to get GSI %u for IRQ %u (%i)\n", gsi, irq, status);
+        return UACPI_STATUS_UNIMPLEMENTED;
+    }
+
+    struct acpi_irq* acpi_irq = mem_alloc(sizeof(*acpi_irq), 0);
+    if (!acpi_irq)
+        return UACPI_STATUS_OUT_OF_MEMORY;
+    acpi_irq->handler.raise = acpi_irq_raise;
+    acpi_irq->handler.line = line;
+    acpi_irq->func = handler;
+    acpi_irq->ctx = ctx;
+
+    irq_line_attach(line, &acpi_irq->handler);
+    irq_line_program(line, trigger, polarity);
+    log("acpi: Installed handler for IRQ %u on GSI %u\n", irq, gsi);
+
+    *out_irq_handle = acpi_irq;
+    return UACPI_STATUS_OK;
+}
+
+uacpi_status uacpi_kernel_uninstall_interrupt_handler(uacpi_interrupt_handler handler, uacpi_handle irq_handle) {
+    struct acpi_irq* acpi_irq = irq_handle;
+    struct irq_line* line = acpi_irq->handler.line;
+    irq_line_detach(line, &acpi_irq->handler);
+    if (list_is_empty(&line->handlers) && line->mask)
+        line->mask(line);
+    mem_free(acpi_irq);
+    return UACPI_STATUS_OK;
+}
+
+uacpi_handle uacpi_kernel_create_spinlock() {
+    struct spinlock* lock = mem_alloc(sizeof(*lock), 0);
+    if (lock)
+        spin_init(lock);
+    return lock;
+}
+
+void uacpi_kernel_free_spinlock(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+uacpi_cpu_flags uacpi_kernel_lock_spinlock(uacpi_handle handle) {
+    spin_lock(handle);
+    return 0;
+}
+
+void uacpi_kernel_unlock_spinlock(uacpi_handle handle, uacpi_cpu_flags) {
+    spin_unlock(handle);
+}
+
+struct acpi_work {
+    struct work work;
+    uacpi_work_handler handler;
+    uacpi_handle ctx;
+};
+
+static void acpi_work_fn(struct work* work) {
+    struct acpi_work* item = containerof(work, struct acpi_work, work);
+    const uacpi_work_handler handler = item->handler;
+    const uacpi_handle ctx = item->ctx;
+    handler(ctx);
+}
+
+uacpi_handle uacpi_kernel_create_work_item() {
+    struct acpi_work* item = mem_alloc(sizeof(*item), 0);
+    if (item)
+        work_init(&item->work, acpi_work_fn);
+    return item;
+}
+
+void uacpi_kernel_free_work_item(uacpi_handle handle) {
+    mem_free(handle);
+}
+
+void uacpi_kernel_schedule_work(uacpi_work_type, uacpi_handle work_item, uacpi_work_handler handler, uacpi_handle ctx) {
+    struct acpi_work* item = work_item;
+    item->handler = handler;
+    item->ctx = ctx;
+    work_queue(acpi_workqueue, &item->work);
+}
+
+uacpi_status uacpi_kernel_wait_for_work_completion() {
+    workqueue_flush(acpi_workqueue);
+    return UACPI_STATUS_OK;
+}

dev/block/Kconfig

@@ -0,0 +1,5 @@
+menu "Block device drivers"
+
+source "dev/block/nvme/Kconfig"
+
+endmenu

dev/block/bio.c

@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/utils.h>
+
+struct bio* bio_alloc(enum bio_op op, size_t num_segs) {
+    struct bio* bio = mem_alloc(sizeof(struct bio) + num_segs * sizeof(struct bio_seg), 0);
+    if (!bio)
+        return nullptr;
+    bio->op = op;
+    bio->num_segs = num_segs;
+    event_init(&bio->event);
+    atomic_init(&bio->pending, 0);
+    atomic_init(&bio->short_offset, UINT64_MAX);
+    atomic_init(&bio->child_status, 0);
+    return bio;
+}
+
+void bio_free(struct bio* bio) {
+    mem_free(bio);
+}
+
+static void record_short(struct bio* parent, uint64_t offset, errno_t status) {
+    uint64_t current = atomic_load(&parent->short_offset, ATOMIC_RELAXED);
+    while (offset < current) {
+        if (atomic_compare_exchange_weak(&parent->short_offset, &current, offset, ATOMIC_RELAXED, ATOMIC_RELAXED))
+            break;
+    }
+    if (status) {
+        errno_t expected = 0;
+        atomic_compare_exchange(&parent->child_status, &expected, status, ATOMIC_RELAXED, ATOMIC_RELAXED);
+    }
+}
+
+static void parent_put(struct bio* parent) {
+    if (atomic_fetch_sub(&parent->pending, 1, ATOMIC_ACQ_REL) != 1)
+        return;
+    const uint64_t offset = atomic_load(&parent->short_offset, ATOMIC_RELAXED);
+    const size_t done = offset == UINT64_MAX ? parent->num_lbas : offset;
+    const errno_t status = atomic_load(&parent->child_status, ATOMIC_RELAXED);
+    bio_complete(parent, done == 0 ? status : 0, done);
+}
+
+void bio_complete(struct bio* bio, errno_t status, size_t done_lbas) {
+    bio->status = status;
+    bio->done_lbas = done_lbas;
+
+    struct bio* parent = bio->parent;
+    if (parent) {
+        if (status || done_lbas < bio->num_lbas)
+            record_short(parent, bio->parent_offset + done_lbas, status);
+        bio_free(bio);
+        parent_put(parent);
+        return;
+    }
+
+    if (bio->end)
+        bio->end(bio);
+    else
+        event_set(&bio->event);
+}
+
+errno_t bio_wait(struct bio* bio, size_t* out_lbas) {
+    event_wait(&bio->event);
+    if (out_lbas)
+        *out_lbas = bio->done_lbas;
+    return bio->status;
+}
+
+// Submits one bio and completes early if it failed.
+static void submit_one(struct blockdev* dev, struct bio* bio) {
+    errno_t status = dev->ops->submit(dev, bio);
+    if (status)
+        bio_complete(bio, status, 0);
+}
+
+void bio_submit(struct blockdev* dev, struct bio* bio) {
+    bio->dev = dev;
+    if (bio->op == BIO_FLUSH) {
+        submit_one(dev, bio);
+        return;
+    }
+
+    if (bio->op == BIO_READ) {
+        if (bio->lba >= dev->lba_count) {
+            bio_complete(bio, 0, 0);
+            return;
+        }
+        bio->num_lbas = MIN(bio->num_lbas, dev->lba_count - bio->lba);
+    }
+    if (bio->num_lbas == 0) {
+        bio_complete(bio, 0, 0);
+        return;
+    }
+
+    const size_t lba_size = dev->lba_size;
+    const size_t lbas_per_page = MAX(arch_mem_page_size() / lba_size, 1);
+    size_t max_lbas = dev->limits.max_lbas ? dev->limits.max_lbas : SIZE_MAX;
+    if (max_lbas > lbas_per_page)
+        max_lbas -= max_lbas % lbas_per_page;
+    const size_t max_segs = dev->limits.max_segs ? dev->limits.max_segs : SIZE_MAX;
+
+    if (bio->num_lbas <= max_lbas && bio->num_segs <= max_segs) {
+        submit_one(dev, bio);
+        return;
+    }
+
+    atomic_store(&bio->pending, 1, ATOMIC_RELAXED);
+    atomic_store(&bio->short_offset, UINT64_MAX, ATOMIC_RELAXED);
+    atomic_store(&bio->child_status, 0, ATOMIC_RELAXED);
+
+    size_t seg = 0;
+    size_t seg_off = 0;
+    uint64_t consumed = 0;
+    while (consumed < bio->num_lbas) {
+        const size_t cap = MIN(max_segs, bio->num_segs - seg);
+        struct bio* child = cap ? bio_alloc(bio->op, cap) : nullptr;
+        if (!child) {
+            record_short(bio, consumed, cap ? ENOMEM : EINVAL);
+            break;
+        }
+
+        size_t count = 0;
+        size_t lbas = 0;
+        while (lbas < max_lbas && count < cap && consumed + lbas < bio->num_lbas) {
+            const struct bio_seg* src = &bio->segs[seg];
+            const size_t take =
+                MIN(MIN(src->len - seg_off, (max_lbas - lbas) * lba_size),
+                    (bio->num_lbas - consumed - lbas) * lba_size);
+            if (take < lba_size)
+                break;
+            child->segs[count++] = (struct bio_seg){.addr = src->addr + seg_off, .len = take};
+            lbas += take / lba_size;
+            seg_off += take;
+            if (seg_off == src->len) {
+                seg++;
+                seg_off = 0;
+            }
+        }
+
+        if (lbas == 0) {
+            bio_free(child);
+            record_short(bio, consumed, EINVAL);
+            break;
+        }
+
+        child->num_segs = count;
+        child->num_lbas = lbas;
+        child->lba = bio->lba + consumed;
+        child->parent = bio;
+        child->parent_offset = consumed;
+        child->dev = dev;
+        consumed += lbas;
+        atomic_fetch_add(&bio->pending, 1, ATOMIC_RELAXED);
+        submit_one(dev, child);
+    }
+
+    parent_put(bio);
+}

dev/block/blockdev.c

@@ -0,0 +1,490 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "block: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+#include <zinnia/initgraph.h>
+#include <zinnia/print.h>
+#include <zinnia/tailq.h>
+#include <zinnia/usercopy.h>
+#include <zinnia/utils.h>
+#include <zinnia/vec.h>
+#include <zinnia/vma.h>
+#include <string.h>
+#include <uapi/block.h>
+
+static size_t lbas_per_page(struct blockdev* dev) {
+    return arch_mem_page_size() / dev->lba_size;
+}
+
+static errno_t cache_get_pages(struct vmo* vmo, uint64_t index, struct page** pages, size_t count, size_t* out_valid) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    const size_t page_size = arch_mem_page_size();
+    const size_t per_page = lbas_per_page(dev);
+    const uint64_t lba = index * per_page;
+
+    if (lba >= dev->lba_count) {
+        *out_valid = 0;
+        return 0;
+    }
+
+    const size_t num_lbas = MIN((uint64_t)count * per_page, dev->lba_count - lba);
+    size_t done;
+    errno_t status = blockdev_rw_pages(dev, BIO_READ, lba, num_lbas, pages, count, &done);
+    if (status)
+        return status;
+
+    const size_t valid = done / per_page + (done % per_page != 0);
+    if (done % per_page != 0) {
+        const size_t tail = (done % per_page) * dev->lba_size;
+        memset((uint8_t*)HHDM_PTR(vm_page_addr(pages[valid - 1])) + tail, 0, page_size - tail);
+    }
+    *out_valid = valid;
+    return valid == 0 && num_lbas > 0 ? EIO : 0;
+}
+
+static errno_t cache_put_pages(struct vmo* vmo, uint64_t index, struct page** pages, size_t count) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    const size_t per_page = lbas_per_page(dev);
+    const uint64_t lba = index * per_page;
+    if (lba >= dev->lba_count)
+        return 0;
+
+    const size_t num_lbas = MIN((uint64_t)count * per_page, dev->lba_count - lba);
+    size_t done;
+    errno_t status = blockdev_rw_pages(dev, BIO_WRITE, lba, num_lbas, pages, count, &done);
+    if (status)
+        return status;
+    return done < num_lbas ? EIO : 0;
+}
+
+static void cache_destroy(struct vmo* vmo) {
+    struct blockdev* dev = containerof(vmo, struct blockdev, cache);
+    if (dev->ops->destroy)
+        dev->ops->destroy(dev);
+}
+
+static const struct vmo_pager_ops cache_pager = {
+    .get_pages = cache_get_pages,
+    .put_pages = cache_put_pages,
+    .max_cluster = 64,
+    .destroy = cache_destroy,
+};
+
+void blockdev_init(
+    struct blockdev* dev,
+    const struct blockdev_ops* ops,
+    uint32_t major,
+    uint32_t minor,
+    size_t lba_size,
+    uint64_t lba_count
+) {
+    vmo_init(&dev->cache, &cache_pager);
+    dev->ops = ops;
+    dev->dev = dev_make(major, minor);
+    dev->lba_size = lba_size;
+    dev->lba_count = lba_count;
+    atomic_init(&dev->claimed, false);
+}
+
+void blockdev_ref(struct blockdev* dev) {
+    vmo_ref(&dev->cache);
+}
+
+void blockdev_unref(struct blockdev* dev) {
+    vmo_unref(&dev->cache);
+}
+
+errno_t blockdev_rw(
+    struct blockdev* dev,
+    enum bio_op op,
+    uint64_t lba,
+    size_t num_lbas,
+    const struct bio_seg* segs,
+    size_t num_segs,
+    size_t* out_lbas
+) {
+    struct bio* bio = bio_alloc(op, num_segs);
+    if (!bio)
+        return ENOMEM;
+    memcpy(bio->segs, segs, num_segs * sizeof(struct bio_seg));
+    bio->lba = lba;
+    bio->num_lbas = num_lbas;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, out_lbas);
+    bio_free(bio);
+    return status;
+}
+
+errno_t blockdev_rw_pages(
+    struct blockdev* dev,
+    enum bio_op op,
+    uint64_t lba,
+    size_t num_lbas,
+    struct page** pages,
+    size_t count,
+    size_t* out_lbas
+) {
+    const size_t page_size = arch_mem_page_size();
+    size_t bytes = num_lbas * dev->lba_size;
+    if (bytes > count * page_size)
+        return EINVAL;
+
+    struct bio* bio = bio_alloc(op, count);
+    if (!bio)
+        return ENOMEM;
+
+    size_t num_segs = 0;
+    for (size_t i = 0; i < count && bytes > 0; i++) {
+        const size_t len = MIN(bytes, page_size);
+        bio->segs[num_segs++] = (struct bio_seg){.addr = vm_page_addr(pages[i]), .len = len};
+        bytes -= len;
+    }
+    bio->num_segs = num_segs;
+    bio->lba = lba;
+    bio->num_lbas = num_lbas;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, out_lbas);
+    bio_free(bio);
+    return status;
+}
+
+errno_t blockdev_flush(struct blockdev* dev) {
+    struct bio* bio = bio_alloc(BIO_FLUSH, 0);
+    if (!bio)
+        return ENOMEM;
+    bio_submit(dev, bio);
+    errno_t status = bio_wait(bio, nullptr);
+    bio_free(bio);
+    return status;
+}
+
+static struct blockdev* file_dev(struct vfs_file* file) {
+    return file->priv;
+}
+
+static uint64_t dev_size(struct blockdev* dev) {
+    return dev->lba_count * dev->lba_size;
+}
+
+static errno_t blockdev_file_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    struct blockdev* dev = file_dev(file);
+    *out_read = 0;
+    if (offset >= dev_size(dev))
+        return 0;
+
+    size_t progress;
+    errno_t status =
+        vmo_read_iter(&dev->cache, offset, buffer, MIN(iovec_iter_len(buffer), dev_size(dev) - offset), &progress);
+    *out_read = progress;
+    return progress > 0 ? 0 : status;
+}
+
+static errno_t blockdev_file_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    struct blockdev* dev = file_dev(file);
+    *out_written = 0;
+    if (iovec_iter_len(buffer) == 0)
+        return 0;
+    if (offset >= dev_size(dev))
+        return ENOSPC;
+
+    size_t progress;
+    errno_t status =
+        vmo_write_iter(&dev->cache, offset, buffer, MIN(iovec_iter_len(buffer), dev_size(dev) - offset), &progress);
+    *out_written = progress;
+    return progress > 0 ? 0 : status;
+}
+
+static errno_t blockdev_file_ioctl(struct vfs_file* file, uintptr_t request, uintptr_t arg, uintptr_t* out_result) {
+    struct blockdev* dev = file_dev(file);
+    switch (request) {
+    case __BLKGETSIZE64: {
+        const uint64_t size = dev_size(dev);
+        if (!usercopy_write((__user void*)arg, &size, sizeof(size)))
+            return EFAULT;
+        *out_result = 0;
+        return 0;
+    }
+    case __BLKSSZGET: {
+        const unsigned int lba_size = dev->lba_size;
+        if (!usercopy_write((__user void*)arg, &lba_size, sizeof(lba_size)))
+            return EFAULT;
+        *out_result = 0;
+        return 0;
+    }
+    case __BLKRRPART: {
+        errno_t status = blockdev_rescan(dev);
+        if (!status)
+            *out_result = 0;
+        return status;
+    }
+    default:
+        if (dev->ops->ioctl)
+            return dev->ops->ioctl(dev, file, request, arg, out_result);
+        return ENOTTY;
+    }
+}
+
+static errno_t blockdev_file_sync(struct vfs_file* file, bool data_only) {
+    struct blockdev* dev = file_dev(file);
+    errno_t status = vmo_sync(&dev->cache, 0, UINT64_MAX);
+    if (status)
+        return status;
+    return blockdev_flush(dev);
+}
+
+static errno_t blockdev_file_mmap(
+    struct vfs_file* file,
+    struct vma* vma,
+    uintptr_t addr,
+    size_t len,
+    enum prot_flags prot,
+    enum map_flags flags,
+    off_t offset,
+    uintptr_t* out_addr
+) {
+    errno_t status =
+        vma_map(vma, &file_dev(file)->cache, addr, len, prot, (flags & MAP_SHARED) ? VMA_SHARED : 0, offset);
+    if (status)
+        return status;
+    *out_addr = addr;
+    return 0;
+}
+
+static void blockdev_file_release(struct vfs_file* file) {
+    blockdev_unref(file_dev(file));
+}
+
+static const struct vfs_file_ops blockdev_file_ops = {
+    .read = blockdev_file_read,
+    .write = blockdev_file_write,
+    .ioctl = blockdev_file_ioctl,
+    .sync = blockdev_file_sync,
+    .mmap = blockdev_file_mmap,
+    .release = blockdev_file_release,
+};
+
+static errno_t blockdev_node_open(struct vfs_node* node, struct vfs_file* file) {
+    struct blockdev* dev = node->priv;
+    blockdev_ref(dev);
+    file->priv = dev;
+    file->ops = &blockdev_file_ops;
+    return 0;
+}
+
+static void blockdev_node_destroy(struct vfs_node* node) {
+    blockdev_unref(node->priv);
+}
+
+static const struct vfs_node_ops blockdev_node_ops = {
+    .destroy = blockdev_node_destroy,
+    .device = {
+        .open = blockdev_node_open,
+    },
+};
+
+errno_t devtmpfs_add_blockdev(const char* name, struct blockdev* dev, mode_t mode) {
+    struct vfs_mknod_target target = {
+        .type = VNODE_BLOCKDEV,
+        .ops = &blockdev_node_ops,
+        .priv = dev,
+        .rdev = dev->dev,
+    };
+
+    blockdev_ref(dev);
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mknod(root, root, name, mode, &target, ident_kernel(), nullptr);
+    vfs_path_drop(root);
+    if (status) {
+        blockdev_unref(dev);
+        return status;
+    }
+    devctl_notify_create(name);
+    return 0;
+}
+
+errno_t blockdev_from_node(struct vfs_node* node, struct blockdev** out) {
+    if (node->type != VNODE_BLOCKDEV || node->ops != &blockdev_node_ops)
+        return ENOTBLK;
+
+    struct blockdev* dev = node->priv;
+    blockdev_ref(dev);
+    *out = dev;
+    return 0;
+}
+
+errno_t blockdev_claim(struct blockdev* dev) {
+    bool expected = false;
+    if (!atomic_compare_exchange(&dev->claimed, &expected, true, ATOMIC_ACQUIRE, ATOMIC_RELAXED))
+        return EBUSY;
+    return 0;
+}
+
+void blockdev_release(struct blockdev* dev) {
+    atomic_store(&dev->claimed, false, ATOMIC_RELEASE);
+}
+
+struct disk_node {
+    char name[64];
+    struct blockdev* dev;
+};
+
+struct disk {
+    char name[32];
+    struct blockdev* dev;
+    vec(struct disk_node) nodes;
+    tailq_link(struct disk) link;
+};
+
+static tailq_head(struct disk) disks;
+static mutex_t disks_lock;
+
+static void add_node(struct disk* disk, const char* name, struct blockdev* dev) {
+    struct disk_node node = {.dev = dev};
+    ksnprintf(node.name, sizeof(node.name), "%s", name);
+    if (vec_push(&disk->nodes, node) && dev)
+        blockdev_unref(dev);
+}
+
+static void scan_partitions(struct disk* disk) {
+    struct gpt_partition* parts;
+    size_t count;
+    if (gpt_scan(disk->dev, &parts, &count))
+        return;
+
+    for (size_t i = 0; i < count; i++) {
+        const struct gpt_partition* part = &parts[i];
+        if (part->end_lba < part->start_lba || part->end_lba >= disk->dev->lba_count) {
+            log_warn("Ignoring partition %zu of \"%s\" with invalid range\n", i + 1, disk->name);
+            continue;
+        }
+
+        char part_name[48];
+        ksnprintf(part_name, sizeof(part_name), "%sp%zu", disk->name, i + 1);
+
+        struct blockdev* dev;
+        errno_t status = partition_new(disk->dev, part->start_lba, part->end_lba - part->start_lba + 1, &dev);
+        if (status) {
+            log_warn("Unable to create partition \"%s\" (%i)\n", part_name, status);
+            continue;
+        }
+
+        char path[64];
+        ksnprintf(path, sizeof(path), "block/%s", part_name);
+        status = devtmpfs_add_blockdev(path, dev, 0660);
+        if (status) {
+            blockdev_unref(dev);
+            log_warn("Unable to register \"%s\" (%i)\n", path, status);
+            continue;
+        }
+        add_node(disk, path, dev);
+
+        char type_str[GUID_STR_LEN];
+        char uuid_str[GUID_STR_LEN];
+        guid_format(&part->type_guid, type_str);
+        guid_format(&part->unique_guid, uuid_str);
+
+        ksnprintf(path, sizeof(path), "block/parttype-%s", type_str);
+        status = devtmpfs_symlink(path, part_name);
+        if (!status)
+            add_node(disk, path, nullptr);
+        else if (status == EEXIST)
+            log("Partition type %s already claimed, no \"%s\" link for \"%s\"\n", type_str, path, part_name);
+
+        ksnprintf(path, sizeof(path), "block/partuuid-%s", uuid_str);
+        status = devtmpfs_symlink(path, part_name);
+        if (!status)
+            add_node(disk, path, nullptr);
+
+        log("Partition %zu: \"%s\" Type: %s UUID: %s\n", i + 1, part_name, type_str, uuid_str);
+    }
+
+    mem_free(parts);
+}
+
+errno_t blockdev_register(const char* name, struct blockdev* dev) {
+    struct disk* disk = mem_alloc(sizeof(struct disk), 0);
+    if (!disk)
+        return ENOMEM;
+    ksnprintf(disk->name, sizeof(disk->name), "%s", name);
+    vec_init(&disk->nodes);
+
+    char path[64];
+    ksnprintf(path, sizeof(path), "block/%s", name);
+    errno_t status = devtmpfs_add_blockdev(path, dev, 0660);
+    if (status) {
+        mem_free(disk);
+        return status;
+    }
+
+    blockdev_ref(dev);
+    disk->dev = dev;
+    log("Registered block device \"%s\"\n", name);
+
+    mutex_lock(&disks_lock);
+    tailq_insert_tail(&disks, disk, link);
+    scan_partitions(disk);
+    mutex_unlock(&disks_lock);
+    return 0;
+}
+
+errno_t blockdev_rescan(struct blockdev* dev) {
+    mutex_lock(&disks_lock);
+    struct disk* disk;
+    for (disk = tailq_first(&disks); disk; disk = tailq_next(disk, link)) {
+        if (disk->dev == dev)
+            break;
+    }
+    if (!disk) {
+        mutex_unlock(&disks_lock);
+        return ENOTTY;
+    }
+
+    for (size_t i = 0; i < vec_length(&disk->nodes); i++) {
+        struct disk_node* node = &vec_elem_get(&disk->nodes, i);
+        errno_t status = devtmpfs_remove(node->name);
+        if (status)
+            log_warn("Unable to remove stale node \"%s\" (%i)\n", node->name, status);
+        if (node->dev)
+            blockdev_unref(node->dev);
+    }
+    disk->nodes.length = 0;
+
+    scan_partitions(disk);
+    mutex_unlock(&disks_lock);
+    return 0;
+}
+
+errno_t blockdev_sync_all() {
+    errno_t result = 0;
+    mutex_lock(&disks_lock);
+    for (struct disk* disk = tailq_first(&disks); disk; disk = tailq_next(disk, link)) {
+        for (size_t i = 0; i < vec_length(&disk->nodes); i++) {
+            struct blockdev* part = vec_elem_get(&disk->nodes, i).dev;
+            errno_t status = part ? vmo_sync(&part->cache, 0, UINT64_MAX) : 0;
+            if (status && !result)
+                result = status;
+        }
+        errno_t status = vmo_sync(&disk->dev->cache, 0, UINT64_MAX);
+        if (!status)
+            status = blockdev_flush(disk->dev);
+        if (status && !result)
+            result = status;
+    }
+    mutex_unlock(&disks_lock);
+    return result;
+}
+
+static void block_stage_init(struct boot_info* info) {
+    tailq_init(&disks);
+    mutex_init(&disks_lock);
+    errno_t status = devtmpfs_mkdir("block", 0755);
+    if (status && status != EEXIST)
+        log_warn("Unable to create /dev/block (%i)\n", status);
+}
+
+INIT_TASK(block_stage, "dev.block", block_stage_init);
+INIT_DEPENDS(block_stage, devtmpfs_stage);

dev/block/gpt.c

@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/blockdev.h>
+#include <string.h>
+
+struct [[__packed]] gpt_header {
+    char signature[8];
+    uint32_t revision;
+    uint32_t header_size;
+    uint32_t header_crc32;
+    uint32_t reserved;
+    uint64_t my_lba;
+    uint64_t alternate_lba;
+    uint64_t first_usable_lba;
+    uint64_t last_usable_lba;
+    struct guid disk_guid;
+    uint64_t partition_entry_lba;
+    uint32_t num_partition_entries;
+    uint32_t partition_entry_size;
+    uint32_t partition_entry_array_crc32;
+};
+
+struct [[__packed]] gpt_entry {
+    struct guid type_guid;
+    struct guid unique_guid;
+    uint64_t start_lba;
+    uint64_t end_lba;
+    uint64_t attributes;
+    uint16_t name[36];
+};
+
+constexpr size_t GPT_MAX_ENTRIES = 1024;
+
+errno_t gpt_scan(struct blockdev* dev, struct gpt_partition** out, size_t* out_count) {
+    struct gpt_header header;
+    size_t read;
+    errno_t status = vmo_read(&dev->cache, dev->lba_size, &header, sizeof(header), &read);
+    if (status)
+        return status;
+    if (read != sizeof(header) || memcmp(header.signature, "EFI PART", 8) != 0)
+        return ENODATA;
+    if (header.partition_entry_size < sizeof(struct gpt_entry) || header.num_partition_entries > GPT_MAX_ENTRIES)
+        return EINVAL;
+
+    struct gpt_partition* parts = mem_alloc(MAX(header.num_partition_entries, 1) * sizeof(struct gpt_partition), 0);
+    if (!parts)
+        return ENOMEM;
+
+    size_t count = 0;
+    const uint64_t base = header.partition_entry_lba * dev->lba_size;
+    for (size_t i = 0; i < header.num_partition_entries; i++) {
+        struct gpt_entry entry;
+        status = vmo_read(&dev->cache, base + i * header.partition_entry_size, &entry, sizeof(entry), &read);
+        if (status || read != sizeof(entry))
+            break;
+        if (guid_is_zero(&entry.type_guid))
+            continue;
+
+        struct gpt_partition* part = &parts[count++];
+        part->type_guid = entry.type_guid;
+        part->unique_guid = entry.unique_guid;
+        part->start_lba = entry.start_lba;
+        part->end_lba = entry.end_lba;
+    }
+
+    *out = parts;
+    *out_count = count;
+    return 0;
+}

dev/block/meson.build

@@ -0,0 +1,11 @@
+kernel_sources += files(
+    'bio.c',
+    'blockdev.c',
+    'gpt.c',
+    'partition.c',
+    'ram.c',
+)
+
+if kconfig.get('CONFIG_NVME', 'n') != 'n'
+    subdir('nvme')
+endif

dev/block/nvme/Kconfig

@@ -0,0 +1,4 @@
+config NVME
+  bool "NVMe block devices"
+  default y
+  depends on PCI

dev/block/nvme/command.c

@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/mmio.h>
+#include "nvme.h"
+
+void nvme_rw_cmd_write(const struct nvme_rw_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, cmd->do_write ? NVME_CMD_WRITE : NVME_CMD_READ) |
+        mmio_field(NVME_SQE_CID, cmd->cid) | mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_RW_SLBA_LOW, (uint32_t)cmd->start_lba);
+
+    uint32_t cdw11 = mmio_field(NVME_RW_SLBA_HIGH, (uint32_t)(cmd->start_lba >> 32));
+
+    uint32_t cdw12 = mmio_field(NVME_RW_NLB, (uint16_t)(cmd->num_lbas - 1));
+    uint32_t cdw13 = 0;
+    uint32_t cdw14 = 0;
+    uint32_t cdw15 = 0;
+
+    mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->prp1);
+    mmio_reg_write(sqe, NVME_SQE_DPTR1, cmd->prp2);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+    mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
+    mmio_reg_write(sqe, NVME_SQE_CDW12, cdw12);
+    mmio_reg_write(sqe, NVME_SQE_CDW13, cdw13);
+    mmio_reg_write(sqe, NVME_SQE_CDW14, cdw14);
+    mmio_reg_write(sqe, NVME_SQE_CDW15, cdw15);
+}
+
+void nvme_flush_cmd_write(const struct nvme_flush_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_CMD_FLUSH) | mmio_field(NVME_SQE_CID, cmd->cid);
+
+    mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+}
+
+void nvme_create_cq_cmd_write(const struct nvme_create_cq_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_CREATE_CQ) |
+        mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_CREATE_CQ_QSIZE, (uint16_t)(cmd->depth - 1)) |
+        mmio_field(NVME_CREATE_CQ_QID, (uint16_t)cmd->qid);
+
+    uint32_t cdw11 = mmio_field(NVME_CREATE_CQ_IV, cmd->irq_vector) |
+        mmio_field(NVME_CREATE_CQ_IEN, cmd->irqs_enabled ? 1 : 0) |
+        mmio_field(NVME_CREATE_CQ_PC, 1); // Our buffer is physically contiguous.
+
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->cq_addr);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+    mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
+}
+
+void nvme_create_sq_cmd_write(const struct nvme_create_sq_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_CREATE_SQ) |
+        mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_CREATE_SQ_QSIZE, (uint16_t)(cmd->depth - 1)) |
+        mmio_field(NVME_CREATE_SQ_QID, (uint16_t)cmd->qid);
+
+    uint32_t cdw11 = mmio_field(NVME_CREATE_SQ_CQID, (uint16_t)cmd->cqid) |
+        mmio_field(NVME_CREATE_SQ_QPRIO, 0) | // Priority is the highest.
+        mmio_field(NVME_CREATE_SQ_PC, 1);     // Our buffer is physically contiguous.
+
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->sq_addr);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+    mmio_reg_write(sqe, NVME_SQE_CDW11, cdw11);
+}
+
+void nvme_identify_cmd_write(const struct nvme_identify_cmd* cmd, struct nvme_sqe* sqe) {
+    uint32_t cdw0 = mmio_field(NVME_SQE_OPC, NVME_ADMIN_IDENTIFY) |
+        mmio_field(NVME_SQE_PSDT, 0); // We always want to use PRP for this.
+
+    uint32_t cdw10 = mmio_field(NVME_IDENTIFY_CNTID, cmd->controller_id) | mmio_field(NVME_IDENTIFY_CNS, cmd->cns);
+
+    mmio_reg_write(sqe, NVME_SQE_DPTR0, cmd->buffer);
+    mmio_reg_write(sqe, NVME_SQE_CDW0, cdw0);
+    mmio_reg_write(sqe, NVME_SQE_NSID, cmd->nsid);
+    mmio_reg_write(sqe, NVME_SQE_CDW10, cdw10);
+}

dev/block/nvme/controller.c

@@ -0,0 +1,388 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/clock.h>
+#include <zinnia/mmio.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <arch/mem.h>
+#include <string.h>
+#include "nvme.h"
+
+constexpr size_t ADMIN_QUEUE_SIZE = 32;
+constexpr size_t IO_QUEUE_SIZE_MAX = 1024;
+constexpr size_t IDENTIFY_SIZE = 4096;
+constexpr size_t NS_LIST_LEN = 1024;
+
+constexpr mmio_reg8 ID_MDTS = {77, MMIO_LE};
+constexpr mmio_reg32 ID_NN = {516, MMIO_LE};
+constexpr mmio_reg64 ID_NSZE = {0, MMIO_LE};
+constexpr mmio_reg8 ID_FLBAS = {26, MMIO_LE};
+constexpr mmio_field8 ID_LBAF_IDX = {ID_FLBAS, 0, 4};
+constexpr mmio_reg32 ID_LBAF0 = {128, MMIO_LE};
+constexpr mmio_field32 ID_LBADS = {ID_LBAF0, 16, 8};
+
+size_t nvme_ctrl_max_transfer_bytes(struct nvme_ctrl* ctrl) {
+    const size_t cap = 512 * arch_mem_page_size();
+    if (ctrl->mdts != 0)
+        return MIN(ctrl->mdts, cap);
+    return cap;
+}
+
+errno_t nvme_ctrl_new_pci(void* regs, struct dma_tag* dma_tag, struct nvme_ctrl** out) {
+    // Read controller version.
+    uint32_t vs = mmio_reg_read(regs, NVME_REG_VS);
+    log("NVMe controller version %u.%u.%u\n",
+        mmio_field_get(NVME_VS_MJR, vs),
+        mmio_field_get(NVME_VS_MNR, vs),
+        mmio_field_get(NVME_VS_TER, vs));
+
+    uint64_t cap = mmio_reg_read(regs, NVME_REG_CAP);
+
+    // Check if our host page size is supported.
+    const size_t mps_max = (size_t)1 << (mmio_field_get(NVME_CAP_MPSMAX, cap) + 12);
+    const size_t mps_min = (size_t)1 << (mmio_field_get(NVME_CAP_MPSMIN, cap) + 12);
+    const size_t page_size = arch_mem_page_size();
+    if (page_size < mps_min || page_size > mps_max) {
+        log_err("Host page size is not supported on this NVMe!\n");
+        return ENOTSUP;
+    }
+
+    struct nvme_ctrl* ctrl = mem_alloc(sizeof(*ctrl), 0);
+    if (!ctrl)
+        return ENOMEM;
+
+    ctrl->regs = regs;
+    ctrl->dma_tag = dma_tag;
+    ctrl->doorbell_stride = (size_t)4 << mmio_field_get(NVME_CAP_DSTRD, cap);
+    ctrl->mps_min = mps_min;
+    ctrl->mps = __builtin_ctzl(page_size) - 12;
+    mutex_init(&ctrl->admin_lock);
+    spin_init(&ctrl->io_lock);
+    atomic_init(&ctrl->ns_count, 0);
+
+    *out = ctrl;
+    return 0;
+}
+
+// Sets the CC.EN flag.
+static errno_t set_status(struct nvme_ctrl* ctrl, bool enable) {
+    uint64_t cap = mmio_reg_read(ctrl->regs, NVME_REG_CAP);
+    // CAP.TO is in 500ms units.
+    const uint64_t timeout = MAX((uint64_t)mmio_field_get(NVME_CAP_TO, cap), 1) * 500'000'000;
+    const uint64_t deadline = clock_get_elapsed_ns() + timeout;
+
+    uint32_t cc = mmio_reg_read(ctrl->regs, NVME_REG_CC);
+    cc &= ~mmio_field(NVME_CC_EN, UINT32_MAX);
+    cc |= mmio_field(NVME_CC_EN, enable ? 1 : 0);
+    mmio_reg_write(ctrl->regs, NVME_REG_CC, cc);
+
+    while (true) {
+        uint32_t csts = mmio_reg_read(ctrl->regs, NVME_REG_CSTS);
+
+        if (mmio_field_get(NVME_CSTS_CFS, csts) != 0)
+            return EIO;
+
+        bool ready = mmio_field_get(NVME_CSTS_RDY, csts) != 0;
+        if (ready == enable)
+            break;
+
+        if (clock_get_elapsed_ns() >= deadline)
+            return ETIMEDOUT;
+    }
+
+    return 0;
+}
+
+static errno_t admin_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe, struct nvme_completion* out) {
+    errno_t status = nvme_queue_submit_cmd(queue, sqe);
+    if (status)
+        return status;
+    return nvme_queue_next_completion(queue, out);
+}
+
+static errno_t identify(struct nvme_queue* queue, const struct nvme_identify_cmd* cmd) {
+    struct nvme_sqe sqe = {};
+    nvme_identify_cmd_write(cmd, &sqe);
+    struct nvme_completion comp;
+    errno_t status = admin_cmd(queue, &sqe, &comp);
+    if (status)
+        return status;
+    if (!nvme_status_is_success(comp.status)) {
+        log_err("Identification failed with status %x\n", comp.status);
+        return EIO;
+    }
+    return 0;
+}
+
+errno_t nvme_ctrl_reset(struct nvme_ctrl* ctrl, bool irqs_enabled, uint16_t io_irq_vector) {
+    // Disable the controller.
+    errno_t status = set_status(ctrl, false);
+    if (status)
+        return status;
+
+    uint64_t cap = mmio_reg_read(ctrl->regs, NVME_REG_CAP);
+
+    const size_t queue_depth = MIN((size_t)mmio_field_get(NVME_CAP_MQES, cap) + 1, IO_QUEUE_SIZE_MAX);
+
+    // Create an admin queue first so we can create more queues.
+    struct nvme_queue* admin_queue;
+    status = nvme_queue_new(ctrl->dma_tag, ctrl->regs, ctrl->doorbell_stride, 0, ADMIN_QUEUE_SIZE, &admin_queue);
+    if (status)
+        return status;
+
+    // Set the admin queue sizes.
+    uint32_t aqa = 0;
+    aqa |= mmio_field(NVME_AQA_ACQS, admin_queue->depth - 1);
+    aqa |= mmio_field(NVME_AQA_ASQS, admin_queue->depth - 1);
+    mmio_reg_write(ctrl->regs, NVME_REG_AQA, aqa);
+
+    // Set the addresses to the admin submission and completion queue.
+    mmio_reg_write(ctrl->regs, NVME_REG_ASQ, admin_queue->sq.addr);
+    mmio_reg_write(ctrl->regs, NVME_REG_ACQ, admin_queue->cq.addr);
+
+    uint32_t cc = mmio_reg_read(ctrl->regs, NVME_REG_CC);
+    cc &=
+        ~(mmio_field(NVME_CC_MPS, UINT32_MAX) | mmio_field(NVME_CC_IOCQES, UINT32_MAX) |
+          mmio_field(NVME_CC_IOSQES, UINT32_MAX));
+
+    cc |= mmio_field(NVME_CC_MPS, ctrl->mps);
+    cc |= mmio_field(NVME_CC_IOCQES, __builtin_ctzl(sizeof(struct nvme_cqe)));
+    cc |= mmio_field(NVME_CC_IOSQES, __builtin_ctzl(sizeof(struct nvme_sqe)));
+
+    mmio_reg_write(ctrl->regs, NVME_REG_CC, cc);
+
+    // Re-enable the controller.
+    status = set_status(ctrl, true);
+    if (status)
+        goto fail_admin;
+
+    struct nvme_io_queue* io_queue;
+    status =
+        nvme_io_queue_new(ctrl->dma_tag, ctrl->regs, ctrl->doorbell_stride, 1, queue_depth, !irqs_enabled, &io_queue);
+    if (status)
+        goto fail_admin;
+
+    struct nvme_sqe sqe = {};
+    struct nvme_completion comp;
+    nvme_create_cq_cmd_write(
+        &(struct nvme_create_cq_cmd){
+            .cq_addr = io_queue->cq.addr,
+            .depth = io_queue->depth,
+            .qid = io_queue->queue_id,
+            .irqs_enabled = irqs_enabled,
+            .irq_vector = irqs_enabled ? io_irq_vector : 0,
+        },
+        &sqe
+    );
+    status = admin_cmd(admin_queue, &sqe, &comp);
+    if (!status && !nvme_status_is_success(comp.status))
+        status = EIO;
+    if (status)
+        goto fail_io;
+
+    sqe = (struct nvme_sqe){};
+    nvme_create_sq_cmd_write(
+        &(struct nvme_create_sq_cmd){
+            .sq_addr = io_queue->sq.addr,
+            .depth = io_queue->depth,
+            .qid = io_queue->queue_id,
+            .cqid = io_queue->queue_id,
+        },
+        &sqe
+    );
+    status = admin_cmd(admin_queue, &sqe, &comp);
+    if (!status && !nvme_status_is_success(comp.status))
+        status = EIO;
+    if (status)
+        goto fail_io;
+
+    mutex_lock(&ctrl->admin_lock);
+    struct nvme_queue* old_admin = ctrl->admin_queue;
+    ctrl->admin_queue = admin_queue;
+    mutex_unlock(&ctrl->admin_lock);
+    if (old_admin)
+        nvme_queue_free(old_admin);
+
+    spin_lock(&ctrl->io_lock);
+    ctrl->io_queue = io_queue;
+    spin_unlock(&ctrl->io_lock);
+
+    return 0;
+
+fail_io:
+    nvme_io_queue_free(io_queue);
+fail_admin:
+    nvme_queue_free(admin_queue);
+    return status;
+}
+
+static void trim_ascii_end(char* str, size_t len) {
+    while (len > 0 &&
+           (str[len - 1] == ' ' || str[len - 1] == '\t' || str[len - 1] == '\n' || str[len - 1] == '\r' ||
+            str[len - 1] == '\f'))
+        len--;
+    str[len] = '\0';
+}
+
+errno_t nvme_ctrl_identify(struct nvme_ctrl* ctrl) {
+    struct dma_buffer identify_buffer;
+    errno_t status = dma_buffer_alloc(ctrl->dma_tag, IDENTIFY_SIZE, &identify_buffer);
+    if (status)
+        return ENOMEM;
+
+    mutex_lock(&ctrl->admin_lock);
+    if (!ctrl->admin_queue) {
+        status = ENXIO;
+        goto out;
+    }
+
+    status = identify(
+        ctrl->admin_queue,
+        &(struct nvme_identify_cmd){
+            .buffer = identify_buffer.addr,
+            .controller_id = 0, // TODO: Use a real number here
+            .cns = 1,
+            .nsid = 0,
+        }
+    );
+    // Check if the identification was successful.
+    if (status)
+        goto out;
+
+    dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
+
+    // Read the model strings.
+    const uint8_t* view = identify_buffer.virt;
+    memcpy(ctrl->serial, view + 4, sizeof(ctrl->serial) - 1);
+    memcpy(ctrl->model, view + 24, sizeof(ctrl->model) - 1);
+    memcpy(ctrl->revision, view + 64, sizeof(ctrl->revision) - 1);
+    trim_ascii_end(ctrl->serial, sizeof(ctrl->serial) - 1);
+    trim_ascii_end(ctrl->model, sizeof(ctrl->model) - 1);
+    trim_ascii_end(ctrl->revision, sizeof(ctrl->revision) - 1);
+    log("Model \"%s\"\n", ctrl->model);
+
+    atomic_store(&ctrl->ns_count, mmio_reg_read(identify_buffer.virt, ID_NN), ATOMIC_RELEASE);
+
+    uint8_t mdts = mmio_reg_read(identify_buffer.virt, ID_MDTS);
+    if (mdts != 0 && mdts < sizeof(size_t) * 8)
+        ctrl->mdts = ((size_t)1 << mdts) * ctrl->mps_min;
+
+out:
+    mutex_unlock(&ctrl->admin_lock);
+    dma_buffer_free(&identify_buffer);
+    return status;
+}
+
+// Scans all existing namespaces.
+errno_t nvme_ctrl_scan_namespaces(struct nvme_ctrl* ctrl, nvme_ns_vec_t* out) {
+    vec_init(out);
+
+    uint32_t* ns_list = mem_alloc(NS_LIST_LEN * sizeof(uint32_t), 0);
+    if (!ns_list)
+        return ENOMEM;
+
+    struct dma_buffer identify_buffer;
+    errno_t status = dma_buffer_alloc(ctrl->dma_tag, IDENTIFY_SIZE, &identify_buffer);
+    if (status) {
+        mem_free(ns_list);
+        return ENOMEM;
+    }
+
+    mutex_lock(&ctrl->admin_lock);
+    struct nvme_queue* aq = ctrl->admin_queue;
+    if (!aq) {
+        status = ENXIO;
+        goto out;
+    }
+
+    uint32_t nsid = 0;
+    while (true) {
+        const struct nvme_identify_cmd admin_ident_cmd = {
+            .buffer = identify_buffer.addr,
+            .controller_id = 0,
+            .cns = 2,
+            .nsid = nsid,
+        };
+        status = identify(aq, &admin_ident_cmd);
+
+        if (status)
+            goto out;
+
+        dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
+
+        // Parse the identifier list until we hit an namespace value of 0.
+        size_t ns_list_len = 0;
+        const uint32_t* raw_list = identify_buffer.virt;
+        while (ns_list_len < NS_LIST_LEN && raw_list[ns_list_len] != 0) {
+            ns_list[ns_list_len] = raw_list[ns_list_len];
+            ns_list_len++;
+        }
+
+        for (size_t i = 0; i < ns_list_len; i++) {
+            const uint32_t id = ns_list[i];
+            const struct nvme_identify_cmd ns_ident_cmd = {
+                .buffer = identify_buffer.addr,
+                .controller_id = 0,
+                .cns = 0,
+                .nsid = id,
+            };
+            status = identify(aq, &ns_ident_cmd);
+
+            if (status)
+                goto out;
+
+            dma_buffer_sync(&identify_buffer, 0, IDENTIFY_SIZE, DMA_SYNC_POSTREAD);
+
+            const uint64_t lba_count = mmio_reg_read(identify_buffer.virt, ID_NSZE);
+
+            // To get the block size of a namespace, we need to read a descriptor (see LBAF0-15).
+            // Get the format descriptor index used by this namespace.
+            const uint8_t lba_format_idx = mmio_field_get(ID_LBAF_IDX, mmio_reg_read(identify_buffer.virt, ID_FLBAS));
+
+            const mmio_reg32 lbaf = {ID_LBAF0.offset + lba_format_idx * sizeof(uint32_t), MMIO_LE};
+            const uint32_t lba_format = mmio_reg_read(identify_buffer.virt, lbaf);
+
+            uint8_t lba_shift = mmio_field_get(ID_LBADS, lba_format);
+            // If LBADS doesn't contain a non-zero value, we need to assume a default. Use 512 bytes here.
+            if (lba_shift == 0)
+                lba_shift = 9;
+
+            struct nvme_ns* ns;
+            status = nvme_ns_new(ctrl, id, lba_shift, lba_count, nvme_ctrl_max_transfer_bytes(ctrl), &ns);
+            if (status)
+                goto out;
+            status = vec_push(out, ns);
+            if (status) {
+                blockdev_unref(&ns->blk);
+                goto out;
+            }
+        }
+
+        // If there is no 0 in the buffer, there's more entries than can fit in `identify_buffer`.
+        if (ns_list_len < NS_LIST_LEN)
+            break;
+        nsid = ns_list[ns_list_len - 1];
+    }
+
+out:
+    mutex_unlock(&ctrl->admin_lock);
+    dma_buffer_free(&identify_buffer);
+    mem_free(ns_list);
+    if (status) {
+        vec_foreach(out, i) {
+            blockdev_unref(&vec_elem_get(out, i)->blk);
+        }
+        vec_destroy(out);
+    }
+    return status;
+}
+
+struct nvme_io_queue* nvme_ctrl_io_queue(struct nvme_ctrl* ctrl) {
+    spin_lock(&ctrl->io_lock);
+    struct nvme_io_queue* queue = ctrl->io_queue;
+    spin_unlock(&ctrl->io_lock);
+    return queue;
+}

dev/block/nvme/meson.build

@@ -0,0 +1,13 @@
+modules += {
+    'nvme': {
+        'modular': kconfig.get('CONFIG_NVME') == 'm',
+        'sources': files(
+            'command.c',
+            'controller.c',
+            'namespace.c',
+            'pci.c',
+            'prp.c',
+            'queue.c',
+        ),
+    },
+}

dev/block/nvme/namespace.c

@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/clock.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <stdckdint.h>
+#include "nvme.h"
+
+static errno_t nvme_ns_submit(struct blockdev* blk, struct bio* bio) {
+    struct nvme_ns* ns = containerof(blk, struct nvme_ns, blk);
+
+    if (bio->op != BIO_FLUSH) {
+        uint64_t end_lba;
+        if (ckd_add(&end_lba, bio->lba, (uint64_t)bio->num_lbas)) {
+            bio_complete(bio, EOVERFLOW, 0);
+            return 0;
+        }
+        if (end_lba > ns->lba_count) {
+            bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0);
+            return 0;
+        }
+    }
+
+    struct nvme_io_queue* queue = nvme_ctrl_io_queue(ns->ctrl);
+    if (!queue) {
+        bio_complete(bio, EIO, 0);
+        return 0;
+    }
+
+    size_t bytes = 0;
+    for (size_t i = 0; i < bio->num_segs; i++)
+        bytes += bio->segs[i].len;
+
+    _Atomic(bool) done = false;
+    const bool poll = queue->poll;
+
+    uint16_t cid = nvme_io_queue_submit(
+        queue,
+        bio,
+        (struct nvme_rw_cmd){
+            .prp1 = 0,
+            .prp2 = 0,
+            .cid = 0,
+            .do_write = bio->op == BIO_WRITE,
+            .start_lba = bio->lba,
+            .num_lbas = bio->num_lbas,
+            .bytes = bytes,
+            .control = 0,
+            .ds_mgmt = 0,
+            .ref_tag = 0,
+            .app_tag = 0,
+            .app_mask = 0,
+            .nsid = ns->nsid,
+        },
+        poll ? &done : nullptr
+    );
+
+    // If we're not interrupt-driven, poll the IO completion queue.
+    if (poll) {
+        const uint64_t deadline = clock_get_elapsed_ns() + NVME_COMPLETION_TIMEOUT_NS;
+        while (!atomic_load(&done, ATOMIC_ACQUIRE)) {
+            nvme_io_queue_drain(queue);
+            if (atomic_load(&done, ATOMIC_ACQUIRE))
+                break;
+            if (clock_get_elapsed_ns() >= deadline && nvme_io_queue_abandon(queue, cid, bio)) {
+                bio_complete(bio, EIO, 0);
+                break;
+            }
+        }
+    }
+
+    return 0;
+}
+
+static void nvme_ns_destroy(struct blockdev* blk) {
+    mem_free(containerof(blk, struct nvme_ns, blk));
+}
+
+static const struct blockdev_ops nvme_ns_ops = {
+    .submit = nvme_ns_submit,
+    .destroy = nvme_ns_destroy,
+};
+
+errno_t nvme_ns_new(
+    struct nvme_ctrl* ctrl,
+    uint32_t nsid,
+    uint8_t lba_shift,
+    uint64_t lba_count,
+    size_t max_transfer_bytes,
+    struct nvme_ns** out
+) {
+    log("Namespace %u: LBA size %zu bytes, %zu MBs total\n",
+        nsid,
+        (size_t)1 << lba_shift,
+        (size_t)(lba_count << lba_shift) / 1024 / 1024);
+
+    struct nvme_ns* ns = mem_alloc(sizeof(*ns), 0);
+    if (!ns)
+        return ENOMEM;
+
+    blockdev_init(&ns->blk, &nvme_ns_ops, NVME_MAJOR, nsid, (size_t)1 << lba_shift, lba_count);
+    ns->blk.limits = (struct blockdev_limits){
+        .max_lbas = MAX(max_transfer_bytes >> lba_shift, 1),
+        .max_segs = 512,
+    };
+    ns->ctrl = ctrl;
+    ns->nsid = nsid;
+    ns->lba_shift = lba_shift;
+    ns->lba_count = lba_count;
+    ns->max_transfer_bytes = max_transfer_bytes;
+
+    *out = ns;
+    return 0;
+}

dev/block/nvme/nvme.h

@@ -0,0 +1,221 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/atomic.h>
+#include <zinnia/blockdev.h>
+#include <zinnia/dma.h>
+#include <zinnia/errno.h>
+#include <zinnia/event.h>
+#include <zinnia/mutex.h>
+#include <zinnia/pci.h>
+#include <zinnia/spin.h>
+#include <zinnia/vec.h>
+#include <stddef.h>
+#include <stdint.h>
+#include "spec.h"
+
+constexpr uint32_t NVME_MAJOR = 159;
+constexpr uint64_t NVME_COMPLETION_TIMEOUT_NS = 30'000'000'000;
+
+// An entry in the completion queue.
+struct nvme_completion {
+    uint32_t result;
+    uint16_t sq_head;
+    uint16_t sq_id;
+    uint16_t cmd_id;
+    bool phase_tag;
+    uint16_t status;
+};
+
+static inline bool nvme_status_is_success(uint16_t status) {
+    return status == 0;
+}
+
+struct nvme_rw_cmd {
+    uint64_t prp1;
+    uint64_t prp2;
+    uint16_t cid;
+    bool do_write;
+    uint64_t start_lba;
+    size_t num_lbas;
+    size_t bytes;
+    uint16_t control;
+    uint32_t ds_mgmt;
+    uint32_t ref_tag;
+    uint16_t app_tag;
+    uint16_t app_mask;
+    uint32_t nsid;
+};
+
+struct nvme_flush_cmd {
+    uint16_t cid;
+    uint32_t nsid;
+};
+
+struct nvme_create_cq_cmd {
+    dma_addr_t cq_addr;
+    size_t depth;
+    size_t qid;
+    bool irqs_enabled;
+    uint16_t irq_vector;
+};
+
+struct nvme_create_sq_cmd {
+    dma_addr_t sq_addr;
+    size_t depth;
+    size_t qid;
+    size_t cqid;
+};
+
+struct nvme_identify_cmd {
+    dma_addr_t buffer;
+    uint16_t controller_id;
+    uint8_t cns;
+    uint32_t nsid;
+};
+
+void nvme_rw_cmd_write(const struct nvme_rw_cmd* cmd, struct nvme_sqe* sqe);
+void nvme_flush_cmd_write(const struct nvme_flush_cmd* cmd, struct nvme_sqe* sqe);
+void nvme_create_cq_cmd_write(const struct nvme_create_cq_cmd* cmd, struct nvme_sqe* sqe);
+void nvme_create_sq_cmd_write(const struct nvme_create_sq_cmd* cmd, struct nvme_sqe* sqe);
+void nvme_identify_cmd_write(const struct nvme_identify_cmd* cmd, struct nvme_sqe* sqe);
+
+struct nvme_queue {
+    // Amount of queue entries.
+    size_t depth;
+    size_t doorbells_offset;
+    size_t doorbell_stride;
+    void* regs;
+    // Physical buffer for the completion queue.
+    struct dma_buffer cq;
+    // The index of the current completion queue entry.
+    size_t cq_head;
+    // Determines whether a completion queue entry is new.
+    uint8_t cq_phase;
+    // Physical buffer for the submission queue.
+    struct dma_buffer sq;
+    // The index of the current submission queue entry.
+    size_t sq_tail;
+};
+
+errno_t nvme_queue_new(
+    struct dma_tag* tag,
+    void* regs,
+    size_t doorbell_stride,
+    size_t queue_id,
+    size_t depth,
+    struct nvme_queue** out
+);
+void nvme_queue_free(struct nvme_queue* queue);
+errno_t nvme_queue_submit_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe);
+errno_t nvme_queue_next_completion(struct nvme_queue* queue, struct nvme_completion* out);
+
+struct nvme_inflight {
+    bool active;
+    struct bio* bio;
+    size_t lbas;
+    struct dma_map* map;
+    struct dma_buffer prps;
+    _Atomic(bool)* done;
+};
+
+struct nvme_io_queue {
+    size_t queue_id;
+    size_t depth;
+    size_t doorbells_offset;
+    size_t doorbell_stride;
+    void* regs;
+    bool poll;
+    struct dma_tag* tag;
+    struct dma_buffer cq;
+    struct dma_buffer sq;
+    struct spinlock sq_lock;
+    size_t sq_tail;
+    struct spinlock cq_lock;
+    size_t cq_head;
+    uint8_t cq_phase;
+    struct spinlock cmds_lock;
+    struct nvme_inflight* slots;
+    uint16_t* free;
+    size_t num_free;
+    struct event slots_free;
+};
+
+errno_t nvme_io_queue_new(
+    struct dma_tag* tag,
+    void* regs,
+    size_t doorbell_stride,
+    size_t queue_id,
+    size_t depth,
+    bool poll,
+    struct nvme_io_queue** out
+);
+void nvme_io_queue_free(struct nvme_io_queue* queue);
+uint16_t nvme_io_queue_submit(
+    struct nvme_io_queue* queue,
+    struct bio* bio,
+    struct nvme_rw_cmd cmd,
+    _Atomic(bool)* done
+);
+bool nvme_io_queue_abandon(struct nvme_io_queue* queue, uint16_t cid, struct bio* bio);
+size_t nvme_io_queue_drain(struct nvme_io_queue* queue);
+
+errno_t nvme_build_prps(
+    struct dma_tag* tag,
+    struct bio* bio,
+    struct nvme_inflight* slot,
+    uint64_t* prp1,
+    uint64_t* prp2
+);
+void nvme_release_prps(struct nvme_inflight* slot);
+
+struct nvme_ns;
+
+struct nvme_ctrl {
+    void* regs;
+    struct dma_tag* dma_tag;
+    size_t doorbell_stride; // The amount of bytes between two doorbells.
+    size_t mps_min;
+    uint8_t mps;
+
+    char serial[20 + 1];  // Serial number of the controller.
+    char model[40 + 1];   // Model name of the controller.
+    char revision[8 + 1]; // Revision of the controller's firmware.
+
+    struct nvme_queue* admin_queue; // Queue to submit admin commands to.
+    mutex_t admin_lock;
+
+    struct nvme_io_queue* io_queue; // Queue to submit IO commands to.
+    struct spinlock io_lock;
+
+    _Atomic(uint32_t) ns_count; // Amount of namespaces.
+    size_t mdts;                // Maximum data transfer size.
+};
+
+typedef vec(struct nvme_ns*) nvme_ns_vec_t;
+
+errno_t nvme_ctrl_new_pci(void* regs, struct dma_tag* dma_tag, struct nvme_ctrl** out);
+size_t nvme_ctrl_max_transfer_bytes(struct nvme_ctrl* ctrl);
+errno_t nvme_ctrl_reset(struct nvme_ctrl* ctrl, bool irqs_enabled, uint16_t io_irq_vector);
+errno_t nvme_ctrl_identify(struct nvme_ctrl* ctrl);
+errno_t nvme_ctrl_scan_namespaces(struct nvme_ctrl* ctrl, nvme_ns_vec_t* out);
+struct nvme_io_queue* nvme_ctrl_io_queue(struct nvme_ctrl* ctrl);
+
+struct nvme_ns {
+    struct blockdev blk;
+    struct nvme_ctrl* ctrl;
+    uint32_t nsid;
+    uint8_t lba_shift;
+    uint64_t lba_count;
+    size_t max_transfer_bytes;
+};
+
+errno_t nvme_ns_new(
+    struct nvme_ctrl* ctrl,
+    uint32_t nsid,
+    uint8_t lba_shift,
+    uint64_t lba_count,
+    size_t max_transfer_bytes,
+    struct nvme_ns** out
+);

dev/block/nvme/pci.c

@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/errno.h>
+#include <zinnia/irq.h>
+#include <zinnia/module.h>
+#include <zinnia/pci.h>
+#include <zinnia/pmap.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <zinnia/virt.h>
+#include <zinnia/vma.h>
+#include <arch/mem.h>
+#include "nvme.h"
+
+static _Atomic(size_t) nvme_counter;
+
+struct nvme_irq_handler {
+    struct irq_handler handler;
+    struct nvme_ctrl* controller;
+};
+
+static enum irq_status nvme_irq_raise(struct irq_handler* self) {
+    struct nvme_irq_handler* handler = containerof(self, struct nvme_irq_handler, handler);
+    struct nvme_io_queue* queue = nvme_ctrl_io_queue(handler->controller);
+    if (queue)
+        nvme_io_queue_drain(queue);
+    return IRQ_STATUS_HANDLED;
+}
+
+static errno_t map_bar(struct pci_bar* bar, void** out) {
+    const size_t page_size = arch_mem_page_size();
+    const size_t len = ALIGN_UP(bar->size, page_size);
+    uintptr_t vaddr = vm_alloc(len);
+    if (!vaddr)
+        return ENOMEM;
+
+    for (size_t i = 0; i < len; i += page_size) {
+        errno_t status = pmap_map(&kernel_vma.pmap, vaddr + i, bar->address + i, PTE_READ | PTE_WRITE, CACHE_MMIO);
+        if (status)
+            return status;
+    }
+
+    *out = (void*)vaddr;
+    return 0;
+}
+
+static errno_t nvme_pci_probe(struct pci_device* dev, const struct pci_match* match) {
+    pci_log(dev, "Probing NVMe device\n");
+
+    // Enable MMIO decoding and DMA. We only support MSI-X or polling, so keep legacy INTx disabled.
+    uint16_t cmd = pci_read16(dev, PCI_COMMAND);
+    pci_write16(dev, PCI_COMMAND, cmd | PCI_COMMAND_MEM | PCI_COMMAND_MASTER | PCI_COMMAND_INTX_DISABLE);
+
+    struct irq_line* irq_line = nullptr;
+    if (pci_setup_msix(dev, &irq_line)) {
+        pci_log_warn(dev, "NVMe MSI-X setup failed, falling back to polling completions\n");
+        irq_line = nullptr;
+    }
+
+    struct pci_bar bar;
+    errno_t status = pci_bar_get(dev, 0, &bar);
+    if (status)
+        return ENXIO;
+    if (bar.type != PCI_BAR_MMIO32 && bar.type != PCI_BAR_MMIO64)
+        return ENXIO;
+
+    void* regs;
+    status = map_bar(&bar, &regs);
+    if (status)
+        return status;
+
+    struct nvme_ctrl* controller;
+    status = nvme_ctrl_new_pci(regs, dev->dma_tag, &controller);
+    if (status) {
+        pci_log_err(dev, "Failed to probe controller: %i\n", status);
+        return ENODEV;
+    }
+
+    if (irq_line) {
+        struct nvme_irq_handler* handler = mem_alloc(sizeof(*handler), 0);
+        if (!handler)
+            return ENOMEM;
+        handler->handler.raise = nvme_irq_raise;
+        handler->controller = controller;
+        irq_line_attach(irq_line, &handler->handler);
+        if (irq_line->unmask)
+            irq_line->unmask(irq_line);
+    }
+
+    // Reset the controller to initialize all queues and other structures.
+    status = nvme_ctrl_reset(controller, irq_line != nullptr, 0);
+    if (status) {
+        pci_log_err(dev, "Failed to reset controller: %i\n", status);
+        return ENODEV;
+    }
+
+    status = nvme_ctrl_identify(controller);
+    if (status) {
+        pci_log_err(dev, "Failed to identify controller: %i\n", status);
+        return ENODEV;
+    }
+
+    nvme_ns_vec_t namespaces;
+    status = nvme_ctrl_scan_namespaces(controller, &namespaces);
+    if (status) {
+        pci_log_err(dev, "Failed to identify controller: %i\n", status);
+        return ENODEV;
+    }
+
+    const size_t nvme_id = atomic_fetch_add(&nvme_counter, 1, ATOMIC_SEQ_CST);
+    vec_foreach(&namespaces, i) {
+        struct nvme_ns* ns = vec_elem_get(&namespaces, i);
+        char path[32];
+        ksnprintf(path, sizeof(path), "nvme%zun%u", nvme_id, ns->nsid);
+        status = blockdev_register(path, &ns->blk);
+        if (status)
+            break;
+    }
+    vec_destroy(&namespaces);
+
+    return status;
+}
+
+static const struct pci_match nvme_match[] = {
+    // Generic NVMe device. Assume no quirks or any funny business.
+    {PCI_MATCH_CLASS(1, 8, 2), .data = 0},
+};
+
+MATCH_TABLE("pci", nvme_match);
+
+static struct pci_driver nvme_driver = {
+    .name = "nvme",
+    .match = nvme_match,
+    .num_matches = ARRAY_SIZE(nvme_match),
+    .probe = nvme_pci_probe,
+};
+
+errno_t nvme_init(struct module* mod) {
+    return pci_driver_register(&nvme_driver);
+}
+
+errno_t nvme_exit(struct module* mod) {
+    // TODO: Unregister
+    return 0;
+}
+
+MODULE_ENTRY(nvme_init);
+MODULE_EXIT(nvme_exit);
+
+MODULE_DESC("NVMe Block Devices");
+MODULE_AUTHOR("Marvin Friedrich");
+MODULE_LICENSE("ISC");
+MODULE_VERSION("1.0");

dev/block/nvme/prp.c

@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/utils.h>
+#include <arch/mem.h>
+#include "nvme.h"
+
+static size_t count_pages(struct dma_map* map, size_t page_size) {
+    size_t count = 0;
+    for (size_t i = 0; i < map->num_segs; i++) {
+        const struct dma_seg* seg = &map->segs[i];
+        if (seg->len == 0)
+            continue;
+        const dma_addr_t end = seg->addr + seg->len;
+        for (dma_addr_t page = ALIGN_DOWN(seg->addr, page_size); page < end; page += page_size)
+            count++;
+    }
+    return count;
+}
+
+static void fill_pages(struct dma_map* map, size_t page_size, uint64_t* out) {
+    bool first_page = true;
+    size_t idx = 0;
+    for (size_t i = 0; i < map->num_segs; i++) {
+        const struct dma_seg* seg = &map->segs[i];
+        if (seg->len == 0)
+            continue;
+        const dma_addr_t end = seg->addr + seg->len;
+        for (dma_addr_t page = ALIGN_DOWN(seg->addr, page_size); page < end; page += page_size) {
+            if (first_page)
+                first_page = false;
+            else
+                out[idx++] = page;
+        }
+    }
+}
+
+errno_t nvme_build_prps(
+    struct dma_tag* tag,
+    struct bio* bio,
+    struct nvme_inflight* slot,
+    uint64_t* prp1,
+    uint64_t* prp2
+) {
+    const size_t page_size = arch_mem_page_size();
+    slot->map = nullptr;
+    slot->prps = (struct dma_buffer){};
+
+    if (bio->num_segs == 0)
+        return EIO;
+
+    errno_t status = dma_map_create(tag, SIZE_MAX, bio->num_segs, SIZE_MAX, 0, &slot->map);
+    if (status)
+        return status;
+
+    status = dma_map_load_bio(slot->map, bio);
+    if (status)
+        goto fail;
+    if (slot->map->num_segs == 0) {
+        status = EIO;
+        goto fail;
+    }
+
+    dma_map_sync(slot->map, 0, slot->map->mapped_len, bio->op == BIO_WRITE ? DMA_SYNC_PREWRITE : DMA_SYNC_PREREAD);
+
+    *prp1 = slot->map->segs[0].addr;
+
+    // Enumerate the base of every page the transfer touches in order.
+    const size_t pages = count_pages(slot->map, page_size);
+    const size_t rest_pages = pages ? pages - 1 : 0;
+
+    if (rest_pages == 0) {
+        *prp2 = 0;
+        return 0;
+    }
+
+    if (rest_pages == 1) {
+        uint64_t second;
+        fill_pages(slot->map, page_size, &second);
+        *prp2 = second;
+        return 0;
+    }
+
+    // We don't want to chain across pages.
+    const size_t entries_per_page = page_size / sizeof(uint64_t);
+    if (rest_pages > entries_per_page) {
+        status = ENOMEM;
+        goto fail;
+    }
+
+    status = dma_buffer_alloc(tag, page_size, &slot->prps);
+    if (status)
+        goto fail;
+
+    fill_pages(slot->map, page_size, slot->prps.virt);
+    dma_buffer_sync(&slot->prps, 0, rest_pages * sizeof(uint64_t), DMA_SYNC_PREWRITE);
+
+    *prp2 = slot->prps.addr;
+    return 0;
+
+fail:
+    nvme_release_prps(slot);
+    return status;
+}
+
+void nvme_release_prps(struct nvme_inflight* slot) {
+    if (slot->map) {
+        dma_map_sync(
+            slot->map,
+            0,
+            slot->map->mapped_len,
+            slot->bio && slot->bio->op == BIO_WRITE ? DMA_SYNC_POSTWRITE : DMA_SYNC_POSTREAD
+        );
+        dma_map_unload(slot->map);
+        dma_map_destroy(slot->map);
+        slot->map = nullptr;
+    }
+    dma_buffer_free(&slot->prps);
+}

dev/block/nvme/queue.c

@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: ISC
+
+#define pr_fmt(fmt) "nvme: " fmt
+
+#include <zinnia/alloc.h>
+#include <zinnia/clock.h>
+#include <zinnia/mmio.h>
+#include <zinnia/print.h>
+#include <zinnia/utils.h>
+#include <string.h>
+#include "nvme.h"
+
+constexpr size_t DOORBELL_OFFSET = 0x1000;
+
+static void ring_tail(void* regs, size_t doorbells_offset, size_t tail) {
+    mmio_reg_write(regs, ((mmio_reg32){doorbells_offset, MMIO_LE}), (uint32_t)tail);
+}
+
+static void ring_head(void* regs, size_t doorbells_offset, size_t doorbell_stride, size_t head) {
+    mmio_reg_write(regs, ((mmio_reg32){doorbells_offset + doorbell_stride, MMIO_LE}), (uint32_t)head);
+}
+
+// Creates a new submission and completion queue pair.
+errno_t nvme_queue_new(
+    struct dma_tag* tag,
+    void* regs,
+    size_t doorbell_stride,
+    size_t queue_id,
+    size_t depth,
+    struct nvme_queue** out
+) {
+    const size_t align = 0x1000;
+    const size_t sq_size = ALIGN_UP(depth * sizeof(struct nvme_sqe), align);
+    const size_t cq_size = ALIGN_UP(depth * sizeof(struct nvme_cqe), align);
+
+    struct nvme_queue* queue = mem_alloc(sizeof(*queue), 0);
+    if (!queue)
+        return ENOMEM;
+
+    // Allocate memory the completion queue.
+    errno_t status = dma_buffer_alloc(tag, cq_size, &queue->cq);
+    if (status)
+        goto fail;
+
+    // Allocate memory for the submission queue.
+    status = dma_buffer_alloc(tag, sq_size, &queue->sq);
+    if (status)
+        goto fail;
+
+    // Calculate the offset of the doorbell registers. The stride is already precomputed here.
+    queue->doorbells_offset = DOORBELL_OFFSET + (queue_id * 2 * doorbell_stride);
+    queue->doorbell_stride = doorbell_stride;
+    queue->depth = depth;
+    queue->regs = regs;
+    queue->cq_head = 0;
+    queue->cq_phase = 1; // When the controller is enabled, the first phase is 1.
+    queue->sq_tail = 0;
+
+    *out = queue;
+    return 0;
+
+fail:
+    nvme_queue_free(queue);
+    return status;
+}
+
+void nvme_queue_free(struct nvme_queue* queue) {
+    dma_buffer_free(&queue->cq);
+    dma_buffer_free(&queue->sq);
+    mem_free(queue);
+}
+
+// Submits a command to this queue.
+errno_t nvme_queue_submit_cmd(struct nvme_queue* queue, const struct nvme_sqe* sqe) {
+    struct nvme_sqe* entries = queue->sq.virt;
+    memcpy(&entries[queue->sq_tail], sqe, sizeof(*sqe));
+    dma_buffer_sync(&queue->sq, queue->sq_tail * sizeof(*sqe), sizeof(*sqe), DMA_SYNC_PREWRITE);
+
+    queue->sq_tail++;
+    if (queue->sq_tail == queue->depth)
+        queue->sq_tail = 0;
+
+    // Notify the controller of the new tail index.
+    ring_tail(queue->regs, queue->doorbells_offset, queue->sq_tail);
+
+    return 0;
+}
+
+// Reads the next completion entry from the queue.
+errno_t nvme_queue_next_completion(struct nvme_queue* queue, struct nvme_completion* out) {
+    struct nvme_cqe* entry = &((struct nvme_cqe*)queue->cq.virt)[queue->cq_head];
+
+    // Wait until the phase for this entry has changed.
+    uint32_t dw3;
+    const uint64_t deadline = clock_get_elapsed_ns() + NVME_COMPLETION_TIMEOUT_NS;
+    while (true) {
+        dma_buffer_sync(&queue->cq, queue->cq_head * sizeof(*entry), sizeof(*entry), DMA_SYNC_POSTREAD);
+        dw3 = mmio_reg_read(entry, NVME_CQE_DW3);
+
+        // The controller will flip the phase bit of the current entry when writing.
+        if (mmio_field_get(NVME_CQE_PHASE_TAG, dw3) == queue->cq_phase)
+            break;
+
+        if (clock_get_elapsed_ns() >= deadline)
+            return ETIMEDOUT;
+    }
+
+    // Then, read the rest of the completion queue entry.
+    uint32_t dw0 = mmio_reg_read(entry, NVME_CQE_DW0);
+    uint32_t dw2 = mmio_reg_read(entry, NVME_CQE_DW2);
+
+    *out = (struct nvme_completion){
+        .result = dw0,
+        .sq_head = mmio_field_get(NVME_CQE_SQ_HEAD, dw2),
+        .sq_id = mmio_field_get(NVME_CQE_SQ_IDENT, dw2),
+        .cmd_id = mmio_field_get(NVME_CQE_CID, dw3),
+        .status = mmio_field_get(NVME_CQE_STATUS, dw3),
+        .phase_tag = mmio_field_get(NVME_CQE_PHASE_TAG, dw3) != 0,
+    };
+
+    queue->cq_head++;
+    if (queue->cq_head == queue->depth) {
+        queue->cq_head = 0;
+        queue->cq_phase ^= 1; // Flip the phase bit.
+    }
+
+    // Notify the controller of the new head index.
+    ring_head(queue->regs, queue->doorbells_offset, queue->doorbell_stride, queue->cq_head);
+
+    return 0;
+}
+
+errno_t nvme_io_queue_new(
+    struct dma_tag* tag,
+    void* regs,
+    size_t doorbell_stride,
+    size_t queue_id,
+    size_t depth,
+    bool poll,
+    struct nvme_io_queue** out
+) {
+    const size_t align = 0x1000;
+    const size_t sq_size = ALIGN_UP(depth * sizeof(struct nvme_sqe), align);
+    const size_t cq_size = ALIGN_UP(depth * sizeof(struct nvme_cqe), align);
+
+    struct nvme_io_queue* queue = mem_alloc(sizeof(*queue), 0);
+    if (!queue)
+        return ENOMEM;
+
+    errno_t status = dma_buffer_alloc(tag, cq_size, &queue->cq);
+    if (status)
+        goto fail;
+
+    status = dma_buffer_alloc(tag, sq_size, &queue->sq);
+    if (status)
+        goto fail;
+
+    const size_t slots = depth - 1;
+    queue->slots = mem_alloc(slots * sizeof(*queue->slots), 0);
+    queue->free = mem_alloc(slots * sizeof(*queue->free), 0);
+    if (!queue->slots || !queue->free) {
+        status = ENOMEM;
+        goto fail;
+    }
+    for (size_t i = 0; i < slots; i++)
+        queue->free[i] = slots - 1 - i;
+    queue->num_free = slots;
+
+    queue->queue_id = queue_id;
+    queue->depth = depth;
+    queue->doorbells_offset = DOORBELL_OFFSET + (queue_id * 2 * doorbell_stride);
+    queue->doorbell_stride = doorbell_stride;
+    queue->regs = regs;
+    queue->poll = poll;
+    queue->tag = tag;
+    queue->sq_tail = 0;
+    queue->cq_head = 0;
+    queue->cq_phase = 1;
+    spin_init(&queue->sq_lock);
+    spin_init(&queue->cq_lock);
+    spin_init(&queue->cmds_lock);
+    event_init(&queue->slots_free);
+
+    *out = queue;
+    return 0;
+
+fail:
+    nvme_io_queue_free(queue);
+    return status;
+}
+
+void nvme_io_queue_free(struct nvme_io_queue* queue) {
+    dma_buffer_free(&queue->cq);
+    dma_buffer_free(&queue->sq);
+    mem_free(queue->slots);
+    mem_free(queue->free);
+    mem_free(queue);
+}
+
+static uint16_t acquire_slot(struct nvme_io_queue* queue) {
+    while (true) {
+        spin_lock(&queue->cmds_lock);
+        if (queue->num_free) {
+            uint16_t cid = queue->free[--queue->num_free];
+            spin_unlock(&queue->cmds_lock);
+            return cid;
+        }
+        spin_unlock(&queue->cmds_lock);
+
+        if (queue->poll) {
+            nvme_io_queue_drain(queue);
+            continue;
+        }
+
+        nvme_io_queue_drain(queue);
+        event_reset(&queue->slots_free);
+        spin_lock(&queue->cmds_lock);
+        bool empty = queue->num_free == 0;
+        spin_unlock(&queue->cmds_lock);
+        if (empty)
+            event_wait(&queue->slots_free);
+    }
+}
+
+static void release_slot(struct nvme_io_queue* queue, uint16_t cid) {
+    spin_lock(&queue->cmds_lock);
+    queue->free[queue->num_free++] = cid;
+    spin_unlock(&queue->cmds_lock);
+}
+
+static void write_sqe(struct nvme_io_queue* queue, const struct nvme_sqe* sqe) {
+    spin_lock(&queue->sq_lock);
+    struct nvme_sqe* entries = queue->sq.virt;
+    memcpy(&entries[queue->sq_tail], sqe, sizeof(*sqe));
+    dma_buffer_sync(&queue->sq, queue->sq_tail * sizeof(*sqe), sizeof(*sqe), DMA_SYNC_PREWRITE);
+    queue->sq_tail++;
+    if (queue->sq_tail == queue->depth)
+        queue->sq_tail = 0;
+    ring_tail(queue->regs, queue->doorbells_offset, queue->sq_tail);
+    spin_unlock(&queue->sq_lock);
+}
+
+uint16_t nvme_io_queue_submit(
+    struct nvme_io_queue* queue,
+    struct bio* bio,
+    struct nvme_rw_cmd cmd,
+    _Atomic(bool)* done
+) {
+    const uint16_t cid = acquire_slot(queue);
+    struct nvme_inflight* slot = &queue->slots[cid];
+
+    struct nvme_sqe sqe = {};
+    if (bio->op == BIO_FLUSH) {
+        slot->map = nullptr;
+        slot->prps = (struct dma_buffer){};
+        nvme_flush_cmd_write(&(struct nvme_flush_cmd){.cid = cid, .nsid = cmd.nsid}, &sqe);
+    } else {
+        errno_t status = nvme_build_prps(queue->tag, bio, slot, &cmd.prp1, &cmd.prp2);
+        if (status) {
+            release_slot(queue, cid);
+            bio_complete(bio, EIO, 0);
+            if (done)
+                atomic_store(done, true, ATOMIC_RELEASE);
+            return cid;
+        }
+        cmd.cid = cid;
+        nvme_rw_cmd_write(&cmd, &sqe);
+    }
+
+    spin_lock(&queue->cmds_lock);
+    slot->active = true;
+    slot->bio = bio;
+    slot->lbas = bio->num_lbas;
+    slot->done = done;
+    spin_unlock(&queue->cmds_lock);
+
+    write_sqe(queue, &sqe);
+    return cid;
+}
+
+bool nvme_io_queue_abandon(struct nvme_io_queue* queue, uint16_t cid, struct bio* bio) {
+    spin_lock(&queue->cmds_lock);
+    struct nvme_inflight* slot = &queue->slots[cid];
+    bool abandoned = slot->active && slot->bio == bio;
+    if (abandoned) {
+        slot->bio = nullptr;
+        slot->done = nullptr;
+    }
+    spin_unlock(&queue->cmds_lock);
+    return abandoned;
+}
+
+size_t nvme_io_queue_drain(struct nvme_io_queue* queue) {
+    spin_lock(&queue->cq_lock);
+    size_t completed = 0;
+
+    while (true) {
+        struct nvme_cqe* entry = &((struct nvme_cqe*)queue->cq.virt)[queue->cq_head];
+        dma_buffer_sync(&queue->cq, queue->cq_head * sizeof(*entry), sizeof(*entry), DMA_SYNC_POSTREAD);
+        uint32_t dw3 = mmio_reg_read(entry, NVME_CQE_DW3);
+        if (mmio_field_get(NVME_CQE_PHASE_TAG, dw3) != queue->cq_phase)
+            break;
+
+        uint16_t cid = mmio_field_get(NVME_CQE_CID, dw3);
+        uint16_t status = mmio_field_get(NVME_CQE_STATUS, dw3);
+
+        queue->cq_head++;
+        if (queue->cq_head == queue->depth) {
+            queue->cq_head = 0;
+            queue->cq_phase ^= 1;
+        }
+
+        struct nvme_inflight inflight = {};
+        spin_lock(&queue->cmds_lock);
+        if (cid < queue->depth - 1 && queue->slots[cid].active) {
+            inflight = queue->slots[cid];
+            queue->slots[cid].active = false;
+            queue->free[queue->num_free++] = cid;
+        }
+        spin_unlock(&queue->cmds_lock);
+
+        if (inflight.active) {
+            nvme_release_prps(&inflight);
+            if (inflight.bio) {
+                if (nvme_status_is_success(status))
+                    bio_complete(inflight.bio, 0, inflight.lbas);
+                else
+                    bio_complete(inflight.bio, EIO, 0);
+            }
+            if (inflight.done)
+                atomic_store(inflight.done, true, ATOMIC_RELEASE);
+        }
+
+        completed++;
+    }
+
+    if (completed > 0)
+        ring_head(queue->regs, queue->doorbells_offset, queue->doorbell_stride, queue->cq_head);
+
+    spin_unlock(&queue->cq_lock);
+
+    if (completed > 0)
+        event_set(&queue->slots_free);
+
+    return completed;
+}

dev/block/nvme/spec.h

@@ -0,0 +1,152 @@
+// SPDX-License-Identifier: ISC
+
+#pragma once
+
+#include <zinnia/compiler.h>
+#include <zinnia/mmio.h>
+#include <stddef.h>
+#include <stdint.h>
+
+// Submission queue entry.
+struct [[__packed]] nvme_sqe {
+    uint32_t cdw0;
+
+    uint32_t nsid; // Namespace Identifier
+    uint32_t cdw2;
+    uint32_t cdw3;
+
+    uint64_t mptr; // Metadata Pointer
+
+    uint64_t dptr0; // Data Pointer 0
+    uint64_t dptr1; // Data Pointer 1
+
+    uint32_t cdw10;
+    uint32_t cdw11;
+    uint32_t cdw12;
+    uint32_t cdw13;
+    uint32_t cdw14;
+    uint32_t cdw15;
+};
+static_assert(sizeof(struct nvme_sqe) == 0x40);
+
+constexpr mmio_reg32 NVME_SQE_CDW0 = {offsetof(struct nvme_sqe, cdw0), MMIO_LE};
+constexpr mmio_field32 NVME_SQE_CID = {NVME_SQE_CDW0, 16, 16}; // Command Identifier
+constexpr mmio_field32 NVME_SQE_PSDT = {NVME_SQE_CDW0, 14, 2}; // PRP or SGL for Data Transfer
+constexpr mmio_field32 NVME_SQE_FUSE = {NVME_SQE_CDW0, 8, 2};  // Fused Operation
+constexpr mmio_field32 NVME_SQE_OPC = {NVME_SQE_CDW0, 0, 8};   // Opcode
+
+constexpr mmio_reg32 NVME_SQE_NSID = {offsetof(struct nvme_sqe, nsid), MMIO_LE};   // Namespace Identifier
+constexpr mmio_reg64 NVME_SQE_MPTR = {offsetof(struct nvme_sqe, mptr), MMIO_LE};   // Metadata Pointer
+constexpr mmio_reg64 NVME_SQE_DPTR0 = {offsetof(struct nvme_sqe, dptr0), MMIO_LE}; // Data Pointer 0
+constexpr mmio_reg64 NVME_SQE_DPTR1 = {offsetof(struct nvme_sqe, dptr1), MMIO_LE}; // Data Pointer 1
+
+constexpr mmio_reg32 NVME_SQE_CDW10 = {offsetof(struct nvme_sqe, cdw10), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW11 = {offsetof(struct nvme_sqe, cdw11), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW12 = {offsetof(struct nvme_sqe, cdw12), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW13 = {offsetof(struct nvme_sqe, cdw13), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW14 = {offsetof(struct nvme_sqe, cdw14), MMIO_LE};
+constexpr mmio_reg32 NVME_SQE_CDW15 = {offsetof(struct nvme_sqe, cdw15), MMIO_LE};
+
+constexpr uint8_t NVME_CREATE_CQ_OPCODE = 0x05;
+constexpr mmio_field32 NVME_CREATE_CQ_QSIZE = {NVME_SQE_CDW10, 16, 16}; // Queue Size
+constexpr mmio_field32 NVME_CREATE_CQ_QID = {NVME_SQE_CDW10, 0, 16};    // Queue Identifier
+constexpr mmio_field32 NVME_CREATE_CQ_IV = {NVME_SQE_CDW11, 16, 16};    // Interrupt Vector
+constexpr mmio_field32 NVME_CREATE_CQ_IEN = {NVME_SQE_CDW11, 1, 1};     // Interrupts Enabled
+constexpr mmio_field32 NVME_CREATE_CQ_PC = {NVME_SQE_CDW11, 0, 1};      // Physically Contiguous
+
+constexpr uint8_t NVME_CREATE_SQ_OPCODE = 0x01;
+constexpr mmio_field32 NVME_CREATE_SQ_QSIZE = {NVME_SQE_CDW10, 16, 16}; // Queue Size
+constexpr mmio_field32 NVME_CREATE_SQ_QID = {NVME_SQE_CDW10, 0, 16};    // Queue Identifier
+constexpr mmio_field32 NVME_CREATE_SQ_CQID = {NVME_SQE_CDW11, 16, 16};  // Completion Queue Identifier
+constexpr mmio_field32 NVME_CREATE_SQ_QPRIO = {NVME_SQE_CDW11, 1, 2};   // Queue Priority
+constexpr mmio_field32 NVME_CREATE_SQ_PC = {NVME_SQE_CDW11, 0, 1};      // Physically Contiguous
+
+constexpr mmio_field32 NVME_IDENTIFY_CNTID = {NVME_SQE_CDW10, 16, 16}; // Controller Identifier
+constexpr mmio_field32 NVME_IDENTIFY_CNS = {NVME_SQE_CDW10, 0, 8};     // Controller or Namespace Structure
+
+constexpr mmio_field32 NVME_RW_SLBA_LOW = {NVME_SQE_CDW10, 0, 32};  // Starting LBA Lower Bits
+constexpr mmio_field32 NVME_RW_SLBA_HIGH = {NVME_SQE_CDW11, 0, 32}; // Starting LBA Higher Bits
+constexpr mmio_field32 NVME_RW_LR = {NVME_SQE_CDW12, 31, 1};        // Limited Retry
+constexpr mmio_field32 NVME_RW_FUA = {NVME_SQE_CDW12, 30, 1};       // Force Unit Access
+constexpr mmio_field32 NVME_RW_PRINFO = {NVME_SQE_CDW12, 26, 4};    // Protection Information Field
+constexpr mmio_field32 NVME_RW_NLB = {NVME_SQE_CDW12, 0, 16};       // Number of Logical Blocks
+
+struct [[__packed]] nvme_cqe {
+    uint32_t dw0;
+    uint32_t dw1;
+    uint32_t dw2;
+    uint32_t dw3;
+};
+static_assert(sizeof(struct nvme_cqe) == 0x10);
+
+constexpr mmio_reg32 NVME_CQE_DW0 = {offsetof(struct nvme_cqe, dw0), MMIO_LE};
+constexpr mmio_reg32 NVME_CQE_DW2 = {offsetof(struct nvme_cqe, dw2), MMIO_LE};
+constexpr mmio_field32 NVME_CQE_SQ_IDENT = {NVME_CQE_DW2, 16, 16};
+constexpr mmio_field32 NVME_CQE_SQ_HEAD = {NVME_CQE_DW2, 0, 16};
+constexpr mmio_reg32 NVME_CQE_DW3 = {offsetof(struct nvme_cqe, dw3), MMIO_LE};
+constexpr mmio_field32 NVME_CQE_STATUS = {NVME_CQE_DW3, 17, 15};
+constexpr mmio_field32 NVME_CQE_PHASE_TAG = {NVME_CQE_DW3, 16, 1};
+constexpr mmio_field32 NVME_CQE_CID = {NVME_CQE_DW3, 0, 16};
+
+constexpr mmio_reg64 NVME_REG_CAP = {0x00, MMIO_LE};            // Controller Capabilities
+constexpr mmio_field64 NVME_CAP_MPSMAX = {NVME_REG_CAP, 52, 4}; // Memory Page Size Maximum
+constexpr mmio_field64 NVME_CAP_MPSMIN = {NVME_REG_CAP, 48, 4}; // Memory Page Size Minimum
+constexpr mmio_field64 NVME_CAP_BPS = {NVME_REG_CAP, 45, 1};    // Boot Partition Support
+constexpr mmio_field64 NVME_CAP_CSS = {NVME_REG_CAP, 37, 8};    // Command Sets Supported
+constexpr mmio_field64 NVME_CAP_NSSRS = {NVME_REG_CAP, 36, 1};  // NVM Subsystem Reset Supported
+constexpr mmio_field64 NVME_CAP_DSTRD = {NVME_REG_CAP, 32, 4};  // Doorbell Stride
+constexpr mmio_field64 NVME_CAP_TO = {NVME_REG_CAP, 24, 8};     // Timeout
+constexpr mmio_field64 NVME_CAP_AMS = {NVME_REG_CAP, 17, 2};    // Arbitration Mechanism Supported
+constexpr mmio_field64 NVME_CAP_CQR = {NVME_REG_CAP, 16, 1};    // Contiguous Queues Required
+constexpr mmio_field64 NVME_CAP_MQES = {NVME_REG_CAP, 0, 16};   // Maximum Queue Entries Supported
+
+constexpr mmio_reg32 NVME_REG_VS = {0x08, MMIO_LE};         // Version
+constexpr mmio_field32 NVME_VS_MJR = {NVME_REG_VS, 16, 16}; // Major Version Number
+constexpr mmio_field32 NVME_VS_MNR = {NVME_REG_VS, 8, 8};   // Minor Version Number
+constexpr mmio_field32 NVME_VS_TER = {NVME_REG_VS, 0, 8};   // Tertiary Version Number
+
+constexpr mmio_reg32 NVME_REG_INTMS = {0x0C, MMIO_LE}; // Interrupt Mask Set
+constexpr mmio_reg32 NVME_REG_INTMC = {0x10, MMIO_LE}; // Interrupt Mask Clear
+
+constexpr mmio_reg32 NVME_REG_CC = {0x14, MMIO_LE};           // Controller Configuration
+constexpr mmio_field32 NVME_CC_IOCQES = {NVME_REG_CC, 20, 4}; // I/O Completion Queue Entry Size
+constexpr mmio_field32 NVME_CC_IOSQES = {NVME_REG_CC, 16, 4}; // I/O Submission Queue Entry Size
+constexpr mmio_field32 NVME_CC_SHN = {NVME_REG_CC, 14, 2};    // Shutdown Notification
+constexpr mmio_field32 NVME_CC_AMS = {NVME_REG_CC, 11, 3};    // Arbitration Mechanism Selected
+constexpr mmio_field32 NVME_CC_MPS = {NVME_REG_CC, 7, 4};     // Memory Page Size
+constexpr mmio_field32 NVME_CC_CSS = {NVME_REG_CC, 4, 3};     // I/O Command Set Selected
+constexpr mmio_field32 NVME_CC_EN = {NVME_REG_CC, 0, 1};      // Enable
+
+constexpr mmio_reg32 NVME_REG_CSTS = {0x1C, MMIO_LE};           // Controller Status
+constexpr mmio_field32 NVME_CSTS_PP = {NVME_REG_CSTS, 5, 1};    // Processing Paused
+constexpr mmio_field32 NVME_CSTS_NSSRO = {NVME_REG_CSTS, 4, 1}; // NVM Subsystem Reset Occurred
+constexpr mmio_field32 NVME_CSTS_SHST = {NVME_REG_CSTS, 2, 2};  // Shutdown Status
+constexpr mmio_field32 NVME_CSTS_CFS = {NVME_REG_CSTS, 1, 1};   // Controller Fatal Status
+constexpr mmio_field32 NVME_CSTS_RDY = {NVME_REG_CSTS, 0, 1};   // Ready
+
+constexpr mmio_reg32 NVME_REG_AQA = {0x24, MMIO_LE}; // Admin Queue Attributes
+
+constexpr mmio_field32 NVME_AQA_ACQS = {NVME_REG_AQA, 0, 12};  // Admin Completion Queue Size
+constexpr mmio_field32 NVME_AQA_ASQS = {NVME_REG_AQA, 16, 12}; // Admin Submission Queue Size
+
+constexpr mmio_reg64 NVME_REG_ASQ = {0x28, MMIO_LE}; // Admin Submission Queue
+constexpr mmio_reg64 NVME_REG_ACQ = {0x30, MMIO_LE}; // Admin Completion Queue
+
+// Generic Command Set
+enum nvme_cmd : uint8_t {
+    NVME_CMD_FLUSH = 0x00,
+    NVME_CMD_WRITE = 0x01,
+    NVME_CMD_READ = 0x02,
+};
+
+// Admin Command Set
+enum nvme_admin_cmd : uint8_t {
+    NVME_ADMIN_DELETE_SQ = 0x00,
+    NVME_ADMIN_CREATE_SQ = 0x01,
+    NVME_ADMIN_DELETE_CQ = 0x04,
+    NVME_ADMIN_CREATE_CQ = 0x05,
+    NVME_ADMIN_IDENTIFY = 0x06,
+    NVME_ADMIN_ABORT = 0x08,
+    NVME_ADMIN_SET_FEATURES = 0x09,
+    NVME_ADMIN_GET_FEATURES = 0x0A,
+};

dev/block/partition.c

@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/blockdev.h>
+#include <stdint.h>
+
+struct partition {
+    struct blockdev blk;
+    struct blockdev* parent;
+    uint64_t start;
+};
+
+static _Atomic(uint32_t) next_minor;
+
+static errno_t partition_submit(struct blockdev* blk, struct bio* bio) {
+    struct partition* part = containerof(blk, struct partition, blk);
+    struct blockdev* parent = part->parent;
+
+    if (bio->op != BIO_FLUSH) {
+        if (bio->lba >= blk->lba_count) {
+            bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0);
+            return 0;
+        }
+        if (bio->op == BIO_WRITE && bio->num_lbas > blk->lba_count - bio->lba) {
+            bio_complete(bio, ENOSPC, 0);
+            return 0;
+        }
+        bio->lba += part->start;
+    }
+    return parent->ops->submit(parent, bio);
+}
+
+static errno_t partition_ioctl(
+    struct blockdev* blk,
+    struct vfs_file* file,
+    uintptr_t request,
+    uintptr_t arg,
+    uintptr_t* out
+) {
+    struct blockdev* parent = containerof(blk, struct partition, blk)->parent;
+    if (!parent->ops->ioctl)
+        return ENOTTY;
+    return parent->ops->ioctl(parent, file, request, arg, out);
+}
+
+static void partition_destroy(struct blockdev* blk) {
+    struct partition* part = containerof(blk, struct partition, blk);
+    blockdev_unref(part->parent);
+    mem_free(part);
+}
+
+static const struct blockdev_ops partition_ops = {
+    .submit = partition_submit,
+    .ioctl = partition_ioctl,
+    .destroy = partition_destroy,
+};
+
+errno_t partition_new(struct blockdev* parent, uint64_t start_lba, uint64_t lba_count, struct blockdev** out) {
+    struct partition* part = mem_alloc(sizeof(struct partition), 0);
+    if (!part)
+        return ENOMEM;
+
+    blockdev_init(
+        &part->blk,
+        &partition_ops,
+        259,
+        atomic_fetch_add(&next_minor, 1, ATOMIC_RELAXED),
+        parent->lba_size,
+        lba_count
+    );
+    part->blk.limits = parent->limits;
+    blockdev_ref(parent);
+    part->parent = parent;
+    part->start = start_lba;
+    *out = &part->blk;
+    return 0;
+}

dev/block/ram.c

@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/atomic.h>
+#include <zinnia/blockdev.h>
+#include <string.h>
+
+constexpr uint32_t RAM_MAJOR = 1;
+constexpr size_t RAM_LBA_SIZE = 512;
+
+struct ramdisk {
+    struct blockdev blk;
+    phys_t base;
+};
+
+static atomic_uint next_minor;
+
+static errno_t ramdisk_submit(struct blockdev* blk, struct bio* bio) {
+    struct ramdisk* ram = containerof(blk, struct ramdisk, blk);
+
+    if (bio->op == BIO_FLUSH) {
+        bio_complete(bio, 0, 0);
+        return 0;
+    }
+    if (bio->lba >= blk->lba_count || bio->num_lbas > blk->lba_count - bio->lba) {
+        bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0);
+        return 0;
+    }
+
+    uint64_t lba = bio->lba;
+    for (size_t i = 0; i < bio->num_segs; i++) {
+        const struct bio_seg* seg = &bio->segs[i];
+        void* store = HHDM_PTR(ram->base + lba * RAM_LBA_SIZE);
+        void* buffer = HHDM_PTR(seg->addr);
+        if (bio->op == BIO_READ)
+            memcpy(buffer, store, seg->len);
+        else
+            memcpy(store, buffer, seg->len);
+        lba += seg->len / RAM_LBA_SIZE;
+    }
+
+    bio_complete(bio, 0, bio->num_lbas);
+    return 0;
+}
+
+static void ramdisk_destroy(struct blockdev* blk) {
+    mem_free(containerof(blk, struct ramdisk, blk));
+}
+
+static const struct blockdev_ops ramdisk_ops = {
+    .submit = ramdisk_submit,
+    .destroy = ramdisk_destroy,
+};
+
+errno_t ramdisk_new(phys_t base, size_t length, struct blockdev** out) {
+    struct ramdisk* ram = mem_alloc(sizeof(struct ramdisk), 0);
+    if (!ram)
+        return ENOMEM;
+
+    blockdev_init(
+        &ram->blk,
+        &ramdisk_ops,
+        RAM_MAJOR,
+        atomic_fetch_add(&next_minor, 1, ATOMIC_RELAXED),
+        RAM_LBA_SIZE,
+        length / RAM_LBA_SIZE
+    );
+    ram->base = base;
+    *out = &ram->blk;
+    return 0;
+}

dev/char/chardev.c

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+
+void chardev_init(
+    struct chardev* dev,
+    const struct chardev_ops* ops,
+    const struct vfs_file_ops* file_ops,
+    uint32_t major,
+    uint32_t minor
+) {
+    refcount_init(&dev->rc);
+    dev->ops = ops;
+    dev->file_ops = file_ops;
+    dev->dev = dev_make(major, minor);
+}
+
+void chardev_ref(struct chardev* dev) {
+    refcount_inc(&dev->rc);
+}
+
+void chardev_unref(struct chardev* dev) {
+    if (refcount_dec(&dev->rc) && dev->ops && dev->ops->destroy)
+        dev->ops->destroy(dev);
+}
+
+static errno_t chardev_node_open(struct vfs_node* node, struct vfs_file* file) {
+    struct chardev* dev = node->priv;
+    file->ops = dev->file_ops;
+    if (dev->ops && dev->ops->open)
+        return dev->ops->open(dev, file);
+    return 0;
+}
+
+static void chardev_node_destroy(struct vfs_node* node) {
+    chardev_unref(node->priv);
+}
+
+static const struct vfs_node_ops chardev_node_ops = {
+    .destroy = chardev_node_destroy,
+    .device = {
+        .open = chardev_node_open,
+    },
+};
+
+errno_t devtmpfs_add_chardev(const char* name, struct chardev* dev, mode_t mode) {
+    struct vfs_mknod_target target = {
+        .type = VNODE_CHARDEV,
+        .ops = &chardev_node_ops,
+        .priv = dev,
+        .rdev = dev->dev,
+    };
+
+    chardev_ref(dev);
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mknod(root, root, name, mode, &target, ident_kernel(), nullptr);
+    vfs_path_drop(root);
+    if (status) {
+        chardev_unref(dev);
+        return status;
+    }
+    devctl_notify_create(name);
+    return 0;
+}
+
+errno_t devtmpfs_mkdir(const char* name, mode_t mode) {
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_mkdir(root, root, name, mode, ident_kernel());
+    vfs_path_drop(root);
+    return status;
+}
+
+errno_t devtmpfs_remove(const char* name) {
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_unlink(root, root, name, ident_kernel());
+    vfs_path_drop(root);
+    if (status)
+        return status;
+    devctl_notify_destroy(name);
+    return 0;
+}
+
+errno_t devtmpfs_symlink(const char* name, const char* target) {
+    struct vfs_path root = devtmpfs_root();
+    errno_t status = vfs_symlink(root, root, name, target, ident_kernel());
+    vfs_path_drop(root);
+    return status;
+}

dev/char/devctl.c

@@ -0,0 +1,262 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/alloc.h>
+#include <zinnia/assert.h>
+#include <zinnia/atomic.h>
+#include <zinnia/chardev.h>
+#include <zinnia/devctl.h>
+#include <zinnia/mutex.h>
+#include <zinnia/poll.h>
+#include <zinnia/tailq.h>
+#include <string.h>
+
+constexpr size_t DEVCTL_QUEUE_CAPACITY = 256;
+constexpr size_t DEVCTL_BACKLOG_CAPACITY = 256;
+
+struct devctl_line {
+    tailq_link(struct devctl_line) link;
+    size_t len;
+    char data[];
+};
+
+struct devctl_queue {
+    tailq_head(struct devctl_line) lines;
+    size_t count;
+};
+
+struct devctl_reader {
+    tailq_link(struct devctl_reader) link;
+    struct devctl_queue queue;
+};
+
+static mutex_t devctl_lock;
+static tailq_head(struct devctl_reader) devctl_readers;
+static struct devctl_queue devctl_backlog;
+static bool devctl_had_reader;
+static struct poll_event devctl_event;
+static atomic_bool devctl_ready;
+static struct chardev devctl_dev;
+
+static void queue_init(struct devctl_queue* queue) {
+    tailq_init(&queue->lines);
+    queue->count = 0;
+}
+
+static struct devctl_line* queue_pop(struct devctl_queue* queue) {
+    struct devctl_line* line = tailq_first(&queue->lines);
+    if (line) {
+        tailq_remove(&queue->lines, line, link);
+        queue->count--;
+    }
+    return line;
+}
+
+static void queue_push(struct devctl_queue* queue, struct devctl_line* line, size_t capacity) {
+    if (queue->count >= capacity)
+        mem_free(queue_pop(queue));
+    tailq_insert_tail(&queue->lines, line, link);
+    queue->count++;
+}
+
+static void queue_clear(struct devctl_queue* queue) {
+    struct devctl_line* line;
+    while ((line = queue_pop(queue)))
+        mem_free(line);
+}
+
+static struct devctl_line* line_new(const char* data, size_t len) {
+    struct devctl_line* line = mem_alloc(sizeof(*line) + len, 0);
+    if (!line)
+        return nullptr;
+    line->len = len;
+    memcpy(line->data, data, len);
+    return line;
+}
+
+static void devctl_notify(const char* type, const char* name) {
+    if (!atomic_load(&devctl_ready, ATOMIC_ACQUIRE))
+        return;
+
+    while (*name == '/')
+        name++;
+
+    const char prefix[] = "!system=DEVFS subsystem=CDEV type=";
+    const char middle[] = " cdev=";
+    const size_t prefix_len = sizeof(prefix) - 1;
+    const size_t type_len = strlen(type);
+    const size_t middle_len = sizeof(middle) - 1;
+    const size_t name_len = strlen(name);
+    const size_t len = prefix_len + type_len + middle_len + name_len + 1;
+
+    char* buf = mem_alloc(len, 0);
+    if (!buf)
+        return;
+    char* pos = buf;
+    memcpy(pos, prefix, prefix_len);
+    pos += prefix_len;
+    memcpy(pos, type, type_len);
+    pos += type_len;
+    memcpy(pos, middle, middle_len);
+    pos += middle_len;
+    memcpy(pos, name, name_len);
+    pos += name_len;
+    *pos = '\n';
+
+    mutex_lock(&devctl_lock);
+    if (tailq_empty(&devctl_readers)) {
+        if (!devctl_had_reader) {
+            struct devctl_line* line = line_new(buf, len);
+            if (line)
+                queue_push(&devctl_backlog, line, DEVCTL_BACKLOG_CAPACITY);
+        }
+    } else {
+        for (struct devctl_reader* reader = tailq_first(&devctl_readers); reader; reader = tailq_next(reader, link)) {
+            struct devctl_line* line = line_new(buf, len);
+            if (line)
+                queue_push(&reader->queue, line, DEVCTL_QUEUE_CAPACITY);
+        }
+    }
+    mutex_unlock(&devctl_lock);
+    mem_free(buf);
+
+    poll_event_notify(&devctl_event);
+}
+
+void devctl_notify_create(const char* name) {
+    devctl_notify("CREATE", name);
+}
+
+void devctl_notify_destroy(const char* name) {
+    devctl_notify("DESTROY", name);
+}
+
+static bool file_nonblocking(struct vfs_file* file) {
+    return atomic_load(&file->flags, ATOMIC_RELAXED) & O_NONBLOCK;
+}
+
+static errno_t drain_lines(struct devctl_queue* queue, iovec_iter_t* buffer, size_t* out_total) {
+    size_t total = 0;
+    errno_t status = 0;
+    struct devctl_line* line;
+    while ((line = tailq_first(&queue->lines))) {
+        if (line->len > iovec_iter_len(buffer)) {
+            if (total == 0)
+                status = EINVAL;
+            break;
+        }
+        queue_pop(queue);
+        size_t copied;
+        status = iovec_iter_copy_to(buffer, line->data, line->len, &copied);
+        mem_free(line);
+        total += copied;
+        if (status)
+            break;
+    }
+    *out_total = total;
+    return status;
+}
+
+static errno_t devctl_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    struct devctl_reader* reader = file->priv;
+
+    struct poll_table table;
+    poll_table_init(&table);
+    poll_wait(&table, &devctl_event);
+
+    errno_t status = 0;
+    size_t read = 0;
+    mutex_lock(&devctl_lock);
+    while (true) {
+        if (reader->queue.count) {
+            status = drain_lines(&reader->queue, buffer, &read);
+            break;
+        }
+        if (file_nonblocking(file)) {
+            status = EAGAIN;
+            break;
+        }
+        if (poll_table_sleep(&table, &devctl_lock, nullptr) == EINTR) {
+            status = ERESTART;
+            break;
+        }
+    }
+    mutex_unlock(&devctl_lock);
+    poll_table_fini(&table);
+
+    *out_read = read;
+    return status;
+}
+
+static errno_t devctl_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    return EBADF;
+}
+
+static enum poll_flags devctl_poll(struct vfs_file* file, enum poll_flags events, struct poll_table* table) {
+    struct devctl_reader* reader = file->priv;
+
+    if (events & (POLLIN | POLLRDNORM))
+        poll_wait(table, &devctl_event);
+
+    enum poll_flags revents = 0;
+    mutex_lock(&devctl_lock);
+    if (reader->queue.count)
+        revents |= POLLIN | POLLRDNORM;
+    mutex_unlock(&devctl_lock);
+    return revents;
+}
+
+static void devctl_release(struct vfs_file* file) {
+    struct devctl_reader* reader = file->priv;
+
+    mutex_lock(&devctl_lock);
+    tailq_remove(&devctl_readers, reader, link);
+    mutex_unlock(&devctl_lock);
+
+    queue_clear(&reader->queue);
+    mem_free(reader);
+}
+
+static const struct vfs_file_ops devctl_file_ops = {
+    .read = devctl_read,
+    .write = devctl_write,
+    .poll = devctl_poll,
+    .release = devctl_release,
+};
+
+static errno_t devctl_open(struct chardev* dev, struct vfs_file* file) {
+    struct devctl_reader* reader = mem_alloc(sizeof(*reader), 0);
+    if (!reader)
+        return ENOMEM;
+    queue_init(&reader->queue);
+
+    mutex_lock(&devctl_lock);
+    if (!devctl_had_reader) {
+        devctl_had_reader = true;
+        struct devctl_line* line;
+        while ((line = queue_pop(&devctl_backlog)))
+            queue_push(&reader->queue, line, DEVCTL_QUEUE_CAPACITY);
+    }
+    tailq_insert_tail(&devctl_readers, reader, link);
+    mutex_unlock(&devctl_lock);
+
+    file->priv = reader;
+    vfs_stream_open(file);
+    return 0;
+}
+
+static const struct chardev_ops devctl_dev_ops = {.open = devctl_open};
+
+static void devctl_stage_init(struct boot_info* info) {
+    mutex_init(&devctl_lock);
+    tailq_init(&devctl_readers);
+    queue_init(&devctl_backlog);
+    poll_event_init(&devctl_event);
+    atomic_store(&devctl_ready, true, ATOMIC_RELEASE);
+
+    chardev_init(&devctl_dev, &devctl_dev_ops, &devctl_file_ops, 5, 3);
+    errno_t status = devtmpfs_add_chardev("devctl", &devctl_dev, 0600);
+    assert(status == 0, "Unable to create /dev/devctl (%i)", status);
+}
+
+INIT_TASK(devctl_stage, "dev.devctl", devctl_stage_init);
+INIT_DEPENDS(devctl_stage, devtmpfs_stage);

dev/char/memfiles.c

@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: ISC
+
+#include <zinnia/assert.h>
+#include <zinnia/chardev.h>
+#include <zinnia/random.h>
+#include <zinnia/utils.h>
+
+constexpr size_t MEMFILE_CHUNK = 64;
+
+static const uint8_t zero_chunk[MEMFILE_CHUNK];
+
+static errno_t memfile_discard(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    *out_written = iovec_iter_len(buffer);
+    return 0;
+}
+
+static errno_t memfile_read_zero(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    size_t total = 0;
+    errno_t status = 0;
+    while (iovec_iter_len(buffer)) {
+        size_t copied;
+        status = iovec_iter_copy_to(buffer, zero_chunk, MIN(iovec_iter_len(buffer), MEMFILE_CHUNK), &copied);
+        total += copied;
+        if (status)
+            break;
+    }
+    *out_read = total;
+    return status;
+}
+
+static errno_t null_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    *out_read = 0;
+    return 0;
+}
+
+static const struct vfs_file_ops null_file_ops = {
+    .read = null_read,
+    .write = memfile_discard,
+};
+
+static const struct vfs_file_ops zero_file_ops = {
+    .read = memfile_read_zero,
+    .write = memfile_discard,
+};
+
+static errno_t full_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) {
+    *out_written = 0;
+    return ENOSPC;
+}
+
+static const struct vfs_file_ops full_file_ops = {
+    .read = memfile_read_zero,
+    .write = full_write,
+};
+
+static errno_t random_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) {
+    uint8_t chunk[MEMFILE_CHUNK];
+    size_t total = 0;
+    errno_t status = 0;
+    while (iovec_iter_len(buffer)) {
+        const size_t len = MIN(iovec_iter_len(buffer), sizeof(chunk));
+
+        // TODO: This function sucks and is probably not useful for cryptography.
+        random_fill(chunk, len);
+
+        size_t copied;
+        status = iovec_iter_copy_to(buffer, chunk, len, &copied);
+        total += copied;
+        if (status)
+            break;
+    }
+    *out_read = total;
+    return status;
+}
+
+static const struct vfs_file_ops random_file_ops = {
+    .read = random_read,
+    .write = memfile_discard,
+};
+
+static struct chardev null_dev;
+static struct chardev zero_dev;
+static struct chardev full_dev;
+static struct chardev random_dev;
+static struct chardev urandom_dev;
+
+static void memfiles_add(const char* name, struct chardev* dev, const struct vfs_file_ops* ops, uint32_t minor) {
+    chardev_init(dev, nullptr, ops, 1, minor);
+    errno_t status = devtmpfs_add_chardev(name, dev, 0666);
+    assert(status == 0, "Unable to create /dev/%s (%i)", name, status);
+}
+
+static void memfiles_stage_init(struct boot_info* info) {
+    memfiles_add("null", &null_dev, &null_file_ops, 3);
+    memfiles_add("zero", &zero_dev, &zero_file_ops, 5);
+    memfiles_add("full", &full_dev, &full_file_ops, 7);
+    memfiles_add("random", &random_dev, &random_file_ops, 8);
+    memfiles_add("urandom", &urandom_dev, &random_file_ops, 9);
+}
+
+INIT_TASK(memfiles_stage, "dev.memfiles", memfiles_stage_init);
+INIT_DEPENDS(memfiles_stage, devtmpfs_stage, random_stage);

dev/char/meson.build

@@ -0,0 +1,5 @@
+kernel_sources += files(
+    'chardev.c',
+    'devctl.c',
+    'memfiles.c',
+)